push cometbft-0.38.2 with scrtlabs changes in

DOCKER/Dockerfile

@@ -1,6 +1,6 @@
 # Use a build arg to ensure that both stages use the same,
 # hopefully current, go version.
-ARG GOLANG_BASE_IMAGE=golang:1.19-alpine
+ARG GOLANG_BASE_IMAGE=golang:1.21-alpine
 
 # stage 1 Generate CometBFT Binary
 FROM --platform=$BUILDPLATFORM $GOLANG_BASE_IMAGE as builder

DOCKER/README.md

@@ -8,7 +8,7 @@ Official releases can be found [here](https://github.com/cometbft/cometbft/relea
 
 The Dockerfile for CometBFT is not expected to change in the near future. The main file used for all builds can be found [here](https://raw.githubusercontent.com/cometbft/cometbft/main/DOCKER/Dockerfile).
 
-Respective versioned files can be found <https://raw.githubusercontent.com/cometbft/cometbft/vX.XX.XX/DOCKER/Dockerfile> (replace the Xs with the version number).
+Respective versioned files can be found at `https://raw.githubusercontent.com/cometbft/cometbft/vX.XX.XX/DOCKER/Dockerfile` (replace the Xs with the version number).
 
 ## Quick reference
 
@@ -20,9 +20,9 @@ Respective versioned files can be found <https://raw.githubusercontent.com/comet
 
 CometBFT is Byzantine Fault Tolerant (BFT) middleware that takes a state transition machine, written in any programming language, and securely replicates it on many machines.
 
-For more background, see the [the docs](https://docs.cometbft.com/v0.34/introduction/#quick-start).
+For more background, see the [the docs](https://docs.cometbft.com/v0.38.x/introduction/#quick-start).
 
-To get started developing applications, see the [application developers guide](https://docs.cometbft.com/v0.34/introduction/quick-start.html).
+To get started developing applications, see the [application developers guide](https://docs.cometbft.com/v0.38.x/introduction/quick-start.html).
 
 ## How to use this image
 
@@ -37,7 +37,7 @@ docker run -it --rm -v "/tmp:/cometbft" cometbft/cometbft node --proxy_app=kvsto
 
 ## Local cluster
 
-To run a 4-node network, see the `Makefile` in the root of [the repo](https://github.com/cometbft/cometbft/blob/v0.34.x/Makefile) and run:
+To run a 4-node network, see the `Makefile` in the root of [the repo](https://github.com/cometbft/cometbft/blob/v0.38.x/Makefile) and run:
 
 ```sh
 make build-linux
@@ -49,8 +49,8 @@ Note that this will build and use a different image than the ones provided here.
 
 ## License
 
-- CometBFT's license is [Apache 2.0](https://github.com/cometbft/cometbft/blob/main/LICENSE).
+- CometBFT's license is [Apache 2.0](https://github.com/cometbft/cometbft/blob/v0.38.x/LICENSE).
 
 ## Contributing
 
-Contributions are most welcome! See the [contributing file](https://github.com/cometbft/cometbft/blob/v0.34.x/CONTRIBUTING.md) for more information.
+Contributions are most welcome! See the [contributing file](https://github.com/cometbft/cometbft/blob/v0.38.x/CONTRIBUTING.md) for more information.

DOCKER/build.sh

@@ -17,7 +17,7 @@ echo
 if [[ $REPLY =~ ^[Yy]$ ]]
 then
 		docker build \
- 			-t "cometbft/cometbft" \
- 			-t "cometbft/cometbft:$TAG" \
- 			-t "cometbft/cometbft:$TAG_NO_PATCH" .
+			-t "cometbft/cometbft" \
+			-t "cometbft/cometbft:$TAG" \
+			-t "cometbft/cometbft:$TAG_NO_PATCH" .
 fi

Makefile

@@ -1,52 +1,12 @@
+include common.mk
+
 PACKAGES=$(shell go list ./...)
 BUILDDIR?=$(CURDIR)/build
 OUTPUT?=$(BUILDDIR)/cometbft
 
-BUILD_TAGS?=cometbft
-
-COMMIT_HASH := $(shell git rev-parse --short HEAD)
-LD_FLAGS = -X github.com/cometbft/cometbft/version.TMGitCommitHash=$(COMMIT_HASH)
-BUILD_FLAGS = -mod=readonly -ldflags "$(LD_FLAGS)"
 HTTPS_GIT := https://github.com/cometbft/cometbft.git
 CGO_ENABLED ?= 0
 
-# handle nostrip
-ifeq (,$(findstring nostrip,$(COMETBFT_BUILD_OPTIONS)))
-  BUILD_FLAGS += -trimpath
-  LD_FLAGS += -s -w
-endif
-
-# handle race
-ifeq (race,$(findstring race,$(COMETBFT_BUILD_OPTIONS)))
-  CGO_ENABLED=1
-  BUILD_FLAGS += -race
-endif
-
-# handle cleveldb
-ifeq (cleveldb,$(findstring cleveldb,$(COMETBFT_BUILD_OPTIONS)))
-  CGO_ENABLED=1
-  BUILD_TAGS += cleveldb
-endif
-
-# handle badgerdb
-ifeq (badgerdb,$(findstring badgerdb,$(COMETBFT_BUILD_OPTIONS)))
-  BUILD_TAGS += badgerdb
-endif
-
-# handle rocksdb
-ifeq (rocksdb,$(findstring rocksdb,$(COMETBFT_BUILD_OPTIONS)))
-  CGO_ENABLED=1
-  BUILD_TAGS += rocksdb
-endif
-
-# handle boltdb
-ifeq (boltdb,$(findstring boltdb,$(COMETBFT_BUILD_OPTIONS)))
-  BUILD_TAGS += boltdb
-endif
-
-# allow users to pass additional flags via the conventional LDFLAGS variable
-LD_FLAGS += $(LDFLAGS)
-
 # Process Docker environment varible TARGETPLATFORM
 # in order to build binary with correspondent ARCH
 # by default will always build for linux/amd64
@@ -125,6 +85,20 @@ install:
 	CGO_ENABLED=$(CGO_ENABLED) go install $(BUILD_FLAGS) -tags $(BUILD_TAGS) ./cmd/cometbft
 .PHONY: install
 
+###############################################################################
+###                               Metrics                                   ###
+###############################################################################
+
+metrics: testdata-metrics
+	go generate -run="scripts/metricsgen" ./...
+.PHONY: metrics
+
+# By convention, the go tool ignores subdirectories of directories named
+# 'testdata'. This command invokes the generate command on the folder directly
+# to avoid this.
+testdata-metrics:
+	ls ./scripts/metricsgen/testdata | xargs -I{} go generate -v -run="scripts/metricsgen" ./scripts/metricsgen/testdata/{}
+.PHONY: testdata-metrics
 
 ###############################################################################
 ###                                Mocks                                    ###
@@ -140,7 +114,7 @@ mockery:
 
 check-proto-deps:
 ifeq (,$(shell which protoc-gen-gogofaster))
-	@go install github.com/gogo/protobuf/protoc-gen-gogofaster@latest
+	@go install github.com/cosmos/gogoproto/protoc-gen-gogofaster@latest
 endif
 .PHONY: check-proto-deps
 
@@ -154,6 +128,7 @@ proto-gen: check-proto-deps
 	@echo "Generating Protobuf files"
 	@go run github.com/bufbuild/buf/cmd/buf generate
 	@mv ./proto/tendermint/abci/types.pb.go ./abci/types/
+	@cp ./proto/tendermint/rpc/grpc/types.pb.go ./rpc/grpc
 .PHONY: proto-gen
 
 # These targets are provided for convenience and are intended for local
@@ -256,7 +231,7 @@ format:
 
 lint:
 	@echo "--> Running linter"
-	@go run github.com/golangci/golangci-lint/cmd/golangci-lint run
+	@go run github.com/golangci/golangci-lint/cmd/golangci-lint@latest run
 .PHONY: lint
 
 vulncheck:
@@ -294,7 +269,7 @@ build-docker:
 
 # Build linux binary on other platforms
 build-linux:
-	GOOS=linux GOARCH=amd64 $(MAKE) build
+	GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) $(MAKE) build
 .PHONY: build-linux
 
 build-docker-localnode:
@@ -337,3 +312,25 @@ endif
 contract-tests:
 	dredd
 .PHONY: contract-tests
+
+# Implements test splitting and running. This is pulled directly from
+# the github action workflows for better local reproducibility.
+
+GO_TEST_FILES != find $(CURDIR) -name "*_test.go"
+
+# default to four splits by default
+NUM_SPLIT ?= 4
+
+$(BUILDDIR):
+	mkdir -p $@
+
+# The format statement filters out all packages that don't have tests.
+# Note we need to check for both in-package tests (.TestGoFiles) and
+# out-of-package tests (.XTestGoFiles).
+$(BUILDDIR)/packages.txt:$(GO_TEST_FILES) $(BUILDDIR)
+	go list -f "{{ if (or .TestGoFiles .XTestGoFiles) }}{{ .ImportPath }}{{ end }}" ./... | sort > $@
+
+split-test-packages:$(BUILDDIR)/packages.txt
+	split -d -n l/$(NUM_SPLIT) $< $<.
+test-group-%:split-test-packages
+	cat $(BUILDDIR)/packages.txt.$* | xargs go test -mod=readonly -timeout=15m -race -coverprofile=$(BUILDDIR)/$*.profile.out

README.md

@@ -10,11 +10,12 @@
 [![License][license-badge]][license-url]
 [![Sourcegraph][sg-badge]][sg-url]
 
-| Branch  | Tests                                    | Linting                               |
-|---------|------------------------------------------|---------------------------------------|
-| main    | [![Tests][tests-badge]][tests-url]       | [![Lint][lint-badge]][lint-url]       |
-| v0.37.x | [![Tests][tests-badge-v037x]][tests-url] | [![Lint][lint-badge-v037x]][lint-url] |
-| v0.34.x | [![Tests][tests-badge-v034x]][tests-url] | [![Lint][lint-badge-v034x]][lint-url] |
+| Branch  | Tests                                          | Linting                                     |
+|---------|------------------------------------------------|---------------------------------------------|
+| main    | [![Tests][tests-badge]][tests-url]             | [![Lint][lint-badge]][lint-url]             |
+| v0.38.x | [![Tests][tests-badge-v038x]][tests-url-v038x] | [![Lint][lint-badge-v038x]][lint-url-v038x] |
+| v0.37.x | [![Tests][tests-badge-v037x]][tests-url-v037x] | [![Lint][lint-badge-v037x]][lint-url-v037x] |
+| v0.34.x | [![Tests][tests-badge-v034x]][tests-url-v034x] | [![Lint][lint-badge-v034x]][lint-url-v034x] |
 
 CometBFT is a Byzantine Fault Tolerant (BFT) middleware that takes a
 state transition machine - written in any programming language - and securely
@@ -58,17 +59,20 @@ looking for, see [our security policy](SECURITY.md).
 
 ## Minimum requirements
 
-| Requirement | Notes             |
-|-------------|-------------------|
-| Go version  | Go 1.19 or higher |
+| CometBFT version | Requirement | Notes             |
+|------------------|-------------|-------------------|
+| main             | Go version  | Go 1.20 or higher |
+| v0.38.x          | Go version  | Go 1.20 or higher |
+| v0.37.x          | Go version  | Go 1.20 or higher |
+| v0.34.x          | Go version  | Go 1.19 or higher |
 
 ### Install
 
-See the [install instructions](./docs/introduction/install.md).
+See the [install guide](./docs/guides/install.md).
 
 ### Quick Start
 
-- [Single node](./docs/introduction/quick-start.md)
+- [Single node](./docs/guides/quick-start.md)
 - [Local cluster using docker-compose](./docs/networks/docker-compose.md)
 
 ## Contributing
@@ -118,9 +122,12 @@ CometBFT up-to-date. Upgrading instructions can be found in
 
 Currently supported versions include:
 
+- v0.38.x: CometBFT v0.38 introduces ABCI 2.0, which implements the entirety of
+  ABCI++
+- v0.37.x: CometBFT v0.37 introduces ABCI 1.0, which is the first major step
+  towards the full ABCI++ implementation in ABCI 2.0
 - v0.34.x: The CometBFT v0.34 series is compatible with the Tendermint Core
   v0.34 series
-- v0.37.x: (release candidate)
 
 ## Resources
 
@@ -142,7 +149,7 @@ Currently supported versions include:
 ### Research
 
 Below are links to the original Tendermint consensus algorithm and relevant
-whitepapers which CosmosBFT will continue to build on.
+whitepapers which CometBFT will continue to build on.
 
 - [The latest gossip on BFT consensus](https://arxiv.org/abs/1807.04938)
 - [Master's Thesis on Tendermint](https://atrium.lib.uoguelph.ca/xmlui/handle/10214/9769)
@@ -165,7 +172,7 @@ maintains [cometbft.com](https://cometbft.com).
 [version-url]: https://github.com/cometbft/cometbft/releases/latest
 [api-badge]: https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667
 [api-url]: https://pkg.go.dev/github.com/cometbft/cometbft
-[go-badge]: https://img.shields.io/badge/go-1.19-blue.svg
+[go-badge]: https://img.shields.io/badge/go-1.20-blue.svg
 [go-url]: https://github.com/moovweb/gvm
 [discord-badge]: https://img.shields.io/discord/669268347736686612.svg
 [discord-url]: https://discord.gg/cosmosnetwork
@@ -174,11 +181,19 @@ maintains [cometbft.com](https://cometbft.com).
 [sg-badge]: https://sourcegraph.com/github.com/cometbft/cometbft/-/badge.svg
 [sg-url]: https://sourcegraph.com/github.com/cometbft/cometbft?badge
 [tests-url]: https://github.com/cometbft/cometbft/actions/workflows/tests.yml
+[tests-url-v038x]: https://github.com/cometbft/cometbft/actions/workflows/tests.yml?query=branch%3Av0.38.x
+[tests-url-v037x]: https://github.com/cometbft/cometbft/actions/workflows/tests.yml?query=branch%3Av0.37.x
+[tests-url-v034x]: https://github.com/cometbft/cometbft/actions/workflows/tests.yml?query=branch%3Av0.34.x
 [tests-badge]: https://github.com/cometbft/cometbft/actions/workflows/tests.yml/badge.svg?branch=main
+[tests-badge-v038x]: https://github.com/cometbft/cometbft/actions/workflows/tests.yml/badge.svg?branch=v0.38.x
 [tests-badge-v037x]: https://github.com/cometbft/cometbft/actions/workflows/tests.yml/badge.svg?branch=v0.37.x
 [tests-badge-v034x]: https://github.com/cometbft/cometbft/actions/workflows/tests.yml/badge.svg?branch=v0.34.x
 [lint-badge]: https://github.com/cometbft/cometbft/actions/workflows/lint.yml/badge.svg?branch=main
 [lint-badge-v034x]: https://github.com/cometbft/cometbft/actions/workflows/lint.yml/badge.svg?branch=v0.34.x
 [lint-badge-v037x]: https://github.com/cometbft/cometbft/actions/workflows/lint.yml/badge.svg?branch=v0.37.x
+[lint-badge-v038x]: https://github.com/cometbft/cometbft/actions/workflows/lint.yml/badge.svg?branch=v0.38.x
 [lint-url]: https://github.com/cometbft/cometbft/actions/workflows/lint.yml
+[lint-url-v034x]: https://github.com/cometbft/cometbft/actions/workflows/lint.yml?query=branch%3Av0.34.x
+[lint-url-v037x]: https://github.com/cometbft/cometbft/actions/workflows/lint.yml?query=branch%3Av0.37.x
+[lint-url-v038x]: https://github.com/cometbft/cometbft/actions/workflows/lint.yml?query=branch%3Av0.38.x
 [tm-core]: https://github.com/tendermint/tendermint

RELEASES.md

@@ -93,6 +93,15 @@ the 0.38.x line.
    Be sure to merge this PR before making other changes on the newly-created
    backport branch.
 
+5. Ensure that the RPC docs' `version` field in `rpc/openapi/openapi.yaml` has
+   been updated from `main` to the backport branch version.
+
+6. Prepare the [CometBFT documentation
+   repository](https://github.com/cometbft/cometbft-docs) to build the release
+   branch's version by updating the
+   [VERSIONS](https://github.com/cometbft/cometbft-docs/blob/main/VERSIONS)
+   file.
+
 After doing these steps, go back to `main` and do the following:
 
 1. Create a new workflow to run e2e nightlies for the new backport branch. (See
@@ -148,7 +157,7 @@ backport branch (see above). Otherwise:
 1. Start from the backport branch (e.g. `v0.38.x`).
 2. Run the integration tests and the E2E nightlies
    (which can be triggered from the GitHub UI;
-   e.g., <https://github.com/cometbft/cometbft/actions/workflows/e2e-nightly-37x.yml>).
+   e.g., <https://github.com/cometbft/cometbft/actions/workflows/e2e-manual.yml>).
 3. Prepare the pre-release documentation:
    * Build the changelog with [unclog] _without_ doing an unclog release, and
      commit the built changelog. This ensures that all changelog entries appear
@@ -316,8 +325,8 @@ and limitation that real-world  deployments of CometBFT experience in production
 #### 200 Node Testnet
 
 To test the stability and performance of CometBFT in a real world scenario,
-a 200 node test network is run. The network comprises 5 seed nodes, 100
-validators and 95 non-validating full nodes. All nodes begin by dialing
+a 200 node test network is run. The network comprises 5 seed nodes, 175
+validators and 20 non-validating full nodes. All nodes begin by dialing
 a subset of the seed nodes to discover peers. The network is run for several
 days, with metrics being collected continuously. In cases of changes to performance
 critical systems, testnets of larger sizes should be considered.
@@ -333,6 +342,20 @@ to blocksync to the head of the chain and begins producing blocks using
 consensus it is stopped. Once stopped, a new node is started and
 takes its place. This network is run for several days.
 
+#### Vote-extension Testnet
+
+CometBFT v0.38.0 introduced **vote-extensions**, which are added as the name suggests, to precommit votes sent by validators.
+The Vote-extension Testnet is used to determine how vote-extensions affect the performance of CometBFT, under various settings.
+The application used in the experiment is the same used on the (#200-node-testnet), but is configured differently to gauge de effects of varying vote extension sizes.
+In the (#200-node-testnet) the application extends pre-commit votes with a 64 bit number encoded with variable compression.
+In the Vote-extension Testnet, pre-commit votes are extended with a non-compressed extension of configurable size.
+Experiments are run with multiple sizes to determine their impact and, for comparison sake, we include a run with the same settings as in the (#200-node-testnet).
+
+The testnet consists of 175 validators, 20 non-validator full-nodes, and 5 seed nodes.
+All 195 full-nodes begin by dialing a subset of the seed nodes to discover peers.
+Once all full-nodes are started, a 5 minute period is waited before starting an experiment.
+For each experiment, the load generators issue requests at a constant rate during 150 seconds, then wait for 5 minutes to allow the system to quiesce, then repeat the load generation; the load generation step is repeated 5 times for each experiment.
+
 #### Network Partition Testnet
 
 CometBFT is expected to recover from network partitions. A partition where no