diff --git a/.github/scripts/offline-upgrade-test.sh b/.github/scripts/offline-upgrade-test.sh new file mode 100644 index 0000000000..5fda03c416 --- /dev/null +++ b/.github/scripts/offline-upgrade-test.sh @@ -0,0 +1,185 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +readonly REPO_ROOT="$(git rev-parse --show-toplevel)" +readonly RUN_ROOT="${RUNNER_TEMP:-/tmp}/sei-offline-upgrade-${GITHUB_RUN_ID:-$$}" +readonly SOURCE_WORKTREE="$RUN_ROOT/source" +readonly TARGET_WORKTREE="$RUN_ROOT/target" +readonly ARTIFACT_ROOT="$RUN_ROOT/artifacts" +readonly STATE_ARTIFACT="$ARTIFACT_ROOT/state" + +FROM_REF="${FROM_REF:-}" +TO_REF="${TO_REF:-${GITHUB_SHA:-HEAD}}" +BOUNDARY_FROM= +BOUNDARY_TO= +UPGRADE_TAG= + +log() { + printf '\n[%s] %s\n' "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" "$*" +} + +die() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +prepare_boundary() { + BOUNDARY_FROM="$(go run ./upgradetest/cmd/boundary from)" + BOUNDARY_TO="$(go run ./upgradetest/cmd/boundary to)" + UPGRADE_TAG="$(go run ./upgradetest/cmd/boundary tag)" + FROM_REF="${FROM_REF:-release/$BOUNDARY_FROM}" +} + +validate_inputs() { + [[ "$FROM_REF" =~ ^release/v[0-9]+\.[0-9]+(\.[0-9]+)?(-branch)?$ ]] || + die "from_ref must look like release/v6.6 or release/v6.2.0-branch" + [[ "$TO_REF" =~ ^[A-Za-z0-9][A-Za-z0-9._/-]*$ ]] && + [[ "$TO_REF" != *..* ]] && [[ "$TO_REF" != *@\{* ]] || + die "to_ref contains characters Git cannot safely resolve" + + local phase + for phase in source target; do + [[ -f "$REPO_ROOT/app/${UPGRADE_TAG}_offline_${phase}_test.go" ]] || + die "$UPGRADE_TAG has no offline $phase test" + done + grep -q 'func Test.*OfflineUpgradeReopen(' \ + "$REPO_ROOT/app/${UPGRADE_TAG}_offline_source_test.go" || + die "$UPGRADE_TAG has no offline reopen test" + [[ -f "$REPO_ROOT/app/upgrade_offline_harness_test.go" ]] || + die "offline upgrade harness is missing" +} + +resolve_ref() { + local ref="$1" + if [[ "$ref" == "HEAD" || "$ref" =~ ^[0-9a-fA-F]{40}$ ]]; then + git rev-parse --verify "$ref^{commit}" + return + fi + + git fetch --no-tags origin "$ref" >&2 + git rev-parse --verify FETCH_HEAD +} + +prepare_worktrees() { + local source_sha + local target_sha + source_sha="$(resolve_ref "$FROM_REF")" || + die "unable to resolve source ref $FROM_REF" + target_sha="$(resolve_ref "$TO_REF")" || + die "unable to resolve target ref $TO_REF" + + git worktree add --detach "$SOURCE_WORKTREE" "$source_sha" + git worktree add --detach "$TARGET_WORKTREE" "$target_sha" + + grep -Fxq "$BOUNDARY_FROM" "$SOURCE_WORKTREE/app/tags" || + die "$FROM_REF does not contain source upgrade $BOUNDARY_FROM" + ! grep -Fxq "$BOUNDARY_TO" "$SOURCE_WORKTREE/app/tags" || + die "$FROM_REF already contains target upgrade $BOUNDARY_TO" + grep -Fxq "$BOUNDARY_TO" "$TARGET_WORKTREE/app/tags" || + die "$TO_REF does not contain target upgrade $BOUNDARY_TO" + + { + printf 'source_ref=%s\n' "$FROM_REF" + printf 'source_sha=%s\n' "$source_sha" + printf 'target_ref=%s\n' "$TO_REF" + printf 'target_sha=%s\n' "$target_sha" + printf 'boundary_from=%s\n' "$BOUNDARY_FROM" + printf 'boundary_to=%s\n' "$BOUNDARY_TO" + printf 'upgrade_tag=%s\n' "$UPGRADE_TAG" + } >"$ARTIFACT_ROOT/revisions.txt" +} + +install_phase_tests() { + local worktree="$1" + local phase="$2" + install -m 0644 \ + "$REPO_ROOT/app/upgrade_offline_harness_test.go" \ + "$worktree/app/upgrade_offline_harness_test.go" + install -m 0644 \ + "$REPO_ROOT/app/${UPGRADE_TAG}_offline_${phase}_test.go" \ + "$worktree/app/${UPGRADE_TAG}_offline_${phase}_test.go" +} + +run_phase() { + local phase="$1" + local worktree="$2" + local test_suffix + local upgrade_list + local file_phase + case "$phase" in + source) + test_suffix=Source + upgrade_list= + file_phase=source + ;; + target) + test_suffix=Target + upgrade_list="$BOUNDARY_TO" + file_phase=target + ;; + reopen) + test_suffix=Reopen + upgrade_list= + file_phase=source + ;; + *) die "unknown offline upgrade phase $phase" ;; + esac + install_phase_tests "$worktree" "$file_phase" + + log "Running $UPGRADE_TAG offline $phase phase against $(git -C "$worktree" rev-parse --short HEAD)" + ( + cd "$worktree" + local tests + local listing_stdout="$ARTIFACT_ROOT/$phase-list.stdout" + local listing_stderr="$ARTIFACT_ROOT/$phase-list.stderr" + if ! go test \ + -tags="$UPGRADE_TAG,offline_upgrade,upgrade_$file_phase" \ + -list "^Test.*OfflineUpgrade${test_suffix}$" \ + ./app >"$listing_stdout" 2>"$listing_stderr"; then + cat "$listing_stdout" "$listing_stderr" >&2 + die "$UPGRADE_TAG $phase phase failed to list offline tests" + fi + tests="$(awk '/^Test.*OfflineUpgrade(Source|Target|Reopen)$/ { print }' "$listing_stdout")" + if [[ -z "$tests" ]]; then + cat "$listing_stderr" >&2 + die "$UPGRADE_TAG $phase phase selected no offline test" + fi + rm -f "$listing_stdout" "$listing_stderr" + + UPGRADE_TEST_PHASE="$phase" \ + UPGRADE_TEST_ARTIFACT="$STATE_ARTIFACT" \ + UPGRADE_VERSION_LIST="$upgrade_list" \ + go test \ + -tags="$UPGRADE_TAG,offline_upgrade,upgrade_$file_phase" \ + -run "^Test.*OfflineUpgrade${test_suffix}$" \ + -count=1 \ + -timeout=10m \ + ./app + ) 2>&1 | tee "$ARTIFACT_ROOT/$phase.log" +} + +cleanup() { + local exit_code=$? + trap - EXIT + set +e + git -C "$REPO_ROOT" worktree remove --force "$SOURCE_WORKTREE" 2>/dev/null + git -C "$REPO_ROOT" worktree remove --force "$TARGET_WORKTREE" 2>/dev/null + exit "$exit_code" +} + +main() { + prepare_boundary + validate_inputs + mkdir -p "$RUN_ROOT" "$ARTIFACT_ROOT" "$STATE_ARTIFACT" + exec > >(tee "$ARTIFACT_ROOT/runner.log") 2>&1 + trap cleanup EXIT + + prepare_worktrees + run_phase source "$SOURCE_WORKTREE" + run_phase target "$TARGET_WORKTREE" + run_phase reopen "$SOURCE_WORKTREE" + log "Offline upgrade $BOUNDARY_FROM -> $BOUNDARY_TO succeeded" +} + +main "$@" diff --git a/.github/scripts/release-upgrade-test.sh b/.github/scripts/release-upgrade-test.sh index f6559c60e7..692eba7904 100755 --- a/.github/scripts/release-upgrade-test.sh +++ b/.github/scripts/release-upgrade-test.sh @@ -7,14 +7,20 @@ readonly RUN_ROOT="${RUNNER_TEMP:-/tmp}/sei-release-upgrade-${GITHUB_RUN_ID:-$$} readonly MAIN_WORKTREE="$RUN_ROOT/main" readonly RELEASE_WORKTREE="$RUN_ROOT/release" readonly BUILD_ROOT="$RUN_ROOT/bin" -readonly ARTIFACT_ROOT="$REPO_ROOT/artifacts/release-upgrade" +readonly ARTIFACT_ROOT="$RUN_ROOT/artifacts" readonly NODE_COUNT=4 +readonly CROSS_VERSION_ARTIFACT="$ARTIFACT_ROOT/cross-version.json" RELEASE_BRANCH="${RELEASE_BRANCH:-}" +MAIN_REF="${MAIN_REF:-${GITHUB_SHA:-HEAD}}" UPGRADE_LEAD_SECONDS="${UPGRADE_LEAD_SECONDS:-60}" POST_UPGRADE_BLOCKS="${POST_UPGRADE_BLOCKS:-10}" CLUSTER_STARTED=false MAIN_BINARY_HASH= +BOUNDARY_FROM= +UPGRADE_NAME= +UPGRADE_TAG= +CROSS_VERSION_TESTS= log() { printf '\n[%s] %s\n' "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" "$*" @@ -40,41 +46,9 @@ validate_inputs() { [[ "$RELEASE_BRANCH" =~ ^release/v[0-9]+\.[0-9]+(\.[0-9]+)?(-branch)?$ ]] || die "release_branch must look like release/v6.6 or release/v6.2.0-branch" fi -} - -resolve_latest_release_branch() { - local heads_file="$RUN_ROOT/release-heads" - git ls-remote --heads origin 'release/v*' >"$heads_file" - - python3 - "$heads_file" <<'PY' -import re -import sys - -patterns = ( - (re.compile(r"^refs/heads/(release/v(\d+)\.(\d+))$"), 2), - (re.compile(r"^refs/heads/(release/v(\d+)\.(\d+)\.(\d+))$"), 1), - (re.compile(r"^refs/heads/(release/v(\d+)\.(\d+)\.(\d+)-branch)$"), 0), -) - -candidates = [] -with open(sys.argv[1], encoding="utf-8") as heads: - for line in heads: - _, ref = line.split() - for pattern, preference in patterns: - match = pattern.fullmatch(ref) - if not match: - continue - branch = match.group(1) - numbers = tuple(int(part) for part in match.groups()[1:]) - version = numbers if len(numbers) == 3 else (*numbers, 0) - candidates.append((version, preference, branch)) - break - -if not candidates: - raise SystemExit("no official release/v* branch found") - -print(max(candidates)[2]) -PY + [[ "$MAIN_REF" =~ ^[A-Za-z0-9][A-Za-z0-9._/-]*$ ]] && + [[ "$MAIN_REF" != *..* ]] && [[ "$MAIN_REF" != *@\{* ]] || + die "main_ref contains characters Git cannot safely resolve" } latest_upgrade_tag() { @@ -97,55 +71,44 @@ print(max(versions)[1]) PY } -version_greater_than() { - python3 - "$1" "$2" <<'PY' -import re -import sys - -def parse(value): - match = re.fullmatch(r"v(\d+)\.(\d+)(?:\.(\d+))?", value) - if not match: - raise SystemExit(f"invalid upgrade version: {value}") - return tuple(int(part or 0) for part in match.groups()) - -raise SystemExit(0 if parse(sys.argv[1]) > parse(sys.argv[2]) else 1) -PY +has_upgrade_tag() { + local source_dir="$1" + local upgrade_name="$2" + grep -Fxq "$upgrade_name" "$source_dir/app/tags" } -next_minor_version() { - python3 - "$1" "$2" <<'PY' -import re -import sys +prepare_boundary() { + BOUNDARY_FROM="$(go run ./upgradetest/cmd/boundary from)" + UPGRADE_NAME="$(go run ./upgradetest/cmd/boundary to)" + UPGRADE_TAG="$(go run ./upgradetest/cmd/boundary tag)" + RELEASE_BRANCH="${RELEASE_BRANCH:-release/$BOUNDARY_FROM}" +} -def parse(value): - match = re.fullmatch(r"v(\d+)\.(\d+)(?:\.(\d+))?", value) - if not match: - raise SystemExit(f"invalid upgrade version: {value}") - return tuple(int(part or 0) for part in match.groups()) +resolve_ref() { + local ref="$1" + if [[ "$ref" == "HEAD" || "$ref" =~ ^[0-9a-fA-F]{40}$ ]]; then + git rev-parse --verify "$ref^{commit}" + return + fi -highest = max(parse(sys.argv[1]), parse(sys.argv[2])) -print(f"v{highest[0]}.{highest[1] + 1}") -PY + git fetch --no-tags origin "$ref" >&2 + git rev-parse --verify FETCH_HEAD } prepare_worktrees() { - if [[ -z "$RELEASE_BRANCH" ]]; then - RELEASE_BRANCH="$(resolve_latest_release_branch)" - fi - - log "Pinning origin/main and $RELEASE_BRANCH" - git fetch --no-tags origin main + log "Pinning $MAIN_REF and $RELEASE_BRANCH" local main_sha - main_sha="$(git rev-parse FETCH_HEAD)" - - git fetch --no-tags origin "$RELEASE_BRANCH" + main_sha="$(resolve_ref "$MAIN_REF")" || + die "unable to resolve target ref $MAIN_REF" local release_sha - release_sha="$(git rev-parse FETCH_HEAD)" + release_sha="$(resolve_ref "$RELEASE_BRANCH")" || + die "unable to resolve source ref $RELEASE_BRANCH" git worktree add --detach "$MAIN_WORKTREE" "$main_sha" git worktree add --detach "$RELEASE_WORKTREE" "$release_sha" { + printf 'main_ref=%s\n' "$MAIN_REF" printf 'main_sha=%s\n' "$main_sha" printf 'release_branch=%s\n' "$RELEASE_BRANCH" printf 'release_sha=%s\n' "$release_sha" @@ -158,34 +121,59 @@ prepare_upgrade_name() { release_upgrade="$(latest_upgrade_tag "$RELEASE_WORKTREE")" main_upgrade="$(latest_upgrade_tag "$MAIN_WORKTREE")" - if version_greater_than "$main_upgrade" "$release_upgrade"; then - UPGRADE_NAME="$main_upgrade" - log "Using main upgrade name $UPGRADE_NAME" - else - UPGRADE_NAME="$(next_minor_version "$main_upgrade" "$release_upgrade")" - log "Generating synthetic main upgrade name $UPGRADE_NAME" - python3 - "$MAIN_WORKTREE/app/tags" "$UPGRADE_NAME" <<'PY' -import pathlib -import sys + has_upgrade_tag "$RELEASE_WORKTREE" "$BOUNDARY_FROM" || + die "$RELEASE_BRANCH does not contain source upgrade $BOUNDARY_FROM" + ! has_upgrade_tag "$RELEASE_WORKTREE" "$UPGRADE_NAME" || + die "$RELEASE_BRANCH already contains $UPGRADE_NAME; it cannot test that boundary" + has_upgrade_tag "$MAIN_WORKTREE" "$UPGRADE_NAME" || + die "$MAIN_REF does not contain target upgrade $UPGRADE_NAME" -tags_path = pathlib.Path(sys.argv[1]) -content = tags_path.read_bytes() -separator = b"" if not content or content.endswith(b"\n") else b"\n" -tags_path.write_bytes(content + separator + sys.argv[2].encode() + b"\n") -PY - ( - cd "$MAIN_WORKTREE" - go run ./scripts/bump_version - ) 2>&1 | tee "$ARTIFACT_ROOT/precompile-generation.log" - fi + log "Testing $RELEASE_BRANCH ($release_upgrade) -> $MAIN_REF ($main_upgrade) with $UPGRADE_NAME" { printf 'release_upgrade=%s\n' "$release_upgrade" - printf 'main_upgrade_before_generation=%s\n' "$main_upgrade" + printf 'main_latest_upgrade=%s\n' "$main_upgrade" + printf 'boundary_from=%s\n' "$BOUNDARY_FROM" printf 'test_upgrade=%s\n' "$UPGRADE_NAME" + printf 'upgrade_tag=%s\n' "$UPGRADE_TAG" } | tee -a "$ARTIFACT_ROOT/revisions.txt" } +discover_cross_version_tests() { + local listed + local listing_stdout="$ARTIFACT_ROOT/cross-version-list.stdout" + local listing_stderr="$ARTIFACT_ROOT/cross-version-list.stderr" + if ! go test -tags "$UPGRADE_TAG" -list '^Test.*CrossVersion$' ./app \ + >"$listing_stdout" 2>"$listing_stderr"; then + cat "$listing_stdout" "$listing_stderr" >&2 + die "failed to list Test*CrossVersion assertions for build tag $UPGRADE_TAG" + fi + listed="$(awk '/^Test.*CrossVersion$/ { print }' "$listing_stdout")" + if [[ -z "$listed" ]]; then + cat "$listing_stderr" >&2 + die "build tag $UPGRADE_TAG defines no Test*CrossVersion assertion" + fi + rm -f "$listing_stdout" "$listing_stderr" + CROSS_VERSION_TESTS="$(paste -sd'|' - <<<"$listed")" +} + +run_cross_version_phase() { + local phase="$1" + log "Running $UPGRADE_TAG cross-version assertions ($phase)" + UPGRADE_TEST_PHASE="$phase" \ + UPGRADE_TEST_ARTIFACT="$CROSS_VERSION_ARTIFACT" \ + UPGRADE_TEST_NODE="sei-node-0" \ + UPGRADE_TEST_UPGRADE_NAME="$UPGRADE_NAME" \ + UPGRADE_TEST_TARGET_HEIGHT="${TARGET_HEIGHT:-}" \ + UPGRADE_TEST_RELEASE_BINARY="/tmp/seid.release" \ + go test -tags "$UPGRADE_TAG" \ + -run "^($CROSS_VERSION_TESTS)$" \ + -count=1 \ + -timeout=15m \ + ./app 2>&1 | + tee "$ARTIFACT_ROOT/cross-version-$phase.log" +} + build_localnode_image() { log "Building the localnode toolchain image" ( @@ -198,8 +186,12 @@ build_binary() { local source_dir="$1" local output_path="$2" local label="$3" + local source_commit + local source_version local go_mod_cache local go_build_cache + source_commit="$(git -C "$source_dir" rev-parse HEAD)" + source_version="$(git -C "$source_dir" describe --tags --always)" go_mod_cache="$(go env GOMODCACHE)" go_build_cache="$(go env GOCACHE)" mkdir -p "$go_mod_cache" "$go_build_cache" @@ -213,8 +205,11 @@ build_binary() { -v "$go_build_cache:/root/.cache/go-build:Z" \ -w /sei-protocol/sei-chain \ -e LEDGER_ENABLED=false \ + -e GOFLAGS=-buildvcs=false \ + -e "BUILD_COMMIT=$source_commit" \ + -e "BUILD_VERSION=$source_version" \ sei-chain/localnode \ - bash -c 'export PATH=/usr/local/go/bin:$PATH && make clean && make build-linux' + bash -c 'export PATH=/usr/local/go/bin:$PATH && make clean && make VERSION="$BUILD_VERSION" COMMIT="$BUILD_COMMIT" build-linux' install -m 0755 "$source_dir/build/seid" "$output_path" sha256sum "$output_path" | tee "$ARTIFACT_ROOT/$label.sha256" @@ -353,6 +348,14 @@ stage_main_binary() { local node local actual_hash + # Every validator keeps its own copy of the binary it is running, because a + # test may put the old binary back on any node, not only the one it queries. + for ((i = 0; i < NODE_COUNT; i++)); do + node="sei-node-$i" + docker exec --user root "$node" \ + sh -c 'cp /root/go/bin/seid /tmp/seid.release && chmod 0755 /tmp/seid.release' + done + for ((i = 0; i < NODE_COUNT; i++)); do node="sei-node-$i" docker cp "$BUILD_ROOT/main-seid" "$node:/tmp/seid.next" @@ -530,16 +533,19 @@ verify_post_upgrade() { local current for ((i = 0; i < NODE_COUNT; i++)); do current="$(height "sei-node-$i")" - ((current > maximum)) && maximum="$current" + if ((current > maximum)); then + maximum="$current" + fi if [[ -z "$minimum" ]] || ((current < minimum)); then minimum="$current" fi + printf 'post_upgrade_node_%s_height=%s\n' "$i" "$current" | + tee -a "$ARTIFACT_ROOT/revisions.txt" done ((maximum - minimum <= 3)) || die "validators are not synchronized after upgrade (min=$minimum max=$maximum)" - printf 'post_upgrade_min_height=%s\n' "$minimum" | - tee -a "$ARTIFACT_ROOT/revisions.txt" + run_cross_version_phase after log "Upgrade $UPGRADE_NAME succeeded" } @@ -577,16 +583,20 @@ cleanup() { } main() { + prepare_boundary validate_inputs mkdir -p "$RUN_ROOT" "$BUILD_ROOT" "$ARTIFACT_ROOT" + exec > >(tee "$ARTIFACT_ROOT/runner.log") 2>&1 trap cleanup EXIT prepare_worktrees prepare_upgrade_name + discover_cross_version_tests build_localnode_image build_binary "$RELEASE_WORKTREE" "$BUILD_ROOT/release-seid" release build_binary "$MAIN_WORKTREE" "$BUILD_ROOT/main-seid" main start_release_cluster + run_cross_version_phase before stage_main_binary submit_upgrade upgrade_nodes_as_they_halt diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index f0364023f7..c2875a2585 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -56,6 +56,18 @@ jobs: echo "$PKGS" | xargs go test \ -timeout=${{ env.GO_TEST_TIMEOUT }} + # The file describing the upgrade being shipped sits behind a build tag, + # so the run above compiles none of it. The target derives the tag from + # app/tags rather than taking one from here, which is what stops this step + # from going on running an upgrade that already shipped. + - name: Minor upgrade test + run: make upgrade-test + + # Files for upgrades that already shipped are no longer run, but they have + # to keep compiling. The run above and golangci-lint both skip them. + - name: Minor upgrade tests compile + run: make upgrade-test-vet + coverage: name: Coverage runs-on: ${{ github.event_name == 'merge_group' && 'ubuntu-latest' || 'uci-default' }} diff --git a/.github/workflows/offline-upgrade-test.yml b/.github/workflows/offline-upgrade-test.yml new file mode 100644 index 0000000000..18e6f35fb3 --- /dev/null +++ b/.github/workflows/offline-upgrade-test.yml @@ -0,0 +1,73 @@ +name: Offline Minor Upgrade Test + +on: + pull_request: + branches: + - main + paths: + - .github/scripts/offline-upgrade-test.sh + - .github/workflows/offline-upgrade-test.yml + - Makefile + - app/app.go + - app/tags + - app/upgrade_*_offline_*_test.go + - app/upgrade_offline_harness_test.go + - app/upgrades.go + - upgradetest/** + workflow_dispatch: + inputs: + source_ref: + description: Source branch (empty derives release/vMAJOR.MINOR) + required: false + type: string + target_ref: + description: Target branch or commit (empty uses this workflow commit) + required: false + type: string + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + offline-upgrade: + name: Persisted Go boundary + runs-on: uci-default + timeout-minutes: 30 + env: + FROM_REF: ${{ inputs.source_ref || '' }} + TO_REF: ${{ inputs.target_ref || github.sha }} + steps: + # See: https://github.com/actions/checkout/releases/tag/v7.0.0 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + fetch-depth: 0 + + # See: https://github.com/actions/setup-go/releases/tag/v6 + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 + with: + go-version: '1.25.6' + cache: false + + - name: Run persisted cross-branch Go test + run: make upgrade-test-offline FROM_REF="$FROM_REF" TO_REF="$TO_REF" + + - name: Upload offline upgrade diagnostics + if: ${{ always() }} + # See: https://github.com/actions/upload-artifact/releases/tag/v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: offline-upgrade-${{ github.run_id }} + path: | + ${{ runner.temp }}/sei-offline-upgrade-${{ github.run_id }}/artifacts/*.log + ${{ runner.temp }}/sei-offline-upgrade-${{ github.run_id }}/artifacts/revisions.txt + ${{ runner.temp }}/sei-offline-upgrade-${{ github.run_id }}/artifacts/state/manifest.json + if-no-files-found: warn + retention-days: 7 diff --git a/.github/workflows/release-upgrade-test.yml b/.github/workflows/release-upgrade-test.yml index de7f73374c..fb64fbfa6e 100644 --- a/.github/workflows/release-upgrade-test.yml +++ b/.github/workflows/release-upgrade-test.yml @@ -1,4 +1,4 @@ -name: Release to Main Upgrade Test +name: Minor Release Upgrade Test on: pull_request: @@ -7,12 +7,23 @@ on: paths: - .github/workflows/release-upgrade-test.yml - .github/scripts/release-upgrade-test.sh + - Makefile + - app/app.go + - app/tags + - app/upgrade_v*_test.go + - app/upgrades.go + - docker/** + - upgradetest/** schedule: - cron: '17 7 * * *' workflow_dispatch: inputs: release_branch: - description: Official release branch to test (empty selects the latest) + description: Source branch (empty derives release/vMAJOR.MINOR from the tagged boundary) + required: false + type: string + target_ref: + description: Target branch or commit (empty uses this workflow commit) required: false type: string upgrade_lead_seconds: @@ -39,10 +50,11 @@ defaults: jobs: release-to-main: - name: Latest release to main + name: Minor release boundary runs-on: ubuntu-large timeout-minutes: 90 env: + MAIN_REF: ${{ inputs.target_ref || github.sha }} RELEASE_BRANCH: ${{ inputs.release_branch || '' }} UPGRADE_LEAD_SECONDS: ${{ inputs.upgrade_lead_seconds || '60' }} POST_UPGRADE_BLOCKS: ${{ inputs.post_upgrade_blocks || '10' }} @@ -75,6 +87,6 @@ jobs: uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: release-upgrade-${{ github.run_id }} - path: artifacts/release-upgrade + path: ${{ runner.temp }}/sei-release-upgrade-${{ github.run_id }}/artifacts if-no-files-found: warn retention-days: 7 diff --git a/AGENTS.md b/AGENTS.md index 3e25195fe0..04b19a611c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,6 +14,37 @@ progressively the deeper you go. Existing package guides include: - `evmrpc/AGENTS.md` — EVM JSON-RPC (`eth_*`, `sei_*`, `debug_*`) semantics - `x/evm/AGENTS.md` — EVM module: address association, StateDB bridge, precompiles, pointers - `sei-tendermint/AGENTS.md` — sei-tendermint module conventions +- `testutil/configtest/AGENTS.md` — configuration characterization: how to pin a new key, section, or default +- `upgradetest/AGENTS.md` — upgrade boundary tests: how a release's upgrade coverage is scoped and selected + +## Configuration reads + +How a seid node resolves configuration is pinned by the characterization suite in +`testutil/configtest`. Renaming a key the suite covers, changing a default, or changing +how a value is cast will fail that suite. **Adding** a key does not always: the completeness +check compares struct fields, so a second key landing in a field some row already +claims is uncaught and the row has to be written by hand. Where there is a failure it +is the review prompt: record the new behavior so the old and new value land in a diff, +rather than skipping the row or widening the assertion until it passes. Read +[`testutil/configtest/AGENTS.md`](testutil/configtest/AGENTS.md) before changing a +configuration read, and before adding one. + +## Upgrade names + +Appending a name to `app/tags` moves the upgrade boundary this build ships, and +`upgradetest` derives from that list which test set CI runs. The move fails +`TestCurrentBoundaryHasATestFile` until a file for the new boundary exists, and +that failure is the review prompt: state what the upgrade changes and what it may +not, rather than carrying the previous release's cases forward or deleting them. +Read [`upgradetest/AGENTS.md`](upgradetest/AGENTS.md) before adding an upgrade +name, and before changing an upgrade handler. + +Create the v6.7 definition with +`make new-upgrade-test FROM=v6.6 TO=v6.7`; do not hand-name its build tag. +Exercise its persisted Go boundary with +`make upgrade-test-offline FROM_REF=release/v6.6 TO_REF=release/v6.7`, and its +real node boundary with +`make upgrade-test-cross-version FROM_REF=release/v6.6 TO_REF=release/v6.7`. ## Code style diff --git a/Makefile b/Makefile index fcce17fb4d..63ec78b856 100644 --- a/Makefile +++ b/Makefile @@ -631,6 +631,85 @@ giga-mixed-integration-test: .PHONY: giga-mixed-integration-test +# Create the tagged app file in which a minor upgrade test is defined. +# +# make new-upgrade-test FROM=v6.6 TO=v6.7 +new-upgrade-test: + @if [ -z "$(FROM)" ] || [ -z "$(TO)" ]; then \ + echo "usage: make new-upgrade-test FROM=v6.6 TO=v6.7" >&2; \ + exit 2; \ + fi + @go run ./upgradetest/cmd/new -from "$(FROM)" -to "$(TO)" +.PHONY: new-upgrade-test + +# Run only the tests added by the version-specific file for the minor upgrade +# this build ships. Both the build tag and test names are discovered, so this +# target keeps selecting the right file without naming a version. +upgrade-test: + @set -e; \ + boundary=$$(go run ./upgradetest/cmd/boundary); \ + tag=$$(go run ./upgradetest/cmd/boundary tag) && \ + tmp=$$(mktemp -d) && trap 'rm -rf "$$tmp"' 0; \ + if ! go test -list '^Test' ./app > "$$tmp/base.stdout" 2> "$$tmp/base.stderr"; then \ + echo "failed to list untagged app tests:" >&2; \ + cat "$$tmp/base.stdout" "$$tmp/base.stderr" >&2; \ + exit 1; \ + fi; \ + awk '/^Test/ { print }' "$$tmp/base.stdout" | sort > "$$tmp/base"; \ + if ! go test -tags "$$tag" -list '^Test' ./app > "$$tmp/tagged.stdout" 2> "$$tmp/tagged.stderr"; then \ + echo "failed to list app tests with build tag $$tag:" >&2; \ + cat "$$tmp/tagged.stdout" "$$tmp/tagged.stderr" >&2; \ + exit 1; \ + fi; \ + awk '/^Test/ { print }' "$$tmp/tagged.stdout" | sort > "$$tmp/tagged"; \ + comm -13 "$$tmp/base" "$$tmp/tagged" > "$$tmp/selected"; \ + if [ ! -s "$$tmp/selected" ]; then \ + cat "$$tmp/tagged.stderr" >&2; \ + echo "no tests were added by build tag $$tag" >&2; \ + exit 1; \ + fi; \ + tests=$$(paste -sd'|' "$$tmp/selected"); \ + echo "=== Upgrade boundary $$boundary (-tags $$tag) ==="; \ + go test -tags "$$tag" -run "^($$tests)$$" -count=1 -timeout=10m ./app +.PHONY: upgrade-test + +# Compile the current boundary's source phase against one ref, persist its app +# database, compile the target phase against another ref to apply the upgrade, +# then reopen the migrated database with the source branch. +# +# make upgrade-test-offline \ +# FROM_REF=release/v6.6 TO_REF=release/v6.7 +upgrade-test-offline: + @FROM_REF="$(FROM_REF)" TO_REF="$(TO_REF)" \ + bash .github/scripts/offline-upgrade-test.sh +.PHONY: upgrade-test-offline + +# Build two refs, create state with the source binary, coordinate the on-chain +# upgrade, and run the current build tag's CrossVersion test before and after. +# +# make upgrade-test-cross-version \ +# FROM_REF=release/v6.6 TO_REF=release/v6.7 +upgrade-test-cross-version: + @RELEASE_BRANCH="$(FROM_REF)" MAIN_REF="$(TO_REF)" \ + bash .github/scripts/release-upgrade-test.sh +.PHONY: upgrade-test-cross-version + +# Compile every version-specific app upgrade test, including versions that have +# already shipped. Offline phase files are compiled against their release side +# because source-only APIs may no longer exist in the current checkout. +upgrade-test-vet: + @set -e; \ + for file in app/upgrade_v*_test.go; do \ + [ -f "$$file" ] || continue; \ + case "$$file" in *_offline_source_test.go|*_offline_target_test.go) continue ;; esac; \ + tag=$$(basename "$$file" _test.go); \ + echo "=== Compiling $$file (-tags $$tag) ==="; \ + go test -tags "$$tag" -run '^$$' ./app; \ + done; \ + bash upgradetest/compile_offline.sh +.PHONY: upgrade-test-vet + + # Implements test splitting and running. This is pulled directly from # the github action workflows for better local reproducibility. diff --git a/app/upgrade_offline_harness_test.go b/app/upgrade_offline_harness_test.go new file mode 100644 index 0000000000..5d89750d10 --- /dev/null +++ b/app/upgrade_offline_harness_test.go @@ -0,0 +1,574 @@ +//go:build offline_upgrade + +package app + +import ( + "context" + "encoding/base64" + "encoding/hex" + "encoding/json" + "io" + "os" + "path/filepath" + "reflect" + "sort" + "testing" + + serverconfig "github.com/sei-protocol/sei-chain/sei-cosmos/server/config" + sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" + upgradetypes "github.com/sei-protocol/sei-chain/sei-cosmos/x/upgrade/types" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" + abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" + "github.com/sei-protocol/sei-chain/sei-tendermint/config" + tmproto "github.com/sei-protocol/sei-chain/sei-tendermint/proto/tendermint/types" + "github.com/sei-protocol/sei-chain/sei-wasmd/x/wasm" + evmtypes "github.com/sei-protocol/sei-chain/x/evm/types" + "github.com/stretchr/testify/require" + dbm "github.com/tendermint/tm-db" +) + +const ( + offlineUpgradeArtifactEnv = "UPGRADE_TEST_ARTIFACT" + offlineUpgradeChainID = "sei-test" + offlineUpgradePhaseEnv = "UPGRADE_TEST_PHASE" + offlineUpgradeSnapshotHomeEnv = "UPGRADE_TEST_SNAPSHOT_HOME" + // offlineUpgradeMigratedDir is the artifact subdirectory that holds the + // committed post-upgrade database. + offlineUpgradeMigratedDir = "clean" +) + +type offlineUpgradeArtifact struct { + Upgrade string `json:"upgrade"` + SourceHeight int64 `json:"source_height"` + UpgradeHeight int64 `json:"upgrade_height"` + ModuleVersions []string `json:"module_versions"` + Stores map[string]map[string]string `json:"stores"` + Retained offlineUpgradeRetainedState `json:"retained"` + MigratedRoot string `json:"migrated_root"` + UpgradeHash string `json:"upgrade_hash"` +} + +// offlineUpgradeRetainedState identifies the state the source phase wrote: the +// store keys of each retired module, the bank balances behind IBC escrow and +// voucher denoms, and the accounts a post-upgrade transaction moves funds +// between. TxSenderKey is a throwaway key generated for the fixture database, +// which the target phase needs in order to sign as that account. +type offlineUpgradeRetainedState struct { + FeegrantGranter string `json:"feegrant_granter"` + FeegrantGrantee string `json:"feegrant_grantee"` + FeegrantKey string `json:"feegrant_key"` + CapabilityName string `json:"capability_name"` + CapabilityIndex uint64 `json:"capability_index"` + CapabilityOwnersKey string `json:"capability_owners_key"` + IBCClientID string `json:"ibc_client_id"` + IBCClientStateKey string `json:"ibc_client_state_key"` + IBCConnectionID string `json:"ibc_connection_id"` + IBCConnectionKey string `json:"ibc_connection_key"` + IBCPortID string `json:"ibc_port_id"` + IBCChannelID string `json:"ibc_channel_id"` + IBCChannelKey string `json:"ibc_channel_key"` + TransferDenomHash string `json:"transfer_denom_hash"` + TransferIBCDenom string `json:"transfer_ibc_denom"` + TransferTraceKey string `json:"transfer_trace_key"` + EscrowAddress string `json:"escrow_address"` + EscrowAmount string `json:"escrow_amount"` + EscrowSupply string `json:"escrow_supply"` + VoucherHolder string `json:"voucher_holder"` + VoucherAmount string `json:"voucher_amount"` + VoucherSupply string `json:"voucher_supply"` + TxSender string `json:"tx_sender"` + TxSenderKey string `json:"tx_sender_key"` + TxRecipient string `json:"tx_recipient"` +} + +func requireOfflineUpgradePhase(t *testing.T, want string) string { + t.Helper() + require.Equal(t, want, os.Getenv(offlineUpgradePhaseEnv), + "%s must select this test phase", offlineUpgradePhaseEnv) + root := os.Getenv(offlineUpgradeArtifactEnv) + require.NotEmpty(t, root, "%s is required", offlineUpgradeArtifactEnv) + absolute, err := filepath.Abs(root) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(absolute, 0o750)) + return absolute +} + +type offlineUpgradeChainIDOpts struct { + TestAppOpts + chainID string +} + +func (o offlineUpgradeChainIDOpts) Get(s string) interface{} { + if s == "chain-id" { + return o.chainID + } + return o.TestAppOpts.Get(s) +} + +// requireOfflineUpgradeSnapshotHome returns the node home named by +// UPGRADE_TEST_SNAPSHOT_HOME. It skips when the variable is unset and fails +// when the path cannot be opened as a node home. +func requireOfflineUpgradeSnapshotHome(t *testing.T) string { + t.Helper() + home := os.Getenv(offlineUpgradeSnapshotHomeEnv) + if home == "" { + t.Skipf("%s is unset; set it to a node home directory to run retained-state checks against a real snapshot", + offlineUpgradeSnapshotHomeEnv) + } + absolute, err := filepath.Abs(home) + require.NoError(t, err, "%s=%q is not a usable path", offlineUpgradeSnapshotHomeEnv, home) + info, err := os.Stat(absolute) + require.NoError(t, err, "%s=%q does not exist", offlineUpgradeSnapshotHomeEnv, home) + require.True(t, info.IsDir(), "%s=%q is not a directory", offlineUpgradeSnapshotHomeEnv, home) + + genesis := filepath.Join(absolute, "config", "genesis.json") + _, err = os.Stat(genesis) + require.NoError(t, err, "%s=%q is not a node home: missing config/genesis.json", + offlineUpgradeSnapshotHomeEnv, home) + + // The store path a node resolves depends on which layout it was created + // with, so ask the same resolver the app itself uses rather than guessing. + commitStore := utils.GetCosmosSCStorePath(absolute) + require.True(t, utils.DirExists(commitStore), + "%s=%q has no state commitment store at %s", + offlineUpgradeSnapshotHomeEnv, home, commitStore) + return absolute +} + +func readOfflineUpgradeGenesisChainID(t *testing.T, home string) string { + t.Helper() + raw, err := os.ReadFile(filepath.Join(home, "config", "genesis.json")) + require.NoError(t, err, "read %s/config/genesis.json", home) + var genesis struct { + ChainID string `json:"chain_id"` + } + require.NoError(t, json.Unmarshal(raw, &genesis), "parse %s/config/genesis.json", home) + require.NotEmpty(t, genesis.ChainID, "%s/config/genesis.json has no chain_id", home) + return genesis.ChainID +} + +func openOfflineUpgradeSnapshotApp(t *testing.T, home, chainID string) *App { + t.Helper() + require.NotEmpty(t, chainID) + encodingConfig := MakeEncodingConfig() + options := []AppOption{ + func(app *App) { + receiptStore, receiptErr := setupReceiptStore(app.keys[evmtypes.StoreKey]) + require.NoError(t, receiptErr) + app.receiptStore = receiptStore + }, + } + testApp := New( + dbm.NewMemDB(), + nil, + true, + map[int64]bool{}, + home, + 1, + false, + config.TestConfig(), + encodingConfig, + wasm.EnableAllProposals, + offlineUpgradeChainIDOpts{chainID: chainID}, + EmptyWasmOpts, + options, + ) + requireOfflineUpgradeExecutionConfig(t, testApp) + require.Positive(t, testApp.LastBlockHeight(), + "%s opened with LastBlockHeight 0; the application database is empty or unreadable", home) + return testApp +} + +func openOfflineUpgradeApp(t *testing.T, root string, initialize bool) *App { + t.Helper() + dbDir := filepath.Join(root, "application") + require.NoError(t, os.MkdirAll(dbDir, 0o750)) + db, err := dbm.NewGoLevelDB("application", dbDir) + require.NoError(t, err) + + encodingConfig := MakeEncodingConfig() + var genesisStateBytes []byte + if initialize { + genesisState := NewDefaultGenesisState(encodingConfig.Marshaler) + genesisStateBytes, err = json.Marshal(genesisState) + require.NoError(t, err) + } + options := []AppOption{ + func(app *App) { + receiptStore, receiptErr := setupReceiptStore(app.keys[evmtypes.StoreKey]) + require.NoError(t, receiptErr) + app.receiptStore = receiptStore + }, + } + testApp := New( + db, + nil, + true, + map[int64]bool{}, + filepath.Join(root, "home"), + 1, + false, + config.TestConfig(), + encodingConfig, + wasm.EnableAllProposals, + TestAppOpts{}, + EmptyWasmOpts, + options, + ) + requireOfflineUpgradeExecutionConfig(t, testApp) + if initialize { + initializeOfflineUpgradeApp(t, testApp, genesisStateBytes) + } + return testApp +} + +// requireOfflineUpgradeExecutionConfig asserts that this harness constructs an +// app with OCC disabled and DefaultConcurrencyWorkers. The fleet sets +// occ-enabled = true and the live harness sets concurrency-workers = 4. +func requireOfflineUpgradeExecutionConfig(t *testing.T, testApp *App) { + t.Helper() + require.False(t, testApp.OccEnabled(), + "offline upgrade tests ran with BaseApp.OccEnabled()=%v, want false; this layer's application-hash determinism is not the fleet's (occ-enabled = true)", + testApp.OccEnabled()) + require.Equal(t, serverconfig.DefaultConcurrencyWorkers, testApp.ConcurrencyWorkers(), + "offline upgrade tests ran with BaseApp.ConcurrencyWorkers()=%d, want DefaultConcurrencyWorkers=%d; the live harness sets 4", + testApp.ConcurrencyWorkers(), serverconfig.DefaultConcurrencyWorkers) +} + +// initializeOfflineUpgradeApp calls the InitChain signature provided by the +// branch under test. Both signatures accept the same request. +func initializeOfflineUpgradeApp(t *testing.T, testApp *App, stateBytes []byte) { + t.Helper() + request := &abci.RequestInitChain{ + ConsensusParams: DefaultConsensusParams, + ChainId: offlineUpgradeChainID, + AppStateBytes: stateBytes, + } + + method := reflect.ValueOf(testApp).MethodByName("InitChain") + require.True(t, method.IsValid(), "app has no InitChain method") + args := []reflect.Value{reflect.ValueOf(request)} + if method.Type().NumIn() == 2 { + args = append([]reflect.Value{reflect.ValueOf(context.Background())}, args...) + } + require.Len(t, args, method.Type().NumIn(), "unsupported InitChain signature") + results := method.Call(args) + require.NotEmpty(t, results, "InitChain returned no values") + if last := results[len(results)-1]; last.Type().Implements(reflect.TypeFor[error]()) && !last.IsNil() { + t.Fatalf("InitChain: %v", last.Interface()) + } +} + +func closeOfflineUpgradeApp(t *testing.T, testApp *App) { + t.Helper() + require.NoError(t, testApp.Close()) +} + +// copyOfflineUpgradeDatabase copies the application database directories under +// srcRoot to a new dstRoot. +func copyOfflineUpgradeDatabase(t *testing.T, srcRoot, dstRoot string) { + t.Helper() + require.NoError(t, os.RemoveAll(dstRoot)) + require.NoError(t, os.MkdirAll(dstRoot, 0o750)) + for _, name := range []string{"application", "home"} { + src := filepath.Join(srcRoot, name) + if _, err := os.Stat(src); os.IsNotExist(err) { + continue + } + copyOfflineUpgradeTree(t, src, filepath.Join(dstRoot, name)) + } + _, err := os.Stat(filepath.Join(dstRoot, "application")) + require.NoError(t, err, "copied database has no application directory") +} + +func copyOfflineUpgradeTree(t *testing.T, src, dst string) { + t.Helper() + info, err := os.Stat(src) + require.NoError(t, err) + require.True(t, info.IsDir(), "%s is not a directory", src) + require.NoError(t, os.MkdirAll(dst, 0o750)) + entries, err := os.ReadDir(src) + require.NoError(t, err) + for _, entry := range entries { + from := filepath.Join(src, entry.Name()) + to := filepath.Join(dst, entry.Name()) + info, err := os.Lstat(from) + require.NoError(t, err) + switch { + case info.Mode()&os.ModeSymlink != 0: + copyOfflineUpgradeSymlink(t, from, to) + case info.IsDir(): + copyOfflineUpgradeTree(t, from, to) + default: + copyOfflineUpgradeFile(t, from, to) + } + } +} + +func copyOfflineUpgradeSymlink(t *testing.T, src, dst string) { + t.Helper() + target, err := os.Readlink(src) + require.NoError(t, err) + require.NoError(t, os.Symlink(target, dst)) +} + +func copyOfflineUpgradeFile(t *testing.T, src, dst string) { + t.Helper() + in, err := os.Open(src) + require.NoError(t, err) + defer func() { + require.NoError(t, in.Close()) + }() + info, err := in.Stat() + require.NoError(t, err) + out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, info.Mode().Perm()) + require.NoError(t, err) + defer func() { + require.NoError(t, out.Close()) + }() + _, err = io.Copy(out, in) + require.NoError(t, err) +} + +func commitOfflineUpgradeApp(t *testing.T, testApp *App) { + t.Helper() + _, err := testApp.Commit(context.Background()) + require.NoError(t, err) +} + +func committedOfflineUpgradeHash(t *testing.T, testApp *App) []byte { + t.Helper() + hash := append([]byte(nil), testApp.LastCommitID().Hash...) + require.NotEmpty(t, hash, "committed application hash is empty") + return hash +} + +func offlineUpgradeHashString(hash []byte) string { + return hex.EncodeToString(hash) +} + +// offlineUpgradeMigratedDatabase returns the committed post-upgrade database +// named by artifact.MigratedRoot under root. +func offlineUpgradeMigratedDatabase(t *testing.T, root string, artifact offlineUpgradeArtifact) string { + t.Helper() + require.NotEmpty(t, artifact.MigratedRoot, "target phase did not record the migrated database path") + require.Equal(t, filepath.Base(artifact.MigratedRoot), artifact.MigratedRoot, + "migrated database path must be a directory name under the artifact root") + migrated := filepath.Join(root, artifact.MigratedRoot) + _, err := os.Stat(filepath.Join(migrated, "application")) + require.NoError(t, err, "migrated database %s has no application directory", migrated) + return migrated +} + +func offlineUpgradeReadContext(testApp *App, height int64) sdk.Context { + return offlineUpgradeContext(testApp, height, offlineUpgradeChainID) +} + +func offlineUpgradeContext(testApp *App, height int64, chainID string) sdk.Context { + return testApp.NewUncachedContext(false, tmproto.Header{ + ChainID: chainID, + Height: height, + }) +} + +func snapshotOfflineUpgradeStore( + t *testing.T, + testApp *App, + ctx sdk.Context, + storeName string, +) map[string]string { + t.Helper() + storeKey := testApp.GetKey(storeName) + require.NotNil(t, storeKey, "%s store is not mounted", storeName) + iterator := ctx.KVStore(storeKey).Iterator(nil, nil) + defer func() { + require.NoError(t, iterator.Close()) + }() + + entries := map[string]string{} + for ; iterator.Valid(); iterator.Next() { + key := encodeOfflineUpgradeKey(iterator.Key()) + entries[key] = base64.StdEncoding.EncodeToString(iterator.Value()) + } + return entries +} + +func snapshotOfflineUpgradeStores( + t *testing.T, + testApp *App, + ctx sdk.Context, + storeNames []string, +) map[string]map[string]string { + t.Helper() + stores := make(map[string]map[string]string, len(storeNames)) + for _, storeName := range storeNames { + stores[storeName] = snapshotOfflineUpgradeStore(t, testApp, ctx, storeName) + } + return stores +} + +func encodeOfflineUpgradeKey(key []byte) string { + return base64.StdEncoding.EncodeToString(key) +} + +func requireOfflineUpgradeStoreKey(t *testing.T, snapshot map[string]string, name, key string) { + t.Helper() + require.NotEmpty(t, key, "%s was not recorded", name) + _, ok := snapshot[key] + require.True(t, ok, "retained %s disappeared", name) +} + +// committedOfflineUpgradePlan returns the pending upgrade plan in the committed +// upgrade store. +func committedOfflineUpgradePlan(t *testing.T, testApp *App) (upgradetypes.Plan, bool) { + t.Helper() + return testApp.UpgradeKeeper.GetUpgradePlan(offlineUpgradeReadContext(testApp, testApp.LastBlockHeight())) +} + +func committedUpgradeStore(t *testing.T, testApp *App) sdk.KVStore { + t.Helper() + key := testApp.GetKey(upgradetypes.StoreKey) + require.NotNil(t, key, "upgrade store is not mounted") + store := testApp.CommitMultiStore().GetCommitKVStore(key) + require.NotNil(t, store, "upgrade store is not in the commit multistore") + return store +} + +func offlineUpgradeModuleVersionKey(module string) []byte { + return append([]byte{upgradetypes.VersionMapByte}, []byte(module)...) +} + +// offlineUpgradeModuleVersions returns module names stored under the version-map +// prefix in the committed upgrade store. +func offlineUpgradeModuleVersions(t *testing.T, testApp *App) []string { + t.Helper() + iterator := sdk.KVStorePrefixIterator(committedUpgradeStore(t, testApp), []byte{upgradetypes.VersionMapByte}) + defer func() { + require.NoError(t, iterator.Close()) + }() + + names := make([]string, 0) + for ; iterator.Valid(); iterator.Next() { + key := iterator.Key() + require.Greater(t, len(key), 1) + names = append(names, string(key[1:])) + } + sort.Strings(names) + return names +} + +func offlineUpgradeHasModuleVersion(t *testing.T, testApp *App, module string) bool { + t.Helper() + return committedUpgradeStore(t, testApp).Has(offlineUpgradeModuleVersionKey(module)) +} + +func requireOfflineUpgradeStoresMounted(t *testing.T, testApp *App, storeNames []string) { + t.Helper() + cms := testApp.CommitMultiStore() + mounted := map[string]struct{}{} + for _, key := range cms.StoreKeys() { + mounted[key.Name()] = struct{}{} + } + for _, name := range storeNames { + _, ok := mounted[name] + require.True(t, ok, "%s is not present in the commit multistore", name) + key := testApp.GetKey(name) + require.NotNil(t, key, "%s store is not mounted", name) + require.NotNil(t, cms.GetCommitKVStore(key), "%s is not in the commit multistore", name) + } +} + +func snapshotCommittedOfflineUpgradeStore(t *testing.T, testApp *App, storeName string) map[string]string { + t.Helper() + storeKey := testApp.GetKey(storeName) + require.NotNil(t, storeKey, "%s store is not mounted", storeName) + store := testApp.CommitMultiStore().GetCommitKVStore(storeKey) + require.NotNil(t, store, "%s is not in the commit multistore", storeName) + iterator := store.Iterator(nil, nil) + defer func() { + require.NoError(t, iterator.Close()) + }() + entries := map[string]string{} + for ; iterator.Valid(); iterator.Next() { + entries[encodeOfflineUpgradeKey(iterator.Key())] = base64.StdEncoding.EncodeToString(iterator.Value()) + } + return entries +} + +// requireOfflineUpgradeStoreProof verifies a commitment proof for the +// lexicographically first encoded key in snapshot. +func requireOfflineUpgradeStoreProof(t *testing.T, testApp *App, storeName string, snapshot map[string]string) { + t.Helper() + require.NotEmpty(t, snapshot, "%s snapshot is empty; a proof query would be vacuous", storeName) + encodedKeys := make([]string, 0, len(snapshot)) + for encodedKey := range snapshot { + encodedKeys = append(encodedKeys, encodedKey) + } + sort.Strings(encodedKeys) + encodedKey := encodedKeys[0] + key, err := base64.StdEncoding.DecodeString(encodedKey) + require.NoError(t, err) + want, err := base64.StdEncoding.DecodeString(snapshot[encodedKey]) + require.NoError(t, err) + queryable, ok := testApp.CommitMultiStore().(sdk.Queryable) + require.True(t, ok, "commit multistore does not support queries") + resp := queryable.Query(context.Background(), abci.RequestQuery{ + Path: "/" + storeName + "/key", + Data: key, + Prove: true, + }) + require.Equal(t, uint32(0), resp.Code, "query /%s/key: %s", storeName, resp.Log) + require.Equal(t, want, resp.Value, "query /%s/key returned a different value", storeName) + require.NotNil(t, resp.ProofOps, "%s is missing from the commitment set", storeName) + require.NotEmpty(t, resp.ProofOps.Ops, "%s is missing from the commitment set", storeName) +} + +func requireOfflineUpgradeRetainedStores(t *testing.T, testApp *App, want map[string]map[string]string) { + t.Helper() + names := make([]string, 0, len(want)) + for name := range want { + names = append(names, name) + } + sort.Strings(names) + requireOfflineUpgradeStoresMounted(t, testApp, names) + for _, name := range names { + got := snapshotCommittedOfflineUpgradeStore(t, testApp, name) + require.Equal(t, want[name], got, "v6.7 changed retained %s state", name) + requireOfflineUpgradeStoreProof(t, testApp, name, want[name]) + } +} + +func writeOfflineUpgradeArtifact(t *testing.T, root string, artifact offlineUpgradeArtifact) { + t.Helper() + content, err := json.MarshalIndent(artifact, "", " ") + require.NoError(t, err) + content = append(content, '\n') + require.NoError(t, os.WriteFile(filepath.Join(root, "manifest.json"), content, 0o600)) +} + +func readOfflineUpgradeArtifact(t *testing.T, root string) offlineUpgradeArtifact { + t.Helper() + content, err := os.ReadFile(filepath.Join(root, "manifest.json")) + require.NoError(t, err) + var artifact offlineUpgradeArtifact + require.NoError(t, json.Unmarshal(content, &artifact)) + require.NotEmpty(t, artifact.Upgrade) + require.NotEmpty(t, artifact.ModuleVersions) + require.NotEmpty(t, artifact.Stores) + return artifact +} + +func offlineUpgradeDifference(left, right []string) []string { + rightSet := make(map[string]struct{}, len(right)) + for _, value := range right { + rightSet[value] = struct{}{} + } + var difference []string + for _, value := range left { + if _, ok := rightSet[value]; !ok { + difference = append(difference, value) + } + } + sort.Strings(difference) + return difference +} diff --git a/app/upgrade_orphan_test.go b/app/upgrade_orphan_test.go new file mode 100644 index 0000000000..8f76c83d0a --- /dev/null +++ b/app/upgrade_orphan_test.go @@ -0,0 +1,116 @@ +package app + +import ( + "testing" + + upgradetypes "github.com/sei-protocol/sei-chain/sei-cosmos/x/upgrade/types" + storekeys "github.com/sei-protocol/sei-chain/sei-db/common/keys" + tmproto "github.com/sei-protocol/sei-chain/sei-tendermint/proto/tendermint/types" + "github.com/stretchr/testify/require" +) + +// retainedStores are mounted KV stores that no registered module owns. A store +// stays on this list when its history must remain readable at the store level +// after its module is gone; dropping it instead is application-hash breaking +// and needs a StoreUpgrades{Deleted} entry at a specific upgrade height. +var retainedStores = map[string]string{ + feegrantModuleName: "module removed in v6.7; allowances kept for historical state access", + capabilityModuleName: "module removed in v6.7; capabilities kept for freeze-mode historical state access", + transferModuleName: "module removed in v6.7; transfer state kept for historical state access", + storekeys.IBCStoreKey: "module removed in v6.7; client, connection and channel state kept for " + + "historical state access", +} + +// storeKeyOwners names the owning module for the KV stores whose key differs +// from the module name. Every other mounted store is keyed by its own module. +var storeKeyOwners = map[string]string{ + "acc": "auth", +} + +func owningModuleName(storeKey string) string { + if owner, ok := storeKeyOwners[storeKey]; ok { + return owner + } + return storeKey +} + +// Removing a module from the manager does not remove it from the stored module +// version map: SetModuleVersionMap only writes the keys it is given and never +// deletes the ones it is not, so a departing module's entry survives every +// later upgrade unless a handler calls DeleteModuleVersion for it. This asserts +// the whole map rather than the names v6.7 happens to drop, so the next module +// removal that forgets the call fails here instead of leaving a version entry +// on chain forever. +func TestLatestUpgradeLeavesNoOrphanedModuleVersions(t *testing.T) { + previousUpgrades := upgradesList + t.Cleanup(func() { upgradesList = previousUpgrades }) + + t.Setenv("UPGRADE_VERSION_LIST", LatestUpgrade) + testApp := Setup(t, false, false, false) + testApp.RegisterUpgradeHandlers() + ctx := testApp.NewContext(false, tmproto.Header{}) + + registered := make(map[string]struct{}, len(testApp.mm.Modules)) + for name := range testApp.mm.Modules { + registered[name] = struct{}{} + } + + // Model a chain that carried every retained store's module version across + // earlier upgrades, which is what a real node upgrading into this release + // has in state. + versionMap := testApp.UpgradeKeeper.GetModuleVersionMap(ctx) + for name := range retainedStores { + versionMap[name] = 1 + } + testApp.UpgradeKeeper.SetModuleVersionMap(ctx, versionMap) + + testApp.UpgradeKeeper.ApplyUpgrade(ctx, upgradetypes.Plan{ + Name: LatestUpgrade, + Height: ctx.BlockHeight(), + }) + + for name := range testApp.UpgradeKeeper.GetModuleVersionMap(ctx) { + require.Contains(t, registered, name, + "module version map still carries %q, which no registered module owns; "+ + "the upgrade handler needs a DeleteModuleVersion call for it", name) + } +} + +// Every mounted KV store should be owned by a registered module or be named on +// the retained list. An unowned store that nobody declared is state the chain +// keeps paying for with no code able to read it, which is how a module removal +// half lands: the manager entry goes, the store stays, and nothing says whether +// that was the intent. +func TestMountedStoresAreOwnedOrExplicitlyRetained(t *testing.T) { + testApp := Setup(t, false, false, false) + + var unowned []string + for _, storeKey := range kvStoreKeyNames { + if _, ok := testApp.mm.Modules[owningModuleName(storeKey)]; ok { + continue + } + if _, ok := retainedStores[storeKey]; ok { + continue + } + unowned = append(unowned, storeKey) + } + + require.Empty(t, unowned, + "these KV stores are mounted but no registered module owns them; for each, either "+ + "delete it at an upgrade height, add it to retainedStores with the reason, or "+ + "record its owning module in storeKeyOwners") +} + +// A retained store is only worth retaining if it can still be read. This fails +// if a later change drops one from the mount list while leaving it declared +// retained, which would make the declaration a comment rather than a fact. +func TestRetainedStoresRemainMounted(t *testing.T) { + testApp := Setup(t, false, false, false) + + for storeKey, reason := range retainedStores { + require.Contains(t, kvStoreKeyNames, storeKey, + "%q is declared retained (%s) but is not mounted", storeKey, reason) + require.NotNil(t, testApp.GetKey(storeKey), + "%q is declared retained (%s) but has no store key", storeKey, reason) + } +} diff --git a/app/upgrade_test.go b/app/upgrade_test.go index 08a0c269d7..bfc02c91c2 100644 --- a/app/upgrade_test.go +++ b/app/upgrade_test.go @@ -1,16 +1,20 @@ package app_test import ( + "fmt" + "strings" "testing" "time" "github.com/sei-protocol/sei-chain/app" "github.com/sei-protocol/sei-chain/sei-cosmos/crypto/keys/secp256k1" sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" + "github.com/sei-protocol/sei-chain/sei-cosmos/x/upgrade" "github.com/sei-protocol/sei-chain/sei-cosmos/x/upgrade/types" storekeys "github.com/sei-protocol/sei-chain/sei-db/common/keys" abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" tmproto "github.com/sei-protocol/sei-chain/sei-tendermint/proto/tendermint/types" + "github.com/sei-protocol/sei-chain/upgradetest" "github.com/stretchr/testify/require" ) @@ -21,6 +25,76 @@ func TestUpgradesListIsSorted(t *testing.T) { testWrapper.App.RegisterUpgradeHandlers() } +func TestUpgradePlanNamesMatchRegisteredHandlers(t *testing.T) { + shipped := app.ReleaseUpgrades() + require.NotEmpty(t, shipped) + t.Setenv("UPGRADE_VERSION_LIST", strings.Join(shipped, "\n")) + + tm := time.Now().UTC() + valPub := secp256k1.GenPrivKey().PubKey() + testWrapper := app.NewTestWrapper(t, tm, valPub, false) + testWrapper.App.RegisterUpgradeHandlers() + + for _, name := range shipped { + require.True(t, testWrapper.App.UpgradeKeeper.HasHandler(name), + "no handler registered for shipped upgrade %q", name) + } + + boundary, err := upgradetest.Current() + require.NoError(t, err) + require.Contains(t, shipped, boundary.To, + "go run ./upgradetest/cmd/boundary to prints %q, which is not a shipped upgrade", boundary.To) + + names := []string{app.LatestUpgrade} + if boundary.To != app.LatestUpgrade { + names = append(names, boundary.To) + } + for _, name := range names { + for _, miss := range upgradeNameNearMisses(name) { + require.NotEqual(t, name, miss) + require.False(t, testWrapper.App.UpgradeKeeper.HasHandler(miss), + "handler registered for near-miss name %q of %q", miss, name) + + require.NoError(t, testWrapper.App.UpgradeKeeper.ScheduleUpgrade(testWrapper.Ctx, types.Plan{ + Name: miss, + Height: testWrapper.Ctx.BlockHeight(), + })) + var panicked any + func() { + defer func() { panicked = recover() }() + upgrade.BeginBlocker(testWrapper.App.UpgradeKeeper, testWrapper.Ctx) + }() + requireUpgradeNeededPanic(t, panicked, miss, testWrapper.Ctx.BlockHeight()) + require.Zero(t, testWrapper.App.UpgradeKeeper.GetDoneHeight(testWrapper.Ctx, name), + "near-miss plan %q applied the handler for %q", miss, name) + require.Zero(t, testWrapper.App.UpgradeKeeper.GetDoneHeight(testWrapper.Ctx, miss), + "near-miss plan %q was applied", miss) + plan, found := testWrapper.App.UpgradeKeeper.GetUpgradePlan(testWrapper.Ctx) + require.True(t, found, "near-miss plan %q was cleared instead of halting", miss) + require.Equal(t, miss, plan.Name) + } + } +} + +// upgradeNameNearMisses returns plausible misspellings of an upgrade plan name. +func upgradeNameNearMisses(name string) []string { + return []string{ + name + ".0", + strings.ToUpper(name), + " " + name + " ", + } +} + +func requireUpgradeNeededPanic(t *testing.T, panicked any, name string, height int64) { + t.Helper() + require.NotNil(t, panicked, "plan %q was produced without a handler", name) + msg := fmt.Sprint(panicked) + require.Contains(t, msg, fmt.Sprintf(`UPGRADE "%s" NEEDED`, name), + "halt panic is missing the upgrade name: %v", panicked) + require.Contains(t, msg, fmt.Sprintf("height: %d", height), + "halt panic is missing plan height %d: %v", height, panicked) +} + // Test community tax param is set to 0 as part of upgrade 1.2.3beta func TestDistributionCommunityTaxParamMigration(t *testing.T) { tm := time.Now().UTC() diff --git a/app/upgrade_v67_offline_source_test.go b/app/upgrade_v67_offline_source_test.go new file mode 100644 index 0000000000..1ebdb115dd --- /dev/null +++ b/app/upgrade_v67_offline_source_test.go @@ -0,0 +1,419 @@ +//go:build upgrade_v67 && offline_upgrade && upgrade_source + +package app + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/sei-protocol/sei-chain/sei-cosmos/crypto/keys/secp256k1" + sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" + capabilitytypes "github.com/sei-protocol/sei-chain/sei-cosmos/x/capability/types" + "github.com/sei-protocol/sei-chain/sei-cosmos/x/feegrant" + upgradetypes "github.com/sei-protocol/sei-chain/sei-cosmos/x/upgrade/types" + ibctransfertypes "github.com/sei-protocol/sei-chain/sei-ibc-go/modules/apps/transfer/types" + ibcclienttypes "github.com/sei-protocol/sei-chain/sei-ibc-go/modules/core/02-client/types" + connectiontypes "github.com/sei-protocol/sei-chain/sei-ibc-go/modules/core/03-connection/types" + channeltypes "github.com/sei-protocol/sei-chain/sei-ibc-go/modules/core/04-channel/types" + commitmenttypes "github.com/sei-protocol/sei-chain/sei-ibc-go/modules/core/23-commitment/types" + ibchost "github.com/sei-protocol/sei-chain/sei-ibc-go/modules/core/24-host" + ibctmtypes "github.com/sei-protocol/sei-chain/sei-ibc-go/modules/light-clients/07-tendermint/types" + abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" + tmproto "github.com/sei-protocol/sei-chain/sei-tendermint/proto/tendermint/types" + minttypes "github.com/sei-protocol/sei-chain/x/mint/types" + "github.com/stretchr/testify/require" +) + +var v67OfflineSourceStores = []string{ + "feegrant", + "capability", + "ibc", + "transfer", +} + +const ( + v67OfflineEscrowAmount int64 = 777_777 + v67OfflineVoucherAmount int64 = 1_234_567 +) + +func TestV67OfflineUpgradeSource(t *testing.T) { + root := requireOfflineUpgradePhase(t, "source") + testApp := openOfflineUpgradeApp(t, root, true) + ctx := testApp.GetContextForDeliverTx(nil).WithBlockTime(time.Now().UTC()) + + retained := seedV67OfflineUpgradeState(t, testApp, ctx) + stores := snapshotV67OfflineUpgradeStores(t, testApp, ctx) + upgradeHeight := ctx.BlockHeight() + 2 + require.NoError(t, testApp.UpgradeKeeper.ScheduleUpgrade(ctx, upgradetypes.Plan{ + Name: "v6.7", + Height: upgradeHeight, + })) + + commitOfflineUpgradeApp(t, testApp) + sourceHeight := testApp.LastBlockHeight() + plan, found := committedOfflineUpgradePlan(t, testApp) + require.True(t, found, "scheduled upgrade plan was not committed") + require.Equal(t, "v6.7", plan.Name) + require.Equal(t, upgradeHeight, plan.Height) + moduleVersions := offlineUpgradeModuleVersions(t, testApp) + expectedModules := append([]string(nil), v67OfflineSourceStores...) + expectedModules = append(expectedModules, "oracle") + for _, module := range expectedModules { + require.Contains(t, moduleVersions, module, + "v6.6 module version map does not contain %s", module) + } + closeOfflineUpgradeApp(t, testApp) + + writeOfflineUpgradeArtifact(t, root, offlineUpgradeArtifact{ + Upgrade: plan.Name, + SourceHeight: sourceHeight, + UpgradeHeight: upgradeHeight, + ModuleVersions: moduleVersions, + Stores: stores, + Retained: retained, + }) + + requireV67OfflineUnupgradedHalt(t, root, sourceHeight, upgradeHeight) +} + +func TestV67OfflineUpgradeReopen(t *testing.T) { + root := requireOfflineUpgradePhase(t, "reopen") + artifact := readOfflineUpgradeArtifact(t, root) + require.Equal(t, "v6.7", artifact.Upgrade) + require.NotEmpty(t, artifact.UpgradeHash, "target phase did not record the post-upgrade application hash") + + migrated := offlineUpgradeMigratedDatabase(t, root, artifact) + reopenRoot := filepath.Join(root, "reopen") + copyOfflineUpgradeDatabase(t, migrated, reopenRoot) + + testApp := openOfflineUpgradeApp(t, reopenRoot, false) + defer closeOfflineUpgradeApp(t, testApp) + + require.Equal(t, artifact.UpgradeHeight, testApp.LastBlockHeight(), + "v6.6 opened the migrated database at a different height than v6.7 left it") + openedHash := offlineUpgradeHashString(committedOfflineUpgradeHash(t, testApp)) + require.NotEqual(t, artifact.UpgradeHash, openedHash, + "v6.6 opened the migrated database with the same application hash v6.7 committed at height %d; an operator rolling back would not fork at the upgrade height\nv6.7=%s\nv6.6=%s", + artifact.UpgradeHeight, artifact.UpgradeHash, openedHash) + + versions := offlineUpgradeModuleVersions(t, testApp) + for _, module := range v67OfflineSourceStores { + require.NotContains(t, versions, module, + "v6.6 still sees a version-map entry for %s after v6.7 deleted it", module) + } + require.Contains(t, versions, "oracle") + + ctx := offlineUpgradeReadContext(testApp, testApp.LastBlockHeight()) + _, havePlan := testApp.UpgradeKeeper.GetUpgradePlan(ctx) + require.False(t, havePlan, + "v6.6 still sees a pending upgrade plan on the migrated database") + + requireOfflineUpgradeStoresMounted(t, testApp, v67OfflineSourceStores) + for _, storeName := range v67OfflineSourceStores { + got := snapshotCommittedOfflineUpgradeStore(t, testApp, storeName) + require.Equal(t, artifact.Stores[storeName], got, + "v6.6 cannot read the retained %s state the upgrade left behind", storeName) + } + + lastName, lastHeight := testApp.UpgradeKeeper.GetLastCompletedUpgrade(ctx) + require.Equal(t, artifact.Upgrade, lastName) + require.Equal(t, artifact.UpgradeHeight, lastHeight) + require.False(t, testApp.UpgradeKeeper.HasHandler("v6.7"), + "v6.6 registered a v6.7 upgrade handler") + + var panicked any + func() { + defer func() { panicked = recover() }() + _, err := testApp.FinalizeBlock(context.Background(), &abci.RequestFinalizeBlock{ + Hash: []byte("offline-upgrade-reopen"), + Header: &tmproto.Header{ + ChainID: offlineUpgradeChainID, + Height: artifact.UpgradeHeight + 1, + }, + }) + require.NoError(t, err, "v6.6 returned from FinalizeBlock without panicking") + }() + require.NotNil(t, panicked, "v6.6 produced a block on the migrated database") + require.Contains(t, fmt.Sprint(panicked), "upgrade handler is missing for v6.7 upgrade plan", + "v6.6 panicked for a different reason: %v", panicked) +} + +// requireV67OfflineUnupgradedHalt drives a copy of the pre-upgrade database +// through the v6.7 plan height. +func requireV67OfflineUnupgradedHalt(t *testing.T, root string, sourceHeight, upgradeHeight int64) { + t.Helper() + haltRoot := filepath.Join(root, "unupgraded-halt") + copyOfflineUpgradeDatabase(t, root, haltRoot) + + testApp := openOfflineUpgradeApp(t, haltRoot, false) + require.Equal(t, sourceHeight, testApp.LastBlockHeight()) + require.False(t, testApp.UpgradeKeeper.HasHandler("v6.7"), + "v6.6 registered a v6.7 upgrade handler") + + var panicked any + func() { + defer func() { panicked = recover() }() + _, err := testApp.FinalizeBlock(context.Background(), &abci.RequestFinalizeBlock{ + Hash: []byte("offline-upgrade-unupgraded-halt"), + Header: &tmproto.Header{ + ChainID: offlineUpgradeChainID, + Height: upgradeHeight, + }, + }) + require.NoError(t, err, "v6.6 returned from FinalizeBlock without panicking") + }() + require.NotNil(t, panicked, "v6.6 produced the v6.7 plan height") + msg := fmt.Sprint(panicked) + require.Contains(t, msg, `UPGRADE "v6.7" NEEDED`, + "halt panic is missing the upgrade name: %v", panicked) + require.Contains(t, msg, fmt.Sprintf("height: %d", upgradeHeight), + "halt panic is missing plan height %d: %v", upgradeHeight, panicked) + require.Equal(t, sourceHeight, testApp.LastBlockHeight(), + "v6.6 committed the v6.7 plan height") + closeOfflineUpgradeApp(t, testApp) + + reopened := openOfflineUpgradeApp(t, haltRoot, false) + defer closeOfflineUpgradeApp(t, reopened) + require.Equal(t, sourceHeight, reopened.LastBlockHeight(), + "v6.6 left committed state behind after halting at the v6.7 plan height") +} + +// seedV67OfflineUpgradeState writes real feegrant, capability, IBC, transfer, +// escrow and voucher state through the v6.6 keepers. +func seedV67OfflineUpgradeState(t *testing.T, testApp *App, ctx sdk.Context) offlineUpgradeRetainedState { + t.Helper() + retained := offlineUpgradeRetainedState{} + seedV67Feegrant(t, testApp, ctx, &retained) + seedV67Capability(t, testApp, ctx, &retained) + seedV67IBC(t, testApp, ctx, &retained) + seedV67DeliverableTxAccount(t, testApp, ctx, &retained) + seedV67Transfer(t, testApp, ctx, &retained) + return retained +} + +func seedV67Feegrant(t *testing.T, testApp *App, ctx sdk.Context, retained *offlineUpgradeRetainedState) { + t.Helper() + granter := fundOfflineUpgradeAccount(t, testApp, ctx) + grantee := fundOfflineUpgradeAccount(t, testApp, ctx) + secondGrantee := fundOfflineUpgradeAccount(t, testApp, ctx) + allowance := &feegrant.BasicAllowance{ + SpendLimit: sdk.NewCoins(sdk.NewInt64Coin("usei", 1_000_000)), + } + require.NoError(t, testApp.FeeGrantKeeper.GrantAllowance(ctx, granter, grantee, allowance)) + require.NoError(t, testApp.FeeGrantKeeper.GrantAllowance(ctx, granter, secondGrantee, &feegrant.BasicAllowance{ + SpendLimit: sdk.NewCoins(sdk.NewInt64Coin("usei", 1)), + })) + + got, err := testApp.FeeGrantKeeper.GetAllowance(ctx, granter, grantee) + require.NoError(t, err) + require.NotNil(t, got) + + retained.FeegrantGranter = granter.String() + retained.FeegrantGrantee = grantee.String() + retained.FeegrantKey = encodeOfflineUpgradeKey(feegrant.FeeAllowanceKey(granter, grantee)) +} + +func seedV67Capability(t *testing.T, testApp *App, ctx sdk.Context, retained *offlineUpgradeRetainedState) { + t.Helper() + const name = "offline-upgrade" + capability, err := testApp.ScopedIBCKeeper.NewCapability(ctx, name) + require.NoError(t, err) + require.NoError(t, testApp.ScopedTransferKeeper.ClaimCapability(ctx, capability, name)) + + fetched, ok := testApp.ScopedIBCKeeper.GetCapability(ctx, name) + require.True(t, ok) + require.Equal(t, capability.Index, fetched.Index) + + owners, found := testApp.CapabilityKeeper.GetOwners(ctx, capability.Index) + require.True(t, found) + require.Len(t, owners.Owners, 2) + + retained.CapabilityName = name + retained.CapabilityIndex = capability.Index + retained.CapabilityOwnersKey = encodeOfflineUpgradeKey(append( + append([]byte{}, capabilitytypes.KeyPrefixIndexCapability...), + capabilitytypes.IndexToKey(capability.Index)..., + )) +} + +func seedV67IBC(t *testing.T, testApp *App, ctx sdk.Context, retained *offlineUpgradeRetainedState) { + t.Helper() + clientState := ibctmtypes.NewClientState( + "testchain", + ibctmtypes.DefaultTrustLevel, + 14*24*time.Hour, + 21*24*time.Hour, + 10*time.Second, + ibcclienttypes.NewHeight(0, 1), + commitmenttypes.GetSDKSpecs(), + []string{"upgrade", "upgradedIBCState"}, + true, + true, + ) + hash := sha256.Sum256([]byte("offline-upgrade-consensus")) + consensusState := ibctmtypes.NewConsensusState( + time.Unix(1_700_000_000, 0).UTC(), + commitmenttypes.NewMerkleRoot(hash[:]), + hash[:], + ) + clientID, err := testApp.IBCKeeper.ClientKeeper.CreateClient(ctx, clientState, consensusState) + require.NoError(t, err) + storedClient, found := testApp.IBCKeeper.ClientKeeper.GetClientState(ctx, clientID) + require.True(t, found) + require.Equal(t, clientState.ClientType(), storedClient.ClientType()) + + counterparty := connectiontypes.NewCounterparty( + "07-tendermint-1", + "connection-0", + commitmenttypes.NewMerklePrefix([]byte("ibc")), + ) + connectionID, err := testApp.IBCKeeper.ConnectionKeeper.ConnOpenInit(ctx, clientID, counterparty, nil, 0) + require.NoError(t, err) + _, found = testApp.IBCKeeper.ConnectionKeeper.GetConnection(ctx, connectionID) + require.True(t, found) + + channelID := testApp.IBCKeeper.ChannelKeeper.GenerateChannelIdentifier(ctx) + channel := channeltypes.NewChannel( + channeltypes.OPEN, + channeltypes.UNORDERED, + channeltypes.NewCounterparty(ibctransfertypes.PortID, "channel-0"), + []string{connectionID}, + ibctransfertypes.Version, + ) + testApp.IBCKeeper.ChannelKeeper.SetChannel(ctx, ibctransfertypes.PortID, channelID, channel) + testApp.IBCKeeper.ChannelKeeper.SetNextSequenceSend(ctx, ibctransfertypes.PortID, channelID, 1) + storedChannel, found := testApp.IBCKeeper.ChannelKeeper.GetChannel(ctx, ibctransfertypes.PortID, channelID) + require.True(t, found) + require.Equal(t, channeltypes.OPEN, storedChannel.State) + + retained.IBCClientID = clientID + retained.IBCClientStateKey = encodeOfflineUpgradeKey(ibchost.FullClientStateKey(clientID)) + retained.IBCConnectionID = connectionID + retained.IBCConnectionKey = encodeOfflineUpgradeKey(ibchost.ConnectionKey(connectionID)) + retained.IBCPortID = ibctransfertypes.PortID + retained.IBCChannelID = channelID + retained.IBCChannelKey = encodeOfflineUpgradeKey(ibchost.ChannelKey(ibctransfertypes.PortID, channelID)) +} + +func seedV67Transfer(t *testing.T, testApp *App, ctx sdk.Context, retained *offlineUpgradeRetainedState) { + t.Helper() + trace := ibctransfertypes.DenomTrace{ + Path: "transfer/channel-0", + BaseDenom: "uatom", + } + testApp.TransferKeeper.SetDenomTrace(ctx, trace) + got, found := testApp.TransferKeeper.GetDenomTrace(ctx, trace.Hash()) + require.True(t, found) + require.Equal(t, trace, got) + require.Equal(t, "ibc/"+strings.ToUpper(hex.EncodeToString(trace.Hash())), got.IBCDenom()) + + retained.TransferDenomHash = strings.ToUpper(hex.EncodeToString(trace.Hash())) + retained.TransferIBCDenom = got.IBCDenom() + retained.TransferTraceKey = encodeOfflineUpgradeKey(append( + append([]byte{}, ibctransfertypes.DenomTraceKey...), + trace.Hash()..., + )) + + seedV67TransferEscrow(t, testApp, ctx, retained) + seedV67TransferVoucher(t, testApp, ctx, retained) + recordV67EscrowSupply(t, testApp, ctx, retained) +} + +// recordV67EscrowSupply records the current usei total supply. +func recordV67EscrowSupply(t *testing.T, testApp *App, ctx sdk.Context, retained *offlineUpgradeRetainedState) { + t.Helper() + supply := testApp.BankKeeper.GetSupply(ctx, "usei") + escrowAmt, ok := sdk.NewIntFromString(retained.EscrowAmount) + require.True(t, ok) + require.True(t, supply.Amount.GTE(escrowAmt), + "escrowed coins are not counted in total supply") + retained.EscrowSupply = supply.Amount.String() +} + +// seedV67TransferEscrow locks native coins in the escrow account for the seeded +// port and channel. +func seedV67TransferEscrow(t *testing.T, testApp *App, ctx sdk.Context, retained *offlineUpgradeRetainedState) { + t.Helper() + escrowAddr := ibctransfertypes.GetEscrowAddress(retained.IBCPortID, retained.IBCChannelID) + sender := fundOfflineUpgradeAccount(t, testApp, ctx) + locked := sdk.NewInt64Coin("usei", v67OfflineEscrowAmount) + require.NoError(t, testApp.BankKeeper.SendCoins(ctx, sender, escrowAddr, sdk.NewCoins(locked))) + + got := testApp.BankKeeper.GetBalance(ctx, escrowAddr, locked.Denom) + require.Equal(t, locked, got, "escrow account was not credited") + + retained.EscrowAddress = escrowAddr.String() + retained.EscrowAmount = got.Amount.String() +} + +// seedV67TransferVoucher mints the seeded denom trace's IBC voucher to a holder +// account. +func seedV67TransferVoucher(t *testing.T, testApp *App, ctx sdk.Context, retained *offlineUpgradeRetainedState) { + t.Helper() + require.NotEmpty(t, retained.TransferIBCDenom) + holder := fundOfflineUpgradeAccount(t, testApp, ctx) + voucher := sdk.NewInt64Coin(retained.TransferIBCDenom, v67OfflineVoucherAmount) + require.NoError(t, testApp.BankKeeper.MintCoins(ctx, ibctransfertypes.ModuleName, sdk.NewCoins(voucher))) + require.NoError(t, testApp.BankKeeper.SendCoinsFromModuleToAccount( + ctx, ibctransfertypes.ModuleName, holder, sdk.NewCoins(voucher))) + + got := testApp.BankKeeper.GetBalance(ctx, holder, voucher.Denom) + require.Equal(t, voucher, got, "voucher holder was not credited") + supply := testApp.BankKeeper.GetSupply(ctx, voucher.Denom) + require.Equal(t, got, supply, "voucher total supply does not match the holder balance") + require.False(t, supply.IsZero(), "voucher total supply is zero") + + retained.VoucherHolder = holder.String() + retained.VoucherAmount = got.Amount.String() + retained.VoucherSupply = supply.Amount.String() +} + +func fundOfflineUpgradeAccount(t *testing.T, testApp *App, ctx sdk.Context) sdk.AccAddress { + t.Helper() + addr := sdk.AccAddress(secp256k1.GenPrivKey().PubKey().Address()) + acc := testApp.AccountKeeper.NewAccountWithAddress(ctx, addr) + testApp.AccountKeeper.SetAccount(ctx, acc) + coins := sdk.NewCoins(sdk.NewInt64Coin("usei", 1_000_000_000)) + require.NoError(t, testApp.BankKeeper.MintCoins(ctx, minttypes.ModuleName, coins)) + require.NoError(t, testApp.BankKeeper.SendCoinsFromModuleToAccount(ctx, minttypes.ModuleName, addr, coins)) + require.False(t, testApp.BankKeeper.GetBalance(ctx, addr, "usei").IsZero()) + return addr +} + +// seedV67DeliverableTxAccount records a funded sender and an empty recipient +// for a signed bank send after the upgrade. +func seedV67DeliverableTxAccount(t *testing.T, testApp *App, ctx sdk.Context, retained *offlineUpgradeRetainedState) { + t.Helper() + priv := secp256k1.GenPrivKey() + sender := sdk.AccAddress(priv.PubKey().Address()) + acc := testApp.AccountKeeper.NewAccountWithAddress(ctx, sender) + require.NoError(t, acc.SetPubKey(priv.PubKey())) + testApp.AccountKeeper.SetAccount(ctx, acc) + coins := sdk.NewCoins(sdk.NewInt64Coin("usei", 1_000_000_000)) + require.NoError(t, testApp.BankKeeper.MintCoins(ctx, minttypes.ModuleName, coins)) + require.NoError(t, testApp.BankKeeper.SendCoinsFromModuleToAccount(ctx, minttypes.ModuleName, sender, coins)) + + recipient := sdk.AccAddress(secp256k1.GenPrivKey().PubKey().Address()) + testApp.AccountKeeper.SetAccount(ctx, testApp.AccountKeeper.NewAccountWithAddress(ctx, recipient)) + + retained.TxSender = sender.String() + retained.TxSenderKey = hex.EncodeToString(priv.Bytes()) + retained.TxRecipient = recipient.String() +} + +func snapshotV67OfflineUpgradeStores(t *testing.T, testApp *App, ctx sdk.Context) map[string]map[string]string { + t.Helper() + stores := snapshotOfflineUpgradeStores(t, testApp, ctx, v67OfflineSourceStores) + for storeName, entries := range stores { + require.Greater(t, len(entries), 1, + "%s store has %d keys after keeper writes; a sentinel write produces one", + storeName, len(entries)) + } + return stores +} diff --git a/app/upgrade_v67_offline_target_test.go b/app/upgrade_v67_offline_target_test.go new file mode 100644 index 0000000000..e66a1ba0b6 --- /dev/null +++ b/app/upgrade_v67_offline_target_test.go @@ -0,0 +1,417 @@ +//go:build upgrade_v67 && offline_upgrade && upgrade_target + +package app + +import ( + "context" + "encoding/hex" + "path/filepath" + "testing" + "time" + + "github.com/sei-protocol/sei-chain/sei-cosmos/crypto/keys/secp256k1" + sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" + "github.com/sei-protocol/sei-chain/sei-cosmos/types/tx/signing" + xauthsigning "github.com/sei-protocol/sei-chain/sei-cosmos/x/auth/signing" + banktypes "github.com/sei-protocol/sei-chain/sei-cosmos/x/bank/types" + upgradetypes "github.com/sei-protocol/sei-chain/sei-cosmos/x/upgrade/types" + abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" + tmproto "github.com/sei-protocol/sei-chain/sei-tendermint/proto/tendermint/types" + "github.com/stretchr/testify/require" +) + +var v67OfflineRemovedModules = []string{ + "capability", + "feegrant", + "ibc", + "transfer", +} + +var v67OfflineUpgradeBlockTime = time.Unix(1_700_000_000, 0).UTC() + +func TestV67OfflineUpgradeTarget(t *testing.T) { + t.Run("fixture", testV67OfflineUpgradeTargetFixture) + t.Run("snapshot", testV67OfflineUpgradeTargetSnapshot) +} + +func testV67OfflineUpgradeTargetFixture(t *testing.T) { + root := requireOfflineUpgradePhase(t, "target") + artifact := readOfflineUpgradeArtifact(t, root) + require.Equal(t, "v6.7", artifact.Upgrade) + requireV67OfflineRetainedIdentities(t, artifact.Retained) + require.Equal(t, artifact.SourceHeight+1, artifact.UpgradeHeight) + + t.Setenv("UPGRADE_VERSION_LIST", LatestUpgrade) + + cleanRoot := filepath.Join(root, offlineUpgradeMigratedDir) + crashRoot := filepath.Join(root, "crash") + copyOfflineUpgradeDatabase(t, root, cleanRoot) + copyOfflineUpgradeDatabase(t, root, crashRoot) + + cleanHash := applyV67OfflineUpgradeClean(t, cleanRoot, artifact) + crashHash := applyV67OfflineUpgradeCrashReplay(t, crashRoot, artifact) + require.Equalf(t, cleanHash, crashHash, + "crash-replay application hash diverged from the clean single-pass hash: clean=%x crash-replay=%x", + cleanHash, crashHash) + + artifact.MigratedRoot = offlineUpgradeMigratedDir + artifact.UpgradeHash = offlineUpgradeHashString(cleanHash) + writeOfflineUpgradeArtifact(t, root, artifact) + + txRoot := filepath.Join(root, "post-upgrade-tx") + copyOfflineUpgradeDatabase(t, cleanRoot, txRoot) + requireV67OfflineDeliveredBankSend(t, txRoot, artifact.Retained) +} + +func applyV67OfflineUpgradeClean(t *testing.T, root string, artifact offlineUpgradeArtifact) []byte { + t.Helper() + testApp := openOfflineUpgradeApp(t, root, false) + requireV67OfflinePersistedPlanHasHandler(t, testApp, artifact) + require.Equal(t, artifact.ModuleVersions, offlineUpgradeModuleVersions(t, testApp), + "v6.7 did not reopen the v6.6 module version map") + requireV67OfflineRetainedStores(t, testApp, artifact) + requireV67OfflineBankState(t, testApp, artifact.Retained) + + finalizeV67OfflineUpgrade(t, testApp, artifact.UpgradeHeight) + commitOfflineUpgradeApp(t, testApp) + closeOfflineUpgradeApp(t, testApp) + + reopened := openOfflineUpgradeApp(t, root, false) + defer closeOfflineUpgradeApp(t, reopened) + require.Equal(t, artifact.UpgradeHeight, reopened.LastBlockHeight()) + requireV67OfflineAppliedName(t, reopened, artifact) + requireV67OfflineVersionMap(t, reopened, artifact.ModuleVersions) + requireV67OfflineRetainedStores(t, reopened, artifact) + requireV67OfflineBankState(t, reopened, artifact.Retained) + requireV67OfflineVoucherSend(t, reopened, artifact.Retained) + return committedOfflineUpgradeHash(t, reopened) +} + +func applyV67OfflineUpgradeCrashReplay(t *testing.T, root string, artifact offlineUpgradeArtifact) []byte { + t.Helper() + testApp := openOfflineUpgradeApp(t, root, false) + requireV67OfflinePersistedPlanHasHandler(t, testApp, artifact) + require.Equal(t, artifact.SourceHeight, testApp.LastBlockHeight()) + finalizeV67OfflineUpgrade(t, testApp, artifact.UpgradeHeight) + closeOfflineUpgradeApp(t, testApp) + + interrupted := openOfflineUpgradeApp(t, root, false) + require.Equal(t, artifact.SourceHeight, interrupted.LastBlockHeight(), + "closing without commit left the crash-replay database above the pre-upgrade height") + require.Equal(t, artifact.ModuleVersions, offlineUpgradeModuleVersions(t, interrupted), + "closing without commit mutated the pre-upgrade version map") + finalizeV67OfflineUpgrade(t, interrupted, artifact.UpgradeHeight) + commitOfflineUpgradeApp(t, interrupted) + closeOfflineUpgradeApp(t, interrupted) + + reopened := openOfflineUpgradeApp(t, root, false) + defer closeOfflineUpgradeApp(t, reopened) + require.Equal(t, artifact.UpgradeHeight, reopened.LastBlockHeight()) + requireV67OfflineAppliedName(t, reopened, artifact) + requireV67OfflineVersionMap(t, reopened, artifact.ModuleVersions) + return committedOfflineUpgradeHash(t, reopened) +} + +func requireV67OfflinePersistedPlanHasHandler(t *testing.T, testApp *App, artifact offlineUpgradeArtifact) { + t.Helper() + plan, found := committedOfflineUpgradePlan(t, testApp) + require.True(t, found, "committed upgrade plan did not survive the process boundary") + require.Equal(t, artifact.Upgrade, plan.Name) + require.Equal(t, artifact.UpgradeHeight, plan.Height) + require.Equal(t, LatestUpgrade, plan.Name) + require.True(t, testApp.UpgradeKeeper.HasHandler(plan.Name), + "target binary has no handler for persisted plan name %q", plan.Name) +} + +func requireV67OfflineAppliedName(t *testing.T, testApp *App, artifact offlineUpgradeArtifact) { + t.Helper() + lastName, lastHeight := testApp.UpgradeKeeper.GetLastCompletedUpgrade( + offlineUpgradeReadContext(testApp, testApp.LastBlockHeight())) + require.Equal(t, artifact.Upgrade, lastName) + require.Equal(t, artifact.UpgradeHeight, lastHeight) + require.True(t, testApp.UpgradeKeeper.HasHandler(lastName), + "target binary has no handler for applied plan name %q", lastName) +} + +func finalizeV67OfflineUpgrade(t *testing.T, testApp *App, height int64) { + t.Helper() + _, err := testApp.FinalizeBlock(context.Background(), &abci.RequestFinalizeBlock{ + Hash: []byte("offline-upgrade"), + Header: &tmproto.Header{ + ChainID: offlineUpgradeChainID, + Height: height, + Time: v67OfflineUpgradeBlockTime, + }, + }) + require.NoError(t, err) +} + +func testV67OfflineUpgradeTargetSnapshot(t *testing.T) { + home := requireOfflineUpgradeSnapshotHome(t) + t.Setenv("UPGRADE_VERSION_LIST", "v6.7") + chainID := readOfflineUpgradeGenesisChainID(t, home) + + testApp := openOfflineUpgradeSnapshotApp(t, home, chainID) + sourceHeight := testApp.LastBlockHeight() + readCtx := offlineUpgradeContext(testApp, sourceHeight, chainID) + beforeVersions := offlineUpgradeModuleVersions(t, testApp) + for _, module := range v67OfflineRemovedModules { + require.Contains(t, beforeVersions, module, + "%s is not a pre-v6.7 snapshot: module version map is missing %s", home, module) + } + require.Contains(t, beforeVersions, "oracle") + beforeStores := snapshotOfflineUpgradeStores(t, testApp, readCtx, v67OfflineRemovedModules) + + require.True(t, testApp.UpgradeKeeper.HasHandler("v6.7"), + "v6.7 upgrade handler is not registered; set UPGRADE_VERSION_LIST=v6.7") + upgradeHeight := sourceHeight + 1 + upgradeCtx := offlineUpgradeContext(testApp, upgradeHeight, chainID) + testApp.UpgradeKeeper.ApplyUpgrade(upgradeCtx, upgradetypes.Plan{ + Name: "v6.7", + Height: upgradeHeight, + }) + testApp.CommitMultiStore().Commit(true) + closeOfflineUpgradeApp(t, testApp) + + reopened := openOfflineUpgradeSnapshotApp(t, home, chainID) + defer closeOfflineUpgradeApp(t, reopened) + requireV67OfflineVersionMap(t, reopened, beforeVersions) + requireOfflineUpgradeRetainedStores(t, reopened, beforeStores) +} + +func requireV67OfflineVersionMap(t *testing.T, testApp *App, before []string) { + t.Helper() + after := offlineUpgradeModuleVersions(t, testApp) + require.Equal(t, v67OfflineRemovedModules, + offlineUpgradeDifference(before, after), + "v6.7 removed an unexpected set of module versions") + require.Contains(t, after, "oracle") + require.True(t, offlineUpgradeHasModuleVersion(t, testApp, "oracle"), + "upgrade store dropped the oracle version-map entry") + for _, module := range v67OfflineRemovedModules { + require.False(t, offlineUpgradeHasModuleVersion(t, testApp, module), + "upgrade store still has a version-map entry for %s", module) + } +} + +func requireV67OfflineRetainedStores(t *testing.T, testApp *App, artifact offlineUpgradeArtifact) { + t.Helper() + requireOfflineUpgradeRetainedStores(t, testApp, artifact.Stores) + for storeName, want := range artifact.Stores { + requireV67OfflineRetainedStoreKeys(t, storeName, want, artifact.Retained) + } +} + +func requireV67OfflineRetainedIdentities(t *testing.T, retained offlineUpgradeRetainedState) { + t.Helper() + require.NotEmpty(t, retained.FeegrantGranter) + require.NotEmpty(t, retained.FeegrantGrantee) + require.NotEmpty(t, retained.FeegrantKey) + require.NotEmpty(t, retained.CapabilityName) + require.NotZero(t, retained.CapabilityIndex) + require.NotEmpty(t, retained.CapabilityOwnersKey) + require.NotEmpty(t, retained.IBCClientID) + require.NotEmpty(t, retained.IBCClientStateKey) + require.NotEmpty(t, retained.IBCConnectionID) + require.NotEmpty(t, retained.IBCConnectionKey) + require.NotEmpty(t, retained.IBCPortID) + require.NotEmpty(t, retained.IBCChannelID) + require.NotEmpty(t, retained.IBCChannelKey) + require.NotEmpty(t, retained.TransferDenomHash) + require.NotEmpty(t, retained.TransferIBCDenom) + require.NotEmpty(t, retained.TransferTraceKey) + require.NotEmpty(t, retained.EscrowAddress) + require.NotEmpty(t, retained.EscrowAmount) + require.NotEmpty(t, retained.EscrowSupply) + require.NotEmpty(t, retained.VoucherHolder) + require.NotEmpty(t, retained.VoucherAmount) + require.NotEmpty(t, retained.VoucherSupply) + require.NotEmpty(t, retained.TxSender) + require.NotEmpty(t, retained.TxSenderKey) + require.NotEmpty(t, retained.TxRecipient) +} + +// requireV67OfflineBankState asserts the recorded IBC escrow and voucher bank +// balances and total supplies. +func requireV67OfflineBankState(t *testing.T, testApp *App, retained offlineUpgradeRetainedState) { + t.Helper() + ctx := offlineUpgradeReadContext(testApp, testApp.LastBlockHeight()) + + escrowAddr, err := sdk.AccAddressFromBech32(retained.EscrowAddress) + require.NoError(t, err) + escrowCoin := offlineUpgradeRecordedCoin(t, "usei", retained.EscrowAmount) + require.Equal(t, escrowCoin, testApp.BankKeeper.GetBalance(ctx, escrowAddr, escrowCoin.Denom), + "v6.7 changed the IBC escrow balance") + require.Equal(t, offlineUpgradeRecordedCoin(t, "usei", retained.EscrowSupply), + testApp.BankKeeper.GetSupply(ctx, "usei"), + "v6.7 changed usei total supply; escrowed coins must remain counted") + + holder, err := sdk.AccAddressFromBech32(retained.VoucherHolder) + require.NoError(t, err) + voucherCoin := offlineUpgradeRecordedCoin(t, retained.TransferIBCDenom, retained.VoucherAmount) + require.Equal(t, voucherCoin, testApp.BankKeeper.GetBalance(ctx, holder, voucherCoin.Denom), + "v6.7 changed the IBC voucher holder balance") + require.Equal(t, offlineUpgradeRecordedCoin(t, retained.TransferIBCDenom, retained.VoucherSupply), + testApp.BankKeeper.GetSupply(ctx, voucherCoin.Denom), + "v6.7 changed IBC voucher total supply") +} + +// requireV67OfflineVoucherSend sends the recorded IBC voucher from its holder to +// another account through the bank keeper. +func requireV67OfflineVoucherSend(t *testing.T, testApp *App, retained offlineUpgradeRetainedState) { + t.Helper() + ctx, _ := offlineUpgradeReadContext(testApp, testApp.LastBlockHeight()).CacheContext() + holder, err := sdk.AccAddressFromBech32(retained.VoucherHolder) + require.NoError(t, err) + voucher := offlineUpgradeRecordedCoin(t, retained.TransferIBCDenom, retained.VoucherAmount) + require.True(t, voucher.Amount.GT(sdk.OneInt()), "voucher amount is too small to send") + + recipient := sdk.AccAddress("v67-voucher-receiver") + testApp.AccountKeeper.SetAccount(ctx, testApp.AccountKeeper.NewAccountWithAddress(ctx, recipient)) + send := sdk.NewCoin(voucher.Denom, sdk.OneInt()) + require.NoError(t, testApp.BankKeeper.SendCoins(ctx, holder, recipient, sdk.NewCoins(send)), + "bank send of an IBC voucher failed after v6.7") + require.Equal(t, send, testApp.BankKeeper.GetBalance(ctx, recipient, voucher.Denom), + "bank send of an IBC voucher did not credit the recipient") + require.Equal(t, voucher.Sub(send), testApp.BankKeeper.GetBalance(ctx, holder, voucher.Denom), + "bank send of an IBC voucher did not debit the holder") + require.Equal(t, offlineUpgradeRecordedCoin(t, voucher.Denom, retained.VoucherSupply), + testApp.BankKeeper.GetSupply(ctx, voucher.Denom), + "sending an IBC voucher changed its total supply") +} + +func offlineUpgradeRecordedCoin(t *testing.T, denom, amount string) sdk.Coin { + t.Helper() + parsed, ok := sdk.NewIntFromString(amount) + require.True(t, ok, "invalid recorded amount %q for denom %s", amount, denom) + return sdk.NewCoin(denom, parsed) +} + +func requireV67OfflineRetainedStoreKeys( + t *testing.T, + storeName string, + snapshot map[string]string, + retained offlineUpgradeRetainedState, +) { + t.Helper() + switch storeName { + case "feegrant": + requireOfflineUpgradeStoreKey(t, snapshot, "feegrant allowance", retained.FeegrantKey) + case "capability": + requireOfflineUpgradeStoreKey(t, snapshot, "capability owner set", retained.CapabilityOwnersKey) + case "ibc": + requireOfflineUpgradeStoreKey(t, snapshot, "IBC client "+retained.IBCClientID, retained.IBCClientStateKey) + requireOfflineUpgradeStoreKey(t, snapshot, "IBC connection "+retained.IBCConnectionID, retained.IBCConnectionKey) + requireOfflineUpgradeStoreKey(t, snapshot, "IBC channel "+retained.IBCPortID+"/"+retained.IBCChannelID, retained.IBCChannelKey) + case "transfer": + requireOfflineUpgradeStoreKey(t, snapshot, "transfer denom trace "+retained.TransferIBCDenom, retained.TransferTraceKey) + } +} + +const ( + v67OfflinePostUpgradeSendAmt int64 = 4242 + v67OfflinePostUpgradeFee int64 = 200000 +) + +// requireV67OfflineDeliveredBankSend delivers a signed bank send against the +// migrated database and requires that the committed balances and sequence moved. +func requireV67OfflineDeliveredBankSend(t *testing.T, root string, retained offlineUpgradeRetainedState) { + t.Helper() + testApp := openOfflineUpgradeApp(t, root, false) + upgradeHeight := testApp.LastBlockHeight() + + privBytes, err := hex.DecodeString(retained.TxSenderKey) + require.NoError(t, err) + priv := &secp256k1.PrivKey{Key: privBytes} + sender, err := sdk.AccAddressFromBech32(retained.TxSender) + require.NoError(t, err) + require.Equal(t, sender, sdk.AccAddress(priv.PubKey().Address()), + "recorded sender key does not match recorded sender address") + recipient, err := sdk.AccAddressFromBech32(retained.TxRecipient) + require.NoError(t, err) + + beforeCtx := offlineUpgradeReadContext(testApp, upgradeHeight) + senderBefore := testApp.BankKeeper.GetBalance(beforeCtx, sender, "usei") + recvBefore := testApp.BankKeeper.GetBalance(beforeCtx, recipient, "usei") + acc := testApp.AccountKeeper.GetAccount(beforeCtx, sender) + require.NotNil(t, acc, "sender account is missing from the migrated database") + seqBefore := acc.GetSequence() + + txBz := signV67OfflineBankSend(t, testApp, priv, recipient, v67OfflinePostUpgradeSendAmt, v67OfflinePostUpgradeFee) + res, err := testApp.FinalizeBlock(context.Background(), &abci.RequestFinalizeBlock{ + Hash: []byte("offline-upgrade-tx"), + Header: &tmproto.Header{ + ChainID: offlineUpgradeChainID, + Height: upgradeHeight + 1, + Time: v67OfflineUpgradeBlockTime.Add(time.Second), + }, + Txs: [][]byte{txBz}, + }) + require.NoError(t, err) + require.Len(t, res.TxResults, 1) + require.Equal(t, uint32(abci.CodeTypeOK), res.TxResults[0].Code, res.TxResults[0].Log) + require.Positive(t, res.TxResults[0].GasUsed) + commitOfflineUpgradeApp(t, testApp) + closeOfflineUpgradeApp(t, testApp) + + reopened := openOfflineUpgradeApp(t, root, false) + defer closeOfflineUpgradeApp(t, reopened) + require.Equal(t, upgradeHeight+1, reopened.LastBlockHeight()) + afterCtx := offlineUpgradeReadContext(reopened, reopened.LastBlockHeight()) + require.Equal(t, recvBefore.Add(sdk.NewInt64Coin("usei", v67OfflinePostUpgradeSendAmt)), + reopened.BankKeeper.GetBalance(afterCtx, recipient, "usei"), + "delivered bank send did not credit the recipient") + require.Equal(t, senderBefore.Sub(sdk.NewInt64Coin("usei", v67OfflinePostUpgradeSendAmt+v67OfflinePostUpgradeFee)), + reopened.BankKeeper.GetBalance(afterCtx, sender, "usei"), + "delivered bank send did not debit the sender including the fee") + require.Equal(t, seqBefore+1, + reopened.AccountKeeper.GetAccount(afterCtx, sender).GetSequence(), + "delivered bank send did not advance the sender sequence") +} + +func signV67OfflineBankSend(t *testing.T, testApp *App, priv *secp256k1.PrivKey, to sdk.AccAddress, amount, fee int64) []byte { + t.Helper() + from := sdk.AccAddress(priv.PubKey().Address()) + txConfig := testApp.GetTxConfig() + txBuilder := txConfig.NewTxBuilder() + require.NoError(t, txBuilder.SetMsgs(banktypes.NewMsgSend(from, to, sdk.NewCoins(sdk.NewInt64Coin("usei", amount))))) + txBuilder.SetGasLimit(1_000_000) + txBuilder.SetFeeAmount(sdk.NewCoins(sdk.NewInt64Coin("usei", fee))) + + ctx := offlineUpgradeReadContext(testApp, testApp.LastBlockHeight()) + acc := testApp.AccountKeeper.GetAccount(ctx, from) + require.NotNil(t, acc, "sender account is missing from the migrated database") + + signerData := xauthsigning.SignerData{ + ChainID: offlineUpgradeChainID, + AccountNumber: acc.GetAccountNumber(), + Sequence: acc.GetSequence(), + } + sigData := signing.SingleSignatureData{ + SignMode: txConfig.SignModeHandler().DefaultMode(), + Signature: nil, + } + sig := signing.SignatureV2{ + PubKey: priv.PubKey(), + Data: &sigData, + Sequence: acc.GetSequence(), + } + require.NoError(t, txBuilder.SetSignatures(sig)) + bytesToSign, err := txConfig.SignModeHandler().GetSignBytes(txConfig.SignModeHandler().DefaultMode(), signerData, txBuilder.GetTx()) + require.NoError(t, err) + sigBytes, err := priv.Sign(bytesToSign) + require.NoError(t, err) + sigData = signing.SingleSignatureData{ + SignMode: txConfig.SignModeHandler().DefaultMode(), + Signature: sigBytes, + } + sig = signing.SignatureV2{ + PubKey: priv.PubKey(), + Data: &sigData, + Sequence: acc.GetSequence(), + } + require.NoError(t, txBuilder.SetSignatures(sig)) + bz, err := txConfig.TxEncoder()(txBuilder.GetTx()) + require.NoError(t, err) + return bz +} diff --git a/app/upgrade_v67_test.go b/app/upgrade_v67_test.go new file mode 100644 index 0000000000..fb1f22cb0a --- /dev/null +++ b/app/upgrade_v67_test.go @@ -0,0 +1,1612 @@ +//go:build upgrade_v67 + +package app_test + +import ( + "context" + "encoding/hex" + "encoding/json" + "fmt" + "sort" + "strconv" + "strings" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + serverconfig "github.com/sei-protocol/sei-chain/sei-cosmos/server/config" + sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" + "github.com/sei-protocol/sei-chain/sei-cosmos/types/address" + sdkerrors "github.com/sei-protocol/sei-chain/sei-cosmos/types/errors" + "github.com/sei-protocol/sei-chain/sei-cosmos/x/auth/signing" + upgradetypes "github.com/sei-protocol/sei-chain/sei-cosmos/x/upgrade/types" + "github.com/sei-protocol/sei-chain/sei-db/common/keys" + abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" + tmproto "github.com/sei-protocol/sei-chain/sei-tendermint/proto/tendermint/types" + "github.com/sei-protocol/sei-chain/testutil/processblock" + "github.com/sei-protocol/sei-chain/testutil/processblock/msgs" + "github.com/sei-protocol/sei-chain/upgradetest" + evmtypes "github.com/sei-protocol/sei-chain/x/evm/types" + oracletypes "github.com/sei-protocol/sei-chain/x/oracle/types" + "github.com/stretchr/testify/require" +) + +// v6.7 retires the feegrant, capability, ibc and transfer modules, and +// deprecates the oracle handlers. These tests cover opaque writes into the +// retired stores, transactions aimed at retired surfaces, ordinary +// transactions after the handler, bank balances left behind by IBC transfer, +// and the upgrade handler itself. + +const ( + v67UpgradeName = "v6.7" + v67KeyringPassword = "12345678\n" + txFee = 200000 + // v67FeegrantAllowancePrefix is the fee-allowance key prefix. + v67FeegrantAllowancePrefix byte = 0x00 + // DenomTrace.IBCDenom lives on the pre-upgrade branch. + v67IBCVoucherDenomShape = "ibc/0000000000000000000000000000000000000000000000000000000000000067" + v67IBCVoucherShapeAmount int64 = 1_234_567 + v67EscrowStyleAmount int64 = 7_777_777 + v67EscrowStyleSendAmount = "1234567usei" + v67PostUpgradeSendAmt int64 = 4242 + v67PostUpgradeSendAmount = "4242usei" + v67EVMNativeSendTo = "0x0000000000000000000000000000000000000067" +) + +// GetEscrowAddress lives on the pre-upgrade branch; this is not a real escrow account. +var v67EscrowStyleAddress = sdk.AccAddress{ + 0xec, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x67, +} + +var v67PostUpgradeBankReceiver = sdk.AccAddress{ + 0x42, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x67, +} + +// retiredModuleTx is a transaction aimed at a module surface v6.7 retires, +// paired with the rejection the v6.7 binary must produce for it. +type retiredModuleTx struct { + name string + // sign builds the transaction. Each case signs from an account of its own so + // that a rejection which consumes no sequence number cannot disturb the + // other cases sharing the block. + sign func(a *processblock.App, signer, granter sdk.AccAddress) signing.Tx + // code and codespace are the ABCI rejection the transaction must produce. + code uint32 + codespace string + // logContains is a distinguishing fragment of the rejection message. + logContains string + // chargesFee records whether the sender still pays for the failed + // transaction. Rejections raised by the message handler run after the fee + // deduction; rejections raised by ValidateBasic run before it. + chargesFee bool +} + +func retiredModuleTxCases() []retiredModuleTx { + return []retiredModuleTx{ + { + name: "oracle aggregate exchange rate vote", + sign: func(a *processblock.App, signer, _ sdk.AccAddress) signing.Tx { + return a.Sign(signer, txFee, oracletypes.NewMsgAggregateExchangeRateVote( + "1.5uatom", signer, sdk.ValAddress(signer))) + }, + code: uint32(oracletypes.ErrOracleDeprecated.ABCICode()), + codespace: oracletypes.ErrOracleDeprecated.Codespace(), + logContains: "oracle module is deprecated", + chargesFee: true, + }, + { + name: "oracle delegate feed consent", + sign: func(a *processblock.App, signer, _ sdk.AccAddress) signing.Tx { + return a.Sign(signer, txFee, oracletypes.NewMsgDelegateFeedConsent( + sdk.ValAddress(signer), signer)) + }, + code: uint32(oracletypes.ErrOracleDeprecated.ABCICode()), + codespace: oracletypes.ErrOracleDeprecated.Codespace(), + logContains: "oracle module is deprecated", + chargesFee: true, + }, + { + name: "transaction nominating a distinct fee granter", + sign: func(a *processblock.App, signer, granter sdk.AccAddress) signing.Tx { + return a.SignWithFeeGranter(signer, granter, txFee, + oracletypes.NewMsgDelegateFeedConsent(sdk.ValAddress(signer), signer)) + }, + code: uint32(sdkerrors.ErrInvalidRequest.ABCICode()), + codespace: sdkerrors.ErrInvalidRequest.Codespace(), + logContains: "fee grants are not enabled", + chargesFee: false, + }, + { + // The granter field stays on the wire even though nothing grants + // any more, so a client that always populates it with its own + // address keeps working. Only a granter that differs from the payer + // is refused. + name: "transaction nominating itself as fee granter", + sign: func(a *processblock.App, signer, _ sdk.AccAddress) signing.Tx { + return a.SignWithFeeGranter(signer, signer, txFee, + msgs.Send(signer, signer, 1)) + }, + code: abci.CodeTypeOK, + codespace: "", + chargesFee: true, + }, + } +} + +// retiredModuleTxAccounts holds one signer and one fee granter per case. +type retiredModuleTxAccounts struct { + signers []sdk.AccAddress + granters []sdk.AccAddress +} + +// fundRetiredModuleTxAccounts creates and funds the accounts one phase needs. +// +// Every phase's accounts must be created before the first block runs. The +// process block harness hands out the deliver context of the block it just +// committed, so an account created after that point is written into a cache +// that has already been flushed and never reaches committed state. +func fundRetiredModuleTxAccounts(a *processblock.App, phase string) retiredModuleTxAccounts { + cases := retiredModuleTxCases() + accounts := retiredModuleTxAccounts{ + signers: make([]sdk.AccAddress, len(cases)), + granters: make([]sdk.AccAddress, len(cases)), + } + for i, c := range cases { + accounts.signers[i] = a.NewSignableAccount(phase + "/signer/" + c.name) + accounts.granters[i] = a.NewSignableAccount(phase + "/granter/" + c.name) + a.FundAccount(accounts.signers[i], 1000000000) + a.FundAccount(accounts.granters[i], 1000000000) + } + return accounts +} + +type ordinaryTxAccounts struct { + bankSender sdk.AccAddress + bankReceiver sdk.AccAddress + evmSender sdk.AccAddress +} + +func fundOrdinaryTxAccounts(a *processblock.App, phase string) ordinaryTxAccounts { + accounts := ordinaryTxAccounts{ + bankSender: a.NewSignableAccount(phase + "/bank-sender"), + bankReceiver: a.NewAccount(), + evmSender: a.NewSignableAccount(phase + "/evm-sender"), + } + a.FundAccount(accounts.bankSender, 1000000000) + a.FundAccount(accounts.evmSender, 1000000000) + return accounts +} + +// newV67Chain returns a chain whose only registered upgrade is v6.7, so that +// applying it exercises the handler under test rather than an earlier one. The +// common preset supplies the bonded validators a block needs a proposer from. +func newV67Chain(t *testing.T) *processblock.App { + t.Helper() + t.Setenv("UPGRADE_VERSION_LIST", v67UpgradeName) + a := processblock.NewTestApp(t) + processblock.CommonPreset(a) + a.RegisterUpgradeHandlers() + requireV67GoLayerExecutionConfig(t, a) + return a +} + +func applyV67(t *testing.T, a *processblock.App) { + t.Helper() + a.UpgradeKeeper.ApplyUpgrade(a.Ctx(), upgradetypes.Plan{ + Name: v67UpgradeName, + Height: a.Ctx().BlockHeight(), + }) +} + +// applyV67ToCommitStore applies the v6.7 handler against the committed multistore. +func applyV67ToCommitStore(t *testing.T, a *processblock.App) { + t.Helper() + // Writes into the deliver context the harness hands back are dropped rather + // than committed, so a committed-store assertion needs the upgrade applied + // through a context that reaches the commit multistore. + ctx, write := a.NewUncachedContext(false, a.Ctx().BlockHeader()).CacheContext() + a.UpgradeKeeper.ApplyUpgrade(ctx, upgradetypes.Plan{ + Name: v67UpgradeName, + Height: ctx.BlockHeight(), + }) + write() +} + +// newV67ChainWithoutHandler returns a chain whose registered upgrade list does +// not contain v6.7. +func newV67ChainWithoutHandler(t *testing.T) *processblock.App { + t.Helper() + t.Setenv("UPGRADE_VERSION_LIST", "v6.6") + a := processblock.NewTestApp(t) + processblock.CommonPreset(a) + a.RegisterUpgradeHandlers() + requireV67GoLayerExecutionConfig(t, a) + return a +} + +// requireV67GoLayerExecutionConfig asserts that in-process upgrade tests run +// with OCC disabled and DefaultConcurrencyWorkers. The fleet sets occ-enabled +// = true and the live harness sets concurrency-workers = 4. +func requireV67GoLayerExecutionConfig(t *testing.T, a *processblock.App) { + t.Helper() + require.False(t, a.OccEnabled(), + "in-process v6.7 tests ran with BaseApp.OccEnabled()=%v, want false; this layer's application-hash determinism is not the fleet's (occ-enabled = true)", + a.OccEnabled()) + require.Equal(t, serverconfig.DefaultConcurrencyWorkers, a.ConcurrencyWorkers(), + "in-process v6.7 tests ran with BaseApp.ConcurrencyWorkers()=%d, want DefaultConcurrencyWorkers=%d; the live harness sets 4", + a.ConcurrencyWorkers(), serverconfig.DefaultConcurrencyWorkers) +} + +func finalizeV67Block(a *processblock.App, height int64) (*abci.ResponseFinalizeBlock, error) { + votes := a.GetVotes() + var proposer []byte + if len(votes) > 0 { + proposer = votes[0].Validator.Address + } + return a.FinalizeBlock(context.Background(), &abci.RequestFinalizeBlock{ + DecidedLastCommit: abci.CommitInfo{Round: 0, Votes: votes}, + ByzantineValidators: []abci.Misbehavior{}, + Hash: []byte("abc"), + Header: &tmproto.Header{ + ChainID: a.ChainID, + Height: height, + ProposerAddress: proposer, + Time: time.Now(), + }, + }) +} + +func requireV67UpgradeNeededPanic(t *testing.T, panicked any, height int64) { + t.Helper() + require.NotNil(t, panicked, "the v6.7 plan height was produced without a handler") + msg := fmt.Sprint(panicked) + require.Contains(t, msg, `UPGRADE "v6.7" NEEDED`, + "halt panic is missing the upgrade name: %v", panicked) + require.Contains(t, msg, fmt.Sprintf("height: %d", height), + "halt panic is missing plan height %d: %v", height, panicked) +} + +func scheduleV67Plan(t *testing.T, a *processblock.App, height int64) { + t.Helper() + require.NoError(t, a.UpgradeKeeper.ScheduleUpgrade(a.Ctx(), upgradetypes.Plan{ + Name: v67UpgradeName, + Height: height, + })) + plan, found := a.UpgradeKeeper.GetUpgradePlan(a.Ctx()) + require.True(t, found) + require.Equal(t, v67UpgradeName, plan.Name) + require.Equal(t, height, plan.Height) +} + +// TestV67UnupgradedBinaryHaltsAtPlanHeight asserts that a binary without a v6.7 +// handler panics at the plan height without committing, and that a binary with +// the handler runs that height. +func TestV67UnupgradedBinaryHaltsAtPlanHeight(t *testing.T) { + t.Run("without handler", func(t *testing.T) { + a := newV67ChainWithoutHandler(t) + require.False(t, a.UpgradeKeeper.HasHandler(v67UpgradeName)) + + const planHeight int64 = 3 + scheduleV67Plan(t, a, planHeight) + a.RunBlock([]signing.Tx{}) + a.RunBlock([]signing.Tx{}) + require.Equal(t, planHeight-1, a.Ctx().BlockHeight()) + _, havePlan := a.UpgradeKeeper.GetUpgradePlan(a.Ctx()) + require.True(t, havePlan, "scheduled v6.7 plan was not committed") + + var panicked any + var finalizeErr error + func() { + defer func() { panicked = recover() }() + _, finalizeErr = finalizeV67Block(a, planHeight) + }() + require.NoError(t, finalizeErr, "FinalizeBlock returned an error instead of panicking") + requireV67UpgradeNeededPanic(t, panicked, planHeight) + require.Equal(t, planHeight-1, a.Ctx().BlockHeight()) + require.Equal(t, planHeight-1, a.LastBlockHeight(), + "an un-upgraded binary committed the v6.7 plan height") + require.Zero(t, a.UpgradeKeeper.GetDoneHeight(a.Ctx(), v67UpgradeName), + "an un-upgraded binary applied v6.7") + _, havePlan = a.UpgradeKeeper.GetUpgradePlan(a.Ctx()) + require.True(t, havePlan, "an un-upgraded binary cleared the v6.7 plan") + }) + + t.Run("with handler", func(t *testing.T) { + a := newV67Chain(t) + require.True(t, a.UpgradeKeeper.HasHandler(v67UpgradeName)) + + const planHeight int64 = 1 + scheduleV67Plan(t, a, planHeight) + require.NotPanics(t, func() { a.RunBlock([]signing.Tx{}) }) + require.Equal(t, planHeight, a.Ctx().BlockHeight()) + require.Equal(t, planHeight, a.UpgradeKeeper.GetDoneHeight(a.Ctx(), v67UpgradeName)) + _, havePlan := a.UpgradeKeeper.GetUpgradePlan(a.Ctx()) + require.False(t, havePlan) + }) +} + +// TestV67ApplyUpgradeTwice applies the v6.7 handler a second time against the +// state the first application produced. +func TestV67ApplyUpgradeTwice(t *testing.T) { + a := newV67Chain(t) + seeded := seedRetiredStores(t, a) + + applyV67(t, a) + onceVersions := a.UpgradeKeeper.GetModuleVersionMap(a.Ctx()) + onceDone := a.UpgradeKeeper.GetDoneHeight(a.Ctx(), v67UpgradeName) + onceAppVersion := a.AppVersion() + onceStores := snapshotDeliverRetiredStores(t, a) + _, havePlan := a.UpgradeKeeper.GetUpgradePlan(a.Ctx()) + require.False(t, havePlan) + require.Equal(t, a.Ctx().BlockHeight(), onceDone) + for _, store := range retiredStoreKeys { + require.NotContains(t, onceVersions, store) + require.Equal(t, seeded[store], onceStores[store]["seeded"]) + } + + require.NotPanics(t, func() { applyV67(t, a) }) + + require.Equal(t, onceVersions, a.UpgradeKeeper.GetModuleVersionMap(a.Ctx()), + "second ApplyUpgrade changed the module version map") + require.Equal(t, onceDone, a.UpgradeKeeper.GetDoneHeight(a.Ctx(), v67UpgradeName), + "second ApplyUpgrade changed the done height") + require.Equal(t, onceStores, snapshotDeliverRetiredStores(t, a), + "second ApplyUpgrade changed retired store state") + _, havePlan = a.UpgradeKeeper.GetUpgradePlan(a.Ctx()) + require.False(t, havePlan) + require.Equal(t, onceAppVersion+1, a.AppVersion(), + "second ApplyUpgrade is not a no-op: ApplyUpgrade increments protocol version on every call") +} + +// TestV67CrossVersion creates retired-module state with the v6.6 binary and +// verifies the same chain after its validators restart on v6.7. +func TestV67CrossVersion(t *testing.T) { + upgradetest.RunCrossVersion(t, seedV66State, verifyV67State) +} + +func seedV66State(t *testing.T, chain *upgradetest.CrossVersion) { + require.Equal(t, v67UpgradeName, chain.UpgradeName(t)) + chain.Record(t, v67LiveHarnessConfigKey, v67ReadLiveHarnessConfig(t, chain)) + + granter := chain.KeyAddress(t, "sei-node-0", "admin") + grantee := chain.KeyAddress(t, "sei-node-0", "node_admin") + chain.Record(t, "feegrant_granter", granter) + chain.Record(t, "feegrant_grantee", grantee) + + grant := chain.Seid(v67KeyringPassword, + "tx", "feegrant", "grant", granter, grantee, + "--spend-limit", "100000000usei", + "--from", "admin", + "--chain-id", "sei", + "--fees", "200000usei", + "--gas", "2000000", + "--broadcast-mode", "sync", + "--yes", + "--output", "json", + ) + chain.RequireDeliverTxSuccess(t, "v6.6 fee allowance grant", grant) + + allowanceBefore := chain.MustSeid(t, "", + "q", "feegrant", "grant", granter, grantee, "--output", "json") + spendLimitBefore := v67FeegrantSpendLimit(t, allowanceBefore) + chain.WriteDiagnostic(t, "v66-feegrant-before-spend.json", []byte(allowanceBefore)) + + spend := chain.Seid(v67KeyringPassword, + "tx", "bank", "send", "node_admin", granter, "1usei", + "--fee-account", granter, + "--from", "node_admin", + "--chain-id", "sei", + "--fees", "200000usei", + "--gas", "2000000", + "--broadcast-mode", "sync", + "--yes", + "--output", "json", + ) + chain.RequireDeliverTxSuccess(t, "v6.6 fee-granted bank send", spend) + + allowanceAfter := chain.MustSeid(t, "", + "q", "feegrant", "grant", granter, grantee, "--output", "json") + spendLimitAfter := v67FeegrantSpendLimit(t, allowanceAfter) + require.True(t, spendLimitAfter.LT(spendLimitBefore), + "the v6.6 transaction did not spend its fee allowance: %s to %s", + spendLimitBefore, spendLimitAfter) + chain.WriteDiagnostic(t, "v66-feegrant-after-spend.json", []byte(allowanceAfter)) + + oracleQuery := chain.Seid("", "q", "oracle", "exchange-rates", "--output", "json") + chain.WriteDiagnostic(t, "v66-oracle-query.stdout", []byte(oracleQuery.Stdout)) + chain.WriteDiagnostic(t, "v66-oracle-query.stderr", []byte(oracleQuery.Stderr)) + require.NoError(t, oracleQuery.Err, "v6.6 must still expose the oracle query") + + seedV66EscrowShapedBankState(t, chain) + + moduleVersions := chain.ModuleVersions(t) + for _, module := range append(retiredStoreKeys, oracletypes.ModuleName) { + require.Contains(t, moduleVersions, module, + "v6.6 module version map does not contain %s", module) + require.NotEmpty(t, chain.QueryStore(t, upgradetypes.StoreKey, v67ModuleVersionKey(module)), + "v6.6 upgrade store is missing a version-map entry for %s", module) + } + chain.Record(t, "module_versions", moduleVersions) + + feegrantKey := v67FeegrantAllowanceKey(t, granter, grantee) + feegrantValue := chain.QueryStore(t, keys.FeegrantStoreKey, feegrantKey) + require.NotEmpty(t, feegrantValue, "v6.6 feegrant store does not serve the granted allowance") + chain.Record(t, "feegrant_store_key", feegrantKey) + chain.Record(t, "feegrant_store_value", feegrantValue) + + preserveV67UnupgradedHome(t, chain) +} + +func verifyV67State(t *testing.T, chain *upgradetest.CrossVersion) { + require.Equal(t, v67UpgradeName, chain.UpgradeName(t)) + + var beforeConfig map[string]v67ValidatorRuntimeConfig + chain.Replay(t, v67LiveHarnessConfigKey, &beforeConfig) + require.Equal(t, beforeConfig, v67ReadLiveHarnessConfig(t, chain), + "the upgrade changed a validator's SeiDB, OCC, or pruning settings; every other assertion in this run is about a configuration the binary is no longer running") + + planName := requireV67RecordedPlanName(t, chain) + appliedOutput := chain.MustSeid(t, "", + "q", "upgrade", "applied", planName, "--output", "json") + var applied struct { + Header struct { + Height json.RawMessage `json:"height"` + } `json:"header"` + } + require.NoError(t, json.Unmarshal([]byte(appliedOutput), &applied)) + appliedHeight := v67JSONInt(t, applied.Header.Height) + require.Equal(t, chain.TargetHeight(t), appliedHeight) + chain.Record(t, "applied_height", appliedHeight) + // A header carries the application hash of the state its parent produced, so + // the upgrade block's own result appears one height above the applied height. + chain.RequireBlockAgreement(t, + appliedHeight, + appliedHeight+1, + appliedHeight+2, + appliedHeight+3, + ) + + var beforeVersions []string + chain.Replay(t, "module_versions", &beforeVersions) + afterVersions := chain.ModuleVersions(t) + chain.Record(t, "module_versions_after", afterVersions) + require.Equal(t, + sortedStrings(retiredStoreKeys), + stringDifference(beforeVersions, afterVersions), + "v6.7 removed an unexpected set of module versions", + ) + require.Contains(t, afterVersions, oracletypes.ModuleName, + "v6.7 removed oracle even though its blocker is still registered") + + requireV67UpgradeStoreVersions(t, chain) + require.NotEmpty(t, chain.QueryStore(t, upgradetypes.StoreKey, v67ModuleVersionKey(oracletypes.ModuleName)), + "v6.7 removed oracle from the upgrade store even though its blocker is still registered") + + var feegrantKey []byte + var feegrantValue []byte + chain.Replay(t, "feegrant_store_key", &feegrantKey) + chain.Replay(t, "feegrant_store_value", &feegrantValue) + require.Equal(t, feegrantValue, chain.QueryStore(t, keys.FeegrantStoreKey, feegrantKey), + "v6.7 node no longer serves the feegrant store value written by v6.6") + chain.WaitForBlocks(t, 5) + require.Equal(t, feegrantValue, chain.QueryStore(t, keys.FeegrantStoreKey, feegrantKey), + "a later block changed the retained feegrant store") + requireV67UpgradeStoreVersions(t, chain) + + var granter string + var grantee string + chain.Replay(t, "feegrant_granter", &granter) + chain.Replay(t, "feegrant_grantee", &grantee) + + feegrantSpend := chain.Seid(v67KeyringPassword, + "tx", "bank", "send", "node_admin", granter, "1usei", + "--fee-account", granter, + "--from", "node_admin", + "--chain-id", "sei", + "--fees", "200000usei", + "--gas", "2000000", + "--broadcast-mode", "sync", + "--yes", + "--output", "json", + ) + chain.WriteDiagnostic(t, "v67-feegrant-spend.stdout", []byte(feegrantSpend.Stdout)) + chain.WriteDiagnostic(t, "v67-feegrant-spend.stderr", []byte(feegrantSpend.Stderr)) + require.Contains(t, feegrantSpend.Combined(), "fee grants are not enabled", + "v6.7 accepted the fee-granted transaction that v6.6 executed") + + feegrantQuery := chain.Seid("", + "q", "feegrant", "grant", granter, grantee, "--output", "json") + chain.WriteDiagnostic(t, "v67-feegrant-query.stdout", []byte(feegrantQuery.Stdout)) + chain.WriteDiagnostic(t, "v67-feegrant-query.stderr", []byte(feegrantQuery.Stderr)) + require.Error(t, feegrantQuery.Err, "v6.7 still exposes the retired feegrant query command") + + oracleQuery := chain.Seid("", "q", "oracle", "exchange-rates", "--output", "json") + chain.WriteDiagnostic(t, "v67-oracle-query.stdout", []byte(oracleQuery.Stdout)) + chain.WriteDiagnostic(t, "v67-oracle-query.stderr", []byte(oracleQuery.Stderr)) + require.Contains(t, oracleQuery.Combined(), "oracle module is deprecated") + + requireV67EscrowShapedBankState(t, chain) + requireV67IBCTransferQueriesGone(t, chain) + + requireV67PostUpgradeTxs(t, chain) + + requireV67CrashRecovery(t, chain) + requireV67OldBinaryOnMigratedNode(t, chain) + requireV67UnupgradedBinaryHalts(t, chain) + + chain.StopNode(t) + + currentGenesis := chain.Export(t, v67RunningSeid, "v67-export") + for _, module := range retiredStoreKeys { + require.NotContains(t, currentGenesis.AppState, module, + "v6.7 export unexpectedly contains retired module %s", module) + } + + releaseGenesis := chain.Export(t, chain.ReleaseBinary(t), "v66-export-after-v67") + for _, module := range retiredStoreKeys { + require.Contains(t, releaseGenesis.AppState, module, + "v6.6 cannot read retained %s state after the v6.7 store reload", module) + } + require.Positive(t, v67ExportedFeegrantAllowanceCount(t, releaseGenesis), + "the fee allowance written by v6.6 did not survive v6.7") +} + +func v67FeegrantSpendLimit(t *testing.T, output string) sdk.Int { + t.Helper() + var value any + require.NoError(t, json.Unmarshal([]byte(output), &value)) + amount, ok := findV67SpendLimit(value) + require.True(t, ok, "feegrant response has no usei spend limit: %s", output) + return amount +} + +func findV67SpendLimit(value any) (sdk.Int, bool) { + switch value := value.(type) { + case map[string]any: + if spendLimit, ok := value["spend_limit"].([]any); ok { + for _, coin := range spendLimit { + fields, ok := coin.(map[string]any) + if !ok || fields["denom"] != "usei" { + continue + } + if amount, ok := sdk.NewIntFromString(fmt.Sprint(fields["amount"])); ok { + return amount, true + } + } + } + for _, nested := range value { + if amount, ok := findV67SpendLimit(nested); ok { + return amount, true + } + } + case []any: + for _, nested := range value { + if amount, ok := findV67SpendLimit(nested); ok { + return amount, true + } + } + } + return sdk.Int{}, false +} + +func v67ExportedFeegrantAllowanceCount(t *testing.T, genesis upgradetest.ExportedGenesis) int { + t.Helper() + var feegrant struct { + Allowances []json.RawMessage `json:"allowances"` + } + require.NoError(t, json.Unmarshal(genesis.AppState[keys.FeegrantStoreKey], &feegrant)) + return len(feegrant.Allowances) +} + +func v67JSONInt(t *testing.T, encoded json.RawMessage) int64 { + t.Helper() + var text string + if err := json.Unmarshal(encoded, &text); err == nil { + value, parseErr := strconv.ParseInt(text, 10, 64) + require.NoError(t, parseErr) + return value + } + var value int64 + require.NoError(t, json.Unmarshal(encoded, &value)) + return value +} + +func requireV67RecordedPlanName(t *testing.T, chain *upgradetest.CrossVersion) string { + t.Helper() + want := chain.UpgradeName(t) + proposals := chain.MustSeid(t, "", "q", "gov", "proposals", "--output", "json") + chain.WriteDiagnostic(t, "v67-gov-proposals.json", []byte(proposals)) + planName := softwareUpgradePlanName(t, proposals) + require.Equal(t, want, planName) + + doneKey := append([]byte{upgradetypes.DoneByte}, []byte(planName)...) + require.NotEmpty(t, chain.QueryStore(t, upgradetypes.StoreKey, doneKey), + "upgrade store has no done entry for recorded plan name %q", planName) + + for _, miss := range upgradeNameNearMisses(planName) { + missKey := append([]byte{upgradetypes.DoneByte}, []byte(miss)...) + require.Empty(t, chain.QueryStore(t, upgradetypes.StoreKey, missKey), + "upgrade store has a done entry for near-miss name %q", miss) + applied := chain.Seid("", "q", "upgrade", "applied", miss, "--output", "json") + require.Error(t, applied.Err, "applied query succeeded for near-miss name %q", miss) + require.Contains(t, applied.Combined(), "no upgrade found") + } + return planName +} + +func softwareUpgradePlanName(t *testing.T, proposalsJSON string) string { + t.Helper() + var envelope struct { + Proposals []struct { + Content struct { + Type string `json:"@type"` + Plan struct { + Name string `json:"name"` + } `json:"plan"` + } `json:"content"` + } `json:"proposals"` + } + require.NoError(t, json.Unmarshal([]byte(proposalsJSON), &envelope)) + var names []string + for _, proposal := range envelope.Proposals { + if !strings.HasSuffix(proposal.Content.Type, ".SoftwareUpgradeProposal") { + continue + } + if strings.HasSuffix(proposal.Content.Type, ".CancelSoftwareUpgradeProposal") { + continue + } + require.NotEmpty(t, proposal.Content.Plan.Name) + names = append(names, proposal.Content.Plan.Name) + } + require.Len(t, names, 1, "want one software-upgrade proposal, got %q", names) + return names[0] +} + +func TestV67SoftwareUpgradePlanName(t *testing.T) { + got := softwareUpgradePlanName(t, `{ + "proposals": [ + {"content": {"@type": "/cosmos.gov.v1beta1.TextProposal", "title": "x"}}, + {"content": {"@type": "/cosmos.upgrade.v1beta1.CancelSoftwareUpgradeProposal", "title": "v6.7.0"}}, + {"content": {"@type": "/cosmos.upgrade.v1beta1.SoftwareUpgradeProposal", "title": "v6.7", "plan": {"name": "v6.7", "height": "42"}}} + ] +}`) + require.Equal(t, "v6.7", got) +} + +func v67ModuleVersionKey(module string) []byte { + return append([]byte{upgradetypes.VersionMapByte}, []byte(module)...) +} + +func requireV67UpgradeStoreVersions(t *testing.T, chain *upgradetest.CrossVersion) { + t.Helper() + for _, module := range retiredStoreKeys { + require.Empty(t, chain.QueryStore(t, upgradetypes.StoreKey, v67ModuleVersionKey(module)), + "v6.7 upgrade store still has a version-map entry for %s", module) + } +} + +func v67FeegrantAllowanceKey(t *testing.T, granter, grantee string) []byte { + t.Helper() + granterAddr, err := sdk.AccAddressFromBech32(granter) + require.NoError(t, err) + granteeAddr, err := sdk.AccAddressFromBech32(grantee) + require.NoError(t, err) + // Grantee then granter, matching feegrant.FeeAllowanceKey on v6.6. + key := []byte{v67FeegrantAllowancePrefix} + key = append(key, address.MustLengthPrefix(granteeAddr)...) + key = append(key, address.MustLengthPrefix(granterAddr)...) + return key +} + +type v67IBCTransferQuery struct { + name string + args []string +} + +func v67IBCTransferQueries() []v67IBCTransferQuery { + return []v67IBCTransferQuery{ + {name: "denom-traces", args: []string{"q", "ibc-transfer", "denom-traces", "--output", "json"}}, + {name: "params", args: []string{"q", "ibc-transfer", "params", "--output", "json"}}, + {name: "escrow-address", args: []string{"q", "ibc-transfer", "escrow-address", "transfer", "channel-0"}}, + } +} + +// seedV66EscrowShapedBankState funds the v6.6 escrow address for +// transfer/channel-0 by a bank send and records that address and balance. +func seedV66EscrowShapedBankState(t *testing.T, chain *upgradetest.CrossVersion) { + t.Helper() + escrow := strings.TrimSpace(chain.MustSeid(t, "", + "q", "ibc-transfer", "escrow-address", "transfer", "channel-0")) + require.NotEmpty(t, escrow, "v6.6 escrow-address query returned no address") + chain.Record(t, "escrow_style_address", escrow) + chain.WriteDiagnostic(t, "v66-ibc-transfer-escrow-address.stdout", []byte(escrow+"\n")) + + send := chain.Seid(v67KeyringPassword, + "tx", "bank", "send", "admin", escrow, v67EscrowStyleSendAmount, + "--from", "admin", + "--chain-id", "sei", + "--fees", "200000usei", + "--gas", "2000000", + "--broadcast-mode", "sync", + "--yes", + "--output", "json", + ) + chain.RequireDeliverTxSuccess(t, "v6.6 bank send to escrow-shaped address", send) + + balanceOut := chain.MustSeid(t, "", + "q", "bank", "balances", escrow, "--denom", "usei", "--output", "json") + chain.WriteDiagnostic(t, "v66-escrow-style-balance.json", []byte(balanceOut)) + _, balanceAmt := v67BankCoin(t, balanceOut) + require.Equal(t, "1234567", balanceAmt, + "the v6.6 bank send did not credit the escrow-shaped address") + chain.Record(t, "escrow_style_balance", balanceAmt) + + supplyOut := chain.MustSeid(t, "", + "q", "bank", "total", "--denom", "usei", "--output", "json") + chain.WriteDiagnostic(t, "v66-usei-supply.json", []byte(supplyOut)) + _, supplyAmt := v67BankCoin(t, supplyOut) + v67RequireSupplyCoversEscrow(t, supplyAmt, balanceAmt) + + for _, q := range v67IBCTransferQueries() { + out := chain.MustSeid(t, "", q.args...) + chain.WriteDiagnostic(t, "v66-ibc-transfer-"+q.name+".stdout", []byte(out)) + } +} + +// requireV67EscrowShapedBankState asserts the recorded escrow-shaped address +// still holds its balance and that usei supply still covers that balance. +func requireV67EscrowShapedBankState(t *testing.T, chain *upgradetest.CrossVersion) { + t.Helper() + var escrow, wantBalance string + chain.Replay(t, "escrow_style_address", &escrow) + chain.Replay(t, "escrow_style_balance", &wantBalance) + + balanceOut := chain.MustSeid(t, "", + "q", "bank", "balances", escrow, "--denom", "usei", "--output", "json") + chain.WriteDiagnostic(t, "v67-escrow-style-balance.json", []byte(balanceOut)) + _, gotBalance := v67BankCoin(t, balanceOut) + require.Equal(t, wantBalance, gotBalance, + "v6.7 changed the escrow-shaped address balance") + + supplyOut := chain.MustSeid(t, "", + "q", "bank", "total", "--denom", "usei", "--output", "json") + chain.WriteDiagnostic(t, "v67-usei-supply.json", []byte(supplyOut)) + _, gotSupply := v67BankCoin(t, supplyOut) + v67RequireSupplyCoversEscrow(t, gotSupply, wantBalance) +} + +// v67RequireSupplyCoversEscrow asserts usei total supply is at least the +// escrow-shaped balance. +func v67RequireSupplyCoversEscrow(t *testing.T, supplyAmt, escrowAmt string) { + t.Helper() + supply, ok := sdk.NewIntFromString(supplyAmt) + require.True(t, ok, "invalid usei supply %q", supplyAmt) + escrow, ok := sdk.NewIntFromString(escrowAmt) + require.True(t, ok, "invalid escrow-shaped balance %q", escrowAmt) + require.True(t, supply.GTE(escrow), + "usei total supply %s is below the escrow-shaped balance %s; inflation may raise supply, but the escrowed coins must still be counted", + supplyAmt, escrowAmt) +} + +// requireV67IBCTransferQueriesGone asserts that the v6.6 ibc-transfer query +// commands are no longer exposed. +func requireV67IBCTransferQueriesGone(t *testing.T, chain *upgradetest.CrossVersion) { + t.Helper() + for _, q := range v67IBCTransferQueries() { + result := chain.Seid("", q.args...) + chain.WriteDiagnostic(t, "v67-ibc-transfer-"+q.name+".stdout", []byte(result.Stdout)) + chain.WriteDiagnostic(t, "v67-ibc-transfer-"+q.name+".stderr", []byte(result.Stderr)) + require.Error(t, result.Err, "v6.7 still exposes q ibc-transfer %s", q.name) + } +} + +func v67BankCoin(t *testing.T, output string) (denom, amount string) { + t.Helper() + var coin struct { + Denom string `json:"denom"` + Amount json.RawMessage `json:"amount"` + } + require.NoError(t, json.Unmarshal([]byte(strings.TrimSpace(output)), &coin), output) + require.NotEmpty(t, coin.Denom, "bank coin JSON has no denom: %s", output) + return coin.Denom, v67CoinAmount(t, coin.Amount).String() +} + +// v67CoinAmount decodes a coin amount. A chain's supply reaches far beyond an +// int64, and proto JSON quotes the field as a string. +func v67CoinAmount(t *testing.T, encoded json.RawMessage) sdk.Int { + t.Helper() + text := strings.Trim(strings.TrimSpace(string(encoded)), `"`) + amount, ok := sdk.NewIntFromString(text) + require.True(t, ok, "invalid coin amount %s", encoded) + return amount +} + +// requireV67PostUpgradeTxs broadcasts a bank send and an EVM native-send after +// the upgrade and requires that each is included and changes state. +func requireV67PostUpgradeTxs(t *testing.T, chain *upgradetest.CrossVersion) { + t.Helper() + sender := chain.KeyAddress(t, chain.Node(), "admin") + receiver := v67PostUpgradeBankReceiver.String() + + seqBefore := v67AccountSequence(t, chain, sender) + recvBefore := v67UseiBalance(t, chain, receiver) + + chain.RequireDeliverTxSuccess(t, "v6.7 bank send", chain.Seid(v67KeyringPassword, + "tx", "bank", "send", "admin", receiver, v67PostUpgradeSendAmount, + "--from", "admin", + "--chain-id", "sei", + "--fees", "200000usei", + "--gas", "2000000", + "--broadcast-mode", "sync", + "--yes", + "--output", "json", + )) + require.Equal(t, seqBefore+1, v67AccountSequence(t, chain, sender), + "bank send did not advance the sender sequence") + require.Equal(t, recvBefore.AddRaw(v67PostUpgradeSendAmt).String(), + v67UseiBalance(t, chain, receiver).String(), + "bank send did not credit the receiver") + + evmRecv := v67EVMCastSeiAddress(t, v67EVMNativeSendTo) + evmBefore := v67UseiBalance(t, chain, evmRecv) + chain.RequireDeliverTxSuccess(t, "v6.7 evm native-send", chain.Seid(v67KeyringPassword, + "tx", "evm", "native-send", "admin", v67EVMNativeSendTo, v67PostUpgradeSendAmount, + "--from", "admin", + "--chain-id", "sei", + "--fees", "200000usei", + "--gas", "2000000", + "--broadcast-mode", "sync", + "--yes", + "--output", "json", + )) + require.Equal(t, seqBefore+2, v67AccountSequence(t, chain, sender), + "evm native-send did not advance the sender sequence") + require.Equal(t, evmBefore.AddRaw(v67PostUpgradeSendAmt).String(), + v67UseiBalance(t, chain, evmRecv).String(), + "evm native-send did not credit the cast Sei address") +} + +func v67AccountSequence(t *testing.T, chain *upgradetest.CrossVersion, address string) int64 { + t.Helper() + out := chain.MustSeid(t, "", "q", "account", address, "--output", "json") + var acc struct { + Sequence json.RawMessage `json:"sequence"` + } + require.NoError(t, json.Unmarshal([]byte(out), &acc), out) + if len(acc.Sequence) == 0 { + return 0 + } + return v67JSONInt(t, acc.Sequence) +} + +func v67UseiBalance(t *testing.T, chain *upgradetest.CrossVersion, address string) sdk.Int { + t.Helper() + out := chain.MustSeid(t, "", + "q", "bank", "balances", address, "--denom", "usei", "--output", "json") + _, amount := v67BankCoin(t, out) + value, ok := sdk.NewIntFromString(amount) + require.True(t, ok, "invalid usei balance %q", amount) + return value +} + +func v67EVMCastSeiAddress(t *testing.T, evmHex string) string { + t.Helper() + raw, err := hex.DecodeString(strings.TrimPrefix(evmHex, "0x")) + require.NoError(t, err, "decode EVM address %s", evmHex) + require.Len(t, raw, 20, "EVM address %s is not 20 bytes", evmHex) + return sdk.AccAddress(raw).String() +} + +func stringDifference(left, right []string) []string { + rightSet := make(map[string]struct{}, len(right)) + for _, value := range right { + rightSet[value] = struct{}{} + } + var difference []string + for _, value := range left { + if _, ok := rightSet[value]; !ok { + difference = append(difference, value) + } + } + sort.Strings(difference) + return difference +} + +func sortedStrings(values []string) []string { + sorted := append([]string(nil), values...) + sort.Strings(sorted) + return sorted +} + +// runRetiredModuleTxs submits one transaction per case in a single block and +// asserts the rejection and fee outcome each case declares. +func runRetiredModuleTxs(t *testing.T, a *processblock.App, phase string, accounts retiredModuleTxAccounts) { + t.Helper() + cases := retiredModuleTxCases() + + balancesBefore := make([]sdk.Coin, len(cases)) + txs := make([]signing.Tx, len(cases)) + for i, c := range cases { + balancesBefore[i] = a.BankKeeper.GetBalance(a.Ctx(), accounts.signers[i], "usei") + txs[i] = c.sign(a, accounts.signers[i], accounts.granters[i]) + } + + results := a.RunBlockDetailed(txs) + require.Len(t, results, len(cases)) + + for i, c := range cases { + t.Run(phase+"/"+c.name, func(t *testing.T) { + res := results[i] + require.Equal(t, c.code, res.Code, "log was %q", res.Log) + require.Equal(t, c.codespace, res.Codespace) + require.Contains(t, res.Log, c.logContains) + + spent := balancesBefore[i].Sub(a.BankKeeper.GetBalance(a.Ctx(), accounts.signers[i], "usei")) + if c.chargesFee { + require.Equal(t, sdk.NewInt64Coin("usei", txFee), spent, + "a transaction rejected by the message handler still pays its fee") + } else { + require.True(t, spent.IsZero(), + "a transaction rejected before fee deduction must cost nothing, spent %s", spent) + } + }) + } +} + +// runOrdinaryTxs submits a bank send and an EVM native send in one block and +// requires each delivered result to succeed and to move balances and sequence. +func runOrdinaryTxs(t *testing.T, a *processblock.App, phase string, accounts ordinaryTxAccounts) { + t.Helper() + const sendAmt int64 = 1000 + evmTo := common.HexToAddress(v67EVMNativeSendTo) + evmRecv := sdk.AccAddress(evmTo.Bytes()) + + bankSenderBefore := a.BankKeeper.GetBalance(a.Ctx(), accounts.bankSender, "usei") + bankRecvBefore := a.BankKeeper.GetBalance(a.Ctx(), accounts.bankReceiver, "usei") + bankSeqBefore := a.AccountKeeper.GetAccount(a.Ctx(), accounts.bankSender).GetSequence() + evmSenderBefore := a.BankKeeper.GetBalance(a.Ctx(), accounts.evmSender, "usei") + evmRecvBefore := a.BankKeeper.GetBalance(a.Ctx(), evmRecv, "usei") + evmSeqBefore := a.AccountKeeper.GetAccount(a.Ctx(), accounts.evmSender).GetSequence() + + results := a.RunBlockDetailed([]signing.Tx{ + a.Sign(accounts.bankSender, txFee, msgs.Send(accounts.bankSender, accounts.bankReceiver, sendAmt)), + a.Sign(accounts.evmSender, txFee, evmtypes.NewMsgSend( + accounts.evmSender, evmTo, sdk.NewCoins(sdk.NewInt64Coin("usei", sendAmt)))), + }) + require.Len(t, results, 2) + + t.Run(phase+"/bank send", func(t *testing.T) { + require.Equal(t, uint32(abci.CodeTypeOK), results[0].Code, results[0].Log) + require.Positive(t, results[0].GasUsed) + require.Equal(t, sdk.NewInt64Coin("usei", sendAmt), + a.BankKeeper.GetBalance(a.Ctx(), accounts.bankReceiver, "usei").Sub(bankRecvBefore)) + require.Equal(t, sdk.NewInt64Coin("usei", sendAmt+txFee), + bankSenderBefore.Sub(a.BankKeeper.GetBalance(a.Ctx(), accounts.bankSender, "usei"))) + require.Equal(t, bankSeqBefore+1, + a.AccountKeeper.GetAccount(a.Ctx(), accounts.bankSender).GetSequence()) + }) + t.Run(phase+"/evm native send", func(t *testing.T) { + require.Equal(t, uint32(abci.CodeTypeOK), results[1].Code, results[1].Log) + require.Positive(t, results[1].GasUsed) + require.Equal(t, sdk.NewInt64Coin("usei", sendAmt), + a.BankKeeper.GetBalance(a.Ctx(), evmRecv, "usei").Sub(evmRecvBefore)) + require.Equal(t, sdk.NewInt64Coin("usei", sendAmt+txFee), + evmSenderBefore.Sub(a.BankKeeper.GetBalance(a.Ctx(), accounts.evmSender, "usei"))) + require.Equal(t, evmSeqBefore+1, + a.AccountKeeper.GetAccount(a.Ctx(), accounts.evmSender).GetSequence()) + }) +} + +// Transactions aimed at retired modules must follow the v6.7 tombstone +// behavior after the upgrade handler runs. TestV67CrossVersion covers the +// behavior transition from the old binary. +func TestV67RejectsRetiredModuleTxsAfterUpgrade(t *testing.T) { + a := newV67Chain(t) + afterAccounts := fundRetiredModuleTxAccounts(a, "after-upgrade") + + applyV67(t, a) + runRetiredModuleTxs(t, a, "after-upgrade", afterAccounts) +} + +// TestV67AcceptsOrdinaryTxsAcrossUpgrade delivers a bank send and an EVM native +// send before and after applyV67. +func TestV67AcceptsOrdinaryTxsAcrossUpgrade(t *testing.T) { + a := newV67Chain(t) + before := fundOrdinaryTxAccounts(a, "before-upgrade") + after := fundOrdinaryTxAccounts(a, "after-upgrade") + + runOrdinaryTxs(t, a, "before-upgrade", before) + applyV67ToCommitStore(t, a) + runOrdinaryTxs(t, a, "after-upgrade", after) +} + +// Spamming a retired module is not free. Every oracle transaction is rejected, +// yet it still pays its fee, consumes a sequence number, and occupies block +// gas. This is what stops a retired handler from becoming a free denial of +// service, so it is asserted rather than left as an implementation detail. +func TestRetiredOracleTxsAreRejectedButStillCharged(t *testing.T) { + a := newV67Chain(t) + applyV67(t, a) + + const spamCount = 25 + spammer := a.NewSignableAccount("oracle-spammer") + a.FundAccount(spammer, 1000000000) + + balanceBefore := a.BankKeeper.GetBalance(a.Ctx(), spammer, "usei") + sequenceBefore := a.AccountKeeper.GetAccount(a.Ctx(), spammer).GetSequence() + + txs := make([]signing.Tx, spamCount) + for i := range txs { + txs[i] = a.Sign(spammer, txFee, oracletypes.NewMsgAggregateExchangeRateVote( + "1.5uatom", spammer, sdk.ValAddress(spammer))) + } + + for i, res := range a.RunBlockDetailed(txs) { + require.Equal(t, uint32(oracletypes.ErrOracleDeprecated.ABCICode()), res.Code, + "spam transaction %d: %s", i, res.Log) + require.Positive(t, res.GasUsed, "spam transaction %d consumed no gas", i) + } + + spent := balanceBefore.Sub(a.BankKeeper.GetBalance(a.Ctx(), spammer, "usei")) + require.Equal(t, sdk.NewInt64Coin("usei", txFee*spamCount), spent) + require.Equal(t, sequenceBefore+spamCount, + a.AccountKeeper.GetAccount(a.Ctx(), spammer).GetSequence()) + + // Nothing the spam carried reached oracle state. + _, err := a.OracleKeeper.GetAggregateExchangeRateVote(a.Ctx(), sdk.ValAddress(spammer)) + require.ErrorIs(t, err, oracletypes.ErrNoAggregateVote, + "a rejected vote must not be recorded") +} + +// TestV67RetainsRetiredModuleStateWrittenBeforeUpgrade pins that v6.7 leaves +// the retired stores mounted with their committed key/value sets unchanged, +// and removes their version-map entries from the upgrade store's committed bytes. +func TestV67RetainsRetiredModuleStateWrittenBeforeUpgrade(t *testing.T) { + a := newV67Chain(t) + + seeded := seedRetiredStores(t, a) + a.RunBlock([]signing.Tx{}) + requireRetiredStoresMounted(t, a) + before := snapshotRetiredStores(t, a) + for _, store := range retiredStoreKeys { + require.Equal(t, seeded[store], before[store]["seeded"], + "seeded %s state is missing from the committed store", store) + require.True(t, committedModuleVersionExists(t, a, store), + "seeded %s module version is missing from the committed upgrade store", store) + requireRetiredStoreInCommitment(t, a, store, []byte("seeded"), seeded[store]) + } + + applyV67ToCommitStore(t, a) + a.RunBlock([]signing.Tx{}) + requireRetiredStoresUnchanged(t, a, before, seeded) + + for range 5 { + a.RunBlock([]signing.Tx{}) + } + requireRetiredStoresUnchanged(t, a, before, seeded) +} + +// TestV67LeavesBankBalancesAndSupplyUntouched asserts that applying v6.7 does +// not move or burn bank balances or total supply, including an ibc/-shaped +// denom and coins at an escrow-style address. +func TestV67LeavesBankBalancesAndSupplyUntouched(t *testing.T) { + a := newV67Chain(t) + + voucherHolder := a.NewAccount() + a.FundAccountWithDenom(voucherHolder, v67IBCVoucherShapeAmount, v67IBCVoucherDenomShape) + a.FundAccountWithDenom(v67EscrowStyleAddress, v67EscrowStyleAmount, "usei") + + voucherBefore := a.BankKeeper.GetBalance(a.Ctx(), voucherHolder, v67IBCVoucherDenomShape) + escrowBefore := a.BankKeeper.GetBalance(a.Ctx(), v67EscrowStyleAddress, "usei") + require.Equal(t, sdk.NewInt64Coin(v67IBCVoucherDenomShape, v67IBCVoucherShapeAmount), voucherBefore) + require.Equal(t, sdk.NewInt64Coin("usei", v67EscrowStyleAmount), escrowBefore) + require.Equal(t, voucherBefore, a.BankKeeper.GetSupply(a.Ctx(), v67IBCVoucherDenomShape)) + require.True(t, a.BankKeeper.GetSupply(a.Ctx(), "usei").IsGTE(escrowBefore)) + + holderBalancesBefore := a.BankKeeper.GetAllBalances(a.Ctx(), voucherHolder) + escrowBalancesBefore := a.BankKeeper.GetAllBalances(a.Ctx(), v67EscrowStyleAddress) + suppliesBefore := v67BankSupplySnapshot(a) + + applyV67(t, a) + + require.Equal(t, suppliesBefore, v67BankSupplySnapshot(a), + "v6.7 moved or burned bank supply") + require.Equal(t, holderBalancesBefore, a.BankKeeper.GetAllBalances(a.Ctx(), voucherHolder), + "v6.7 changed balances of an ibc/ voucher holder") + require.Equal(t, escrowBalancesBefore, a.BankKeeper.GetAllBalances(a.Ctx(), v67EscrowStyleAddress), + "v6.7 changed balances at an escrow-style address") +} + +func v67BankSupplySnapshot(a *processblock.App) map[string]string { + supplies := map[string]string{} + a.BankKeeper.IterateTotalSupply(a.Ctx(), func(c sdk.Coin) bool { + supplies[c.Denom] = c.Amount.String() + return false + }) + return supplies +} + +func seedRetiredStores(t *testing.T, a *processblock.App) map[string][]byte { + t.Helper() + seeded := map[string][]byte{} + versionMap := a.UpgradeKeeper.GetModuleVersionMap(a.Ctx()) + for _, store := range retiredStoreKeys { + key := a.GetKey(store) + require.NotNil(t, key, "%s store is no longer mounted", store) + value := []byte("pre-upgrade/" + store) + a.Ctx().KVStore(key).Set([]byte("seeded"), value) + seeded[store] = value + versionMap[store] = 1 + } + a.UpgradeKeeper.SetModuleVersionMap(a.Ctx(), versionMap) + return seeded +} + +func requireRetiredStoresUnchanged(t *testing.T, a *processblock.App, before map[string]map[string][]byte, seeded map[string][]byte) { + t.Helper() + requireRetiredStoresMounted(t, a) + require.Equal(t, before, snapshotRetiredStores(t, a), + "a retired store's committed key/value set changed after v6.7") + for _, store := range retiredStoreKeys { + require.False(t, committedModuleVersionExists(t, a, store), + "committed upgrade store still has a version-map entry for %s", store) + requireRetiredStoreInCommitment(t, a, store, []byte("seeded"), seeded[store]) + } +} + +const ( + v67ValidatorHome = "/root/.sei" + v67UnupgradedHaltNode = "sei-node-3" + v67UnupgradedHomeSnapshot = "/tmp/v67-unupgraded-home" + v67UpgradedHomeSnapshot = "/tmp/v67-upgraded-home" + v67OldBinaryHomeSnapshot = "/tmp/v67-pre-old-binary" + v67RunningSeid = "/root/go/bin/seid" + v67LiveHarnessConfigKey = "validator_runtime_config" + v67LiveHarnessPruning = "nothing" +) + +const v67LiveHarnessPruningMessage = "the live upgrade harness pins pruning = %q so historical queries survive this run; mainnet validators prune, so this suite does not prove that a pruning validator still serves retained retired-store data after v6.7" + +// v67ValidatorRuntimeConfig is the subset of a validator's app.toml that an +// upgrade must not silently change. +type v67ValidatorRuntimeConfig struct { + SCEnable bool `json:"sc_enable"` + SSEnable bool `json:"ss_enable"` + OCCEnabled bool `json:"occ_enabled"` + Pruning string `json:"pruning"` +} + +// v67ReadLiveHarnessConfig reads each validator's running app.toml and requires +// SeiDB and OCC enabled and pruning = "nothing". Mainnet validators prune, so +// this cluster does not prove retained retired-store data survives on a pruning node. +func v67ReadLiveHarnessConfig(t *testing.T, chain *upgradetest.CrossVersion) map[string]v67ValidatorRuntimeConfig { + t.Helper() + nodes := chain.Nodes() + configs := make(map[string]v67ValidatorRuntimeConfig, len(nodes)) + for _, node := range nodes { + cfg := v67ReadValidatorRuntimeConfig(t, chain, node) + v67RequireLiveHarnessSettings(t, node, cfg) + configs[node] = cfg + } + return configs +} + +func v67ReadValidatorRuntimeConfig(t *testing.T, chain *upgradetest.CrossVersion, node string) v67ValidatorRuntimeConfig { + t.Helper() + path := v67ValidatorHome + "/config/app.toml" + result := chain.BinaryOn(node, "", "cat", path) + chain.WriteDiagnostic(t, node+"-app.toml", []byte(result.Stdout)) + require.NoError(t, result.Err, "read %s %s: %s", node, path, result.Combined()) + require.NotEmpty(t, result.Stdout, "%s %s is empty", node, path) + return v67ParseValidatorRuntimeConfig(t, node, result.Stdout) +} + +func v67ParseValidatorRuntimeConfig(t *testing.T, node, tomlText string) v67ValidatorRuntimeConfig { + t.Helper() + return v67ValidatorRuntimeConfig{ + SCEnable: v67TomlBool(t, node, tomlText, "sc-enable"), + SSEnable: v67TomlBool(t, node, tomlText, "ss-enable"), + OCCEnabled: v67TomlBool(t, node, tomlText, "occ-enabled"), + Pruning: v67TomlScalar(t, node, tomlText, "pruning"), + } +} + +func v67RequireLiveHarnessSettings(t *testing.T, node string, cfg v67ValidatorRuntimeConfig) { + t.Helper() + require.True(t, cfg.SCEnable, "%s sc-enable=%v, want true", node, cfg.SCEnable) + require.True(t, cfg.SSEnable, "%s ss-enable=%v, want true", node, cfg.SSEnable) + require.True(t, cfg.OCCEnabled, + "%s occ-enabled=%v, want true; the live cluster must run OCC because the fleet does", + node, cfg.OCCEnabled) + require.Equal(t, v67LiveHarnessPruning, cfg.Pruning, + "%s pruning=%q, want %q; "+v67LiveHarnessPruningMessage, + node, cfg.Pruning, v67LiveHarnessPruning, v67LiveHarnessPruning) +} + +func v67TomlBool(t *testing.T, node, tomlText, key string) bool { + t.Helper() + raw := v67TomlScalar(t, node, tomlText, key) + value, err := strconv.ParseBool(raw) + require.NoError(t, err, "%s %s=%q is not a bool", node, key, raw) + return value +} + +func v67TomlScalar(t *testing.T, node, tomlText, key string) string { + t.Helper() + value, ok := v67LastTomlScalar(tomlText, key) + require.True(t, ok, "%s app.toml has no %s key", node, key) + return value +} + +func v67LastTomlScalar(tomlText, key string) (string, bool) { + found := false + value := "" + for _, line := range strings.Split(tomlText, "\n") { + line = strings.TrimSpace(strings.TrimSuffix(line, "\r")) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + if i := strings.Index(line, " #"); i >= 0 { + line = strings.TrimSpace(line[:i]) + } + name, raw, ok := strings.Cut(line, "=") + if !ok { + continue + } + if strings.TrimSpace(name) != key { + continue + } + raw = strings.TrimSpace(raw) + if unquoted, err := strconv.Unquote(raw); err == nil { + raw = unquoted + } + value = raw + found = true + } + return value, found +} + +func TestV67TomlScalarParser(t *testing.T) { + text := ` +# occ-enabled = false +pruning = "nothing" # archive +occ-enabled = true +sc-enable = true +ss-enable = true + +[giga_executor] +occ_enabled = false + +[state-commit] +sc-enable = false +` + sc, ok := v67LastTomlScalar(text, "sc-enable") + require.True(t, ok) + require.Equal(t, "false", sc) + + pruning, ok := v67LastTomlScalar(text, "pruning") + require.True(t, ok) + require.Equal(t, "nothing", pruning) + + occ, ok := v67LastTomlScalar(text, "occ-enabled") + require.True(t, ok) + require.Equal(t, "true", occ) + + _, ok = v67LastTomlScalar(text, "occ_enabled") + require.True(t, ok) + + _, ok = v67LastTomlScalar(text, "concurrency-workers") + require.False(t, ok) + + cfg := v67ParseValidatorRuntimeConfig(t, "fixture", ` +pruning = "nothing" +occ-enabled = true +sc-enable = true +ss-enable = true +`) + require.Equal(t, v67ValidatorRuntimeConfig{ + SCEnable: true, + SSEnable: true, + OCCEnabled: true, + Pruning: v67LiveHarnessPruning, + }, cfg) +} + +func copyV67ValidatorHome(t *testing.T, chain *upgradetest.CrossVersion, node, dst string) { + t.Helper() + result := chain.BinaryOn(node, "", "sh", "-c", + fmt.Sprintf("rm -rf %s && cp -a %s %s", dst, v67ValidatorHome, dst)) + require.NoError(t, result.Err, "copy %s home to %s: %s", node, dst, result.Combined()) +} + +func replaceV67ValidatorHome(t *testing.T, chain *upgradetest.CrossVersion, node, src string) { + t.Helper() + result := chain.BinaryOn(node, "", "sh", "-c", + fmt.Sprintf("rm -rf %s && mv %s %s", v67ValidatorHome, src, v67ValidatorHome)) + require.NoError(t, result.Err, "replace %s home from %s: %s", node, src, result.Combined()) +} + +func requireV67UpgradeNeededLog(t *testing.T, log string, height int64) { + t.Helper() + expected := fmt.Sprintf(`UPGRADE "%s" NEEDED at height: %d`, v67UpgradeName, height) + escaped := fmt.Sprintf(`UPGRADE \"%s\" NEEDED at height: %d`, v67UpgradeName, height) + require.True(t, strings.Contains(log, expected) || strings.Contains(log, escaped), + "missing upgrade-needed halt at height %d\nlog:\n%s", height, log) +} + +func restoreV67Validator(t *testing.T, chain *upgradetest.CrossVersion, node, snapshot string) { + t.Helper() + chain.StopNodeOn(t, node) + replaceV67ValidatorHome(t, chain, node, snapshot) + chain.StartNodeOn(t, node, v67RunningSeid) + chain.RequireBlockAgreement(t, chain.Height(t)) +} + +// preserveV67UnupgradedHome copies a non-primary validator home while seid is +// stopped, then requires that validator to rejoin. +func preserveV67UnupgradedHome(t *testing.T, chain *upgradetest.CrossVersion) { + t.Helper() + require.NotEqual(t, chain.Node(), v67UnupgradedHaltNode) + chain.StopNodeOn(t, v67UnupgradedHaltNode) + copyV67ValidatorHome(t, chain, v67UnupgradedHaltNode, v67UnupgradedHomeSnapshot) + chain.StartNodeOn(t, v67UnupgradedHaltNode, v67RunningSeid) + chain.RequireBlockAgreement(t, chain.Height(t)) + chain.Record(t, "unupgraded_home_node", v67UnupgradedHaltNode) +} + +// requireV67CrashRecovery asserts that a validator killed without a clean +// shutdown after the upgrade replays its last block on restart and agrees with +// its peers. It covers recovery near the boundary, not a crash inside the +// upgrade block, which no reliably timed test can produce. +func requireV67CrashRecovery(t *testing.T, chain *upgradetest.CrossVersion) { + t.Helper() + const peer = "sei-node-1" + killedAt := chain.HeightOn(t, peer) + chain.KillNodeOn(t, peer) + chain.WaitForBlocks(t, 2) + chain.StartNodeOn(t, peer, v67RunningSeid) + chain.WaitForHeightOn(t, peer, killedAt, 3*time.Minute) + chain.RequireBlockAgreement(t, chain.Height(t)) +} + +// requireV67OldBinaryOnMigratedNode starts the v6.6 binary against an upgraded +// validator database, then restores the v6.7 binary. +func requireV67OldBinaryOnMigratedNode(t *testing.T, chain *upgradetest.CrossVersion) { + t.Helper() + const node = "sei-node-2" + require.NotEqual(t, chain.Node(), node) + + upgradeHeight := chain.TargetHeight(t) + stoppedAt := chain.HeightOn(t, node) + require.GreaterOrEqual(t, stoppedAt, upgradeHeight) + + chain.StopNodeOn(t, node) + copyV67ValidatorHome(t, chain, node, v67OldBinaryHomeSnapshot) + + restored := false + restore := func() { + if restored { + return + } + restored = true + t.Logf("restoring v6.7 binary on %s after old-binary observation", node) + restoreV67Validator(t, chain, node, v67OldBinaryHomeSnapshot) + } + defer restore() + + observed := chain.StartNodeObserving(t, node, chain.ReleaseBinary(t), 45*time.Second) + chain.WriteDiagnostic(t, "v66-on-v67-restart.log", []byte(observed.Log)) + + require.False(t, observed.Running, + "v6.6 seid stayed up on a v6.7 database\nlog:\n%s", observed.Log) + require.False(t, observed.Height > stoppedAt, + "v6.6 seid advanced from height %d to %d on a v6.7 database\nlog:\n%s", + stoppedAt, observed.Height, observed.Log) + require.True(t, + strings.Contains(observed.Log, "state.AppHash does not match AppHash after replay") || + strings.Contains(observed.Log, "upgrade handler is missing for v6.7 upgrade plan"), + "v6.6 seid neither failed the handshake nor panicked on the missing v6.7 handler:\n%s", observed.Log) + + restore() +} + +// requireV67UnupgradedBinaryHalts starts the v6.6 binary against a pre-upgrade +// validator home and requires it to halt at the v6.7 plan height. +func requireV67UnupgradedBinaryHalts(t *testing.T, chain *upgradetest.CrossVersion) { + t.Helper() + var node string + chain.Replay(t, "unupgraded_home_node", &node) + require.NotEqual(t, chain.Node(), node) + require.Equal(t, v67UnupgradedHaltNode, node) + + upgradeHeight := chain.TargetHeight(t) + chain.StopNodeOn(t, node) + copyV67ValidatorHome(t, chain, node, v67UpgradedHomeSnapshot) + replaceV67ValidatorHome(t, chain, node, v67UnupgradedHomeSnapshot) + + restored := false + restore := func() { + if restored { + return + } + restored = true + t.Logf("restoring upgraded home on %s after un-upgraded halt", node) + restoreV67Validator(t, chain, node, v67UpgradedHomeSnapshot) + } + defer restore() + + observed := chain.StartNodeObserving(t, node, chain.ReleaseBinary(t), 3*time.Minute) + chain.WriteDiagnostic(t, "v66-unupgraded-halt.log", []byte(observed.Log)) + + require.False(t, observed.Running, + "un-upgraded seid stayed up through the v6.7 plan height\nlog:\n%s", observed.Log) + require.False(t, observed.Height >= upgradeHeight, + "un-upgraded seid committed at or past plan height %d (last observed %d)\nlog:\n%s", + upgradeHeight, observed.Height, observed.Log) + requireV67UpgradeNeededLog(t, observed.Log, upgradeHeight) + + restore() +} + +func snapshotDeliverRetiredStores(t *testing.T, a *processblock.App) map[string]map[string][]byte { + t.Helper() + stores := make(map[string]map[string][]byte, len(retiredStoreKeys)) + for _, name := range retiredStoreKeys { + stores[name] = snapshotDeliverStore(t, a, name) + } + return stores +} + +func snapshotDeliverStore(t *testing.T, a *processblock.App, storeName string) map[string][]byte { + t.Helper() + storeKey := a.GetKey(storeName) + require.NotNil(t, storeKey, "%s store is not mounted", storeName) + iterator := a.Ctx().KVStore(storeKey).Iterator(nil, nil) + defer func() { + require.NoError(t, iterator.Close()) + }() + entries := map[string][]byte{} + for ; iterator.Valid(); iterator.Next() { + entries[string(iterator.Key())] = append([]byte(nil), iterator.Value()...) + } + return entries +} + +func requireRetiredStoresMounted(t *testing.T, a *processblock.App) { + t.Helper() + cms := a.CommitMultiStore() + mounted := map[string]struct{}{} + for _, key := range cms.StoreKeys() { + mounted[key.Name()] = struct{}{} + } + for _, name := range retiredStoreKeys { + _, ok := mounted[name] + require.True(t, ok, "%s is not present in the commit multistore", name) + key := a.GetKey(name) + require.NotNil(t, key, "%s store is no longer mounted", name) + require.NotNil(t, cms.GetCommitKVStore(key), "%s is not in the commit multistore", name) + } +} + +func snapshotRetiredStores(t *testing.T, a *processblock.App) map[string]map[string][]byte { + t.Helper() + stores := make(map[string]map[string][]byte, len(retiredStoreKeys)) + for _, name := range retiredStoreKeys { + stores[name] = snapshotCommittedStore(t, a, name) + } + return stores +} + +func snapshotCommittedStore(t *testing.T, a *processblock.App, storeName string) map[string][]byte { + t.Helper() + storeKey := a.GetKey(storeName) + require.NotNil(t, storeKey, "%s store is not mounted", storeName) + store := a.CommitMultiStore().GetCommitKVStore(storeKey) + require.NotNil(t, store, "%s is not in the commit multistore", storeName) + iterator := store.Iterator(nil, nil) + defer func() { + require.NoError(t, iterator.Close()) + }() + entries := map[string][]byte{} + for ; iterator.Valid(); iterator.Next() { + entries[string(iterator.Key())] = append([]byte(nil), iterator.Value()...) + } + return entries +} + +func requireRetiredStoreInCommitment(t *testing.T, a *processblock.App, storeName string, key, want []byte) { + t.Helper() + queryable, ok := a.CommitMultiStore().(sdk.Queryable) + require.True(t, ok, "commit multistore does not support queries") + resp := queryable.Query(context.Background(), abci.RequestQuery{ + Path: "/" + storeName + "/key", + Data: key, + Prove: true, + }) + require.Equal(t, uint32(0), resp.Code, "query /%s/key: %s", storeName, resp.Log) + require.Equal(t, want, resp.Value, "query /%s/key returned a different value", storeName) + require.NotNil(t, resp.ProofOps, "%s is missing from the commitment set", storeName) + require.NotEmpty(t, resp.ProofOps.Ops, "%s is missing from the commitment set", storeName) +} + +func committedModuleVersionExists(t *testing.T, a *processblock.App, module string) bool { + t.Helper() + key := a.GetKey(upgradetypes.StoreKey) + require.NotNil(t, key, "upgrade store is not mounted") + store := a.CommitMultiStore().GetCommitKVStore(key) + require.NotNil(t, store, "upgrade store is not in the commit multistore") + return store.Has(append([]byte{upgradetypes.VersionMapByte}, []byte(module)...)) +} + +// Deprecating the oracle handlers stopped transactions from reaching oracle +// state, but the module is still in the manager and its mid blocker still runs +// every vote period. With no votes to tally it marks every bonded validator +// absent, so the store keeps being written after the upgrade even though no +// client can put anything in it. +// +// This is characterization, not a defect: it records the write that a later +// oracle removal has to stop before it can drop the store, because a module +// still writing at the height its store is deleted is how an upgrade halts a +// chain. If oracle stops writing, this test should be deleted along with the +// blocker, not adjusted to keep passing. +func TestOracleKeepsWritingStateAfterV67(t *testing.T) { + a := newV67Chain(t) + applyV67(t, a) + + validator := a.GetAllValidators()[0] + operator, err := sdk.ValAddressFromBech32(validator.OperatorAddress) + require.NoError(t, err) + + require.Zero(t, a.OracleKeeper.GetVotePenaltyCounter(a.Ctx(), operator).AbstainCount) + + // A vote period is two blocks by default, so this spans several of them. + for i := 0; i < 8; i++ { + a.RunBlock([]signing.Tx{}) + } + + require.Positive(t, a.OracleKeeper.GetVotePenaltyCounter(a.Ctx(), operator).AbstainCount, + "oracle no longer records abstentions after v6.7; if its blocker was removed, "+ + "remove this test with it") + _, err = a.OracleKeeper.GetAggregateExchangeRateVote(a.Ctx(), operator) + require.ErrorIs(t, err, oracletypes.ErrNoAggregateVote, + "the abstentions must come from the blocker, not from a vote that got through") +} + +// The retired stores survive the upgrade, but no module claims them, so genesis +// export cannot emit them. An opaque write made before the upgrade is still in +// the store after export, and is absent from the exported genesis document. +func TestV67RetainedStateIsAbsentFromExportedGenesis(t *testing.T) { + a := newV67Chain(t) + for _, store := range retiredStoreKeys { + a.Ctx().KVStore(a.GetKey(store)).Set([]byte("seeded"), []byte("pre-upgrade")) + } + + applyV67(t, a) + a.RunBlock([]signing.Tx{}) + + exported, err := a.ExportAppStateAndValidators(false, nil) + require.NoError(t, err) + + var genesis map[string]json.RawMessage + require.NoError(t, json.Unmarshal(exported.AppState, &genesis)) + for _, store := range retiredStoreKeys { + require.Equal(t, []byte("pre-upgrade"), a.Ctx().KVStore(a.GetKey(store)).Get([]byte("seeded")), + "the %s state must still be in the store, or its absence from the export proves nothing", store) + require.NotContains(t, genesis, store, + "an exported genesis with a %s section would need a module to import it", store) + } +} + +// retiredStoreKeys are the stores whose modules v6.7 removes while keeping the +// store mounted. Declared here in terms of the sei-db key constants so this +// external test package does not need the unexported names in package app. +var retiredStoreKeys = []string{ + keys.FeegrantStoreKey, + keys.CapabilityStoreKey, + keys.IBCStoreKey, + keys.IBCTransferStoreKey, +} diff --git a/app/upgrades.go b/app/upgrades.go index 1923f8a69f..6617abceee 100644 --- a/app/upgrades.go +++ b/app/upgrades.go @@ -3,6 +3,7 @@ package app import ( "embed" "os" + "slices" "strings" sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" @@ -20,6 +21,12 @@ var f embed.FS // in a missing value in a log statement for which the fix is not released var upgradesList []string +// releaseUpgrades is the embedded list, kept apart from upgradesList because +// UPGRADE_VERSION_LIST replaces the latter in place and never restores it. A +// caller asking which upgrades this build ships has to be answered from a value +// no test can have already overwritten. +var releaseUpgrades []string + var LatestUpgrade string func init() { @@ -27,8 +34,15 @@ func init() { if err != nil { panic(err) } - upgradesList = parseUpgradesList(string(content)) - LatestUpgrade = upgradesList[len(upgradesList)-1] + releaseUpgrades = parseUpgradesList(string(content)) + upgradesList = slices.Clone(releaseUpgrades) + LatestUpgrade = releaseUpgrades[len(releaseUpgrades)-1] +} + +// ReleaseUpgrades returns the upgrade names this build embeds, in semver order, +// the last of which is LatestUpgrade. UPGRADE_VERSION_LIST does not affect it. +func ReleaseUpgrades() []string { + return slices.Clone(releaseUpgrades) } func parseUpgradesList(list string) []string { diff --git a/app/upgrades_test.go b/app/upgrades_test.go index 9dcb63ccc0..da2536ba55 100644 --- a/app/upgrades_test.go +++ b/app/upgrades_test.go @@ -46,6 +46,34 @@ func TestOverrideList(t *testing.T) { } } +// UPGRADE_VERSION_LIST replaces upgradesList in place and nothing puts it back, +// so a test that pins the handler set leaves the global holding its value for +// every test after it. ReleaseUpgrades has to stay the answer to what this build +// ships regardless, because that is what upgradetest derives the boundary being +// shipped from, and a boundary that moved with the override would let a test set +// select itself. +func TestReleaseUpgradesIgnoresTheOverride(t *testing.T) { + shipped := ReleaseUpgrades() + assert.Equal(t, shipped[len(shipped)-1], LatestUpgrade) + + defaultList := upgradesList + t.Cleanup(func() { upgradesList = defaultList }) + + t.Setenv("UPGRADE_VERSION_LIST", "v6.7") + overrideList() + + assert.Equal(t, []string{"v6.7"}, upgradesList) + assert.Equal(t, shipped, ReleaseUpgrades(), "the override moved which upgrades this build ships") +} + +// ReleaseUpgrades hands out a copy, so a caller that sorts or truncates what it +// gets back cannot change what the next caller sees. +func TestReleaseUpgradesCopies(t *testing.T) { + shipped := ReleaseUpgrades() + shipped[0] = "clobbered" + assert.NotEqual(t, "clobbered", ReleaseUpgrades()[0]) +} + func TestParseUpgradesList(t *testing.T) { tests := []struct { name string diff --git a/testutil/processblock/common.go b/testutil/processblock/common.go index 2bf1943211..2dd572fe81 100644 --- a/testutil/processblock/common.go +++ b/testutil/processblock/common.go @@ -70,6 +70,13 @@ func (a *App) Ctx() sdk.Context { // Assumes all validators voted with equal weight, and there are no byzantine validators. // Proposer is rotated among all validators round-robin. func (a *App) RunBlock(txs []signing.Tx) (resultCodes []uint32) { + return utils.Map(a.RunBlockDetailed(txs), func(r *types.ExecTxResult) uint32 { return r.Code }) +} + +// RunBlockDetailed processes and commits a block of transactions the same way +// RunBlock does, returning the full per-transaction results. Callers that need +// the error codespace, log, or gas of a rejected transaction use this. +func (a *App) RunBlockDetailed(txs []signing.Tx) []*types.ExecTxResult { defer func() { a.lastCtx = a.GetContextForDeliverTx([]byte{}) // Commit will set deliver tx ctx to nil so we need to cache it here for testing queries before the next block is FinalizeBlock'ed (which will set deliver tx ctx) _, err := a.Commit(context.Background()) @@ -105,7 +112,7 @@ func (a *App) RunBlock(txs []signing.Tx) (resultCodes []uint32) { if err != nil { panic(err) } - return utils.Map(res.TxResults, func(r *types.ExecTxResult) uint32 { return r.Code }) + return res.TxResults } func (a *App) GetVotes() []types.VoteInfo { diff --git a/testutil/processblock/tx.go b/testutil/processblock/tx.go index 1cdcfa5f80..cc5177548e 100644 --- a/testutil/processblock/tx.go +++ b/testutil/processblock/tx.go @@ -16,6 +16,17 @@ var Marshaler = codec.NewProtoCodec(InterfaceReg) var TxConfig = tx.NewTxConfig(Marshaler, tx.DefaultSignModes) func (a *App) Sign(account sdk.AccAddress, fee int64, msgs ...sdk.Msg) xauthsigning.Tx { + return a.sign(account, nil, fee, msgs...) +} + +// SignWithFeeGranter signs a transaction that nominates feeGranter to pay its +// fee. Passing a feeGranter other than account produces the wire shape a +// feegrant-using client sent before the module was removed. +func (a *App) SignWithFeeGranter(account, feeGranter sdk.AccAddress, fee int64, msgs ...sdk.Msg) xauthsigning.Tx { + return a.sign(account, feeGranter, fee, msgs...) +} + +func (a *App) sign(account, feeGranter sdk.AccAddress, fee int64, msgs ...sdk.Msg) xauthsigning.Tx { txBuilder := TxConfig.NewTxBuilder() if err := txBuilder.SetMsgs(msgs...); err != nil { panic(err) @@ -24,6 +35,9 @@ func (a *App) Sign(account sdk.AccAddress, fee int64, msgs ...sdk.Msg) xauthsign txBuilder.SetFeeAmount([]sdk.Coin{ sdk.NewCoin("usei", sdk.NewInt(fee)), }) + if feeGranter != nil { + txBuilder.SetFeeGranter(feeGranter) + } acc := a.AccountKeeper.GetAccount(a.Ctx(), account) seqNum := acc.GetSequence() diff --git a/upgradetest/AGENTS.md b/upgradetest/AGENTS.md new file mode 100644 index 0000000000..ce50152dba --- /dev/null +++ b/upgradetest/AGENTS.md @@ -0,0 +1,77 @@ +# Version-specific upgrade tests + +The tests themselves stay in `app`, following the existing naming and helper +structure: + +- `app/upgrade_test.go` and `app/upgrade_orphan_test.go` hold generic checks. +- `app/upgrade_v67_test.go` holds checks specific to the v6.6 -> v6.7 change. +- A version-specific file has the matching build tag, such as `upgrade_v67`. + +Do not move app upgrade tests into this package. This package only provides the +small amount of automation needed to select, validate and scaffold those files. + +## Adding a minor upgrade + +Run: + +```bash +make new-upgrade-test FROM=v6.6 TO=v6.7 +``` + +The command creates `app/upgrade_v67_test.go` plus separately compiled +`upgrade_v67_offline_source_test.go` and +`upgrade_v67_offline_target_test.go`. The main file has the matching +`upgrade_v67` constraint, the `newV67Chain` / `applyV67` shape, and +`TestV67CrossVersion` callbacks for the live two-binary path. +The source file also has a reopen TODO so the runner's third phase has a +test to select. Its TODOs fail in the layer that reaches them until real +assertions replace them. + +Appending `v6.7` to `app/tags` makes that pair the current boundary. +`make upgrade-test` derives the build tag from the embedded list and runs the +app package with the file enabled. The workflow must not name a version. + +`make upgrade-test-vet` compiles every version-specific app test. It compiles +historical offline phases in detached worktrees at their release refs and the +current target phase in the current checkout because those files may use APIs +available only on one side of the boundary. Ordinary untagged tests and +`golangci-lint run` do not type-check these build-tagged files. + +Run a real branch boundary with: + +```bash +make upgrade-test-offline \ + FROM_REF=release/v6.6 TO_REF=release/v6.7 + +make upgrade-test-cross-version \ + FROM_REF=release/v6.6 TO_REF=release/v6.7 +``` + +Both runners build from detached worktrees pinned to resolved commits. +`upgrade-test-offline` injects the tagged source test into the disposable old +worktree, writes a committed application database, injects the target test into +the new worktree to apply the handler, then runs a reopen phase that compiles +the source test again against the migrated database. `upgrade-test-cross-version` +starts validators on the source binary, runs the tagged test's `before` +callback, executes the governance halt and binary replacement, then runs its +`after` callback against the same node homes. + +## Scope + +Keep the checks in the same direct style as the existing v6.7 file. Use +`testutil/processblock` for transaction and state assertions, and call +`ApplyUpgrade` through a version-specific helper. + +`make upgrade-test` is the fast, in-process layer and uses the current checkout +on both sides. `make upgrade-test-offline` is the persisted, three-phase Go +layer: it reaches no consensus or node lifecycle code. Its source and target +files may use APIs available only on their respective branch because each is +compiled separately. The reopen phase compiles the source file against the +migrated database the target phase left behind. The target phase also accepts +`UPGRADE_TEST_SNAPSHOT_HOME` pointing at a node home: when set, +`TestV67OfflineUpgradeTarget/snapshot` opens that database, applies the +upgrade, and runs the retained-store and version-map assertions against it. +When unset the subtest skips; a path that is not a usable node home fails. +`make upgrade-test-cross-version` owns the full node lifecycle. Keep all +version-specific definitions in tagged app test files; `upgradetest` only +provides selection and coordination. diff --git a/upgradetest/README.md b/upgradetest/README.md new file mode 100644 index 0000000000..1e4ad9550c --- /dev/null +++ b/upgradetest/README.md @@ -0,0 +1,91 @@ +# Minor upgrade tests + +Version-specific upgrade tests stay beside the existing app upgrade tests. The +v6.7 test is `app/upgrade_v67_test.go`; its build tag is `upgrade_v67`. + +## Define the v6.7 upgrade + +From the repository root: + +```bash +make new-upgrade-test FROM=v6.6 TO=v6.7 +``` + +This creates `app/upgrade_v67_test.go` and its offline source/target files with: + +- the `upgrade_v67` build tag; +- the `newV67Chain` / `applyV67` structure; +- before/after callbacks for the real two-binary boundary; +- separately compiled source/target TODOs for the persisted Go boundary, plus a + reopen TODO in the source file so the runner's third phase has a test to + select; +- a failing TODO test so an empty scaffold cannot pass CI. + +Replace the TODO with ordinary Go tests in that file. Keep generic upgrade +tests, such as orphaned module-version checks, in the existing untagged +`app/upgrade_test.go` and `app/upgrade_orphan_test.go`. + +When `v6.7` is appended to `app/tags`, CI derives `upgrade_v67` automatically. +No version string is hard-coded in the Makefile or workflow. + +## Run it + +```bash +make upgrade-test +make upgrade-test-vet +``` + +`upgrade-test` reads the last two minor versions from `app/tags`, derives the +current build tag, and runs the app tests with that file enabled. + +`upgrade-test-vet` type-checks every `app/upgrade_v*_test.go`, including tests +for upgrades that already shipped. Historical offline phases compile in +detached worktrees at their release refs, while the current target phase +compiles in the current checkout, so each can keep using branch-specific APIs. + +## Run the real boundary + +```bash +make upgrade-test-offline \ + FROM_REF=release/v6.6 TO_REF=release/v6.7 + +make upgrade-test-cross-version \ + FROM_REF=release/v6.6 TO_REF=release/v6.7 +``` + +`upgrade-test-offline` compiles three Go test processes. The source process uses +the old branch's app code to write a committed database; the target process +uses the new branch's app code to reopen that database, apply the handler, and +verify the persisted result; the reopen process compiles the source file again +against the migrated database and records what the old binary does with it. The +runner copies the target branch's tagged test definitions into disposable +worktrees while each phase still links against its own branch. + +To point the target phase at a real node home instead of the synthetic +fixture, set `UPGRADE_TEST_SNAPSHOT_HOME` to that directory and run only the +snapshot subtest (the home is written in place): + +```bash +UPGRADE_TEST_SNAPSHOT_HOME=/path/to/node/home \ +UPGRADE_VERSION_LIST=v6.7 \ +go test -tags=upgrade_v67,offline_upgrade,upgrade_target \ + -run 'TestV67OfflineUpgradeTarget/snapshot' -count=1 ./app +``` + +The path must be a node home: a `config/genesis.json` plus the state +commitment store, at whichever of the two layouts the node was created with. +An unusable path fails; an unset variable skips that subtest and leaves the +fixture path unchanged. +The snapshot must still carry the retired modules in its version map +(pre-v6.7 state). + +`upgrade-test-cross-version` builds one `seid` from each ref. Four validators +create fixtures with the source binary, pass the governance upgrade height, +halt, and restart with the target binary against the same homes. The same +`upgrade_v67` tag selects its before- and after-upgrade assertions. + +The source callback must prove its fixture worked before recording it. The +target callback must test the transition, not merely repeat a behavior of the +target binary. `upgradetest.CrossVersion.Record` and `Replay` carry fixture +identities and observations between the two test processes; chain state itself +must stay in the validator database. diff --git a/upgradetest/boundary.go b/upgradetest/boundary.go new file mode 100644 index 0000000000..0596f883bc --- /dev/null +++ b/upgradetest/boundary.go @@ -0,0 +1,144 @@ +// Package upgradetest selects and scaffolds version-specific upgrade tests. +package upgradetest + +import ( + "fmt" + "regexp" + "strconv" + + "github.com/sei-protocol/sei-chain/app" +) + +// A Boundary is an upgrade as an operator performs it: the chain has applied +// From and is about to apply To. +type Boundary struct { + From string + To string +} + +// Current returns the latest minor-version boundary this build embeds. Patch +// upgrade names are ignored because they do not define a new minor test. +func Current() (Boundary, error) { + var names []string + for _, name := range app.ReleaseUpgrades() { + if minorVersion.MatchString(name) { + names = append(names, name) + } + } + if len(names) < 2 { + return Boundary{}, fmt.Errorf( + "upgradetest: this build embeds %d minor upgrade names, and a boundary needs two", len(names)) + } + return NewMinorBoundary(names[len(names)-2], names[len(names)-1]) +} + +func (b Boundary) String() string { + return b.From + " -> " + b.To +} + +// Tag returns the build tag compiling this boundary's app test file. +func (b Boundary) Tag() (string, error) { + return TagFor(b.To) +} + +// TestFile returns the name of this boundary's app test file. +func (b Boundary) TestFile() (string, error) { + return TestFileFor(b.To) +} + +// OfflineSourceTestFile returns the source-phase Go test for this boundary. +func (b Boundary) OfflineSourceTestFile() (string, error) { + return OfflineSourceTestFileFor(b.To) +} + +// OfflineTargetTestFile returns the target-phase Go test for this boundary. +func (b Boundary) OfflineTargetTestFile() (string, error) { + return OfflineTargetTestFileFor(b.To) +} + +// TagFor returns the build tag compiling the test set for an upgrade name: +// v6.7 gives upgrade_v67, matching app/upgrade_v67_test.go. +func TagFor(upgrade string) (string, error) { + suffix, err := versionSuffix(upgrade) + if err != nil { + return "", err + } + return "upgrade_" + suffix, nil +} + +// TestFileFor returns the app test file for an upgrade name. +func TestFileFor(upgrade string) (string, error) { + suffix, err := versionSuffix(upgrade) + if err != nil { + return "", err + } + return "upgrade_" + suffix + "_test.go", nil +} + +// OfflineSourceTestFileFor returns the persisted source-phase app test. +func OfflineSourceTestFileFor(upgrade string) (string, error) { + suffix, err := versionSuffix(upgrade) + if err != nil { + return "", err + } + return "upgrade_" + suffix + "_offline_source_test.go", nil +} + +// OfflineTargetTestFileFor returns the persisted target-phase app test. +func OfflineTargetTestFileFor(upgrade string) (string, error) { + suffix, err := versionSuffix(upgrade) + if err != nil { + return "", err + } + return "upgrade_" + suffix + "_offline_target_test.go", nil +} + +// NewMinorBoundary returns a boundary between two minor versions of the same +// major version. The target has to be newer than the source. +func NewMinorBoundary(from, to string) (Boundary, error) { + fromMajor, fromMinor, err := parseMinorVersion(from) + if err != nil { + return Boundary{}, fmt.Errorf("from version: %w", err) + } + toMajor, toMinor, err := parseMinorVersion(to) + if err != nil { + return Boundary{}, fmt.Errorf("to version: %w", err) + } + if fromMajor != toMajor { + return Boundary{}, fmt.Errorf( + "minor upgrade %s -> %s crosses major versions", from, to) + } + if toMinor <= fromMinor { + return Boundary{}, fmt.Errorf( + "minor upgrade target %s must be newer than source %s", to, from) + } + return Boundary{From: from, To: to}, nil +} + +var minorVersion = regexp.MustCompile(`^v(0|[1-9]\d*)\.(0|[1-9]\d*)$`) + +func parseMinorVersion(version string) (uint64, uint64, error) { + parts := minorVersion.FindStringSubmatch(version) + if parts == nil { + return 0, 0, fmt.Errorf( + "%q is not a minor version (want vMAJOR.MINOR)", version) + } + major, err := strconv.ParseUint(parts[1], 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("major version in %q: %w", version, err) + } + minor, err := strconv.ParseUint(parts[2], 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("minor version in %q: %w", version, err) + } + return major, minor, nil +} + +func versionSuffix(version string) (string, error) { + parts := minorVersion.FindStringSubmatch(version) + if parts == nil { + return "", fmt.Errorf( + "upgradetest: %q is not a minor version (want vMAJOR.MINOR)", version) + } + return "v" + parts[1] + parts[2], nil +} diff --git a/upgradetest/boundary_test.go b/upgradetest/boundary_test.go new file mode 100644 index 0000000000..279e7c04e4 --- /dev/null +++ b/upgradetest/boundary_test.go @@ -0,0 +1,113 @@ +package upgradetest_test + +import ( + "testing" + + "github.com/sei-protocol/sei-chain/app" + "github.com/sei-protocol/sei-chain/upgradetest" + "github.com/stretchr/testify/require" +) + +func TestTagAndFileSpelling(t *testing.T) { + for _, tc := range []struct { + upgrade string + tag string + file string + offlineSource string + offlineTarget string + }{ + { + upgrade: "v6.6", tag: "upgrade_v66", file: "upgrade_v66_test.go", + offlineSource: "upgrade_v66_offline_source_test.go", + offlineTarget: "upgrade_v66_offline_target_test.go", + }, + { + upgrade: "v6.7", tag: "upgrade_v67", file: "upgrade_v67_test.go", + offlineSource: "upgrade_v67_offline_source_test.go", + offlineTarget: "upgrade_v67_offline_target_test.go", + }, + { + upgrade: "v7.0", tag: "upgrade_v70", file: "upgrade_v70_test.go", + offlineSource: "upgrade_v70_offline_source_test.go", + offlineTarget: "upgrade_v70_offline_target_test.go", + }, + } { + t.Run(tc.upgrade, func(t *testing.T) { + tag, err := upgradetest.TagFor(tc.upgrade) + require.NoError(t, err) + require.Equal(t, tc.tag, tag) + + file, err := upgradetest.TestFileFor(tc.upgrade) + require.NoError(t, err) + require.Equal(t, tc.file, file) + + source, err := upgradetest.OfflineSourceTestFileFor(tc.upgrade) + require.NoError(t, err) + require.Equal(t, tc.offlineSource, source) + + target, err := upgradetest.OfflineTargetTestFileFor(tc.upgrade) + require.NoError(t, err) + require.Equal(t, tc.offlineTarget, target) + }) + } +} + +func TestTagForRequiresAMinorVersion(t *testing.T) { + for _, upgrade := range []string{ + "", "6.7", "v6", "v6.7.1", "1.0.4beta", "v4.0.0-evm-devnet", + } { + _, err := upgradetest.TagFor(upgrade) + require.Error(t, err, "TagFor(%q) should be refused", upgrade) + } +} + +func TestNewMinorBoundaryRejectsInvalidBumps(t *testing.T) { + for _, tc := range []struct { + name string + from string + to string + err string + }{ + {name: "source patch", from: "v6.6.3", to: "v6.7", err: "want vMAJOR.MINOR"}, + {name: "target patch", from: "v6.6", to: "v6.7.1", err: "want vMAJOR.MINOR"}, + {name: "major bump", from: "v6.7", to: "v7.0", err: "crosses major versions"}, + {name: "same version", from: "v6.7", to: "v6.7", err: "must be newer"}, + {name: "backwards", from: "v6.7", to: "v6.6", err: "must be newer"}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := upgradetest.NewMinorBoundary(tc.from, tc.to) + require.ErrorContains(t, err, tc.err) + }) + } +} + +func TestCurrentIsTheLastTwoEmbeddedMinorUpgrades(t *testing.T) { + var names []string + for _, name := range app.ReleaseUpgrades() { + if _, err := upgradetest.TagFor(name); err == nil { + names = append(names, name) + } + } + require.GreaterOrEqual(t, len(names), 2) + + boundary, err := upgradetest.Current() + require.NoError(t, err) + require.Equal(t, names[len(names)-2], boundary.From) + require.Equal(t, names[len(names)-1], boundary.To) +} + +// Minor names lose their dots in file and tag names. Keep that shortening +// one-to-one across the embedded release list. +func TestEveryMinorUpgradeHasItsOwnTag(t *testing.T) { + spelledBy := map[string]string{} + for _, upgrade := range app.ReleaseUpgrades() { + tag, err := upgradetest.TagFor(upgrade) + if err != nil { + continue + } + require.NotContains(t, spelledBy, tag, + "%s and %s both spell %s", spelledBy[tag], upgrade, tag) + spelledBy[tag] = upgrade + } + require.NotEmpty(t, spelledBy) +} diff --git a/upgradetest/cmd/boundary/main.go b/upgradetest/cmd/boundary/main.go new file mode 100644 index 0000000000..06fdef16a4 --- /dev/null +++ b/upgradetest/cmd/boundary/main.go @@ -0,0 +1,64 @@ +// Command boundary prints the upgrade boundary this build ships, so that a +// Makefile target or a CI step can select the boundary's test set without +// naming a version. Naming one there is how a workflow comes to run the test +// set for an upgrade that already shipped. +// +// boundary the boundary, as "v6.6 -> v6.7" +// boundary from the source version, as "v6.6" +// boundary to the target upgrade name, as "v6.7" +// boundary tag the build tag compiling its test, as "upgrade_v67" +// boundary file the app test file, as "upgrade_v67_test.go" +package main + +import ( + "fmt" + "io" + "os" + + "github.com/sei-protocol/sei-chain/upgradetest" +) + +func main() { + if err := run(os.Args[1:], os.Stdout); err != nil { + fmt.Fprintf(os.Stderr, "boundary: %v\n", err) + os.Exit(1) + } +} + +func run(args []string, out io.Writer) error { + what := "boundary" + switch len(args) { + case 0: + case 1: + what = args[0] + default: + return fmt.Errorf("want at most one of boundary, from, to, tag or file, got %d arguments", len(args)) + } + + boundary, err := upgradetest.Current() + if err != nil { + return err + } + + var answer string + switch what { + case "boundary": + answer = boundary.String() + case "from": + answer = boundary.From + case "to": + answer = boundary.To + case "tag": + answer, err = boundary.Tag() + case "file": + answer, err = boundary.TestFile() + default: + return fmt.Errorf("unknown request %q; want boundary, from, to, tag or file", what) + } + if err != nil { + return err + } + + _, err = fmt.Fprintln(out, answer) + return err +} diff --git a/upgradetest/cmd/boundary/main_test.go b/upgradetest/cmd/boundary/main_test.go new file mode 100644 index 0000000000..2ad74e0b7c --- /dev/null +++ b/upgradetest/cmd/boundary/main_test.go @@ -0,0 +1,50 @@ +package main + +import ( + "bytes" + "strings" + "testing" + + "github.com/sei-protocol/sei-chain/app" + "github.com/sei-protocol/sei-chain/upgradetest" + "github.com/stretchr/testify/require" +) + +func TestRunPrintsCurrentBoundaryFields(t *testing.T) { + boundary, err := upgradetest.Current() + require.NoError(t, err) + tag, err := boundary.Tag() + require.NoError(t, err) + file, err := boundary.TestFile() + require.NoError(t, err) + + for _, tc := range []struct { + request string + want string + }{ + {request: "from", want: boundary.From + "\n"}, + {request: "to", want: boundary.To + "\n"}, + {request: "tag", want: tag + "\n"}, + {request: "file", want: file + "\n"}, + } { + t.Run(tc.request, func(t *testing.T) { + var output bytes.Buffer + require.NoError(t, run([]string{tc.request}, &output)) + require.Equal(t, tc.want, output.String()) + }) + } +} + +func TestRunRejectsUnknownField(t *testing.T) { + err := run([]string{"directory"}, &bytes.Buffer{}) + require.ErrorContains(t, err, "want boundary, from, to, tag or file") +} + +func TestRunToPrintsAShippedUpgradeName(t *testing.T) { + var output bytes.Buffer + require.NoError(t, run([]string{"to"}, &output)) + printed := strings.TrimSuffix(output.String(), "\n") + require.Equal(t, printed, strings.TrimSpace(printed), + "boundary to padded the upgrade name with whitespace") + require.Contains(t, app.ReleaseUpgrades(), printed) +} diff --git a/upgradetest/cmd/new/main.go b/upgradetest/cmd/new/main.go new file mode 100644 index 0000000000..7cc83bcfb9 --- /dev/null +++ b/upgradetest/cmd/new/main.go @@ -0,0 +1,51 @@ +// Command new scaffolds the tagged app test file for a minor upgrade. +// +// go run ./upgradetest/cmd/new -from v6.6 -to v6.7 +package main + +import ( + "flag" + "fmt" + "io" + "os" + + "github.com/sei-protocol/sei-chain/upgradetest" +) + +func main() { + if err := run(os.Args[1:], os.Stdout); err != nil { + fmt.Fprintf(os.Stderr, "new upgrade test: %v\n", err) + os.Exit(1) + } +} + +func run(args []string, out io.Writer) error { + flags := flag.NewFlagSet("new", flag.ContinueOnError) + flags.SetOutput(io.Discard) + var from, to, root string + flags.StringVar(&from, "from", "", "source minor version, for example v6.6") + flags.StringVar(&to, "to", "", "target minor version, for example v6.7") + flags.StringVar(&root, "root", "app", "app directory") + if err := flags.Parse(args); err != nil { + return err + } + if flags.NArg() != 0 { + return fmt.Errorf("unexpected positional arguments: %v", flags.Args()) + } + if from == "" || to == "" { + return fmt.Errorf("both -from and -to are required") + } + + path, err := upgradetest.Scaffold(root, from, to) + if err != nil { + return err + } + if _, err := fmt.Fprintf(out, "created %s\n", path); err != nil { + return err + } + _, err = fmt.Fprintln( + out, + "next: replace the in-process and cross-version TODOs, then run make upgrade-test", + ) + return err +} diff --git a/upgradetest/cmd/new/main_test.go b/upgradetest/cmd/new/main_test.go new file mode 100644 index 0000000000..6243717ea0 --- /dev/null +++ b/upgradetest/cmd/new/main_test.go @@ -0,0 +1,23 @@ +package main + +import ( + "bytes" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRunScaffoldsTheRequestedUpgrade(t *testing.T) { + root := t.TempDir() + var out bytes.Buffer + + err := run([]string{"-from", "v6.6", "-to", "v6.7", "-root", root}, &out) + require.NoError(t, err) + require.Contains(t, out.String(), filepath.Join(root, "upgrade_v67_test.go")) +} + +func TestRunRequiresBothVersions(t *testing.T) { + err := run([]string{"-from", "v6.6"}, &bytes.Buffer{}) + require.ErrorContains(t, err, "both -from and -to are required") +} diff --git a/upgradetest/compile_offline.sh b/upgradetest/compile_offline.sh new file mode 100644 index 0000000000..13b154cc67 --- /dev/null +++ b/upgradetest/compile_offline.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +readonly REPO_ROOT="$(git rev-parse --show-toplevel)" +readonly RUN_ROOT="$(mktemp -d "${RUNNER_TEMP:-/tmp}/sei-offline-upgrade-compile.XXXXXX")" +readonly WORKTREE_ROOT="$RUN_ROOT/worktrees" +readonly TEST_SETS="$RUN_ROOT/test-sets.tsv" + +die() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +cleanup() { + local exit_code=$? + trap - EXIT + set +e + if [[ -d "$WORKTREE_ROOT" ]]; then + local worktree + for worktree in "$WORKTREE_ROOT"/*; do + [[ -d "$worktree" ]] || continue + git -C "$REPO_ROOT" worktree remove --force "$worktree" 2>/dev/null + done + fi + rm -rf "$RUN_ROOT" + exit "$exit_code" +} + +discover_test_sets() { + python3 - "$REPO_ROOT/app/tags" "$REPO_ROOT/app" >"$TEST_SETS" <<'PY' +import pathlib +import re +import sys + +tags_path = pathlib.Path(sys.argv[1]) +app_dir = pathlib.Path(sys.argv[2]) +versions = [] +for line in tags_path.read_text(encoding="utf-8").splitlines(): + match = re.fullmatch(r"v(\d+)\.(\d+)", line.strip()) + if match: + versions.append((int(match.group(1)), int(match.group(2)), line.strip())) +versions.sort() + +version_by_tag = {} +for index, (major, minor, target) in enumerate(versions): + tag = f"upgrade_v{major}{minor}" + if tag in version_by_tag: + raise SystemExit(f"{tag} is ambiguous between {version_by_tag[tag][2]} and {target}") + version_by_tag[tag] = (index, major, target) + +source_suffix = "_offline_source_test.go" +target_suffix = "_offline_target_test.go" +source_files = { + path.name.removesuffix(source_suffix): path.name + for path in app_dir.glob(f"upgrade_v*{source_suffix}") +} +target_files = { + path.name.removesuffix(target_suffix): path.name + for path in app_dir.glob(f"upgrade_v*{target_suffix}") +} +test_sets = [] +for tag in source_files.keys() | target_files.keys(): + if tag not in source_files or tag not in target_files: + raise SystemExit(f"{tag} must define both offline phase files") + if tag not in version_by_tag: + raise SystemExit(f"{tag} does not match a minor version in {tags_path}") + index, major, target = version_by_tag[tag] + if index == 0 or versions[index - 1][0] != major: + raise SystemExit(f"{tag} has no preceding minor release") + source = versions[index - 1][2] + test_sets.append((major, versions[index][1], source, target, tag)) + +for _, _, source, target, tag in sorted(test_sets): + print(source, target, tag, source_files[tag], target_files[tag], sep="\t") +PY +} + +resolve_release() { + local version="$1" + local branch="release/$version" + local ref + for ref in "refs/remotes/origin/$branch" "refs/heads/$branch"; do + if git -C "$REPO_ROOT" rev-parse --verify "$ref^{commit}" >/dev/null 2>&1; then + git -C "$REPO_ROOT" rev-parse --verify "$ref^{commit}" + return + fi + done + + git -C "$REPO_ROOT" fetch --no-tags origin "$branch" >&2 + git -C "$REPO_ROOT" rev-parse --verify FETCH_HEAD +} + +prepare_release_worktree() { + local version="$1" + local worktree="$2" + if [[ ! -d "$worktree" ]]; then + local commit + commit="$(resolve_release "$version")" || + die "unable to resolve release/$version" + git -C "$REPO_ROOT" worktree add --detach "$worktree" "$commit" >&2 + fi +} + +compile_phase() { + local checkout="$1" + local file="$2" + local tags="$3" + if [[ "$checkout" != "$REPO_ROOT" ]]; then + install -m 0644 \ + "$REPO_ROOT/app/upgrade_offline_harness_test.go" \ + "$checkout/app/upgrade_offline_harness_test.go" + install -m 0644 "$REPO_ROOT/app/$file" "$checkout/app/$file" + fi + + local included + if ! included="$( + cd "$checkout" + go list -tags "$tags" \ + -f '{{range .TestGoFiles}}{{println .}}{{end}}{{range .XTestGoFiles}}{{println .}}{{end}}' \ + ./app + )"; then + die "failed to list app tests while compiling $file" + fi + grep -Fxq "$file" <<<"$included" || + die "$file is not selected by build tags $tags" + + printf '=== Compiling app/%s against %s (-tags %s) ===\n' \ + "$file" "$(git -C "$checkout" rev-parse --short HEAD)" "$tags" + ( + cd "$checkout" + go test -tags "$tags" -run '^$' ./app + ) +} + +main() { + mkdir -p "$WORKTREE_ROOT" + trap cleanup EXIT + discover_test_sets + + local current_target + current_target="$(go run ./upgradetest/cmd/boundary to)" + local source + local target + local tag + local source_file + local target_file + local source_checkout + local target_checkout + while IFS=$'\t' read -r source target tag source_file target_file; do + [[ -n "$source" ]] || continue + source_checkout="$WORKTREE_ROOT/${source//./_}" + prepare_release_worktree "$source" "$source_checkout" + compile_phase \ + "$source_checkout" \ + "$source_file" \ + "$tag,offline_upgrade,upgrade_source" + + if [[ "$target" == "$current_target" ]]; then + target_checkout="$REPO_ROOT" + else + target_checkout="$WORKTREE_ROOT/${target//./_}" + prepare_release_worktree "$target" "$target_checkout" + fi + compile_phase \ + "$target_checkout" \ + "$target_file" \ + "$tag,offline_upgrade,upgrade_target" + done <"$TEST_SETS" +} + +main "$@" diff --git a/upgradetest/crossversion.go b/upgradetest/crossversion.go new file mode 100644 index 0000000000..19f7149ae0 --- /dev/null +++ b/upgradetest/crossversion.go @@ -0,0 +1,1026 @@ +package upgradetest + +import ( + "bytes" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "strings" + "testing" + "time" +) + +const ( + crossVersionPhaseEnv = "UPGRADE_TEST_PHASE" + crossVersionArtifactEnv = "UPGRADE_TEST_ARTIFACT" + crossVersionNodeEnv = "UPGRADE_TEST_NODE" + crossVersionUpgradeNameEnv = "UPGRADE_TEST_UPGRADE_NAME" + crossVersionTargetHeightEnv = "UPGRADE_TEST_TARGET_HEIGHT" + crossVersionReleaseBinaryEnv = "UPGRADE_TEST_RELEASE_BINARY" + validatorCount = 4 +) + +// A CrossVersion gives a tagged test access to the validator running each side +// of an upgrade and to the artifact shared between its two test processes. +type CrossVersion struct { + node string + artifactPath string + values map[string]json.RawMessage +} + +// CommandResult is the complete result of a command executed in a validator. +type CommandResult struct { + Stdout string + Stderr string + Err error +} + +// ExportedGenesis is the application state emitted by seid export. +type ExportedGenesis struct { + AppState map[string]json.RawMessage `json:"app_state"` +} + +type crossVersionArtifact struct { + Values map[string]json.RawMessage `json:"values"` +} + +// RunCrossVersion runs the callback selected by UPGRADE_TEST_PHASE. Without a +// phase it skips the test so ordinary tagged app tests remain self-contained. +func RunCrossVersion( + t *testing.T, + before func(*testing.T, *CrossVersion), + after func(*testing.T, *CrossVersion), +) { + t.Helper() + + phase := os.Getenv(crossVersionPhaseEnv) + if phase == "" { + t.Skip("cross-version phase is driven by make upgrade-test-cross-version") + } + if phase != "before" && phase != "after" { + t.Fatalf("%s must be before or after, got %q", crossVersionPhaseEnv, phase) + } + + artifactPath := os.Getenv(crossVersionArtifactEnv) + if artifactPath == "" { + t.Fatalf("%s is required", crossVersionArtifactEnv) + } + node := os.Getenv(crossVersionNodeEnv) + if node == "" { + t.Fatalf("%s is required", crossVersionNodeEnv) + } + if err := os.MkdirAll(filepath.Dir(artifactPath), 0o750); err != nil { + t.Fatalf("create cross-version artifact directory: %v", err) + } + + crossVersion := &CrossVersion{ + node: node, + artifactPath: artifactPath, + values: map[string]json.RawMessage{}, + } + if phase == "after" { + crossVersion.load(t) + after(t, crossVersion) + } else { + before(t, crossVersion) + } + crossVersion.save(t) +} + +// Record stores a JSON value for the other side of the upgrade. +func (c *CrossVersion) Record(t *testing.T, name string, value any) { + t.Helper() + if name == "" { + t.Fatal("cross-version artifact key must not be empty") + } + encoded, err := json.Marshal(value) + if err != nil { + t.Fatalf("encode cross-version artifact %q: %v", name, err) + } + c.values[name] = encoded +} + +// Replay decodes a value recorded on the other side of the upgrade. +func (c *CrossVersion) Replay(t *testing.T, name string, target any) { + t.Helper() + encoded, ok := c.values[name] + if !ok { + t.Fatalf("cross-version artifact has no %q value", name) + } + if err := json.Unmarshal(encoded, target); err != nil { + t.Fatalf("decode cross-version artifact %q: %v", name, err) + } +} + +// UpgradeName returns the upgrade name selected by the orchestrator. +func (c *CrossVersion) UpgradeName(t *testing.T) string { + t.Helper() + return requiredEnv(t, crossVersionUpgradeNameEnv) +} + +// TargetHeight returns the height selected by the governance proposal. +func (c *CrossVersion) TargetHeight(t *testing.T) int64 { + t.Helper() + value := requiredEnv(t, crossVersionTargetHeightEnv) + height, err := strconv.ParseInt(value, 10, 64) + if err != nil { + t.Fatalf("parse %s=%q: %v", crossVersionTargetHeightEnv, value, err) + } + return height +} + +// ReleaseBinary returns the path at which the validator keeps the old binary. +func (c *CrossVersion) ReleaseBinary(t *testing.T) string { + t.Helper() + return requiredEnv(t, crossVersionReleaseBinaryEnv) +} + +// Node returns the validator this process is bound to. +func (c *CrossVersion) Node() string { + return c.node +} + +// Nodes returns the validator names in the cluster. +func (c *CrossVersion) Nodes() []string { + nodes := make([]string, validatorCount) + for i := range nodes { + nodes[i] = fmt.Sprintf("sei-node-%d", i) + } + return nodes +} + +// Seid executes the running seid binary in the primary validator. +func (c *CrossVersion) Seid(input string, args ...string) CommandResult { + return c.SeidOn(c.node, input, args...) +} + +// SeidOn executes the running seid binary in a named validator. +func (c *CrossVersion) SeidOn(node, input string, args ...string) CommandResult { + return c.BinaryOn(node, input, "/root/go/bin/seid", args...) +} + +// Binary executes a seid-compatible binary in the primary validator. +func (c *CrossVersion) Binary(input, binary string, args ...string) CommandResult { + return c.BinaryOn(c.node, input, binary, args...) +} + +// BinaryOn executes a seid-compatible binary in a named validator. +func (c *CrossVersion) BinaryOn(node, input, binary string, args ...string) CommandResult { + command := append([]string{binary}, args...) + return runDocker(node, input, command...) +} + +// MustSeid executes seid and fails the test when the process exits unsuccessfully. +func (c *CrossVersion) MustSeid(t *testing.T, input string, args ...string) string { + t.Helper() + result := c.Seid(input, args...) + if result.Err != nil { + t.Fatalf("seid %s failed: %v\n%s", strings.Join(args, " "), result.Err, result.Combined()) + } + return result.Stdout +} + +// RequireBlockAgreement requires every validator to report the same application +// hash and block hash at each height. +func (c *CrossVersion) RequireBlockAgreement(t *testing.T, heights ...int64) { + t.Helper() + if len(heights) == 0 { + t.Fatal("no heights to compare") + } + maxHeight := heights[0] + for _, height := range heights { + if height <= 0 { + t.Fatalf("height must be positive, got %d", height) + } + if height > maxHeight { + maxHeight = height + } + } + nodes := c.Nodes() + for _, node := range nodes { + c.WaitForHeightOn(t, node, maxHeight, 3*time.Minute) + } + for _, height := range heights { + views := make([]blockView, 0, len(nodes)) + for _, node := range nodes { + views = append(views, c.blockAt(t, node, height)) + } + if err := validatorBlockAgreementError(height, views); err != nil { + t.Fatal(err) + } + } +} + +func (c *CrossVersion) blockAt(t *testing.T, node string, height int64) blockView { + t.Helper() + if node == "" { + t.Fatal("node must not be empty") + } + if height <= 0 { + t.Fatalf("height must be positive, got %d", height) + } + body, err := json.Marshal(map[string]any{ + "jsonrpc": "2.0", + "id": 1, + "method": "block", + "params": map[string]any{ + "height": strconv.FormatInt(height, 10), + }, + }) + if err != nil { + t.Fatalf("encode block query: %v", err) + } + result := c.BinaryOn(node, "", "curl", "-sf", "-H", "Content-Type: application/json", + "-d", string(body), "http://127.0.0.1:26657") + label := fmt.Sprintf("block-%s-%d", node, height) + c.WriteDiagnostic(t, label+".stdout", []byte(result.Stdout)) + c.WriteDiagnostic(t, label+".stderr", []byte(result.Stderr)) + if result.Err != nil { + t.Fatalf("block query at height %d on %s failed: %v\n%s", height, node, result.Err, result.Combined()) + } + parsed, err := parseBlockIdentity([]byte(result.Stdout)) + if err != nil { + t.Fatalf("decode block query at height %d on %s: %v\n%s", height, node, err, result.Stdout) + } + if parsed.height != height { + t.Fatalf("block query at height %d on %s returned height %d\n%s", + height, node, parsed.height, result.Stdout) + } + return blockView{node: node, appHash: parsed.appHash, blockHash: parsed.blockHash} +} + +// QueryStore reads key from storeName on the running node via ABCI query. +func (c *CrossVersion) QueryStore(t *testing.T, storeName string, key []byte) []byte { + t.Helper() + if storeName == "" { + t.Fatal("store name must not be empty") + } + body, err := json.Marshal(map[string]any{ + "jsonrpc": "2.0", + "id": 1, + "method": "abci_query", + "params": map[string]any{ + "path": "/store/" + storeName + "/key", + "data": hex.EncodeToString(key), + "prove": false, + }, + }) + if err != nil { + t.Fatalf("encode ABCI query: %v", err) + } + result := c.Binary("", "curl", "-sf", "-H", "Content-Type: application/json", + "-d", string(body), "http://127.0.0.1:26657") + label := fmt.Sprintf("abci-query-%s-%s", storeName, hex.EncodeToString(key)) + c.WriteDiagnostic(t, label+".stdout", []byte(result.Stdout)) + c.WriteDiagnostic(t, label+".stderr", []byte(result.Stderr)) + if result.Err != nil { + t.Fatalf("ABCI query /store/%s/key failed: %v\n%s", storeName, result.Err, result.Combined()) + } + value, code, log, err := parseABCIQueryResponse([]byte(result.Stdout)) + if err != nil { + t.Fatalf("decode ABCI query /store/%s/key: %v\n%s", storeName, err, result.Stdout) + } + if code != 0 { + t.Fatalf("ABCI query /store/%s/key returned code %d: %s", storeName, code, log) + } + return value +} + +// KeyAddress returns an address from a validator's test keyring. +func (c *CrossVersion) KeyAddress(t *testing.T, node, key string, extraArgs ...string) string { + t.Helper() + args := append([]string{"keys", "show", key, "-a"}, extraArgs...) + result := c.SeidOn(node, "12345678\n", args...) + if result.Err != nil { + t.Fatalf("%s key %s: %v\n%s", node, key, result.Err, result.Combined()) + } + address := strings.TrimSpace(result.Stdout) + if address == "" { + t.Fatalf("%s key %s returned an empty address", node, key) + } + return address +} + +// DeliveredTx is the committed DeliverTx result of a broadcast transaction. +type DeliveredTx struct { + Hash string + Height int64 + Code int64 + GasUsed int64 + RawLog string +} + +// RequireDeliverTxSuccess waits until a synchronously broadcast transaction is +// included in a committed block and requires that its DeliverTx result succeeded. +func (c *CrossVersion) RequireDeliverTxSuccess(t *testing.T, label string, result CommandResult) DeliveredTx { + t.Helper() + slug := strings.ReplaceAll(label, " ", "-") + c.WriteDiagnostic(t, slug+".broadcast.stdout", []byte(result.Stdout)) + c.WriteDiagnostic(t, slug+".broadcast.stderr", []byte(result.Stderr)) + c.requireCheckTxSuccess(t, label, result) + hash, err := parseBroadcastTxHash(result.Stdout) + if err != nil { + t.Fatalf("%s: %v\n%s", label, err, result.Stdout) + } + + deadline := time.Now().Add(3 * time.Minute) + var last CommandResult + for time.Now().Before(deadline) { + last = c.Seid("", "q", "tx", hash, "--output", "json") + if last.Err == nil { + c.WriteDiagnostic(t, slug+".included.json", []byte(last.Stdout)) + delivered, err := parseDeliveredTx(last.Stdout) + if err != nil { + t.Fatalf("%s: decode included tx: %v\n%s", label, err, last.Stdout) + } + if delivered.Hash != "" && !strings.EqualFold(delivered.Hash, hash) { + t.Fatalf("%s query returned hash %s, want %s", label, delivered.Hash, hash) + } + if delivered.Height <= 0 { + t.Fatalf("%s was not included in a block (hash %s)", label, hash) + } + if delivered.Code != 0 { + t.Fatalf("%s delivered with code %d: %s", label, delivered.Code, delivered.RawLog) + } + if delivered.GasUsed <= 0 { + t.Fatalf("%s consumed no gas", label) + } + delivered.Hash = hash + return delivered + } + time.Sleep(time.Second) + } + c.WriteDiagnostic(t, slug+".query.stdout", []byte(last.Stdout)) + c.WriteDiagnostic(t, slug+".query.stderr", []byte(last.Stderr)) + t.Fatalf("%s was not included within 3m (hash %s): %v\n%s", + label, hash, last.Err, last.Combined()) + return DeliveredTx{} +} + +// requireCheckTxSuccess requires that a synchronous broadcast's CheckTx result succeeded. +func (c *CrossVersion) requireCheckTxSuccess(t *testing.T, label string, result CommandResult) { + t.Helper() + if result.Err != nil { + t.Fatalf("%s failed: %v\n%s", label, result.Err, result.Combined()) + } + var response struct { + Code json.RawMessage `json:"code"` + RawLog string `json:"raw_log"` + } + if err := json.Unmarshal([]byte(result.Stdout), &response); err != nil { + t.Fatalf("%s did not return JSON: %v\n%s", label, err, result.Stdout) + } + if len(response.Code) == 0 { + return + } + code, err := parseJSONInt(response.Code) + if err != nil { + t.Fatalf("%s returned an invalid code: %v", label, err) + } + if code != 0 { + t.Fatalf("%s was rejected with code %d: %s", label, code, response.RawLog) + } +} + +// ModuleVersions returns the sorted names in the on-chain module version map. +func (c *CrossVersion) ModuleVersions(t *testing.T) []string { + t.Helper() + output := c.MustSeid(t, "", "q", "upgrade", "module_versions", "--output", "json") + var response struct { + ModuleVersions []struct { + Name string `json:"name"` + } `json:"module_versions"` + } + if err := json.Unmarshal([]byte(output), &response); err != nil { + t.Fatalf("decode module version map: %v\n%s", err, output) + } + names := make([]string, 0, len(response.ModuleVersions)) + for _, version := range response.ModuleVersions { + names = append(names, version.Name) + } + sort.Strings(names) + return names +} + +// Height returns the primary validator's latest committed height. +func (c *CrossVersion) Height(t *testing.T) int64 { + t.Helper() + return c.HeightOn(t, c.node) +} + +// HeightOn returns a named validator's latest committed height. +func (c *CrossVersion) HeightOn(t *testing.T, node string) int64 { + t.Helper() + height, output, err := c.tryHeightOn(node) + if err != nil { + t.Fatalf("query %s height: %v\n%s", node, err, output) + } + return height +} + +// WaitForHeight waits until the primary validator reaches a committed height. +func (c *CrossVersion) WaitForHeight(t *testing.T, target int64, timeout time.Duration) { + t.Helper() + c.WaitForHeightOn(t, c.node, target, timeout) +} + +// WaitForHeightOn waits until a named validator reaches a committed height. +func (c *CrossVersion) WaitForHeightOn(t *testing.T, node string, target int64, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + var lastOutput string + var lastErr error + for time.Now().Before(deadline) { + height, output, err := c.tryHeightOn(node) + lastOutput, lastErr = output, err + if err == nil && height >= target { + return + } + time.Sleep(time.Second) + } + t.Fatalf("%s did not reach height %d within %s: %v\n%s", + node, target, timeout, lastErr, lastOutput) +} + +// WaitForBlocks waits for the primary validator to commit more blocks. +func (c *CrossVersion) WaitForBlocks(t *testing.T, blocks int64) { + t.Helper() + c.WaitForHeight(t, c.Height(t)+blocks, 3*time.Minute) +} + +// StopNode stops seid in the primary validator. +func (c *CrossVersion) StopNode(t *testing.T) { + t.Helper() + c.StopNodeOn(t, c.node) +} + +// StopNodeOn stops seid in a named validator. +func (c *CrossVersion) StopNodeOn(t *testing.T, node string) { + t.Helper() + c.signalSeidOn(t, node, "TERM") +} + +// KillNodeOn sends SIGKILL to seid in a named validator and waits until it has exited. +func (c *CrossVersion) KillNodeOn(t *testing.T, node string) { + t.Helper() + c.signalSeidOn(t, node, "KILL") +} + +// StartObservation is the state of a validator after StartNodeObserving returns. +type StartObservation struct { + // Running reports whether seid was still alive at the end of the window. + Running bool + // Height is the last committed height a status query returned. It is 0 when + // no query succeeded. + Height int64 + // Log is the restart log written by this start. + Log string +} + +// StartNodeOn starts seid in a named validator using binary. +func (c *CrossVersion) StartNodeOn(t *testing.T, node, binary string) { + t.Helper() + c.launchSeidOn(t, node, binary, seidNodeLogPath(node), false) + c.waitForSeidState(t, node, "running", time.Minute) +} + +// StartNodeObserving starts seid on node with binary and watches it until timeout. +// A process that exits during the window is a returned outcome, not a test failure. +func (c *CrossVersion) StartNodeObserving(t *testing.T, node, binary string, timeout time.Duration) StartObservation { + t.Helper() + if timeout <= 0 { + t.Fatalf("observation timeout must be positive, got %s", timeout) + } + c.launchSeidOn(t, node, binary, seidObservedLogPath(node), true) + + deadline := time.Now().Add(timeout) + var observed StartObservation + var lastInspectErr error + sawRunning := false + for time.Now().Before(deadline) { + state, err := c.processStateOn(node) + if err != nil { + lastInspectErr = err + time.Sleep(time.Second) + continue + } + lastInspectErr = nil + if state == "running" { + sawRunning = true + observed.Running = true + if height, _, heightErr := c.tryHeightOn(node); heightErr == nil { + observed.Height = height + } + } else { + observed.Running = false + if sawRunning { + break + } + } + time.Sleep(time.Second) + } + if lastInspectErr != nil && !sawRunning { + t.Fatalf("inspect seid in %s: %v", node, lastInspectErr) + } + observed.Log = c.readSeidLog(node, seidObservedLogPath(node)) + return observed +} + +func (c *CrossVersion) launchSeidOn(t *testing.T, node, binary, logPath string, fresh bool) { + t.Helper() + if node == "" { + t.Fatal("node must not be empty") + } + if binary == "" { + t.Fatal("binary must not be empty") + } + state, err := c.processStateOn(node) + if err != nil { + t.Fatalf("inspect seid in %s: %v", node, err) + } + if state == "running" { + t.Fatalf("seid in %s is already running", node) + } + + redirect := ">>" + if fresh { + redirect = ">" + } + script := fmt.Sprintf( + "exec env -u UPGRADE_VERSION_LIST %s start --chain-id sei --inv-check-period 10 %s %s 2>&1", + strconv.Quote(binary), + redirect, + logPath, + ) + result := runDockerDetached(node, "sh", "-c", script) + if result.Err != nil { + t.Fatalf("start seid in %s: %v\n%s", node, result.Err, result.Combined()) + } +} + +// seidNodeLogPath is the log a validator writes from the moment the cluster +// starts it. The orchestrator reads this file to recognise an upgrade halt, so +// a restart has to keep appending to it rather than divert output elsewhere. +func seidNodeLogPath(node string) string { + return "build/generated/logs/seid-" + strings.TrimPrefix(node, "sei-node-") + ".log" +} + +// seidObservedLogPath is the log a single observed start writes. It is truncated +// per launch, so a halt found in it belongs to that launch and not to one the +// validator logged earlier. +func seidObservedLogPath(node string) string { + return "build/generated/logs/seid-" + strings.TrimPrefix(node, "sei-node-") + "-observed.log" +} + +func (c *CrossVersion) readSeidLog(node, path string) string { + result := c.BinaryOn(node, "", "cat", path) + if result.Err != nil { + return result.Combined() + } + return result.Stdout +} + +func (c *CrossVersion) signalSeidOn(t *testing.T, node, signal string) { + t.Helper() + if node == "" { + t.Fatal("node must not be empty") + } + if signal != "TERM" && signal != "KILL" { + t.Fatalf("unsupported seid signal %q", signal) + } + result := runDocker(node, "", "sh", "-c", seidSignalScript(signal)) + if result.Err != nil { + t.Fatalf("signal seid in %s: %v\n%s", node, result.Err, result.Combined()) + } + c.waitForSeidState(t, node, "stopped", time.Minute) +} + +func seidSignalScript(signal string) string { + return fmt.Sprintf(` +for comm in /proc/[0-9]*/comm; do + [ -r "$comm" ] || continue + read -r name <"$comm" || continue + if [ "$name" = seid ]; then + process_dir="${comm%%/comm}" + read -r stat_pid stat_comm process_state stat_rest <"$process_dir/stat" || continue + [ "$process_state" = Z ] && continue + pid="${comm#/proc/}" + pid="${pid%%/comm}" + kill -%s "$pid" + fi +done`, signal) +} + +func (c *CrossVersion) waitForSeidState(t *testing.T, node, want string, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + var lastState string + var lastErr error + for time.Now().Before(deadline) { + state, err := c.processStateOn(node) + lastState, lastErr = state, err + if err == nil && state == want { + return + } + time.Sleep(time.Second) + } + if lastErr != nil { + t.Fatalf("seid in %s did not become %s within %s: %v", node, want, timeout, lastErr) + } + t.Fatalf("seid in %s did not become %s within %s; last state %s", node, want, timeout, lastState) +} + +// Export runs a binary against the stopped validator and returns its app state. +func (c *CrossVersion) Export(t *testing.T, binary, label string) ExportedGenesis { + t.Helper() + var last CommandResult + for attempt := 1; attempt <= 10; attempt++ { + last = c.Binary("", binary, "export", "--home", "/root/.sei", "--chain-id", "sei") + c.WriteDiagnostic(t, fmt.Sprintf("%s-%d.stdout", label, attempt), []byte(last.Stdout)) + c.WriteDiagnostic(t, fmt.Sprintf("%s-%d.stderr", label, attempt), []byte(last.Stderr)) + if last.Err == nil { + genesis, err := extractGenesis([]byte(last.Stdout)) + if err == nil { + encoded, marshalErr := json.Marshal(genesis) + if marshalErr != nil { + t.Fatalf("encode %s export: %v", label, marshalErr) + } + c.WriteDiagnostic(t, label+".json", encoded) + return genesis + } + last.Err = err + } + time.Sleep(2 * time.Second) + } + t.Fatalf("%s export failed: %v\n%s", label, last.Err, last.Combined()) + return ExportedGenesis{} +} + +// WriteDiagnostic writes one file beside the cross-version artifact. +func (c *CrossVersion) WriteDiagnostic(t *testing.T, name string, content []byte) { + t.Helper() + if name == "" || filepath.Base(name) != name { + t.Fatalf("invalid cross-version diagnostic name %q", name) + } + path := filepath.Join(filepath.Dir(c.artifactPath), name) + if err := os.WriteFile(path, content, 0o600); err != nil { + t.Fatalf("write cross-version diagnostic %s: %v", path, err) + } +} + +// Combined returns stdout and stderr as one diagnostic string. +func (r CommandResult) Combined() string { + switch { + case r.Stdout == "": + return r.Stderr + case r.Stderr == "": + return r.Stdout + default: + return r.Stdout + "\n" + r.Stderr + } +} + +func (c *CrossVersion) tryHeightOn(node string) (int64, string, error) { + result := c.SeidOn(node, "", "status") + output := result.Combined() + if result.Err != nil { + return 0, output, result.Err + } + var response struct { + SyncInfo struct { + LatestBlockHeight json.RawMessage `json:"latest_block_height"` + } `json:"SyncInfo"` + } + if err := json.Unmarshal([]byte(result.Stdout), &response); err != nil { + return 0, output, err + } + height, err := parseJSONInt(response.SyncInfo.LatestBlockHeight) + return height, output, err +} + +func (c *CrossVersion) processStateOn(node string) (string, error) { + result := runDocker(node, "", "sh", "-c", ` +for comm in /proc/[0-9]*/comm; do + [ -r "$comm" ] || continue + read -r name <"$comm" || continue + if [ "$name" = seid ]; then + process_dir="${comm%/comm}" + read -r stat_pid stat_comm process_state stat_rest <"$process_dir/stat" || continue + [ "$process_state" = Z ] || { printf running; exit 0; } + fi +done +printf stopped`) + if result.Err != nil { + return "", fmt.Errorf("%w: %s", result.Err, result.Combined()) + } + state := strings.TrimSpace(result.Stdout) + if state != "running" && state != "stopped" { + return "", fmt.Errorf("invalid process state %q", state) + } + return state, nil +} + +func (c *CrossVersion) load(t *testing.T) { + t.Helper() + content, err := os.ReadFile(c.artifactPath) + if err != nil { + t.Fatalf("read cross-version artifact %s: %v", c.artifactPath, err) + } + var artifact crossVersionArtifact + if err := json.Unmarshal(content, &artifact); err != nil { + t.Fatalf("decode cross-version artifact %s: %v", c.artifactPath, err) + } + if artifact.Values == nil { + t.Fatalf("cross-version artifact %s has no values", c.artifactPath) + } + c.values = artifact.Values +} + +func (c *CrossVersion) save(t *testing.T) { + t.Helper() + content, err := json.MarshalIndent(crossVersionArtifact{Values: c.values}, "", " ") + if err != nil { + t.Fatalf("encode cross-version artifact: %v", err) + } + content = append(content, '\n') + if err := os.MkdirAll(filepath.Dir(c.artifactPath), 0o750); err != nil { + t.Fatalf("create cross-version artifact directory: %v", err) + } + temporary := c.artifactPath + ".tmp" + if err := os.WriteFile(temporary, content, 0o600); err != nil { + t.Fatalf("write cross-version artifact: %v", err) + } + if err := os.Rename(temporary, c.artifactPath); err != nil { + t.Fatalf("publish cross-version artifact: %v", err) + } +} + +func runDocker(node, input string, command ...string) CommandResult { + args := []string{"exec"} + if input != "" { + args = append(args, "-i") + } + args = append(args, node) + args = append(args, command...) + + cmd := exec.Command("docker", args...) //nolint:gosec // test-controlled commands run only in disposable validators + if input != "" { + cmd.Stdin = strings.NewReader(input) + } + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + return CommandResult{Stdout: stdout.String(), Stderr: stderr.String(), Err: err} +} + +func runDockerDetached(node string, command ...string) CommandResult { + args := append([]string{"exec", "-d", node}, command...) + cmd := exec.Command("docker", args...) //nolint:gosec // test-controlled commands run only in disposable validators + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + return CommandResult{Stdout: stdout.String(), Stderr: stderr.String(), Err: err} +} + +func extractGenesis(output []byte) (ExportedGenesis, error) { + offset := 0 + for _, line := range bytes.SplitAfter(output, []byte{'\n'}) { + candidate := bytes.TrimLeft(line, " \t\r\n") + if len(candidate) == 0 || candidate[0] != '{' { + offset += len(line) + continue + } + start := offset + len(line) - len(candidate) + var genesis ExportedGenesis + decoder := json.NewDecoder(bytes.NewReader(output[start:])) + if err := decoder.Decode(&genesis); err == nil && genesis.AppState != nil { + return genesis, nil + } + offset += len(line) + } + return ExportedGenesis{}, fmt.Errorf("no genesis document found") +} + +func parseJSONInt(encoded json.RawMessage) (int64, error) { + if len(encoded) == 0 { + return 0, fmt.Errorf("value is empty") + } + var number json.Number + decoder := json.NewDecoder(bytes.NewReader(encoded)) + decoder.UseNumber() + if err := decoder.Decode(&number); err == nil { + return strconv.ParseInt(number.String(), 10, 64) + } + var text string + if err := json.Unmarshal(encoded, &text); err != nil { + return 0, fmt.Errorf("decode integer %s: %w", encoded, err) + } + return strconv.ParseInt(text, 10, 64) +} + +func parseBroadcastTxHash(stdout string) (string, error) { + var resp struct { + TxHash string `json:"txhash"` + } + if err := json.Unmarshal([]byte(stdout), &resp); err != nil { + return "", fmt.Errorf("decode broadcast JSON: %w", err) + } + if resp.TxHash == "" { + return "", fmt.Errorf("broadcast JSON has no txhash") + } + return resp.TxHash, nil +} + +func parseDeliveredTx(stdout string) (DeliveredTx, error) { + var resp struct { + TxHash string `json:"txhash"` + Height json.RawMessage `json:"height"` + Code json.RawMessage `json:"code"` + GasUsed json.RawMessage `json:"gas_used"` + RawLog string `json:"raw_log"` + } + if err := json.Unmarshal([]byte(stdout), &resp); err != nil { + return DeliveredTx{}, fmt.Errorf("decode included tx JSON: %w", err) + } + var delivered DeliveredTx + delivered.Hash = resp.TxHash + delivered.RawLog = resp.RawLog + if len(resp.Height) > 0 { + height, err := parseJSONInt(resp.Height) + if err != nil { + return DeliveredTx{}, fmt.Errorf("decode height: %w", err) + } + delivered.Height = height + } + if len(resp.Code) > 0 { + code, err := parseJSONInt(resp.Code) + if err != nil { + return DeliveredTx{}, fmt.Errorf("decode code: %w", err) + } + delivered.Code = code + } + if len(resp.GasUsed) > 0 { + gasUsed, err := parseJSONInt(resp.GasUsed) + if err != nil { + return DeliveredTx{}, fmt.Errorf("decode gas_used: %w", err) + } + delivered.GasUsed = gasUsed + } + return delivered, nil +} + +func parseABCIQueryResponse(output []byte) ([]byte, uint32, string, error) { + var envelope struct { + Error json.RawMessage `json:"error"` + Result struct { + Response struct { + Code json.RawMessage `json:"code"` + Log string `json:"log"` + Value json.RawMessage `json:"value"` + Codespace string `json:"codespace"` + } `json:"response"` + } `json:"result"` + } + if err := json.Unmarshal(output, &envelope); err != nil { + return nil, 0, "", fmt.Errorf("decode JSON-RPC envelope: %w", err) + } + if len(envelope.Error) > 0 && string(envelope.Error) != "null" { + return nil, 0, "", fmt.Errorf("JSON-RPC error: %s", envelope.Error) + } + code := uint32(0) + if len(envelope.Result.Response.Code) > 0 && string(envelope.Result.Response.Code) != "null" { + parsed, err := parseJSONInt(envelope.Result.Response.Code) + if err != nil { + return nil, 0, "", fmt.Errorf("decode ABCI code: %w", err) + } + if parsed < 0 { + return nil, 0, "", fmt.Errorf("negative ABCI code %d", parsed) + } + code = uint32(parsed) + } + value, err := decodeABCIBytes(envelope.Result.Response.Value) + if err != nil { + return nil, 0, "", err + } + return value, code, envelope.Result.Response.Log, nil +} + +func decodeABCIBytes(raw json.RawMessage) ([]byte, error) { + if len(raw) == 0 || string(raw) == "null" { + return nil, nil + } + var text string + if err := json.Unmarshal(raw, &text); err != nil { + return nil, fmt.Errorf("decode ABCI bytes %s: %w", raw, err) + } + if text == "" { + return nil, nil + } + decoded, err := base64.StdEncoding.DecodeString(text) + if err != nil { + return nil, fmt.Errorf("decode ABCI value %q: %w", text, err) + } + return decoded, nil +} + +type blockView struct { + node string + appHash []byte + blockHash []byte +} + +type parsedBlock struct { + appHash []byte + blockHash []byte + height int64 +} + +func parseBlockIdentity(output []byte) (parsedBlock, error) { + var envelope struct { + Error json.RawMessage `json:"error"` + Result struct { + BlockID struct { + Hash string `json:"hash"` + } `json:"block_id"` + Block struct { + Header struct { + Height json.RawMessage `json:"height"` + AppHash string `json:"app_hash"` + } `json:"header"` + } `json:"block"` + } `json:"result"` + } + if err := json.Unmarshal(output, &envelope); err != nil { + return parsedBlock{}, fmt.Errorf("decode JSON-RPC envelope: %w", err) + } + if len(envelope.Error) > 0 && string(envelope.Error) != "null" { + return parsedBlock{}, fmt.Errorf("JSON-RPC error: %s", envelope.Error) + } + appHash, err := decodeRPCHex("app_hash", envelope.Result.Block.Header.AppHash) + if err != nil { + return parsedBlock{}, err + } + blockHash, err := decodeRPCHex("block_hash", envelope.Result.BlockID.Hash) + if err != nil { + return parsedBlock{}, err + } + height, err := parseJSONInt(envelope.Result.Block.Header.Height) + if err != nil { + return parsedBlock{}, fmt.Errorf("decode block height: %w", err) + } + return parsedBlock{appHash: appHash, blockHash: blockHash, height: height}, nil +} + +func decodeRPCHex(label, text string) ([]byte, error) { + text = strings.TrimPrefix(text, "0x") + if text == "" { + return nil, fmt.Errorf("block has no %s", label) + } + decoded, err := hex.DecodeString(text) + if err != nil { + return nil, fmt.Errorf("decode %s %q: %w", label, text, err) + } + return decoded, nil +} + +func validatorBlockAgreementError(height int64, views []blockView) error { + if len(views) < 2 { + return fmt.Errorf("need at least two validators to compare at height %d", height) + } + first := views[0] + for _, view := range views[1:] { + if bytes.Equal(view.appHash, first.appHash) && bytes.Equal(view.blockHash, first.blockHash) { + continue + } + return fmt.Errorf("%s", formatValidatorBlockDisagreement(height, views)) + } + return nil +} + +func formatValidatorBlockDisagreement(height int64, views []blockView) string { + var b strings.Builder + fmt.Fprintf(&b, "validators disagreed at height %d:", height) + for _, view := range views { + fmt.Fprintf(&b, "\n %s app_hash=%x block_hash=%x", view.node, view.appHash, view.blockHash) + } + return b.String() +} + +func requiredEnv(t *testing.T, name string) string { + t.Helper() + value := os.Getenv(name) + if value == "" { + t.Fatalf("%s is required", name) + } + return value +} diff --git a/upgradetest/crossversion_test.go b/upgradetest/crossversion_test.go new file mode 100644 index 0000000000..00ba3df95f --- /dev/null +++ b/upgradetest/crossversion_test.go @@ -0,0 +1,203 @@ +package upgradetest + +import ( + "encoding/json" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCrossVersionArtifactRoundTrip(t *testing.T) { + path := filepath.Join(t.TempDir(), "artifact.json") + before := &CrossVersion{ + node: "sei-node-0", + artifactPath: path, + values: map[string]json.RawMessage{}, + } + before.Record(t, "modules", []string{"feegrant", "ibc"}) + before.save(t) + + after := &CrossVersion{ + node: "sei-node-0", + artifactPath: path, + values: map[string]json.RawMessage{}, + } + after.load(t) + var modules []string + after.Replay(t, "modules", &modules) + require.Equal(t, []string{"feegrant", "ibc"}, modules) +} + +func TestExtractGenesisSkipsLogsAndReadsPrettyJSON(t *testing.T) { + genesis, err := extractGenesis([]byte(`starting export +{"level":"info","message":"loading"} +{ + "app_state": { + "bank": {"params": {}} + } +} +finished +`)) + require.NoError(t, err) + require.Contains(t, genesis.AppState, "bank") +} + +func TestExtractGenesisRejectsOutputWithoutAppState(t *testing.T) { + _, err := extractGenesis([]byte(`{"level":"info","message":"loading"}`)) + require.ErrorContains(t, err, "no genesis document") +} + +func TestParseJSONIntAcceptsNumbersAndStrings(t *testing.T) { + for _, encoded := range []string{"67", `"67"`} { + value, err := parseJSONInt(json.RawMessage(encoded)) + require.NoError(t, err) + require.Equal(t, int64(67), value) + } + + _, err := parseJSONInt(json.RawMessage(`"not-a-height"`)) + require.Error(t, err) +} + +func TestParseBroadcastTxHash(t *testing.T) { + hash, err := parseBroadcastTxHash(`{"txhash":"ABCDEF","code":0}`) + require.NoError(t, err) + require.Equal(t, "ABCDEF", hash) + + _, err = parseBroadcastTxHash(`{"code":0}`) + require.ErrorContains(t, err, "no txhash") +} + +func TestParseDeliveredTx(t *testing.T) { + got, err := parseDeliveredTx(`{ + "txhash": "ABCDEF", + "height": "42", + "code": 0, + "gas_used": "12345", + "raw_log": "[]" +}`) + require.NoError(t, err) + require.Equal(t, DeliveredTx{ + Hash: "ABCDEF", + Height: 42, + Code: 0, + GasUsed: 12345, + RawLog: "[]", + }, got) +} + +func TestParseABCIQueryResponse(t *testing.T) { + value, code, log, err := parseABCIQueryResponse([]byte(`{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "response": { + "code": 0, + "log": "", + "value": "aGVsbG8=" + } + } +}`)) + require.NoError(t, err) + require.Equal(t, uint32(0), code) + require.Equal(t, []byte("hello"), value) + require.Empty(t, log) + + value, code, log, err = parseABCIQueryResponse([]byte(`{ + "result": { + "response": { + "code": "0", + "value": null + } + } +}`)) + require.NoError(t, err) + require.Equal(t, uint32(0), code) + require.Empty(t, value) + require.Empty(t, log) +} + +func TestCrossVersionNodes(t *testing.T) { + require.Equal(t, []string{"sei-node-0", "sei-node-1", "sei-node-2", "sei-node-3"}, + (&CrossVersion{}).Nodes()) +} + +func TestParseBlockIdentity(t *testing.T) { + parsed, err := parseBlockIdentity([]byte(`{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "block_id": { + "hash": "0x01020304" + }, + "block": { + "header": { + "height": "42", + "app_hash": "0x0a0b0c0d" + } + } + } +}`)) + require.NoError(t, err) + require.Equal(t, int64(42), parsed.height) + require.Equal(t, []byte{0x0a, 0x0b, 0x0c, 0x0d}, parsed.appHash) + require.Equal(t, []byte{0x01, 0x02, 0x03, 0x04}, parsed.blockHash) + + _, err = parseBlockIdentity([]byte(`{"result":{"block_id":{"hash":"01"},"block":{"header":{"height":"1","app_hash":""}}}}`)) + require.ErrorContains(t, err, "no app_hash") + + _, err = parseBlockIdentity([]byte(`{"result":{"block_id":{"hash":""},"block":{"header":{"height":"1","app_hash":"0a"}}}}`)) + require.ErrorContains(t, err, "no block_hash") +} + +func TestValidatorBlockAgreementError(t *testing.T) { + agreeing := []blockView{ + {node: "sei-node-0", appHash: []byte{0xaa}, blockHash: []byte{0xbb}}, + {node: "sei-node-1", appHash: []byte{0xaa}, blockHash: []byte{0xbb}}, + {node: "sei-node-2", appHash: []byte{0xaa}, blockHash: []byte{0xbb}}, + {node: "sei-node-3", appHash: []byte{0xaa}, blockHash: []byte{0xbb}}, + } + require.NoError(t, validatorBlockAgreementError(67, agreeing)) + require.ErrorContains(t, validatorBlockAgreementError(67, agreeing[:1]), + "need at least two validators to compare at height 67") + + splitState := []blockView{ + {node: "sei-node-0", appHash: []byte{0xaa}, blockHash: []byte{0xbb}}, + {node: "sei-node-1", appHash: []byte{0xcc}, blockHash: []byte{0xbb}}, + {node: "sei-node-2", appHash: []byte{0xaa}, blockHash: []byte{0xbb}}, + {node: "sei-node-3", appHash: []byte{0xaa}, blockHash: []byte{0xbb}}, + } + err := validatorBlockAgreementError(67, splitState) + require.Error(t, err) + require.ErrorContains(t, err, "validators disagreed at height 67") + require.ErrorContains(t, err, "sei-node-0 app_hash=aa block_hash=bb") + require.ErrorContains(t, err, "sei-node-1 app_hash=cc block_hash=bb") + + splitChain := []blockView{ + {node: "sei-node-0", appHash: []byte{0xaa}, blockHash: []byte{0xbb}}, + {node: "sei-node-1", appHash: []byte{0xaa}, blockHash: []byte{0xdd}}, + {node: "sei-node-2", appHash: []byte{0xaa}, blockHash: []byte{0xbb}}, + {node: "sei-node-3", appHash: []byte{0xaa}, blockHash: []byte{0xbb}}, + } + err = validatorBlockAgreementError(68, splitChain) + require.Error(t, err) + require.ErrorContains(t, err, "validators disagreed at height 68") + require.ErrorContains(t, err, "sei-node-0 app_hash=aa block_hash=bb") + require.ErrorContains(t, err, "sei-node-1 app_hash=aa block_hash=dd") +} + +func TestSeidSignalScript(t *testing.T) { + script := seidSignalScript("KILL") + require.Contains(t, script, `kill -KILL "$pid"`) + require.Contains(t, script, `${comm%/comm}`) +} + +// A restarted validator must keep writing to the log the orchestrator greps for +// an upgrade halt, while an observed start needs a log of its own so a halt +// found there cannot be one an earlier launch logged. +func TestSeidLogPaths(t *testing.T) { + require.Equal(t, "build/generated/logs/seid-2.log", seidNodeLogPath("sei-node-2")) + require.Equal(t, "build/generated/logs/seid-0.log", seidNodeLogPath("sei-node-0")) + require.NotEqual(t, seidNodeLogPath("sei-node-3"), seidObservedLogPath("sei-node-3")) + require.Equal(t, "build/generated/logs/seid-3-observed.log", seidObservedLogPath("sei-node-3")) +} diff --git a/upgradetest/scaffold.go b/upgradetest/scaffold.go new file mode 100644 index 0000000000..568a2c9e47 --- /dev/null +++ b/upgradetest/scaffold.go @@ -0,0 +1,146 @@ +package upgradetest + +import ( + "fmt" + "go/format" + "os" + "path/filepath" + "strings" +) + +// Scaffold creates the in-process, offline, and live definitions for one +// version-specific app upgrade test. It returns the main tagged test path. +func Scaffold(root, from, to string) (string, error) { + boundary, err := NewMinorBoundary(from, to) + if err != nil { + return "", err + } + tag, err := boundary.Tag() + if err != nil { + return "", err + } + fileName, err := boundary.TestFile() + if err != nil { + return "", err + } + offlineSourceName, err := boundary.OfflineSourceTestFile() + if err != nil { + return "", err + } + offlineTargetName, err := boundary.OfflineTargetTestFile() + if err != nil { + return "", err + } + suffix, err := versionSuffix(to) + if err != nil { + return "", err + } + exportedSuffix := strings.ToUpper(suffix[:1]) + suffix[1:] + + testPath := filepath.Join(root, fileName) + paths := []string{ + testPath, + filepath.Join(root, offlineSourceName), + filepath.Join(root, offlineTargetName), + } + for _, path := range paths { + if _, err := os.Stat(path); err == nil { + return "", fmt.Errorf("upgrade test %s already exists", path) + } else if !os.IsNotExist(err) { + return "", fmt.Errorf("inspect upgrade test %s: %w", path, err) + } + } + + source := []byte(fmt.Sprintf(`//go:build %[1]s + +package app_test + +import ( + "testing" + + upgradetypes "github.com/sei-protocol/sei-chain/sei-cosmos/x/upgrade/types" + "github.com/sei-protocol/sei-chain/testutil/processblock" + "github.com/sei-protocol/sei-chain/upgradetest" +) + +const %[2]sUpgradeName = %[3]q + +func new%[4]sChain(t *testing.T) *processblock.App { + t.Helper() + t.Setenv("UPGRADE_VERSION_LIST", %[2]sUpgradeName) + app := processblock.NewTestApp(t) + processblock.CommonPreset(app) + app.RegisterUpgradeHandlers() + return app +} + +func apply%[4]s(t *testing.T, app *processblock.App) { + t.Helper() + app.UpgradeKeeper.ApplyUpgrade(app.Ctx(), upgradetypes.Plan{ + Name: %[2]sUpgradeName, + Height: app.Ctx().BlockHeight(), + }) +} + +func Test%[4]sUpgrade(t *testing.T) { + app := new%[4]sChain(t) + apply%[4]s(t, app) + t.Fatal("TODO: define the %[5]s upgrade assertions") +} + +func Test%[4]sCrossVersion(t *testing.T) { + upgradetest.RunCrossVersion(t, + func(t *testing.T, chain *upgradetest.CrossVersion) { + t.Fatal("TODO: create %[3]s state with the source binary") + }, + func(t *testing.T, chain *upgradetest.CrossVersion) { + t.Fatal("TODO: verify %[3]s state with the target binary") + }, + ) +} +`, tag, suffix, to, exportedSuffix, boundary)) + offlineSource := []byte(fmt.Sprintf(`//go:build %[1]s && offline_upgrade && upgrade_source + +package app + +import "testing" + +func Test%[2]sOfflineUpgradeSource(t *testing.T) { + _ = requireOfflineUpgradePhase(t, "source") + t.Fatal("TODO: create committed %[3]s source state") +} + +func Test%[2]sOfflineUpgradeReopen(t *testing.T) { + _ = requireOfflineUpgradePhase(t, "reopen") + t.Fatal("TODO: reopen the migrated database with the source binary") +} +`, tag, exportedSuffix, to)) + offlineTarget := []byte(fmt.Sprintf(`//go:build %[1]s && offline_upgrade && upgrade_target + +package app + +import "testing" + +func Test%[2]sOfflineUpgradeTarget(t *testing.T) { + _ = requireOfflineUpgradePhase(t, "target") + t.Fatal("TODO: reopen and verify committed %[3]s target state") +} +`, tag, exportedSuffix, to)) + + sources := [][]byte{source, offlineSource, offlineTarget} + for i := range sources { + sources[i], err = format.Source(sources[i]) + if err != nil { + return "", fmt.Errorf("format generated upgrade test %s: %w", paths[i], err) + } + } + for i, path := range paths { + if err := os.WriteFile(path, sources[i], 0o644); err != nil { //nolint:gosec // generated Go source uses repository file permissions + for _, written := range paths[:i+1] { + _ = os.Remove(written) + } + return "", fmt.Errorf("write upgrade test %s: %w", path, err) + } + } + return testPath, nil +} diff --git a/upgradetest/scaffold_test.go b/upgradetest/scaffold_test.go new file mode 100644 index 0000000000..dabd345452 --- /dev/null +++ b/upgradetest/scaffold_test.go @@ -0,0 +1,87 @@ +package upgradetest_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/sei-protocol/sei-chain/upgradetest" + "github.com/stretchr/testify/require" +) + +func TestScaffoldCreatesATaggedAppUpgradeTest(t *testing.T) { + root := t.TempDir() + + path, err := upgradetest.Scaffold(root, "v6.6", "v6.7") + require.NoError(t, err) + require.Equal(t, filepath.Join(root, "upgrade_v67_test.go"), path) + + source, err := os.ReadFile(path) + require.NoError(t, err) + require.Contains(t, string(source), "//go:build upgrade_v67") + require.Contains(t, string(source), "package app_test") + require.Contains(t, string(source), `const v67UpgradeName = "v6.7"`) + require.Contains(t, string(source), "func newV67Chain") + require.Contains(t, string(source), "func applyV67") + require.Contains(t, string(source), "TODO: define the v6.6 -> v6.7 upgrade assertions") + require.Contains(t, string(source), "func TestV67CrossVersion") + require.Contains(t, string(source), "upgradetest.RunCrossVersion") + require.Contains(t, string(source), "TODO: create v6.7 state with the source binary") + require.Contains(t, string(source), "TODO: verify v6.7 state with the target binary") + + offlineSource, err := os.ReadFile(filepath.Join(root, "upgrade_v67_offline_source_test.go")) + require.NoError(t, err) + require.Contains(t, string(offlineSource), + "//go:build upgrade_v67 && offline_upgrade && upgrade_source") + require.Contains(t, string(offlineSource), "func TestV67OfflineUpgradeSource") + require.Contains(t, string(offlineSource), "TODO: create committed v6.7 source state") + require.Contains(t, string(offlineSource), "func TestV67OfflineUpgradeReopen") + require.Contains(t, string(offlineSource), "TODO: reopen the migrated database with the source binary") + + offlineTarget, err := os.ReadFile(filepath.Join(root, "upgrade_v67_offline_target_test.go")) + require.NoError(t, err) + require.Contains(t, string(offlineTarget), + "//go:build upgrade_v67 && offline_upgrade && upgrade_target") + require.Contains(t, string(offlineTarget), "func TestV67OfflineUpgradeTarget") + require.Contains(t, string(offlineTarget), "TODO: reopen and verify committed v6.7 target state") + + file, err := upgradetest.ReadTestFile(root, "upgrade_v67_test.go") + require.NoError(t, err) + require.Equal(t, "upgrade_v67", file.Tag) +} + +func TestScaffoldRefusesToOverwriteASet(t *testing.T) { + root := t.TempDir() + path, err := upgradetest.Scaffold(root, "v6.6", "v6.7") + require.NoError(t, err) + before, err := os.ReadFile(path) + require.NoError(t, err) + + _, err = upgradetest.Scaffold(root, "v6.6", "v6.7") + require.ErrorContains(t, err, "already exists") + after, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, before, after) +} + +func TestScaffoldRefusesToOverwriteAnOfflinePhase(t *testing.T) { + root := t.TempDir() + offline := filepath.Join(root, "upgrade_v67_offline_source_test.go") + require.NoError(t, os.WriteFile(offline, []byte("existing"), 0o600)) + + _, err := upgradetest.Scaffold(root, "v6.6", "v6.7") + require.ErrorContains(t, err, "already exists") + _, statErr := os.Stat(filepath.Join(root, "upgrade_v67_test.go")) + require.ErrorIs(t, statErr, os.ErrNotExist) +} + +func TestScaffoldValidatesBeforeWriting(t *testing.T) { + root := t.TempDir() + + _, err := upgradetest.Scaffold(root, "v6.6.1", "v6.7") + require.ErrorContains(t, err, "want vMAJOR.MINOR") + + entries, err := os.ReadDir(root) + require.NoError(t, err) + require.Empty(t, entries) +} diff --git a/upgradetest/set.go b/upgradetest/set.go new file mode 100644 index 0000000000..0823a3440a --- /dev/null +++ b/upgradetest/set.go @@ -0,0 +1,83 @@ +package upgradetest + +import ( + "bufio" + "fmt" + "go/build/constraint" + "os" + "path/filepath" + "regexp" + "strings" +) + +// A TestFile is one version-specific app upgrade test and its build tag. +type TestFile struct { + Name string + Tag string +} + +var testFileName = regexp.MustCompile(`^upgrade_v\d+_test\.go$`) + +// TestFiles returns every version-specific upgrade test in root. +func TestFiles(root string) ([]TestFile, error) { + entries, err := os.ReadDir(root) + if err != nil { + return nil, err + } + var files []TestFile + for _, entry := range entries { + if entry.IsDir() || !testFileName.MatchString(entry.Name()) { + continue + } + file, err := ReadTestFile(root, entry.Name()) + if err != nil { + return nil, err + } + files = append(files, file) + } + return files, nil +} + +// ReadTestFile returns the version-specific upgrade test named under root. +func ReadTestFile(root, name string) (TestFile, error) { + tag, err := declaredTag(filepath.Join(root, name)) + if err != nil { + return TestFile{}, err + } + return TestFile{Name: name, Tag: tag}, nil +} + +// ExpectedTag returns the build tag implied by the file's name. +func (f TestFile) ExpectedTag() string { + return strings.TrimSuffix(f.Name, "_test.go") +} + +// declaredTag returns the single build tag a Go file is constrained by, and the +// empty string when the file declares no constraint or a compound one. +func declaredTag(path string) (string, error) { + file, err := os.Open(path) //nolint:gosec // path is a Go file listed from the app directory + if err != nil { + return "", err + } + defer func() { _ = file.Close() }() + + lines := bufio.NewScanner(file) + for lines.Scan() { + line := strings.TrimSpace(lines.Text()) + if strings.HasPrefix(line, "package ") { + return "", nil + } + if !constraint.IsGoBuild(line) { + continue + } + expr, err := constraint.Parse(line) + if err != nil { + return "", fmt.Errorf("%s: %w", path, err) + } + if tag, ok := expr.(*constraint.TagExpr); ok { + return tag.Tag, nil + } + return "", nil + } + return "", lines.Err() +} diff --git a/upgradetest/set_test.go b/upgradetest/set_test.go new file mode 100644 index 0000000000..4179ed9623 --- /dev/null +++ b/upgradetest/set_test.go @@ -0,0 +1,69 @@ +package upgradetest_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/sei-protocol/sei-chain/upgradetest" + "github.com/stretchr/testify/require" +) + +const appDir = "../app" + +// Appending a minor version to app/tags without defining its version-specific +// test fails here. The ordinary Go test run cannot see a tagged test file. +func TestCurrentBoundaryHasATestFile(t *testing.T) { + boundary, err := upgradetest.Current() + require.NoError(t, err) + fileName, err := boundary.TestFile() + require.NoError(t, err) + tag, err := boundary.Tag() + require.NoError(t, err) + + file, err := upgradetest.ReadTestFile(appDir, fileName) + require.NoError(t, err, + "the %s boundary has no app/%s; create it with make new-upgrade-test FROM=%s TO=%s", + boundary, fileName, boundary.From, boundary.To) + require.Equal(t, tag, file.Tag, + "app/%s has build tag %q; want %q", fileName, file.Tag, tag) +} + +func TestCurrentBoundaryHasOfflinePhaseFiles(t *testing.T) { + boundary, err := upgradetest.Current() + require.NoError(t, err) + source, err := boundary.OfflineSourceTestFile() + require.NoError(t, err) + target, err := boundary.OfflineTargetTestFile() + require.NoError(t, err) + + for _, file := range []string{source, target} { + _, err := os.Stat(filepath.Join(appDir, file)) + require.NoError(t, err, + "the %s boundary has no app/%s; create it with make new-upgrade-test FROM=%s TO=%s", + boundary, file, boundary.From, boundary.To) + } +} + +// Each version-specific file carries the tag implied by its existing app file +// name. The generic app/upgrade_test.go is deliberately outside this check. +func TestVersionSpecificUpgradeTestsCarryTheirOwnTag(t *testing.T) { + files, err := upgradetest.TestFiles(appDir) + require.NoError(t, err) + require.NotEmpty(t, files, "no version-specific app upgrade tests found") + + for _, file := range files { + t.Run(file.Name, func(t *testing.T) { + require.Equal(t, file.ExpectedTag(), file.Tag, + "app/%s is constrained by %s; want %s", + file.Name, describeTag(file.Tag), file.ExpectedTag()) + }) + } +} + +func describeTag(tag string) string { + if tag == "" { + return "no single build tag" + } + return tag +}