From babcb4b7e2b610b039e2424c4599a334949c0baa Mon Sep 17 00:00:00 2001 From: Angelo De Caro Date: Thu, 24 Sep 2026 12:56:36 +0000 Subject: [PATCH] feat(selector): add sherdlock lock-contention diagnostics Phases 1-2 of #2395: make sherdlock's lock contention observable, and give the behaviour a reproducible baseline before changing the selector itself. - Metrics: a lock_conflicts_total counter and a distinct_tokens_attempted histogram. The histogram is observed exactly once per Select() call, with the attempted-token set accumulated in the caller and unioned across every backoff round, so a StubbornSelector retry does not emit one sample per round; a call that never attempted a lock records nothing rather than a zero sample below the first bucket. - A reproducible hot-token contention baseline test, plus shared selector test cases. - driver.TokenLockStore.ListLocks and driver.LockRecord: read every held lock joined with the status of its consuming transaction. The SQL implementation checks rows.Err() after draining the iterator, because the shared rowIterator does not, and a mid-scan failure would otherwise return a truncated list with a nil error. - cond.Exists, for correlated-subquery conditions. - tokendiag: a new CGO-free CLI (cmd/tokendiag) that reports the currently held token locks with their age and consumer status, flags locks whose consumer has already reached a terminal status, and prints a script-friendly summary. Configuration is a YAML file; every scalar key can be overridden by a CORE_* environment variable, each bound up front so that viper's Unmarshal, which walks AllKeys(), actually visits keys supplied only through the environment. - Documentation: docs/development/tokendiag.md, cmd/tokendiag/README.md and the metrics reference. Signed-off-by: Angelo De Caro Co-authored-by: AkramBitar Signed-off-by: AkramBitar --- Makefile | 7 +- cmd/tokendiag/README.md | 165 +++++++++ cmd/tokendiag/cobra/config/cmd.go | 98 ++++++ cmd/tokendiag/cobra/locks/cmd.go | 74 ++++ cmd/tokendiag/cobra/locks/config.go | 93 +++++ cmd/tokendiag/cobra/locks/config_test.go | 110 ++++++ cmd/tokendiag/cobra/locks/runner.go | 108 ++++++ cmd/tokendiag/cobra/locks/stores.go | 94 ++++++ cmd/tokendiag/cobra/locks/stores_test.go | 126 +++++++ cmd/tokendiag/go.mod | 115 +++++++ cmd/tokendiag/go.sum | 319 ++++++++++++++++++ cmd/tokendiag/main.go | 52 +++ docs/README.md | 1 + docs/development/development.md | 1 + docs/development/metrics.md | 10 +- docs/development/tokendiag.md | 20 ++ .../metricsdoc/testdata/metrics.golden | 2 + .../selector/sherdlock/contention_test.go | 161 +++++++++ .../selector/sherdlock/inmemory/locker.go | 10 + .../selector/sherdlock/manager_test.go | 18 +- token/services/selector/sherdlock/metrics.go | 19 ++ token/services/selector/sherdlock/selector.go | 66 +++- .../selector/sherdlock/selector_test.go | 69 ++++ .../services/selector/testutils/test_cases.go | 30 ++ token/services/storage/db/dbtest/tokenlock.go | 112 +++++- token/services/storage/db/driver/token.go | 20 ++ .../storage/db/sql/common/tokenlock.go | 85 +++++ .../db/sql/query/cond/condition_test.go | 11 + .../storage/db/sql/query/cond/exists.go | 12 + 29 files changed, 1982 insertions(+), 26 deletions(-) create mode 100644 cmd/tokendiag/README.md create mode 100644 cmd/tokendiag/cobra/config/cmd.go create mode 100644 cmd/tokendiag/cobra/locks/cmd.go create mode 100644 cmd/tokendiag/cobra/locks/config.go create mode 100644 cmd/tokendiag/cobra/locks/config_test.go create mode 100644 cmd/tokendiag/cobra/locks/runner.go create mode 100644 cmd/tokendiag/cobra/locks/stores.go create mode 100644 cmd/tokendiag/cobra/locks/stores_test.go create mode 100644 cmd/tokendiag/go.mod create mode 100644 cmd/tokendiag/go.sum create mode 100644 cmd/tokendiag/main.go create mode 100644 docs/development/tokendiag.md create mode 100644 token/services/selector/sherdlock/contention_test.go diff --git a/Makefile b/Makefile index 452eb03990..1b60a2b526 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ TOP = . include $(TOP)/checks.mk # Define all Go module directories -GO_MODULES := . integration token/services/storage/db/kvs/hashicorp cmd/artifactgen cmd/tokengen cmd/token_validation_service cmd/profiler cmd/skicleanup cmd/node x/token/services/network/evm +GO_MODULES := . integration token/services/storage/db/kvs/hashicorp cmd/artifactgen cmd/tokengen cmd/token_validation_service cmd/profiler cmd/skicleanup cmd/tokendiag cmd/node x/token/services/network/evm TIDY_GO_MODULES := $(GO_MODULES) tools # include fabricx target @@ -219,6 +219,11 @@ artifactgen: skicleanup: @cd ./cmd/skicleanup/; CGO_ENABLED=0 go install github.com/LFDT-Panurus/panurus/cmd/skicleanup +.PHONY: tokendiag +# install tokendiag tool (must build without cgo; see #1445) +tokendiag: + @cd ./cmd/tokendiag/; CGO_ENABLED=0 go install github.com/LFDT-Panurus/panurus/cmd/tokendiag + .PHONY: traceinspector # install traceinspector tool traceinspector: diff --git a/cmd/tokendiag/README.md b/cmd/tokendiag/README.md new file mode 100644 index 0000000000..99702638d7 --- /dev/null +++ b/cmd/tokendiag/README.md @@ -0,0 +1,165 @@ +# tokendiag + +`tokendiag` is a diagnostic command-line tool for inspecting token-selector state in a +Panurus token database. It was added for +[#2395](https://github.com/LFDT-Panurus/panurus/issues/2395), a stress load test that +showed severe lock contention on a small number of hot tokens. + +## Build + +```bash +make tokendiag +``` + +The binary is installed to `$GOPATH/bin/tokendiag`. + +## Commands + +### `config example` + +Prints a fully-annotated YAML configuration file to stdout. Use this to bootstrap a new +configuration: + +```bash +tokendiag config example > config.yaml +``` + +No flags required. + +### `locks` + +Reads every currently held row in the `token_locks` table, joined with the status of +its consuming transaction, and reports: + +- every held lock, with its age and the consumer's status, oldest first; +- locks whose consumer has already reached a **terminal** status (`Confirmed`, + `Deleted`, or `Orphan`) — these are **leaked** locks: nothing on the success path + released them, so they sit until the next lease-age sweep (see mechanism 4 in #2395); +- a summary line (total locks, leaked count, oldest age) suitable for scripting. + +This is a **read-only** operation. No data is modified or deleted. + +```bash +tokendiag locks --config +``` + +**Flags:** + +| Flag | Required | Default | Description | +|------|----------|---------|-------------| +| `--config` | Yes | — | Path to the YAML configuration file | + +**Output format:** + +``` +--- Held locks (oldest first) --- + token=: consumer_tx_id= age= status=[ [LEAKED: ...]] + +--- Summary --- + Total locks held : + Leaked (terminal consumer): + Oldest lock age : +``` + +A single snapshot cannot distinguish "one token repeatedly re-contended" from "one +token held a long time" — that comparison requires running `locks` more than once and +diffing. The ranking here is by lock age only. + +## Configuration + +The tool reads a YAML file that describes how to connect to the target database. +Generate a starter file with: + +```bash +tokendiag config example > config.yaml +``` + +### SQLite example + +```yaml +driver: sqlite +dataSource: /var/lib/panurus/node/data.db +tablePrefix: "" +skipPrefix: false +tableNames: {} +tableNameParams: [] +``` + +### PostgreSQL example + +```yaml +driver: postgres +dataSource: "host=db.example.com port=5432 user=panurus password=secret dbname=panurus sslmode=require" +tablePrefix: "prod_" +skipPrefix: false +tableNames: {} +tableNameParams: [] +``` + +### Table name params (network / channel / namespace) + +If the Panurus node was started with a non-empty TMS identity — network, channel +and/or namespace — those values were passed as params when the node derived its own +table names, and become part of every table name alongside `tablePrefix`. Set the same +values here, in the same order (network, channel, namespace), so `tokendiag` resolves +the same tables: + +```yaml +driver: postgres +dataSource: "postgres://user:pass@localhost:5432/panurus?sslmode=disable" +tablePrefix: "" +skipPrefix: false +tableNames: {} +tableNameParams: ["mynetwork", "mychannel", "mynamespace"] +``` + +Leave `tableNameParams` empty if the node was started without any of these +identifiers. See [`docs/services/storage.md`](../../docs/services/storage.md#table-name-customisation) +for the escaping rules that apply to each param. + +### Skipping the prefix + +If the Panurus node was started with `token.storage.skipPrefix: true`, set the same +flag here so the tool resolves the same unprefixed table names: + +```yaml +driver: postgres +dataSource: "postgres://user:pass@localhost:5432/panurus?sslmode=disable" +tablePrefix: "" +skipPrefix: true +tableNames: {} +tableNameParams: [] +``` + +### Table name overrides + +If the Panurus node was started with non-default table names (using the +`token.storage.tableNames` config option), set the same overrides here so the tool +connects to the correct tables. The `locks` command reads `tkn_locks` and `requests`: + +```yaml +driver: postgres +dataSource: "postgres://user:pass@localhost:5432/panurus?sslmode=disable" +tablePrefix: "" +skipPrefix: false +tableNames: + tkn_locks: my_token_locks +tableNameParams: [] +``` + +## Environment variables + +Configuration values can be overridden with environment variables prefixed `CORE_`, +using `_` in place of `.`: + +```bash +CORE_DATASOURCE="postgres://..." tokendiag locks --config config.yaml +``` + +A list value is supplied as a single comma-separated variable: + +```bash +CORE_TABLENAMEPARAMS="testnetwork,testchannel,tokenns" tokendiag locks --config config.yaml +``` + +`tableNames` is a map and cannot be set this way; put it in the config file. diff --git a/cmd/tokendiag/cobra/config/cmd.go b/cmd/tokendiag/cobra/config/cmd.go new file mode 100644 index 0000000000..4e518df727 --- /dev/null +++ b/cmd/tokendiag/cobra/config/cmd.go @@ -0,0 +1,98 @@ +/* +Copyright IBM Corp. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +// Package config provides CLI commands for working with tokendiag configuration. +package config + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +// exampleConfig is a fully-annotated YAML configuration that users can adapt. +// +//nolint:gosec +const exampleConfig = `# tokendiag configuration file +# +# driver selects the database backend. +# Supported values: "sqlite", "postgres" +driver: postgres + +# dataSource is the DSN (Data Source Name) passed directly to the database driver. +# +# PostgreSQL DSN formats: +# URL format: "postgres://user:pass@host:5432/dbname?sslmode=disable" +# Key=value: "host=localhost port=5432 user=panurus password=secret dbname=panurus sslmode=require" +# +# SQLite format (file path): +# dataSource: /var/lib/panurus/node/data.db +dataSource: "postgres://user:pass@localhost:5432/panurus?sslmode=disable" + +# tablePrefix is the optional prefix that was used when the Panurus node created +# its database tables. Leave empty if the node was configured without a prefix. +tablePrefix: "" + +# skipPrefix controls whether the FSC-generated prefix is omitted from all table +# names. Set this to true when the Panurus node was started with +# token.storage.skipPrefix: true. Default is false. +skipPrefix: false + +# tableNames is an optional map of short-code overrides for SQL table names. +# Each key is a canonical short code and the value is the replacement short code +# that will be used when generating the final SQL table name. The FSC-generated +# prefix and params are still applied around the replacement value (unless +# skipPrefix is true). Unknown keys produce a warning and are ignored. +# +# The locks command reads the tkn_locks and requests tables: +# tkn_locks -> fsc_tkn_locks__ +# requests -> fsc_requests__ +# +# Example — rename the lock table: +# tableNames: +# tkn_locks: my_token_locks +tableNames: {} + +# tableNameParams carries the TMS identity (network, channel, namespace, in that +# order) that the Panurus node passed when it derived its own table names. These +# become part of every table name alongside tablePrefix, so they must match the +# node's configuration exactly. Leave empty if the node was started without any of +# these identifiers. +# +# Example: +# tableNameParams: ["mynetwork", "mychannel", "mynamespace"] +tableNameParams: [] +` + +// Cmd returns the Cobra Command for the config subcommand group. +func Cmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "config", + Short: "Configuration helpers.", + Long: `Commands for working with the tokendiag configuration file.`, + } + + cmd.AddCommand(exampleCmd()) + + return cmd +} + +func exampleCmd() *cobra.Command { + return &cobra.Command{ + Use: "example", + Short: "Print an annotated example configuration file.", + Long: `Print a fully-annotated YAML configuration to stdout. + +Redirect the output to a file to create a starting configuration: + + tokendiag config example > config.yaml`, + RunE: func(cmd *cobra.Command, _ []string) error { + _, err := fmt.Fprint(cmd.OutOrStdout(), exampleConfig) + + return err + }, + } +} diff --git a/cmd/tokendiag/cobra/locks/cmd.go b/cmd/tokendiag/cobra/locks/cmd.go new file mode 100644 index 0000000000..b4d1a8288e --- /dev/null +++ b/cmd/tokendiag/cobra/locks/cmd.go @@ -0,0 +1,74 @@ +/* +Copyright IBM Corp. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +// Package locks provides the tokendiag "locks" diagnostic subcommand: a read-only +// inspection of the token_locks table, added for #2395 to let an operator answer +// "which tokens are locked right now, for how long, and is any of that a leak" +// without racing a second Lock call against the primary key. +package locks + +import ( + "context" + "time" + + "github.com/hyperledger-labs/fabric-smart-client/pkg/utils/errors" + "github.com/spf13/cobra" +) + +// Cmd returns the Cobra Command for the locks subcommand. +func Cmd() *cobra.Command { + c := &command{} + + cmd := &cobra.Command{ + Use: "locks", + Short: "Inspect currently held token locks.", + Long: `Reads every currently held row in the token_locks table, joined with the +status of its consuming transaction, and prints: + - every held lock, with its age and the consumer's status; + - locks whose consumer has already reached a terminal status (Confirmed, + Deleted, or Orphan) - these are leaked locks: nothing released them on + settlement, so they will sit until the next lease-age sweep; + - a summary line suitable for scripting. + +This is a read-only operation. No data is modified or deleted.`, + RunE: c.run, + } + + flags := cmd.Flags() + flags.StringVar(&c.configPath, "config", "", "Path to the YAML configuration file (required)") + + if err := cmd.MarkFlagRequired("config"); err != nil { + // MarkFlagRequired only errors if the flag does not exist — this is a programming error. + panic(err) + } + + return cmd +} + +type command struct { + configPath string +} + +func (c *command) run(cmd *cobra.Command, _ []string) error { + cmd.SilenceUsage = true + + cfg, err := LoadConfig(c.configPath) + if err != nil { + return errors.Wrap(err, "failed to load config") + } + + stores, err := NewStores(cfg) + if err != nil { + return errors.Wrap(err, "failed to open stores") + } + defer func() { + if err := stores.Close(); err != nil { + cmd.PrintErrf("warning: failed to close stores: %v\n", err) + } + }() + + return Run(context.Background(), cmd.OutOrStdout(), stores, time.Now()) +} diff --git a/cmd/tokendiag/cobra/locks/config.go b/cmd/tokendiag/cobra/locks/config.go new file mode 100644 index 0000000000..02cfa1bbc5 --- /dev/null +++ b/cmd/tokendiag/cobra/locks/config.go @@ -0,0 +1,93 @@ +/* +Copyright IBM Corp. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package locks + +import ( + "strings" + + "github.com/hyperledger-labs/fabric-smart-client/pkg/utils/errors" + "github.com/spf13/viper" +) + +// Config holds the database connection parameters loaded from the YAML config file. +type Config struct { + // Driver is the database driver to use: "sqlite" or "postgres". + Driver string `mapstructure:"driver"` + // DataSource is the DSN / file path for the target database. + DataSource string `mapstructure:"dataSource"` + // TablePrefix is the optional prefix used when deriving table names. + TablePrefix string `mapstructure:"tablePrefix"` + // SkipPrefix disables the FSC-generated prefix on all table names when true. + // Set this to true when the Panurus node was configured with + // token.storage.skipPrefix: true. Default is false. + SkipPrefix bool `mapstructure:"skipPrefix"` + // TableNames holds optional per-table short-code overrides. + // Each key is a canonical short code (e.g. "tkn_locks") and the value is + // the replacement short code to use when generating the final SQL table name. + // The FSC-generated prefix and params are still applied around the replacement + // (unless SkipPrefix is true). Unknown keys are warned and ignored. + TableNames map[string]string `mapstructure:"tableNames"` + // TableNameParams holds the TMS identity (network, channel, namespace, in that + // order) that the Panurus node passed as the variadic params to + // GetTableNamesWithOverrides/GetTableNamesWithConfig when it derived its table + // names. These become part of every table name alongside TablePrefix, so they + // must match the node's configuration exactly or the tool will resolve the + // wrong (or nonexistent) tables. Leave empty if the node was started without + // any of these identifiers. + TableNameParams []string `mapstructure:"tableNameParams"` +} + +// envBoundKeys lists every config key a CORE_ environment variable may override. +// +// AutomaticEnv on its own is not enough. Unmarshal walks AllKeys(), which holds only +// the keys viper already knows from the config file or from an explicit binding, while +// AutomaticEnv resolves lazily on Get. A key present only in the environment is +// therefore never visited and its override silently does nothing - so CORE_DATASOURCE +// fails the "dataSource must not be empty" check, and CORE_TABLEPREFIX / +// CORE_TABLENAMEPARAMS fail silently and resolve the wrong tables. Binding each key up +// front puts it in AllKeys() so Unmarshal sees it. +// +// tableNames is deliberately absent: it is a map, which a single environment variable +// cannot express. +var envBoundKeys = []string{ + "driver", "dataSource", "tablePrefix", "skipPrefix", "tableNameParams", +} + +// LoadConfig reads the YAML config file at the given path and returns a Config. +func LoadConfig(path string) (Config, error) { + v := viper.New() + v.SetConfigFile(path) + // The prefix must be set on this instance: AutomaticEnv on a fresh viper would + // otherwise bind bare DATASOURCE/DRIVER, letting an unrelated variable silently + // repoint the tool at another database, and would ignore the documented CORE_ form. + v.SetEnvPrefix("core") + v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + v.AutomaticEnv() + for _, key := range envBoundKeys { + if err := v.BindEnv(key); err != nil { + return Config{}, errors.Wrapf(err, "failed to bind environment variable for key %q", key) + } + } + + if err := v.ReadInConfig(); err != nil { + return Config{}, errors.Wrapf(err, "failed to read config file %q", path) + } + + var cfg Config + if err := v.Unmarshal(&cfg); err != nil { + return Config{}, errors.Wrap(err, "failed to unmarshal config") + } + + if cfg.Driver != "sqlite" && cfg.Driver != "postgres" { + return Config{}, errors.Errorf("unsupported driver %q: must be \"sqlite\" or \"postgres\"", cfg.Driver) + } + if cfg.DataSource == "" { + return Config{}, errors.New("dataSource must not be empty") + } + + return cfg, nil +} diff --git a/cmd/tokendiag/cobra/locks/config_test.go b/cmd/tokendiag/cobra/locks/config_test.go new file mode 100644 index 0000000000..6ed8e93d36 --- /dev/null +++ b/cmd/tokendiag/cobra/locks/config_test.go @@ -0,0 +1,110 @@ +/* +Copyright IBM Corp. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package locks + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// writeConfig writes a config file and returns its path. +func writeConfig(t *testing.T, body string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "config.yaml") + require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) + + return path +} + +const minimalConfig = `driver: postgres +dataSource: "postgres://user:pass@localhost:5432/from-file?sslmode=disable" +tablePrefix: "pfx" +skipPrefix: false +tableNameParams: + - testnetwork + - testchannel + - tokenns +` + +func TestLoadConfig(t *testing.T) { + cfg, err := LoadConfig(writeConfig(t, minimalConfig)) + require.NoError(t, err) + assert.Equal(t, "postgres", cfg.Driver) + assert.Equal(t, "postgres://user:pass@localhost:5432/from-file?sslmode=disable", cfg.DataSource) + assert.Equal(t, "pfx", cfg.TablePrefix) + assert.False(t, cfg.SkipPrefix) + assert.Equal(t, []string{"testnetwork", "testchannel", "tokenns"}, cfg.TableNameParams) +} + +func TestLoadConfigRejectsBadInput(t *testing.T) { + t.Run("unsupported driver", func(t *testing.T) { + _, err := LoadConfig(writeConfig(t, "driver: mysql\ndataSource: x\n")) + require.ErrorContains(t, err, "unsupported driver") + }) + + t.Run("empty dataSource", func(t *testing.T) { + _, err := LoadConfig(writeConfig(t, "driver: sqlite\n")) + require.ErrorContains(t, err, "dataSource must not be empty") + }) + + t.Run("missing file", func(t *testing.T) { + _, err := LoadConfig(filepath.Join(t.TempDir(), "absent.yaml")) + require.ErrorContains(t, err, "failed to read config file") + }) +} + +// TestLoadConfigEnvOverrides pins the CORE_ overrides documented in the README. Binding +// each key explicitly is what makes them work: AutomaticEnv resolves lazily on Get, +// while Unmarshal only walks AllKeys(), so a key present solely in the environment would +// otherwise never be visited - CORE_DATASOURCE would fail the empty-dataSource check and +// the table-name keys would fail silently, resolving the wrong tables. +func TestLoadConfigEnvOverrides(t *testing.T) { + t.Run("keys supplied only by the environment", func(t *testing.T) { + t.Setenv("CORE_DATASOURCE", "postgres://user:pass@localhost:5432/from-env?sslmode=disable") + t.Setenv("CORE_TABLEPREFIX", "envpfx") + t.Setenv("CORE_SKIPPREFIX", "true") + // A list arrives as one comma-separated value. + t.Setenv("CORE_TABLENAMEPARAMS", "envnetwork,envchannel,envns") + + cfg, err := LoadConfig(writeConfig(t, "driver: postgres\n")) + require.NoError(t, err) + assert.Equal(t, "postgres://user:pass@localhost:5432/from-env?sslmode=disable", cfg.DataSource) + assert.Equal(t, "envpfx", cfg.TablePrefix) + assert.True(t, cfg.SkipPrefix) + assert.Equal(t, []string{"envnetwork", "envchannel", "envns"}, cfg.TableNameParams) + }) + + t.Run("environment overrides the file", func(t *testing.T) { + t.Setenv("CORE_TABLEPREFIX", "envpfx") + + cfg, err := LoadConfig(writeConfig(t, minimalConfig)) + require.NoError(t, err) + assert.Equal(t, "envpfx", cfg.TablePrefix) + }) + + t.Run("file is kept when no variable is set", func(t *testing.T) { + cfg, err := LoadConfig(writeConfig(t, minimalConfig)) + require.NoError(t, err) + assert.Equal(t, "pfx", cfg.TablePrefix) + assert.Equal(t, []string{"testnetwork", "testchannel", "tokenns"}, cfg.TableNameParams) + }) + + // An unprefixed variable must not be picked up: the prefix is what stops an + // unrelated DATASOURCE in the environment from silently repointing the tool at + // another database. + t.Run("unprefixed variable is ignored", func(t *testing.T) { + t.Setenv("DATASOURCE", "postgres://evil@localhost:5432/other?sslmode=disable") + + cfg, err := LoadConfig(writeConfig(t, minimalConfig)) + require.NoError(t, err) + assert.Equal(t, "postgres://user:pass@localhost:5432/from-file?sslmode=disable", cfg.DataSource) + }) +} diff --git a/cmd/tokendiag/cobra/locks/runner.go b/cmd/tokendiag/cobra/locks/runner.go new file mode 100644 index 0000000000..eb7538354d --- /dev/null +++ b/cmd/tokendiag/cobra/locks/runner.go @@ -0,0 +1,108 @@ +/* +Copyright IBM Corp. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package locks + +import ( + "context" + "fmt" + "io" + "sort" + "strconv" + "time" + + driver3 "github.com/LFDT-Panurus/panurus/token/services/storage/db/driver" + "github.com/hyperledger-labs/fabric-smart-client/pkg/utils/errors" +) + +// isTerminal reports whether status is a terminal status of a consuming transaction — +// i.e. one after which the lock it holds should already have been released. A lock +// still present with a terminal-status consumer is the mechanism-4 leak from #2395: +// nothing on the success path calls UnlockByTxID, so the row survives until the +// next lease-age sweep. +func isTerminal(status *driver3.TxStatus) bool { + if status == nil { + return false + } + + switch *status { + case driver3.Confirmed, driver3.Deleted, driver3.Orphan: + return true + default: + return false + } +} + +// statusName renders status for display, or "unknown" if nil. +func statusName(status *driver3.TxStatus) string { + if status == nil { + return "unknown" + } + if name, ok := driver3.TxStatusMessage[*status]; ok { + return name + } + + return strconv.Itoa(*status) +} + +// Run reads every currently held lock via TokenLockStore.ListLocks and reports: +// - every held lock, with its age and the status of its consuming transaction; +// - locks whose consumer has already reached a terminal status (the leak +// mechanism-4 in #2395 describes — the lock is not released on settlement, +// so it sits until the next lease-age sweep); +// - the same list ordered oldest lock first, which is as close to a hot-token +// ranking as one snapshot gets: it cannot distinguish "repeatedly +// re-contended" from "held a long time" without comparing against an +// earlier run; +// - a summary line intended for scripting (total, leaked, oldest age). +func Run(ctx context.Context, w io.Writer, stores *Stores, now time.Time) error { + records, err := stores.TokenLock.ListLocks(ctx) + if err != nil { + return errors.Wrap(err, "list locks") + } + + sort.Slice(records, func(i, j int) bool { + return records[i].CreatedAt.Before(records[j].CreatedAt) + }) + + if _, err := fmt.Fprintf(w, "--- Held locks (oldest first) ---\n"); err != nil { + return err + } + var leaked int + var oldest time.Duration + for i, r := range records { + age := now.Sub(r.CreatedAt) + if i == 0 { + oldest = age + } + terminalMark := "" + if isTerminal(r.Status) { + leaked++ + terminalMark = " [LEAKED: consumer is terminal, lock should have been released]" + } + if _, err := fmt.Fprintf(w, " token=%s:%d consumer_tx_id=%s age=%s status=%s%s\n", + r.TokenID.TxId, r.TokenID.Index, r.ConsumerTxID, age.Round(time.Second), statusName(r.Status), terminalMark); err != nil { + return err + } + } + + if _, err := fmt.Fprintf(w, "\n--- Summary ---\n"); err != nil { + return err + } + if _, err := fmt.Fprintf(w, " Total locks held : %d\n", len(records)); err != nil { + return err + } + if _, err := fmt.Fprintf(w, " Leaked (terminal consumer): %d\n", leaked); err != nil { + return err + } + if len(records) > 0 { + if _, err := fmt.Fprintf(w, " Oldest lock age : %s\n", oldest.Round(time.Second)); err != nil { + return err + } + } + + return nil +} diff --git a/cmd/tokendiag/cobra/locks/stores.go b/cmd/tokendiag/cobra/locks/stores.go new file mode 100644 index 0000000000..a417bfcdbc --- /dev/null +++ b/cmd/tokendiag/cobra/locks/stores.go @@ -0,0 +1,94 @@ +/* +Copyright IBM Corp. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package locks + +import ( + "database/sql" + + driver3 "github.com/LFDT-Panurus/panurus/token/services/storage/db/driver" + sqlcommon "github.com/LFDT-Panurus/panurus/token/services/storage/db/sql/common" + "github.com/LFDT-Panurus/panurus/token/services/storage/db/sql/postgres" + "github.com/LFDT-Panurus/panurus/token/services/storage/db/sql/sqlite" + "github.com/hyperledger-labs/fabric-smart-client/pkg/utils/errors" + scommon "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/storage/driver/common" + _ "github.com/jackc/pgx/v5/stdlib" + _ "modernc.org/sqlite" +) + +// Stores groups the store(s) needed by the locks command. +type Stores struct { + TokenLock driver3.TokenLockStore +} + +// Close closes the underlying database connection. TokenLock owns the +// *sql.DB handle it was built from and closes it, so there is nothing else +// to release here. +func (s *Stores) Close() error { + if err := s.TokenLock.Close(); err != nil { + return errors.Wrap(err, "token lock store close") + } + + return nil +} + +// NewStores opens the database described by cfg and returns a TokenLockStore +// pointing at the existing schema (no CREATE TABLE is issued). +func NewStores(cfg Config) (*Stores, error) { + storeCfg := sqlcommon.StorageConfig{ + TableNames: cfg.TableNames, + SkipPrefix: cfg.SkipPrefix, + } + tableNames, err := sqlcommon.GetTableNamesWithConfig(cfg.TablePrefix, storeCfg, cfg.TableNameParams...) + if err != nil { + return nil, errors.Wrap(err, "derive table names") + } + + switch cfg.Driver { + case "sqlite": + return newSQLiteStores(cfg.DataSource, tableNames) + case "postgres": + return newPostgresStores(cfg.DataSource, tableNames) + default: + return nil, errors.Errorf("unsupported driver: %s", cfg.Driver) + } +} + +func newSQLiteStores(dataSource string, tableNames sqlcommon.TableNames) (*Stores, error) { + db, err := sql.Open("sqlite", dataSource) + if err != nil { + return nil, errors.Wrap(err, "open sqlite db") + } + + dbs := &scommon.RWDB{ReadDB: db, WriteDB: db} + + tokenLockStore, err := sqlite.NewTokenLockStore(dbs, tableNames) + if err != nil { + _ = db.Close() + + return nil, errors.Wrap(err, "create sqlite token lock store") + } + + return &Stores{TokenLock: tokenLockStore}, nil +} + +func newPostgresStores(dataSource string, tableNames sqlcommon.TableNames) (*Stores, error) { + db, err := sql.Open("pgx", dataSource) + if err != nil { + return nil, errors.Wrap(err, "open postgres db") + } + + dbs := &scommon.RWDB{ReadDB: db, WriteDB: db} + + tokenLockStore, err := postgres.NewTokenLockStore(dbs, tableNames) + if err != nil { + _ = db.Close() + + return nil, errors.Wrap(err, "create postgres token lock store") + } + + return &Stores{TokenLock: tokenLockStore}, nil +} diff --git a/cmd/tokendiag/cobra/locks/stores_test.go b/cmd/tokendiag/cobra/locks/stores_test.go new file mode 100644 index 0000000000..782fca8c46 --- /dev/null +++ b/cmd/tokendiag/cobra/locks/stores_test.go @@ -0,0 +1,126 @@ +/* +Copyright IBM Corp. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package locks + +import ( + "context" + "database/sql" + "path/filepath" + "testing" + "time" + + driver3 "github.com/LFDT-Panurus/panurus/token/services/storage/db/driver" + sqlcommon "github.com/LFDT-Panurus/panurus/token/services/storage/db/sql/common" + _ "modernc.org/sqlite" + + "github.com/stretchr/testify/require" +) + +// TestNewStores_TableNameParams checks that NewStores forwards Config.TableNameParams +// to GetTableNamesWithConfig, so a deployment that started its Panurus node with a +// non-empty network/channel/namespace identity resolves the same table names here. +// Before this test's corresponding fix, NewStores called GetTableNamesWithConfig with +// zero params regardless of what Config carried (there was no field to carry them at +// all), so the tool would query the wrong (or nonexistent) tables against any real +// deployment using non-empty params. +func TestNewStores_TableNameParams(t *testing.T) { + const prefix = "pfx" + params := []string{"net1", "chan1", "ns1"} + + // Compute the table names the exact same way NewStores is expected to: via + // GetTableNamesWithConfig with the config's prefix, overrides/skip-prefix and + // params. This is the independent expectation the wiring in stores.go must match. + storageCfg := sqlcommon.StorageConfig{} + expected, err := sqlcommon.GetTableNamesWithConfig(prefix, storageCfg, params...) + require.NoError(t, err) + + dataSource := filepath.Join(t.TempDir(), "test.db") + seedSQLiteSchema(t, dataSource, expected) + + cfg := Config{ + Driver: "sqlite", + DataSource: dataSource, + TablePrefix: prefix, + TableNameParams: params, + } + + stores, err := NewStores(cfg) + require.NoError(t, err) + defer func() { require.NoError(t, stores.Close()) }() + + records, err := stores.TokenLock.ListLocks(context.Background()) + require.NoError(t, err) + require.Len(t, records, 1) + require.Equal(t, "tx1", records[0].TokenID.TxId) + require.Equal(t, "consumer1", records[0].ConsumerTxID) + require.NotNil(t, records[0].Status) + require.Equal(t, driver3.Confirmed, *records[0].Status) +} + +// TestNewStores_MissingTableNameParams checks that omitting TableNameParams resolves +// different (here, nonexistent) table names than the ones the schema was seeded +// under, demonstrating that the params genuinely participate in the derived name +// rather than being silently ignored. +func TestNewStores_MissingTableNameParams(t *testing.T) { + const prefix = "pfx" + params := []string{"net1", "chan1", "ns1"} + + storageCfg := sqlcommon.StorageConfig{} + expected, err := sqlcommon.GetTableNamesWithConfig(prefix, storageCfg, params...) + require.NoError(t, err) + + dataSource := filepath.Join(t.TempDir(), "test.db") + seedSQLiteSchema(t, dataSource, expected) + + // Same prefix, but no TableNameParams: this must resolve a different table name + // than the one the schema was created under, so the query fails. + cfg := Config{ + Driver: "sqlite", + DataSource: dataSource, + TablePrefix: prefix, + } + + stores, err := NewStores(cfg) + require.NoError(t, err) + defer func() { require.NoError(t, stores.Close()) }() + + _, err = stores.TokenLock.ListLocks(context.Background()) + require.Error(t, err) +} + +// seedSQLiteSchema creates a minimal TokenLocks/Requests schema under the given table +// names and inserts a single held lock whose consuming transaction is Confirmed. +func seedSQLiteSchema(t *testing.T, dataSource string, tableNames sqlcommon.TableNames) { + t.Helper() + + db, err := sql.Open("sqlite", dataSource) + require.NoError(t, err) + defer func() { require.NoError(t, db.Close()) }() + + _, err = db.Exec(`CREATE TABLE ` + tableNames.Requests + ` ( + tx_id TEXT NOT NULL PRIMARY KEY, + status INT NOT NULL + )`) + require.NoError(t, err) + + _, err = db.Exec(`CREATE TABLE ` + tableNames.TokenLocks + ` ( + tx_id TEXT NOT NULL, + idx INT NOT NULL, + consumer_tx_id TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY(tx_id, idx) + )`) + require.NoError(t, err) + + _, err = db.Exec(`INSERT INTO `+tableNames.Requests+` (tx_id, status) VALUES (?, ?)`, //nolint:gosec // table name comes from GetTableNamesWithConfig, not attacker input + "consumer1", driver3.Confirmed) + require.NoError(t, err) + + _, err = db.Exec(`INSERT INTO `+tableNames.TokenLocks+` (tx_id, idx, consumer_tx_id, created_at) VALUES (?, ?, ?, ?)`, //nolint:gosec // table name comes from GetTableNamesWithConfig, not attacker input + "tx1", 0, "consumer1", time.Now().Round(0).String()) + require.NoError(t, err) +} diff --git a/cmd/tokendiag/go.mod b/cmd/tokendiag/go.mod new file mode 100644 index 0000000000..483d9fb818 --- /dev/null +++ b/cmd/tokendiag/go.mod @@ -0,0 +1,115 @@ +module github.com/LFDT-Panurus/panurus/cmd/tokendiag + +go 1.27.1 + +replace github.com/LFDT-Panurus/panurus => ./../../ + +require ( + github.com/LFDT-Panurus/panurus v0.10.1 + github.com/hyperledger-labs/fabric-smart-client v0.23.0 + github.com/jackc/pgx/v5 v5.11.0 + github.com/spf13/cobra v1.10.2 + github.com/spf13/viper v1.21.0 + github.com/stretchr/testify v1.12.1 + modernc.org/sqlite v1.59.0 +) + +require ( + github.com/DATA-DOG/go-sqlmock v1.5.2 // indirect + github.com/IBM/mathlib v0.3.2 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/XSAM/otelsql v0.44.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/bits-and-blooms/bitset v1.24.6 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cockroachdb/errors v1.14.0 // indirect + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/redact v1.1.5 // indirect + github.com/consensys/gnark-crypto v0.21.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/dgraph-io/ristretto/v2 v2.4.2 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/go-connections v0.7.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/fsnotify/fsnotify v1.10.1 // indirect + github.com/getsentry/sentry-go v0.46.0 // indirect + github.com/go-logfmt/logfmt v0.6.1 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect + github.com/hashicorp/go-uuid v1.0.3 // indirect + github.com/hyperledger/fabric-amcl v0.0.0-20230602173724-9e02669dceb2 // indirect + github.com/hyperledger/fabric-lib-go v1.1.5-0.20260708100132-163bcc919208 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgxlisten v0.0.0-20250802141604-12b92425684c // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/knadh/koanf/maps v0.1.2 // indirect + github.com/knadh/koanf/parsers/yaml v1.1.1 // indirect + github.com/knadh/koanf/providers/env/v2 v2.0.1 // indirect + github.com/knadh/koanf/providers/file v1.2.1 // indirect + github.com/knadh/koanf/providers/rawbytes v1.0.1 // indirect + github.com/knadh/koanf/v2 v2.3.7 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/moby/api v1.56.0 // indirect + github.com/moby/moby/client v0.6.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/onsi/gomega v1.44.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/pelletier/go-toml/v2 v2.3.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_golang v1.24.1 // indirect + github.com/prometheus/client_model v0.6.3 // indirect + github.com/prometheus/common v0.71.0 // indirect + github.com/prometheus/procfs v0.21.1 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/sagikazarmark/locafero v0.12.0 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 // indirect + go.opentelemetry.io/otel v1.46.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.46.0 // indirect + go.opentelemetry.io/otel/metric v1.46.0 // indirect + go.opentelemetry.io/otel/sdk v1.46.0 // indirect + go.opentelemetry.io/otel/trace v1.46.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.28.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sync v0.23.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.42.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5 // indirect + google.golang.org/grpc v1.84.0 // indirect + google.golang.org/protobuf v1.36.12 // indirect + modernc.org/libc v1.75.7 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.12.1 // indirect +) diff --git a/cmd/tokendiag/go.sum b/cmd/tokendiag/go.sum new file mode 100644 index 0000000000..1e249110cf --- /dev/null +++ b/cmd/tokendiag/go.sum @@ -0,0 +1,319 @@ +github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= +github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= +github.com/IBM/idemix v0.3.1 h1:Hzlad1hsij+Nk8PoGB4E2MmexHUyE9KLyZTJVkK/I5A= +github.com/IBM/idemix v0.3.1/go.mod h1:S5SXwuwJQXFrY1LwCdvqwfQWOKCb8hVoY18ZcAFc8rQ= +github.com/IBM/mathlib v0.3.2 h1:4XWONT9OBwzsgVPdqWOZhB/bxETPSdi7t+1u36X9oNo= +github.com/IBM/mathlib v0.3.2/go.mod h1:SoGBO5m5bHQ4hph+nhf3o5gmBgRLXXZ77/YVa1X7xbc= +github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= +github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/XSAM/otelsql v0.44.0 h1:KxCiv26Fh4okTPlgROE2BWk+lgi20pdgMGxuSwgbRls= +github.com/XSAM/otelsql v0.44.0/go.mod h1:FySZIr4R4WWMqvIjf2Iah7C0LAlpKvs9XRkaX7rE608= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bits-and-blooms/bitset v1.24.6 h1:qcrftZUVBIwfs+m+nhoCBAPT+ZPZZjti8SbHbDQQkZ4= +github.com/bits-and-blooms/bitset v1.24.6/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cockroachdb/errors v1.14.0 h1:EfdVEJpN3z8rPMo43Yit59LxoiIa470fSXpZXuEs+ZI= +github.com/cockroachdb/errors v1.14.0/go.mod h1:xRa70jZ9sNBQmISt5KmJmAD++E4dQHm89oCRiZGEdq0= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/consensys/gnark-crypto v0.21.0 h1:FDHibVIk4T5LkOKAkiN38g8gEvOxNcM10mLHOqvFTD0= +github.com/consensys/gnark-crypto v0.21.0/go.mod h1:hdTjDNjdkYJ1oVuc8emh9XEhfM1SbyZhJigFqItiOLk= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/ristretto/v2 v2.4.2 h1:x0cvjmUKxt764Yxdk2nr94we1AvPPAMh1rh5TQ+Jo80= +github.com/dgraph-io/ristretto/v2 v2.4.2/go.mod h1:0KsrXtXvnv0EqnzyowllbVJB8yBonswa2lTCK2gGo9E= +github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38= +github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= +github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= +github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= +github.com/getsentry/sentry-go v0.46.0 h1:mbdDaarbUdOt9X+dx6kDdntkShLEX3/+KyOsVDTPDj0= +github.com/getsentry/sentry-go v0.46.0/go.mod h1:evVbw2qotNUdYG8KxXbAdjOQWWvWIwKxpjdZZIvcIPw= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-logfmt/logfmt v0.6.1 h1:4hvbpePJKnIzH1B+8OR/JPbTx37NktoI9LE2QZBBkvE= +github.com/go-logfmt/logfmt v0.6.1/go.mod h1:EV2pOAQoZaT1ZXZbqDl5hrymndi4SY9ED9/z6CO0XAk= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hyperledger-labs/fabric-smart-client v0.23.0 h1:6U0y5JWkfGHPaNykNGnPjfIWcHmZQQhU73qj9b59kTw= +github.com/hyperledger-labs/fabric-smart-client v0.23.0/go.mod h1:LfR2cp6FZLKHG+gRx71BlZzluLvyAW/Z+tQ5A5Q0DLc= +github.com/hyperledger/fabric-amcl v0.0.0-20230602173724-9e02669dceb2 h1:B1Nt8hKb//KvgGRprk0h1t4lCnwhE9/ryb1WqfZbV+M= +github.com/hyperledger/fabric-amcl v0.0.0-20230602173724-9e02669dceb2/go.mod h1:X+DIyUsaTmalOpmpQfIvFZjKHQedrURQ5t4YqquX7lE= +github.com/hyperledger/fabric-lib-go v1.1.5-0.20260708100132-163bcc919208 h1:qA49XOMwyNPxggVyW+HSDAg7I3GdtlGWDTbh40/rwZk= +github.com/hyperledger/fabric-lib-go v1.1.5-0.20260708100132-163bcc919208/go.mod h1:EKqTudBsC2yovZdcJq5ekWvkq3USF1mbau07I0y8zMs= +github.com/hyperledger/fabric-protos-go-apiv2 v0.3.7 h1:sQ5qv8vQQfwewa1JlCiSCC8dLElmaU2/frLolpgibEY= +github.com/hyperledger/fabric-protos-go-apiv2 v0.3.7/go.mod h1:bJnwzfv03oZQeCc863pdGTDgf5nmCy6Za3RAE7d2XsQ= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/pgxlisten v0.0.0-20250802141604-12b92425684c h1:on93qgQJqBwjHYjUv51BuS6Q8UlALFAHhsvuR5WVaos= +github.com/jackc/pgxlisten v0.0.0-20250802141604-12b92425684c/go.mod h1:ygR1JwoRvIb4hhLukHQxSB3u/sRQT4Laylx0rDtNEhE= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= +github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpbo= +github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/parsers/yaml v1.1.1 h1:u70vV5IyaM0HvONh8HoqBC97oTgO33KcpZbTLiKVinU= +github.com/knadh/koanf/parsers/yaml v1.1.1/go.mod h1:HHmcHXUrp9cOPcuC+2wrr44GTUB0EC+PyfN3HZD9tFg= +github.com/knadh/koanf/providers/env/v2 v2.0.1 h1:a3KagndPqhcWHQv6Pz4OZmwkI/yMeTjkiZye6ZCkyW0= +github.com/knadh/koanf/providers/env/v2 v2.0.1/go.mod h1:1g01PE+Ve1gBfWNNw2wmULRP0tc8RJrjn5p2N/jNCIc= +github.com/knadh/koanf/providers/file v1.2.1 h1:bEWbtQwYrA+W2DtdBrQWyXqJaJSG3KrP3AESOJYp9wM= +github.com/knadh/koanf/providers/file v1.2.1/go.mod h1:bp1PM5f83Q+TOUu10J/0ApLBd9uIzg+n9UgthfY+nRA= +github.com/knadh/koanf/providers/rawbytes v1.0.1 h1:JCQoly+djX23Okr8kqtS19R7UXKleTAp62Vib2VrVYs= +github.com/knadh/koanf/providers/rawbytes v1.0.1/go.mod h1:KxwYJf1uezTKy6PBtfE+m725NGp4GPVA7XoNTJ/PtLo= +github.com/knadh/koanf/v2 v2.3.7 h1:amceufOeoQcq6VFKjm7/ggJ3t0Dkqaxy5fza4j3YgTA= +github.com/knadh/koanf/v2 v2.3.7/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leanovate/gopter v0.2.11 h1:vRjThO1EKPb/1NsDXuDrzldR28RLkBflWYcU9CvzWu4= +github.com/leanovate/gopter v0.2.11/go.mod h1:aK3tzZP/C+p1m3SPRE4SYZFGP7jjkuSI4f7Xvpt0S9c= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/miekg/pkcs11 v1.1.2 h1:/VxmeAX5qU6Q3EwafypogwWbYryHFmF2RpkJmw3m4MQ= +github.com/miekg/pkcs11 v1.1.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/moby/api v1.56.0 h1:GQzua3NA599ASSIICx0iFgiJeO9YkdDARvQsm23ZZuQ= +github.com/moby/moby/api v1.56.0/go.mod h1:sZ+THbVWkjOmBPPfbnzdD/G1LuIexWhqlSHHPTDQ1Uk= +github.com/moby/moby/client v0.6.0 h1:AJjEB21QPbXSXjDsZorFBoDZPhMrfbpaPLgSMAW9Bgs= +github.com/moby/moby/client v0.6.0/go.mod h1:OCo00wNRyA3m4lmJ228W3JbyCN4ZNNYjpOXiJydBdcQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/onsi/ginkgo/v2 v2.33.0 h1:C8gBA6Uc2ZEubiV+SXiu5tZnMTwEmXHgkJwGozKtZf8= +github.com/onsi/ginkgo/v2 v2.33.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= +github.com/onsi/gomega v1.44.0 h1:eAiGl3Pw5jz5GQdDff0BcxYpAX1JxW8xD7mFUuwNfZQ= +github.com/onsi/gomega v1.44.0/go.mod h1:e/C2HwaZ1DhvjzXXuFhcR7hY7Sh9pl7MmoWKEjzwcdA= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pelletier/go-toml/v2 v2.3.1 h1:MYEvvGnQjeNkRF1qUuGolNtNExTDwct51yp7olPtrEc= +github.com/pelletier/go-toml/v2 v2.3.1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= +github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= +github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo= +github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM= +github.com/prometheus/common v0.71.0 h1:9KDAKb7Mj3HEVKyFCK6Dc/HIwlBzZIN2l7/lrHl3KK8= +github.com/prometheus/common v0.71.0/go.mod h1:CLJ5H8TEsGX8bl31BdMkfhIZ+QmZ9tBPPotUxUbfcmk= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4= +github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= +github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 h1:B2h3uqicet1CT2N5TOFhS+Gq++9i0/CLmaxvhmhtP5s= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0/go.mod h1:dylvB+ZiiwMvsDij9O84Uy7SijLgHMX4mbkncds+4Sw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU= +go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc= +go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 h1:OFnwLJr+pF3iHrlGSzbxyuo6/6HyBlnlN1CWEJmBVcw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0/go.mod h1:716wFneO0ov19A2beH5hjfh9AK5z/VWNAtDijp1Y0/g= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 h1:w53CDeOA/Kurp7yRsegSr6pbbr759dOvJ+yNmWM6Hxs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0/go.mod h1:BOmGMCbAtvcJiSJ+hLuhgPLdDbimnraSl8irz3iY8sY= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.46.0 h1:KdRxPiAoMptR3vfWzvjjvutTsSiwbC2uG0496rzZNfo= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.46.0/go.mod h1:K/qSA+3G7Eovxi4K09wzrAgkWRnosS0DAOZeEpve7sM= +go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8= +go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o= +go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI= +go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM= +go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE= +go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4= +go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c= +go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597 h1:qLvzZeaANDgyVOA8pyHCOStGlXn0rseXma+GQjeuv2g= +golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 h1:ax2KzoSRIZU/M0cIxri3pKxy99vniH1PVxWC6si/eZI= +google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688/go.mod h1:1RJ9BQGyNdZwkGc1eTqkErfRZ6RJyYPHZo73BZ1vQqI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5 h1:1VUiZAXyC+zmiFYi+WLtBzr68Cj8wOofHjjrA/kkizc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA= +google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0= +google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= +modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8= +modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w= +modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= +modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.75.7 h1:o3DTP9/0p9pKmY2WCKQaySW6wIiZhNM7wc2lUoyhfew= +modernc.org/libc v1.75.7/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.59.0 h1:X1es1GpqBlS/5T+vbM4HLUdaa8OtQx468DF2vrx+38A= +modernc.org/sqlite v1.59.0/go.mod h1:+paeT2A3iPRHkQDwG7oA6Tk0zQd5woMEI8q7orfry8k= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= diff --git a/cmd/tokendiag/main.go b/cmd/tokendiag/main.go new file mode 100644 index 0000000000..8274749a82 --- /dev/null +++ b/cmd/tokendiag/main.go @@ -0,0 +1,52 @@ +/* +Copyright IBM Corp. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package main + +import ( + "os" + "strings" + + "github.com/LFDT-Panurus/panurus/cmd/tokendiag/cobra/config" + "github.com/LFDT-Panurus/panurus/cmd/tokendiag/cobra/locks" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +// CmdRoot is the prefix for environment variables. +const CmdRoot = "core" + +// mainCmd is the root command for the tokendiag tool. +var mainCmd = &cobra.Command{Use: "tokendiag"} + +// main is the entry point for the tokendiag command. +func main() { + if err := Execute(); err != nil { + os.Exit(1) + } +} + +// Execute adds all child commands to the root command and sets flags appropriately. +func Execute() error { + viper.SetEnvPrefix(CmdRoot) + viper.AutomaticEnv() + replacer := strings.NewReplacer(".", "_") + viper.SetEnvKeyReplacer(replacer) + + mainFlags := mainCmd.PersistentFlags() + mainFlags.String("logging-level", "", "Legacy logging level flag") + if err := viper.BindPFlag("logging_level", mainFlags.Lookup("logging-level")); err != nil { + return err + } + if err := mainFlags.MarkHidden("logging-level"); err != nil { + return err + } + + mainCmd.AddCommand(config.Cmd()) + mainCmd.AddCommand(locks.Cmd()) + + return mainCmd.Execute() +} diff --git a/docs/README.md b/docs/README.md index cd2ca4a9db..ccce28c096 100644 --- a/docs/README.md +++ b/docs/README.md @@ -30,6 +30,7 @@ Panurus ships several standalone CLI tools, each living in its own Go module und | [**tokengen**](../cmd/tokengen/README.md) | Generates public parameters, token chaincode packages, and other cryptographic artifacts. Used to pre-configure development and test environments. | | [**artifactgen**](../cmd/artifactgen/README.md) | Topology-driven artifact generation (previously part of `tokengen`). Kept separate to avoid pulling in the `integration/nwo` test framework. | | [**skicleanup**](../cmd/skicleanup/README.md) | Diagnostic tool that lists orphaned signer entries and their derived SKIs. Connects directly to an existing Panurus database (SQLite or PostgreSQL). | +| [**tokendiag**](../cmd/tokendiag/README.md) | Diagnostic tool for inspecting token-selector state, starting with currently held token locks and settlement-status leaks. Connects directly to an existing Panurus database (SQLite or PostgreSQL). | ## Development diff --git a/docs/development/development.md b/docs/development/development.md index 4388d34def..776950b3d5 100644 --- a/docs/development/development.md +++ b/docs/development/development.md @@ -13,6 +13,7 @@ This page contains link to the development guidelines and more. - [Mock Files Generation](./mock.md) - [AI Agents Best Practices](./ai_agents.md) - [Tools: tokengen](./tokengen.md) +- [Tools: tokendiag](./tokendiag.md) - [Makefile Guide](./makefile.md) ## Useful resources diff --git a/docs/development/metrics.md b/docs/development/metrics.md index 894447a418..68a7da28ec 100644 --- a/docs/development/metrics.md +++ b/docs/development/metrics.md @@ -178,7 +178,13 @@ Source: `token/services/auditor/metrics.go`. Recorded by the default token selector. `outcome` is one of `success`, `insufficient_funds`, `locked_funds` or `error`; `fetcher_type` is `eager` or `lazy`. `selection_immediate_retries` shows how -often a selection had to retry because of concurrent lock contention. +often a selection had to retry because of concurrent lock contention. `lock_conflicts_total` and +`distinct_tokens_attempted` were added for [#2395](https://github.com/LFDT-Panurus/panurus/issues/2395) +to distinguish "one hot token retried many times" from "many tokens each contended once"; +`lock_conflicts_total` is deliberately unlabeled by token id or wallet id to avoid unbounded +cardinality — per-token attribution belongs in the selector's debug-level log line (`Lost lock +race on token [...]`, visible once the sherdlock package's logger is at debug level) and in the +[`tokendiag locks`](../../cmd/tokendiag/README.md) command. | Metric | Type | Labels | Description | |---|---|---|---| @@ -186,6 +192,8 @@ often a selection had to retry because of concurrent lock contention. | `panurus_services_selector_sherdlock_selection_duration_seconds` | histogram | — | Duration of a token selection call in seconds | | `panurus_services_selector_sherdlock_selection_outcome_total` | counter | `outcome` | Total number of token selection outcomes by result type | | `panurus_services_selector_sherdlock_selection_immediate_retries` | histogram | — | Distribution of immediate retry counts per token selection call | +| `panurus_services_selector_sherdlock_lock_conflicts_total` | counter | — | Total number of lost lock races (a token was already locked by another process) | +| `panurus_services_selector_sherdlock_distinct_tokens_attempted` | histogram | — | Distribution of the number of distinct tokens a lock was attempted on (won, lost, or rate-limited) per token selection call | Source: `token/services/selector/sherdlock/metrics.go`. diff --git a/docs/development/tokendiag.md b/docs/development/tokendiag.md new file mode 100644 index 0000000000..4a682d66a5 --- /dev/null +++ b/docs/development/tokendiag.md @@ -0,0 +1,20 @@ +# Tools: tokendiag + +- [`tokendiag`](../../cmd/tokendiag/README.md) is a read-only diagnostic tool for + inspecting token-selector state directly against an existing Panurus database + (SQLite or PostgreSQL). It was added for + [#2395](https://github.com/LFDT-Panurus/panurus/issues/2395), a stress load test that + showed a small number of hot tokens absorbing the vast majority of lock-contention + errors. +- The `locks` subcommand lists every currently held row in the `token_locks` table, + flags locks whose consuming transaction has already reached a terminal status + (`Confirmed`, `Deleted`, or `Orphan`) as **leaked** — the row should have been + released on settlement but was not, and will otherwise sit until the next + lease-age sweep — and prints a summary suitable for scripting. +- Mirrors `skicleanup`'s configuration format (`driver`, `dataSource`, `tablePrefix`, + `skipPrefix`, `tableNames`), so an existing `skicleanup` config file can usually be + reused as-is, plus a `tableNameParams` field (network/channel/namespace, in that + order) that carries the same params a node passes to `GetTableNamesWithConfig` when + it derives its own table names — required whenever the node runs with a non-empty + TMS identity, otherwise `tokendiag` resolves the wrong table names; see the tool's + own README for details and examples. diff --git a/token/services/metricsdoc/testdata/metrics.golden b/token/services/metricsdoc/testdata/metrics.golden index 888e2018ac..f8b640a0ac 100644 --- a/token/services/metricsdoc/testdata/metrics.golden +++ b/token/services/metricsdoc/testdata/metrics.golden @@ -35,6 +35,8 @@ panurus_services_network_fabricx_finality_queue_finality_queue_enqueue_drops_tot panurus_services_network_fabricx_finality_queue_finality_queue_pending_events | gauge | - | token/services/network/fabricx/finality/queue/metrics.go | Current number of finality events waiting in the queue buffer panurus_services_network_fabricx_finality_queue_finality_queue_processing_duration_seconds | histogram | - | token/services/network/fabricx/finality/queue/metrics.go | Histogram of successful event processing time in worker goroutines (seconds) panurus_services_network_fabricx_finality_queue_finality_queue_processing_errors_total | counter | - | token/services/network/fabricx/finality/queue/metrics.go | Total number of errors returned by event.Process in worker goroutines +panurus_services_selector_sherdlock_distinct_tokens_attempted | histogram | - | token/services/selector/sherdlock/metrics.go | Distribution of the number of distinct tokens a lock was attempted on (won, lost, or rate-limited) per token selection call +panurus_services_selector_sherdlock_lock_conflicts_total | counter | - | token/services/selector/sherdlock/metrics.go | Total number of lost lock races (a token was already locked by another process) panurus_services_selector_sherdlock_selection_duration_seconds | histogram | - | token/services/selector/sherdlock/metrics.go | Duration of a token selection call in seconds panurus_services_selector_sherdlock_selection_immediate_retries | histogram | - | token/services/selector/sherdlock/metrics.go | Distribution of immediate retry counts per token selection call panurus_services_selector_sherdlock_selection_outcome_total | counter | outcome | token/services/selector/sherdlock/metrics.go | Total number of token selection outcomes by result type diff --git a/token/services/selector/sherdlock/contention_test.go b/token/services/selector/sherdlock/contention_test.go new file mode 100644 index 0000000000..d27f83c7b6 --- /dev/null +++ b/token/services/selector/sherdlock/contention_test.go @@ -0,0 +1,161 @@ +/* +Copyright IBM Corp. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package sherdlock + +import ( + "context" + "maps" + "sync" + "testing" + "time" + + "github.com/LFDT-Panurus/panurus/token/services/selector/testutils" + "github.com/LFDT-Panurus/panurus/token/services/storage/db/driver" + "github.com/LFDT-Panurus/panurus/token/services/utils/types/transaction" + token2 "github.com/LFDT-Panurus/panurus/token/token" + "github.com/hyperledger-labs/fabric-smart-client/pkg/utils/errors" + "github.com/stretchr/testify/require" +) + +// countingLocker decorates a Locker to record, per token ID, how many times +// Lock was attempted and how many of those attempts lost the race (returned +// driver.ErrTokenAlreadyLocked). It exists only for TestHotTokenContention, +// to turn the incident reported in #2395 (a handful of tokens absorbing the +// overwhelming majority of lock conflicts) into a number a test can assert +// on and a PR description can cite as a baseline. +type countingLocker struct { + Locker + mu sync.Mutex + attempts map[token2.ID]int + conflicts map[token2.ID]int +} + +func newCountingLocker(l Locker) *countingLocker { + return &countingLocker{ + Locker: l, + attempts: make(map[token2.ID]int), + conflicts: make(map[token2.ID]int), + } +} + +// Lock records the attempt against tokenID, then delegates to the wrapped +// Locker. A lost race (errors.Is(err, driver.ErrTokenAlreadyLocked)) is +// additionally counted as a conflict. +func (c *countingLocker) Lock(ctx context.Context, tokenID *token2.ID, consumerTxID transaction.ID, walletID string) error { + err := c.Locker.Lock(ctx, tokenID, consumerTxID, walletID) + + c.mu.Lock() + c.attempts[*tokenID]++ + if errors.Is(err, driver.ErrTokenAlreadyLocked) { + c.conflicts[*tokenID]++ + } + c.mu.Unlock() + + return err +} + +// snapshot returns a defensive copy of the current attempts and conflicts, +// safe to read after all concurrent Lock calls have completed. +func (c *countingLocker) snapshot() (attempts, conflicts map[token2.ID]int) { + c.mu.Lock() + defer c.mu.Unlock() + + attempts = make(map[token2.ID]int, len(c.attempts)) + maps.Copy(attempts, c.attempts) + conflicts = make(map[token2.ID]int, len(c.conflicts)) + maps.Copy(conflicts, c.conflicts) + + return attempts, conflicts +} + +// startManagersWithLockCounters is like startManagers, but wraps each +// replica's Locker in a countingLocker and returns the counters alongside +// the replicas, so a test can aggregate lock-conflict distribution across +// all replicas once the concurrent workload has finished. +func startManagersWithLockCounters(t *testing.T, number int, backoff time.Duration, maxRetries int) ([]testutils.EnhancedManager, []*countingLocker, func()) { + t.Helper() + terminate, pgConnStr := startContainer(t) + replicas := make([]testutils.EnhancedManager, number) + counters := make([]*countingLocker, number) + + for i := range number { + var counter *countingLocker + replica, err := createManagerWithLocker(t, pgConnStr, backoff, maxRetries, func(l Locker) Locker { + counter = newCountingLocker(l) + + return counter + }) + require.NoError(t, err) + replicas[i] = replica + counters[i] = counter + } + + return replicas, counters, terminate +} + +// TestHotTokenContention reproduces the incident reported in #2395 against a +// real Postgres-backed selector: a wallet with a few small tokens and one +// much larger, rotating hot token, and far more concurrent requests than +// tokens. It records, per token ID across all replicas, how many lock +// attempts were made and how many lost the race, so contention can be +// quantified rather than merely observed as an occasional flaky failure. +// +// The only hard assertion is the functional invariant that must hold no +// matter how contention is distributed: total demand exactly equals the +// wallet's total balance, so any Select error is spurious (contention- +// induced), never a genuine insufficient-funds. Phases 3-5 of #2395 are +// expected to reduce the conflict counts and the max single-token share +// logged here; this test's log output is the baseline they should be +// compared against. +func TestHotTokenContention(t *testing.T) { + replicas, counters, terminate := startManagersWithLockCounters(t, 3, 2*time.Second, 60) + defer terminate() + + testutils.TestHotTokenContention(t, replicas) + + totalAttempts, totalConflicts := 0, 0 + distinctTokensAttempted := make(map[token2.ID]struct{}) + perTokenConflicts := make(map[token2.ID]int) + for _, c := range counters { + attempts, conflicts := c.snapshot() + for id, n := range attempts { + totalAttempts += n + distinctTokensAttempted[id] = struct{}{} + } + for id, n := range conflicts { + totalConflicts += n + perTokenConflicts[id] += n + } + } + + maxConflictsForToken := 0 + for _, n := range perTokenConflicts { + if n > maxConflictsForToken { + maxConflictsForToken = n + } + } + + maxShare := 0.0 + if totalConflicts > 0 { + maxShare = float64(maxConflictsForToken) / float64(totalConflicts) + } + conflictRate := 0.0 + if totalAttempts > 0 { + conflictRate = float64(totalConflicts) / float64(totalAttempts) + } + + // conflictRate is the number that mirrors the stress report's headline + // figure (95.2% of lock violations): here it is the share of every lock + // attempt, across all replicas, that lost the race. maxShare is diluted + // by design: deleteTokensAndStoreChange mints a fresh token ID each time + // the hot token is spent, so the same *lineage* of change stays hot + // across the run without any single ID accumulating a large share. + t.Logf( + "#2395 contention baseline: distinct tokens attempted=%d, total lock attempts=%d, total conflicts=%d, conflict rate=%.2f, distinct tokens conflicted=%d, max single-token conflict share=%.2f", + len(distinctTokensAttempted), totalAttempts, totalConflicts, conflictRate, len(perTokenConflicts), maxShare, + ) +} diff --git a/token/services/selector/sherdlock/inmemory/locker.go b/token/services/selector/sherdlock/inmemory/locker.go index df567c0ce7..032c7f7cae 100644 --- a/token/services/selector/sherdlock/inmemory/locker.go +++ b/token/services/selector/sherdlock/inmemory/locker.go @@ -10,9 +10,11 @@ import ( "context" "time" + simpleinmemory "github.com/LFDT-Panurus/panurus/token/services/selector/simple/inmemory" "github.com/LFDT-Panurus/panurus/token/services/storage/db/driver" "github.com/LFDT-Panurus/panurus/token/services/utils/types/transaction" "github.com/LFDT-Panurus/panurus/token/token" + "github.com/hyperledger-labs/fabric-smart-client/pkg/utils/errors" ) type Locker interface { @@ -32,8 +34,16 @@ func NewLocker(l Locker) *locker { return &locker{Locker: l} } +// Lock delegates to the underlying in-memory locker and normalizes a lost +// lock race to driver.ErrTokenAlreadyLocked, the same sentinel the SQL-backed +// TokenLockStore returns for the equivalent case (see common/tokenlock.go). +// Without this, selector.go could not tell "lost the race" from a real error +// on this backend, since simple/inmemory.AlreadyLockedError is a distinct value. func (l *locker) Lock(ctx context.Context, tokenID *token.ID, consumerTxID transaction.ID, walletID string) error { _, err := l.Locker.Lock(ctx, walletID, tokenID, consumerTxID, false) + if err != nil && errors.Is(err, simpleinmemory.AlreadyLockedError) { + return driver.ErrTokenAlreadyLocked + } return err } diff --git a/token/services/selector/sherdlock/manager_test.go b/token/services/selector/sherdlock/manager_test.go index 57715939e9..6f37e40d9a 100644 --- a/token/services/selector/sherdlock/manager_test.go +++ b/token/services/selector/sherdlock/manager_test.go @@ -98,6 +98,17 @@ func startManagers(t *testing.T, number int, backoff time.Duration, maxRetries i func createManager(t *testing.T, pgConnStr string, backoff time.Duration, maxRetries int) (testutils.EnhancedManager, error) { t.Helper() + + return createManagerWithLocker(t, pgConnStr, backoff, maxRetries, nil) +} + +// createManagerWithLocker is like createManager, but lets the caller decorate +// the raw Locker before it is handed to NewManager. wrap may be nil. This +// exists so TestHotTokenContention can wrap the Locker in a countingLocker to +// record per-token lock attempts and conflicts, without duplicating the rest +// of the manager wiring. +func createManagerWithLocker(t *testing.T, pgConnStr string, backoff time.Duration, maxRetries int, wrap func(Locker) Locker) (testutils.EnhancedManager, error) { + t.Helper() d := postgres.NewDriverWithDbProvider(multiplexed.MockTypeConfig(postgres2.Persistence, postgres2.Config{ TablePrefix: "test", DataSource: pgConnStr, @@ -116,9 +127,14 @@ func createManager(t *testing.T, pgConnStr string, backoff time.Duration, maxRet return nil, errors.Join(err, tokenDB.Close()) } + var locker Locker = lockDB + if wrap != nil { + locker = wrap(locker) + } + m := NewMetrics(&disabled.Provider{}) fetcher := newMixedFetcher(tokenDB.(dbtest.TestTokenDB), m, 0, 0, 0) - manager := NewManager(fetcher, lockDB, testutils.TokenQuantityPrecision, backoff, maxRetries, 0, 0, m) + manager := NewManager(fetcher, locker, testutils.TokenQuantityPrecision, backoff, maxRetries, 0, 0, m) return testutils.NewEnhancedManager(t, manager, tokenDB.(dbtest.TestTokenDB)), nil } diff --git a/token/services/selector/sherdlock/metrics.go b/token/services/selector/sherdlock/metrics.go index 81bd90aed5..2ac67eb3dc 100644 --- a/token/services/selector/sherdlock/metrics.go +++ b/token/services/selector/sherdlock/metrics.go @@ -28,6 +28,16 @@ type Metrics struct { SelectionOutcome metrics.Counter // ImmediateRetries tracks the distribution of immediate retry counts per Select() call. ImmediateRetries metrics.Histogram + // LockConflicts counts every lost lock race (TryLock finding a token already + // locked by another process). Deliberately unlabeled by token id or wallet id: + // either would be unbounded cardinality. Per-token attribution belongs in the + // debug-level log line in selector.go and in the `tokendiag locks` command. + LockConflicts metrics.Counter + // DistinctTokensAttempted tracks, per Select() call, how many distinct tokens + // were tried: the lock was won, lost to another process, or denied by the rate + // limiter. This is what distinguishes "one hot token retried many times" from + // "many tokens each contended once". + DistinctTokensAttempted metrics.Histogram } func NewMetrics(p metrics.Provider) *Metrics { @@ -54,5 +64,14 @@ func NewMetrics(p metrics.Provider) *Metrics { Help: "Distribution of immediate retry counts per token selection call", Buckets: []float64{0, 1, 2, 3, 4, 5}, }), + LockConflicts: p.NewCounter(metrics.CounterOpts{ + Name: "lock_conflicts_total", + Help: "Total number of lost lock races (a token was already locked by another process)", + }), + DistinctTokensAttempted: p.NewHistogram(metrics.HistogramOpts{ + Name: "distinct_tokens_attempted", + Help: "Distribution of the number of distinct tokens a lock was attempted on (won, lost, or rate-limited) per token selection call", + Buckets: []float64{1, 2, 5, 10, 25, 50, 100}, + }), } } diff --git a/token/services/selector/sherdlock/selector.go b/token/services/selector/sherdlock/selector.go index 53c582cb4c..c6cc0e899b 100644 --- a/token/services/selector/sherdlock/selector.go +++ b/token/services/selector/sherdlock/selector.go @@ -15,6 +15,7 @@ import ( "github.com/LFDT-Panurus/panurus/token" "github.com/LFDT-Panurus/panurus/token/services/logging" + "github.com/LFDT-Panurus/panurus/token/services/storage/db/driver" "github.com/LFDT-Panurus/panurus/token/services/utils/types/transaction" token2 "github.com/LFDT-Panurus/panurus/token/token" "github.com/hyperledger-labs/fabric-smart-client/pkg/utils/errors" @@ -59,8 +60,13 @@ type StubbornSelector struct { func (m *StubbornSelector) Select(ctx context.Context, ownerFilter token.OwnerFilter, q string, tokenType token2.Type) ([]*token2.ID, token2.Quantity, error) { start := time.Now() + // One set for the whole call: each backoff round runs a fresh inner selection, but + // the histogram reports per-Select() fan-out, so the distinct tokens seen across + // every round are unioned here and observed exactly once. + attempted := collections.NewSet[token2.ID]() + defer observeDistinctTokensAttempted(m.metrics, attempted) for retriesAfterBackoff := 0; retriesAfterBackoff <= m.maxRetriesAfterBackoff; retriesAfterBackoff++ { - if tokens, quantity, err := m.selectWithoutMetrics(ctx, ownerFilter, q, tokenType); err == nil || !errors.Is(err, token.SelectorSufficientButLockedFunds) { + if tokens, quantity, err := m.selectWithoutMetrics(ctx, ownerFilter, q, tokenType, attempted); err == nil || !errors.Is(err, token.SelectorSufficientButLockedFunds) { m.metrics.SelectionDuration.Observe(time.Since(start).Seconds()) if err == nil { m.metrics.SelectionOutcome.With(outcomeLabel, "success").Add(1) @@ -119,9 +125,23 @@ func NewSelector(logger logging.Logger, tokenDB TokenFetcher, lockDB TokenLocker } } +// observeDistinctTokensAttempted records the per-Select() fan-out. It skips the +// observation when no lock was ever attempted: the early bails in selectInternal and the +// plain empty-wallet path reach here with an empty set, and observing 0 would land under +// the histogram's first bucket while still inflating _count - pulling the mean and the +// quantiles toward zero and blurring the very signal this histogram exists to give. No +// attempt is absence of data, not a measurement of zero. +func observeDistinctTokensAttempted(m *Metrics, attempted collections.Set[token2.ID]) { + if n := attempted.Length(); n > 0 { + m.DistinctTokensAttempted.Observe(float64(n)) + } +} + func (s *Selector) Select(ctx context.Context, owner token.OwnerFilter, q string, tokenType token2.Type) ([]*token2.ID, token2.Quantity, error) { start := time.Now() - ids, quantity, immediateRetries, err := s.selectInternal(ctx, owner, q, tokenType) + attempted := collections.NewSet[token2.ID]() + defer observeDistinctTokensAttempted(s.metrics, attempted) + ids, quantity, immediateRetries, err := s.selectInternal(ctx, owner, q, tokenType, attempted) if err != nil { if err2 := s.locker.UnlockAll(ctx); err2 != nil { s.logger.Warnf("failed to unlock tokens after selection error: %v", err2) @@ -143,8 +163,8 @@ func (s *Selector) Select(ctx context.Context, owner token.OwnerFilter, q string } // selectWithoutMetrics is used by StubbornSelector to avoid double-counting metrics. -func (s *Selector) selectWithoutMetrics(ctx context.Context, owner token.OwnerFilter, q string, tokenType token2.Type) ([]*token2.ID, token2.Quantity, error) { - ids, quantity, _, err := s.selectInternal(ctx, owner, q, tokenType) +func (s *Selector) selectWithoutMetrics(ctx context.Context, owner token.OwnerFilter, q string, tokenType token2.Type, attempted collections.Set[token2.ID]) ([]*token2.ID, token2.Quantity, error) { + ids, quantity, _, err := s.selectInternal(ctx, owner, q, tokenType, attempted) if err != nil { if err2 := s.locker.UnlockAll(ctx); err2 != nil { s.logger.Warnf("failed to unlock tokens after selection error: %v", err2) @@ -154,7 +174,11 @@ func (s *Selector) selectWithoutMetrics(ctx context.Context, owner token.OwnerFi return ids, quantity, err } -func (s *Selector) selectInternal(ctx context.Context, owner token.OwnerFilter, q string, tokenType token2.Type) ([]*token2.ID, token2.Quantity, int, error) { +// selectInternal performs one selection attempt. attempted is owned by the caller and +// records every distinct token this attempt tried to lock; a StubbornSelector reuses the +// same set across all of its backoff rounds so that DistinctTokensAttempted is observed +// once per Select() call, counting each distinct token once. +func (s *Selector) selectInternal(ctx context.Context, owner token.OwnerFilter, q string, tokenType token2.Type, attempted collections.Set[token2.ID]) ([]*token2.ID, token2.Quantity, int, error) { if s.isClosed() { return nil, nil, 0, errors.Errorf("selector is already closed") } @@ -199,14 +223,32 @@ func (s *Selector) selectInternal(ctx context.Context, owner token.OwnerFilter, immediateRetries++ tokensLockedByOthersExist = false - } else if locked, lockErr := s.locker.TryLock(ctx, &t.Id, owner.ID()); !locked { - // A rate-limit denial from the locker is a hard stop: abort instead of retrying. - if errors.Is(lockErr, token.SelectorRateLimited) { - return nil, nil, immediateRetries, lockErr - } - s.logger.DebugfContext(ctx, "Tried to lock token [%v], but it was already locked by another process", t) - tokensLockedByOthersExist = true } else { + // Counted once here, before the outcome is known, so a later third + // outcome branch cannot forget to record the attempt. + attempted.Add(t.Id) + if locked, lockErr := s.locker.TryLock(ctx, &t.Id, owner.ID()); !locked { + // A rate-limit denial from the locker is a hard stop: abort instead of retrying. + if errors.Is(lockErr, token.SelectorRateLimited) { + return nil, nil, immediateRetries, lockErr + } + if errors.Is(lockErr, driver.ErrTokenAlreadyLocked) { + // Lost the race: someone else holds this token. This is the + // expected, common case under contention, not a DB error. + s.metrics.LockConflicts.Add(1) + s.logger.DebugfContext(ctx, "Lost lock race on token [%s:%d]: already locked by another process", t.Id.TxId, t.Id.Index) + } else { + // A real store error (not a lock conflict) collapsed into the + // same !locked branch by TryLock. Only the log line separates + // the two: to the caller this still reads as ordinary + // contention, so a store outage surfaces as locked funds + // rather than as an error. See #2395. + s.logger.WarnfContext(ctx, "Failed to lock token [%s:%d]: %v", t.Id.TxId, t.Id.Index, lockErr) + } + tokensLockedByOthersExist = true + + continue + } s.logger.DebugfContext(ctx, "Got the lock on token [%v]", t) q, err := token2.ToQuantity(t.Quantity, s.precision) if err != nil { diff --git a/token/services/selector/sherdlock/selector_test.go b/token/services/selector/sherdlock/selector_test.go index efe4b4308a..8419d2cc74 100644 --- a/token/services/selector/sherdlock/selector_test.go +++ b/token/services/selector/sherdlock/selector_test.go @@ -14,8 +14,10 @@ import ( "github.com/LFDT-Panurus/panurus/token" "github.com/LFDT-Panurus/panurus/token/services/selector/sherdlock" "github.com/LFDT-Panurus/panurus/token/services/selector/sherdlock/mocks" + "github.com/LFDT-Panurus/panurus/token/services/storage/db/driver" token2 "github.com/LFDT-Panurus/panurus/token/token" "github.com/hyperledger-labs/fabric-smart-client/pkg/utils/errors" + metricsa "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/metrics" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -245,3 +247,70 @@ func setupMetricsMocks() (*mocks.FakeProvider, *sherdlock.Metrics) { return metricsProvider, sherdlock.NewMetrics(metricsProvider) } + +// setupNamedHistogramMocks builds a Metrics whose histograms are distinct fakes, keyed +// by metric name, so a test can assert on one histogram without the others' observations +// landing on the same fake. +func setupNamedHistogramMocks() (map[string]*mocks.FakeHistogram, *sherdlock.Metrics) { + mockCounter := &mocks.FakeCounter{} + mockCounter.WithReturns(mockCounter) + histograms := map[string]*mocks.FakeHistogram{} + metricsProvider := &mocks.FakeProvider{} + metricsProvider.NewCounterReturns(mockCounter) + metricsProvider.NewHistogramCalls(func(opts metricsa.HistogramOpts) metricsa.Histogram { + h := &mocks.FakeHistogram{} + h.WithReturns(h) + histograms[opts.Name] = h + + return h + }) + + return histograms, sherdlock.NewMetrics(metricsProvider) +} + +// TestDistinctTokensAttemptedObservedOncePerSelect pins the contract documented on +// Metrics.DistinctTokensAttempted and in docs/development/metrics.md: exactly one +// observation per Select() call, carrying the number of distinct tokens the whole call +// tried to lock. A StubbornSelector runs its inner selection once per backoff round, so +// observing inside that inner call would emit one sample per round, each counting only +// that round's tokens - inflating the sample count and understating per-call fan-out, +// which is the opposite of what #2395 needs the histogram for. +func TestDistinctTokensAttemptedObservedOncePerSelect(t *testing.T) { + const retriesAfterBackoff = 2 + + histograms, metrics := setupNamedHistogramMocks() + + mockFetcher := &mocks.FakeTokenFetcher{} + mockLocker := &mocks.FakeTokenLocker{} + // Two tokens, both always held by someone else: selection can never complete, so + // the stubborn selector exhausts every backoff round and calls its inner selection + // retriesAfterBackoff+1 times. + mockFetcher.UnspentTokensIteratorByCalls(func(context.Context, string, token2.Type) (sherdlock.Iterator[*token2.UnspentTokenInWallet], error) { + it := &mocks.FakeIterator[*token2.UnspentTokenInWallet]{} + it.NextReturnsOnCall(0, &token2.UnspentTokenInWallet{ + Id: token2.ID{TxId: "tx1", Index: 0}, Type: "ABC", Quantity: "100", + }, nil) + it.NextReturnsOnCall(1, &token2.UnspentTokenInWallet{ + Id: token2.ID{TxId: "tx2", Index: 0}, Type: "ABC", Quantity: "100", + }, nil) + it.NextReturns(nil, nil) + + return it, nil + }) + mockLocker.TryLockReturns(false, driver.ErrTokenAlreadyLocked) + + s := sherdlock.NewStubbornSelector(sherdlock.Logger(), mockFetcher, mockLocker, 64, time.Millisecond, retriesAfterBackoff, metrics) + _, _, err := s.Select(t.Context(), &unitTestMockOwnerFilter{id: "alice"}, "50", "ABC") + require.Error(t, err) + + // Precondition: the inner selection really did run more than once, otherwise this + // test would pass even with the observation left in the per-attempt path. + require.Greater(t, mockFetcher.UnspentTokensIteratorByCallCount(), retriesAfterBackoff, + "expected the stubborn selector to retry, so that per-attempt observation would be visible") + + attempted := histograms["distinct_tokens_attempted"] + require.NotNil(t, attempted, "distinct_tokens_attempted histogram was never created") + require.Equal(t, 1, attempted.ObserveCallCount(), "DistinctTokensAttempted must be observed once per Select() call") + // The count is an exact small integer, so an epsilon comparison would be noise. + assert.InDelta(t, 2, attempted.ObserveArgsForCall(0), 0, "both distinct tokens must be counted once each, across all retries") +} diff --git a/token/services/selector/testutils/test_cases.go b/token/services/selector/testutils/test_cases.go index 9566161336..590ab1971c 100644 --- a/token/services/selector/testutils/test_cases.go +++ b/token/services/selector/testutils/test_cases.go @@ -78,6 +78,36 @@ func TestSufficientTokensBigDenominationsManyReplicas(t *testing.T, replicas []E assert.Empty(t, errs) } +// TestHotTokenContention mirrors the incident reported in #2395: a wallet +// with a few small tokens and one much larger one, and far more concurrent +// requests than tokens. Every request asks for CHF1, satisfiable either by a +// small token directly or by the large one — which recycles most of its +// value back as a freshly-minted token via deleteTokensAndStoreChange, so at +// any instant there is exactly one "big" token in the pool. That is the hot +// token every losing goroutine keeps re-targeting after a lost lock race, +// absent a per-attempt blacklist (mechanism 1 in #2395) or an anti-join +// against already-locked tokens (mechanism 3). +// +// Unlike the other cases in this file, callers are expected to also inspect +// lock-conflict counts (see sherdlock's TestHotTokenContention, which wraps +// the Locker to record them) and assert on their distribution — this +// function only asserts the functional invariant that must hold regardless +// of how contention is distributed: total demand exactly matches the wallet +// balance, so no error here can be a genuine insufficient-funds; any error +// is spurious, caused by contention. +func TestHotTokenContention(t *testing.T, replicas []EnhancedManager) { + small := newToken(1) + big := newToken(296) + unspentTokens := createDefaultTokens(append(collections.Repeat(small, 4), big)...) + err := storeTokens(replicas[0], unspentTokens) + require.NoError(t, err) + + // 3 replicas x 100 requests of CHF1 = CHF300, exactly the total balance. + item := newToken(1) + errs := parallelSelect(t, replicas, collections.Repeat(item, 100)) + assert.Empty(t, errs, "spurious insufficient-funds under lock contention (#2395)") +} + func TestInsufficientTokensOneReplica(t *testing.T, replica EnhancedManager) { // Create 2 tokens of value CHF1 each (total CHF2) item := newToken(1) diff --git a/token/services/storage/db/dbtest/tokenlock.go b/token/services/storage/db/dbtest/tokenlock.go index 0d6baa4299..8cfb734b4a 100644 --- a/token/services/storage/db/dbtest/tokenlock.go +++ b/token/services/storage/db/dbtest/tokenlock.go @@ -15,7 +15,6 @@ import ( driver3 "github.com/LFDT-Panurus/panurus/token/services/storage/db/driver" "github.com/LFDT-Panurus/panurus/token/services/utils" "github.com/LFDT-Panurus/panurus/token/token" - fscerrors "github.com/hyperledger-labs/fabric-smart-client/pkg/utils/errors" "github.com/stretchr/testify/require" ) @@ -66,6 +65,7 @@ var tokenLockDBCases = []struct { {"TestKeepSiblingIndices", TestKeepSiblingIndices}, {"TestReleaseOnAgedLease", TestReleaseOnAgedLease}, {"TestKeepFreshPendingLock", TestKeepFreshPendingLock}, + {"TestListLocks", TestListLocks}, } func TestFully(t *testing.T, tokenDB driver3.TokenStore, tokenLockDB driver3.TokenLockStore, tokenTransactionDB driver3.TokenTransactionStore) { @@ -224,6 +224,86 @@ func TestKeepFreshPendingLock(t *testing.T, tokenDB driver3.TokenStore, tokenLoc requireLockHeld(t, tokenLockDB, tokenID) } +// TestListLocks verifies that ListLocks reports every held lock together with the +// current status of its consuming transaction, and that a released lock disappears +// from the listing. This is the diagnostic reader added for #2395. +func TestListLocks(t *testing.T, tokenDB driver3.TokenStore, tokenLockDB driver3.TokenLockStore, tokenTransactionDB driver3.TokenTransactionStore) { + ctx := t.Context() + held := token.ID{TxId: "producer", Index: 0} + toRelease := token.ID{TxId: "producer", Index: 1} + orphaned := token.ID{TxId: "producer", Index: 2} + + addTokenRequest(t, tokenTransactionDB, "producer") + addTokenRequest(t, tokenTransactionDB, "pending-consumer") + addTokenRequest(t, tokenTransactionDB, "settled-consumer") + storeTokens(t, tokenDB, "producer", 0, 1, 2) + require.NoError(t, tokenLockDB.Lock(ctx, &held, "pending-consumer", "owner1")) + require.NoError(t, tokenLockDB.Lock(ctx, &toRelease, "settled-consumer", "owner1")) + // No token request is registered for "assembling-consumer": the selector takes + // the lock while the transfer is still being assembled, so a lock can legitimately + // precede its consumer's requests row. Only consumer_tx_id has no foreign key, so + // this shape is representable, and ListLocks LEFT JOINs requests to keep it + // visible - dropping it would hide exactly the in-flight and abandoned locks + // #2395 is about. + require.NoError(t, tokenLockDB.Lock(ctx, &orphaned, "assembling-consumer", "owner1")) + + locks, err := tokenLockDB.ListLocks(ctx) + require.NoError(t, err) + require.Len(t, locks, 3) + + byTokenID := map[token.ID]driver3.LockRecord{} + for _, l := range locks { + byTokenID[l.TokenID] = l + } + + pending, ok := byTokenID[held] + require.True(t, ok, "lock on %s should be reported", held) + require.Equal(t, "pending-consumer", pending.ConsumerTxID) + require.NotNil(t, pending.Status) + require.Equal(t, driver3.Pending, *pending.Status) + + // A lock whose consumer has no requests row is reported with a nil Status, + // which is the documented contract of driver.LockRecord.Status. + orphan, ok := byTokenID[orphaned] + require.True(t, ok, "lock on %s should be reported even with no consumer requests row", orphaned) + require.Equal(t, "assembling-consumer", orphan.ConsumerTxID) + require.Nil(t, orphan.Status, "consumer has no requests row, so status must be nil") + + // Mark the second consumer settled: the lock is still held (this is the leak + // TestListLocks is meant to surface - see mechanism 4 in #2395), and ListLocks + // must show its consumer's terminal status rather than silently dropping it. + require.NoError(t, tokenTransactionDB.SetStatus(ctx, "settled-consumer", driver3.Confirmed, "")) + + locks, err = tokenLockDB.ListLocks(ctx) + require.NoError(t, err) + require.Len(t, locks, 3) + byTokenID = map[token.ID]driver3.LockRecord{} + for _, l := range locks { + byTokenID[l.TokenID] = l + } + settled, ok := byTokenID[toRelease] + require.True(t, ok, "lock on %s should still be reported after its consumer settles", toRelease) + require.NotNil(t, settled.Status) + require.Equal(t, driver3.Confirmed, *settled.Status) + + // Releasing the lock removes it from the listing. + require.NoError(t, tokenLockDB.UnlockByTxID(ctx, "settled-consumer")) + locks, err = tokenLockDB.ListLocks(ctx) + require.NoError(t, err) + require.Len(t, locks, 2) + require.NotContains(t, collectTokenIDs(locks), toRelease) +} + +// collectTokenIDs returns the token IDs of the given lock records. +func collectTokenIDs(locks []driver3.LockRecord) []token.ID { + ids := make([]token.ID, 0, len(locks)) + for _, l := range locks { + ids = append(ids, l.TokenID) + } + + return ids +} + // addTokenRequest registers a token request for txID, so that its status can later be // moved to a terminal one with SetStatus. func addTokenRequest(t *testing.T, tokenTransactionDB driver3.TokenTransactionStore, txID string) { @@ -260,18 +340,13 @@ func storeTokens(t *testing.T, tokenDB driver3.TokenStore, txID string, indices require.NoError(t, tx.Commit()) } -// requireLockHeld asserts that the lock on tokenID survived cleanup. The store exposes -// no read API, so the probe is a second Lock on the same token: the (tx_id, idx) -// primary key rejects it for as long as the row is there. -// We assert on driver3.ErrTokenAlreadyLocked specifically so that an unrelated Lock -// failure (e.g. a future rule that rejects locks on Deleted producers) does not make -// "Keep" tests pass vacuously. +// requireLockHeld asserts that the lock on tokenID survived cleanup, via +// TokenLockStore.ListLocks (see #2395) rather than probing with a second Lock call. func requireLockHeld(t *testing.T, tokenLockDB driver3.TokenLockStore, tokenID token.ID) { t.Helper() - err := tokenLockDB.Lock(t.Context(), &tokenID, "probe-"+tokenID.String(), "owner1") - require.True(t, fscerrors.Is(err, driver3.ErrTokenAlreadyLocked), - "lock on token %s should still be held (want ErrTokenAlreadyLocked, got %v)", tokenID, err) + require.True(t, lockExists(t, tokenLockDB, tokenID), + "lock on token %s should still be held", tokenID) } // requireLockReleased asserts that cleanup collected the lock on tokenID: the row is @@ -279,6 +354,21 @@ func requireLockHeld(t *testing.T, tokenLockDB driver3.TokenLockStore, tokenID t func requireLockReleased(t *testing.T, tokenLockDB driver3.TokenLockStore, tokenID token.ID) { t.Helper() - require.NoError(t, tokenLockDB.Lock(t.Context(), &tokenID, "probe-"+tokenID.String(), "owner1"), + require.False(t, lockExists(t, tokenLockDB, tokenID), "lock on token %s should have been released", tokenID) } + +// lockExists reports whether tokenID appears among the currently held locks. +func lockExists(t *testing.T, tokenLockDB driver3.TokenLockStore, tokenID token.ID) bool { + t.Helper() + + locks, err := tokenLockDB.ListLocks(t.Context()) + require.NoError(t, err) + for _, l := range locks { + if l.TokenID == tokenID { + return true + } + } + + return false +} diff --git a/token/services/storage/db/driver/token.go b/token/services/storage/db/driver/token.go index 51c24b94fb..53a5ec78a7 100644 --- a/token/services/storage/db/driver/token.go +++ b/token/services/storage/db/driver/token.go @@ -342,6 +342,21 @@ type TokenNotifier interface { UnsubscribeAll() error } +// LockRecord describes a single held token lock, joined with the terminal-status +// view of its consuming transaction. Status is nil when the consuming transaction has +// no matching row in the requests table (should not normally happen, since a lock is +// only ever created for a transaction that has one). +type LockRecord struct { + // TokenID is the locked token (the (tx_id, idx) of the transaction that created it). + TokenID token.ID + // ConsumerTxID is the transaction attempting to spend TokenID. + ConsumerTxID transaction.ID + // CreatedAt is when the lock was taken (see LockAt). + CreatedAt time.Time + // Status is the current status of ConsumerTxID, or nil if unknown. + Status *TxStatus +} + // TokenLockStore enforces that a token be used only by one process // A housekeeping job can clean up expired locks (e.g. created_at is more than 5 minutes ago) in order to: // - avoid that the table grows infinitely @@ -360,6 +375,11 @@ type TokenLockStore interface { LockAt(ctx context.Context, tokenID *token.ID, consumerTxID transaction.ID, walletID string, createdAt time.Time) error // UnlockByTxID unlocks all tokens locked by the consumer TX UnlockByTxID(ctx context.Context, consumerTxID transaction.ID) error + // ListLocks returns every currently held lock, joined with the status of its + // consuming transaction. It is a read-only diagnostic: it exists so that operators + // and tests can inspect lock state without racing a second Lock call against the + // primary key (see #2395). It does no filtering; callers select/rank as needed. + ListLocks(ctx context.Context) ([]LockRecord, error) // Cleanup removes stale token locks. A lock is stale when either: // 1. The *consuming* transaction (the one trying to spend the token, // identified by consumer_tx_id) has reached a terminal failure status diff --git a/token/services/storage/db/sql/common/tokenlock.go b/token/services/storage/db/sql/common/tokenlock.go index 972bfa2411..df1467307d 100644 --- a/token/services/storage/db/sql/common/tokenlock.go +++ b/token/services/storage/db/sql/common/tokenlock.go @@ -20,6 +20,7 @@ import ( "github.com/LFDT-Panurus/panurus/token/services/utils/types/transaction" "github.com/LFDT-Panurus/panurus/token/token" "github.com/hyperledger-labs/fabric-smart-client/pkg/utils/errors" + "github.com/hyperledger-labs/fabric-smart-client/platform/common/utils/collections/iterators" fscdriver "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/storage/driver" common2 "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/storage/driver/common" "github.com/hyperledger-labs/fabric-smart-client/platform/view/services/storage/driver/sql/common" @@ -104,6 +105,90 @@ func (db *TokenLockStore) UnlockByTxID(ctx context.Context, consumerTxID transac return err } +// ListLocks returns every currently held lock, joined with the status of its consuming +// transaction. It reuses the same TokenLocks/Requests join as IsStaleLock/Cleanup, so +// the notion of "consuming transaction" stays consistent across the diagnostic reader +// and the actual expiry logic. See driver.TokenLockStore.ListLocks and #2395. +func (db *TokenLockStore) ListLocks(ctx context.Context) ([]driver.LockRecord, error) { + tokenLocks, tokenRequests := q.Table(db.Table.TokenLocks), q.Table(db.Table.Requests) + + query, args := q.Select(). + Fields( + tokenLocks.Field("consumer_tx_id"), tokenLocks.Field("tx_id"), tokenLocks.Field("idx"), + tokenRequests.Field("status"), tokenLocks.Field("created_at"), + ). + From(tokenLocks.Join(tokenRequests, cond.Cmp(tokenLocks.Field("consumer_tx_id"), "=", tokenRequests.Field("tx_id")))). + Format(db.ci) + logging.Debug(logger, query, args) + + rows, err := db.ReadDB.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + it := common.NewIterator(rows, func(entry *driver.LockRecord) error { + var createdAt scannableTime + if err := rows.Scan(&entry.ConsumerTxID, &entry.TokenID.TxId, &entry.TokenID.Index, &entry.Status, &createdAt); err != nil { + return err + } + entry.CreatedAt = createdAt.Time + + return nil + }) + + records, err := iterators.ReadAllValues(it) + if err != nil { + return nil, err + } + // ReadAllValues stops as soon as rows.Next() reports false, and the shared + // rowIterator never consults rows.Err(), so a mid-scan failure (connection reset, + // statement timeout on this unbounded scan) would otherwise return a truncated + // slice with a nil error - and the caller would report fewer locks than are + // actually held, with no indication anything went wrong. + if err := rows.Err(); err != nil { + return nil, errors.Wrap(err, "failed to read lock records") + } + + return records, nil +} + +// scannableTime scans a created_at value regardless of dialect. Postgres hands the +// database/sql driver a native time.Time; sqlite (modernc.org/sqlite) only performs +// that conversion for columns declared DATE/DATETIME/TIMESTAMP, and our shared schema +// declares created_at as TIMESTAMPTZ (deliberately, for timezone-consistent comparison +// against Postgres's NOW() - see Cleanup), so on sqlite the driver hands back the raw +// text it wrote the value as instead. Rather than weaken the shared schema, accept +// either shape here. +type scannableTime struct { + time.Time +} + +// sqliteTimeFormat is the layout modernc.org/sqlite writes a bound time.Time parameter +// as by default (time.Time.String), absent a _time_format DSN option we don't set. +const sqliteTimeFormat = "2006-01-02 15:04:05.999999999 -0700 MST" + +func (s *scannableTime) Scan(src any) error { + switch v := src.(type) { + case time.Time: + s.Time = v + + return nil + case string: + t, err := time.Parse(sqliteTimeFormat, v) + if err != nil { + return errors.Wrapf(err, "cannot parse created_at value [%s]", v) + } + s.Time = t + + return nil + case []byte: + return s.Scan(string(v)) + default: + return errors.Errorf("cannot scan value of type [%T] into time.Time", src) + } +} + func (db *TokenLockStore) GetSchema() string { return fmt.Sprintf(` -- TokenLocks diff --git a/token/services/storage/db/sql/query/cond/condition_test.go b/token/services/storage/db/sql/query/cond/condition_test.go index 1f6a089c10..9a665e4d1e 100644 --- a/token/services/storage/db/sql/query/cond/condition_test.go +++ b/token/services/storage/db/sql/query/cond/condition_test.go @@ -107,6 +107,17 @@ var testMatrix = []testCase{ expectedQuery: "EXISTS (SELECT 1 FROM requests WHERE status = $0)", expectedParams: []common3.Param{3}, }, + { + name: "not exists sub-query", + condition: cond2.NotExists( + q.Select(). + Fields(common3.FieldName("1")). + From(common3.NewTable("requests")). + Where(cond2.Eq("status", 3)), + ), + expectedQuery: "NOT EXISTS (SELECT 1 FROM requests WHERE status = $0)", + expectedParams: []common3.Param{3}, + }, { // A single field and a single value collapses to plain equality. name: "in-tuple, one field one value", diff --git a/token/services/storage/db/sql/query/cond/exists.go b/token/services/storage/db/sql/query/cond/exists.go index 609a8c47ed..d8957489a1 100644 --- a/token/services/storage/db/sql/query/cond/exists.go +++ b/token/services/storage/db/sql/query/cond/exists.go @@ -17,6 +17,7 @@ type subquery interface { type existsCond struct { subquery subquery + negate bool } // Exists creates an EXISTS (SELECT ...) condition wrapping the given subquery. @@ -24,7 +25,18 @@ func Exists(sq subquery) Condition { return &existsCond{subquery: sq} } +// NotExists creates a NOT EXISTS (SELECT ...) condition wrapping the given +// subquery. NOT EXISTS renders identically on every dialect, so - per the +// "Adding a condition" checklist in docs/development/sql-query-dsl.md - this +// implements WriteString directly rather than adding a CondInterpreter method. +func NotExists(sq subquery) Condition { + return &existsCond{subquery: sq, negate: true} +} + func (e *existsCond) WriteString(ci common.CondInterpreter, sb common.Builder) { + if e.negate { + sb.WriteString("NOT ") + } sb.WriteString("EXISTS (") e.subquery.FormatTo(ci, sb) sb.WriteRune(')')