sherdlock: add lock-contention diagnostics and a reproducible baseline test (Phases 1-2 of #2395)

Makefile

@@ -42,7 +42,7 @@ TOP = .
 include $(TOP)/checks.mk
 
 # Define all Go module directories
-GO_MODULES := . integration token/services/storage/db/kvs/hashicorp cmd/artifactgen cmd/tokengen cmd/token_validation_service cmd/profiler cmd/skicleanup cmd/node x/token/services/network/evm
+GO_MODULES := . integration token/services/storage/db/kvs/hashicorp cmd/artifactgen cmd/tokengen cmd/token_validation_service cmd/profiler cmd/skicleanup cmd/tokendiag cmd/node x/token/services/network/evm
 TIDY_GO_MODULES := $(GO_MODULES) tools
 
 # include fabricx target
@@ -219,6 +219,11 @@ artifactgen:
 skicleanup:
 	@cd ./cmd/skicleanup/; CGO_ENABLED=0 go install github.com/LFDT-Panurus/panurus/cmd/skicleanup
 
+.PHONY: tokendiag
+# install tokendiag tool (must build without cgo; see #1445)
+tokendiag:
+	@cd ./cmd/tokendiag/; CGO_ENABLED=0 go install github.com/LFDT-Panurus/panurus/cmd/tokendiag
+
 .PHONY: traceinspector
 # install traceinspector tool
 traceinspector:

cmd/tokendiag/README.md

@@ -0,0 +1,165 @@
+# tokendiag
+
+`tokendiag` is a diagnostic command-line tool for inspecting token-selector state in a
+Panurus token database. It was added for
+[#2395](https://github.com/LFDT-Panurus/panurus/issues/2395), a stress load test that
+showed severe lock contention on a small number of hot tokens.
+
+## Build
+
+```bash
+make tokendiag
+```
+
+The binary is installed to `$GOPATH/bin/tokendiag`.
+
+## Commands
+
+### `config example`
+
+Prints a fully-annotated YAML configuration file to stdout. Use this to bootstrap a new
+configuration:
+
+```bash
+tokendiag config example > config.yaml
+```
+
+No flags required.
+
+### `locks`
+
+Reads every currently held row in the `token_locks` table, joined with the status of
+its consuming transaction, and reports:
+
+- every held lock, with its age and the consumer's status, oldest first;
+- locks whose consumer has already reached a **terminal** status (`Confirmed`,
+  `Deleted`, or `Orphan`) — these are **leaked** locks: nothing on the success path
+  released them, so they sit until the next lease-age sweep (see mechanism 4 in #2395);
+- a summary line (total locks, leaked count, oldest age) suitable for scripting.
+
+This is a **read-only** operation. No data is modified or deleted.
+
+```bash
+tokendiag locks --config <path-to-config.yaml>
+```
+
+**Flags:**
+
+| Flag | Required | Default | Description |
+|------|----------|---------|-------------|
+| `--config` | Yes | — | Path to the YAML configuration file |
+
+**Output format:**
+
+```
+--- Held locks (oldest first) ---
+  token=<tx_id>:<idx> consumer_tx_id=<tx_id> age=<duration> status=<status>[  [LEAKED: ...]]
+
+--- Summary ---
+  Total locks held         : <n>
+  Leaked (terminal consumer): <n>
+  Oldest lock age           : <duration>
+```
+
+A single snapshot cannot distinguish "one token repeatedly re-contended" from "one
+token held a long time" — that comparison requires running `locks` more than once and
+diffing. The ranking here is by lock age only.
+
+## Configuration
+
+The tool reads a YAML file that describes how to connect to the target database.
+Generate a starter file with:
+
+```bash
+tokendiag config example > config.yaml
+```
+
+### SQLite example
+
+```yaml
+driver: sqlite
+dataSource: /var/lib/panurus/node/data.db
+tablePrefix: ""
+skipPrefix: false
+tableNames: {}
+tableNameParams: []
+```
+
+### PostgreSQL example
+
+```yaml
+driver: postgres
+dataSource: "host=db.example.com port=5432 user=panurus password=secret dbname=panurus sslmode=require"
+tablePrefix: "prod_"
+skipPrefix: false
+tableNames: {}
+tableNameParams: []
+```
+
+### Table name params (network / channel / namespace)
+
+If the Panurus node was started with a non-empty TMS identity — network, channel
+and/or namespace — those values were passed as params when the node derived its own
+table names, and become part of every table name alongside `tablePrefix`. Set the same
+values here, in the same order (network, channel, namespace), so `tokendiag` resolves
+the same tables:
+
+```yaml
+driver: postgres
+dataSource: "postgres://user:pass@localhost:5432/panurus?sslmode=disable"
+tablePrefix: ""
+skipPrefix: false
+tableNames: {}
+tableNameParams: ["mynetwork", "mychannel", "mynamespace"]
+```
+
+Leave `tableNameParams` empty if the node was started without any of these
+identifiers. See [`docs/services/storage.md`](../../docs/services/storage.md#table-name-customisation)
+for the escaping rules that apply to each param.
+
+### Skipping the prefix
+
+If the Panurus node was started with `token.storage.skipPrefix: true`, set the same
+flag here so the tool resolves the same unprefixed table names:
+
+```yaml
+driver: postgres
+dataSource: "postgres://user:pass@localhost:5432/panurus?sslmode=disable"
+tablePrefix: ""
+skipPrefix: true
+tableNames: {}
+tableNameParams: []
+```
+
+### Table name overrides
+
+If the Panurus node was started with non-default table names (using the
+`token.storage.tableNames` config option), set the same overrides here so the tool
+connects to the correct tables. The `locks` command reads `tkn_locks` and `requests`:
+
+```yaml
+driver: postgres
+dataSource: "postgres://user:pass@localhost:5432/panurus?sslmode=disable"
+tablePrefix: ""
+skipPrefix: false
+tableNames:
+  tkn_locks: my_token_locks
+tableNameParams: []
+```
+
+## Environment variables
+
+Configuration values can be overridden with environment variables prefixed `CORE_`,
+using `_` in place of `.`:
+
+```bash
+CORE_DATASOURCE="postgres://..." tokendiag locks --config config.yaml
+```
+
+A list value is supplied as a single comma-separated variable:
+
+```bash
+CORE_TABLENAMEPARAMS="testnetwork,testchannel,tokenns" tokendiag locks --config config.yaml
+```
+
+`tableNames` is a map and cannot be set this way; put it in the config file.

cmd/tokendiag/cobra/config/cmd.go

@@ -0,0 +1,98 @@
+/*
+Copyright IBM Corp. All Rights Reserved.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+// Package config provides CLI commands for working with tokendiag configuration.
+package config
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+// exampleConfig is a fully-annotated YAML configuration that users can adapt.
+//
+//nolint:gosec
+const exampleConfig = `# tokendiag configuration file
+#
+# driver selects the database backend.
+# Supported values: "sqlite", "postgres"
+driver: postgres
+
+# dataSource is the DSN (Data Source Name) passed directly to the database driver.
+#
+# PostgreSQL DSN formats:
+#   URL format:  "postgres://user:pass@host:5432/dbname?sslmode=disable"
+#   Key=value:   "host=localhost port=5432 user=panurus password=secret dbname=panurus sslmode=require"
+#
+# SQLite format (file path):
+#   dataSource: /var/lib/panurus/node/data.db
+dataSource: "postgres://user:pass@localhost:5432/panurus?sslmode=disable"
+
+# tablePrefix is the optional prefix that was used when the Panurus node created
+# its database tables. Leave empty if the node was configured without a prefix.
+tablePrefix: ""
+
+# skipPrefix controls whether the FSC-generated prefix is omitted from all table
+# names. Set this to true when the Panurus node was started with
+# token.storage.skipPrefix: true. Default is false.
+skipPrefix: false
+
+# tableNames is an optional map of short-code overrides for SQL table names.
+# Each key is a canonical short code and the value is the replacement short code
+# that will be used when generating the final SQL table name. The FSC-generated
+# prefix and params are still applied around the replacement value (unless
+# skipPrefix is true). Unknown keys produce a warning and are ignored.
+#
+# The locks command reads the tkn_locks and requests tables:
+#   tkn_locks -> fsc_tkn_locks_<prefix>_<params>
+#   requests  -> fsc_requests_<prefix>_<params>
+#
+# Example — rename the lock table:
+# tableNames:
+#   tkn_locks: my_token_locks
+tableNames: {}
+
+# tableNameParams carries the TMS identity (network, channel, namespace, in that
+# order) that the Panurus node passed when it derived its own table names. These
+# become part of every table name alongside tablePrefix, so they must match the
+# node's configuration exactly. Leave empty if the node was started without any of
+# these identifiers.
+#
+# Example:
+# tableNameParams: ["mynetwork", "mychannel", "mynamespace"]
+tableNameParams: []
+`
+
+// Cmd returns the Cobra Command for the config subcommand group.
+func Cmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "config",
+		Short: "Configuration helpers.",
+		Long:  `Commands for working with the tokendiag configuration file.`,
+	}
+
+	cmd.AddCommand(exampleCmd())
+
+	return cmd
+}
+
+func exampleCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   "example",
+		Short: "Print an annotated example configuration file.",
+		Long: `Print a fully-annotated YAML configuration to stdout.
+
+Redirect the output to a file to create a starting configuration:
+
+  tokendiag config example > config.yaml`,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			_, err := fmt.Fprint(cmd.OutOrStdout(), exampleConfig)
+
+			return err
+		},
+	}
+}

cmd/tokendiag/cobra/locks/cmd.go

@@ -0,0 +1,74 @@
+/*
+Copyright IBM Corp. All Rights Reserved.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+// Package locks provides the tokendiag "locks" diagnostic subcommand: a read-only
+// inspection of the token_locks table, added for #2395 to let an operator answer
+// "which tokens are locked right now, for how long, and is any of that a leak"
+// without racing a second Lock call against the primary key.
+package locks
+
+import (
+	"context"
+	"time"
+
+	"github.com/hyperledger-labs/fabric-smart-client/pkg/utils/errors"
+	"github.com/spf13/cobra"
+)
+
+// Cmd returns the Cobra Command for the locks subcommand.
+func Cmd() *cobra.Command {
+	c := &command{}
+
+	cmd := &cobra.Command{
+		Use:   "locks",
+		Short: "Inspect currently held token locks.",
+		Long: `Reads every currently held row in the token_locks table, joined with the
+status of its consuming transaction, and prints:
+  - every held lock, with its age and the consumer's status;
+  - locks whose consumer has already reached a terminal status (Confirmed,
+    Deleted, or Orphan) - these are leaked locks: nothing released them on
+    settlement, so they will sit until the next lease-age sweep;
+  - a summary line suitable for scripting.
+
+This is a read-only operation. No data is modified or deleted.`,
+		RunE: c.run,
+	}
+
+	flags := cmd.Flags()
+	flags.StringVar(&c.configPath, "config", "", "Path to the YAML configuration file (required)")
+
+	if err := cmd.MarkFlagRequired("config"); err != nil {
+		// MarkFlagRequired only errors if the flag does not exist — this is a programming error.
+		panic(err)
+	}
+
+	return cmd
+}
+
+type command struct {
+	configPath string
+}
+
+func (c *command) run(cmd *cobra.Command, _ []string) error {
+	cmd.SilenceUsage = true
+
+	cfg, err := LoadConfig(c.configPath)
+	if err != nil {
+		return errors.Wrap(err, "failed to load config")
+	}
+
+	stores, err := NewStores(cfg)
+	if err != nil {
+		return errors.Wrap(err, "failed to open stores")
+	}
+	defer func() {
+		if err := stores.Close(); err != nil {
+			cmd.PrintErrf("warning: failed to close stores: %v\n", err)
+		}
+	}()
+
+	return Run(context.Background(), cmd.OutOrStdout(), stores, time.Now())
+}