diff --git a/database_config_files/mariadb/mariadb_default.cnf b/database_config_files/mariadb/mariadb_default.cnf new file mode 100644 index 0000000..33edd7b --- /dev/null +++ b/database_config_files/mariadb/mariadb_default.cnf @@ -0,0 +1,148 @@ +# ============================================================================= +# mariadb_default.cnf - MariaDB Reference/Documentation Configuration +# +# Created: July 2026 +# +# PURPOSE: +# Minimal-necessary starting point, documenting every setting also tuned +# by mariadb_cache.cnf / mariadb_simple_2gbp.cnf / mariadb_tidesdb*.cnf. +# Every line below is commented out and carries the upstream MariaDB +# default value plus a link to the reference documentation, so a reviewer +# can see at a glance what changes each of the other profiles makes +# relative to a stock server -- instead of having to look each one up. +# Mirrors database_config_files/postgresql/postgresql_default.conf so the +# two engines have a directly comparable "everything stock" baseline. +# +# TARGET VERSION: +# MariaDB 12.2.2 -- pinned by taf_run.sh's MARIADB_TARBALL_NAME +# (mariadb-12.2.2-linux-systemd-x86_64.tar.gz). Default values below are +# for this version's documentation set (https://mariadb.com/kb/en/); a few +# defaults changed in recent MariaDB releases relative to older MySQL-era +# assumptions -- e.g. character_set_server moved from latin1 to utf8mb4 +# (10.6+), sync_binlog moved from 0 to 1 (10.5+), and innodb_log_file_size +# was superseded by innodb_redo_log_capacity (10.8+). Verify against the +# KB page for any setting before assuming an older MySQL/MariaDB default +# still applies. +# +# USAGE: +# Point taf.db_config_file at this file for a plain, stock-defaults +# MariaDB run -- i.e. the same role postgresql_default.conf plays for +# sysbench_lua_pgsql.properties. It contributes NO active settings of its +# own (every line below is commented); the only non-stock values a +# benchmark run gets are the ones TAF forces regardless of db_config_file +# (see NOTES below). +# To run one of the tuned profiles instead, point taf.db_config_file at +# mariadb_cache.cnf / mariadb_simple_2gbp.cnf / mariadb_tidesdb*.cnf. +# Diff this file against those to see exactly what each one overrides and +# by how much relative to a stock server. +# +# NOTES: +# - datadir, socket, log-error, and pid-file are always forced by TAF as +# explicit mariadbd CLI flags after --defaults-file= +# (mariadb.pm::_db_start_runtime_server); MariaDB's own CLI-over-config +# precedence means any datadir/socket/log-error/pid-file line in this +# file would be silently ignored, not merely overridden -- do not set +# them here. +# - port is NOT forced by TAF (unlike PostgreSQL's port/listen_addresses, +# which postgres.pm always strips and re-forces). Left unset here, the +# server uses the MariaDB compiled-in default (3306). Set it explicitly +# in this file if a run needs a non-default port. +# - bind-address is left unset (MariaDB default: listen on all +# interfaces), since none of the sibling profiles set it either. +# ============================================================================= + +# A section header is required even though every setting below it is +# commented out -- MariaDB's config parser rejects a file with no [section] +# headers at all ("Config file contains no section headers"), unlike +# PostgreSQL's postgresql.conf format (which has no sections). +[mysqld] + +# --------------------------------------------------------------------------- +# "Mandatory" parameters -- i.e. ones without which the server would not +# start, or would start in a way unusable for a benchmark run. +# +# In practice, MariaDB has NO my.cnf setting that lacks a built-in default -- +# mariadbd starts fine with an empty [mysqld] section, using its compiled-in +# boot values throughout. Nothing below is a case of "no default exists"; the +# only two params in this category are handled outside this file entirely: +# +# - datadir -- forced by TAF via --datadir=, not here. +# - socket -- forced by TAF via --socket=, not here. +# +# No other setting in this file is "required" in the no-default sense the +# question implies. If you want to designate one of the tunables below as +# mandatory-with-no-safe-default for a *new* profile (e.g. innodb_buffer_pool_size +# sized to a specific host's RAM, so the compiled-in 128MB would be wrong for +# a benchmark), leave a TODO here and decide the value per-host: +# +# TODO(you): any profile-specific "must be set explicitly" parameter goes here. +# --------------------------------------------------------------------------- + +# --------------------------------------------------------------------------- +# Memory +# Reference: https://mariadb.com/kb/en/innodb-system-variables/ +# Reference: https://mariadb.com/kb/en/server-system-variables/ +# --------------------------------------------------------------------------- +#innodb_buffer_pool_size = 128M # MariaDB default +#innodb_buffer_pool_instances = 1 # MariaDB default when innodb_buffer_pool_size < 1GB; auto-scales up to 8 above that, capped at innodb_buffer_pool_size/1GB +#key_buffer_size = 128M # MariaDB default (MyISAM/Aria key cache; largely unused with InnoDB as default engine) +#sort_buffer_size = 2M # MariaDB default (per-connection) +#join_buffer_size = 256K # MariaDB default (per-connection) +#tmp_table_size = 16M # MariaDB default +#max_heap_table_size = 16M # MariaDB default + +# --------------------------------------------------------------------------- +# InnoDB Durability / Redo Log +# Reference: https://mariadb.com/kb/en/innodb-system-variables/ +# Reference: https://mariadb.com/kb/en/replication-and-binary-log-system-variables/ +# --------------------------------------------------------------------------- +#innodb_flush_log_at_trx_commit = 1 # MariaDB default (full ACID durability) +#innodb_redo_log_capacity = 100M # MariaDB default (10.8+; supersedes innodb_log_file_size/innodb_log_files_in_group) +#innodb_doublewrite = ON # MariaDB default +#innodb_flush_method = fsync # MariaDB default (O_DIRECT is NOT the stock default, unlike some tuned profiles here) +#sync_binlog = 1 # MariaDB default (CHANGED from 0 in MariaDB 10.5+; log-bin itself is off by default on a standalone, non-replica server) +#log_bin = OFF # MariaDB default (binlog disabled unless server_id + log-bin configured) + +# --------------------------------------------------------------------------- +# I/O +# Reference: https://mariadb.com/kb/en/innodb-system-variables/ +# --------------------------------------------------------------------------- +#innodb_io_capacity = 200 # MariaDB default +#innodb_io_capacity_max = 2000 # MariaDB default (auto = 2 * innodb_io_capacity if unset) +#innodb_read_io_threads = 4 # MariaDB default +#innodb_write_io_threads = 4 # MariaDB default +#innodb_file_per_table = ON # MariaDB default + +# --------------------------------------------------------------------------- +# Connections / Caching +# Reference: https://mariadb.com/kb/en/server-system-variables/ +# --------------------------------------------------------------------------- +#max_connections = 151 # MariaDB default +#table_open_cache = 2000 # MariaDB default +#thread_cache_size = -1 # MariaDB default: autosized from max_connections (roughly max_connections/100, min 0) unless explicitly set +#thread_handling = one-thread-per-connection # MariaDB default (thread pool is opt-in, not the stock mode) +#open_files_limit = 0 # MariaDB default (0 = use the OS/ulimit-derived value; not an explicit cap) + +# --------------------------------------------------------------------------- +# Optimizer +# Reference: https://mariadb.com/kb/en/server-system-variables/#optimizer_switch +# --------------------------------------------------------------------------- +#optimizer_switch = # MariaDB default; see KB page -- too many individual flags to usefully enumerate as a single value here +#innodb_stats_on_metadata = OFF # MariaDB default (CHANGED from ON in older MySQL-derived defaults) +#innodb_autoinc_lock_mode = 2 # MariaDB default (interleaved) + +# --------------------------------------------------------------------------- +# Logging +# Reference: https://mariadb.com/kb/en/server-system-variables/ +# --------------------------------------------------------------------------- +#general_log = OFF # MariaDB default +#slow_query_log = OFF # MariaDB default +#log_error = # forced by TAF via --log-error=, not here +#performance_schema = OFF # MariaDB default (unlike MySQL/Percona, where it is ON by default) + +# --------------------------------------------------------------------------- +# Character Set / Collation +# Reference: https://mariadb.com/kb/en/server-system-variables/ +# --------------------------------------------------------------------------- +#character_set_server = utf8mb4 # MariaDB default (CHANGED from latin1 in MariaDB 10.6+) +#collation_server = utf8mb4_uca1400_ai_ci # MariaDB default (CHANGED in MariaDB 10.10+; verify against the KB page for 12.2 specifically before relying on this exact name) diff --git a/database_config_files/postgresql/postgresql_analytics.conf b/database_config_files/postgresql/postgresql_analytics.conf new file mode 100644 index 0000000..047d74a --- /dev/null +++ b/database_config_files/postgresql/postgresql_analytics.conf @@ -0,0 +1,94 @@ +# ============================================================================= +# postgresql_analytics.conf - PostgreSQL Configuration for Analytical Workloads +# +# Created: June 2026 +# +# PURPOSE: +# Provide tuned PostgreSQL settings for OLAP / analytical workloads +# (HammerDB TPROCH, large aggregation queries). Optimized for query +# throughput, parallel execution, and large sort/hash operations. +# +# TARGET VERSION: +# PostgreSQL 18.4 (see TAF/tests/setup_almalinux10.sh EXPECTED_PG_VERSION +# and postgresql_default.conf for the full stock-default reference). +# +# NOTES: +# - Increase work_mem cautiously: each sort/hash node per-connection +# can use up to work_mem. With many concurrent queries, total memory +# usage = max_connections * max_sort_operations * work_mem. +# - parallel workers are enabled for analytical parallelism. +# ============================================================================= + +# --------------------------------------------------------------------------- +# Memory +# --------------------------------------------------------------------------- +shared_buffers = 4GB +work_mem = 256MB +maintenance_work_mem = 1GB +effective_cache_size = 12GB +temp_buffers = 64MB + +# --------------------------------------------------------------------------- +# WAL / Checkpointing +# --------------------------------------------------------------------------- +wal_buffers = 64MB +checkpoint_completion_target = 0.9 +checkpoint_timeout = 30min +max_wal_size = 8GB +min_wal_size = 2GB +synchronous_commit = off + +# --------------------------------------------------------------------------- +# Parallelism +# --------------------------------------------------------------------------- +max_worker_processes = 16 +max_parallel_workers_per_gather = 4 +max_parallel_workers = 16 +parallel_setup_cost = 100 +parallel_tuple_cost = 0.01 +min_parallel_table_scan_size = 8MB +min_parallel_index_scan_size = 512kB + +# --------------------------------------------------------------------------- +# I/O Workers (PostgreSQL 18+) +# Large sequential scans/sorts benefit from more concurrent prefetch than the +# stock 3 IO workers can service, especially with parallel workers each +# issuing scans concurrently. +# --------------------------------------------------------------------------- +io_method = worker +io_workers = 16 + +# --------------------------------------------------------------------------- +# Connections +# --------------------------------------------------------------------------- +max_connections = 100 + +# --------------------------------------------------------------------------- +# Planner +# --------------------------------------------------------------------------- +random_page_cost = 1.1 +effective_io_concurrency = 200 +default_statistics_target = 500 +enable_hashagg = on +enable_hashjoin = on +enable_sort = on + +# --------------------------------------------------------------------------- +# JIT (PostgreSQL 11+) +# --------------------------------------------------------------------------- +jit = on + +# --------------------------------------------------------------------------- +# Logging (minimal for benchmarking) +# --------------------------------------------------------------------------- +log_min_duration_statement = -1 +log_connections = off +log_disconnections = off +log_checkpoints = off +log_autovacuum_min_duration = -1 + +# --------------------------------------------------------------------------- +# Autovacuum +# --------------------------------------------------------------------------- +autovacuum = on +autovacuum_max_workers = 3 diff --git a/database_config_files/postgresql/postgresql_default.conf b/database_config_files/postgresql/postgresql_default.conf new file mode 100644 index 0000000..cf53e08 --- /dev/null +++ b/database_config_files/postgresql/postgresql_default.conf @@ -0,0 +1,169 @@ +# ============================================================================= +# postgresql_default.conf - PostgreSQL Reference/Documentation Configuration +# +# Created: July 2026 +# +# PURPOSE: +# Minimal-necessary starting point, documenting every setting also tuned +# by postgresql_oltp.conf / postgresql_analytics.conf / postgresql_minimal.conf. +# Every line below is commented out and carries the upstream PostgreSQL +# default value plus a link to the reference documentation, so a reviewer +# can see at a glance what changes each of the other three profiles makes +# relative to a stock server -- instead of having to look each one up. +# +# TARGET VERSION: +# PostgreSQL 18.4 -- the current stable release (released 2026-05-14, +# https://www.postgresql.org/docs/release/18.4/). TAF/tests/setup_almalinux10.sh +# pins EXPECTED_PG_VERSION=18.4 and hard-fails setup if the installed +# server doesn't match exactly, across all three --method installers +# (percona / pgdg / appstream). +# All default values and doc links below are for the PostgreSQL 18 +# documentation set (https://www.postgresql.org/docs/18/), not "current" +# (which silently repoints at whatever the newest major version is once +# PG19 ships) -- defaults do change between major versions, e.g. +# effective_io_concurrency's default was 1 in PG16 but is 16 in PG18, and +# log_connections changed from a boolean to a string-typed GUC. +# +# USAGE: +# This is the default taf.db_config_file for sysbench_lua_pgsql.properties +# (and hammerdb_tprocc_pgsql.properties) -- i.e. what a plain run_me.sh +# invocation actually runs against, with every GUC left at its PG18.4 +# stock default. It contributes NO active settings of its own (every +# line below is commented out); the only non-stock values a benchmark +# run gets are the ones TAF forces regardless of db_config_file (port, +# listen_addresses, ssl -- see NOTES below). +# To run one of the tuned profiles instead, point taf.db_config_file at +# postgresql_oltp.conf / postgresql_analytics.conf / postgresql_minimal.conf. +# Diff this file against those to see exactly what each one overrides +# and by how much relative to a stock server. +# +# NOTES: +# - Port and listen_addresses are always set by TAF (postgres.pm +# _db_apply_postgresql_conf); do not set them here -- any line +# matching /^\s*(port|listen_addresses|ssl)\s*=/i is stripped from +# whatever db_config_file is supplied before it's appended. +# - ssl settings are managed by TAF via db_ssl_mode; do not set ssl here. +# ============================================================================= + +# --------------------------------------------------------------------------- +# "Mandatory" parameters -- i.e. ones without which the server would not +# start, or would start in a way unusable for a benchmark run. +# +# In practice, PostgreSQL has NO postgresql.conf setting that lacks a +# built-in default -- `initdb` generates a fully valid postgresql.conf with +# every parameter left at its compiled-in boot_val, and the server starts +# fine from that alone. Nothing below is a case of "no default exists"; it's +# listed here only because TAF's benchmark harness would be unusable (not +# because postgres itself would refuse to start) if left at the stock value: +# +# - port (PG default: 5432) -- forced by TAF, not here. +# - listen_addresses (PG default: localhost) -- forced by TAF, not here. +# Stock 'localhost' would block the TCP connections TAF/sysbench make +# from the control host; TAF always overrides this to '*' regardless +# of what (if anything) is set in this file. +# - unix_socket_directories (PG default: /tmp on Linux) -- left at stock +# default; not overridden anywhere in TAF. No action needed. +# +# No other setting in this file is "required" in the no-default sense the +# question implies. If you want to designate one of the tunables below as +# mandatory-with-no-safe-default for a *new* profile (e.g. shared_buffers +# sized to a specific host's RAM, so the compiled-in 128MB would be wrong +# for a benchmark), leave a TODO here and decide the value per-host: +# +# TODO(you): any profile-specific "must be set explicitly" parameter goes here. +# --------------------------------------------------------------------------- + +# --------------------------------------------------------------------------- +# Memory +# Reference: https://www.postgresql.org/docs/18/runtime-config-resource.html +# --------------------------------------------------------------------------- +#shared_buffers = 128MB # PG18 default +#work_mem = 4MB # PG18 default +#maintenance_work_mem = 64MB # PG18 default +#temp_buffers = 8MB # PG18 default + +# --------------------------------------------------------------------------- +# WAL / Checkpointing +# Reference: https://www.postgresql.org/docs/18/runtime-config-wal.html +# --------------------------------------------------------------------------- +#wal_buffers = -1 # PG18 default: -1 = 1/32 of shared_buffers (min 64kB, max one WAL segment, typically 16MB) +#synchronous_commit = on # PG18 default +#checkpoint_completion_target = 0.9 # PG18 default +#checkpoint_timeout = 5min # PG18 default +#max_wal_size = 1GB # PG18 default +#min_wal_size = 80MB # PG18 default + +# --------------------------------------------------------------------------- +# Parallelism +# Reference: https://www.postgresql.org/docs/18/runtime-config-resource.html +# --------------------------------------------------------------------------- +#max_worker_processes = 8 # PG18 default +#max_parallel_workers_per_gather = 2 # PG18 default +#max_parallel_workers = 8 # PG18 default + +# --------------------------------------------------------------------------- +# I/O Workers (PostgreSQL 18+ -- new async I/O subsystem) +# Reference: https://www.postgresql.org/docs/18/runtime-config-resource.html +# +# PG18 introduces io_method: reads (and on some platforms writes) can be +# issued asynchronously via a small pool of dedicated "IO worker" processes +# (io_method=worker, the default) instead of the backend blocking on each +# syscall itself. This is the direct answer to "how many IO threads/workers +# does a run use" now that we're on 18.4 -- in PG16 there was no such pool at +# all; effective_io_concurrency there was just an advisory prefetch depth, +# not an actual worker count. +# --------------------------------------------------------------------------- +#io_method = worker # PG18 default (worker | sync | io_uring) +#io_workers = 3 # PG18 default -- number of IO worker processes (server-wide, not per-connection) +#io_combine_limit = 128kB # PG18 default -- largest single I/O size when combining adjacent block reads +#io_max_combine_limit = 128kB # PG18 default (platform-dependent ceiling for io_combine_limit; PGC_POSTMASTER) +#io_max_concurrency = -1 # PG18 default -- auto-selected from shared_buffers/max processes, capped at 64 + +# --------------------------------------------------------------------------- +# Connections +# Reference: https://www.postgresql.org/docs/18/runtime-config-connection.html +# --------------------------------------------------------------------------- +#max_connections = 100 # PG18 default (may be lower if the kernel can't support it, per initdb) + +# --------------------------------------------------------------------------- +# Planner +# Reference: https://www.postgresql.org/docs/18/runtime-config-query.html +# Reference: https://www.postgresql.org/docs/18/runtime-config-resource.html (effective_io_concurrency) +# --------------------------------------------------------------------------- +#effective_cache_size = 4GB # PG18 default +#random_page_cost = 4.0 # PG18 default +#effective_io_concurrency = 16 # PG18 default -- CHANGED from PG16's default of 1 (raised as part of the async-I/O rework) +#default_statistics_target = 100 # PG18 default +#parallel_setup_cost = 1000 # PG18 default +#parallel_tuple_cost = 0.1 # PG18 default +#min_parallel_table_scan_size = 8MB # PG18 default +#min_parallel_index_scan_size = 512kB # PG18 default +#enable_hashagg = on # PG18 default +#enable_hashjoin = on # PG18 default +#enable_sort = on # PG18 default + +# --------------------------------------------------------------------------- +# JIT (PostgreSQL 11+) +# Reference: https://www.postgresql.org/docs/18/runtime-config-query.html +# --------------------------------------------------------------------------- +#jit = on # PG18 default + +# --------------------------------------------------------------------------- +# Logging +# Reference: https://www.postgresql.org/docs/18/runtime-config-logging.html +# --------------------------------------------------------------------------- +#log_min_duration_statement = -1 # PG18 default (disabled) +#log_connections = '' # PG18 default -- CHANGED type: string GUC (connection-phase selector), not a boolean as in PG16; '' disables all connection logging +#log_disconnections = off # PG18 default +#log_checkpoints = on # PG18 default +#log_autovacuum_min_duration = 10min # PG18 default + +# --------------------------------------------------------------------------- +# Autovacuum +# Reference: https://www.postgresql.org/docs/18/runtime-config-autovacuum.html +# --------------------------------------------------------------------------- +#autovacuum = on # PG18 default +#autovacuum_max_workers = 3 # PG18 default -- max autovacuum processes running concurrently +#autovacuum_worker_slots = 16 # PG18 default -- reserved backend slots for autovacuum workers (PGC_POSTMASTER, separate pool from autovacuum_max_workers) +#autovacuum_naptime = 1min # PG18 default +#autovacuum_vacuum_cost_delay = 2ms # PG18 default diff --git a/database_config_files/postgresql/postgresql_minimal.conf b/database_config_files/postgresql/postgresql_minimal.conf new file mode 100644 index 0000000..e5b0827 --- /dev/null +++ b/database_config_files/postgresql/postgresql_minimal.conf @@ -0,0 +1,60 @@ +# ============================================================================= +# postgresql_minimal.conf - PostgreSQL Minimal Configuration for Development +# +# Created: June 2026 +# +# PURPOSE: +# Provide a minimal, resource-light PostgreSQL configuration for +# development, functional testing, and low-load environments. Uses +# conservative defaults that work on machines with limited RAM. +# +# TARGET VERSION: +# PostgreSQL 18.4 (see TAF/tests/setup_almalinux10.sh EXPECTED_PG_VERSION +# and postgresql_default.conf for the full stock-default reference). +# No settings below changed default value between PG16 and PG18, so no +# numeric changes were needed here -- this profile intentionally leaves +# effective_io_concurrency/io_workers/io_method at their PG18 stock +# defaults (16 / 3 / worker), matching its "minimal footprint" intent. +# ============================================================================= + +# --------------------------------------------------------------------------- +# Memory (conservative for dev/CI) +# --------------------------------------------------------------------------- +shared_buffers = 256MB +work_mem = 4MB +maintenance_work_mem = 64MB +effective_cache_size = 1GB + +# --------------------------------------------------------------------------- +# WAL / Checkpointing +# --------------------------------------------------------------------------- +wal_buffers = 16MB +checkpoint_completion_target = 0.9 +checkpoint_timeout = 5min +max_wal_size = 1GB +min_wal_size = 80MB + +# --------------------------------------------------------------------------- +# Connections +# --------------------------------------------------------------------------- +max_connections = 200 + +# --------------------------------------------------------------------------- +# Parallelism (disabled for reproducibility in dev) +# --------------------------------------------------------------------------- +max_parallel_workers_per_gather = 0 +max_parallel_workers = 4 + +# --------------------------------------------------------------------------- +# Planner +# --------------------------------------------------------------------------- +random_page_cost = 4.0 +default_statistics_target = 100 + +# --------------------------------------------------------------------------- +# Logging (informative for dev) +# --------------------------------------------------------------------------- +log_min_duration_statement = 1000 +log_connections = off +log_disconnections = off +log_checkpoints = on diff --git a/database_config_files/postgresql/postgresql_oltp.conf b/database_config_files/postgresql/postgresql_oltp.conf new file mode 100644 index 0000000..1c01b8f --- /dev/null +++ b/database_config_files/postgresql/postgresql_oltp.conf @@ -0,0 +1,94 @@ +# ============================================================================= +# postgresql_oltp.conf - PostgreSQL Configuration for OLTP Benchmarks +# +# Created: June 2026 +# +# PURPOSE: +# Provide tuned PostgreSQL settings for OLTP workloads (Sysbench, +# HammerDB TPROCC). Optimized for throughput, low latency, and +# deterministic benchmark behavior. +# +# TARGET VERSION: +# PostgreSQL 18.4 (see TAF/tests/setup_almalinux10.sh EXPECTED_PG_VERSION +# and postgresql_default.conf for the full stock-default reference). +# +# USAGE: +# Set taf.db_config_file=/path/to/this/file in your properties file, +# or pass --property=taf.db_config_file= on the command line. +# +# NOTES: +# - TAF appends these settings to the postgresql.conf generated by initdb. +# - Port and listen_addresses are always set by TAF; do not set them here. +# - ssl settings are managed by TAF via db_ssl_mode; do not set ssl here. +# - Adjust shared_buffers and effective_cache_size to match available RAM +# on your test host (25% and 75% of RAM respectively are typical). +# - synchronous_commit = off is safe for benchmarking; do NOT use in +# production databases where durability is required. +# ============================================================================= + +# --------------------------------------------------------------------------- +# Memory +# --------------------------------------------------------------------------- +shared_buffers = 4GB +work_mem = 16MB +maintenance_work_mem = 256MB +effective_cache_size = 12GB +temp_buffers = 32MB + +# --------------------------------------------------------------------------- +# WAL / Checkpointing +# --------------------------------------------------------------------------- +wal_buffers = 64MB +checkpoint_completion_target = 0.9 +checkpoint_timeout = 15min +max_wal_size = 4GB +min_wal_size = 1GB +synchronous_commit = off + +# --------------------------------------------------------------------------- +# Parallelism +# --------------------------------------------------------------------------- +max_worker_processes = 8 +max_parallel_workers_per_gather = 0 +max_parallel_workers = 8 + +# --------------------------------------------------------------------------- +# I/O Workers (PostgreSQL 18+) +# io_workers raised above the stock default of 3: with up to 210 concurrent +# guests each issuing many prefetch/read requests (high effective_io_concurrency +# below), a small fixed worker pool becomes a bottleneck sooner than on a +# lightly-loaded server. io_method left at the 'worker' default rather than +# io_uring -- not universally available/enabled across the guest kernels in +# this pool, and 'worker' is the safe, portable choice for a mixed VM fleet. +# --------------------------------------------------------------------------- +io_method = worker +io_workers = 16 + +# --------------------------------------------------------------------------- +# Connections +# --------------------------------------------------------------------------- +max_connections = 600 + +# --------------------------------------------------------------------------- +# Planner +# --------------------------------------------------------------------------- +random_page_cost = 1.1 +effective_io_concurrency = 200 +default_statistics_target = 100 + +# --------------------------------------------------------------------------- +# Logging (minimal for benchmarking) +# --------------------------------------------------------------------------- +log_min_duration_statement = -1 +log_connections = off +log_disconnections = off +log_checkpoints = off +log_autovacuum_min_duration = -1 + +# --------------------------------------------------------------------------- +# Autovacuum (enabled but low-priority during benchmark) +# --------------------------------------------------------------------------- +autovacuum = on +autovacuum_max_workers = 3 +autovacuum_naptime = 1min +autovacuum_vacuum_cost_delay = 20ms diff --git a/libs/database_libs/mariadb.pm b/libs/database_libs/mariadb.pm index c83c987..61c5f54 100644 --- a/libs/database_libs/mariadb.pm +++ b/libs/database_libs/mariadb.pm @@ -316,9 +316,61 @@ sub new { # Runtime pidfile now lives under runtime_dir $self->{pidfile} = File::Spec->catfile($runtime_dir, "mariadb_runtime.pid"); + # When TAF runs as root, mariadbd/mariadb-install-db must run as a + # non-root OS user -- mariadbd refuses to start as root outright (unlike + # postgres, which just needs its server process to not be root; MariaDB's + # check is unconditional and has no --allow-run-as-root style override). + # Unlike PostgreSQL's package install (which creates a 'postgres' system + # user via RPM postinstall scriptlet), this plugin targets a plain tarball + # install with no such user created automatically -- create one if it + # doesn't already exist, rather than only detecting a pre-existing one + # like postgres.pm does. + $self->{is_root} = ($> == 0) ? 1 : 0; + if ($self->{is_root}) { + my @pw = getpwnam('mysql'); + unless (@pw) { + PrintVerbose("${_me}::new - running as root and OS user 'mysql' does not exist; creating it"); + system('useradd', '--system', '--no-create-home', '--shell', '/sbin/nologin', 'mysql'); + if ($? != 0) { + PrintWarning("${_me}::new - useradd mysql failed (exit " . ($? >> 8) . ")"); + } + @pw = getpwnam('mysql'); + } + if (@pw) { + $self->{os_user} = 'mysql'; + $self->{os_uid} = $pw[2]; + $self->{os_gid} = $pw[3]; + PrintVerbose("${_me}::new - running as root; server operations will use OS user 'mysql' (uid=$pw[2])"); + } else { + PrintWarning("${_me}::new - running as root but OS user 'mysql' not found/creatable; mariadbd will refuse to start"); + $self->{os_user} = undef; + $self->{os_uid} = undef; + $self->{os_gid} = undef; + } + } + return $self; } +################################################################################ +# _os_prefix +# +# PURPOSE: +# Return the command prefix needed to run a command as the 'mysql' OS user +# when TAF is executing as root. Returns an empty list when not root or +# when the 'mysql' OS user could not be resolved/created. Mirrors +# postgres.pm's _os_prefix() so both engine plugins handle a root-executed +# TAF the same way. +# +# USAGE: +# my @cmd = ($self->_os_prefix(), $binary, @args); +################################################################################ +sub _os_prefix { + my ($self) = @_; + return () unless $self->{is_root} && $self->{os_user}; + return ('runuser', '-u', $self->{os_user}, '--'); +} + ################################################################################ # db_init # @@ -512,6 +564,7 @@ sub db_start { # Build argv list for exec() my @cmd = ( + $self->_os_prefix(), $server, "--defaults-file=$self->{config}", "--datadir=$data_dir", @@ -1854,6 +1907,28 @@ sub _db_prepare_data_dir { return ERROR; }; + # When running as root, hand ownership to the mysql OS user so + # mariadb-install-db and mariadbd (both run via _os_prefix() as 'mysql') + # can read and write the data directory. Also chown tmpdir, since the + # socket, pidfile, and log-error paths mariadbd opens itself all live + # there (Utilities.pm defaults db_socket to "db.sock"). + if ($self->{is_root} && defined $self->{os_uid}) { + chown($self->{os_uid}, $self->{os_gid}, $dir) + or PrintWarning("_db_prepare_data_dir: chown $dir to $self->{os_user} failed: $!"); + if ($self->{tmpdir} && -d $self->{tmpdir}) { + # Recursive, not just the directory itself: unlike data_dir (wiped + # and recreated from scratch above), tmpdir persists across + # attempts, so a prior run's bootstrap/runtime pidfile or log + # (created before this fix existed, or by a run that failed + # before reaching this chown) can already exist there owned by + # root -- a non-recursive chown leaves those files unwritable by + # 'mysql', and mariadbd fails outright when it can't create/write + # its own --pid-file. + system('chown', '-R', "$self->{os_uid}:$self->{os_gid}", $self->{tmpdir}) == 0 + or PrintWarning("_db_prepare_data_dir: recursive chown of tmpdir failed (exit " . ($? >> 8) . ")"); + } + } + return OK; } @@ -2197,6 +2272,7 @@ sub _db_run_install_db { # Build the install-db command line. # NOTE: No embedded quotes. _run_command handles argument quoting safely. my @cmd = ( + $self->_os_prefix(), $install_db, "--no-defaults", "--basedir=$self->{install_root}", @@ -2356,6 +2432,7 @@ sub _db_start_bootstrap { # Build argv list for exec() my @cmd = ( + $self->_os_prefix(), $server, "--no-defaults", "--datadir=$datadir", @@ -2539,6 +2616,30 @@ sub _spawn_background { my $logdir = File::Spec->catpath($vol, $dir, ''); File::Path::make_path($logdir) unless -d $logdir; + # When running as root, pre-create and chown the pidfile to the target OS + # user *before* forking. mariadbd itself opens/writes its own --pid-file= + # path internally, after runuser (via _os_prefix()) has already dropped + # it to that user -- but this same $pidfile path is also written by this + # very function's parent below (`open $fh, '>', $pidfile`), which never + # drops privileges and stays root. Whichever of the two creates the file + # first ends up owning it; if root creates it first (observed in + # practice), mariadbd's own later write fails with "Can't create/write + # to file ... Permission denied" and the server dies. Pre-creating it + # with the right ownership up front means both writers just open an + # *existing* file (which doesn't change ownership) instead of racing to + # create it, regardless of which one gets there first. + if ($self->{is_root} && defined $self->{os_uid}) { + unless (-e $pidfile) { + if (open(my $fh, '>', $pidfile)) { + close $fh; + } else { + PrintWarning($_tag."could not pre-create pidfile $pidfile: $!"); + } + } + chown($self->{os_uid}, $self->{os_gid}, $pidfile) + or PrintWarning($_tag."chown $pidfile to $self->{os_user} failed: $!"); + } + # fork the daemon my $pid = fork(); if (!defined $pid) { diff --git a/libs/database_libs/postgres.pm b/libs/database_libs/postgres.pm new file mode 100644 index 0000000..9b3eac9 --- /dev/null +++ b/libs/database_libs/postgres.pm @@ -0,0 +1,1171 @@ +package postgres; +############################################################################### +# postgres.pm - PostgreSQL Database Plugin for TAF +# +# Created: June 2026 by lukas.oliva@virtuozzo.com using Claude +# Last Modified: June 2026 +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 or later of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 +# +# Licensed under the GNU General Public License, version 2 or later (GPLv2+). +# See https://www.gnu.org/licenses/ for details. +# +# PURPOSE: +# Provide a deterministic, contributor-proof implementation of the +# PostgreSQL backend lifecycle for the Test Automation Framework (TAF). +# This plugin encapsulates all logic required to initialize, configure, +# start, stop, restart, and validate a PostgreSQL server instance under +# TAF control. It receives all configuration at construction time and +# performs all engine-specific behavior behind a stable, version-aware +# plugin API. The plugin is responsible for initdb-based initialization, +# pg_hba.conf and postgresql.conf management, user and database bootstrap, +# runtime startup via pg_ctl, and liveness checks via pg_isready, ensuring +# that every PostgreSQL instance behaves predictably across all environments +# and packaging formats. +# +# ARCHITECTURAL ROLE: +# - Implements the complete PostgreSQL lifecycle: +# init -> initdb -> pg_hba.conf -> postgresql.conf -> users -> start -> stop +# - Encapsulates all engine-specific behavior behind a stable TAF plugin API. +# - Receives all configuration at construction time; does not depend on +# global framework state or the $ctx structure. +# - Normalizes installation layout, runtime paths, and configuration. +# - Provides deterministic fork/exec-free startup via pg_ctl. +# - Provides contributor-proof behavior for: +# * db_init() +# * db_start() +# * db_stop() +# * db_restart() +# * db_ping() +# +# KEY DIFFERENCES FROM MARIADB/MYSQL PLUGINS: +# - Initialization uses initdb, not mysqld --initialize. +# - Server is managed via pg_ctl (no manual fork/exec needed). +# - Configuration files are postgresql.conf + pg_hba.conf (not my.cnf). +# - User and database bootstrap uses psql as the superuser. +# - Authentication is controlled via pg_hba.conf rules. +# - Liveness checks use pg_isready (not mysqladmin ping). +# - Default port is 5432 (not 3306). +# - No socket-only bootstrap mode; pg_hba.conf governs all auth. +# +# NOTE: +# This plugin is fully self-contained. All SQL required for bootstrap, +# user creation, grants, and lifecycle validation is executed through the +# psql client binary. No external SQL libraries are used. +# +# CONTRACT: +# - Must be instantiated via ->new(%args) with all required DB configuration. +# - Must implement db_ping(), db_start(), db_stop(), and db_init() +# without requiring the framework context. +# - Must not modify global TAF state. +# - Must return OK/ERROR codes consistently. +############################################################################### +our $_me = "PostgreSQL"; + +################################################################################ +# Includes +################################################################################ +use strict; +use warnings; +use File::Spec; +use File::Path (); +use File::Basename (); +use Carp; +use POSIX qw(setsid); +use FindBin qw($Bin); +use lib "$Bin/../taf_libs"; +use TAF::Logging qw( + PrintError + PrintWarning + PrintVerbose + StageStart + StageEnd +); + +################################################################################ +# Constants +################################################################################ +use constant OK => 0; +use constant ERROR => 1; +use constant TRUE => 1; +use constant FALSE => 0; + +################################################################################ +# new +# +# PURPOSE: +# Construct and return a new PostgreSQL plugin object. The object captures +# all configuration, paths, binaries, SSL settings, and lifecycle state +# required for deterministic PostgreSQL behavior under TAF. +# +# BEHAVIOR: +# - Stores all constructor arguments directly into the plugin object. +# - Resolves postgres, pg_ctl, psql, initdb, and pg_isready binaries. +# - Validates that required binaries exist and are executable. +# - Sets the default port to 5432 when not supplied. +# +# NOTES: +# - PostgreSQL binaries may live under versioned paths such as +# /usr/pgsql-16/bin/ or /usr/lib/postgresql/16/bin/. The _find_binary() +# helper searches install_root/bin/ first, then common system paths. +# - PGPASSWORD is set in the environment at runtime to avoid interactive +# password prompts when running psql commands. +################################################################################ +sub new { + my ($class, %args) = @_; + + my $self = { + + # Instanced pid + db_pid => undef, + + # Install and data paths + install_root => $args{db_software_install_dir}, + data_dir => $args{db_data_dir}, + trans_logs_dir => $args{db_trans_logs_dir}, + + # Config (postgresql.conf path; pg_hba.conf is derived from data_dir) + config => $args{db_config_file}, + + # Binaries (resolved below) + postgres_bin => undef, + pg_ctl_bin => undef, + psql_bin => undef, + initdb_bin => undef, + pg_isready_bin => undef, + + # Error log + error_log => undef, + + # Connectivity + port => $args{db_port} // 5432, + + # SSL (TAF unified SSL contract) + ssl_mode => $args{db_ssl_mode}, + ssl_ca => $args{db_ssl_ca}, + ssl_cert => $args{db_ssl_cert}, + ssl_key => $args{db_ssl_key}, + + # Database and users + database => $args{database} // 'test', + db_user => $args{db_user} // 'pgsql_tester', + db_user_pass => $args{db_user_pass} // 'PostgresPass_@123', + db_user_permissions => $args{db_user_permissions} // 'ALL PRIVILEGES', + db_root_user => $args{db_root_user} // 'postgres', + db_root_pass => $args{db_root_pass} // 'PostgresPass_@123', + + # Locality and performance + cpus => $args{db_task_set}, + db_start_wait => $args{db_start_wait}, + db_stop_wait => $args{db_stop_wait}, + tmpdir => $args{tmp_dir}, + + # Extras + extra_args => $args{db_extra_args}, + + # State flags + initialized => FALSE, + users_created => FALSE, + + # Version metadata (populated during init) + pg_version => undef, + pg_version_num => undef, + }; + + bless $self, $class; + + # Validate tmpdir early — it is required throughout the lifecycle + unless ($self->{tmpdir} && -d $self->{tmpdir}) { + PrintError("${_me}::new - tmpdir is missing or not a directory: " . + ($self->{tmpdir} // "")); + return undef; + } + + # Validate install_root + unless ($self->{install_root} && -d $self->{install_root}) { + PrintError("${_me}::new - install_root is missing or not a directory: " . + ($self->{install_root} // "")); + return undef; + } + + # Resolve binaries + $self->{postgres_bin} = _find_binary($self->{install_root}, 'postgres'); + $self->{pg_ctl_bin} = _find_binary($self->{install_root}, 'pg_ctl'); + $self->{psql_bin} = _find_binary($self->{install_root}, 'psql'); + $self->{initdb_bin} = _find_binary($self->{install_root}, 'initdb'); + $self->{pg_isready_bin} = _find_binary($self->{install_root}, 'pg_isready'); + + # Validate required binaries + for my $b (qw(postgres_bin pg_ctl_bin psql_bin initdb_bin pg_isready_bin)) { + unless ($self->{$b} && -x $self->{$b}) { + PrintError("${_me}::new - Required binary '$b' not found under " . + $self->{install_root}); + return undef; + } + } + + $self->{log_init} = File::Spec->catfile($self->{tmpdir}, "postgresql_initdb.log"); + $self->{log_start} = File::Spec->catfile($self->{tmpdir}, "postgresql_start.log"); + $self->{pidfile} = File::Spec->catfile($self->{tmpdir}, "postgresql_runtime.pid"); + + # When TAF runs as root, initdb and pg_ctl must run as the postgres OS user. + # Detect this at construction time so all lifecycle methods can wrap commands. + $self->{is_root} = ($> == 0) ? 1 : 0; + if ($self->{is_root}) { + my @pw = getpwnam('postgres'); + if (@pw) { + $self->{os_user} = 'postgres'; + $self->{os_uid} = $pw[2]; + $self->{os_gid} = $pw[3]; + PrintVerbose("${_me}::new - running as root; cluster operations will use OS user 'postgres' (uid=$pw[2])"); + + # If data_dir or tmpdir are under /root (not accessible to postgres), + # redirect them to /tmp where the postgres user can traverse. + my $pg_base = "/tmp/taf_pg_$$"; + if ($self->{data_dir} && $self->{data_dir} =~ m{^/root(/|$)}) { + $self->{data_dir} = "$pg_base/data"; + PrintVerbose("${_me}::new - data_dir relocated to $self->{data_dir} (root path inaccessible to postgres)"); + } + if ($self->{tmpdir} && $self->{tmpdir} =~ m{^/root(/|$)}) { + $self->{tmpdir} = "$pg_base/tmp"; + PrintVerbose("${_me}::new - tmpdir relocated to $self->{tmpdir} (root path inaccessible to postgres)"); + # Ensure log paths are updated + $self->{log_init} = File::Spec->catfile($self->{tmpdir}, "postgresql_initdb.log"); + $self->{log_start} = File::Spec->catfile($self->{tmpdir}, "postgresql_start.log"); + $self->{pidfile} = File::Spec->catfile($self->{tmpdir}, "postgresql_runtime.pid"); + } + # Create and chown the pg_base dirs so postgres can write into them + if ($self->{data_dir} =~ m{^\Q$pg_base\E} || $self->{tmpdir} =~ m{^\Q$pg_base\E}) { + File::Path::make_path("$pg_base/data", "$pg_base/tmp") + or PrintWarning("${_me}::new - could not pre-create $pg_base dirs"); + chown($pw[2], $pw[3], $pg_base, "$pg_base/data", "$pg_base/tmp"); + chmod(0700, $pg_base, "$pg_base/data", "$pg_base/tmp"); + } + } else { + PrintWarning("${_me}::new - running as root but OS user 'postgres' not found; initdb may fail"); + $self->{os_user} = undef; + $self->{os_uid} = undef; + $self->{os_gid} = undef; + } + } + + return $self; +} + +################################################################################ +# db_init +# +# PURPOSE: +# Execute the full PostgreSQL initialization lifecycle. This routine +# prepares the datadir via initdb, writes postgresql.conf and pg_hba.conf, +# starts the server, creates the TAF tester user and database, then stops +# the server. The cluster is left in a clean, initialized state ready for +# db_start() by the framework. +# +# BEHAVIOR: +# 1. Validate binaries and tmpdir. +# 2. Prepare an empty datadir. +# 3. Run initdb to create the PostgreSQL cluster. +# 4. Apply postgresql.conf (user-supplied or built-in defaults). +# 5. Write pg_hba.conf to allow TCP and local connections. +# 6. Start the server. +# 7. Create the tester role and test database via psql. +# 8. Stop the server. +# +# NOTES: +# - pg_hba.conf always allows connections from 127.0.0.1 and ::1 using +# md5 authentication for the tester and postgres users. This is required +# for sysbench and other TCP-based benchmark clients. +# - The superuser password is set during initdb via --pwfile. +################################################################################ +sub db_init { + my ($self) = @_; + my $_init = StageStart("$_me -> Init Database ->"); + + # Validate binaries + return ERROR if $self->_db_validate_binaries() != OK; + + # Prepare empty data directory + return ERROR if $self->_db_prepare_data_dir() != OK; + + # Detect PostgreSQL version + $self->{pg_version} = $self->_detect_pg_version($self->{postgres_bin}); + unless ($self->{pg_version}) { + PrintError("$_init Failed to detect PostgreSQL version"); + return ERROR; + } + PrintVerbose("$_init Detected PostgreSQL version: $self->{pg_version}"); + + # Run initdb to create the cluster + return ERROR if $self->_db_run_initdb() != OK; + + # Apply postgresql.conf + return ERROR if $self->_db_apply_postgresql_conf() != OK; + + # Write pg_hba.conf + return ERROR if $self->_db_write_pg_hba_conf() != OK; + + # Start server for user bootstrap + return ERROR if $self->db_start() != OK; + + # Create tester role and test database + return ERROR if $self->_db_setup_users() != OK; + + # Stop server after bootstrap + return ERROR if $self->db_stop() != OK; + + $self->{initialized} = TRUE; + StageEnd($_init); + return OK; +} + +################################################################################ +# db_start +# +# PURPOSE: +# Start the PostgreSQL server using pg_ctl. Waits for the server to become +# ready using pg_isready before returning OK. +# +# BEHAVIOR: +# - Builds and executes: pg_ctl start -D {data_dir} -l {log} -w -t {timeout} +# - pg_ctl writes the server PID into {data_dir}/postmaster.pid. +# - Reads the PID from postmaster.pid after successful startup. +# - Waits via _wait_for_start() which polls pg_isready. +################################################################################ +sub db_start { + my ($self, $wait_seconds) = @_; + my $_st = StageStart("$_me -> Database Start ->"); + + my $pg_ctl = $self->{pg_ctl_bin}; + my $data_dir = $self->{data_dir}; + my $log = $self->{log_start}; + my $timeout = $wait_seconds // $self->{db_start_wait} // 90; + + unless ($pg_ctl && -x $pg_ctl) { + PrintError("$_st pg_ctl binary not executable: " . ($pg_ctl // "")); + return ERROR; + } + + unless ($data_dir && -d $data_dir) { + PrintError("$_st data_dir does not exist: " . ($data_dir // "")); + return ERROR; + } + + # Port is set in postgresql.conf by _db_apply_postgresql_conf(); no need to + # pass it via -o here. Passing "-o -p N" through _run_command (which uses + # shell string form of system()) would cause shell splitting issues. + my @cmd = ( + $self->_os_prefix(), + $pg_ctl, + 'start', + "-D", $data_dir, + "-l", $log, + "-w", + "-t", $timeout, + ); + + if ($self->{extra_args}) { + push @cmd, "-o", "\"$self->{extra_args}\""; + } + + PrintVerbose("$_st Running: @cmd"); + + my $rc = $self->_run_command(\@cmd, "start", undef); + if ($rc != 0) { + PrintError("$_st pg_ctl start failed (exit $rc), see $log"); + return ERROR; + } + + # Confirm readiness via pg_isready + if ($self->_wait_for_start($timeout) != OK) { + PrintError("$_st PostgreSQL did not become ready, see $log"); + return ERROR; + } + + # Read PID from postmaster.pid + my $pidfile = File::Spec->catfile($data_dir, "postmaster.pid"); + if (-f $pidfile) { + if (open(my $fh, '<', $pidfile)) { + my $pid = <$fh>; + close $fh; + chomp $pid; + if ($pid =~ /^\d+$/) { + $self->{db_pid} = $pid; + PrintVerbose("$_st PostgreSQL runtime PID: $pid"); + } + } + } + + StageEnd($_st); + return OK; +} + +################################################################################ +# db_stop +# +# PURPOSE: +# Stop the PostgreSQL server using pg_ctl stop -m fast. Waits for the +# server to exit before returning OK. +################################################################################ +sub db_stop { + my ($self, $wait_seconds) = @_; + my $_st = StageStart("$_me -> Database Stop ->"); + + my $pg_ctl = $self->{pg_ctl_bin}; + my $data_dir = $self->{data_dir}; + my $timeout = $wait_seconds // $self->{db_stop_wait} // 120; + + unless ($pg_ctl && -x $pg_ctl) { + PrintError("$_st pg_ctl binary not executable: " . ($pg_ctl // "")); + return ERROR; + } + + # Check whether the server is actually running + my $status_rc = $self->_pg_ctl_status(); + if ($status_rc != 0) { + PrintVerbose("$_st PostgreSQL is not running (pg_ctl status=$status_rc); nothing to stop"); + StageEnd($_st); + return OK; + } + + my @cmd = ( + $self->_os_prefix(), + $pg_ctl, + 'stop', + "-D", $data_dir, + "-m", "fast", + "-w", + "-t", $timeout, + ); + + PrintVerbose("$_st Running: @cmd"); + + my $rc = $self->_run_command(\@cmd, "stop", undef); + if ($rc != 0) { + PrintError("$_st pg_ctl stop failed (exit $rc)"); + return ERROR; + } + + $self->{db_pid} = undef; + + PrintVerbose("$_st PostgreSQL stopped"); + StageEnd($_st); + return OK; +} + +################################################################################ +# db_restart +################################################################################ +sub db_restart { + my ($self) = @_; + my $_st = StageStart("$_me -> Database Restart ->"); + + if ($self->db_stop() != OK) { + PrintError("$_st db_stop() failed during restart"); + return ERROR; + } + + if ($self->db_start() != OK) { + PrintError("$_st db_start() failed during restart"); + return ERROR; + } + + StageEnd($_st); + return OK; +} + +################################################################################ +# db_ping +# +# PURPOSE: +# Verify that the PostgreSQL server is responsive using pg_isready, then +# confirm SQL execution via a trivial SELECT 1. +################################################################################ +sub db_ping { + my ($self) = @_; + my $_st = StageStart("$_me -> Ping ->"); + + my $rc = $self->_db_execute_no_return_query("SELECT 1"); + if ($rc != OK) { + PrintError("$_st Ping failed"); + return ERROR; + } + + PrintVerbose("$_st Ping successful"); + StageEnd($_st); + return OK; +} + +################################################################################ +# db_pid +################################################################################ +sub db_pid { + my ($self) = @_; + + my $pid = $self->{db_pid}; + unless (defined $pid && $pid =~ /^\d+$/) { + PrintError("${_me}::db_pid - PID not set or invalid"); + return undef; + } + return $pid; +} + +#=============================================================================== +# Internal Subs +#=============================================================================== + +################################################################################ +# _db_execute_no_return_query +# +# PURPOSE: +# Execute a SQL statement through psql as the root (postgres) superuser. +# Used for bootstrap SQL and liveness checks. Does not return result sets. +# +# BEHAVIOR: +# - Uses TCP connection to 127.0.0.1:{port} (pg_hba.conf must allow it). +# - Sets PGPASSWORD in the environment to avoid interactive prompts. +# - Appends -c to run the statement directly. +# - Returns OK on exit 0, ERROR otherwise. +################################################################################ +sub _db_execute_no_return_query { + my ($self, $sql, $as_root) = @_; + my $_tag = "$_me -> _db_execute_no_return_query ->"; + + my $psql = $self->{psql_bin}; + unless ($psql && -x $psql) { + PrintError("$_tag psql not executable: " . ($psql // "")); + return ERROR; + } + + my $user = $as_root ? $self->{db_root_user} : $self->{db_user}; + my $pass = $as_root ? $self->{db_root_pass} : $self->{db_user_pass}; + my $db = $as_root ? 'postgres' : $self->{database}; + + PrintVerbose("$_tag Executing: $sql"); + + # Set PGPASSWORD to avoid interactive prompt + local $ENV{PGPASSWORD} = $pass if $pass; + + my @cmd = ( + $psql, + "-h", "127.0.0.1", + "-p", $self->{port}, + "-U", $user, + "-d", $db, + "-c", $sql, + "-q", + "--no-psqlrc", + ); + + my $rc = system(@cmd); + if ($rc != 0) { + my $exit = $rc >> 8; + PrintError("$_tag Query failed (exit $exit): $sql"); + return ERROR; + } + + return OK; +} + +################################################################################ +# _db_execute_as_superuser +# +# PURPOSE: +# Execute a SQL statement as the postgres superuser against the postgres +# maintenance database. Used exclusively during bootstrap. +################################################################################ +sub _db_execute_as_superuser { + my ($self, $sql, $db) = @_; + $db //= 'postgres'; # default: maintenance database + my $_tag = "$_me -> _db_execute_as_superuser ->"; + + my $psql = $self->{psql_bin}; + unless ($psql && -x $psql) { + PrintError("$_tag psql not executable"); + return ERROR; + } + + PrintVerbose("$_tag Executing (db=$db): $sql"); + + local $ENV{PGPASSWORD} = $self->{db_root_pass} if $self->{db_root_pass}; + + my @cmd = ( + $psql, + "-h", "127.0.0.1", + "-p", $self->{port}, + "-U", $self->{db_root_user}, + "-d", $db, + "-c", $sql, + "-q", + "--no-psqlrc", + ); + + my $rc = system(@cmd); + if ($rc != 0) { + my $exit = $rc >> 8; + PrintError("$_tag Superuser query failed (exit $exit): $sql"); + return ERROR; + } + + return OK; +} + +################################################################################ +# _db_setup_users +# +# PURPOSE: +# Create the TAF tester role and test database during initialization. +# +# BEHAVIOR: +# - Sets the postgres superuser password. +# - Drops and recreates the tester role. +# - Drops and recreates the test database owned by tester. +# - Grants privileges on the test database to the tester. +# +# CONTRACT: +# - Must be called after db_start() has launched the server. +# - Operates via TCP connections to 127.0.0.1 (pg_hba.conf must allow it). +################################################################################ +sub _db_setup_users { + my ($self) = @_; + my $_st = StageStart("$_me -> Setup Users ->"); + + my $root = $self->{db_root_user}; + my $rootpass = $self->{db_root_pass}; + my $user = $self->{db_user}; + my $pass = $self->{db_user_pass}; + my $db = $self->{database}; + + # Set superuser password + if ($rootpass) { + my $sql = "ALTER USER \"$root\" WITH PASSWORD '$rootpass'"; + return ERROR if $self->_db_execute_as_superuser($sql) != OK; + PrintVerbose("$_st Superuser password set"); + } + + # Drop tester role if exists (clean re-init semantics) + { + my $sql = "DROP DATABASE IF EXISTS \"$db\""; + return ERROR if $self->_db_execute_as_superuser($sql) != OK; + + $sql = "DROP ROLE IF EXISTS \"$user\""; + return ERROR if $self->_db_execute_as_superuser($sql) != OK; + } + + # Create tester role with login and password + { + my $sql = "CREATE ROLE \"$user\" WITH LOGIN PASSWORD '$pass'"; + return ERROR if $self->_db_execute_as_superuser($sql) != OK; + PrintVerbose("$_st Tester role created: $user"); + } + + # Create test database owned by tester + { + my $sql = "CREATE DATABASE \"$db\" OWNER \"$user\""; + return ERROR if $self->_db_execute_as_superuser($sql) != OK; + PrintVerbose("$_st Test database created: $db"); + } + + # Grant all privileges on the database + { + my $sql = "GRANT ALL PRIVILEGES ON DATABASE \"$db\" TO \"$user\""; + return ERROR if $self->_db_execute_as_superuser($sql) != OK; + } + + # PG 15+: GRANT CREATE on the public schema — revoked from PUBLIC by default. + # Must connect to the test database (not postgres) to GRANT on its schema. + { + my $sql = "GRANT ALL ON SCHEMA public TO \"$user\""; + return ERROR if $self->_db_execute_as_superuser($sql, $db) != OK; + PrintVerbose("$_st GRANT public schema to $user (PG15+ requirement)"); + } + + $self->{users_created} = TRUE; + PrintVerbose("$_st Tester user setup complete"); + + StageEnd($_st); + return OK; +} + +################################################################################ +# _db_run_initdb +# +# PURPOSE: +# Initialize a new PostgreSQL cluster using initdb. +# +# BEHAVIOR: +# - Writes a temporary password file for the superuser. +# - Runs: initdb -D {data_dir} -U {db_root_user} -E UTF8 --pwfile= +# - Removes the password file after initdb completes. +################################################################################ +sub _db_run_initdb { + my ($self) = @_; + my $_tag = StageStart("$_me -> RunInitdb ->"); + + my $initdb = $self->{initdb_bin}; + my $data_dir = $self->{data_dir}; + my $root = $self->{db_root_user}; + my $rootpass = $self->{db_root_pass}; + my $log = $self->{log_init}; + + unless ($initdb && -x $initdb) { + PrintError("$_tag initdb not executable: " . ($initdb // "")); + return ERROR; + } + + unless (-d $data_dir) { + PrintError("$_tag data_dir does not exist: $data_dir"); + return ERROR; + } + + # Write superuser password to a temporary pwfile. + # When running as root, the initdb process runs as the postgres OS user and + # cannot access paths under /root. Use /tmp for the pwfile so it is always + # readable, and remove it immediately after initdb completes. + my $pwfile_dir = ($self->{is_root} && $self->{os_user}) ? '/tmp' : $self->{tmpdir}; + my $pwfile = File::Spec->catfile($pwfile_dir, "pg_pwfile_taf_$$.tmp"); + if (open(my $fh, '>', $pwfile)) { + print $fh $rootpass // ''; + close $fh; + # World-readable so the postgres OS user can read it; short-lived file + chmod(($self->{is_root} ? 0644 : 0600), $pwfile); + } else { + PrintError("$_tag Cannot write pwfile: $pwfile"); + return ERROR; + } + + my @cmd = ( + $self->_os_prefix(), + $initdb, + "-D", $data_dir, + "-U", $root, + "-E", "UTF8", + "--locale=C", + "--pwfile=$pwfile", + ); + + PrintVerbose("$_tag Running: @cmd"); + + my $rc = $self->_run_command(\@cmd, "initdb", $log); + unlink $pwfile; + + if ($rc != 0) { + PrintError("$_tag initdb failed (exit $rc), see $log"); + return ERROR; + } + + PrintVerbose("$_tag initdb completed"); + StageEnd($_tag); + return OK; +} + +################################################################################ +# _db_apply_postgresql_conf +# +# PURPOSE: +# Apply postgresql.conf settings. If the user supplied a config file via +# db_config_file, its contents are appended to (not replaced) the +# postgresql.conf created by initdb. This preserves initdb-generated +# defaults while layering TAF-specific tuning on top. +# +# BEHAVIOR: +# - Always writes the port setting to ensure the configured port is used. +# - If a user config file is supplied and readable, appends its contents. +# - If no user config is supplied, writes safe benchmark defaults. +################################################################################ +sub _db_apply_postgresql_conf { + my ($self) = @_; + my $_tag = "$_me -> _db_apply_postgresql_conf ->"; + + my $pg_conf = File::Spec->catfile($self->{data_dir}, "postgresql.conf"); + + unless (-w $pg_conf) { + PrintError("$_tag postgresql.conf not writable: $pg_conf"); + return ERROR; + } + + # Append TAF port setting unconditionally + if (open(my $fh, '>>', $pg_conf)) { + print $fh "\n# === TAF-managed settings ===\n"; + print $fh "port = $self->{port}\n"; + print $fh "listen_addresses = '*'\n"; + + # Unix socket lives in TAF's own tmpdir, not PG's stock default + # (/tmp) -- this is the same directory sysbench-lua.pm's + # SetConnectionArgs() derives (via dirname($options{db_socket})) + # when db_clients_use_unix_socket is set, so the two must agree. + my $socket_dir = $self->{tmpdir}; + $socket_dir =~ s{/+$}{}; + print $fh "unix_socket_directories = '$socket_dir'\n"; + + # SSL settings + my $ssl_mode = lc($self->{ssl_mode} // 'off'); + if ($ssl_mode ne 'off') { + print $fh "ssl = on\n"; + print $fh "ssl_ca_file = '$self->{ssl_ca}'\n" if $self->{ssl_ca}; + print $fh "ssl_cert_file = '$self->{ssl_cert}'\n" if $self->{ssl_cert}; + print $fh "ssl_key_file = '$self->{ssl_key}'\n" if $self->{ssl_key}; + } else { + print $fh "ssl = off\n"; + } + + # If user supplied a config file, append its contents + if ($self->{config} && -r $self->{config}) { + print $fh "\n# === User-supplied TAF config ===\n"; + if (open(my $ufh, '<', $self->{config})) { + while (my $line = <$ufh>) { + # Skip port/listen/ssl/socket-dir — already written above + next if $line =~ /^\s*(port|listen_addresses|ssl|unix_socket_directories)\s*=/i; + print $fh $line; + } + close $ufh; + PrintVerbose("$_tag Appended user config: $self->{config}"); + } + } else { + # Write safe benchmark defaults when no user config supplied + print $fh "\n# === TAF benchmark defaults ===\n"; + print $fh "shared_buffers = 256MB\n"; + print $fh "work_mem = 4MB\n"; + print $fh "maintenance_work_mem = 64MB\n"; + print $fh "effective_cache_size = 1GB\n"; + print $fh "checkpoint_completion_target = 0.9\n"; + print $fh "wal_buffers = 16MB\n"; + print $fh "max_connections = 500\n"; + print $fh "log_min_duration_statement = -1\n"; + print $fh "log_connections = off\n"; + print $fh "log_disconnections = off\n"; + } + + close $fh; + } else { + PrintError("$_tag Cannot open postgresql.conf for writing: $pg_conf"); + return ERROR; + } + + PrintVerbose("$_tag postgresql.conf configured"); + return OK; +} + +################################################################################ +# _db_write_pg_hba_conf +# +# PURPOSE: +# Write pg_hba.conf to allow TCP and local connections for both the +# postgres superuser and the TAF tester user. This is required because +# initdb generates a restrictive pg_hba.conf (peer/ident auth), which +# would block psql TCP connections used by TAF and benchmark clients. +# +# BEHAVIOR: +# - Writes a minimal, TAF-controlled pg_hba.conf. +# - Allows md5 authentication for all users from 127.0.0.1/32 and ::1/128. +# - Allows local (Unix socket) connections for the postgres user for +# pg_ctl and maintenance operations. +# - If ssl_mode is not 'off', adds hostssl rules in addition to host rules. +################################################################################ +sub _db_write_pg_hba_conf { + my ($self) = @_; + my $_tag = "$_me -> _db_write_pg_hba_conf ->"; + + my $hba = File::Spec->catfile($self->{data_dir}, "pg_hba.conf"); + + unless (open(my $fh, '>', $hba)) { + PrintError("$_tag Cannot write pg_hba.conf: $hba"); + return ERROR; + } else { + my $ssl_mode = lc($self->{ssl_mode} // 'off'); + my $auth = "md5"; + + print $fh "# TAF-managed pg_hba.conf\n"; + print $fh "# TYPE DATABASE USER ADDRESS METHOD\n"; + print $fh "\n"; + + # Local (Unix socket) — postgres superuser only, for pg_ctl and psql maintenance + print $fh "local all postgres trust\n"; + print $fh "local all all md5\n"; + print $fh "\n"; + + # TCP IPv4 and IPv6 — all users + print $fh "host all all 127.0.0.1/32 $auth\n"; + print $fh "host all all ::1/128 $auth\n"; + print $fh "\n"; + + # SSL connections when SSL is enabled + if ($ssl_mode ne 'off') { + print $fh "hostssl all all 0.0.0.0/0 $auth\n"; + } + + close $fh; + } + + PrintVerbose("$_tag pg_hba.conf written"); + return OK; +} + +################################################################################ +################################################################################ +# _os_prefix +# +# PURPOSE: +# Return the command prefix needed to run a command as the postgres OS user +# when TAF is executing as root. Returns an empty list when not root or +# when the postgres OS user could not be resolved. +# +# USAGE: +# my @cmd = ($self->_os_prefix(), $binary, @args); +################################################################################ +sub _os_prefix { + my ($self) = @_; + return () unless $self->{is_root} && $self->{os_user}; + return ('runuser', '-u', $self->{os_user}, '--'); +} + +# _db_prepare_data_dir +# +# PURPOSE: +# Ensure the data directory is empty and ready for initdb. Removes any +# existing content. Creates the directory fresh. When running as root, +# also chowns the directory to the postgres OS user so initdb can write it. +################################################################################ +sub _db_prepare_data_dir { + my ($self) = @_; + my $dir = $self->{data_dir}; + + if (-d $dir) { + PrintVerbose("$_me -> Removing existing data directory: $dir"); + File::Path::remove_tree($dir, {error => \my $err}); + if (@$err) { + PrintError("_db_prepare_data_dir: Failed to remove $dir"); + return ERROR; + } + } + + File::Path::make_path($dir) or do { + PrintError("_db_prepare_data_dir: Failed to create $dir"); + return ERROR; + }; + + # When running as root, hand ownership to the postgres OS user so initdb + # and pg_ctl can read and write the cluster directory. + if ($self->{is_root} && defined $self->{os_uid}) { + chown($self->{os_uid}, $self->{os_gid}, $dir) + or PrintWarning("_db_prepare_data_dir: chown $dir to $self->{os_user} failed: $!"); + # Also chown tmpdir so pg_ctl can write the startup log + chown($self->{os_uid}, $self->{os_gid}, $self->{tmpdir}) + or PrintWarning("_db_prepare_data_dir: chown tmpdir failed: $!"); + } + + return OK; +} + +################################################################################ +# _db_validate_binaries +# +# PURPOSE: +# Validate that all required PostgreSQL binaries resolved during new() +# exist and are executable. +################################################################################ +sub _db_validate_binaries { + my ($self) = @_; + my $_tag = "$_me -> _db_validate_binaries ->"; + + for my $b (qw(postgres_bin pg_ctl_bin psql_bin initdb_bin pg_isready_bin)) { + unless ($self->{$b} && -x $self->{$b}) { + PrintError("$_tag Binary '$b' not found or not executable: " . + ($self->{$b} // "")); + return ERROR; + } + } + + PrintVerbose("$_tag All required binaries validated"); + return OK; +} + +################################################################################ +# _detect_pg_version +# +# PURPOSE: +# Detect the PostgreSQL server version. Runs: postgres --version +# Returns the version string (e.g. "16.3") or undef on failure. +################################################################################ +sub _detect_pg_version { + my ($self, $binary) = @_; + + return undef unless defined $binary && -x $binary; + + my $output = `"$binary" --version 2>&1`; + return undef unless defined $output && length $output; + + # Expected: "postgres (PostgreSQL) 16.3" + my ($version) = $output =~ /PostgreSQL\)\s+(\d+\.\d+(?:\.\d+)?)/; + return undef unless $version; + + # Extract numeric major version (e.g. 16 from 16.3) + my ($major) = $version =~ /^(\d+)/; + $self->{pg_version_num} = $major; + + $self->{server_version_raw} = $output; + $self->{server_version_norm} = $version; + + PrintVerbose("${_me}::_detect_pg_version: $version"); + return $version; +} + +################################################################################ +# _wait_for_start +# +# PURPOSE: +# Poll pg_isready until the server is accepting connections or timeout +# is reached. +# +# BEHAVIOR: +# - Calls pg_isready -h 127.0.0.1 -p {port} in a loop. +# - Polls at 1-second intervals up to $timeout seconds. +# - Returns OK when pg_isready exits 0; ERROR on timeout. +################################################################################ +sub _wait_for_start { + my ($self, $timeout) = @_; + my $_tag = "$_me -> _wait_for_start ->"; + + $timeout //= $self->{db_start_wait} // 90; + + my $pg_isready = $self->{pg_isready_bin}; + unless ($pg_isready && -x $pg_isready) { + PrintError("$_tag pg_isready not executable"); + return ERROR; + } + + PrintVerbose("$_tag Waiting up to $timeout seconds for PostgreSQL readiness..."); + + for my $i (1 .. $timeout) { + my @cmd = ( + $pg_isready, + "-h", "127.0.0.1", + "-p", $self->{port}, + "-q", + ); + + my $rc = system(@cmd); + if ($rc == 0) { + PrintVerbose("$_tag PostgreSQL is ready (attempt $i)"); + return OK; + } + + sleep 1; + } + + PrintError("$_tag PostgreSQL did not become ready within $timeout seconds"); + return ERROR; +} + +################################################################################ +# _pg_ctl_status +# +# PURPOSE: +# Run pg_ctl status -D {data_dir} and return the exit code. +# Exit 0 means the server is running; non-zero means it is not. +################################################################################ +sub _pg_ctl_status { + my ($self) = @_; + + my $pg_ctl = $self->{pg_ctl_bin}; + my $data_dir = $self->{data_dir}; + + return 1 unless $pg_ctl && -x $pg_ctl && $data_dir && -d $data_dir; + + # pg_ctl status: -q is not valid for all pg_ctl versions (e.g. PG 11). + # Suppress output by redirecting through the shell instead. + my $cmd = join(' ', ($self->_os_prefix()), $pg_ctl, 'status', "-D", $data_dir, '>/dev/null 2>&1'); + my $rc = system($cmd); + return ($rc == 0) ? 0 : 1; +} + +################################################################################ +# _find_binary +# +# PURPOSE: +# Locate a PostgreSQL binary under the install root or standard system +# paths. Searches in deterministic order: +# /bin/ +# /sbin/ +# / +# +# NOTES: +# - PostgreSQL binaries may also exist under versioned system paths such as +# /usr/pgsql-16/bin/ or /usr/lib/postgresql/16/bin/. The install_root +# passed by TAF::DatabaseSoftwareInstalls should already point to the +# correct versioned prefix; this routine only searches under that root. +################################################################################ +sub _find_binary { + my ($base, $binary) = @_; + + return undef unless defined $base && length $base; + return undef unless defined $binary && length $binary; + + my @paths = ( + File::Spec->catfile($base, "bin", $binary), + File::Spec->catfile($base, "sbin", $binary), + File::Spec->catfile($base, $binary), + ); + + for my $p (@paths) { + return $p if -e $p && -x $p; + } + + return undef; +} + +################################################################################ +# _run_command +# +# PURPOSE: +# Execute a system command from an array reference. Optionally redirects +# stdout/stderr to a logfile. Returns the normalized exit code. +################################################################################ +sub _run_command { + my ($self, $cmd_ref, $tag, $logfile) = @_; + my $_tag = "${_me}::_run_command($tag): "; + + my $cmd_str = join(' ', @$cmd_ref); + + if ($logfile) { + if (open(my $fh, '>>', $logfile)) { + print $fh "=== _run_command [$tag] ===\n"; + print $fh "$cmd_str\n"; + close $fh; + } + $cmd_str .= " >> \"$logfile\" 2>&1"; + } + + PrintVerbose("$_tag $cmd_str"); + + my $rc = system($cmd_str); + + if ($rc == -1) { + PrintError("$_tag Failed to execute: $!"); + return 1; + } + + my $exit = $rc >> 8; + + if ($exit != 0) { + PrintError("$_tag Exit code $exit"); + } + + return $exit; +} + +############################################################################# +# Module terminator +############################################################################# +1; diff --git a/libs/script_tools_lib/ClientCmakeBuild.pm b/libs/script_tools_lib/ClientCmakeBuild.pm index 3028713..94b1c0e 100644 --- a/libs/script_tools_lib/ClientCmakeBuild.pm +++ b/libs/script_tools_lib/ClientCmakeBuild.pm @@ -410,9 +410,7 @@ sub SetLibAndInclude { return _SetLibAndInclude_MySQLFamily($installDir); } elsif ($maker eq 'postgres' || $maker eq 'postgresql') { - # TO BE ADDED - DebugPrint("ERROR: PostgreSQL client builds are not supported by this module"); - return ERROR; + return _SetLibAndInclude_PostgreSQL($installDir); } elsif ($maker eq 'oracle') { # TO BE ADDED @@ -753,9 +751,74 @@ sub _DetectMakerFromInstallDir { return $maker if $path =~ m{/\Q$maker\E[^/]*}i; } + # Fallback: probe for pg_config to detect system-package PostgreSQL (e.g. /usr) + my $pg_config = File::Spec->catfile($installDir, 'bin', 'pg_config'); + return 'postgres' if -x $pg_config; + return undef; } +#------------------------------------------------------------------------------- +# Subroutine: _SetLibAndInclude_PostgreSQL +# +# PURPOSE: +# Resolve include and library directories for PostgreSQL client builds +# using pg_config. Sets $ENV{INC} and $ENV{LIB} for use by cmake. +# +# PARAMETERS: +# $installDir - Root of the PostgreSQL installation. +# +# RETURNS: +# OK - INC and LIB resolved via pg_config. +# ERROR - pg_config not found or returned invalid paths. +#------------------------------------------------------------------------------- +sub _SetLibAndInclude_PostgreSQL { + my ($installDir) = @_; + + DebugPrint("SetLibAndInclude - PostgreSQL family"); + + # Locate pg_config: first under installDir, then system-wide + my $pgConfig; + for my $candidate ( + File::Spec->catfile($installDir, 'bin', 'pg_config'), + '/usr/bin/pg_config', + ) { + if (-x $candidate) { + $pgConfig = $candidate; + last; + } + } + + unless ($pgConfig) { + DebugPrint("ERROR: pg_config not found under $installDir/bin or /usr/bin"); + return ERROR; + } + + DebugPrint("pg_config = $pgConfig"); + + my $includeDir = `$pgConfig --includedir 2>/dev/null`; + my $libDir = `$pgConfig --libdir 2>/dev/null`; + chomp($includeDir); + chomp($libDir); + + unless ($includeDir && -d $includeDir) { + DebugPrint("ERROR: pg_config --includedir returned invalid directory: '$includeDir'"); + return ERROR; + } + unless ($libDir && -d $libDir) { + DebugPrint("ERROR: pg_config --libdir returned invalid directory: '$libDir'"); + return ERROR; + } + + $ENV{INC} = $includeDir; + $ENV{LIB} = $libDir; + + DebugPrint("PostgreSQL INC = $includeDir"); + DebugPrint("PostgreSQL LIB = $libDir"); + + return OK; +} + ############################################################################# # Module terminator ############################################################################# diff --git a/libs/sql_libs/Executor.pm b/libs/sql_libs/Executor.pm index 61e7007..7c894b9 100644 --- a/libs/sql_libs/Executor.pm +++ b/libs/sql_libs/Executor.pm @@ -19,7 +19,7 @@ package sql_libs::Executor; # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 # # Licensed under the GNU General Public License, version 2 or later (GPLv2+). # See https://www.gnu.org/licenses/ for details. @@ -391,8 +391,11 @@ sub DbCreateDatabase { my $db = $ctx->{options}{database} or croak "DbCreateDatabase: ctx->{options}{database} is undefined"; + my $user = $ctx->{options}{db_user}; + my $sql = _LoadDialect($ctx, "create_database"); $sql =~ s/\{db\}/$db/g; + $sql =~ s/\{user\}/$user/g if defined $user; return DbExecuteNoReturnQuery($sql, $ctx); } @@ -570,6 +573,30 @@ sub _BuildCommand { croak "_BuildCommand requires db_port or db_socket in options" unless defined $connection; + my $maker = _NormalizeMaker($ctx->{taf_var}{db_maker} // ''); + + # PostgreSQL uses psql syntax which differs from MySQL-family clients + if ($maker eq 'postgres') { + my $cmd = ''; + $cmd .= "PGPASSWORD='$pass' " if defined $pass; + $cmd .= "$client -U $user"; + # Always use TCP loopback — pg_hba.conf allows 127.0.0.1; db_socket is MySQL-style + my $pg_port = $opt->{db_port} // 5432; + $cmd .= " -h 127.0.0.1 -p $pg_port"; + # Connect to maintenance database for DDL; -q suppresses notices + $cmd .= " -d postgres -q"; + # SSL for postgres + if ($opt->{ssl_enabled}) { + $cmd .= " --set=sslmode=require"; + $cmd .= " --set=sslrootcert=$opt->{ssl_ca}" if $opt->{ssl_ca}; + $cmd .= " --set=sslcert=$opt->{ssl_cert}" if $opt->{ssl_cert}; + $cmd .= " --set=sslkey=$opt->{ssl_key}" if $opt->{ssl_key}; + } + $cmd .= " $extra" if $extra; + $cmd .= " -c \"$sql\""; + return $cmd; + } + my $cmd = "$client -u $user"; if (defined $pass) { @@ -585,7 +612,7 @@ sub _BuildCommand { # SSL options (normalized earlier in TAF) if ($opt->{ssl_enabled}) { - if ($maker eq 'mariadb' || $maker eq 'mysql') { + if ($maker eq 'mariadb' || $maker eq 'mysql') { $cmd .= " --ssl-ca=$opt->{ssl_ca}" if $opt->{ssl_ca}; $cmd .= " --ssl-cert=$opt->{ssl_cert}" if $opt->{ssl_cert}; $cmd .= " --ssl-key=$opt->{ssl_key}" if $opt->{ssl_key}; diff --git a/libs/sql_libs/dialects/postgres.sql b/libs/sql_libs/dialects/postgres.sql index 4908aea..213804f 100644 --- a/libs/sql_libs/dialects/postgres.sql +++ b/libs/sql_libs/dialects/postgres.sql @@ -46,7 +46,114 @@ FROM pg_database; SELECT * FROM pg_stat_database; [create_database] -CREATE DATABASE {db}; +CREATE DATABASE {db} OWNER "{user}"; [drop_database] -DROP DATABASE IF EXISTS {db}; \ No newline at end of file +DROP DATABASE IF EXISTS {db}; + +[active_connections] +SELECT count(*) AS total, + state, + wait_event_type, + wait_event +FROM pg_stat_activity +WHERE pid <> pg_backend_pid() +GROUP BY state, wait_event_type, wait_event +ORDER BY total DESC; + +[wait_events] +SELECT wait_event_type, + wait_event, + count(*) AS count +FROM pg_stat_activity +WHERE wait_event IS NOT NULL + AND pid <> pg_backend_pid() +GROUP BY wait_event_type, wait_event +ORDER BY count DESC; + +[table_stats] +SELECT schemaname, + relname, + seq_scan, + seq_tup_read, + idx_scan, + idx_tup_fetch, + n_live_tup, + n_dead_tup, + last_autovacuum, + last_autoanalyze +FROM pg_stat_user_tables +ORDER BY seq_scan DESC; + +[index_usage] +SELECT schemaname, + tablename, + indexname, + idx_scan, + idx_tup_read, + idx_tup_fetch +FROM pg_stat_user_indexes +ORDER BY idx_scan DESC; + +[bgwriter_stats] +SELECT checkpoints_timed, + checkpoints_req, + checkpoint_write_time, + checkpoint_sync_time, + buffers_checkpoint, + buffers_clean, + maxwritten_clean, + buffers_backend, + buffers_backend_fsync, + buffers_alloc +FROM pg_stat_bgwriter; + +[lock_waits] +SELECT blocked.pid AS blocked_pid, + blocked.query AS blocked_query, + blocking.pid AS blocking_pid, + blocking.query AS blocking_query, + blocked.wait_event, + blocked.wait_event_type +FROM pg_stat_activity AS blocked +JOIN pg_stat_activity AS blocking + ON blocking.pid = ANY(pg_blocking_pids(blocked.pid)) +WHERE blocked.cardinality(pg_blocking_pids(blocked.pid)) > 0; + +[replication_lag] +SELECT client_addr, + state, + sent_lsn, + write_lsn, + flush_lsn, + replay_lsn, + (sent_lsn - replay_lsn) AS lag_bytes +FROM pg_stat_replication; + +[table_bloat_estimate] +SELECT schemaname, + tablename, + pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) AS total_size, + n_dead_tup, + n_live_tup, + ROUND(100.0 * n_dead_tup / NULLIF(n_live_tup + n_dead_tup, 0), 2) AS dead_pct +FROM pg_stat_user_tables +WHERE n_live_tup + n_dead_tup > 0 +ORDER BY dead_pct DESC NULLS LAST; + +[transaction_stats] +SELECT datname, + xact_commit, + xact_rollback, + blks_read, + blks_hit, + ROUND(100.0 * blks_hit / NULLIF(blks_hit + blks_read, 0), 2) AS cache_hit_pct, + tup_returned, + tup_fetched, + tup_inserted, + tup_updated, + tup_deleted, + conflicts, + deadlocks +FROM pg_stat_database +WHERE datname NOT IN ('template0', 'template1', 'postgres'); \ No newline at end of file diff --git a/libs/sql_libs/postgres.sql b/libs/sql_libs/postgres.sql new file mode 100644 index 0000000..53c40f5 --- /dev/null +++ b/libs/sql_libs/postgres.sql @@ -0,0 +1,157 @@ +# ====================================================================== +# MariaDB Foundation - SQL Dialect Definitions +# ---------------------------------------------------------------------- +# File: dialects/sql.dialect +# Purpose: +# Defines named SQL snippets used by TAF for MariaDB diagnostics, +# environment introspection, and database lifecycle operations. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 or later of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 +# +# Licensed under the GNU General Public License, version 2 or later (GPLv2+). +# See https://www.gnu.org/licenses/ for details. +# +# Notes: +# - All blocks must remain deterministic and contributor-proof. +# - Do not modify block names without updating all references. +# ====================================================================== +[version] +SELECT version(); + +[variables] +SHOW ALL; + +[row_count] +SELECT COUNT(*) FROM {table}; + +[db_size] +SELECT pg_database.datname AS db, + pg_database_size(pg_database.datname) AS size_bytes +FROM pg_database; + +[stats] +SELECT * FROM pg_stat_database; + +[create_database] +CREATE DATABASE {db} OWNER "{user}"; + +[drop_database] +DROP DATABASE IF EXISTS {db}; + +[active_connections] +SELECT count(*) AS total, + state, + wait_event_type, + wait_event +FROM pg_stat_activity +WHERE pid <> pg_backend_pid() +GROUP BY state, wait_event_type, wait_event +ORDER BY total DESC; + +[wait_events] +SELECT wait_event_type, + wait_event, + count(*) AS count +FROM pg_stat_activity +WHERE wait_event IS NOT NULL + AND pid <> pg_backend_pid() +GROUP BY wait_event_type, wait_event +ORDER BY count DESC; + +[table_stats] +SELECT schemaname, + relname, + seq_scan, + seq_tup_read, + idx_scan, + idx_tup_fetch, + n_live_tup, + n_dead_tup, + last_autovacuum, + last_autoanalyze +FROM pg_stat_user_tables +ORDER BY seq_scan DESC; + +[index_usage] +SELECT schemaname, + tablename, + indexname, + idx_scan, + idx_tup_read, + idx_tup_fetch +FROM pg_stat_user_indexes +ORDER BY idx_scan DESC; + +[bgwriter_stats] +SELECT checkpoints_timed, + checkpoints_req, + checkpoint_write_time, + checkpoint_sync_time, + buffers_checkpoint, + buffers_clean, + maxwritten_clean, + buffers_backend, + buffers_backend_fsync, + buffers_alloc +FROM pg_stat_bgwriter; + +[lock_waits] +SELECT blocked.pid AS blocked_pid, + blocked.query AS blocked_query, + blocking.pid AS blocking_pid, + blocking.query AS blocking_query, + blocked.wait_event, + blocked.wait_event_type +FROM pg_stat_activity AS blocked +JOIN pg_stat_activity AS blocking + ON blocking.pid = ANY(pg_blocking_pids(blocked.pid)) +WHERE blocked.cardinality(pg_blocking_pids(blocked.pid)) > 0; + +[replication_lag] +SELECT client_addr, + state, + sent_lsn, + write_lsn, + flush_lsn, + replay_lsn, + (sent_lsn - replay_lsn) AS lag_bytes +FROM pg_stat_replication; + +[table_bloat_estimate] +SELECT schemaname, + tablename, + pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) AS total_size, + n_dead_tup, + n_live_tup, + ROUND(100.0 * n_dead_tup / NULLIF(n_live_tup + n_dead_tup, 0), 2) AS dead_pct +FROM pg_stat_user_tables +WHERE n_live_tup + n_dead_tup > 0 +ORDER BY dead_pct DESC NULLS LAST; + +[transaction_stats] +SELECT datname, + xact_commit, + xact_rollback, + blks_read, + blks_hit, + ROUND(100.0 * blks_hit / NULLIF(blks_hit + blks_read, 0), 2) AS cache_hit_pct, + tup_returned, + tup_fetched, + tup_inserted, + tup_updated, + tup_deleted, + conflicts, + deadlocks +FROM pg_stat_database +WHERE datname NOT IN ('template0', 'template1', 'postgres'); diff --git a/libs/taf_libs/TAF/CommandLine.pm b/libs/taf_libs/TAF/CommandLine.pm index 6a7983e..f2570b5 100644 --- a/libs/taf_libs/TAF/CommandLine.pm +++ b/libs/taf_libs/TAF/CommandLine.pm @@ -333,6 +333,7 @@ sub ParseCommandLineOptions { # Debug / tooling #----------------------------------------------------------------------- "tools-debug" => \$tmp_ref->{tools_debug}, + "debug-print-config" => \$flags_ref->{debug_print_config}, #----------------------------------------------------------------------- # Info & commandline flags/options diff --git a/libs/taf_libs/TAF/Run.pm b/libs/taf_libs/TAF/Run.pm index 4dad428..ee81d4a 100644 --- a/libs/taf_libs/TAF/Run.pm +++ b/libs/taf_libs/TAF/Run.pm @@ -134,7 +134,7 @@ use strict; use warnings; use List::Util qw(max all); use sql_libs::Executor; -Executor->import(':all'); +sql_libs::Executor->import(':all'); use profile_libs::Runner; diff --git a/libs/taf_libs/TAF/Utilities.pm b/libs/taf_libs/TAF/Utilities.pm index 553555e..f5c2b88 100644 --- a/libs/taf_libs/TAF/Utilities.pm +++ b/libs/taf_libs/TAF/Utilities.pm @@ -177,8 +177,9 @@ our %PLUGIN_ALIASES = ( mariadbd => 'mariadb', mysql => 'mysql', mysqld => 'mysql', - postgres => 'postgres', - pgsql => 'postgres', + postgres => 'postgres', + pgsql => 'postgres', + postgresql => 'postgres', oracle => 'oracle', sqlplus => 'oracle', ); diff --git a/properties/default/sysbench_lua_default.properties b/properties/default/sysbench_lua_default.properties index 82e0b92..d6c93b3 100644 --- a/properties/default/sysbench_lua_default.properties +++ b/properties/default/sysbench_lua_default.properties @@ -135,7 +135,7 @@ sysbench_lua.ignore_errors = null # --------------------------------------------------------------------------- # Build / compile # --------------------------------------------------------------------------- -sysbench_lua.cmake_args = -DWITH_MYSQL=on -DCMAKE_BUILD_TYPE=Release +sysbench_lua.cmake_args = -DWITH_MYSQL=on -DWITH_PGSQL=on -DCMAKE_BUILD_TYPE=Release # --------------------------------------------------------------------------- # Debugging diff --git a/properties/mariadb/beta/sysbench_lua.properties b/properties/mariadb/beta/sysbench_lua.properties index a534aff..6c6660b 100644 --- a/properties/mariadb/beta/sysbench_lua.properties +++ b/properties/mariadb/beta/sysbench_lua.properties @@ -19,7 +19,7 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 # # Licensed under the GNU General Public License, version 2 or later (GPLv2+). # See https://www.gnu.org/licenses/ for details. @@ -121,7 +121,7 @@ taf.test_type=adhoc taf.tests=POINT_SELECT,OLTP_RO,OLTP_RW # Comma-separated list of thread counts to test. -taf.threads=4,8,16,32,64,128 +taf.threads=8,16,32,64,128 # Warmup duration in seconds. taf.warmup_duration=200 diff --git a/properties/mysql/beta/sysbench_lua.properties b/properties/mysql/beta/sysbench_lua.properties index 23430e3..f38af3f 100644 --- a/properties/mysql/beta/sysbench_lua.properties +++ b/properties/mysql/beta/sysbench_lua.properties @@ -19,7 +19,7 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 # # Licensed under the GNU General Public License, version 2 or later (GPLv2+). # See https://www.gnu.org/licenses/ for details. diff --git a/properties/postgresql/hammerdb_tprocc_pgsql.properties b/properties/postgresql/hammerdb_tprocc_pgsql.properties new file mode 100644 index 0000000..10d2b16 --- /dev/null +++ b/properties/postgresql/hammerdb_tprocc_pgsql.properties @@ -0,0 +1,61 @@ +############################################################################# +# hammerdb_tprocc_pgsql.properties +# +# Created: June 2026 +# +# This file is part of the Test Automation Framework (TAF). +# +# PURPOSE: +# Example TAF properties file for running HammerDB TPC-C benchmarks +# against a PostgreSQL database managed by TAF. +# +# USAGE: +# perl taf.pl --properties-file=properties/postgresql/hammerdb_tprocc_pgsql.properties \ +# --property=taf.action=init-start-db-run-tests +# +# NOTES: +# - Set taf.db_software_install_packages to your PostgreSQL package path. +# - HammerDB must be installed separately; set hammerdb_tprocc.hammerdb_dir. +# - pg_storedprocs=true improves TPC-C performance on PostgreSQL. +############################################################################# + +# --------------------------------------------------------------------------- +# TAF framework +# --------------------------------------------------------------------------- +taf.action = init-start-db-run-tests +taf.taf_db_makers_plugin = postgres + +# --------------------------------------------------------------------------- +# Database software install +# --------------------------------------------------------------------------- +# taf.db_software_install_packages = /path/to/postgresql-18.tar.gz +taf.db_port = 5432 + +# --------------------------------------------------------------------------- +# Database configuration +# --------------------------------------------------------------------------- +taf.db_config_file = database_config_files/postgresql/postgresql_default.conf + +# --------------------------------------------------------------------------- +# Test suite +# --------------------------------------------------------------------------- +taf.test_suite = hammerdb-tprocc + +# --------------------------------------------------------------------------- +# HammerDB TPC-C — PostgreSQL specific +# --------------------------------------------------------------------------- +hammerdb_tprocc.db_type = postgres +hammerdb_tprocc.warehouses = 100 + +# PostgreSQL-specific TPC-C optimizations +hammerdb_tprocc.pg_storedprocs = true +hammerdb_tprocc.pg_vacuum = true +hammerdb_tprocc.pg_oracompat = false +hammerdb_tprocc.pg_cituscompat = false + +# --------------------------------------------------------------------------- +# Workload +# --------------------------------------------------------------------------- +hammerdb_tprocc.def_threads = 8,16,32,64 +hammerdb_tprocc.def_duration = 300 +hammerdb_tprocc.rampup = 2 diff --git a/properties/postgresql/sysbench_lua_pgsql.properties b/properties/postgresql/sysbench_lua_pgsql.properties new file mode 100644 index 0000000..8ff5071 --- /dev/null +++ b/properties/postgresql/sysbench_lua_pgsql.properties @@ -0,0 +1,64 @@ +############################################################################# +# sysbench_lua_pgsql.properties +# +# Created: June 2026 +# +# This file is part of the Test Automation Framework (TAF). +# +# PURPOSE: +# Example TAF properties file for running Sysbench OLTP benchmarks +# against a PostgreSQL database managed by TAF. +# +# USAGE: +# perl taf.pl --properties-file=properties/postgresql/sysbench_lua_pgsql.properties \ +# --property=taf.action=init-start-db-run-tests +# +# NOTES: +# - Set taf.db_software_install_packages to the path of your PostgreSQL +# tarball or RPM packages before running. +# - Adjust thread counts and duration to suit your test host. +# - The default config uses postgresql_oltp.conf; swap for +# postgresql_minimal.conf on low-resource machines. +############################################################################# + +# --------------------------------------------------------------------------- +# TAF framework +# --------------------------------------------------------------------------- +taf.action = init-start-db-run-tests +taf.taf_db_makers_plugin = postgres + +# --------------------------------------------------------------------------- +# Database software install +# --------------------------------------------------------------------------- +# taf.db_software_install_packages = /path/to/postgresql-18.tar.gz +taf.db_port = 5432 + +# --------------------------------------------------------------------------- +# Database configuration +# --------------------------------------------------------------------------- +taf.db_config_file = database_config_files/postgresql/postgresql_default.conf + +# --------------------------------------------------------------------------- +# Test suite +# --------------------------------------------------------------------------- +taf.test_suite = sysbench-lua + +# --------------------------------------------------------------------------- +# Sysbench driver +# --------------------------------------------------------------------------- +sysbench_lua.db_driver = pgsql +sysbench_lua.connector = libpq + +# --------------------------------------------------------------------------- +# Workload +# --------------------------------------------------------------------------- +sysbench_lua.def_threads = 8,16,32,64,128 +sysbench_lua.def_duration = 300 +sysbench_lua.number_of_tables = 8 +sysbench_lua.number_of_rows = 1000000 +sysbench_lua.oltp_skip_trx = on + +# --------------------------------------------------------------------------- +# Tests to run (standard OLTP subset compatible with PostgreSQL) +# --------------------------------------------------------------------------- +taf.tests = OLTP_RO,OLTP_RW,UPDATE_KEY,UPDATE_NO_KEY,POINT_SELECT diff --git a/taf.pl b/taf.pl index 3706be3..a756c87 100644 --- a/taf.pl +++ b/taf.pl @@ -409,6 +409,7 @@ list_test_suites_help => FALSE, list_test_types => FALSE, list_version => FALSE, + debug_print_config => FALSE, purge_archive => FALSE, purge_data_directory => FALSE, purge_results_directory => FALSE, @@ -1132,6 +1133,57 @@ sub _LoadProperties{ # Apply commandline overrides again, now that all properties are known TAF::Properties::ApplyOverrides($ctx, $tmpoptions_ref); + + # Dump the fully resolved configuration when requested + main::_PrintDebugConfig(); +} + +############################################################################### +# _PrintDebugConfig +# +# PURPOSE: +# When --debug-print-config is given, dump the fully resolved %options, +# %dirs, and %files hashes, plus the full process environment (%ENV), to +# STDERR. %options/%dirs/%files are the merged result of default +# properties, user properties, and command-line overrides -- the same +# state the rest of the framework operates on from this point on. %ENV is +# included because TAF and the DB software it drives both pick up +# behavior from inherited environment variables (paths, locale, etc.) +# that never go through the properties/CLI system at all. +# +# CONTRACT: +# - Must run only after _LoadProperties has merged all property sources. +# - Must print to STDERR, never STDOUT. +# - Must not terminate the run; this is a diagnostic side effect only. +# - Must redact known secret-shaped keys (passwords). +# - Must print every line as a YAML comment ("# ..."), with each +# section's entries indented 4 spaces per level, so the dump can be +# pasted straight into a YAML file (e.g. result.yaml) as a readable +# comment block instead of opaque "key = value" text. +############################################################################### +sub _PrintDebugConfig { + return unless $flags{debug_print_config}; + + # Case-insensitive substring match, not an exact-key list: %ENV in + # particular carries secrets under names %options/%dirs/%files never + # use (PGPASSWORD, MYSQL_PWD, AWS_SECRET_*, ...). Mirrors (and adds PWD + # to) get_envinfo_standalone.py's SENSITIVE_ENV_RE -- MYSQL_PWD is what + # collect_db_config.sh itself sets to authenticate, and "PASS" alone + # doesn't match it. + my $redact_re = qr/(AUTH|COOKIE|CREDENTIAL|PASS|PWD|PRIVATE|SECRET|TOKEN)/i; + + print STDERR "\n# === TAF RESOLVED CONFIGURATION (--debug-print-config) ===\n"; + for my $section (["options", \%options], ["dirs", \%dirs], ["files", \%files], ["ENV", \%ENV]) { + my ($name, $href) = @$section; + print STDERR "# $name:\n"; + for my $key (sort keys %$href) { + my $value = $href->{$key}; + $value = defined($value) ? $value : ''; + $value = '***REDACTED***' if $key =~ $redact_re && $value ne '' && $value ne ''; + print STDERR "# $key: $value\n"; + } + } + print STDERR "# === END TAF RESOLVED CONFIGURATION ===\n\n"; } ############################################################################### diff --git a/test_suites/sysbench-lua.pm b/test_suites/sysbench-lua.pm index bed6e50..410aad5 100644 --- a/test_suites/sysbench-lua.pm +++ b/test_suites/sysbench-lua.pm @@ -19,7 +19,7 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 # # Licensed under the GNU General Public License, version 2 or later (GPLv2+). # See https://www.gnu.org/licenses/ for details. @@ -92,6 +92,7 @@ our $ctx = undef; #----------------------------------------------------------------------------- use Cwd; +use File::Basename qw(dirname); use threads; use constant IS_WINDOWS => ($^O =~ /^(mswin)/oi); use FindBin qw($Bin); @@ -923,10 +924,10 @@ sub Help { Print("\tare included in each request."); Print("\tExample: To run 3 simple-range queries per request instead of 1,"); Print("\tset: sysbench_lua.oltp_simple_ranges=3\n"); - + Print("\tPOINTS-COVERED-PK \n"); Print("\tsysbench_lua.random_points_ranges\n"); - + Print("\tPOINTS-COVERED-SI \n"); Print("\tsysbench_lua.random_points_ranges\n"); @@ -943,7 +944,7 @@ sub Help { Print("\tRANGE-COVERED-SI \n"); Print("\t\tsysbench_lua.random_points_ranges\n"); - + Print("\tRANGE-NOTCOVERED-PK \n"); Print("\t\tsysbench_lua.random_points_ranges\n"); @@ -954,7 +955,7 @@ sub Help { Print("\tRANDOM-POINTS \n"); Print("\t\tysbench_lua.random_points_ranges\n"); - + Print("\tHOT-POINTS \n"); Print("\t\tsysbench_lua.random_points_ranges\n"); @@ -1354,13 +1355,15 @@ sub ValidateTargetWithSuite { return OK; } - my $expected = $tsOpt{db_driver}; - if (lc($incoming) eq lc($expected)) { - PrintVerbose($vt."db_driver match db maker $incoming, returning OK."); + my $expected = $tsOpt{db_driver}; + my $normalized = NormalizeDBType($incoming) // lc($incoming); + my $expected_normalized = NormalizeDBType($expected) // lc($expected); + if ($normalized eq $expected_normalized) { + PrintVerbose($vt."db_driver match db maker $incoming (normalized: $normalized), returning OK."); StageEnd($vt); return OK; } else { - PrintError($vt."Mismatch: sysbench_lua.db_driver = $expected, db install shows $incoming"); + PrintError($vt."Mismatch: sysbench_lua.db_driver = $expected, db install shows $incoming (normalized: $normalized)"); return ERROR; } } @@ -1630,7 +1633,7 @@ sub CheckTestsForUpdateRange{ sub ConfigureBMKTestCase{ my ($test_uc) = @_; $test_uc = uc($test_uc); - + my $_cbmk = StageStart($_me." -> ConfigureBMKTestCase ->"); # Here, we handle only BMK-only tests (ie: not optional use_bmk tests) if ($bmkFlags{bmk_sec_index_test_case}) { @@ -1645,25 +1648,25 @@ sub ConfigureBMKTestCase{ if ($test_uc eq "BMK_RW_UPDATE_RANGE") { $tsOpt{oltp_lua_script} = "OLTP_RW$trx_suffix"; - + } elsif ($test_uc eq "BMK_WO_UPDATE_RANGE") { $tsOpt{oltp_lua_script} = "OLTP_RW-write_only$trx_suffix"; - + } elsif ($test_uc eq "BMK_RW_UPDATE_INDEX_RANGE") { $tsOpt{oltp_lua_script} = "OLTP_RW-index_updates$trx_suffix"; - + } elsif ($test_uc eq "BMK_RW_UPDATE_NON_INDEX_RANGE") { $tsOpt{oltp_lua_script} = "OLTP_RW-non_index_updates$trx_suffix"; - + } elsif ($test_uc eq "BMK_RW_PS_UPDATE_RANGE") { $tsOpt{oltp_lua_script} = "OLTP_RW-point_selects$trx_suffix"; - + } elsif ($test_uc eq "BMK_RW_PS_UPDATE_INDEX_RANGE") { $tsOpt{oltp_lua_script} = "OLTP_RW-point_selects-non_index_updates$trx_suffix"; - + } elsif ($test_uc eq "BMK_RW_PS_UPDATE_NON_INDEX_RANGE") { $tsOpt{oltp_lua_script} = "OLTP_RO-non_index_updates$trx_suffix"; - + } elsif ($test_uc eq "CONNECT") { $tsOpt{oltp_lua_script} = "OLTP_RO-point_selects_reconnect$trx_suffix"; $tsOpt{test_args} .= " --point-selects=1 "; @@ -1671,7 +1674,7 @@ sub ConfigureBMKTestCase{ $tsOpt{test_args} .= " --sum-ranges=0 "; $tsOpt{test_args} .= " --order-ranges=0 "; $tsOpt{test_args} .= " --distinct-ranges=0 "; - + } else { PrintError($_cbmk." Invalid test: $test"); return ERROR; @@ -1722,10 +1725,10 @@ sub ConfigureStdTestCase{ # Here, we handle tests which are in the base lua set (and may also be in BMK-kit) my $trx_flag = $tsOpt{oltp_skip_trx}; my $use_bmk = $tsOpt{use_bmk}; - + # Helper for transactional suffix my $trx_suffix = ($trx_flag eq "off") ? "-trx.lua" : "-notrx.lua"; - + # POINT_SELECT if ($test_uc eq "POINT_SELECT") { $tsOpt{oltp_lua_script} = $use_bmk ? "OLTP_RO-point_selects$trx_suffix" : "oltp_point_select.lua"; @@ -1735,7 +1738,7 @@ sub ConfigureStdTestCase{ $tsOpt{test_args} .= " --sum-ranges=0"; $tsOpt{test_args} .= " --order-ranges=0"; $tsOpt{test_args} .= " --distinct-ranges=0"; - + # PARSER } elsif ($test_uc eq "PARSER") { $tsOpt{oltp_lua_script} = "oltp_point_select.lua"; @@ -1789,7 +1792,7 @@ sub ConfigureStdTestCase{ } elsif ($test_uc eq "OLTP_INSERT_INTO") { $tsOpt{oltp_lua_script} = "oltp_insert_into.lua"; $tsOpt{test_args} = " --skip-trx=$trx_flag"; - + # OLTP_RW } elsif ($test_uc eq "OLTP_RW") { $tsOpt{oltp_lua_script} = "oltp_read_write.lua"; @@ -1814,7 +1817,7 @@ sub ConfigureStdTestCase{ } elsif ($test_uc eq "INSERT") { $tsOpt{oltp_lua_script} = "oltp_insert.lua"; $tsOpt{test_args} = " --skip-trx=$trx_flag"; - + # DELETE } elsif ($test_uc eq "DELETE") { $tsOpt{oltp_lua_script} = "oltp_delete.lua"; @@ -1899,19 +1902,19 @@ sub ConfigureStdTestCase{ $tsOpt{test_args} .= " --sum-ranges=$tsOpt{oltp_sum_ranges}"; $tsOpt{test_args} .= " --order-ranges=$tsOpt{oltp_order_ranges}"; $tsOpt{test_args} .= " --distinct-ranges=$tsOpt{oltp_distinct_ranges}"; - #POINTS-COVERED-PK + #POINTS-COVERED-PK } elsif ($test_uc eq "POINTS-COVERED-PK") { $tsOpt{oltp_lua_script} = "oltp_points_covered.lua"; $tsOpt{test_args} = " --skip-trx"; $tsOpt{test_args} .= " --on-id=true"; $tsOpt{test_args} .= " --random-points=$tsOpt{random_points_ranges}"; - #POINTS-COVERED-SI + #POINTS-COVERED-SI } elsif ($test_uc eq "POINTS-COVERED-SI") { $tsOpt{oltp_lua_script} = "oltp_points_covered.lua"; $tsOpt{test_args} = " --skip-trx"; $tsOpt{test_args} .= " --on-id=false"; $tsOpt{test_args} .= " --random-points=$tsOpt{random_points_ranges}"; - #POINTS-NOTCOVERED-PK + #POINTS-NOTCOVERED-PK } elsif ($test_uc eq "POINTS-NOTCOVERED-PK") { $tsOpt{oltp_lua_script} = "oltp_points_covered.lua"; $tsOpt{test_args} = " --skip-trx"; @@ -1925,19 +1928,19 @@ sub ConfigureStdTestCase{ $tsOpt{test_args} .= " --on-id=false"; $tsOpt{test_args} .= " --covered=false"; $tsOpt{test_args} .= " --random-points=$tsOpt{random_points_ranges}"; - #RANGE-COVERED-PK + #RANGE-COVERED-PK } elsif ($test_uc eq "RANGE-COVERED-PK") { $tsOpt{oltp_lua_script} = "oltp_range_covered.lua"; $tsOpt{test_args} = " --skip-trx"; $tsOpt{test_args} .= " --on-id=true"; $tsOpt{test_args} .= " --random-points=$tsOpt{random_points_ranges}"; - #RANGE-COVERED-SI + #RANGE-COVERED-SI } elsif ($test_uc eq "RANGE-COVERED-SI") { $tsOpt{oltp_lua_script} = "oltp_range_covered.lua"; $tsOpt{test_args} = " --skip-trx"; $tsOpt{test_args} .= " --on-id=false"; $tsOpt{test_args} .= " --random-points=$tsOpt{random_points_ranges}"; - #RANGE-NOTCOVERED-PK + #RANGE-NOTCOVERED-PK } elsif ($test_uc eq "RANGE-NOTCOVERED-PK") { $tsOpt{oltp_lua_script} = "oltp_range_covered.lua"; $tsOpt{test_args} = " --skip-trx"; @@ -2578,24 +2581,60 @@ sub SetConnectionArgs { my $args = ""; # Base driver - # Normalize MariaDB aliases to mysql + # Normalize MariaDB aliases to mysql; normalize PostgreSQL aliases to pgsql if ($tsOpt{db_driver} =~ /^maria(db)?$/i) { $tsOpt{db_driver} = "mysql"; } + elsif ($tsOpt{db_driver} =~ /^(postgres|postgresql)$/i) { + $tsOpt{db_driver} = "pgsql"; + } $args .= "$tsState{target_lua} --db-driver=" . $tsOpt{db_driver}; - # Connection method - if ($options{db_clients_use_unix_socket}) { - $args .= " --mysql-socket='" . $options{db_socket} . "'"; + # Connection parameters — branched by driver family + if ($tsOpt{db_driver} eq 'pgsql') { + + # PostgreSQL uses --pgsql-* flags. drv_pgsql.c passes --pgsql-host + # straight into PQsetdbLogin(), and libpq treats a value starting + # with '/' as a unix-socket DIRECTORY rather than a hostname (unlike + # libmysqlclient below, "localhost" here would still mean TCP) -- + # libpq then derives the actual socket filename (.s.PGSQL.) + # by joining that directory with the port. $options{db_socket} is a + # single FILE-shaped path (Utilities.pm defaults it to + # "db.sock", matching MariaDB's single-socket-file + # convention), not a directory -- passing it straight through here + # made libpq look for ".../tmp/db.sock/.s.PGSQL.", which never + # exists. Use its directory instead: postgres.pm configures + # postgresql.conf's unix_socket_directories to that same tmpdir + # (see _db_apply_postgresql_conf), so this is where the server + # actually creates its socket. pg_hba.conf already allows the + # connection ("local all all md5", see + # postgres.pm::_db_write_pg_hba_conf). The port is still required + # even for a socket connection: libpq needs it to build the + # ".s.PGSQL." filename. + if ($options{db_clients_use_unix_socket}) { + $args .= " --pgsql-host='" . dirname($options{db_socket} || '/var/run/postgresql') . "'"; + } else { + $args .= " --pgsql-host='127.0.0.1'"; + } + $args .= " --pgsql-port=" . $options{db_port}; + $args .= " --pgsql-user='" . $options{db_user} . "'"; + $args .= " --pgsql-password='" . $options{db_user_pass} . "'"; + $args .= " --pgsql-db='" . $tmpDatabase . "'"; + } else { - $args .= " --mysql-host='" . $options{host} . "'"; - $args .= " --mysql-port=" . $options{db_port}; - } - # Credentials - $args .= " --mysql-user='" . $options{db_user} . "'"; - $args .= " --mysql-password='" . $options{db_user_pass} . "'"; - $args .= " --mysql-db='" . $tmpDatabase . "'"; + # MySQL / MariaDB + if ($options{db_clients_use_unix_socket}) { + $args .= " --mysql-socket='" . $options{db_socket} . "'"; + } else { + $args .= " --mysql-host='" . $options{host} . "'"; + $args .= " --mysql-port=" . $options{db_port}; + } + + $args .= " --mysql-user='" . $options{db_user} . "'"; + $args .= " --mysql-password='" . $options{db_user_pass} . "'"; + $args .= " --mysql-db='" . $tmpDatabase . "'"; + } # Execution mode if (!$options{use_request_based}) { @@ -2614,16 +2653,18 @@ sub SetConnectionArgs { # Partitioning $args .= " --oltp-num-partitions=" . $tsOpt{number_of_partitions} if defined $tsOpt{number_of_partitions}; - # Shutdown behavior - if($tsOpt{forced_shutdown}){ - $args .= " --forced-shutdown=" . $tsOpt{forced_shutdown_sec} if defined $tsOpt{forced_shutdown_sec}; + # Shutdown behavior (MySQL/MariaDB only — not supported by pgsql driver) + if ($tsOpt{db_driver} ne 'pgsql' && $tsOpt{forced_shutdown}) { + $args .= " --forced-shutdown=" . $tsOpt{forced_shutdown_sec} if defined $tsOpt{forced_shutdown_sec}; } - # Errors to ignore - $args .= " --mysql-ignore-errors=" . $tsOpt{ignore_errors} if defined $tsOpt{ignore_errors}; + # Storage engine (MySQL/MariaDB only — PostgreSQL has no storage engine concept) + if ($tsOpt{db_driver} ne 'pgsql') { + # Errors to ignore + $args .= " --mysql-ignore-errors=" . $tsOpt{ignore_errors} if defined $tsOpt{ignore_errors}; - # Storage engine - $args .= " --mysql-storage-engine=" . lc($options{db_engine}) if defined $options{db_engine}; + $args .= " --mysql-storage-engine=" . lc($options{db_engine}) if defined $options{db_engine}; + } # Per-table CREATE TABLE options (e.g. TidesDB table options) if (defined $tsOpt{create_table_options} && length $tsOpt{create_table_options}) { @@ -2653,22 +2694,29 @@ sub SetConnectionArgs { } $args .= " --thread-init-timeout=" . $tsOpt{thread_init_timeout}; - $args .= " --mysql-ssl=" .$tsOpt{bmk_mysql_ssl} if defined $tsOpt{bmk_mysql_ssl}; + # BMK mysql-ssl flag applies to MySQL/MariaDB only + $args .= " --mysql-ssl=" .$tsOpt{bmk_mysql_ssl} + if defined $tsOpt{bmk_mysql_ssl} && $tsOpt{db_driver} ne 'pgsql'; $args .= " --sync-file='" . $tsOpt{bmk_sync_file} . "'" if defined $tsOpt{bmk_sync_file}; $args .= " --sync-wait=" . $tsOpt{bmk_sync_file_wait_timeout_ms} if defined $tsOpt{bmk_sync_file_wait_timeout_ms}; } else { - $args .= " --mysql-ssl" if defined $tsOpt{mysql_ssl}; + # MySQL/MariaDB non-BMK SSL flag + $args .= " --mysql-ssl" if defined $tsOpt{mysql_ssl} && $tsOpt{db_driver} ne 'pgsql'; } - # SSL certs - $args .= " --mysql-ssl-ca='" . $tsOpt{mysql_ssl_ca} . "'" if defined $tsOpt{mysql_ssl_ca}; - $args .= " --mysql-ssl-cert='" . $tsOpt{mysql_ssl_cert} . "'" if defined $tsOpt{mysql_ssl_cert}; - $args .= " --mysql-ssl-key='" . $tsOpt{mysql_ssl_key} . "'" if defined $tsOpt{mysql_ssl_key}; + # SSL certs — MySQL/MariaDB specific flags; pgsql SSL is configured via postgresql.conf + if ($tsOpt{db_driver} ne 'pgsql') { + $args .= " --mysql-ssl-ca='" . $tsOpt{mysql_ssl_ca} . "'" if defined $tsOpt{mysql_ssl_ca}; + $args .= " --mysql-ssl-cert='" . $tsOpt{mysql_ssl_cert} . "'" if defined $tsOpt{mysql_ssl_cert}; + $args .= " --mysql-ssl-key='" . $tsOpt{mysql_ssl_key} . "'" if defined $tsOpt{mysql_ssl_key}; + } - # Charset and partitioning - $args .= " --mysql-table-partitions=" . $tsOpt{bmk_partitions} if $tsOpt{bmk_partitions} > ZERO; - $args .= " --mysql-check-charset=1" if $tsOpt{bmk_check_character_set} > ZERO; + # Charset and partitioning (MySQL/MariaDB only) + if ($tsOpt{db_driver} ne 'pgsql') { + $args .= " --mysql-table-partitions=" . $tsOpt{bmk_partitions} if $tsOpt{bmk_partitions} > ZERO; + $args .= " --mysql-check-charset=1" if $tsOpt{bmk_check_character_set} > ZERO; + } # Debug flags $args .= " --debug=on" if $tsOpt{debug_sysbench}; @@ -2740,7 +2788,7 @@ sub SetLoadArgs { PrintVerbose($_sla."Lua Script Directory = ".$tsOpt{lua_scripts_dir}); PrintVerbose($_sla."Lua Script = ".$tsOpt{oltp_lua_script}); my $sysbench_args = "'$tsState{target_lua} $tsOpt{args}'"; - $tsOpt{load_args} = $args; + $tsOpt{load_args} = $args; $tsOpt{load_args} .= " --sysbench-args=" . $sysbench_args; $tsOpt{load_args} = $args; PrintVerbose($_sla . "Load Args: ".$tsOpt{load_args}); @@ -2839,7 +2887,7 @@ sub SingleTestRun { $test_case //= ''; $m_threads = int($m_threads // 0) || 1; # ensure a positive integer $m_runType //= ''; - + my $_str = StageStart($_me." -> SingleTestRun ->"); my $m_duration = $options{duration}; $m_runType = uc($m_runType); @@ -2899,6 +2947,13 @@ sub VerifyOptions { my $_vo = "$_me -> VerifyOptions ->"; # Validate oltp_skip_trx + # PostgreSQL sysbench driver does not support skip-trx; force it off. + if (defined $tsOpt{db_driver} && $tsOpt{db_driver} eq 'pgsql') { + if (lc($tsOpt{oltp_skip_trx}) eq 'on') { + PrintWarning($_vo."oltp_skip_trx=on is not supported by pgsql driver; forcing to off"); + $tsOpt{oltp_skip_trx} = "off"; + } + } if (lc($tsOpt{oltp_skip_trx}) ne "on" && lc($tsOpt{oltp_skip_trx}) ne "off") { PrintError($_vo."Invalid value for oltp_skip_trx: $tsOpt{oltp_skip_trx}"); PrintVerbose($_vo."Must be \"on\" or \"off\""); @@ -2958,6 +3013,34 @@ sub VerifyOptions { return OK; } +################################################################################ +# NormalizeDBType +# +# PURPOSE: +# Normalize an incoming database type string to the canonical sysbench +# driver name used in --db-driver. Called by ValidateTargetWithSuite(). +# +# CANONICAL MAPPINGS: +# mariadb, maria, mariadbd -> mysql +# mysql, mysqld -> mysql +# postgres, postgresql, +# pgsql -> pgsql +# +# RETURNS: +# Canonical driver string on success; undef on unknown input. +################################################################################ +sub NormalizeDBType { + my ($t) = @_; + return undef unless defined $t && length $t; + $t = lc $t; + $t =~ s/^\s+|\s+$//g; + + return "mysql" if $t =~ /^(mariadb|maria|mariadbd|mysql|mysqld)$/; + return "pgsql" if $t =~ /^(postgres|postgresql|pgsql)$/; + + return undef; +} + ############################################################################# # Module terminator ############################################################################# diff --git a/tests/run_tests.sh b/tests/run_tests.sh new file mode 100755 index 0000000..6d9c133 --- /dev/null +++ b/tests/run_tests.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# ============================================================================= +# run_tests.sh — Runs the complete TAF PostgreSQL test suite +# +# Usage: +# bash tests/run_tests.sh [pytest arguments...] +# +# Examples: +# bash tests/run_tests.sh # all tests +# bash tests/run_tests.sh -v # verbose +# bash tests/run_tests.sh -v -k TestL5 # L5 only +# bash tests/run_tests.sh --co -q # list tests only +# +# Env variables (override automatic detection): +# TAF_PG_INSTALL_DIR PostgreSQL installation directory +# TAF_PG_PORT port for TAF-managed PG (default: 5433) +# TAF_MARIADB_DIR (optional) for L6 MariaDB regression test +# ============================================================================= +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TAF_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +# --------------------------------------------------------------------------- +# PostgreSQL installation detection +# --------------------------------------------------------------------------- +detect_pg_install() { + # 1. Respect explicitly set variable + if [[ -n "${TAF_PG_INSTALL_DIR:-}" ]]; then + echo "${TAF_PG_INSTALL_DIR}" + return + fi + + # 2. Source .taf_pg_env if it exists (created by setup_almalinux10.sh) + if [[ -f "${TAF_ROOT}/.taf_pg_env" ]]; then + # shellcheck source=/dev/null + source "${TAF_ROOT}/.taf_pg_env" + if [[ -n "${TAF_PG_INSTALL_DIR:-}" ]]; then + echo "${TAF_PG_INSTALL_DIR}" + return + fi + fi + + # 3. Look for Percona tarball (default installation method) + if [[ -x "/opt/pgdistro/percona-postgresql16/bin/postgres" ]]; then + echo "/opt/pgdistro/percona-postgresql16" + return + fi + + # 4. PGDG RPM + for dir in /usr/pgsql-16 /usr/pgsql-15 /usr/pgsql-14; do + if [[ -x "${dir}/bin/postgres" ]]; then + echo "${dir}" + return + fi + done + + # 5. AppStream / system PG + if [[ -x "/usr/bin/postgres" ]]; then + echo "/usr" + return + fi + + echo "" +} + +# --------------------------------------------------------------------------- +# Prerequisites check +# --------------------------------------------------------------------------- +RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; CYAN='\033[0;36m'; NC='\033[0m' +info() { echo -e "${GREEN}[TEST]${NC} $*"; } +warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } +error() { echo -e "${RED}[ERROR]${NC} $*" >&2; exit 1; } + +cd "${TAF_ROOT}" + +PG_INSTALL="$(detect_pg_install)" +if [[ -z "$PG_INSTALL" ]]; then + error "PostgreSQL not found. Run first: sudo bash tests/setup_almalinux10.sh" +fi +export TAF_PG_INSTALL_DIR="${PG_INSTALL}" +export TAF_PG_PORT="${TAF_PG_PORT:-5433}" + +# Add PG bin and lib to PATH/LD_LIBRARY_PATH +export PATH="${PG_INSTALL}/bin:${PATH}" +PG_LIB="${PG_INSTALL}/lib" +if [[ -d "${PG_LIB}" ]]; then + export LD_LIBRARY_PATH="${PG_LIB}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +fi + +# Virtuozzo/VzLinux: add system site-packages to Percona Python +PERCONA_PY_SITEPKGS="/opt/percona-python3/lib/python3.12/site-packages" +if [[ -d "$PERCONA_PY_SITEPKGS" ]]; then + PTH_FILE="${PERCONA_PY_SITEPKGS}/system-sitepackages.pth" + if [[ ! -f "$PTH_FILE" ]]; then + echo "/usr/lib/python3.12/site-packages" > "$PTH_FILE" + echo "/usr/lib64/python3.12/site-packages" >> "$PTH_FILE" + fi +fi + +info "PostgreSQL: ${PG_INSTALL} ($(pg_config --version 2>/dev/null || echo 'version unknown'))" +info "Port: ${TAF_PG_PORT}" +info "Sysbench: ${TAF_ROOT}/client_source/sysbench-lua/sysbench" +[[ -n "${TAF_MARIADB_DIR:-}" ]] && info "MariaDB: ${TAF_MARIADB_DIR} (L6 enabled)" + +# Sysbench binary check +SYSBENCH_BIN="${TAF_ROOT}/client_source/sysbench-lua/sysbench" +if [[ ! -x "${SYSBENCH_BIN}" ]]; then + warn "Sysbench not found (${SYSBENCH_BIN})" + warn "L4/L5 tests will be skipped. Run setup_almalinux10.sh to build." +fi + +# Verify pytest +python3 -m pytest --version >/dev/null 2>&1 || \ + error "pytest not found. Run: pip3 install pytest" + +# --------------------------------------------------------------------------- +# Running tests +# --------------------------------------------------------------------------- +echo "" +echo -e "${CYAN}══════════════════════════════════════════════════${NC}" +echo -e "${CYAN} TAF PostgreSQL Integration Tests${NC}" +echo -e "${CYAN}══════════════════════════════════════════════════${NC}" +echo "" + +PYTEST_ARGS=("tests/test_taf_postgresql.py") + +# Default verbose if no arguments provided +if [[ $# -eq 0 ]]; then + PYTEST_ARGS+=("-v" "--tb=short") +else + PYTEST_ARGS+=("$@") +fi + +python3 -m pytest "${PYTEST_ARGS[@]}" diff --git a/tests/setup_almalinux10.sh b/tests/setup_almalinux10.sh new file mode 100755 index 0000000..e8f9c9a --- /dev/null +++ b/tests/setup_almalinux10.sh @@ -0,0 +1,508 @@ +#!/usr/bin/env bash +# ============================================================================= +# setup_almalinux10.sh — Prerequisites for TAF PostgreSQL tests +# +# Target: RHEL/AlmaLinux/Virtuozzo 8–10 (x86_64, aarch64) +# Usage: sudo bash tests/setup_almalinux10.sh [--method=percona|pgdg|appstream] +# +# PostgreSQL installation methods (--method): +# percona (default) — tarball from downloads.percona.com +# https://docs.percona.com/postgresql/18/tarball.html +# pgdg — RPM from pgdg.postgresql.org +# appstream — system postgresql from dnf (AlmaLinux 10 AppStream ships a +# versioned postgresql18 package alongside the unversioned +# postgresql (16) one; --allowerasing swaps 16 out for 18) +# +# EXPECTED_PG_VERSION (below) is pinned to 18.4 -- the current stable +# PostgreSQL release as of 2026-07 (https://www.postgresql.org/docs/release/18.4/, +# released 2026-05-14). Update it here when a newer release ships. All three +# methods are verified against this after install; a mismatch is a hard error +# (see step 3b) rather than a silent partial upgrade. +# +# After completion: +# - PostgreSQL 18.4 available in $PG_INSTALL_DIR +# - Python 3 + pytest installed +# - Build tools for sysbench ready +# - Env saved to .taf_pg_env (source before tests) +# +# Env variables (can be overridden before pytest): +# TAF_PG_INSTALL_DIR (set automatically) +# TAF_PG_PORT (default: 5433) +# TAF_MARIADB_DIR (optional, for L6 regression test) +# PERCONA_LOCAL_ARCHIVE (--method=percona only) path to an already-downloaded +# percona-postgresql-*.tar.gz; skips the +# downloads.percona.com fetch. Set by taf_manage.py +# --PERCONA_ARCHIVE_LOCAL, which SCPs it here once from +# the control machine instead of every guest fetching +# it independently. +# ============================================================================= +set -euo pipefail + +# --------------------------------------------------------------------------- +# Colors and helper functions +# --------------------------------------------------------------------------- +RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; CYAN='\033[0;36m'; NC='\033[0m' +info() { echo -e "${GREEN}[SETUP]${NC} $*"; } +warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } +error() { echo -e "${RED}[ERROR]${NC} $*" >&2; exit 1; } +step() { echo -e "\n${CYAN}━━━ $* ━━━${NC}"; } + +[[ $EUID -eq 0 ]] || error "Script must be run as root (sudo bash $0)" + +# --------------------------------------------------------------------------- +# Expected PostgreSQL version (pre-test gate, step 3b) +# --------------------------------------------------------------------------- +# Pinned to the current stable PostgreSQL release. Verified 2026-07 against +# https://www.postgresql.org/docs/release/18.4/ (released 2026-05-14). +# Update this when a newer release ships -- installs of any --method that +# don't produce this exact version fail hard rather than silently running +# benchmarks against an unintended/outdated PostgreSQL version. +EXPECTED_PG_VERSION="18.4" + +# --------------------------------------------------------------------------- +# Parametry +# --------------------------------------------------------------------------- +METHOD="percona" +for arg in "$@"; do + case "$arg" in + --method=percona) METHOD=percona ;; + --method=pgdg) METHOD=pgdg ;; + --method=appstream) METHOD=appstream ;; + --help|-h) + echo "Usage: sudo bash $0 [--method=percona|pgdg|appstream]" + echo " percona (default) Tarball from downloads.percona.com" + echo " pgdg RPM from pgdg.postgresql.org" + echo " appstream System postgresql from dnf" + exit 0 ;; + *) warn "Unknown parameter: $arg" ;; + esac +done + +ARCH=$(uname -m) +TAF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +info "PG installation method: ${CYAN}${METHOD}${NC}" +info "Architecture: ${ARCH}" +info "TAF directory: ${TAF_DIR}" + +# --------------------------------------------------------------------------- +# 1. BASE DEPENDENCIES +# --------------------------------------------------------------------------- +step "Installing base dependencies" +dnf install -y epel-release 2>/dev/null || true +dnf install -y \ + gcc gcc-c++ make cmake automake libtool pkg-config \ + perl perl-devel \ + python3 python3-pip \ + git wget curl tar \ + libaio-devel readline-devel \ + openssl openssl-devel \ + acl + +# --------------------------------------------------------------------------- +# 1b. "postgres" OS user/group +# --------------------------------------------------------------------------- +# The appstream/pgdg RPM packages create this via their own %pre scriptlet, +# but --method=percona just extracts a tarball -- nothing ever creates it. +# postgres.pm's new() constructor requires an OS user literally named +# "postgres" to drop root privileges before running initdb (PostgreSQL +# refuses `initdb`/`postgres` as root unconditionally); without it, initdb +# runs as root and fails with "initdb: error: cannot be run as root" on +# every single host. Mirrors the standard RHEL/Fedora postgresql-server RPM +# %pre scriptlet (group+user, system account, home /var/lib/pgsql) so the +# result is identical regardless of --method, and running this unconditionally +# for all three methods is a no-op if the RPM path already created it. +step "Ensuring OS user/group 'postgres' exists" +getent group postgres >/dev/null || groupadd -r postgres +if getent passwd postgres >/dev/null; then + info "OS user 'postgres' already exists" +else + mkdir -p /var/lib/pgsql + useradd -r -g postgres -d /var/lib/pgsql -s /bin/bash -c "PostgreSQL Server" postgres + chown postgres:postgres /var/lib/pgsql + info "OS user 'postgres' created (system account, home /var/lib/pgsql)" +fi + +# --------------------------------------------------------------------------- +# 2. POSTGRESQL — according to chosen method +# --------------------------------------------------------------------------- +step "Installing PostgreSQL ${EXPECTED_PG_VERSION} (method: ${METHOD})" + +PG_INSTALL_DIR="" +LIBPQ_INCDIR="" +LIBPQ_LIBDIR="" + +# ─── 2a. PERCONA TARBALL (default) ───────────────────────────────────────── +# Dokumentace: https://docs.percona.com/postgresql/18/tarball.html +install_percona_tarball() { + local VERSION="18.4" + local INSTALL_BASE="/opt/pgdistro" + local PG_SUBDIR="percona-postgresql18" + + # Detect OpenSSL version → choose tarball variant. + # PG18 tarballs ship three variants (unlike PG16's two: ssl1/ssl3) -- + # ssl1.1, ssl3 (OpenSSL 3.0-3.4), and ssl3.5 (OpenSSL 3.5+). AlmaLinux 10 + # ships OpenSSL 3.5.x, so the major-version-only check used for PG16 + # would silently grab the wrong (but still installable) ssl3 build here; + # compare major.minor instead. + local OPENSSL_VER + OPENSSL_VER=$(openssl version | awk '{print $2}') + local SSL_TAG + local ssl_major="${OPENSSL_VER%%.*}" + local ssl_minor="${OPENSSL_VER#*.}"; ssl_minor="${ssl_minor%%.*}" + case "$ssl_major" in + 1) SSL_TAG="ssl1.1" ;; + 3) + if [[ "$ssl_minor" -ge 5 ]]; then + SSL_TAG="ssl3.5" + else + SSL_TAG="ssl3" + fi + ;; + *) SSL_TAG="ssl3.5"; warn "Unknown OpenSSL version ${OPENSSL_VER}, trying ssl3.5" ;; + esac + + # Map architecture to tarball name + local TARARCH + case "$ARCH" in + x86_64) TARARCH="linux-x86_64" ;; + aarch64) TARARCH="linux-aarch64" ;; + *) error "Unsupported architecture: $ARCH" ;; + esac + + local TARBALL="percona-postgresql-${VERSION}-${SSL_TAG}-${TARARCH}.tar.gz" + local URL="https://downloads.percona.com/downloads/postgresql-distribution-18/${VERSION}/binary/tarball/${TARBALL}" + + info "Tarball: ${TARBALL}" + info "URL: ${URL}" + + # Skip if binary already exists + if [[ -x "${INSTALL_BASE}/${PG_SUBDIR}/bin/postgres" ]]; then + info "Percona PostgreSQL 18 already installed in ${INSTALL_BASE}/${PG_SUBDIR}" + PG_INSTALL_DIR="${INSTALL_BASE}/${PG_SUBDIR}" + return 0 + fi + + mkdir -p "${INSTALL_BASE}" + + local TMPTAR="/tmp/${TARBALL}" + # Fetch-once-distribute: with N guests all running setup at once, N + # independent downloads hammer downloads.percona.com. If the orchestrator + # already staged a copy here (taf_manage.py --PERCONA_ARCHIVE_LOCAL, SCP'd + # to REMOTE_WORKDIR before setup), use it instead of downloading. + if [[ -n "${PERCONA_LOCAL_ARCHIVE:-}" ]] && [[ -f "$PERCONA_LOCAL_ARCHIVE" ]]; then + info "Using pre-staged tarball: ${PERCONA_LOCAL_ARCHIVE}" + local abs_src abs_dst + abs_src=$(readlink -f "$PERCONA_LOCAL_ARCHIVE") + abs_dst=$(readlink -f "$TMPTAR" 2>/dev/null || echo "") + [[ "$abs_src" != "$abs_dst" ]] && cp "$PERCONA_LOCAL_ARCHIVE" "$TMPTAR" + elif [[ ! -f "$TMPTAR" ]]; then + info "Downloading tarball..." + wget -q --show-progress -O "$TMPTAR" "$URL" 2>/dev/null || \ + wget -O "$TMPTAR" "$URL" || \ + curl -fL -o "$TMPTAR" "$URL" + else + info "Tarball already downloaded: ${TMPTAR}" + fi + + info "Extracting to ${INSTALL_BASE}/..." + tar -xf "$TMPTAR" -C "${INSTALL_BASE}/" + + # Move Perl/Python/Tcl modules one level up (per documentation) + for mod in percona-perl percona-python3 percona-tcl; do + if [[ -d "${INSTALL_BASE}/${mod}" ]] && [[ ! -d "/opt/${mod}" ]]; then + info "Moving ${mod} -> /opt/${mod}" + mv "${INSTALL_BASE}/${mod}" "/opt/${mod}" + fi + done + + PG_INSTALL_DIR="${INSTALL_BASE}/${PG_SUBDIR}" + + # LD_LIBRARY_PATH for bundled libraries + local PG_LIB="${PG_INSTALL_DIR}/lib" + if [[ -d "$PG_LIB" ]]; then + export LD_LIBRARY_PATH="${PG_LIB}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + cat > /etc/profile.d/percona-pg18.sh </dev/null; then + info "PGDG repository added" + dnf -qy module disable postgresql 2>/dev/null || true + dnf install -y postgresql18-server postgresql18-devel postgresql18 + PG_INSTALL_DIR="/usr/pgsql-18" + else + warn "PGDG EL10 repo unavailable, falling back to AppStream" + install_appstream + fi +} + +# ─── 2c. APPSTREAM ───────────────────────────────────────────────────────── +install_appstream() { + # AlmaLinux 10 AppStream ships both the unversioned "postgresql" (16, the + # default stream) and a versioned "postgresql18" package set side by + # side; they conflict at the file level (postgresql-any / postgresql- + # server-any virtual provides), so --allowerasing is required to swap 16 + # out for 18 on a base image that already has 16 installed. + # libpq-devel is version-independent (provides libpq-fe.h for sysbench) + # and does not conflict with postgresql18-*. + dnf install -y --allowerasing postgresql18 postgresql18-server libpq-devel + PG_INSTALL_DIR="/usr" + warn "PostgreSQL installed from AppStream into ${PG_INSTALL_DIR}" + warn "AppStream may lag behind the latest point release (EXPECTED_PG_VERSION=${EXPECTED_PG_VERSION:-18.4}) -- the version check in step 3b will fail loudly if so; use --method=percona for a guaranteed exact match." +} + +case "$METHOD" in + percona) install_percona_tarball ;; + pgdg) install_pgdg_rpm ;; + appstream) install_appstream ;; +esac + +# --------------------------------------------------------------------------- +# 3. VERIFY POSTGRESQL BINARIES +# --------------------------------------------------------------------------- +step "Verifying PostgreSQL binaries" +PG_BIN="${PG_INSTALL_DIR}/bin" +MISSING=0 +for BIN in postgres pg_ctl psql initdb pg_isready; do + if [[ -x "${PG_BIN}/${BIN}" ]]; then + info " ✓ ${PG_BIN}/${BIN}" + else + warn " ✗ ${PG_BIN}/${BIN} not found" + MISSING=$((MISSING + 1)) + fi +done +[[ $MISSING -eq 0 ]] || error "Missing binaries — check installation in ${PG_INSTALL_DIR}" + +# pg_config is provided by postgresql-server-devel, which conflicts with +# libpq-devel on EL10 AppStream. It is only needed to locate libpq headers +# for sysbench; fall back to hardcoded AppStream paths when unavailable. +if [[ -x "${PG_BIN}/pg_config" ]]; then + info "PostgreSQL version: $("${PG_BIN}/pg_config" --version)" + LIBPQ_INCDIR=$("${PG_BIN}/pg_config" --includedir) + LIBPQ_LIBDIR=$("${PG_BIN}/pg_config" --libdir) +else + warn "pg_config not found (postgresql-server-devel not installed); using AppStream defaults" + info "PostgreSQL version: $("${PG_BIN}/postgres" --version 2>/dev/null || echo unknown)" + LIBPQ_INCDIR="/usr/include" + LIBPQ_LIBDIR="/usr/lib64" +fi +info "includedir: ${LIBPQ_INCDIR}" +info "libdir: ${LIBPQ_LIBDIR}" + +# --------------------------------------------------------------------------- +# 3b. VERIFY POSTGRESQL VERSION MATCHES EXPECTED CURRENT STABLE RELEASE +# --------------------------------------------------------------------------- +# Any installed version other than EXPECTED_PG_VERSION is a hard error -- +# better to fail loudly here than to silently benchmark an unintended +# PostgreSQL version (e.g. AppStream lagging a point release behind, or a +# stale cached tarball/RPM repo). +step "Verifying PostgreSQL version" +ACTUAL_PG_VERSION=$("${PG_BIN}/postgres" --version | grep -oE '[0-9]+\.[0-9]+(\.[0-9]+)?' | head -1) +if [[ -z "$ACTUAL_PG_VERSION" ]]; then + error "Could not determine installed PostgreSQL version from '${PG_BIN}/postgres --version'" +fi +if [[ "$ACTUAL_PG_VERSION" != "$EXPECTED_PG_VERSION" ]]; then + error "Installed PostgreSQL version ${ACTUAL_PG_VERSION} != expected ${EXPECTED_PG_VERSION} (method=${METHOD}, dir=${PG_INSTALL_DIR}). Either a newer release has shipped (update EXPECTED_PG_VERSION at the top of this script), or this --method's repo/tarball is out of date for the pinned version -- try a different --method (percona guarantees an exact-version tarball)." +fi +info "PostgreSQL version OK: ${ACTUAL_PG_VERSION}" + +# --------------------------------------------------------------------------- +# 4. LIBPQ HEADERS FOR SYSBENCH +# --------------------------------------------------------------------------- +step "Checking libpq-fe.h for sysbench" +if [[ -f "${LIBPQ_INCDIR}/libpq-fe.h" ]]; then + info "libpq-fe.h found: ${LIBPQ_INCDIR}/libpq-fe.h" +else + warn "libpq-fe.h not found, trying system packages..." + dnf install -y --allowerasing postgresql-devel libpq-devel 2>/dev/null || \ + dnf install -y postgresql16-devel 2>/dev/null || \ + warn "libpq-devel unavailable — sysbench build may fail" +fi + +# Export for ./configure sysbench +export PKG_CONFIG_PATH="${LIBPQ_LIBDIR}/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" +export CPPFLAGS="-I${LIBPQ_INCDIR}" +export LDFLAGS="-L${LIBPQ_LIBDIR} -Wl,-rpath,${LIBPQ_LIBDIR}" + +# --------------------------------------------------------------------------- +# 5. PYTHON + PYTEST +# --------------------------------------------------------------------------- +step "Installing pytest" + +# Virtuozzo/VzLinux Python3 uses Percona Python as stdlib +# (/usr/bin/python3 is a small wrapper, sys.prefix=/opt/percona-python3). +# Pytest must be in /opt/percona-python3/lib/python3.12/site-packages/ +# or accessible via a .pth file. System pip3/ssl may not work +# (Percona Python requires OpenSSL 3.3+, system has 3.2.x). + +PERCONA_PY_SITEPKGS="/opt/percona-python3/lib/python3.12/site-packages" +SYS_SITEPKGS="/usr/lib/python3.12/site-packages" + +# Strategy 1: add system site-packages to Percona Python via .pth +if [[ -d "$PERCONA_PY_SITEPKGS" ]]; then + PTH_FILE="${PERCONA_PY_SITEPKGS}/system-sitepackages.pth" + if [[ ! -f "$PTH_FILE" ]]; then + info "Adding system site-packages to Percona Python path..." + echo "/usr/lib/python3.12/site-packages" > "$PTH_FILE" + echo "/usr/lib64/python3.12/site-packages" >> "$PTH_FILE" + fi +fi + +# Strategy 2: install pytest via dnf (preferred for Virtuozzo) +PYTEST_INSTALLED=0 +if python3 -m pytest --version >/dev/null 2>&1; then + PYTEST_INSTALLED=1 +elif dnf install -y python3-pytest >/dev/null 2>&1; then + PYTEST_INSTALLED=1 +else + # Strategy 3: copy pytest from /usr/local (installed by earlier pip) + if [[ -d "/usr/local/lib/python3.12/site-packages/pytest" ]]; then + for pkg in pytest _pytest pluggy iniconfig py.py; do + src="/usr/local/lib/python3.12/site-packages/${pkg}" + [[ -e "$src" ]] && cp -r "$src" "${PERCONA_PY_SITEPKGS}/" 2>/dev/null || true + done + for dist in /usr/local/lib/python3.12/site-packages/{pytest,pluggy,iniconfig}-*.dist-info; do + [[ -e "$dist" ]] && cp -r "$dist" "${PERCONA_PY_SITEPKGS}/" 2>/dev/null || true + done + python3 -m pytest --version >/dev/null 2>&1 && PYTEST_INSTALLED=1 + fi +fi + +[[ $PYTEST_INSTALLED -eq 1 ]] || error "Failed to install pytest" +info "pytest: $(python3 -m pytest --version 2>&1 | head -1)" + +# --------------------------------------------------------------------------- +# 6. SYSBENCH — clone and build if missing +# --------------------------------------------------------------------------- +step "Sysbench" +SYSBENCH_SRC="${TAF_DIR}/client_source/sysbench-lua" + +# Correct state: sysbench is a symlink to src/sysbench (locally built binary) +# *with pgsql support* -- a binary that already exists but was built for a +# different engine (e.g. taf_run.sh's own build_sysbench_mariadb(), which +# builds --with-mysql --without-pgsql into this same client_source/sysbench-lua/ +# tree when MariaDB ran on this host first) must not be trusted as-is, or +# the pgsql prepare/run step fails immediately with "invalid option: +# --pgsql-host=...". Confirm the pgsql driver is actually compiled in. +SYSBENCH_OK=0 +if [[ -L "${SYSBENCH_SRC}/sysbench" ]] && [[ "$(readlink "${SYSBENCH_SRC}/sysbench")" == "src/sysbench" ]] && [[ -x "${SYSBENCH_SRC}/src/sysbench" ]] \ + && "${SYSBENCH_SRC}/sysbench" "${SYSBENCH_SRC}/src/lua/oltp_read_write.lua" --help 2>&1 | grep -qE '^pgsql\b'; then + SYSBENCH_OK=1 + info "Sysbench already built with pgsql driver: $("${SYSBENCH_SRC}/sysbench" --version)" +elif [[ -x "${SYSBENCH_SRC}/sysbench" ]] && [[ ! -L "${SYSBENCH_SRC}/sysbench" ]]; then + warn "sysbench is a binary (not a symlink) — may have been rsync'd from another host" + warn "Rebuilding from source for correct architecture and libpq..." +fi +if [[ $SYSBENCH_OK -eq 0 ]]; then + # Submodule check: LuaJIT Makefile is only present after a proper git clone + # with --recurse-submodules. Zip-extracted source lacks .git/ so submodules + # are empty. Force a fresh clone whenever the submodule is missing. + # + # Retry with backoff: a fleet-wide run clones this from GitHub on every + # guest concurrently (e.g. 210 at once for a full density curve), which + # occasionally hits transient network/server flakiness -- observed as + # `fatal: shallow file has changed since we read it` on a small fraction + # of hosts. A single failed attempt used to abort the whole host's setup + # (and, via taf_manage.py, could burn one of only 2 host-level retries on + # something that a plain retry here would have absorbed). Always + # `rm -rf` before each attempt so a partial/corrupt clone from a failed + # attempt can't linger into the next one. + if [[ ! -f "${SYSBENCH_SRC}/third_party/luajit/luajit/Makefile" ]]; then + mkdir -p "${TAF_DIR}/client_source" + clone_ok=0 + for attempt in 1 2 3; do + info "Cloning sysbench (submodules missing or incomplete, attempt ${attempt}/3)..." + rm -rf "${SYSBENCH_SRC}" + if git clone --depth=1 --recurse-submodules https://github.com/akopytov/sysbench "${SYSBENCH_SRC}"; then + clone_ok=1 + break + fi + warn "sysbench clone attempt ${attempt}/3 failed" + [[ $attempt -lt 3 ]] && sleep $((attempt * 10)) + done + [[ $clone_ok -eq 1 ]] || error "Failed to clone sysbench after 3 attempts" + fi + + info "Building sysbench with pgsql support..." + cd "${SYSBENCH_SRC}" + + # Always start configure from a clean cache: a config.cache left over from + # a previous (possibly killed mid-build, or differently-configured) attempt + # on this same guest gets blindly trusted by autoconf, including for + # checks that should never vary by host (e.g. "checking for stdlib.h... + # (cached) no" was observed leading straight into a bogus "thread-local + # storage is not supported" failure). A stale cache is worse than no cache. + rm -f config.cache + + # Use bash explicitly — files from a zip archive may lack execute bits. + bash autogen.sh + bash configure --without-mysql --with-pgsql \ + --with-pgsql-includes="${LIBPQ_INCDIR}" \ + --with-pgsql-libs="${LIBPQ_LIBDIR}" + make -j"$(nproc)" + + if [[ -f "src/sysbench" ]]; then + # Ensure correct symlink — remove any old binary (e.g. rsync'd from another host) + if [[ ! -L "sysbench" ]] || [[ "$(readlink sysbench)" != "src/sysbench" ]]; then + rm -f sysbench + ln -sf src/sysbench sysbench + info "Symlink: sysbench-lua/sysbench -> src/sysbench" + fi + fi + + cd "${TAF_DIR}" + info "Sysbench: $("${SYSBENCH_SRC}/sysbench" --version)" +fi + +# --------------------------------------------------------------------------- +# 7. /root PERMISSIONS (postgres user needs traverse into TAF dir) +# --------------------------------------------------------------------------- +step "Directory permissions" +ROOT_PERM=$(stat -c '%a' /root) +if [[ "${ROOT_PERM: -1}" == "0" ]]; then + info "Adding o+x on /root" + chmod o+x /root +fi +info "/root: $(stat -c '%a' /root)" + +# --------------------------------------------------------------------------- +# 8. SAVING ENV VARIABLES +# --------------------------------------------------------------------------- +step "Saving environment" +ENV_FILE="${TAF_DIR}/.taf_pg_env" +cat > "${ENV_FILE}" < subprocess.CompletedProcess: + """Runs a command and returns CompletedProcess (does not raise on non-zero).""" + merged_env = None + if env: + merged_env = os.environ.copy() + merged_env.update(env) + return subprocess.run( + list(cmd), + capture_output=True, text=True, + cwd=str(cwd or TAF_ROOT), + timeout=timeout, + env=merged_env, + ) + + +def taf_run(props: dict, timeout: int = 600) -> subprocess.CompletedProcess: + """Runs perl taf.pl with the given properties (dict).""" + cmd = ["perl", str(TAF_ROOT / "taf.pl")] + for k, v in props.items(): + cmd.append(f"--property={k}={v}") + return subprocess.run( + cmd, + capture_output=True, text=True, + cwd=str(TAF_ROOT), + timeout=timeout, + ) + + +def taf_propfile(props_file: Path, extra: dict | None = None, + timeout: int = 600) -> subprocess.CompletedProcess: + """Runs perl taf.pl with a properties file and optionally extra overrides.""" + cmd = ["perl", str(TAF_ROOT / "taf.pl"), + f"--properties-file={props_file}"] + for k, v in (extra or {}).items(): + cmd.append(f"--property={k}={v}") + return subprocess.run( + cmd, + capture_output=True, text=True, + cwd=str(TAF_ROOT), + timeout=timeout, + ) + + +def write_props(path: Path, props: dict) -> None: + """Writes a dict to a .properties file.""" + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w") as f: + for k, v in props.items(): + f.write(f"{k} = {v}\n") + + +def has_no_errors(text: str) -> tuple[bool, str | None]: + """Returns (True, None) if TAF output contains no error lines. + Ignores comments and informational 'error' in values. + """ + for line in text.splitlines(): + stripped = line.strip() + if stripped.startswith("#"): + continue + # Look for ERROR as a token (not 'error' inside values) + if re.search(r'\bERROR\b', line): + return False, line + return True, None + + +def find_pg_data_dir(taf_output: str) -> Path | None: + """Parses verbose TAF output and searches for the data directory path.""" + # TAF logs e.g.: "Preparing data directory: /path/to/data" + # or: "data_dir does not exist: /path" + # or: "Datadir does not exist: /path" + patterns = [ + r'data.dir[:\s]+(/[^\s]+)', + r'Removing existing data directory\s+(/[^\s]+)', + r'Created.*data.*dir.*?(/[^\s]+)', + r'initdb.*?-D\s+(/[^\s]+)', + r'pg_ctl.*?-D\s+(/[^\s]+)', + ] + for pat in patterns: + m = re.search(pat, taf_output, re.IGNORECASE) + if m: + return Path(m.group(1)) + return None + + +def psql(query: str, user: str = TAF_PG_USER, password: str = TAF_PG_PASS, + db: str = TAF_PG_DB, port: int = PG_PORT, + host: str = "127.0.0.1") -> subprocess.CompletedProcess: + """Runs a psql query and returns the result.""" + env = {"PGPASSWORD": password} + return run( + str(PG_BIN / "psql"), + "-h", host, "-p", str(port), "-U", user, "-d", db, + "-c", query, "-q", "--no-psqlrc", "--tuples-only", + env=env, + timeout=15, + ) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +@pytest.fixture(scope="session") +def lifecycle_result(tmp_path_factory): + """L3 fixture: runs TAF init-start-db-exit once per session. + + Returns (CompletedProcess, data_dir_path_or_None). + Ensures PostgreSQL is stopped even on error. + """ + # Stop any leftover PG from a previous session + _shutdown_pg_if_running() + + tmp = tmp_path_factory.mktemp("taf_l3") + props_file = tmp / "lifecycle.properties" + + write_props(props_file, { + "taf.action": "init-start-db-exit", + "taf.taf_db_makers_plugin": "postgres", + "taf.db_software_install_dir": str(PG_INSTALL), + "taf.db_port": str(PG_PORT), + "taf.db_user": TAF_PG_USER, + "taf.db_user_pass": TAF_PG_PASS, + "taf.db_root_user": TAF_PG_ROOT, + "taf.db_root_pass": TAF_PG_ROOT_PASS, + "taf.database": TAF_PG_DB, + "taf.test_suite": "sysbench-lua", + "taf.verbose": "true", + "sysbench_lua.db_driver": "pgsql", + }) + + result = taf_propfile(props_file, timeout=300) + output = result.stdout + result.stderr + data_dir = find_pg_data_dir(output) + + yield result, data_dir + + # Cleanup: ensure PG is shut down if TAF failed midway + _shutdown_pg_if_running() + + +@pytest.fixture(scope="session") +def benchmark_result(tmp_path_factory): + """L5 fixture: runs a short TAF benchmark (OLTP_RO + POINT_SELECT). + + Returns CompletedProcess. + """ + # L3 leaves PostgreSQL running (init-start-db-exit keeps PG alive); stop it + # before TAF tries to start a fresh instance via init-start-db-run-tests. + _shutdown_pg_if_running() + + tmp = tmp_path_factory.mktemp("taf_l5") + props_file = tmp / "benchmark.properties" + + # Sysbench uses autotools (not cmake); binary built via L4 autotools test. + # Use init-start-db-run-tests to skip client build (avoid cmake failure). + # sysbench_lua.exe defaults to relative path client_source/sysbench-lua/sysbench + # which resolves correctly from TAF working dir — no override needed. + write_props(props_file, { + "taf.action": "init-start-db-run-tests", + "taf.taf_db_makers_plugin": "postgres", + "taf.db_software_install_dir": str(PG_INSTALL), + "taf.db_port": str(PG_PORT), + "taf.db_user": TAF_PG_USER, + "taf.db_user_pass": TAF_PG_PASS, + "taf.db_root_user": TAF_PG_ROOT, + "taf.db_root_pass": TAF_PG_ROOT_PASS, + "taf.database": TAF_PG_DB, + "taf.test_suite": "sysbench-lua", + "taf.tests": "OLTP_RO,POINT_SELECT", + "taf.verbose": "true", + "sysbench_lua.db_driver": "pgsql", + "sysbench_lua.connector": "libpq", + "sysbench_lua.def_threads": "4,8", + "sysbench_lua.def_duration": "30", + "sysbench_lua.number_of_tables": "1", + "sysbench_lua.number_of_rows": "10000", + "sysbench_lua.oltp_skip_trx": "off", + }) + + result = taf_propfile(props_file, timeout=900) + + yield result + + _shutdown_pg_if_running() + + +def _shutdown_pg_if_running() -> None: + """Best-effort: stop PG if still running (cleanup after L3/L5). + + pg_ctl stop cannot run as root — we find postmaster.pid and send + SIGTERM directly to the process, or run pg_ctl as postgres user via su. + """ + import signal as _signal + from pathlib import Path + pg_ctl = PG_BIN / "pg_ctl" + # TAF creates the data directory in /tmp/taf_pg_/data/ or in TAF_ROOT/data/ + search_roots = [TAF_ROOT, Path("/tmp")] + for root in search_roots: + if not root.is_dir(): + continue + for pid_file in root.rglob("postmaster.pid"): + data_dir = pid_file.parent + # Try via postgres user (pg_ctl cannot run as root) + if pg_ctl.is_file(): + r = subprocess.run( + ["su", "-s", "/bin/sh", "postgres", "-c", + f"{pg_ctl} stop -D {data_dir} -m fast -w -t 30"], + capture_output=True, timeout=45, + ) + if r.returncode == 0: + continue + # Fallback: kill postmaster directly via SIGTERM + try: + pid = int(pid_file.read_text().splitlines()[0].strip()) + os.kill(pid, _signal.SIGTERM) + except Exception: + pass + + +# =========================================================================== +# LAYER 1 — Static validation +# =========================================================================== + +class TestL1Static: + """Syntax and text validation without running TAF or PostgreSQL.""" + + def test_postgres_pm_exists(self): + """Plugin postgres.pm must exist.""" + plugin = TAF_ROOT / "libs" / "database_libs" / "postgres.pm" + assert plugin.is_file(), f"Missing: {plugin}" + + def test_postgres_pm_package_name(self): + """Package declaration in postgres.pm must be 'package postgres'.""" + plugin = TAF_ROOT / "libs" / "database_libs" / "postgres.pm" + content = plugin.read_text() + assert re.search(r"^\s*package\s+postgres\s*;", content, re.MULTILINE), \ + "postgres.pm does not declare 'package postgres'" + + def test_postgres_pm_syntax(self): + """`perl -c` on postgres.pm must pass (on Linux).""" + result = run("perl", "-c", + str(TAF_ROOT / "libs" / "database_libs" / "postgres.pm")) + assert result.returncode == 0 or \ + "Unsupported OS platform" in result.stderr, \ + f"Syntax error in postgres.pm:\n{result.stderr}" + + def test_sysbench_lua_syntax(self): + """`perl -c` on sysbench-lua.pm must pass.""" + result = run("perl", "-c", + str(TAF_ROOT / "test_suites" / "sysbench-lua.pm")) + assert result.returncode == 0, \ + f"Syntax error in sysbench-lua.pm:\n{result.stderr}" + + def test_pgsql_connection_args_in_sysbench(self): + """sysbench-lua.pm must generate --pgsql-host/port/user/password/db.""" + content = (TAF_ROOT / "test_suites" / "sysbench-lua.pm").read_text() + for flag in ("--pgsql-host", "--pgsql-port", "--pgsql-user", + "--pgsql-password", "--pgsql-db"): + assert flag in content, \ + f"Missing '{flag}' in SetConnectionArgs() — pgsql branch" + + def test_mysql_args_guarded_for_pgsql(self): + """--mysql-storage-engine must be guarded by 'db_driver ne pgsql'.""" + content = (TAF_ROOT / "test_suites" / "sysbench-lua.pm").read_text() + # Find the block with mysql-storage-engine + block = re.search( + r"(Storage engine.*?mysql-storage-engine.*?\n)", content, + re.DOTALL | re.IGNORECASE, + ) + # A condition with 'pgsql' must exist near --mysql-storage-engine + idx = content.find("--mysql-storage-engine") + assert idx >= 0, "--mysql-storage-engine not found in file" + surrounding = content[max(0, idx - 200):idx + 100] + assert "pgsql" in surrounding, \ + "--mysql-storage-engine is not guarded by a db_driver ne pgsql check" + + def test_normalize_db_type_defined(self): + """NormalizeDBType must be defined in sysbench-lua.pm.""" + content = (TAF_ROOT / "test_suites" / "sysbench-lua.pm").read_text() + assert "sub NormalizeDBType" in content, \ + "Missing sub NormalizeDBType in sysbench-lua.pm" + + def test_normalize_db_type_maps_postgres_to_pgsql(self): + """NormalizeDBType must map postgres/postgresql → pgsql.""" + content = (TAF_ROOT / "test_suites" / "sysbench-lua.pm").read_text() + # Extract the function + m = re.search(r"sub NormalizeDBType \{(.+?)\n\}", content, re.DOTALL) + assert m, "NormalizeDBType not found" + body = m.group(1) + assert "pgsql" in body and "postgres" in body, \ + "NormalizeDBType does not contain the postgres → pgsql mapping" + + def test_validate_target_normalizes_incoming(self): + """ValidateTargetWithSuite must normalize $incoming before comparison.""" + content = (TAF_ROOT / "test_suites" / "sysbench-lua.pm").read_text() + # Look for NormalizeDBType($incoming) in the body of ValidateTargetWithSuite + m = re.search( + r"sub ValidateTargetWithSuite \{(.+?)\n\}", + content, re.DOTALL, + ) + assert m, "ValidateTargetWithSuite not found" + body = m.group(1) + assert "NormalizeDBType($incoming)" in body, \ + "ValidateTargetWithSuite does not normalize \\$incoming before comparison" + + def test_cmake_args_contains_pgsql(self): + """sysbench_lua_default.properties must contain -DWITH_PGSQL=on.""" + defaults = ( + TAF_ROOT / "properties" / "default" / "sysbench_lua_default.properties" + ).read_text() + assert "-DWITH_PGSQL=on" in defaults, \ + "cmake_args does not contain -DWITH_PGSQL=on" + + def test_postgres_sql_blocks_count(self): + """postgres.sql must have at least 14 named blocks.""" + content = (TAF_ROOT / "libs" / "sql_libs" / "dialects" / "postgres.sql").read_text() + blocks = re.findall(r"^\[(\w+)\]", content, re.MULTILINE) + assert len(blocks) >= 14, \ + f"postgres.sql has only {len(blocks)} blocks, expected ≥14: {blocks}" + + def test_postgres_sql_has_diagnostic_blocks(self): + """postgres.sql must contain diagnostic blocks for benchmark analysis.""" + content = (TAF_ROOT / "libs" / "sql_libs" / "dialects" / "postgres.sql").read_text() + required = [ + "active_connections", "wait_events", "table_stats", + "index_usage", "bgwriter_stats", "transaction_stats", + ] + for block in required: + assert f"[{block}]" in content, \ + f"Missing diagnostic block [{block}] in postgres.sql" + + def test_utilities_pm_has_postgresql_alias(self): + """Utilities.pm must map 'postgresql' → 'postgres' in PLUGIN_ALIASES.""" + content = (TAF_ROOT / "libs" / "taf_libs" / "TAF" / "Utilities.pm").read_text() + assert re.search(r"postgresql\s*=>\s*['\"]postgres['\"]", content), \ + "Missing 'postgresql => postgres' in PLUGIN_ALIASES (Utilities.pm)" + + def test_postgresql_conf_templates_exist(self): + """At least three postgresql.conf templates must exist.""" + pg_cfg_dir = TAF_ROOT / "database_config_files" / "postgresql" + assert pg_cfg_dir.is_dir(), f"Directory {pg_cfg_dir} does not exist" + configs = list(pg_cfg_dir.glob("*.conf")) + assert len(configs) >= 3, \ + f"Expected ≥3 .conf files, found {len(configs)}: {configs}" + + def test_postgresql_properties_examples_exist(self): + """Example properties files for PostgreSQL must exist.""" + pg_props_dir = TAF_ROOT / "properties" / "postgresql" + assert pg_props_dir.is_dir(), f"Directory {pg_props_dir} does not exist" + props = list(pg_props_dir.glob("*.properties")) + assert len(props) >= 2, \ + f"Expected ≥2 .properties files, found {len(props)}: {props}" + + +# =========================================================================== +# LAYER 2 — Plugin unit test (via Perl subprocess) +# =========================================================================== + +def _build_plugin_unit_script(pg_install: Path, port: int, + user: str, password: str, + root: str, root_pass: str) -> str: + """Builds a Perl unit-test script for the postgres.pm plugin. + + Does not use str.format() — Perl syntax contains {} which would conflict + with Python format placeholders. + """ + taf_libs = str(pg_install.parent.parent / "libs" / "database_libs") \ + if False else str(TAF_ROOT / "libs" / "database_libs") + return ( + "#!/usr/bin/perl\n" + "use strict;\n" + "use warnings;\n" + "\n" + "BEGIN {\n" + " package TAF::Logging;\n" + " use Exporter 'import';\n" + " our @EXPORT_OK = qw(PrintError PrintWarning PrintVerbose StageStart StageEnd);\n" + " sub PrintError { print \"ERR: @_\\n\" }\n" + " sub PrintWarning { print \"WARN: @_\\n\" }\n" + " sub PrintVerbose { }\n" + " sub StageStart { return $_[0] }\n" + " sub StageEnd { }\n" + " $INC{'TAF/Logging.pm'} = __FILE__;\n" + "}\n" + "\n" + f"use lib '{TAF_ROOT}/libs/database_libs';\n" + f"use lib '{TAF_ROOT}/libs/taf_libs';\n" + "\n" + "require 'postgres.pm';\n" + "\n" + "my $pg_install = shift @ARGV or die \"Missing pg_install\\n\";\n" + "my $tmpdir = '/tmp';\n" + "my $pass = 0; my $fail = 0;\n" + "\n" + "sub ok {\n" + " my ($cond, $name) = @_;\n" + " if ($cond) { print \"PASS: $name\\n\"; $pass++ }\n" + " else { print \"FAIL: $name\\n\"; $fail++ }\n" + "}\n" + "\n" + "# Test 1: new() with invalid install_root → undef\n" + "{\n" + " my $pg = postgres->new(\n" + " db_software_install_dir => '/nonexistent_xyz',\n" + " db_data_dir => '/tmp/pg_unit_data',\n" + " tmp_dir => $tmpdir,\n" + " );\n" + " ok(!defined $pg, \"new() rejects invalid install_root\");\n" + "}\n" + "\n" + "# Test 2: new() with valid installation → object\n" + "{\n" + " my $pg = postgres->new(\n" + " db_software_install_dir => $pg_install,\n" + " db_data_dir => '/tmp/pg_unit_data',\n" + " tmp_dir => $tmpdir,\n" + f" db_port => {port},\n" + f" db_user => '{user}',\n" + f" db_user_pass => '{password}',\n" + f" db_root_user => '{root}',\n" + f" db_root_pass => '{root_pass}',\n" + " );\n" + " ok(defined $pg, \"new() returns object with valid installation\");\n" + "\n" + " if (defined $pg) {\n" + " for my $b (qw(postgres_bin pg_ctl_bin psql_bin initdb_bin pg_isready_bin)) {\n" + " ok(defined $pg->{$b} && -x $pg->{$b},\n" + " \"Binary $b found: \" . ($pg->{$b} // ''));\n" + " }\n" + f" ok($pg->{{port}} == {port}, \"Port nastaven na {port}\");\n" + f" ok($pg->{{db_user}} eq '{user}', \"db_user stored correctly\");\n" + f" ok($pg->{{db_root_user}} eq '{root}', \"db_root_user stored correctly\");\n" + " }\n" + "}\n" + "\n" + "print \"\\nResult: $pass passed, $fail failed\\n\";\n" + "exit($fail > 0 ? 1 : 0);\n" + ) + + +class TestL2PluginUnit: + """Unit tests for the postgres.pm plugin via Perl subprocess.""" + + @needs_pg + def test_plugin_unit_all_pass(self, tmp_path): + """All unit tests for postgres.pm must pass.""" + script = _build_plugin_unit_script( + pg_install=PG_INSTALL, + port=PG_PORT, + user=TAF_PG_USER, + password=TAF_PG_PASS, + root=TAF_PG_ROOT, + root_pass=TAF_PG_ROOT_PASS, + ) + + script_file = tmp_path / "pg_plugin_unit.pl" + script_file.write_text(script) + + result = run( + "perl", str(script_file), str(PG_INSTALL), + cwd=TAF_ROOT, timeout=30, + ) + output = result.stdout + result.stderr + + # Extract result + m = re.search(r"Result:\s*(\d+) passed,\s*(\d+) failed", output) + if m: + passed, failed = int(m.group(1)), int(m.group(2)) + else: + passed, failed = 0, 1 + + fails = [ln for ln in output.splitlines() if ln.startswith("FAIL:")] + assert failed == 0, ( + f"Plugin unit tests failed ({failed} failures):\n" + + "\n".join(fails) + + f"\n\nFull output:\n{output}" + ) + + +# =========================================================================== +# LAYER 3 — Database lifecycle +# =========================================================================== + +class TestL3Lifecycle: + """Tests the full TAF PostgreSQL lifecycle: init → start → stop.""" + + @needs_pg + def test_taf_exits_cleanly(self, lifecycle_result): + """TAF init-start-db-exit must finish with exit code 0.""" + result, _ = lifecycle_result + assert result.returncode == 0, ( + f"TAF finished with rc={result.returncode}\n" + f"STDOUT:\n{result.stdout[-3000:]}\n" + f"STDERR:\n{result.stderr[-1000:]}" + ) + + @needs_pg + def test_no_errors_in_taf_output(self, lifecycle_result): + """TAF output must not contain ERROR lines.""" + result, _ = lifecycle_result + output = result.stdout + result.stderr + clean, bad_line = has_no_errors(output) + assert clean, f"ERROR line found in TAF output:\n {bad_line}" + + @needs_pg + def test_initdb_stage_logged(self, lifecycle_result): + """TAF must log PostgreSQL cluster initialization.""" + result, _ = lifecycle_result + output = result.stdout + result.stderr + assert any( + keyword in output.lower() + for keyword in ("initdb", "initialize", "init database") + ), "Missing initdb stage record in TAF output" + + @needs_pg + def test_start_stage_logged(self, lifecycle_result): + """TAF must log PostgreSQL server start.""" + result, _ = lifecycle_result + output = result.stdout + result.stderr + assert any( + keyword in output.lower() + for keyword in ("database start", "pg_ctl start", "server started", + "postgresql.*start", "start.*postgresql") + ), "Missing start stage record in TAF output" + + @needs_pg + def test_stop_stage_logged(self, lifecycle_result): + """TAF must log PostgreSQL server stop.""" + result, _ = lifecycle_result + output = result.stdout + result.stderr + assert any( + keyword in output.lower() + for keyword in ("database stop", "pg_ctl stop", "server stopped") + ), "Missing stop stage record in TAF output" + + @needs_pg + def test_pg_hba_conf_has_tcp_md5_rules(self, lifecycle_result): + """pg_hba.conf must contain md5 rules for TCP access.""" + result, data_dir = lifecycle_result + if data_dir is None: + pytest.skip("Cannot determine data_dir from TAF output") + hba = data_dir / "pg_hba.conf" + assert hba.is_file(), f"pg_hba.conf not found in {data_dir}" + content = hba.read_text() + assert "127.0.0.1" in content, "pg_hba.conf does not contain a rule for 127.0.0.1" + assert "md5" in content, "pg_hba.conf does not use md5 authentication" + + @needs_pg + def test_postgresql_conf_has_correct_port(self, lifecycle_result): + """postgresql.conf must contain the TAF-configured port.""" + result, data_dir = lifecycle_result + if data_dir is None: + pytest.skip("Cannot determine data_dir from TAF output") + conf = data_dir / "postgresql.conf" + assert conf.is_file(), f"postgresql.conf not found in {data_dir}" + content = conf.read_text() + assert f"port = {PG_PORT}" in content, \ + f"postgresql.conf does not contain 'port = {PG_PORT}'" + + @needs_pg + def test_server_running_after_start_exit(self, lifecycle_result): + """After init-start-db-exit, PostgreSQL server must be listening on the port. + + The init-start-db-exit action intentionally leaves the server running + (exit = TAF exit, not db stop). The cleanup fixture stops the server + at the end of the session. + """ + result, _ = lifecycle_result + # Test is only relevant when lifecycle succeeded + if result.returncode != 0: + pytest.skip("Lifecycle failed — skipping server-running check") + pg_isready = PG_BIN / "pg_isready" + if not pg_isready.is_file(): + pytest.skip("pg_isready not found") + check = run( + str(pg_isready), "-h", "127.0.0.1", "-p", str(PG_PORT), "-q", + ) + assert check.returncode == 0, \ + f"PostgreSQL is not listening on port {PG_PORT} after init-start-db-exit" + + +# =========================================================================== +# LAYER 4 — Sysbench build with pgsql driver +# =========================================================================== + +class TestL4SysbenchBuild: + """Verifies that sysbench builds with the pgsql driver. + + Note: Sysbench uses autotools (autogen.sh + configure + make), not cmake. + TAF build-client assumes cmake — so L4 tests the autotools build directly. + The resulting binary is accessible via symlink client_source/sysbench-lua/sysbench → src/sysbench. + """ + + SYSBENCH_SRC = TAF_ROOT / "client_source" / "sysbench-lua" + SYSBENCH_BIN = SYSBENCH_SRC / "sysbench" # symlink na src/sysbench + + @needs_pg + def test_sysbench_build_succeeds(self, tmp_path_factory): + """Sysbench build with pgsql driver (autotools: autogen+configure+make).""" + src = self.SYSBENCH_SRC + if not (src / "configure.ac").is_file(): + pytest.skip(f"Sysbench source not found: {src} — clone repo into client_source/sysbench-lua/") + + # autogen.sh + result = run("bash", "autogen.sh", cwd=src, timeout=60) + assert result.returncode == 0, f"autogen.sh failed:\n{result.stdout}\n{result.stderr}" + + # configure --with-pgsql --without-mysql + result = run("bash", "configure", "--without-mysql", "--with-pgsql", + cwd=src, timeout=120) + assert result.returncode == 0, ( + f"configure failed (rc={result.returncode}):\n" + f"{result.stdout[-2000:]}\n{result.stderr[-500:]}" + ) + + # make + import multiprocessing + nproc = str(multiprocessing.cpu_count()) + result = run("make", f"-j{nproc}", cwd=src, timeout=300) + assert result.returncode == 0, ( + f"make failed (rc={result.returncode}):\n" + f"{result.stdout[-2000:]}\n{result.stderr[-500:]}" + ) + + # Ensure symlink sysbench → src/sysbench exists for TAF + symlink = src / "sysbench" + real_bin = src / "src" / "sysbench" + if real_bin.is_file() and not symlink.exists(): + import os as _os + _os.symlink("src/sysbench", str(symlink)) + + @needs_pg + def test_sysbench_binary_exists_after_build(self): + """Sysbench binary must exist after the build.""" + sysbench_bin = TAF_ROOT / "client_source" / "sysbench-lua" / "sysbench" + assert sysbench_bin.is_file(), \ + f"Sysbench binary not found: {sysbench_bin}" + assert os.access(str(sysbench_bin), os.X_OK), \ + f"Sysbench binary is not executable: {sysbench_bin}" + + @needs_pg + def test_sysbench_pgsql_driver_available(self): + """Sysbench must support --db-driver=pgsql.""" + sysbench_bin = TAF_ROOT / "client_source" / "sysbench-lua" / "sysbench" + if not sysbench_bin.is_file(): + pytest.skip("Sysbench binary not found — run L4 build test first") + + result = run( + str(sysbench_bin), "--db-driver=pgsql", "--help", + timeout=15, + ) + output = result.stdout + result.stderr + # sysbench with pgsql driver should display --pgsql-* options + has_pgsql = any( + keyword in output.lower() + for keyword in ("pgsql", "postgres", "--pgsql-host") + ) + assert has_pgsql, ( + "Sysbench does not support pgsql driver — check cmake build with " + "-DWITH_PGSQL=on and availability of libpq-devel\n" + f"Output: {output[:500]}" + ) + + +# =========================================================================== +# LAYER 5 — Full benchmark run +# =========================================================================== + +class TestL5Benchmark: + """End-to-end test: TAF init → build → benchmark → stop.""" + + @needs_pg + def test_benchmark_exits_cleanly(self, benchmark_result): + """Benchmark run must finish with exit code 0.""" + result = benchmark_result + assert result.returncode == 0, ( + f"Benchmark TAF run failed (rc={result.returncode})\n" + f"STDOUT:\n{result.stdout[-5000:]}\n" + f"STDERR:\n{result.stderr[-1000:]}" + ) + + @needs_pg + def test_benchmark_output_has_no_errors(self, benchmark_result): + """Benchmark output must not contain ERROR lines.""" + result = benchmark_result + output = result.stdout + result.stderr + clean, bad_line = has_no_errors(output) + assert clean, f"ERROR found in benchmark output:\n {bad_line}" + + @needs_pg + def test_benchmark_uses_pgsql_connection_args(self, benchmark_result): + """TAF benchmark must call sysbench with --pgsql-host (not --mysql-host).""" + output = benchmark_result.stdout + benchmark_result.stderr + # Verbose TAF output includes the sysbench command line + assert "--pgsql-host" in output or "--pgsql-port" in output, ( + "sysbench was not run with --pgsql-* arguments. " + "Check SetConnectionArgs() in sysbench-lua.pm\n" + f"Searched in {len(output)} characters of output" + ) + assert "--mysql-host" not in output, \ + "sysbench was run with --mysql-host instead of --pgsql-host!" + + @needs_pg + def test_results_directory_structure(self, benchmark_result): + """TAF must produce results for OLTP_RO or POINT_SELECT. + + TAF archives results to archive/ after the run completes — we search + both in results/ (if TAF does not archive) and in archive/. + """ + found_tests = set() + for search_root in (TAF_ROOT / "results", TAF_ROOT / "archive"): + if not search_root.is_dir(): + continue + for entry in search_root.iterdir(): + if not entry.is_dir(): + continue + name = entry.name + for test_name in ("OLTP_RO", "POINT_SELECT"): + if test_name in name and not name.startswith("Error_"): + found_tests.add(test_name) + + archive_entries = list((TAF_ROOT / "archive").iterdir()) if (TAF_ROOT / "archive").is_dir() else [] + assert len(found_tests) >= 1, ( + f"No results for OLTP_RO or POINT_SELECT in results/ or archive/. " + f"archive/ contents: {[e.name for e in archive_entries]}" + ) + + @needs_pg + def test_benchmark_result_files_contain_metrics(self, benchmark_result): + """Result files must contain sysbench metrics (transactions). + + TAF archives results to archive/ — we search both locations. + """ + search_roots = [TAF_ROOT / "results", TAF_ROOT / "archive"] + + metrics_found = False + for search_root in search_roots: + if not search_root.is_dir(): + continue + result_files = list(search_root.rglob("*.log")) + \ + list(search_root.rglob("*.txt")) + for f in result_files: + content = f.read_text(errors="replace") + if "transactions" in content.lower() or \ + "queries/sec" in content.lower() or \ + "events/sec" in content.lower(): + metrics_found = True + break + if metrics_found: + break + + assert metrics_found, \ + "No result file contains sysbench metrics (transactions/queries)" + + @needs_pg + def test_no_sysbench_fatal_errors(self, benchmark_result): + """Result files must not contain sysbench FATAL errors.""" + fatal_lines = [] + for search_root in (TAF_ROOT / "results", TAF_ROOT / "archive"): + if not search_root.is_dir(): + continue + for f in search_root.rglob("*.log"): + # Skip logs from known-failed archive runs + if "Error_" in str(f): + continue + content = f.read_text(errors="replace") + for line in content.splitlines(): + if re.search(r"\bFATAL\b", line, re.IGNORECASE): + fatal_lines.append(f"{f.name}: {line}") + + assert not fatal_lines, \ + f"FATAL errors found in sysbench output:\n" + "\n".join(fatal_lines[:5]) + + +# =========================================================================== +# LAYER 6 — MariaDB regression test +# =========================================================================== + +class TestL6MariaDBRegression: + """Verifies that changes to sysbench-lua.pm have not broken MySQL/MariaDB behaviour.""" + + @needs_mariadb + def test_mariadb_run_exits_cleanly(self, tmp_path_factory): + """A short MariaDB TAF run must finish with exit code 0.""" + mariadb_dir = Path(MARIADB_DIR) + tmp = tmp_path_factory.mktemp("taf_l6") + props_file = tmp / "mariadb_regression.properties" + + write_props(props_file, { + "taf.action": "start-db-run-tests", + "taf.taf_db_makers_plugin": "mariadb", + "taf.db_software_install_dir": str(mariadb_dir), + "taf.db_port": "3306", + "taf.test_suite": "sysbench-lua", + "taf.tests": "OLTP_RO", + "taf.verbose": "true", + "sysbench_lua.db_driver": "mysql", + "sysbench_lua.def_threads": "4", + "sysbench_lua.def_duration": "30", + "sysbench_lua.number_of_rows": "10000", + }) + + result = taf_propfile(props_file, timeout=300) + assert result.returncode == 0, ( + f"MariaDB regression run failed (rc={result.returncode})\n" + f"STDOUT:\n{result.stdout[-3000:]}" + ) + + @needs_mariadb + def test_mysql_args_used_for_mariadb(self, tmp_path_factory): + """MariaDB run must use --mysql-host (not --pgsql-host).""" + mariadb_dir = Path(MARIADB_DIR) + tmp = tmp_path_factory.mktemp("taf_l6_args") + props_file = tmp / "mariadb_args.properties" + + write_props(props_file, { + "taf.action": "start-db-run-tests", + "taf.taf_db_makers_plugin": "mariadb", + "taf.db_software_install_dir": str(mariadb_dir), + "taf.db_port": "3306", + "taf.test_suite": "sysbench-lua", + "taf.tests": "OLTP_RO", + "taf.verbose": "true", + "sysbench_lua.db_driver": "mysql", + "sysbench_lua.def_threads": "4", + "sysbench_lua.def_duration": "30", + "sysbench_lua.number_of_rows": "10000", + }) + + result = taf_propfile(props_file, timeout=300) + output = result.stdout + result.stderr + + assert "--mysql-host" in output or "--mysql-port" in output, \ + "MariaDB run does not use --mysql-* arguments — SetConnectionArgs() may be broken" + assert "--pgsql-host" not in output, \ + "MariaDB run incorrectly uses --pgsql-host!" + + def test_normalize_db_type_mysql_unchanged(self): + """NormalizeDBType must still map mariadb → mysql (and postgres → pgsql). + + Strategy: we extract the body of sub NormalizeDBType directly from the .pm file + using a regex and embed it into an isolated Perl script — this avoids + sysbench-lua.pm's dependencies on the TAF runtime. + """ + suite_file = TAF_ROOT / "test_suites" / "sysbench-lua.pm" + content = suite_file.read_text() + + m = re.search(r"(sub NormalizeDBType \{.+?\n\})", content, re.DOTALL) + assert m, "NormalizeDBType not found in sysbench-lua.pm — cannot run L6 test" + func_body = m.group(1) + + perl_code = textwrap.dedent("""\ + use strict; + use warnings; + + {func} + + my %tests = ( + mariadb => "mysql", + maria => "mysql", + mysql => "mysql", + mysqld => "mysql", + postgres => "pgsql", + postgresql => "pgsql", + pgsql => "pgsql", + ); + + my $fail = 0; + for my $input (sort keys %tests) {{ + my $expected = $tests{{$input}}; + my $got = NormalizeDBType($input) // ""; + if ($got eq $expected) {{ + print "PASS: NormalizeDBType('$input') = '$got'\\n"; + }} else {{ + print "FAIL: NormalizeDBType('$input') = '$got', expected '$expected'\\n"; + $fail++; + }} + }} + exit $fail; + """).format(func=func_body) + + result = run("perl", "-e", perl_code, cwd=TAF_ROOT, timeout=15) + output = result.stdout + result.stderr + fails = [ln for ln in output.splitlines() if ln.startswith("FAIL:")] + assert result.returncode == 0, ( + f"NormalizeDBType returns incorrect values:\n" + + "\n".join(fails) + + f"\nFull output:\n{output}" + )