Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
77 commits
Select commit Hold shift + click to select a range
a2d107d
ci: test db backup & restore
Nov 9, 2026
fd2d138
add docs
Nov 9, 2026
9f8a466
shellcheck
Nov 9, 2026
e980f78
shellcheck
Nov 9, 2026
96d7c96
streamline ci
Nov 9, 2026
5d7de84
touch env
Nov 9, 2026
acb9b62
touch higher
Nov 9, 2026
cb38fa5
try to separate things abit
Nov 9, 2026
3a93c6d
envstuff
Nov 9, 2026
5eedd7c
background docker
Nov 9, 2026
cb6eb2b
--build
Nov 9, 2026
1e6fc98
-T
Nov 9, 2026
7406fbd
wait-for-http-response
Nov 9, 2026
97fe063
wait-
Nov 9, 2026
9dcdd82
wip
Nov 9, 2026
bc2ec23
create admin user
Nov 9, 2026
28a8584
echo admin password at curl
Nov 9, 2026
56c45da
try wiaint properly; fix port
Nov 9, 2026
9a43749
parameterise serviceHost
Nov 9, 2026
25cca1d
relax waiting
Nov 9, 2026
c247281
skip wait-for-it
Nov 9, 2026
27485e8
more logging, more curling
Nov 9, 2026
59df1bb
add container logs
Nov 9, 2026
bc005bd
move to test-images
Nov 10, 2026
39967c5
fix path
Nov 10, 2026
440f56d
ps
Nov 10, 2026
ebefe6e
try localhost https
Nov 10, 2026
73136a1
insecure
Nov 10, 2026
f19bb0f
alignment
Nov 10, 2026
d6f4dd4
resolve
Nov 10, 2026
fd95251
CIULORP
Nov 10, 2026
c534d04
use --pass
Nov 10, 2026
127f1e2
try --user :
Nov 10, 2026
e300190
centralise password
Nov 10, 2026
dc94d51
user in ocning
Nov 10, 2026
41b04ac
more debug
Nov 10, 2026
6afd045
notraceascii
Nov 10, 2026
a98d88c
restore close-iosses
Nov 12, 2026
ad7ec3d
Avoid curl: (18) transfer closed with outstanding read data remaining
Nov 12, 2026
2e538ed
try smarter
Nov 12, 2026
dd4149d
try mapping
Nov 12, 2026
ec131e1
on/off
Nov 12, 2026
dd5dcda
try separate file & routes
Nov 12, 2026
877ced0
specify backup file; disable remote-header-name
Nov 13, 2026
761604a
don't duplicate backupsDir in path
Nov 13, 2026
dc21f77
reverse conf order
Nov 13, 2026
8a00612
ghttp 1
Nov 13, 2026
0b6db54
fix path?
Nov 13, 2026
0ee83a1
no remote-name
Nov 13, 2026
76a3045
remote http1.1
Nov 13, 2026
20ba18a
try printing pg version
Nov 13, 2026
9aaf0e8
try anoterh version
Nov 13, 2026
c9381d2
print pg_restore version
Nov 13, 2026
fb41a3f
more version checks
Nov 13, 2026
cd4b824
verify with containerised pg_restore
Nov 13, 2026
fc2939f
stop container before restoring
Nov 13, 2026
964b20c
loggg
Nov 13, 2026
a4a28c7
fix uname and db name
Nov 13, 2026
ea5ab7a
bp
Nov 13, 2026
3cbf4d0
drop & recreate db
Nov 13, 2026
bb0d090
exec not urn
Nov 13, 2026
7ccac74
drop and recreate separate
Nov 13, 2026
5b2a1d8
use postggres db
Nov 13, 2026
15566ee
more debugb
Nov 13, 2026
24103c7
more debug
Nov 13, 2026
6c01187
formatting
Nov 13, 2026
ea92400
try reassigning
Nov 13, 2026
2d907e6
try as postgres?
Nov 13, 2026
9912955
try dropping instead
Nov 13, 2026
a50474a
try
Nov 13, 2026
82c97cd
log all on postgres db
Nov 13, 2026
ddc02e4
try vacuum
Nov 13, 2026
6bf862f
try user postgres
Nov 13, 2026
0c43591
Merge branch 'next' into tst-postgres-backup-restre
alxndrsn Nov 16, 2026
3304f12
ci: deprioritise azure ubuntu mirrors (#2329)
Nov 19, 2026
e476548
revert package.json changes
Nov 19, 2026
233272a
Merge branch 'next' into tst-postgres-backup-restre
Nov 19, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 0 additions & 122 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,69 +29,8 @@ jobs:
- run: cd test && npm clean-install --no-audit
- run: cd test && npm run lint
- run: cd test && npm run test:github-actions
test-envsub:
timeout-minutes: 2
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v5
- uses: ./.github/actions/relegate-azure-mirrors
- run: cd test/envsub && ./run-tests.sh
test-nginx:
timeout-minutes: 4
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v5
- uses: ./.github/actions/relegate-azure-mirrors
- uses: actions/setup-node@v5
with:
node-version-file: test/package.json
- run: cd test && npm clean-install --no-audit
- run: cd test/nginx && ./setup-tests.sh
- run: cd test && npm run test:nginx:mocha
- run: cd test/nginx && ./lint-config.sh

- run: cd test && npx playwright install --with-deps chromium-headless-shell
- run: cd test && npm run test:nginx:playwright

- if: always()
run: cd test/nginx && docker compose -f nginx.test.docker-compose.yml logs --no-log-prefix nginx-ssl-selfsign
- if: always()
run: cd test/nginx && docker compose -f nginx.test.docker-compose.yml logs --no-log-prefix nginx-ssl-upstream
- if: always()
run: cd test/nginx && docker compose -f nginx.test.docker-compose.yml logs --no-log-prefix service
- if: always()
run: cd test/nginx && docker compose -f nginx.test.docker-compose.yml logs --no-log-prefix enketo
- if: always()
run: cd test/nginx && docker compose -f nginx.test.docker-compose.yml logs --no-log-prefix sentry-mock
test-secrets:
timeout-minutes: 2
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v5
- uses: ./.github/actions/relegate-azure-mirrors
- run: ./test/test-secrets.sh
test-service:
timeout-minutes: 5
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v5
with:
submodules: true
- uses: ./.github/actions/relegate-azure-mirrors
- uses: actions/setup-node@v5
with:
node-version-file: test/package.json
- run: cd test && npm clean-install --no-audit
- run: touch ./files/allow-postgres14-upgrade
- run: cd test && npm run test:service
test-images:
timeout-minutes: 10
needs:
- test-misc
- test-envsub
- test-nginx
- test-secrets
- test-service
runs-on: ubuntu-24.04 # TODO matrix to run on all expected versions?
steps:
- uses: actions/checkout@v5
Expand All @@ -117,64 +56,3 @@ jobs:

- if: always()
run: docker compose logs
build-push-image:
if: |
(github.event_name == 'workflow_dispatch' && inputs.publish_images == true) ||
(github.event_name != 'workflow_dispatch' && (
github.ref == 'refs/heads/master' ||
startsWith(github.ref, 'refs/tags/v')
))
needs:
- test-images
runs-on: ubuntu-24.04
permissions:
contents: read
packages: write
strategy:
matrix:
image: [nginx, service]
env:
REGISTRY: ghcr.io
steps:
- name: Checkout repository
uses: actions/checkout@v5
with:
fetch-depth: 0
fetch-tags: true
submodules: recursive
- uses: ./.github/actions/relegate-azure-mirrors
- name: Log into registry ${{ env.REGISTRY }}
uses: docker/login-action@v4
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Show Docker Context
run: ./test/check-docker-context.sh --report

- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@v6
with:
images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/central-${{ matrix.image }}

- name: Set up QEMU emulator for multi-arch images
uses: docker/setup-qemu-action@v4

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4

- run: ./files/ci/extract-frontend-version.sh

- name: Build and push ${{ matrix.image }} Docker image
uses: docker/build-push-action@v7
with:
file: ${{ matrix.image }}.dockerfile
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
platforms: 'linux/amd64,linux/arm64'
build-args: |
FRONTEND_BUILD_MODE=fetch
FRONTEND_VERSION=${{ env.FRONTEND_VERSION }}
11 changes: 11 additions & 0 deletions files/nginx/central-backend.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
proxy_hide_header Content-Security-Policy;
add_header Content-Security-Policy $central_backend_csp always;

include /usr/share/odk/nginx/common-headers.conf;

proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://service:8383;
proxy_redirect off;

proxy_request_buffering on;
proxy_read_timeout 2m;
16 changes: 5 additions & 11 deletions files/nginx/odk.conf.template
Original file line number Diff line number Diff line change
Expand Up @@ -203,19 +203,13 @@ server {
# End of Enketo Configuration.

# central-backend
location = /v1/backup {
include /usr/share/odk/nginx/central-backend.conf;
proxy_buffering off;
}
location ~ ^/v\d {
proxy_hide_header Content-Security-Policy;
add_header Content-Security-Policy $central_backend_csp always;

include /usr/share/odk/nginx/common-headers.conf;

proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://service:8383;
proxy_redirect off;

proxy_request_buffering on;
include /usr/share/odk/nginx/central-backend.conf;
proxy_buffering on;
proxy_read_timeout 2m;
}

location @blank.html {
Expand Down
1 change: 1 addition & 0 deletions nginx.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ COPY files/nginx/setup-odk.sh \

COPY files/nginx/redirector.conf /usr/share/odk/nginx/
COPY files/nginx/common-headers.conf /usr/share/odk/nginx/
COPY files/nginx/central-backend.conf /usr/share/odk/nginx/
COPY files/nginx/robots.txt /usr/share/nginx/html
COPY --from=intermediate dist/ /usr/share/nginx/html
COPY --from=intermediate /tmp/version.txt /usr/share/nginx/html
Expand Down
25 changes: 3 additions & 22 deletions test/nginx/setup-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,33 +8,14 @@ docker_compose() {
docker compose --file nginx.test.docker-compose.yml "$@"
}

wait_for_http_response() {
local seconds="$1"
local url="$2"
local expectedStatus="${3-200}"
echo >&2 -n "[$(basename "$0")] [$(date)] Waiting for server: $url "
if timeout "$seconds" bash -c -- "
until [[ \$(curl --max-time 1 --silent --output /dev/null -w '%{http_code}' '$url') = \"$expectedStatus\" ]]; do
printf >&2 .
sleep 0.25
done
"; then
printf >&2 '✅\n'
else
printf >&2 '❌\n'
log "!!! URL timed out: $url"
exit 1
fi
}

log "Starting test services..."
docker_compose up --build --detach

log "Waiting for mock backend..."
wait_for_http_response 5 localhost:8383/__mock_http_server/health 200
../wait-for-http-response.sh 5 localhost:8383/__mock_http_server/health 200
log "Waiting for mock enketo..."
wait_for_http_response 5 localhost:8005/__mock_http_server/health 200
../wait-for-http-response.sh 5 localhost:8005/__mock_http_server/health 200
log "Waiting for nginx..."
wait_for_http_response 5 localhost:9000 421
../wait-for-http-response.sh 5 localhost:9000 421

log "Completed OK."
177 changes: 177 additions & 0 deletions test/test-images.sh
Original file line number Diff line number Diff line change
Expand Up @@ -93,4 +93,181 @@ if [[ "$processCount" != 4 ]]; then
exit 1
fi

log "Verifying db backup/restore..."
# Test integration of various packages & compatibility between versions, including:
#
# * pg_dump & pg_restore inside service container
# * postgres version in postgres14 container
# * openssl version in service container
# * openssl version in postgres14 container
#
# NOTE the backup, verify & restore commands, processes & config used in these
# tests should follow the official documentation as closely as possible.
#
# See: https://docs.getodk.org/central-backup/

adminUserEmail=odk-central-admin@example.test
adminUserPword="topSecret-$RANDOM|123"
nginxHost="https://localhost"

show_db_ownership() {
log "Checking ownerships..."
docker compose exec postgres14 psql -U postgres postgres -c "
SELECT n.nspname AS schema_name
, c.relname AS object_name
, CASE c.relkind
WHEN 'r' THEN 'table'
WHEN 'v' THEN 'view'
WHEN 'm' THEN 'materialized view'
WHEN 'S' THEN 'sequence'
WHEN 'f' THEN 'foreign table'
ELSE c.relkind::text
END AS object_type
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
JOIN pg_authid a ON a.oid = c.relowner
WHERE a.rolname = 'odk'
AND n.nspname NOT IN ('pg_catalog', 'information_schema')
"
log "---"
docker compose exec postgres14 psql -U postgres postgres -c "
SELECT b.classid::regclass AS dependent_type
, b.objid AS dependent_object_id
, b.refclassid::regclass AS referenced_type
, d.extname AS extension_name
FROM pg_depend b
LEFT JOIN pg_extension d ON d.oid = b.objid
WHERE b.refobjid = (SELECT oid FROM pg_roles WHERE rolname = 'odk')
"
docker compose exec postgres14 psql -U postgres postgres -c "
(
SELECT 'Table/View/Seq' AS object_type
, n.nspname || '.' || c.relname AS object_name
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
JOIN pg_authid a ON a.oid = c.relowner
WHERE a.rolname = 'odk' AND n.nspname NOT IN ('pg_catalog', 'information_schema')
)
UNION ALL
(
SELECT 'Function/Procedure' AS object_type
, n.nspname || '.' || p.proname AS object_name
FROM pg_proc p
JOIN pg_namespace n ON n.oid = p.pronamespace
JOIN pg_authid a ON a.oid = p.proowner
WHERE a.rolname = 'odk' AND n.nspname NOT IN ('pg_catalog', 'information_schema')
)
UNION ALL
(
SELECT 'Type' AS object_type
, n.nspname || '.' || t.typname AS object_name
FROM pg_type t
JOIN pg_namespace n ON n.oid = t.typnamespace
JOIN pg_authid a ON a.oid = t.typowner
WHERE a.rolname = 'odk' AND n.nspname NOT IN ('pg_catalog', 'information_schema')
)
UNION ALL
(
SELECT 'Extension' AS object_type
, extname AS object_name
FROM pg_extension e
JOIN pg_authid a ON a.oid = e.extowner
WHERE a.rolname = 'odk'
);
"
log "---"
}

test_backup_restore() {
local testDescription="$1"
local passphrase="$2"
local tmpConfig
tmpConfig="$(mktemp)"
backupsDir="$(mktemp --directory)"
backupFile="$backupsDir/backup.enc.bin"

log "Checking pg_dump version..."
docker compose exec service pg_dump --version # FIXME delete this - just for debug
log "Checking container pg_restore version..."
docker compose exec service pg_restore --version # FIXME delete this - just for debug
log "Checking local pg_restore version..."
pg_restore --version

log "Testing backup $testDescription..."

log "Writing curl config file..."
cat >"$tmpConfig" <<EOF
url = "https://local/v1/backup"
user = "$adminUserEmail:$adminUserPword"
json = {"passphrase":"$passphrase"}
#output-dir = $backupsDir
#http1.1
show-error
fail
#remote-name
#remote-header-name # disabled; maybe not a good recommendation? FIXME research if docs should be updated to remove this
remove-on-error
no-clobber
create-dirs
write-out = "%{onerror}%{stderr}\n\n\n\tAN ERROR OCCURRED AND YOUR BACKUP WAS NOT SAVED\n\n\n"
EOF

log "Backing up to $backupFile ..."
curl \
--verbose \
--insecure \
--resolve local:443:127.0.0.1 \
--output "$backupFile" \
--config "$tmpConfig" \
--http1.0
# FIXME remove http1.0 - just trying to get things working, but this is not a permanent solution

log "Verifying backup..."
# TODO this may require documentation updates for ensuring the pg_restore version used to check is the same as the version used inside the container
openssl enc -d -chacha20 -pbkdf2 -pass "pass:$passphrase" <"$backupFile" | docker compose exec -T service pg_restore --file /dev/null

log "Stopping service container to allow db restore..."
docker compose stop service

show_db_ownership

log "Manually dropping and recreating DB to avoid weird errors..."
log "Dropping db..."
docker compose exec postgres14 psql -U postgres postgres -c "DROP DATABASE IF EXISTS odk"
docker compose exec postgres14 psql -U postgres postgres -c "VACUUM FULL ANALYZE;"
show_db_ownership
log "DB dropped; recreating..."
docker compose exec postgres14 psql -U postgres postgres -c "CREATE DATABASE odk OWNER odk"
log "DB recreated."

show_db_ownership

docker compose exec postgres14 psql -U postgres odk -c "DROP EXTENSION IF EXISTS plpgsql CASCADE"
#docker compose exec postgres14 psql -U postgres postgres -c "ALTER EXTENSION plpgsql OWNER TO postgres"

show_db_ownership

log "Restoring backup..."
# N.B. "useless use of cat" deliberately echoes documented command at https://docs.getodk.org/central-backup/#restoring-a-backup
# shellcheck disable=SC2002
cat "$backupFile" | docker compose run --rm -T service node /usr/odk/lib/bin/restore.js - "$passphrase"

log "Restarting service container..."
docker compose start service
check_path 5 /v1/projects '{"message":"Could not authenticate with the provided credentials.","code":401.2}'
}

# FIXME debug remove
docker ps

log "Waiting for service to become available..."
./test/wait-for-http-response.sh 30 "$nginxHost" || true

log "Ensuring admin user exists..."
echo "$adminUserPword" | docker compose exec -T service node /usr/odk/lib/bin/cli --email "$adminUserEmail" user-create
docker compose exec -T service node /usr/odk/lib/bin/cli --email "$adminUserEmail" user-promote
log "Starting backup/restore tests..."
test_backup_restore "without passphrase" ""
test_backup_restore "with passphrase" "hunter2"

log "All OK."
Loading
Loading