diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b58005faf..3b75b0853 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -29,69 +29,8 @@ jobs: - run: cd test && npm clean-install --no-audit - run: cd test && npm run lint - run: cd test && npm run test:github-actions - test-envsub: - timeout-minutes: 2 - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v5 - - uses: ./.github/actions/relegate-azure-mirrors - - run: cd test/envsub && ./run-tests.sh - test-nginx: - timeout-minutes: 4 - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v5 - - uses: ./.github/actions/relegate-azure-mirrors - - uses: actions/setup-node@v5 - with: - node-version-file: test/package.json - - run: cd test && npm clean-install --no-audit - - run: cd test/nginx && ./setup-tests.sh - - run: cd test && npm run test:nginx:mocha - - run: cd test/nginx && ./lint-config.sh - - - run: cd test && npx playwright install --with-deps chromium-headless-shell - - run: cd test && npm run test:nginx:playwright - - - if: always() - run: cd test/nginx && docker compose -f nginx.test.docker-compose.yml logs --no-log-prefix nginx-ssl-selfsign - - if: always() - run: cd test/nginx && docker compose -f nginx.test.docker-compose.yml logs --no-log-prefix nginx-ssl-upstream - - if: always() - run: cd test/nginx && docker compose -f nginx.test.docker-compose.yml logs --no-log-prefix service - - if: always() - run: cd test/nginx && docker compose -f nginx.test.docker-compose.yml logs --no-log-prefix enketo - - if: always() - run: cd test/nginx && docker compose -f nginx.test.docker-compose.yml logs --no-log-prefix sentry-mock - test-secrets: - timeout-minutes: 2 - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v5 - - uses: ./.github/actions/relegate-azure-mirrors - - run: ./test/test-secrets.sh - test-service: - timeout-minutes: 5 - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v5 - with: - submodules: true - - uses: ./.github/actions/relegate-azure-mirrors - - uses: actions/setup-node@v5 - with: - node-version-file: test/package.json - - run: cd test && npm clean-install --no-audit - - run: touch ./files/allow-postgres14-upgrade - - run: cd test && npm run test:service test-images: timeout-minutes: 10 - needs: - - test-misc - - test-envsub - - test-nginx - - test-secrets - - test-service runs-on: ubuntu-24.04 # TODO matrix to run on all expected versions? steps: - uses: actions/checkout@v5 @@ -117,64 +56,3 @@ jobs: - if: always() run: docker compose logs - build-push-image: - if: | - (github.event_name == 'workflow_dispatch' && inputs.publish_images == true) || - (github.event_name != 'workflow_dispatch' && ( - github.ref == 'refs/heads/master' || - startsWith(github.ref, 'refs/tags/v') - )) - needs: - - test-images - runs-on: ubuntu-24.04 - permissions: - contents: read - packages: write - strategy: - matrix: - image: [nginx, service] - env: - REGISTRY: ghcr.io - steps: - - name: Checkout repository - uses: actions/checkout@v5 - with: - fetch-depth: 0 - fetch-tags: true - submodules: recursive - - uses: ./.github/actions/relegate-azure-mirrors - - name: Log into registry ${{ env.REGISTRY }} - uses: docker/login-action@v4 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Show Docker Context - run: ./test/check-docker-context.sh --report - - - name: Extract Docker metadata - id: meta - uses: docker/metadata-action@v6 - with: - images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/central-${{ matrix.image }} - - - name: Set up QEMU emulator for multi-arch images - uses: docker/setup-qemu-action@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 - - - run: ./files/ci/extract-frontend-version.sh - - - name: Build and push ${{ matrix.image }} Docker image - uses: docker/build-push-action@v7 - with: - file: ${{ matrix.image }}.dockerfile - context: . - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - platforms: 'linux/amd64,linux/arm64' - build-args: | - FRONTEND_BUILD_MODE=fetch - FRONTEND_VERSION=${{ env.FRONTEND_VERSION }} diff --git a/files/nginx/central-backend.conf b/files/nginx/central-backend.conf new file mode 100644 index 000000000..91b53fc24 --- /dev/null +++ b/files/nginx/central-backend.conf @@ -0,0 +1,11 @@ +proxy_hide_header Content-Security-Policy; +add_header Content-Security-Policy $central_backend_csp always; + +include /usr/share/odk/nginx/common-headers.conf; + +proxy_set_header X-Forwarded-Proto $scheme; +proxy_pass http://service:8383; +proxy_redirect off; + +proxy_request_buffering on; +proxy_read_timeout 2m; diff --git a/files/nginx/odk.conf.template b/files/nginx/odk.conf.template index b31eb5f7a..21b221995 100644 --- a/files/nginx/odk.conf.template +++ b/files/nginx/odk.conf.template @@ -203,19 +203,13 @@ server { # End of Enketo Configuration. # central-backend + location = /v1/backup { + include /usr/share/odk/nginx/central-backend.conf; + proxy_buffering off; + } location ~ ^/v\d { - proxy_hide_header Content-Security-Policy; - add_header Content-Security-Policy $central_backend_csp always; - - include /usr/share/odk/nginx/common-headers.conf; - - proxy_set_header X-Forwarded-Proto $scheme; - proxy_pass http://service:8383; - proxy_redirect off; - - proxy_request_buffering on; + include /usr/share/odk/nginx/central-backend.conf; proxy_buffering on; - proxy_read_timeout 2m; } location @blank.html { diff --git a/nginx.dockerfile b/nginx.dockerfile index 65eeb07d4..797d2739e 100644 --- a/nginx.dockerfile +++ b/nginx.dockerfile @@ -42,6 +42,7 @@ COPY files/nginx/setup-odk.sh \ COPY files/nginx/redirector.conf /usr/share/odk/nginx/ COPY files/nginx/common-headers.conf /usr/share/odk/nginx/ +COPY files/nginx/central-backend.conf /usr/share/odk/nginx/ COPY files/nginx/robots.txt /usr/share/nginx/html COPY --from=intermediate dist/ /usr/share/nginx/html COPY --from=intermediate /tmp/version.txt /usr/share/nginx/html diff --git a/test/nginx/setup-tests.sh b/test/nginx/setup-tests.sh index 44774afaa..fb375ecb0 100755 --- a/test/nginx/setup-tests.sh +++ b/test/nginx/setup-tests.sh @@ -8,33 +8,14 @@ docker_compose() { docker compose --file nginx.test.docker-compose.yml "$@" } -wait_for_http_response() { - local seconds="$1" - local url="$2" - local expectedStatus="${3-200}" - echo >&2 -n "[$(basename "$0")] [$(date)] Waiting for server: $url " - if timeout "$seconds" bash -c -- " - until [[ \$(curl --max-time 1 --silent --output /dev/null -w '%{http_code}' '$url') = \"$expectedStatus\" ]]; do - printf >&2 . - sleep 0.25 - done - "; then - printf >&2 '✅\n' - else - printf >&2 '❌\n' - log "!!! URL timed out: $url" - exit 1 - fi -} - log "Starting test services..." docker_compose up --build --detach log "Waiting for mock backend..." -wait_for_http_response 5 localhost:8383/__mock_http_server/health 200 +../wait-for-http-response.sh 5 localhost:8383/__mock_http_server/health 200 log "Waiting for mock enketo..." -wait_for_http_response 5 localhost:8005/__mock_http_server/health 200 +../wait-for-http-response.sh 5 localhost:8005/__mock_http_server/health 200 log "Waiting for nginx..." -wait_for_http_response 5 localhost:9000 421 +../wait-for-http-response.sh 5 localhost:9000 421 log "Completed OK." diff --git a/test/test-images.sh b/test/test-images.sh index 3849fa3c0..5f99b846b 100755 --- a/test/test-images.sh +++ b/test/test-images.sh @@ -93,4 +93,181 @@ if [[ "$processCount" != 4 ]]; then exit 1 fi +log "Verifying db backup/restore..." +# Test integration of various packages & compatibility between versions, including: +# +# * pg_dump & pg_restore inside service container +# * postgres version in postgres14 container +# * openssl version in service container +# * openssl version in postgres14 container +# +# NOTE the backup, verify & restore commands, processes & config used in these +# tests should follow the official documentation as closely as possible. +# +# See: https://docs.getodk.org/central-backup/ + +adminUserEmail=odk-central-admin@example.test +adminUserPword="topSecret-$RANDOM|123" +nginxHost="https://localhost" + +show_db_ownership() { + log "Checking ownerships..." + docker compose exec postgres14 psql -U postgres postgres -c " + SELECT n.nspname AS schema_name + , c.relname AS object_name + , CASE c.relkind + WHEN 'r' THEN 'table' + WHEN 'v' THEN 'view' + WHEN 'm' THEN 'materialized view' + WHEN 'S' THEN 'sequence' + WHEN 'f' THEN 'foreign table' + ELSE c.relkind::text + END AS object_type + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_authid a ON a.oid = c.relowner + WHERE a.rolname = 'odk' + AND n.nspname NOT IN ('pg_catalog', 'information_schema') + " + log "---" + docker compose exec postgres14 psql -U postgres postgres -c " + SELECT b.classid::regclass AS dependent_type + , b.objid AS dependent_object_id + , b.refclassid::regclass AS referenced_type + , d.extname AS extension_name + FROM pg_depend b + LEFT JOIN pg_extension d ON d.oid = b.objid + WHERE b.refobjid = (SELECT oid FROM pg_roles WHERE rolname = 'odk') + " + docker compose exec postgres14 psql -U postgres postgres -c " + ( + SELECT 'Table/View/Seq' AS object_type + , n.nspname || '.' || c.relname AS object_name + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_authid a ON a.oid = c.relowner + WHERE a.rolname = 'odk' AND n.nspname NOT IN ('pg_catalog', 'information_schema') + ) + UNION ALL + ( + SELECT 'Function/Procedure' AS object_type + , n.nspname || '.' || p.proname AS object_name + FROM pg_proc p + JOIN pg_namespace n ON n.oid = p.pronamespace + JOIN pg_authid a ON a.oid = p.proowner + WHERE a.rolname = 'odk' AND n.nspname NOT IN ('pg_catalog', 'information_schema') + ) + UNION ALL + ( + SELECT 'Type' AS object_type + , n.nspname || '.' || t.typname AS object_name + FROM pg_type t + JOIN pg_namespace n ON n.oid = t.typnamespace + JOIN pg_authid a ON a.oid = t.typowner + WHERE a.rolname = 'odk' AND n.nspname NOT IN ('pg_catalog', 'information_schema') + ) + UNION ALL + ( + SELECT 'Extension' AS object_type + , extname AS object_name + FROM pg_extension e + JOIN pg_authid a ON a.oid = e.extowner + WHERE a.rolname = 'odk' + ); + " + log "---" +} + +test_backup_restore() { + local testDescription="$1" + local passphrase="$2" + local tmpConfig + tmpConfig="$(mktemp)" + backupsDir="$(mktemp --directory)" + backupFile="$backupsDir/backup.enc.bin" + + log "Checking pg_dump version..." + docker compose exec service pg_dump --version # FIXME delete this - just for debug + log "Checking container pg_restore version..." + docker compose exec service pg_restore --version # FIXME delete this - just for debug + log "Checking local pg_restore version..." + pg_restore --version + + log "Testing backup $testDescription..." + + log "Writing curl config file..." + cat >"$tmpConfig" <&2 "[$(basename "$0")] $*"; } + +seconds="$1" +url="$2" +expectedStatus="${3-200}" + +echo >&2 -n "[$(basename "$0")] [$(date)] Waiting for server: $url " +if timeout "$seconds" bash -c -- " + until [[ \$(curl --max-time 1 --silent --output /dev/null -w '%{http_code}' '$url') = \"$expectedStatus\" ]]; do + printf >&2 . + sleep 0.25 + done +"; then + printf >&2 '✅\n' +else + printf >&2 '❌\n' + log "!!! URL timed out: $url" + exit 1 +fi