From 3baccac2e2581e2f3df483cb604f4e472a2f982d Mon Sep 17 00:00:00 2001 From: Jesse Jaara Date: Thu, 8 Oct 2026 10:58:48 +0300 Subject: [PATCH 1/3] Update Databse images to PostgreSQL 18 --- azuredbmock/Dockerfile | 16 +++++----------- azuredbmock/archive.sources.list | 7 ------- mapmatching/Dockerfile | 27 ++++++++++++--------------- mapmatching/archive.sources.list | 7 ------- 4 files changed, 17 insertions(+), 40 deletions(-) delete mode 100644 azuredbmock/archive.sources.list delete mode 100644 mapmatching/archive.sources.list diff --git a/azuredbmock/Dockerfile b/azuredbmock/Dockerfile index c309cf0..0889fd9 100644 --- a/azuredbmock/Dockerfile +++ b/azuredbmock/Dockerfile @@ -2,14 +2,10 @@ ## relase ARM64 versions of the image at the moment (they did in the past). ## And thus, we have to build the ARM version our selves. -FROM docker.io/postgres:15-bullseye AS postgis +FROM docker.io/postgres:18-trixie AS postgis ENV POSTGIS_MAJOR=3 -ENV POSTGIS_VERSION=3.5.2+dfsg-1.pgdg110+1 - -# Switch to Debian Archive for EOL disto version -RUN rm /etc/apt/sources.list -ADD archive.sources.list /etc/apt/sources.list +ENV POSTGIS_VERSION=3.6.4+dfsg-2.pgdg13+1 RUN apt-get update \ && apt-cache showpkg postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR \ @@ -24,18 +20,16 @@ RUN apt-get update \ RUN mkdir -p /docker-entrypoint-initdb.d -# As of 2026-09, in Azure PostgreSQL Flexible Server v15, the most recent -# available version of the PostGIS extension is 3.6 and it is in use. -# But postgis docker images only go up to version 3.5. + FROM postgis -# fix collations to use fi_FI +# Change collations to use fi_FI RUN localedef -i fi_FI -c -f UTF-8 -A /usr/share/locale/locale.alias fi_FI.UTF-8 ENV LANG=fi_FI.utf8 RUN mkdir -p -m 0755 /jore4/migrations /jore4/scripts -# download script for reading docker secrets +# Download script for reading docker secrets ADD --chmod=755 https://raw.githubusercontent.com/HSLdevcom/jore4-tools/main/docker/read-secrets.sh /jore4/scripts/read-secrets.sh COPY --chmod=644 migrations/ /jore4/migrations/ diff --git a/azuredbmock/archive.sources.list b/azuredbmock/archive.sources.list deleted file mode 100644 index 743a072..0000000 --- a/azuredbmock/archive.sources.list +++ /dev/null @@ -1,7 +0,0 @@ -deb [check-valid-until=no] http://archive.debian.org/debian bullseye main - -# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1147093 -# Debian efd up, and forgot to move the security repository into archive before deleting it. - -deb [check-valid-until=no] http://snapshot.debian.org/archive/debian-security/20260831T235959Z/ bullseye-security main -deb [check-valid-until=no] http://archive.debian.org/debian bullseye-updates main diff --git a/mapmatching/Dockerfile b/mapmatching/Dockerfile index c9f18f5..b2bc7f2 100644 --- a/mapmatching/Dockerfile +++ b/mapmatching/Dockerfile @@ -2,14 +2,10 @@ ## relase ARM64 versions of the image at the moment (they did in the past). ## And thus, we have to build the ARM version our selves. -FROM docker.io/postgres:15-bullseye AS postgis +FROM docker.io/postgres:18-trixie AS postgis ENV POSTGIS_MAJOR=3 -ENV POSTGIS_VERSION=3.5.2+dfsg-1.pgdg110+1 - -# Switch to Debian Archive for EOL disto version -RUN rm /etc/apt/sources.list -ADD archive.sources.list /etc/apt/sources.list +ENV POSTGIS_VERSION=3.6.4+dfsg-2.pgdg13+1 RUN apt-get update \ && apt-cache showpkg postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR \ @@ -30,17 +26,18 @@ FROM postgis AS pgrouting ENV PGROUTING_VERSION=3.8.0 ENV PGROUTING_SHA256=b8a5f0472934fdf7cda3fb4754d01945378d920cdaddc01f378617ddbb9c447f +ENV LIBBOOST_VERSION=1.83.0 RUN set -ex \ && apt update \ && apt install -y \ - libboost-atomic1.74.0 \ - libboost-chrono1.74.0 \ - libboost-graph1.74.0 \ - libboost-date-time1.74.0 \ - libboost-program-options1.74.0 \ - libboost-system1.74.0 \ - libboost-thread1.74.0 \ + libboost-atomic${LIBBOOST_VERSION} \ + libboost-chrono${LIBBOOST_VERSION} \ + libboost-graph${LIBBOOST_VERSION} \ + libboost-date-time${LIBBOOST_VERSION} \ + libboost-program-options${LIBBOOST_VERSION} \ + libboost-system${LIBBOOST_VERSION} \ + libboost-thread${LIBBOOST_VERSION} \ && apt install -y \ build-essential \ cmake \ @@ -61,11 +58,11 @@ RUN set -ex \ && mkdir build \ && cd build \ && cmake .. \ - && make \ + && make -j $(nproc --all) \ && make install \ && cd / \ && rm -rf /usr/src/pgrouting \ - && apt-mark manual postgresql-15 \ + && apt-mark manual postgresql-18 \ && apt purge -y --autoremove \ build-essential \ cmake \ diff --git a/mapmatching/archive.sources.list b/mapmatching/archive.sources.list deleted file mode 100644 index 743a072..0000000 --- a/mapmatching/archive.sources.list +++ /dev/null @@ -1,7 +0,0 @@ -deb [check-valid-until=no] http://archive.debian.org/debian bullseye main - -# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1147093 -# Debian efd up, and forgot to move the security repository into archive before deleting it. - -deb [check-valid-until=no] http://snapshot.debian.org/archive/debian-security/20260831T235959Z/ bullseye-security main -deb [check-valid-until=no] http://archive.debian.org/debian bullseye-updates main From c755fd7a702d8f15e4aa6adb2c99c71905c0ca23 Mon Sep 17 00:00:00 2001 From: Jesse Jaara Date: Fri, 9 Oct 2026 14:15:41 +0300 Subject: [PATCH 2/3] Update Github Action versions --- .github/workflows/cd.yml | 8 +- .../workflows/check-renovatebot-config.yml | 2 +- .github/workflows/test-docker-compose.yml | 6 +- azuredbmock/Dockerfile | 45 ------ azuredbmock/docker-entrypoint.sh | 21 --- azuredbmock/migrations/01-create-roles.sql | 15 -- .../migrations/02-create-network-database.sql | 33 ----- .../migrations/03-create-auth-database.sql | 12 -- .../04-create-jore3importer-database.sql | 25 ---- .../05-create-timetables-database.sql | 36 ----- .../06-create-stopregistry-database.sql | 39 ----- .../07-create-jore4-main-database.sql | 124 ---------------- azuredbmock/migrations/10_postgis.sh | 41 ------ ...e-placeholders-in-sql-schema-migrations.sh | 136 ------------------ mapmatching/Dockerfile | 87 ----------- mapmatching/entrypoint.sh | 26 ---- 16 files changed, 8 insertions(+), 648 deletions(-) delete mode 100644 azuredbmock/Dockerfile delete mode 100755 azuredbmock/docker-entrypoint.sh delete mode 100644 azuredbmock/migrations/01-create-roles.sql delete mode 100644 azuredbmock/migrations/02-create-network-database.sql delete mode 100644 azuredbmock/migrations/03-create-auth-database.sql delete mode 100644 azuredbmock/migrations/04-create-jore3importer-database.sql delete mode 100644 azuredbmock/migrations/05-create-timetables-database.sql delete mode 100644 azuredbmock/migrations/06-create-stopregistry-database.sql delete mode 100644 azuredbmock/migrations/07-create-jore4-main-database.sql delete mode 100644 azuredbmock/migrations/10_postgis.sh delete mode 100755 azuredbmock/replace-placeholders-in-sql-schema-migrations.sh delete mode 100644 mapmatching/Dockerfile delete mode 100755 mapmatching/entrypoint.sh diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index b6efb8c..8acb8a5 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -16,7 +16,7 @@ jobs: permissions: id-token: write contents: read - uses: HSLdevcom/jore4-tools/.github/workflows/shared-build-and-publish-docker-image.yml@shared-build-and-publish-docker-image-v6 + uses: HSLdevcom/jore4-tools/.github/workflows/shared-build-and-publish-docker-image.yml@main with: docker_image_name: jore4-postgres-mapmatching build_arm64_image: true @@ -31,7 +31,7 @@ jobs: permissions: id-token: write contents: read - uses: HSLdevcom/jore4-tools/.github/workflows/shared-build-and-publish-docker-image.yml@shared-build-and-publish-docker-image-v6 + uses: HSLdevcom/jore4-tools/.github/workflows/shared-build-and-publish-docker-image.yml@main with: docker_image_name: jore4-postgres-azuredbmock build_arm64_image: true @@ -46,7 +46,7 @@ jobs: needs: - publish_mapmatching_docker_image_to_acr - publish_azuredbmock_docker_image_to_acr - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 env: POSTGRES_DB: testdb POSTGRES_USER: user @@ -151,7 +151,7 @@ jobs: - name: Verify that postgres database is up and can be connected to id: healthcheck continue-on-error: true - uses: HSLdevcom/jore4-tools/github-actions/healthcheck@healthcheck-v1 + uses: HSLdevcom/jore4-tools/github-actions/healthcheck@main with: command: 'psql -h localhost -p 6432 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c diff --git a/.github/workflows/check-renovatebot-config.yml b/.github/workflows/check-renovatebot-config.yml index 66402ae..d4d7ea1 100644 --- a/.github/workflows/check-renovatebot-config.yml +++ b/.github/workflows/check-renovatebot-config.yml @@ -9,6 +9,6 @@ permissions: jobs: validate: name: Validate renovatebot config - uses: HSLdevcom/jore4-tools/.github/workflows/shared-check-renovatebot-config.yml@shared-check-renovatebot-config-v2 + uses: HSLdevcom/jore4-tools/.github/workflows/shared-check-renovatebot-config.yml@main with: config_file_path: .github/renovate.json5 diff --git a/.github/workflows/test-docker-compose.yml b/.github/workflows/test-docker-compose.yml index 2ba9dcf..b679ab5 100644 --- a/.github/workflows/test-docker-compose.yml +++ b/.github/workflows/test-docker-compose.yml @@ -13,7 +13,7 @@ permissions: jobs: test-docker-compose: name: verify docker-compose setup - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 steps: - name: Checkout code @@ -25,7 +25,7 @@ jobs: - name: Verify that mapmatching database is up and can be connected to id: mapmatching_healthcheck continue-on-error: true - uses: HSLdevcom/jore4-tools/github-actions/healthcheck@healthcheck-v1 + uses: HSLdevcom/jore4-tools/github-actions/healthcheck@main env: POSTGRES_USER: username POSTGRES_DB: testdb @@ -38,7 +38,7 @@ jobs: - name: Verify that azuredbmock database is up and can be connected to id: azuredbmock_healthcheck continue-on-error: true - uses: HSLdevcom/jore4-tools/github-actions/healthcheck@healthcheck-v1 + uses: HSLdevcom/jore4-tools/github-actions/healthcheck@main env: POSTGRES_USER: username POSTGRES_DB: testdb diff --git a/azuredbmock/Dockerfile b/azuredbmock/Dockerfile deleted file mode 100644 index 0889fd9..0000000 --- a/azuredbmock/Dockerfile +++ /dev/null @@ -1,45 +0,0 @@ -## Below an inlined version of the PostGIS Dockerfile. PostGIS does not -## relase ARM64 versions of the image at the moment (they did in the past). -## And thus, we have to build the ARM version our selves. - -FROM docker.io/postgres:18-trixie AS postgis - -ENV POSTGIS_MAJOR=3 -ENV POSTGIS_VERSION=3.6.4+dfsg-2.pgdg13+1 - -RUN apt-get update \ - && apt-cache showpkg postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR \ - && apt-get install -y --no-install-recommends \ - # ca-certificates: for accessing remote raster files; - # fix: https://github.com/postgis/docker-postgis/issues/307 - ca-certificates \ - \ - postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR=$POSTGIS_VERSION \ - postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts \ - && rm -rf /var/lib/apt/lists/* - -RUN mkdir -p /docker-entrypoint-initdb.d - - -FROM postgis - -# Change collations to use fi_FI -RUN localedef -i fi_FI -c -f UTF-8 -A /usr/share/locale/locale.alias fi_FI.UTF-8 -ENV LANG=fi_FI.utf8 - -RUN mkdir -p -m 0755 /jore4/migrations /jore4/scripts - -# Download script for reading docker secrets -ADD --chmod=755 https://raw.githubusercontent.com/HSLdevcom/jore4-tools/main/docker/read-secrets.sh /jore4/scripts/read-secrets.sh - -COPY --chmod=644 migrations/ /jore4/migrations/ -COPY --chmod=755 replace-placeholders-in-sql-schema-migrations.sh /jore4/scripts/replace-placeholders-in-sql-schema-migrations.sh - -COPY --chmod=755 docker-entrypoint.sh /jore4/scripts/ - -ENTRYPOINT ["/jore4/scripts/docker-entrypoint.sh"] -CMD ["postgres"] - -# have to source the secrets here as well, otherwise the POSTGRES_USER and/or POSTGRES_DB variables might not exist -HEALTHCHECK --interval=5s --timeout=5s --start-period=10s --retries=20 \ - CMD bash -c "source /jore4/scripts/read-secrets.sh && pg_isready -h localhost -U \${POSTGRES_USER} -d \${POSTGRES_DB}" diff --git a/azuredbmock/docker-entrypoint.sh b/azuredbmock/docker-entrypoint.sh deleted file mode 100755 index c0cdd9b..0000000 --- a/azuredbmock/docker-entrypoint.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -Eeuo pipefail - -# read the secrets to environment variables -source /jore4/scripts/read-secrets.sh - -# copy the migrations scripts to the docker-entrypoint-initdb.d folder so that the postgres entrypoint -# executes them (together with the other migrations that might have been mapped as a volume) -mkdir -p /docker-entrypoint-initdb.d -cp /jore4/migrations/* /docker-entrypoint-initdb.d/ - -# replace placeholders with secrets within the all migration SQL scripts -SECRET_STORE_BASE_PATH="${SECRET_STORE_BASE_PATH:-/run/secrets}" -REPLACE_PLACEHOLDERS_SCRIPT='/jore4/scripts/replace-placeholders-in-sql-schema-migrations.sh' -MIGRATIONS_DIR="/docker-entrypoint-initdb.d/" -"${REPLACE_PLACEHOLDERS_SCRIPT}" "${SECRET_STORE_BASE_PATH}" "${MIGRATIONS_DIR}" - -# call the original entrypoint to continue execution -echo 'Start postgres' -exec /usr/local/bin/docker-entrypoint.sh "$@" diff --git a/azuredbmock/migrations/01-create-roles.sql b/azuredbmock/migrations/01-create-roles.sql deleted file mode 100644 index 9c9e5e1..0000000 --- a/azuredbmock/migrations/01-create-roles.sql +++ /dev/null @@ -1,15 +0,0 @@ --- These database roles are also created in the azure-infra-jore4aks (Azure --- DevOps) repository. -CREATE USER xxx_db_auth_username_xxx PASSWORD 'xxx_db_auth_password_xxx'; -CREATE USER xxx_db_jore3importer_username_xxx PASSWORD 'xxx_db_jore3importer_password_xxx'; -CREATE USER xxx_db_hasura_username_xxx PASSWORD 'xxx_db_hasura_password_xxx'; -CREATE USER xxx_db_tiamat_username_xxx PASSWORD 'xxx_db_tiamat_password_xxx'; -CREATE USER xxx_db_timetables_api_username_xxx PASSWORD 'xxx_db_timetables_api_password_xxx'; -CREATE USER xxx_db_dss_username_xxx PASSWORD 'xxx_db_dss_password_xxx'; - --- Make the hasura role a member of jore3importer role because both roles must --- have ownership of tables and sequences in the default database (network and --- routes) since both are responsible for populating and truncating tables in --- the aforementioned database. In particular, sequence reset requires an --- ownership and cannot be granted as a privilege. -GRANT xxx_db_jore3importer_username_xxx TO xxx_db_hasura_username_xxx; diff --git a/azuredbmock/migrations/02-create-network-database.sql b/azuredbmock/migrations/02-create-network-database.sql deleted file mode 100644 index f1f3061..0000000 --- a/azuredbmock/migrations/02-create-network-database.sql +++ /dev/null @@ -1,33 +0,0 @@ --- Make the JORE4 admin role the owner of the public schema. -ALTER SCHEMA public OWNER TO CURRENT_USER; - --- Create the extensions used, see https://hasura.io/docs/latest/graphql/core/deployment/postgres-requirements.html --- Create the extensions in the public schema, since we'd need to give additional privileges ("use schema") to any --- user who wishes to use these in the future. Also, Hasura would require additional setup to be able to use the --- extensions from another schema. -CREATE EXTENSION IF NOT EXISTS btree_gist WITH SCHEMA public; -CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public; -CREATE EXTENSION IF NOT EXISTS postgis WITH SCHEMA public; - --- Allow Hasura to connect and create new schemas. -GRANT CONNECT, CREATE ON DATABASE xxx_db_hasura_name_xxx TO xxx_db_hasura_username_xxx; - --- Grant required privileges in the public schema to the Hasura user. -GRANT ALL ON SCHEMA public TO xxx_db_hasura_username_xxx; -GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_hasura_username_xxx; -GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_hasura_username_xxx; - --- Grant select permissions on information_schema and pg_catalog to the Hasura --- user. -GRANT SELECT ON ALL TABLES IN SCHEMA information_schema TO xxx_db_hasura_username_xxx; -GRANT SELECT ON ALL TABLES IN SCHEMA pg_catalog TO xxx_db_hasura_username_xxx; - --- Allow the JORE3-Importer role to connect to the network database. -GRANT CONNECT ON DATABASE xxx_db_hasura_name_xxx TO xxx_db_jore3importer_username_xxx; - --- Grant the JORE3-Importer role to access the public schema and the objects and --- functions created by extensions. Other schema-specific privileges are granted --- in Hasura migrations. -GRANT USAGE ON SCHEMA public TO xxx_db_jore3importer_username_xxx; -GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_jore3importer_username_xxx; -GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_jore3importer_username_xxx; diff --git a/azuredbmock/migrations/03-create-auth-database.sql b/azuredbmock/migrations/03-create-auth-database.sql deleted file mode 100644 index 07c86f0..0000000 --- a/azuredbmock/migrations/03-create-auth-database.sql +++ /dev/null @@ -1,12 +0,0 @@ -CREATE DATABASE xxx_db_auth_name_xxx; - --- Allow the auth role to connect and create new schemas. -GRANT CONNECT, CREATE ON DATABASE xxx_db_auth_name_xxx TO xxx_db_auth_username_xxx; - -\connect xxx_db_auth_name_xxx; - --- Make the JORE4 admin role the owner of the public schema. -ALTER SCHEMA public OWNER TO CURRENT_USER; - --- Grant full schema access to the public schema to the auth role. -GRANT ALL ON SCHEMA public TO xxx_db_auth_username_xxx; diff --git a/azuredbmock/migrations/04-create-jore3importer-database.sql b/azuredbmock/migrations/04-create-jore3importer-database.sql deleted file mode 100644 index 40f7ad6..0000000 --- a/azuredbmock/migrations/04-create-jore3importer-database.sql +++ /dev/null @@ -1,25 +0,0 @@ -CREATE DATABASE xxx_db_jore3importer_name_xxx; - --- Allow the jore3importer role to connect and create new schemas. -GRANT CONNECT, CREATE ON DATABASE xxx_db_jore3importer_name_xxx TO xxx_db_jore3importer_username_xxx; - -\connect xxx_db_jore3importer_name_xxx; - --- Make the JORE4 admin role the owner of the public schema. -ALTER SCHEMA public OWNER TO CURRENT_USER; - --- Create the extensions that JORE3-Importer needs. In PostgreSQL v15 server, --- an ordinary user (without admin roles) may not be able to create extensions. -CREATE EXTENSION IF NOT EXISTS btree_gist WITH SCHEMA public; -CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public; - --- Grant privileges in the public schema to the jore3importer role. -GRANT USAGE ON SCHEMA public TO xxx_db_jore3importer_username_xxx; -GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_jore3importer_username_xxx; -GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_jore3importer_username_xxx; - --- Grant permission on the pg_catalog schema to conditionally create an --- extension if the extension is not already created. This must be granted so --- that the JORE3-Importer role can conditionally create the extension depending --- on which database is involved (there are several in the importer's test setup). -GRANT SELECT ON TABLE pg_catalog.pg_extension TO xxx_db_jore3importer_username_xxx; diff --git a/azuredbmock/migrations/05-create-timetables-database.sql b/azuredbmock/migrations/05-create-timetables-database.sql deleted file mode 100644 index 8648b48..0000000 --- a/azuredbmock/migrations/05-create-timetables-database.sql +++ /dev/null @@ -1,36 +0,0 @@ -CREATE DATABASE xxx_db_timetables_name_xxx; - --- Allow Hasura to connect and create new schemas. -GRANT CONNECT, CREATE ON DATABASE xxx_db_timetables_name_xxx TO xxx_db_hasura_username_xxx; - --- Interval outputs by default are using the sql format ('3 4:05:06'). Here we --- are switching to ISO 8601 format ('P3DT4H5M6S'). -ALTER DATABASE xxx_db_timetables_name_xxx SET intervalstyle = 'iso_8601'; - --- Switch database context to be able to add extensions there. -\connect xxx_db_timetables_name_xxx; - --- Make the JORE4 admin role the owner of the public schema. -ALTER SCHEMA public OWNER TO CURRENT_USER; - -CREATE EXTENSION IF NOT EXISTS btree_gist WITH SCHEMA public; -CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public; - --- Grant required privileges in the public schema to Hasura. -GRANT ALL ON SCHEMA public TO xxx_db_hasura_username_xxx; -GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_hasura_username_xxx; -GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_hasura_username_xxx; - --- Grant select permissions on information_schema and pg_catalog to Hasura. -GRANT SELECT ON ALL TABLES IN SCHEMA information_schema TO xxx_db_hasura_username_xxx; -GRANT SELECT ON ALL TABLES IN SCHEMA pg_catalog TO xxx_db_hasura_username_xxx; - --- Allow the timetables-api role to connect to the timetables database. -GRANT CONNECT ON DATABASE xxx_db_timetables_name_xxx TO xxx_db_timetables_api_username_xxx; - --- Grant the timetables-api role to access the public schema and the objects and --- functions created by extensions. Other schema-specific privileges are granted --- in Hasura migrations. -GRANT USAGE ON SCHEMA public TO xxx_db_timetables_api_username_xxx; -GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_timetables_api_username_xxx; -GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_timetables_api_username_xxx; diff --git a/azuredbmock/migrations/06-create-stopregistry-database.sql b/azuredbmock/migrations/06-create-stopregistry-database.sql deleted file mode 100644 index 6ea599f..0000000 --- a/azuredbmock/migrations/06-create-stopregistry-database.sql +++ /dev/null @@ -1,39 +0,0 @@ -CREATE DATABASE xxx_db_tiamat_name_xxx; - --- Allow Tiamat to connect and create new schemas. -GRANT CONNECT, CREATE ON DATABASE xxx_db_tiamat_name_xxx TO xxx_db_tiamat_username_xxx; - --- Switch database context to initialise it to the state where Tiamat can use --- it. -\connect xxx_db_tiamat_name_xxx; - --- Make the JORE4 admin role the owner of the public schema. -ALTER SCHEMA public OWNER TO CURRENT_USER; - -CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public; -CREATE EXTENSION IF NOT EXISTS postgis WITH SCHEMA public; - --- Grant required privileges in the public schema to Tiamat. -GRANT ALL ON SCHEMA public TO xxx_db_tiamat_username_xxx; -GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_tiamat_username_xxx; -GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_tiamat_username_xxx; - --- Create "topology" schema and install the "postgis_topology" extension to it. --- The Tiamat role needs ownership to the schema and its tables. -CREATE SCHEMA IF NOT EXISTS topology; -CREATE EXTENSION IF NOT EXISTS postgis_topology WITH SCHEMA topology; -ALTER SCHEMA topology OWNER TO xxx_db_tiamat_username_xxx; --- The "postgis_topology" extension creates two tables. -ALTER TABLE topology.layer OWNER TO xxx_db_tiamat_username_xxx; -ALTER TABLE topology.topology OWNER TO xxx_db_tiamat_username_xxx; - --- Grant Hasura read permissions to the stop registry database. -GRANT CONNECT ON DATABASE xxx_db_tiamat_name_xxx TO xxx_db_hasura_username_xxx; - -GRANT USAGE ON SCHEMA public TO xxx_db_hasura_username_xxx; -GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_hasura_username_xxx; -ALTER DEFAULT PRIVILEGES FOR USER xxx_db_tiamat_username_xxx IN SCHEMA public GRANT SELECT ON TABLES TO xxx_db_hasura_username_xxx; - -GRANT USAGE ON SCHEMA topology TO xxx_db_hasura_username_xxx; -GRANT SELECT ON ALL TABLES IN SCHEMA topology TO xxx_db_hasura_username_xxx; -ALTER DEFAULT PRIVILEGES FOR USER xxx_db_tiamat_username_xxx IN SCHEMA topology GRANT SELECT ON TABLES TO xxx_db_hasura_username_xxx; diff --git a/azuredbmock/migrations/07-create-jore4-main-database.sql b/azuredbmock/migrations/07-create-jore4-main-database.sql deleted file mode 100644 index 2d268ac..0000000 --- a/azuredbmock/migrations/07-create-jore4-main-database.sql +++ /dev/null @@ -1,124 +0,0 @@ --- This migration replicates initialization of the JORE4 main database in script scripts/create_db_roles_and_extensions.sh --- in azure-infra-jore4aks repository - -CREATE DATABASE xxx_db_jore4_main_name_xxx; - -\connect xxx_db_jore4_main_name_xxx; - - --------------------------- ------ Create Schemas ----- --------------------------- - -CREATE SCHEMA IF NOT EXISTS network AUTHORIZATION xxx_db_hasura_username_xxx; -CREATE SCHEMA IF NOT EXISTS stopregistry AUTHORIZATION xxx_db_tiamat_username_xxx; -CREATE SCHEMA IF NOT EXISTS timetables AUTHORIZATION xxx_db_hasura_username_xxx; -CREATE SCHEMA IF NOT EXISTS hdb_catalog AUTHORIZATION xxx_db_hasura_username_xxx; -CREATE SCHEMA IF NOT EXISTS dssview AUTHORIZATION xxx_db_hasura_username_xxx; - -CREATE SCHEMA IF NOT EXISTS topology AUTHORIZATION xxx_db_tiamat_username_xxx; - - ------------------------------ ------ Create Extensions ----- ------------------------------ - --- Extensions are created in public schema so the extensions are found from the search path -CREATE EXTENSION IF NOT EXISTS btree_gist WITH SCHEMA public; -CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public; -CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public; -CREATE EXTENSION IF NOT EXISTS postgis WITH SCHEMA public; -CREATE EXTENSION IF NOT EXISTS postgis_topology WITH SCHEMA topology; - - ---------------------------------------- ------ Configure Database Settings ----- ---------------------------------------- - --- Interval outputs by default are using the SQL format ('3 4:05:06'). Here we --- switch to ISO 8601 format ('P3DT4H5M6S'). -ALTER DATABASE xxx_db_jore4_main_name_xxx SET intervalstyle = 'iso_8601'; - - --------------------------------------------------- ------ Grant Database Level Access Privileges ----- --------------------------------------------------- - -GRANT CONNECT, CREATE ON DATABASE xxx_db_jore4_main_name_xxx TO xxx_db_hasura_username_xxx; -GRANT CONNECT, CREATE ON DATABASE xxx_db_jore4_main_name_xxx TO xxx_db_tiamat_username_xxx; - -GRANT CONNECT ON DATABASE xxx_db_jore4_main_name_xxx TO xxx_db_jore3importer_username_xxx; -GRANT CONNECT ON DATABASE xxx_db_jore4_main_name_xxx TO xxx_db_timetables_api_username_xxx; -GRANT CONNECT ON DATABASE xxx_db_jore4_main_name_xxx TO xxx_db_dss_username_xxx; - - -------------------------------------------------------- ------ Grant Network Schema Level Access Privileges ---- -------------------------------------------------------- - --- Grant required privileges to Hasura. -GRANT ALL ON SCHEMA network TO xxx_db_hasura_username_xxx; -GRANT SELECT ON ALL TABLES IN SCHEMA network TO xxx_db_hasura_username_xxx; -GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA network TO xxx_db_hasura_username_xxx; - --- Grant the JORE3-Importer role to access the schema and the objects and --- functions created by extensions. Other schema-specific privileges are granted --- in Hasura migrations. -GRANT USAGE ON SCHEMA network TO xxx_db_jore3importer_username_xxx; -GRANT SELECT ON ALL TABLES IN SCHEMA network TO xxx_db_jore3importer_username_xxx; -GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA network TO xxx_db_jore3importer_username_xxx; - - -------------------------------------------------------------- ------ Grant Stop Registry Schema Level Access Privileges ---- -------------------------------------------------------------- - --- Tiamat -GRANT ALL ON SCHEMA stopregistry TO xxx_db_tiamat_username_xxx; -GRANT SELECT ON ALL TABLES IN SCHEMA stopregistry TO xxx_db_tiamat_username_xxx; -GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA stopregistry TO xxx_db_tiamat_username_xxx; - --- Hasura -GRANT USAGE ON SCHEMA stopregistry TO xxx_db_hasura_username_xxx; -GRANT SELECT ON ALL TABLES IN SCHEMA stopregistry TO xxx_db_hasura_username_xxx; -ALTER DEFAULT PRIVILEGES FOR USER xxx_db_tiamat_username_xxx IN SCHEMA stopregistry GRANT SELECT ON TABLES TO xxx_db_hasura_username_xxx; - - --------------------------------------------------------- ------ Grant Topology Schema Level Access Privileges ---- --------------------------------------------------------- - --- See the beginning of the initial database migration in: --- https://github.com/entur/tiamat/blob/master/src/main/resources/db/migration/V1__Base_version.sql -GRANT USAGE ON SCHEMA topology TO xxx_db_hasura_username_xxx; -GRANT SELECT ON ALL TABLES IN SCHEMA topology TO xxx_db_hasura_username_xxx; -ALTER DEFAULT PRIVILEGES FOR USER xxx_db_tiamat_username_xxx IN SCHEMA topology GRANT SELECT ON TABLES TO xxx_db_hasura_username_xxx; - - ----------------------------------------------------------- ------ Grant Timetables Schema Level Access Privileges ---- ----------------------------------------------------------- - -GRANT ALL ON SCHEMA timetables TO xxx_db_hasura_username_xxx; -GRANT SELECT ON ALL TABLES IN SCHEMA timetables TO xxx_db_hasura_username_xxx; -GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA timetables TO xxx_db_hasura_username_xxx; - --- Allow the timetables-api role to use timetables schema -GRANT USAGE ON SCHEMA timetables TO xxx_db_timetables_api_username_xxx; -GRANT SELECT ON ALL TABLES IN SCHEMA timetables TO xxx_db_timetables_api_username_xxx; -GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA timetables TO xxx_db_timetables_api_username_xxx; - - --------------------------------------------------------- ------ Grant DSS View Schema Level Access Privileges ---- --------------------------------------------------------- - --- See the beginning of the initial database migration in: --- https://github.com/entur/tiamat/blob/master/src/main/resources/db/migration/V1__Base_version.sql -GRANT USAGE ON SCHEMA dssview TO xxx_db_hasura_username_xxx; -GRANT ALL ON SCHEMA dssview TO xxx_db_hasura_username_xxx; -GRANT ALL ON ALL TABLES IN SCHEMA dssview TO xxx_db_hasura_username_xxx; - -GRANT USAGE ON SCHEMA dssview TO xxx_db_dss_username_xxx; -GRANT SELECT ON ALL TABLES IN SCHEMA dssview TO xxx_db_dss_username_xxx; -ALTER DEFAULT PRIVILEGES FOR USER xxx_db_dss_username_xxx IN SCHEMA dssview GRANT SELECT ON TABLES TO xxx_db_dss_username_xxx; diff --git a/azuredbmock/migrations/10_postgis.sh b/azuredbmock/migrations/10_postgis.sh deleted file mode 100644 index ae56ee0..0000000 --- a/azuredbmock/migrations/10_postgis.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -set -e - -# Perform all actions as $POSTGRES_USER -export PGUSER="$POSTGRES_USER" - -# Create the 'template_postgis' template db -"${psql[@]}" <<- 'EOSQL' -CREATE DATABASE template_postgis IS_TEMPLATE true; -EOSQL - -# Load PostGIS into both template_database and $POSTGRES_DB -for DB in template_postgis "$POSTGRES_DB"; do - echo "Loading PostGIS extensions into $DB" - "${psql[@]}" --dbname="$DB" <<-'EOSQL' - CREATE EXTENSION IF NOT EXISTS postgis; - CREATE EXTENSION IF NOT EXISTS postgis_topology; - -- Reconnect to update pg_setting.resetval - -- See https://github.com/postgis/docker-postgis/issues/288 - \c - -- - DO $$ - DECLARE - postgis_major integer; - postgis_minor integer; - BEGIN - SELECT substring(postgis_lib_version() from '^([0-9]+)')::integer, - substring(postgis_lib_version() from '^[0-9]+\.([0-9]+)')::integer - INTO postgis_major, postgis_minor; - - -- Install the legacy tiger geocoder stack only for PostGIS versions before 3.7. - -- fuzzystrmatch is required by postgis_tiger_geocoder, which PostGIS 3.7 and later no longer provide. - IF postgis_major < 3 OR (postgis_major = 3 AND postgis_minor < 7) THEN - CREATE EXTENSION IF NOT EXISTS fuzzystrmatch; - CREATE EXTENSION IF NOT EXISTS postgis_tiger_geocoder; - END IF; - END - $$; -EOSQL -done diff --git a/azuredbmock/replace-placeholders-in-sql-schema-migrations.sh b/azuredbmock/replace-placeholders-in-sql-schema-migrations.sh deleted file mode 100755 index a6a23c3..0000000 --- a/azuredbmock/replace-placeholders-in-sql-schema-migrations.sh +++ /dev/null @@ -1,136 +0,0 @@ -#!/bin/sh -# -# Replace placeholders in SQL schema migrations. -# -# Docker or Kubernetes secrets contain the PostgreSQL users we need in the SQL -# schema migrations. As placeholders in the SQL schema migrations, we use -# mangled Docker secret filenames with bracket delimiters. This script replaces -# the placeholders with the contents of the Docker secrets at runtime. -# -# E.g. the username stored in a secret named -# "db-jore3importer-username" is used with the placeholder -# "xxx_db_jore3importer_username_xxx" within the SQL schema migration -# files. - -set -eu - -[ "$#" -ne 2 ] && { - echo 1>&2 "Usage: $(basename "$0") RUNTIME_SECRETS_DIR RUNTIME_MIGRATIONS_DIR" - exit 1 -} - -secrets_dir="$1" -migrations_dir="$2" - -# Choose delimiters that allow the placeholders to act as valid SQL identifiers -# for any static analysis purposes. -delimiter_alnum='xxx' -start_delimiter="${delimiter_alnum}_" -end_delimiter="_${delimiter_alnum}" -placeholder_characters='0-9A-Za-z_' - -cleanup() { - [ -z "${check_duplicate_list+dummy}" ] || rm -f "${check_duplicate_list}" - [ -z "${sed_script_file+dummy}" ] || rm -f "${sed_script_file}" -} -trap cleanup EXIT - -check_duplicate_list="$(mktemp)" -sed_script_file="$(mktemp)" - -# Create a script file for sed to replace the placeholders. -write_to_sed_script_file() { - secret_path="$1" - # As `basename -z` is not supported by the available shell, we cannot handle - # newlines in filenames with this approach. We do not translate the newline - # inserted by basename. - placeholder="$( - basename "${secret_path}" \ - | tr -C "${placeholder_characters}\n" '_' - )" - - # Check for the delimiter in the secret name. - if echo \ - "${placeholder}" \ - | grep \ - -q \ - -e "${delimiter_alnum}"; then - cat <&2 -The basename for secret -${secret_path} -should not contain the alphanumeric part of the delimiter: -${delimiter_alnum} -EOM - exit 4 - fi - - # Check for collisions of secrets. - if grep \ - -q \ - -x \ - -e "${placeholder}" \ - "${check_duplicate_list}"; then - cat <&2 -The basename for secret -${secret_path} -is not unique enough. The basenames of the secrets should differ by alphanumeric -characters. -EOM - exit 2 - fi - echo "${placeholder}" >>"${check_duplicate_list}" - - echo "s|${start_delimiter}${placeholder}${end_delimiter}|$(cat "${secret_path}")|g" \ - >>"${sed_script_file}" -} - -# Replace the placeholders with sed and check for leftover placeholders. -replace_and_check() { - migration_path="$1" - sed \ - -f "${sed_script_file}" \ - -i \ - "${migration_path}" - if leftover_placeholders="$( - grep \ - -o \ - -h \ - -E \ - -e "${start_delimiter}[${placeholder_characters}]+${end_delimiter}" \ - "${migration_path}" - )"; then - sorted_leftovers="$(echo "${leftover_placeholders}" | sort -u)" - cat <&2 -The migration file -${migration_path} -contains some placeholders for which there are no corresponding secrets. The -offending placeholders are: -${sorted_leftovers} -EOM - exit 3 - fi -} - -echo "Starting to replace placeholders in SQL schema migration files." - -# As `export -f` is not supported by the available shell, executing the shell -# functions with `find` does not work. - -# As `read -d ''` is not supported by the available shell, `find -print0` cannot -# be used here. - -# Loop over secrets. -find "${secrets_dir}" -type f \ - | LC_ALL=C sort \ - | while IFS= read -r file; do - write_to_sed_script_file "${file}" - done - -# Loop over SQL schema migrations. -find "${migrations_dir}" -type f -name '*.sql' \ - | LC_ALL=C sort \ - | while IFS= read -r file; do - replace_and_check "${file}" - done - -echo "Replacing placeholders done." diff --git a/mapmatching/Dockerfile b/mapmatching/Dockerfile deleted file mode 100644 index b2bc7f2..0000000 --- a/mapmatching/Dockerfile +++ /dev/null @@ -1,87 +0,0 @@ -## Below an inlined version of the PostGIS Dockerfile. PostGIS does not -## relase ARM64 versions of the image at the moment (they did in the past). -## And thus, we have to build the ARM version our selves. - -FROM docker.io/postgres:18-trixie AS postgis - -ENV POSTGIS_MAJOR=3 -ENV POSTGIS_VERSION=3.6.4+dfsg-2.pgdg13+1 - -RUN apt-get update \ - && apt-cache showpkg postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR \ - && apt-get install -y --no-install-recommends \ - # ca-certificates: for accessing remote raster files; - # fix: https://github.com/postgis/docker-postgis/issues/307 - ca-certificates \ - \ - postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR=$POSTGIS_VERSION \ - postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts \ - && rm -rf /var/lib/apt/lists/* - -RUN mkdir -p /docker-entrypoint-initdb.d - - -# Inlined version of the pgrouting Dockerfile. Needed to add support for the ARM64 version. -FROM postgis AS pgrouting - -ENV PGROUTING_VERSION=3.8.0 -ENV PGROUTING_SHA256=b8a5f0472934fdf7cda3fb4754d01945378d920cdaddc01f378617ddbb9c447f -ENV LIBBOOST_VERSION=1.83.0 - -RUN set -ex \ - && apt update \ - && apt install -y \ - libboost-atomic${LIBBOOST_VERSION} \ - libboost-chrono${LIBBOOST_VERSION} \ - libboost-graph${LIBBOOST_VERSION} \ - libboost-date-time${LIBBOOST_VERSION} \ - libboost-program-options${LIBBOOST_VERSION} \ - libboost-system${LIBBOOST_VERSION} \ - libboost-thread${LIBBOOST_VERSION} \ - && apt install -y \ - build-essential \ - cmake \ - curl \ - libboost-graph-dev \ - libpq-dev \ - postgresql-server-dev-${PG_MAJOR} \ - && curl -Lo pgrouting.tar.gz "https://github.com/pgRouting/pgrouting/archive/v${PGROUTING_VERSION}.tar.gz" \ - && echo "$PGROUTING_SHA256 *pgrouting.tar.gz" | sha256sum -c - \ - && mkdir -p /usr/src/pgrouting \ - && tar \ - --extract \ - --file pgrouting.tar.gz \ - --directory /usr/src/pgrouting \ - --strip-components 1 \ - && rm pgrouting.tar.gz \ - && cd /usr/src/pgrouting \ - && mkdir build \ - && cd build \ - && cmake .. \ - && make -j $(nproc --all) \ - && make install \ - && cd / \ - && rm -rf /usr/src/pgrouting \ - && apt-mark manual postgresql-18 \ - && apt purge -y --autoremove \ - build-essential \ - cmake \ - libpq-dev \ - libboost-graph-dev \ - postgresql-server-dev-${PG_MAJOR} \ - && rm -rf /var/lib/apt/lists/* - - -FROM pgrouting - -# download script for reading docker secrets -ADD --chmod=755 https://raw.githubusercontent.com/HSLdevcom/jore4-tools/main/docker/read-secrets.sh /tmp/read-secrets.sh - -COPY --chmod=755 entrypoint.sh /usr/local/bin/ - -ENTRYPOINT ["entrypoint.sh"] -CMD ["postgres"] - -# have to source the secrets here as well, otherwise the POSTGRES_USER and/or POSTGRES_DB variables might not exist -HEALTHCHECK --interval=5s --timeout=5s --start-period=10s --retries=20 \ - CMD bash -c "source /tmp/read-secrets.sh && pg_isready -h localhost -U \${POSTGRES_USER} -d \${POSTGRES_DB}" diff --git a/mapmatching/entrypoint.sh b/mapmatching/entrypoint.sh deleted file mode 100755 index eab2618..0000000 --- a/mapmatching/entrypoint.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -set -Eeuo pipefail - -# Read the secrets to environment variables. -source /tmp/read-secrets.sh - -# As the database dump does not initialise extensions, create them manually. -cat < /docker-entrypoint-initdb.d/01-initialize.sql -CREATE EXTENSION postgis; -CREATE EXTENSION pgrouting; -EOT - -# Download the routing/map-matching database dump based on Digiroad data. -echo 'Downloading database dump...' -curl -o /docker-entrypoint-initdb.d/02-digiroad_r_routing.sql \ - ${DIGIROAD_ROUTING_DUMP_URL} - -# Include the routing schema in search path. -cat < /docker-entrypoint-initdb.d/03-post-initialize.sql -ALTER DATABASE ${POSTGRES_DB} SET search_path = routing, public; -EOT - -# Call the original entrypoint to continue startup. -echo 'Start postgres' -exec /usr/local/bin/docker-entrypoint.sh "$@" From 0060b550c79ec94d1c8c5d0681e60af466e1de46 Mon Sep 17 00:00:00 2001 From: Jesse Jaara Date: Fri, 9 Oct 2026 14:52:58 +0300 Subject: [PATCH 3/3] WIP --- .github/workflows/cd.yml | 29 +++- azuredbmock/Dockerfile | 11 ++ azuredbmock/entrypoint.sh | 21 +++ azuredbmock/migrations/01-create-roles.sql | 15 ++ .../migrations/02-create-network-database.sql | 33 +++++ .../migrations/03-create-auth-database.sql | 12 ++ .../04-create-jore3importer-database.sql | 25 ++++ .../05-create-timetables-database.sql | 36 +++++ .../06-create-stopregistry-database.sql | 39 +++++ .../07-create-jore4-main-database.sql | 124 ++++++++++++++++ ...e-placeholders-in-sql-schema-migrations.sh | 136 ++++++++++++++++++ docker-compose.yml | 13 +- jore4-postgres/Dockerfile | 90 ++++++++++++ mapmatching/Dockerfile | 11 ++ mapmatching/azuredbmock-entrypoint.sh | 21 +++ mapmatching/entrypoint.sh | 10 ++ .../azuredbmock/01-create-roles.sql | 15 ++ .../02-create-network-database.sql | 33 +++++ .../azuredbmock/03-create-auth-database.sql | 12 ++ .../04-create-jore3importer-database.sql | 25 ++++ .../05-create-timetables-database.sql | 36 +++++ .../06-create-stopregistry-database.sql | 39 +++++ .../07-create-jore4-main-database.sql | 124 ++++++++++++++++ .../migrations/mapmatching/01-initialize.sql | 2 + .../mapmatching/02-digiroad_r_routing.md | 1 + .../mapmatching/03-post-initialize.sql | 1 + ...e-placeholders-in-sql-schema-migrations.sh | 136 ++++++++++++++++++ 27 files changed, 1046 insertions(+), 4 deletions(-) create mode 100644 azuredbmock/Dockerfile create mode 100755 azuredbmock/entrypoint.sh create mode 100644 azuredbmock/migrations/01-create-roles.sql create mode 100644 azuredbmock/migrations/02-create-network-database.sql create mode 100644 azuredbmock/migrations/03-create-auth-database.sql create mode 100644 azuredbmock/migrations/04-create-jore3importer-database.sql create mode 100644 azuredbmock/migrations/05-create-timetables-database.sql create mode 100644 azuredbmock/migrations/06-create-stopregistry-database.sql create mode 100644 azuredbmock/migrations/07-create-jore4-main-database.sql create mode 100755 azuredbmock/replace-placeholders-in-sql-schema-migrations.sh create mode 100644 jore4-postgres/Dockerfile create mode 100644 mapmatching/Dockerfile create mode 100755 mapmatching/azuredbmock-entrypoint.sh create mode 100755 mapmatching/entrypoint.sh create mode 100644 mapmatching/migrations/azuredbmock/01-create-roles.sql create mode 100644 mapmatching/migrations/azuredbmock/02-create-network-database.sql create mode 100644 mapmatching/migrations/azuredbmock/03-create-auth-database.sql create mode 100644 mapmatching/migrations/azuredbmock/04-create-jore3importer-database.sql create mode 100644 mapmatching/migrations/azuredbmock/05-create-timetables-database.sql create mode 100644 mapmatching/migrations/azuredbmock/06-create-stopregistry-database.sql create mode 100644 mapmatching/migrations/azuredbmock/07-create-jore4-main-database.sql create mode 100644 mapmatching/migrations/mapmatching/01-initialize.sql create mode 100644 mapmatching/migrations/mapmatching/02-digiroad_r_routing.md create mode 100644 mapmatching/migrations/mapmatching/03-post-initialize.sql create mode 100755 mapmatching/replace-placeholders-in-sql-schema-migrations.sh diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 8acb8a5..9273618 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -11,16 +11,36 @@ permissions: contents: read jobs: + publish_jore_postgres_docker_image_to_acr: + name: Publish Generic Jore4 PostgreSQL Docker image to ACR + permissions: + id-token: write + contents: read + uses: HSLdevcom/jore4-tools/.github/workflows/shared-build-and-publish-docker-image.yml@native-arm-build + with: + docker_image_name: jore4-postgres + build_arm64_image: true + context: ${{ github.server_url }}/${{ github.repository }}.git#${{ github.ref }}:./jore4-postgres + secrets: + azure_client_id: ${{ secrets.AZURE_CLIENT_ID }} + azure_tenant_id: ${{ secrets.AZURE_TENANT_ID }} + azure_subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + publish_mapmatching_docker_image_to_acr: name: Publish Map Matching Docker image to ACR + needs: + - publish_jore_postgres_docker_image_to_acr permissions: id-token: write contents: read - uses: HSLdevcom/jore4-tools/.github/workflows/shared-build-and-publish-docker-image.yml@main + uses: HSLdevcom/jore4-tools/.github/workflows/shared-build-and-publish-docker-image.yml@native-arm-build with: docker_image_name: jore4-postgres-mapmatching build_arm64_image: true context: ${{ github.server_url }}/${{ github.repository }}.git#${{ github.ref }}:./mapmatching + build_args: | + BASE_IMAGE_LOCATION=crjore4prod001.azurecr.io/ + BASE_IMAGE_VERSION=${{ needs.publish_jore_postgres_docker_image_to_acr.outputs.docker_image }} secrets: azure_client_id: ${{ secrets.AZURE_CLIENT_ID }} azure_tenant_id: ${{ secrets.AZURE_TENANT_ID }} @@ -28,14 +48,19 @@ jobs: publish_azuredbmock_docker_image_to_acr: name: Publish Azure DB Mock Docker image to ACR + needs: + - publish_jore_postgres_docker_image_to_acr permissions: id-token: write contents: read - uses: HSLdevcom/jore4-tools/.github/workflows/shared-build-and-publish-docker-image.yml@main + uses: HSLdevcom/jore4-tools/.github/workflows/shared-build-and-publish-docker-image.yml@native-arm-build with: docker_image_name: jore4-postgres-azuredbmock build_arm64_image: true context: ${{ github.server_url }}/${{ github.repository }}.git#${{ github.ref }}:./azuredbmock + build_args: | + BASE_IMAGE_LOCATION=crjore4prod001.azurecr.io/ + BASE_IMAGE_VERSION=${{ needs.publish_jore_postgres_docker_image_to_acr.outputs.docker_image }} secrets: azure_client_id: ${{ secrets.AZURE_CLIENT_ID }} azure_tenant_id: ${{ secrets.AZURE_TENANT_ID }} diff --git a/azuredbmock/Dockerfile b/azuredbmock/Dockerfile new file mode 100644 index 0000000..4ee26e8 --- /dev/null +++ b/azuredbmock/Dockerfile @@ -0,0 +1,11 @@ +# The primary Database image +ARG BASE_IMAGE_LOCATION='' +ARG BASE_IMAGE_VERSION=latest +FROM ${BASE_IMAGE_LOCATION}jore4-postgres:${BASE_IMAGE_VERSION} AS jore4-postgres-azuredbmock + +COPY --chmod=644 migrations/* /jore4/migrations/ +COPY --chmod=755 replace-placeholders-in-sql-schema-migrations.sh /jore4/scripts/replace-placeholders-in-sql-schema-migrations.sh +COPY --chmod=755 entrypoint.sh /jore4/scripts/ + +ENTRYPOINT ["/jore4/scripts/entrypoint.sh"] +CMD ["postgres"] diff --git a/azuredbmock/entrypoint.sh b/azuredbmock/entrypoint.sh new file mode 100755 index 0000000..dcc56d5 --- /dev/null +++ b/azuredbmock/entrypoint.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -Eeuo pipefail + +# read the secrets to environment variables +source /jore4/scripts/read-secrets.sh + +# Copy the migrations scripts to the docker-entrypoint-initdb.d folder so that the postgres entrypoint +# executes them (together with the other migrations that might have been mapped as a volume) +mkdir -p /docker-entrypoint-initdb.d +cp /jore4/migrations/* /docker-entrypoint-initdb.d/ + +# replace placeholders with secrets within the all migration SQL scripts +SECRET_STORE_BASE_PATH="${SECRET_STORE_BASE_PATH:-/run/secrets}" +REPLACE_PLACEHOLDERS_SCRIPT='/jore4/scripts/replace-placeholders-in-sql-schema-migrations.sh' +MIGRATIONS_DIR="/docker-entrypoint-initdb.d/" +"${REPLACE_PLACEHOLDERS_SCRIPT}" "${SECRET_STORE_BASE_PATH}" "${MIGRATIONS_DIR}" + +# call the original entrypoint to continue execution +echo 'Start postgres' +exec /usr/local/bin/docker-entrypoint.sh "$@" diff --git a/azuredbmock/migrations/01-create-roles.sql b/azuredbmock/migrations/01-create-roles.sql new file mode 100644 index 0000000..9c9e5e1 --- /dev/null +++ b/azuredbmock/migrations/01-create-roles.sql @@ -0,0 +1,15 @@ +-- These database roles are also created in the azure-infra-jore4aks (Azure +-- DevOps) repository. +CREATE USER xxx_db_auth_username_xxx PASSWORD 'xxx_db_auth_password_xxx'; +CREATE USER xxx_db_jore3importer_username_xxx PASSWORD 'xxx_db_jore3importer_password_xxx'; +CREATE USER xxx_db_hasura_username_xxx PASSWORD 'xxx_db_hasura_password_xxx'; +CREATE USER xxx_db_tiamat_username_xxx PASSWORD 'xxx_db_tiamat_password_xxx'; +CREATE USER xxx_db_timetables_api_username_xxx PASSWORD 'xxx_db_timetables_api_password_xxx'; +CREATE USER xxx_db_dss_username_xxx PASSWORD 'xxx_db_dss_password_xxx'; + +-- Make the hasura role a member of jore3importer role because both roles must +-- have ownership of tables and sequences in the default database (network and +-- routes) since both are responsible for populating and truncating tables in +-- the aforementioned database. In particular, sequence reset requires an +-- ownership and cannot be granted as a privilege. +GRANT xxx_db_jore3importer_username_xxx TO xxx_db_hasura_username_xxx; diff --git a/azuredbmock/migrations/02-create-network-database.sql b/azuredbmock/migrations/02-create-network-database.sql new file mode 100644 index 0000000..f1f3061 --- /dev/null +++ b/azuredbmock/migrations/02-create-network-database.sql @@ -0,0 +1,33 @@ +-- Make the JORE4 admin role the owner of the public schema. +ALTER SCHEMA public OWNER TO CURRENT_USER; + +-- Create the extensions used, see https://hasura.io/docs/latest/graphql/core/deployment/postgres-requirements.html +-- Create the extensions in the public schema, since we'd need to give additional privileges ("use schema") to any +-- user who wishes to use these in the future. Also, Hasura would require additional setup to be able to use the +-- extensions from another schema. +CREATE EXTENSION IF NOT EXISTS btree_gist WITH SCHEMA public; +CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public; +CREATE EXTENSION IF NOT EXISTS postgis WITH SCHEMA public; + +-- Allow Hasura to connect and create new schemas. +GRANT CONNECT, CREATE ON DATABASE xxx_db_hasura_name_xxx TO xxx_db_hasura_username_xxx; + +-- Grant required privileges in the public schema to the Hasura user. +GRANT ALL ON SCHEMA public TO xxx_db_hasura_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_hasura_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_hasura_username_xxx; + +-- Grant select permissions on information_schema and pg_catalog to the Hasura +-- user. +GRANT SELECT ON ALL TABLES IN SCHEMA information_schema TO xxx_db_hasura_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA pg_catalog TO xxx_db_hasura_username_xxx; + +-- Allow the JORE3-Importer role to connect to the network database. +GRANT CONNECT ON DATABASE xxx_db_hasura_name_xxx TO xxx_db_jore3importer_username_xxx; + +-- Grant the JORE3-Importer role to access the public schema and the objects and +-- functions created by extensions. Other schema-specific privileges are granted +-- in Hasura migrations. +GRANT USAGE ON SCHEMA public TO xxx_db_jore3importer_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_jore3importer_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_jore3importer_username_xxx; diff --git a/azuredbmock/migrations/03-create-auth-database.sql b/azuredbmock/migrations/03-create-auth-database.sql new file mode 100644 index 0000000..07c86f0 --- /dev/null +++ b/azuredbmock/migrations/03-create-auth-database.sql @@ -0,0 +1,12 @@ +CREATE DATABASE xxx_db_auth_name_xxx; + +-- Allow the auth role to connect and create new schemas. +GRANT CONNECT, CREATE ON DATABASE xxx_db_auth_name_xxx TO xxx_db_auth_username_xxx; + +\connect xxx_db_auth_name_xxx; + +-- Make the JORE4 admin role the owner of the public schema. +ALTER SCHEMA public OWNER TO CURRENT_USER; + +-- Grant full schema access to the public schema to the auth role. +GRANT ALL ON SCHEMA public TO xxx_db_auth_username_xxx; diff --git a/azuredbmock/migrations/04-create-jore3importer-database.sql b/azuredbmock/migrations/04-create-jore3importer-database.sql new file mode 100644 index 0000000..40f7ad6 --- /dev/null +++ b/azuredbmock/migrations/04-create-jore3importer-database.sql @@ -0,0 +1,25 @@ +CREATE DATABASE xxx_db_jore3importer_name_xxx; + +-- Allow the jore3importer role to connect and create new schemas. +GRANT CONNECT, CREATE ON DATABASE xxx_db_jore3importer_name_xxx TO xxx_db_jore3importer_username_xxx; + +\connect xxx_db_jore3importer_name_xxx; + +-- Make the JORE4 admin role the owner of the public schema. +ALTER SCHEMA public OWNER TO CURRENT_USER; + +-- Create the extensions that JORE3-Importer needs. In PostgreSQL v15 server, +-- an ordinary user (without admin roles) may not be able to create extensions. +CREATE EXTENSION IF NOT EXISTS btree_gist WITH SCHEMA public; +CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public; + +-- Grant privileges in the public schema to the jore3importer role. +GRANT USAGE ON SCHEMA public TO xxx_db_jore3importer_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_jore3importer_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_jore3importer_username_xxx; + +-- Grant permission on the pg_catalog schema to conditionally create an +-- extension if the extension is not already created. This must be granted so +-- that the JORE3-Importer role can conditionally create the extension depending +-- on which database is involved (there are several in the importer's test setup). +GRANT SELECT ON TABLE pg_catalog.pg_extension TO xxx_db_jore3importer_username_xxx; diff --git a/azuredbmock/migrations/05-create-timetables-database.sql b/azuredbmock/migrations/05-create-timetables-database.sql new file mode 100644 index 0000000..8648b48 --- /dev/null +++ b/azuredbmock/migrations/05-create-timetables-database.sql @@ -0,0 +1,36 @@ +CREATE DATABASE xxx_db_timetables_name_xxx; + +-- Allow Hasura to connect and create new schemas. +GRANT CONNECT, CREATE ON DATABASE xxx_db_timetables_name_xxx TO xxx_db_hasura_username_xxx; + +-- Interval outputs by default are using the sql format ('3 4:05:06'). Here we +-- are switching to ISO 8601 format ('P3DT4H5M6S'). +ALTER DATABASE xxx_db_timetables_name_xxx SET intervalstyle = 'iso_8601'; + +-- Switch database context to be able to add extensions there. +\connect xxx_db_timetables_name_xxx; + +-- Make the JORE4 admin role the owner of the public schema. +ALTER SCHEMA public OWNER TO CURRENT_USER; + +CREATE EXTENSION IF NOT EXISTS btree_gist WITH SCHEMA public; +CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public; + +-- Grant required privileges in the public schema to Hasura. +GRANT ALL ON SCHEMA public TO xxx_db_hasura_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_hasura_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_hasura_username_xxx; + +-- Grant select permissions on information_schema and pg_catalog to Hasura. +GRANT SELECT ON ALL TABLES IN SCHEMA information_schema TO xxx_db_hasura_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA pg_catalog TO xxx_db_hasura_username_xxx; + +-- Allow the timetables-api role to connect to the timetables database. +GRANT CONNECT ON DATABASE xxx_db_timetables_name_xxx TO xxx_db_timetables_api_username_xxx; + +-- Grant the timetables-api role to access the public schema and the objects and +-- functions created by extensions. Other schema-specific privileges are granted +-- in Hasura migrations. +GRANT USAGE ON SCHEMA public TO xxx_db_timetables_api_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_timetables_api_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_timetables_api_username_xxx; diff --git a/azuredbmock/migrations/06-create-stopregistry-database.sql b/azuredbmock/migrations/06-create-stopregistry-database.sql new file mode 100644 index 0000000..6ea599f --- /dev/null +++ b/azuredbmock/migrations/06-create-stopregistry-database.sql @@ -0,0 +1,39 @@ +CREATE DATABASE xxx_db_tiamat_name_xxx; + +-- Allow Tiamat to connect and create new schemas. +GRANT CONNECT, CREATE ON DATABASE xxx_db_tiamat_name_xxx TO xxx_db_tiamat_username_xxx; + +-- Switch database context to initialise it to the state where Tiamat can use +-- it. +\connect xxx_db_tiamat_name_xxx; + +-- Make the JORE4 admin role the owner of the public schema. +ALTER SCHEMA public OWNER TO CURRENT_USER; + +CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public; +CREATE EXTENSION IF NOT EXISTS postgis WITH SCHEMA public; + +-- Grant required privileges in the public schema to Tiamat. +GRANT ALL ON SCHEMA public TO xxx_db_tiamat_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_tiamat_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_tiamat_username_xxx; + +-- Create "topology" schema and install the "postgis_topology" extension to it. +-- The Tiamat role needs ownership to the schema and its tables. +CREATE SCHEMA IF NOT EXISTS topology; +CREATE EXTENSION IF NOT EXISTS postgis_topology WITH SCHEMA topology; +ALTER SCHEMA topology OWNER TO xxx_db_tiamat_username_xxx; +-- The "postgis_topology" extension creates two tables. +ALTER TABLE topology.layer OWNER TO xxx_db_tiamat_username_xxx; +ALTER TABLE topology.topology OWNER TO xxx_db_tiamat_username_xxx; + +-- Grant Hasura read permissions to the stop registry database. +GRANT CONNECT ON DATABASE xxx_db_tiamat_name_xxx TO xxx_db_hasura_username_xxx; + +GRANT USAGE ON SCHEMA public TO xxx_db_hasura_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_hasura_username_xxx; +ALTER DEFAULT PRIVILEGES FOR USER xxx_db_tiamat_username_xxx IN SCHEMA public GRANT SELECT ON TABLES TO xxx_db_hasura_username_xxx; + +GRANT USAGE ON SCHEMA topology TO xxx_db_hasura_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA topology TO xxx_db_hasura_username_xxx; +ALTER DEFAULT PRIVILEGES FOR USER xxx_db_tiamat_username_xxx IN SCHEMA topology GRANT SELECT ON TABLES TO xxx_db_hasura_username_xxx; diff --git a/azuredbmock/migrations/07-create-jore4-main-database.sql b/azuredbmock/migrations/07-create-jore4-main-database.sql new file mode 100644 index 0000000..2d268ac --- /dev/null +++ b/azuredbmock/migrations/07-create-jore4-main-database.sql @@ -0,0 +1,124 @@ +-- This migration replicates initialization of the JORE4 main database in script scripts/create_db_roles_and_extensions.sh +-- in azure-infra-jore4aks repository + +CREATE DATABASE xxx_db_jore4_main_name_xxx; + +\connect xxx_db_jore4_main_name_xxx; + + +-------------------------- +----- Create Schemas ----- +-------------------------- + +CREATE SCHEMA IF NOT EXISTS network AUTHORIZATION xxx_db_hasura_username_xxx; +CREATE SCHEMA IF NOT EXISTS stopregistry AUTHORIZATION xxx_db_tiamat_username_xxx; +CREATE SCHEMA IF NOT EXISTS timetables AUTHORIZATION xxx_db_hasura_username_xxx; +CREATE SCHEMA IF NOT EXISTS hdb_catalog AUTHORIZATION xxx_db_hasura_username_xxx; +CREATE SCHEMA IF NOT EXISTS dssview AUTHORIZATION xxx_db_hasura_username_xxx; + +CREATE SCHEMA IF NOT EXISTS topology AUTHORIZATION xxx_db_tiamat_username_xxx; + + +----------------------------- +----- Create Extensions ----- +----------------------------- + +-- Extensions are created in public schema so the extensions are found from the search path +CREATE EXTENSION IF NOT EXISTS btree_gist WITH SCHEMA public; +CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public; +CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public; +CREATE EXTENSION IF NOT EXISTS postgis WITH SCHEMA public; +CREATE EXTENSION IF NOT EXISTS postgis_topology WITH SCHEMA topology; + + +--------------------------------------- +----- Configure Database Settings ----- +--------------------------------------- + +-- Interval outputs by default are using the SQL format ('3 4:05:06'). Here we +-- switch to ISO 8601 format ('P3DT4H5M6S'). +ALTER DATABASE xxx_db_jore4_main_name_xxx SET intervalstyle = 'iso_8601'; + + +-------------------------------------------------- +----- Grant Database Level Access Privileges ----- +-------------------------------------------------- + +GRANT CONNECT, CREATE ON DATABASE xxx_db_jore4_main_name_xxx TO xxx_db_hasura_username_xxx; +GRANT CONNECT, CREATE ON DATABASE xxx_db_jore4_main_name_xxx TO xxx_db_tiamat_username_xxx; + +GRANT CONNECT ON DATABASE xxx_db_jore4_main_name_xxx TO xxx_db_jore3importer_username_xxx; +GRANT CONNECT ON DATABASE xxx_db_jore4_main_name_xxx TO xxx_db_timetables_api_username_xxx; +GRANT CONNECT ON DATABASE xxx_db_jore4_main_name_xxx TO xxx_db_dss_username_xxx; + + +------------------------------------------------------- +----- Grant Network Schema Level Access Privileges ---- +------------------------------------------------------- + +-- Grant required privileges to Hasura. +GRANT ALL ON SCHEMA network TO xxx_db_hasura_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA network TO xxx_db_hasura_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA network TO xxx_db_hasura_username_xxx; + +-- Grant the JORE3-Importer role to access the schema and the objects and +-- functions created by extensions. Other schema-specific privileges are granted +-- in Hasura migrations. +GRANT USAGE ON SCHEMA network TO xxx_db_jore3importer_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA network TO xxx_db_jore3importer_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA network TO xxx_db_jore3importer_username_xxx; + + +------------------------------------------------------------- +----- Grant Stop Registry Schema Level Access Privileges ---- +------------------------------------------------------------- + +-- Tiamat +GRANT ALL ON SCHEMA stopregistry TO xxx_db_tiamat_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA stopregistry TO xxx_db_tiamat_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA stopregistry TO xxx_db_tiamat_username_xxx; + +-- Hasura +GRANT USAGE ON SCHEMA stopregistry TO xxx_db_hasura_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA stopregistry TO xxx_db_hasura_username_xxx; +ALTER DEFAULT PRIVILEGES FOR USER xxx_db_tiamat_username_xxx IN SCHEMA stopregistry GRANT SELECT ON TABLES TO xxx_db_hasura_username_xxx; + + +-------------------------------------------------------- +----- Grant Topology Schema Level Access Privileges ---- +-------------------------------------------------------- + +-- See the beginning of the initial database migration in: +-- https://github.com/entur/tiamat/blob/master/src/main/resources/db/migration/V1__Base_version.sql +GRANT USAGE ON SCHEMA topology TO xxx_db_hasura_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA topology TO xxx_db_hasura_username_xxx; +ALTER DEFAULT PRIVILEGES FOR USER xxx_db_tiamat_username_xxx IN SCHEMA topology GRANT SELECT ON TABLES TO xxx_db_hasura_username_xxx; + + +---------------------------------------------------------- +----- Grant Timetables Schema Level Access Privileges ---- +---------------------------------------------------------- + +GRANT ALL ON SCHEMA timetables TO xxx_db_hasura_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA timetables TO xxx_db_hasura_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA timetables TO xxx_db_hasura_username_xxx; + +-- Allow the timetables-api role to use timetables schema +GRANT USAGE ON SCHEMA timetables TO xxx_db_timetables_api_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA timetables TO xxx_db_timetables_api_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA timetables TO xxx_db_timetables_api_username_xxx; + + +-------------------------------------------------------- +----- Grant DSS View Schema Level Access Privileges ---- +-------------------------------------------------------- + +-- See the beginning of the initial database migration in: +-- https://github.com/entur/tiamat/blob/master/src/main/resources/db/migration/V1__Base_version.sql +GRANT USAGE ON SCHEMA dssview TO xxx_db_hasura_username_xxx; +GRANT ALL ON SCHEMA dssview TO xxx_db_hasura_username_xxx; +GRANT ALL ON ALL TABLES IN SCHEMA dssview TO xxx_db_hasura_username_xxx; + +GRANT USAGE ON SCHEMA dssview TO xxx_db_dss_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA dssview TO xxx_db_dss_username_xxx; +ALTER DEFAULT PRIVILEGES FOR USER xxx_db_dss_username_xxx IN SCHEMA dssview GRANT SELECT ON TABLES TO xxx_db_dss_username_xxx; diff --git a/azuredbmock/replace-placeholders-in-sql-schema-migrations.sh b/azuredbmock/replace-placeholders-in-sql-schema-migrations.sh new file mode 100755 index 0000000..a6a23c3 --- /dev/null +++ b/azuredbmock/replace-placeholders-in-sql-schema-migrations.sh @@ -0,0 +1,136 @@ +#!/bin/sh +# +# Replace placeholders in SQL schema migrations. +# +# Docker or Kubernetes secrets contain the PostgreSQL users we need in the SQL +# schema migrations. As placeholders in the SQL schema migrations, we use +# mangled Docker secret filenames with bracket delimiters. This script replaces +# the placeholders with the contents of the Docker secrets at runtime. +# +# E.g. the username stored in a secret named +# "db-jore3importer-username" is used with the placeholder +# "xxx_db_jore3importer_username_xxx" within the SQL schema migration +# files. + +set -eu + +[ "$#" -ne 2 ] && { + echo 1>&2 "Usage: $(basename "$0") RUNTIME_SECRETS_DIR RUNTIME_MIGRATIONS_DIR" + exit 1 +} + +secrets_dir="$1" +migrations_dir="$2" + +# Choose delimiters that allow the placeholders to act as valid SQL identifiers +# for any static analysis purposes. +delimiter_alnum='xxx' +start_delimiter="${delimiter_alnum}_" +end_delimiter="_${delimiter_alnum}" +placeholder_characters='0-9A-Za-z_' + +cleanup() { + [ -z "${check_duplicate_list+dummy}" ] || rm -f "${check_duplicate_list}" + [ -z "${sed_script_file+dummy}" ] || rm -f "${sed_script_file}" +} +trap cleanup EXIT + +check_duplicate_list="$(mktemp)" +sed_script_file="$(mktemp)" + +# Create a script file for sed to replace the placeholders. +write_to_sed_script_file() { + secret_path="$1" + # As `basename -z` is not supported by the available shell, we cannot handle + # newlines in filenames with this approach. We do not translate the newline + # inserted by basename. + placeholder="$( + basename "${secret_path}" \ + | tr -C "${placeholder_characters}\n" '_' + )" + + # Check for the delimiter in the secret name. + if echo \ + "${placeholder}" \ + | grep \ + -q \ + -e "${delimiter_alnum}"; then + cat <&2 +The basename for secret +${secret_path} +should not contain the alphanumeric part of the delimiter: +${delimiter_alnum} +EOM + exit 4 + fi + + # Check for collisions of secrets. + if grep \ + -q \ + -x \ + -e "${placeholder}" \ + "${check_duplicate_list}"; then + cat <&2 +The basename for secret +${secret_path} +is not unique enough. The basenames of the secrets should differ by alphanumeric +characters. +EOM + exit 2 + fi + echo "${placeholder}" >>"${check_duplicate_list}" + + echo "s|${start_delimiter}${placeholder}${end_delimiter}|$(cat "${secret_path}")|g" \ + >>"${sed_script_file}" +} + +# Replace the placeholders with sed and check for leftover placeholders. +replace_and_check() { + migration_path="$1" + sed \ + -f "${sed_script_file}" \ + -i \ + "${migration_path}" + if leftover_placeholders="$( + grep \ + -o \ + -h \ + -E \ + -e "${start_delimiter}[${placeholder_characters}]+${end_delimiter}" \ + "${migration_path}" + )"; then + sorted_leftovers="$(echo "${leftover_placeholders}" | sort -u)" + cat <&2 +The migration file +${migration_path} +contains some placeholders for which there are no corresponding secrets. The +offending placeholders are: +${sorted_leftovers} +EOM + exit 3 + fi +} + +echo "Starting to replace placeholders in SQL schema migration files." + +# As `export -f` is not supported by the available shell, executing the shell +# functions with `find` does not work. + +# As `read -d ''` is not supported by the available shell, `find -print0` cannot +# be used here. + +# Loop over secrets. +find "${secrets_dir}" -type f \ + | LC_ALL=C sort \ + | while IFS= read -r file; do + write_to_sed_script_file "${file}" + done + +# Loop over SQL schema migrations. +find "${migrations_dir}" -type f -name '*.sql' \ + | LC_ALL=C sort \ + | while IFS= read -r file; do + replace_and_check "${file}" + done + +echo "Replacing placeholders done." diff --git a/docker-compose.yml b/docker-compose.yml index c9f2e35..e749ee3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,15 @@ --- services: + jore4-postgres: + image: jore4-postgres + build: + context: ./jore4-postgres + mapmatchingdb: build: - context: "./mapmatching" + context: ./mapmatching + additional_contexts: + 'jore4-postgres:latest': service:jore4-postgres container_name: mapmatchingdb restart: "unless-stopped" networks: @@ -20,7 +27,9 @@ services: azuredbmock: build: - context: "./azuredbmock" + context: ./azuredbmock + additional_contexts: + 'jore4-postgres:latest': service:jore4-postgres container_name: azuredbmock restart: "unless-stopped" networks: diff --git a/jore4-postgres/Dockerfile b/jore4-postgres/Dockerfile new file mode 100644 index 0000000..aadaa00 --- /dev/null +++ b/jore4-postgres/Dockerfile @@ -0,0 +1,90 @@ +## Below an inlined version of the PostGIS Dockerfile. PostGIS does not +## relase ARM64 versions of the image at the moment (they did in the past). +## And thus, we have to build the ARM version our selves. + +FROM docker.io/postgres:18-trixie AS postgis + +ENV POSTGIS_MAJOR=3 +ENV POSTGIS_VERSION=3.6.4+dfsg-2.pgdg13+1 + +RUN apt-get update \ + && apt-cache showpkg postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR \ + && apt-get install -y --no-install-recommends \ + # ca-certificates: for accessing remote raster files; + # fix: https://github.com/postgis/docker-postgis/issues/307 + ca-certificates \ + \ + postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR=$POSTGIS_VERSION \ + postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts \ + && rm -rf /var/lib/apt/lists/* \ + && mkdir -p /docker-entrypoint-initdb.d + + +# Inlined version of the pgrouting Dockerfile. Needed to add support for the ARM64 version. +FROM postgis AS pgrouting + +ENV PGROUTING_VERSION=3.8.0 +ENV PGROUTING_SHA256=b8a5f0472934fdf7cda3fb4754d01945378d920cdaddc01f378617ddbb9c447f +ENV LIBBOOST_VERSION=1.83.0 + +RUN set -ex \ + && apt update \ + && apt install -y \ + libboost-atomic${LIBBOOST_VERSION} \ + libboost-chrono${LIBBOOST_VERSION} \ + libboost-graph${LIBBOOST_VERSION} \ + libboost-date-time${LIBBOOST_VERSION} \ + libboost-program-options${LIBBOOST_VERSION} \ + libboost-system${LIBBOOST_VERSION} \ + libboost-thread${LIBBOOST_VERSION} \ + && apt install -y \ + build-essential \ + cmake \ + curl \ + libboost-graph-dev \ + libpq-dev \ + postgresql-server-dev-${PG_MAJOR} \ + && curl -Lo pgrouting.tar.gz "https://github.com/pgRouting/pgrouting/archive/v${PGROUTING_VERSION}.tar.gz" \ + && echo "$PGROUTING_SHA256 *pgrouting.tar.gz" | sha256sum -c - \ + && mkdir -p /usr/src/pgrouting \ + && tar \ + --extract \ + --file pgrouting.tar.gz \ + --directory /usr/src/pgrouting \ + --strip-components 1 \ + && rm pgrouting.tar.gz \ + && cd /usr/src/pgrouting \ + && mkdir build \ + && cd build \ + && cmake .. \ + && make -j $(nproc --all) \ + && make install \ + && cd / \ + && rm -rf /usr/src/pgrouting \ + && apt-mark manual postgresql-18 \ + && apt purge -y --autoremove \ + build-essential \ + cmake \ + libpq-dev \ + libboost-graph-dev \ + postgresql-server-dev-${PG_MAJOR} \ + && rm -rf /var/lib/apt/lists/* + + +# Create a baseline PostgreSQL server image with PostGIS and PgRouting extensions installed in. +# Can and should be used as the "random postgresql container neede in place X", that is instead +# of pulling in the normal postgres/postgis/pgrouting images where and whenever, use this instead. +FROM pgrouting AS jore4-postgres + +# Change collations to use fi_FI +RUN localedef -i fi_FI -c -f UTF-8 -A /usr/share/locale/locale.alias fi_FI.UTF-8 +ENV LANG=fi_FI.utf8 + +RUN mkdir -p -m 0755 /jore4/migrations /jore4/scripts + +# Download script for reading docker secrets +ADD --chmod=755 https://raw.githubusercontent.com/HSLdevcom/jore4-tools/main/docker/read-secrets.sh /jore4/scripts/read-secrets.sh + +# Read in secrets to ensure POSTGRES_USER and/or POSTGRES_DB are set +HEALTHCHECK --interval=5s --timeout=5s --start-period=10s --retries=20 \ + CMD bash -c "source /tmp/read-secrets.sh && pg_isready -h localhost -U \${POSTGRES_USER} -d \${POSTGRES_DB}" diff --git a/mapmatching/Dockerfile b/mapmatching/Dockerfile new file mode 100644 index 0000000..193654b --- /dev/null +++ b/mapmatching/Dockerfile @@ -0,0 +1,11 @@ +# The Map Matching service's image +ARG BASE_IMAGE_LOCATION='' +ARG BASE_IMAGE_VERSION=latest +FROM ${BASE_IMAGE_LOCATION}jore4-postgres:${BASE_IMAGE_VERSION} AS jore4-postgres-mapmatching + +COPY --chmod=755 entrypoint.sh /jore4/scripts/entrypoint.sh +COPY --chmod=644 migrations/mapmatching /docker-entrypoint-initdb.d/ +ADD --chmod=644 https://stjore4dev001.blob.core.windows.net/jore4-digiroad/2026-08-04_create_routing_schema_digiroad_r_2026-01_mml_2026-08-04.sql /docker-entrypoint-initdb.d/02-digiroad_r_routing.sql + +ENTRYPOINT ["/jore4/scripts/entrypoint.sh"] +CMD ["postgres"] diff --git a/mapmatching/azuredbmock-entrypoint.sh b/mapmatching/azuredbmock-entrypoint.sh new file mode 100755 index 0000000..dcc56d5 --- /dev/null +++ b/mapmatching/azuredbmock-entrypoint.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -Eeuo pipefail + +# read the secrets to environment variables +source /jore4/scripts/read-secrets.sh + +# Copy the migrations scripts to the docker-entrypoint-initdb.d folder so that the postgres entrypoint +# executes them (together with the other migrations that might have been mapped as a volume) +mkdir -p /docker-entrypoint-initdb.d +cp /jore4/migrations/* /docker-entrypoint-initdb.d/ + +# replace placeholders with secrets within the all migration SQL scripts +SECRET_STORE_BASE_PATH="${SECRET_STORE_BASE_PATH:-/run/secrets}" +REPLACE_PLACEHOLDERS_SCRIPT='/jore4/scripts/replace-placeholders-in-sql-schema-migrations.sh' +MIGRATIONS_DIR="/docker-entrypoint-initdb.d/" +"${REPLACE_PLACEHOLDERS_SCRIPT}" "${SECRET_STORE_BASE_PATH}" "${MIGRATIONS_DIR}" + +# call the original entrypoint to continue execution +echo 'Start postgres' +exec /usr/local/bin/docker-entrypoint.sh "$@" diff --git a/mapmatching/entrypoint.sh b/mapmatching/entrypoint.sh new file mode 100755 index 0000000..17e52af --- /dev/null +++ b/mapmatching/entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -Eeuo pipefail + +# Read the secrets to environment variables. +source /jore4/scripts/read-secrets.sh + +# Call the original entrypoint to continue startup. +echo 'Start postgres' +exec /usr/local/bin/docker-entrypoint.sh "$@" diff --git a/mapmatching/migrations/azuredbmock/01-create-roles.sql b/mapmatching/migrations/azuredbmock/01-create-roles.sql new file mode 100644 index 0000000..9c9e5e1 --- /dev/null +++ b/mapmatching/migrations/azuredbmock/01-create-roles.sql @@ -0,0 +1,15 @@ +-- These database roles are also created in the azure-infra-jore4aks (Azure +-- DevOps) repository. +CREATE USER xxx_db_auth_username_xxx PASSWORD 'xxx_db_auth_password_xxx'; +CREATE USER xxx_db_jore3importer_username_xxx PASSWORD 'xxx_db_jore3importer_password_xxx'; +CREATE USER xxx_db_hasura_username_xxx PASSWORD 'xxx_db_hasura_password_xxx'; +CREATE USER xxx_db_tiamat_username_xxx PASSWORD 'xxx_db_tiamat_password_xxx'; +CREATE USER xxx_db_timetables_api_username_xxx PASSWORD 'xxx_db_timetables_api_password_xxx'; +CREATE USER xxx_db_dss_username_xxx PASSWORD 'xxx_db_dss_password_xxx'; + +-- Make the hasura role a member of jore3importer role because both roles must +-- have ownership of tables and sequences in the default database (network and +-- routes) since both are responsible for populating and truncating tables in +-- the aforementioned database. In particular, sequence reset requires an +-- ownership and cannot be granted as a privilege. +GRANT xxx_db_jore3importer_username_xxx TO xxx_db_hasura_username_xxx; diff --git a/mapmatching/migrations/azuredbmock/02-create-network-database.sql b/mapmatching/migrations/azuredbmock/02-create-network-database.sql new file mode 100644 index 0000000..f1f3061 --- /dev/null +++ b/mapmatching/migrations/azuredbmock/02-create-network-database.sql @@ -0,0 +1,33 @@ +-- Make the JORE4 admin role the owner of the public schema. +ALTER SCHEMA public OWNER TO CURRENT_USER; + +-- Create the extensions used, see https://hasura.io/docs/latest/graphql/core/deployment/postgres-requirements.html +-- Create the extensions in the public schema, since we'd need to give additional privileges ("use schema") to any +-- user who wishes to use these in the future. Also, Hasura would require additional setup to be able to use the +-- extensions from another schema. +CREATE EXTENSION IF NOT EXISTS btree_gist WITH SCHEMA public; +CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public; +CREATE EXTENSION IF NOT EXISTS postgis WITH SCHEMA public; + +-- Allow Hasura to connect and create new schemas. +GRANT CONNECT, CREATE ON DATABASE xxx_db_hasura_name_xxx TO xxx_db_hasura_username_xxx; + +-- Grant required privileges in the public schema to the Hasura user. +GRANT ALL ON SCHEMA public TO xxx_db_hasura_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_hasura_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_hasura_username_xxx; + +-- Grant select permissions on information_schema and pg_catalog to the Hasura +-- user. +GRANT SELECT ON ALL TABLES IN SCHEMA information_schema TO xxx_db_hasura_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA pg_catalog TO xxx_db_hasura_username_xxx; + +-- Allow the JORE3-Importer role to connect to the network database. +GRANT CONNECT ON DATABASE xxx_db_hasura_name_xxx TO xxx_db_jore3importer_username_xxx; + +-- Grant the JORE3-Importer role to access the public schema and the objects and +-- functions created by extensions. Other schema-specific privileges are granted +-- in Hasura migrations. +GRANT USAGE ON SCHEMA public TO xxx_db_jore3importer_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_jore3importer_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_jore3importer_username_xxx; diff --git a/mapmatching/migrations/azuredbmock/03-create-auth-database.sql b/mapmatching/migrations/azuredbmock/03-create-auth-database.sql new file mode 100644 index 0000000..07c86f0 --- /dev/null +++ b/mapmatching/migrations/azuredbmock/03-create-auth-database.sql @@ -0,0 +1,12 @@ +CREATE DATABASE xxx_db_auth_name_xxx; + +-- Allow the auth role to connect and create new schemas. +GRANT CONNECT, CREATE ON DATABASE xxx_db_auth_name_xxx TO xxx_db_auth_username_xxx; + +\connect xxx_db_auth_name_xxx; + +-- Make the JORE4 admin role the owner of the public schema. +ALTER SCHEMA public OWNER TO CURRENT_USER; + +-- Grant full schema access to the public schema to the auth role. +GRANT ALL ON SCHEMA public TO xxx_db_auth_username_xxx; diff --git a/mapmatching/migrations/azuredbmock/04-create-jore3importer-database.sql b/mapmatching/migrations/azuredbmock/04-create-jore3importer-database.sql new file mode 100644 index 0000000..40f7ad6 --- /dev/null +++ b/mapmatching/migrations/azuredbmock/04-create-jore3importer-database.sql @@ -0,0 +1,25 @@ +CREATE DATABASE xxx_db_jore3importer_name_xxx; + +-- Allow the jore3importer role to connect and create new schemas. +GRANT CONNECT, CREATE ON DATABASE xxx_db_jore3importer_name_xxx TO xxx_db_jore3importer_username_xxx; + +\connect xxx_db_jore3importer_name_xxx; + +-- Make the JORE4 admin role the owner of the public schema. +ALTER SCHEMA public OWNER TO CURRENT_USER; + +-- Create the extensions that JORE3-Importer needs. In PostgreSQL v15 server, +-- an ordinary user (without admin roles) may not be able to create extensions. +CREATE EXTENSION IF NOT EXISTS btree_gist WITH SCHEMA public; +CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public; + +-- Grant privileges in the public schema to the jore3importer role. +GRANT USAGE ON SCHEMA public TO xxx_db_jore3importer_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_jore3importer_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_jore3importer_username_xxx; + +-- Grant permission on the pg_catalog schema to conditionally create an +-- extension if the extension is not already created. This must be granted so +-- that the JORE3-Importer role can conditionally create the extension depending +-- on which database is involved (there are several in the importer's test setup). +GRANT SELECT ON TABLE pg_catalog.pg_extension TO xxx_db_jore3importer_username_xxx; diff --git a/mapmatching/migrations/azuredbmock/05-create-timetables-database.sql b/mapmatching/migrations/azuredbmock/05-create-timetables-database.sql new file mode 100644 index 0000000..8648b48 --- /dev/null +++ b/mapmatching/migrations/azuredbmock/05-create-timetables-database.sql @@ -0,0 +1,36 @@ +CREATE DATABASE xxx_db_timetables_name_xxx; + +-- Allow Hasura to connect and create new schemas. +GRANT CONNECT, CREATE ON DATABASE xxx_db_timetables_name_xxx TO xxx_db_hasura_username_xxx; + +-- Interval outputs by default are using the sql format ('3 4:05:06'). Here we +-- are switching to ISO 8601 format ('P3DT4H5M6S'). +ALTER DATABASE xxx_db_timetables_name_xxx SET intervalstyle = 'iso_8601'; + +-- Switch database context to be able to add extensions there. +\connect xxx_db_timetables_name_xxx; + +-- Make the JORE4 admin role the owner of the public schema. +ALTER SCHEMA public OWNER TO CURRENT_USER; + +CREATE EXTENSION IF NOT EXISTS btree_gist WITH SCHEMA public; +CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public; + +-- Grant required privileges in the public schema to Hasura. +GRANT ALL ON SCHEMA public TO xxx_db_hasura_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_hasura_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_hasura_username_xxx; + +-- Grant select permissions on information_schema and pg_catalog to Hasura. +GRANT SELECT ON ALL TABLES IN SCHEMA information_schema TO xxx_db_hasura_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA pg_catalog TO xxx_db_hasura_username_xxx; + +-- Allow the timetables-api role to connect to the timetables database. +GRANT CONNECT ON DATABASE xxx_db_timetables_name_xxx TO xxx_db_timetables_api_username_xxx; + +-- Grant the timetables-api role to access the public schema and the objects and +-- functions created by extensions. Other schema-specific privileges are granted +-- in Hasura migrations. +GRANT USAGE ON SCHEMA public TO xxx_db_timetables_api_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_timetables_api_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_timetables_api_username_xxx; diff --git a/mapmatching/migrations/azuredbmock/06-create-stopregistry-database.sql b/mapmatching/migrations/azuredbmock/06-create-stopregistry-database.sql new file mode 100644 index 0000000..6ea599f --- /dev/null +++ b/mapmatching/migrations/azuredbmock/06-create-stopregistry-database.sql @@ -0,0 +1,39 @@ +CREATE DATABASE xxx_db_tiamat_name_xxx; + +-- Allow Tiamat to connect and create new schemas. +GRANT CONNECT, CREATE ON DATABASE xxx_db_tiamat_name_xxx TO xxx_db_tiamat_username_xxx; + +-- Switch database context to initialise it to the state where Tiamat can use +-- it. +\connect xxx_db_tiamat_name_xxx; + +-- Make the JORE4 admin role the owner of the public schema. +ALTER SCHEMA public OWNER TO CURRENT_USER; + +CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public; +CREATE EXTENSION IF NOT EXISTS postgis WITH SCHEMA public; + +-- Grant required privileges in the public schema to Tiamat. +GRANT ALL ON SCHEMA public TO xxx_db_tiamat_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_tiamat_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_tiamat_username_xxx; + +-- Create "topology" schema and install the "postgis_topology" extension to it. +-- The Tiamat role needs ownership to the schema and its tables. +CREATE SCHEMA IF NOT EXISTS topology; +CREATE EXTENSION IF NOT EXISTS postgis_topology WITH SCHEMA topology; +ALTER SCHEMA topology OWNER TO xxx_db_tiamat_username_xxx; +-- The "postgis_topology" extension creates two tables. +ALTER TABLE topology.layer OWNER TO xxx_db_tiamat_username_xxx; +ALTER TABLE topology.topology OWNER TO xxx_db_tiamat_username_xxx; + +-- Grant Hasura read permissions to the stop registry database. +GRANT CONNECT ON DATABASE xxx_db_tiamat_name_xxx TO xxx_db_hasura_username_xxx; + +GRANT USAGE ON SCHEMA public TO xxx_db_hasura_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_hasura_username_xxx; +ALTER DEFAULT PRIVILEGES FOR USER xxx_db_tiamat_username_xxx IN SCHEMA public GRANT SELECT ON TABLES TO xxx_db_hasura_username_xxx; + +GRANT USAGE ON SCHEMA topology TO xxx_db_hasura_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA topology TO xxx_db_hasura_username_xxx; +ALTER DEFAULT PRIVILEGES FOR USER xxx_db_tiamat_username_xxx IN SCHEMA topology GRANT SELECT ON TABLES TO xxx_db_hasura_username_xxx; diff --git a/mapmatching/migrations/azuredbmock/07-create-jore4-main-database.sql b/mapmatching/migrations/azuredbmock/07-create-jore4-main-database.sql new file mode 100644 index 0000000..2d268ac --- /dev/null +++ b/mapmatching/migrations/azuredbmock/07-create-jore4-main-database.sql @@ -0,0 +1,124 @@ +-- This migration replicates initialization of the JORE4 main database in script scripts/create_db_roles_and_extensions.sh +-- in azure-infra-jore4aks repository + +CREATE DATABASE xxx_db_jore4_main_name_xxx; + +\connect xxx_db_jore4_main_name_xxx; + + +-------------------------- +----- Create Schemas ----- +-------------------------- + +CREATE SCHEMA IF NOT EXISTS network AUTHORIZATION xxx_db_hasura_username_xxx; +CREATE SCHEMA IF NOT EXISTS stopregistry AUTHORIZATION xxx_db_tiamat_username_xxx; +CREATE SCHEMA IF NOT EXISTS timetables AUTHORIZATION xxx_db_hasura_username_xxx; +CREATE SCHEMA IF NOT EXISTS hdb_catalog AUTHORIZATION xxx_db_hasura_username_xxx; +CREATE SCHEMA IF NOT EXISTS dssview AUTHORIZATION xxx_db_hasura_username_xxx; + +CREATE SCHEMA IF NOT EXISTS topology AUTHORIZATION xxx_db_tiamat_username_xxx; + + +----------------------------- +----- Create Extensions ----- +----------------------------- + +-- Extensions are created in public schema so the extensions are found from the search path +CREATE EXTENSION IF NOT EXISTS btree_gist WITH SCHEMA public; +CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public; +CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public; +CREATE EXTENSION IF NOT EXISTS postgis WITH SCHEMA public; +CREATE EXTENSION IF NOT EXISTS postgis_topology WITH SCHEMA topology; + + +--------------------------------------- +----- Configure Database Settings ----- +--------------------------------------- + +-- Interval outputs by default are using the SQL format ('3 4:05:06'). Here we +-- switch to ISO 8601 format ('P3DT4H5M6S'). +ALTER DATABASE xxx_db_jore4_main_name_xxx SET intervalstyle = 'iso_8601'; + + +-------------------------------------------------- +----- Grant Database Level Access Privileges ----- +-------------------------------------------------- + +GRANT CONNECT, CREATE ON DATABASE xxx_db_jore4_main_name_xxx TO xxx_db_hasura_username_xxx; +GRANT CONNECT, CREATE ON DATABASE xxx_db_jore4_main_name_xxx TO xxx_db_tiamat_username_xxx; + +GRANT CONNECT ON DATABASE xxx_db_jore4_main_name_xxx TO xxx_db_jore3importer_username_xxx; +GRANT CONNECT ON DATABASE xxx_db_jore4_main_name_xxx TO xxx_db_timetables_api_username_xxx; +GRANT CONNECT ON DATABASE xxx_db_jore4_main_name_xxx TO xxx_db_dss_username_xxx; + + +------------------------------------------------------- +----- Grant Network Schema Level Access Privileges ---- +------------------------------------------------------- + +-- Grant required privileges to Hasura. +GRANT ALL ON SCHEMA network TO xxx_db_hasura_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA network TO xxx_db_hasura_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA network TO xxx_db_hasura_username_xxx; + +-- Grant the JORE3-Importer role to access the schema and the objects and +-- functions created by extensions. Other schema-specific privileges are granted +-- in Hasura migrations. +GRANT USAGE ON SCHEMA network TO xxx_db_jore3importer_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA network TO xxx_db_jore3importer_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA network TO xxx_db_jore3importer_username_xxx; + + +------------------------------------------------------------- +----- Grant Stop Registry Schema Level Access Privileges ---- +------------------------------------------------------------- + +-- Tiamat +GRANT ALL ON SCHEMA stopregistry TO xxx_db_tiamat_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA stopregistry TO xxx_db_tiamat_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA stopregistry TO xxx_db_tiamat_username_xxx; + +-- Hasura +GRANT USAGE ON SCHEMA stopregistry TO xxx_db_hasura_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA stopregistry TO xxx_db_hasura_username_xxx; +ALTER DEFAULT PRIVILEGES FOR USER xxx_db_tiamat_username_xxx IN SCHEMA stopregistry GRANT SELECT ON TABLES TO xxx_db_hasura_username_xxx; + + +-------------------------------------------------------- +----- Grant Topology Schema Level Access Privileges ---- +-------------------------------------------------------- + +-- See the beginning of the initial database migration in: +-- https://github.com/entur/tiamat/blob/master/src/main/resources/db/migration/V1__Base_version.sql +GRANT USAGE ON SCHEMA topology TO xxx_db_hasura_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA topology TO xxx_db_hasura_username_xxx; +ALTER DEFAULT PRIVILEGES FOR USER xxx_db_tiamat_username_xxx IN SCHEMA topology GRANT SELECT ON TABLES TO xxx_db_hasura_username_xxx; + + +---------------------------------------------------------- +----- Grant Timetables Schema Level Access Privileges ---- +---------------------------------------------------------- + +GRANT ALL ON SCHEMA timetables TO xxx_db_hasura_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA timetables TO xxx_db_hasura_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA timetables TO xxx_db_hasura_username_xxx; + +-- Allow the timetables-api role to use timetables schema +GRANT USAGE ON SCHEMA timetables TO xxx_db_timetables_api_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA timetables TO xxx_db_timetables_api_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA timetables TO xxx_db_timetables_api_username_xxx; + + +-------------------------------------------------------- +----- Grant DSS View Schema Level Access Privileges ---- +-------------------------------------------------------- + +-- See the beginning of the initial database migration in: +-- https://github.com/entur/tiamat/blob/master/src/main/resources/db/migration/V1__Base_version.sql +GRANT USAGE ON SCHEMA dssview TO xxx_db_hasura_username_xxx; +GRANT ALL ON SCHEMA dssview TO xxx_db_hasura_username_xxx; +GRANT ALL ON ALL TABLES IN SCHEMA dssview TO xxx_db_hasura_username_xxx; + +GRANT USAGE ON SCHEMA dssview TO xxx_db_dss_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA dssview TO xxx_db_dss_username_xxx; +ALTER DEFAULT PRIVILEGES FOR USER xxx_db_dss_username_xxx IN SCHEMA dssview GRANT SELECT ON TABLES TO xxx_db_dss_username_xxx; diff --git a/mapmatching/migrations/mapmatching/01-initialize.sql b/mapmatching/migrations/mapmatching/01-initialize.sql new file mode 100644 index 0000000..5c3dce9 --- /dev/null +++ b/mapmatching/migrations/mapmatching/01-initialize.sql @@ -0,0 +1,2 @@ +CREATE EXTENSION postgis; +CREATE EXTENSION pgrouting; diff --git a/mapmatching/migrations/mapmatching/02-digiroad_r_routing.md b/mapmatching/migrations/mapmatching/02-digiroad_r_routing.md new file mode 100644 index 0000000..5a68f36 --- /dev/null +++ b/mapmatching/migrations/mapmatching/02-digiroad_r_routing.md @@ -0,0 +1 @@ +Actual SQL file is downloaded and placed in the Dockerfile diff --git a/mapmatching/migrations/mapmatching/03-post-initialize.sql b/mapmatching/migrations/mapmatching/03-post-initialize.sql new file mode 100644 index 0000000..45ba3e1 --- /dev/null +++ b/mapmatching/migrations/mapmatching/03-post-initialize.sql @@ -0,0 +1 @@ +ALTER DATABASE ${POSTGRES_DB} SET search_path = routing, public; diff --git a/mapmatching/replace-placeholders-in-sql-schema-migrations.sh b/mapmatching/replace-placeholders-in-sql-schema-migrations.sh new file mode 100755 index 0000000..a6a23c3 --- /dev/null +++ b/mapmatching/replace-placeholders-in-sql-schema-migrations.sh @@ -0,0 +1,136 @@ +#!/bin/sh +# +# Replace placeholders in SQL schema migrations. +# +# Docker or Kubernetes secrets contain the PostgreSQL users we need in the SQL +# schema migrations. As placeholders in the SQL schema migrations, we use +# mangled Docker secret filenames with bracket delimiters. This script replaces +# the placeholders with the contents of the Docker secrets at runtime. +# +# E.g. the username stored in a secret named +# "db-jore3importer-username" is used with the placeholder +# "xxx_db_jore3importer_username_xxx" within the SQL schema migration +# files. + +set -eu + +[ "$#" -ne 2 ] && { + echo 1>&2 "Usage: $(basename "$0") RUNTIME_SECRETS_DIR RUNTIME_MIGRATIONS_DIR" + exit 1 +} + +secrets_dir="$1" +migrations_dir="$2" + +# Choose delimiters that allow the placeholders to act as valid SQL identifiers +# for any static analysis purposes. +delimiter_alnum='xxx' +start_delimiter="${delimiter_alnum}_" +end_delimiter="_${delimiter_alnum}" +placeholder_characters='0-9A-Za-z_' + +cleanup() { + [ -z "${check_duplicate_list+dummy}" ] || rm -f "${check_duplicate_list}" + [ -z "${sed_script_file+dummy}" ] || rm -f "${sed_script_file}" +} +trap cleanup EXIT + +check_duplicate_list="$(mktemp)" +sed_script_file="$(mktemp)" + +# Create a script file for sed to replace the placeholders. +write_to_sed_script_file() { + secret_path="$1" + # As `basename -z` is not supported by the available shell, we cannot handle + # newlines in filenames with this approach. We do not translate the newline + # inserted by basename. + placeholder="$( + basename "${secret_path}" \ + | tr -C "${placeholder_characters}\n" '_' + )" + + # Check for the delimiter in the secret name. + if echo \ + "${placeholder}" \ + | grep \ + -q \ + -e "${delimiter_alnum}"; then + cat <&2 +The basename for secret +${secret_path} +should not contain the alphanumeric part of the delimiter: +${delimiter_alnum} +EOM + exit 4 + fi + + # Check for collisions of secrets. + if grep \ + -q \ + -x \ + -e "${placeholder}" \ + "${check_duplicate_list}"; then + cat <&2 +The basename for secret +${secret_path} +is not unique enough. The basenames of the secrets should differ by alphanumeric +characters. +EOM + exit 2 + fi + echo "${placeholder}" >>"${check_duplicate_list}" + + echo "s|${start_delimiter}${placeholder}${end_delimiter}|$(cat "${secret_path}")|g" \ + >>"${sed_script_file}" +} + +# Replace the placeholders with sed and check for leftover placeholders. +replace_and_check() { + migration_path="$1" + sed \ + -f "${sed_script_file}" \ + -i \ + "${migration_path}" + if leftover_placeholders="$( + grep \ + -o \ + -h \ + -E \ + -e "${start_delimiter}[${placeholder_characters}]+${end_delimiter}" \ + "${migration_path}" + )"; then + sorted_leftovers="$(echo "${leftover_placeholders}" | sort -u)" + cat <&2 +The migration file +${migration_path} +contains some placeholders for which there are no corresponding secrets. The +offending placeholders are: +${sorted_leftovers} +EOM + exit 3 + fi +} + +echo "Starting to replace placeholders in SQL schema migration files." + +# As `export -f` is not supported by the available shell, executing the shell +# functions with `find` does not work. + +# As `read -d ''` is not supported by the available shell, `find -print0` cannot +# be used here. + +# Loop over secrets. +find "${secrets_dir}" -type f \ + | LC_ALL=C sort \ + | while IFS= read -r file; do + write_to_sed_script_file "${file}" + done + +# Loop over SQL schema migrations. +find "${migrations_dir}" -type f -name '*.sql' \ + | LC_ALL=C sort \ + | while IFS= read -r file; do + replace_and_check "${file}" + done + +echo "Replacing placeholders done."