Skip to content

Commit 908ad32

Browse files
HuulivoideHuulivoide
committed
WIP
1 parent c755fd7 commit 908ad32

16 files changed

Lines changed: 594 additions & 4 deletions

‎.github/workflows/cd.yml‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,31 +11,53 @@ permissions:
1111
contents: read
1212

1313
jobs:
14+
publish_jore_postgres_docker_image_to_acr:
15+
name: Publish Generic Jore4 PostgreSQL Docker image to ACR
16+
permissions:
17+
id-token: write
18+
contents: read
19+
uses: HSLdevcom/jore4-tools/.github/workflows/shared-build-and-publish-docker-image.yml@main
20+
with:
21+
docker_image_name: jore4-postgres
22+
build_arm64_image: true
23+
context: ${{ github.server_url }}/${{ github.repository }}.git#${{ github.ref }}:./joredb
24+
target: jore4-postgres
25+
secrets:
26+
azure_client_id: ${{ secrets.AZURE_CLIENT_ID }}
27+
azure_tenant_id: ${{ secrets.AZURE_TENANT_ID }}
28+
azure_subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
29+
1430
publish_mapmatching_docker_image_to_acr:
1531
name: Publish Map Matching Docker image to ACR
32+
needs:
33+
- publish_jore_postgres_docker_image_to_acr
1634
permissions:
1735
id-token: write
1836
contents: read
1937
uses: HSLdevcom/jore4-tools/.github/workflows/shared-build-and-publish-docker-image.yml@main
2038
with:
2139
docker_image_name: jore4-postgres-mapmatching
2240
build_arm64_image: true
23-
context: ${{ github.server_url }}/${{ github.repository }}.git#${{ github.ref }}:./mapmatching
41+
context: ${{ github.server_url }}/${{ github.repository }}.git#${{ github.ref }}:./joredb
42+
target: jore4-postgres-mapmatching
2443
secrets:
2544
azure_client_id: ${{ secrets.AZURE_CLIENT_ID }}
2645
azure_tenant_id: ${{ secrets.AZURE_TENANT_ID }}
2746
azure_subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
2847

2948
publish_azuredbmock_docker_image_to_acr:
3049
name: Publish Azure DB Mock Docker image to ACR
50+
needs:
51+
- publish_jore_postgres_docker_image_to_acr
3152
permissions:
3253
id-token: write
3354
contents: read
3455
uses: HSLdevcom/jore4-tools/.github/workflows/shared-build-and-publish-docker-image.yml@main
3556
with:
3657
docker_image_name: jore4-postgres-azuredbmock
3758
build_arm64_image: true
38-
context: ${{ github.server_url }}/${{ github.repository }}.git#${{ github.ref }}:./azuredbmock
59+
context: ${{ github.server_url }}/${{ github.repository }}.git#${{ github.ref }}:./joredb
60+
target: jore4-postgres-azuredbmock
3961
secrets:
4062
azure_client_id: ${{ secrets.AZURE_CLIENT_ID }}
4163
azure_tenant_id: ${{ secrets.AZURE_TENANT_ID }}

‎docker-compose.yml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22
services:
33
mapmatchingdb:
44
build:
5-
context: "./mapmatching"
5+
context: ./joredb
6+
target: jore4-postgres-mapmatching
67
container_name: mapmatchingdb
78
restart: "unless-stopped"
89
networks:
@@ -20,7 +21,8 @@ services:
2021

2122
azuredbmock:
2223
build:
23-
context: "./azuredbmock"
24+
context: ./joredb
25+
target: jore4-postgres-azuredbmock
2426
container_name: azuredbmock
2527
restart: "unless-stopped"
2628
networks:

‎joredb/Dockerfile‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
## Below an inlined version of the PostGIS Dockerfile. PostGIS does not
2+
## relase ARM64 versions of the image at the moment (they did in the past).
3+
## And thus, we have to build the ARM version our selves.
4+
5+
FROM docker.io/postgres:18-trixie AS postgis
6+
7+
ENV POSTGIS_MAJOR=3
8+
ENV POSTGIS_VERSION=3.6.4+dfsg-2.pgdg13+1
9+
10+
RUN apt-get update \
11+
&& apt-cache showpkg postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR \
12+
&& apt-get install -y --no-install-recommends \
13+
# ca-certificates: for accessing remote raster files;
14+
# fix: https://github.com/postgis/docker-postgis/issues/307
15+
ca-certificates \
16+
\
17+
postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR=$POSTGIS_VERSION \
18+
postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts \
19+
&& rm -rf /var/lib/apt/lists/* \
20+
&& mkdir -p /docker-entrypoint-initdb.d
21+
22+
23+
# Inlined version of the pgrouting Dockerfile. Needed to add support for the ARM64 version.
24+
FROM postgis AS pgrouting
25+
26+
ENV PGROUTING_VERSION=3.8.0
27+
ENV PGROUTING_SHA256=b8a5f0472934fdf7cda3fb4754d01945378d920cdaddc01f378617ddbb9c447f
28+
ENV LIBBOOST_VERSION=1.83.0
29+
30+
RUN set -ex \
31+
&& apt update \
32+
&& apt install -y \
33+
libboost-atomic${LIBBOOST_VERSION} \
34+
libboost-chrono${LIBBOOST_VERSION} \
35+
libboost-graph${LIBBOOST_VERSION} \
36+
libboost-date-time${LIBBOOST_VERSION} \
37+
libboost-program-options${LIBBOOST_VERSION} \
38+
libboost-system${LIBBOOST_VERSION} \
39+
libboost-thread${LIBBOOST_VERSION} \
40+
&& apt install -y \
41+
build-essential \
42+
cmake \
43+
curl \
44+
libboost-graph-dev \
45+
libpq-dev \
46+
postgresql-server-dev-${PG_MAJOR} \
47+
&& curl -Lo pgrouting.tar.gz "https://github.com/pgRouting/pgrouting/archive/v${PGROUTING_VERSION}.tar.gz" \
48+
&& echo "$PGROUTING_SHA256 *pgrouting.tar.gz" | sha256sum -c - \
49+
&& mkdir -p /usr/src/pgrouting \
50+
&& tar \
51+
--extract \
52+
--file pgrouting.tar.gz \
53+
--directory /usr/src/pgrouting \
54+
--strip-components 1 \
55+
&& rm pgrouting.tar.gz \
56+
&& cd /usr/src/pgrouting \
57+
&& mkdir build \
58+
&& cd build \
59+
&& cmake .. \
60+
&& make -j $(nproc --all) \
61+
&& make install \
62+
&& cd / \
63+
&& rm -rf /usr/src/pgrouting \
64+
&& apt-mark manual postgresql-18 \
65+
&& apt purge -y --autoremove \
66+
build-essential \
67+
cmake \
68+
libpq-dev \
69+
libboost-graph-dev \
70+
postgresql-server-dev-${PG_MAJOR} \
71+
&& rm -rf /var/lib/apt/lists/*
72+
73+
74+
# Create a baseline PostgreSQL server image with PostGIS and PgRouting extensions installed in.
75+
# Can and should be used as the "random postgresql container neede in place X", that is instead
76+
# of pulling in the normal postgres/postgis/pgrouting images where and whenever, use this instead.
77+
FROM pgrouting AS jore4-postgres
78+
79+
# Change collations to use fi_FI
80+
RUN localedef -i fi_FI -c -f UTF-8 -A /usr/share/locale/locale.alias fi_FI.UTF-8
81+
ENV LANG=fi_FI.utf8
82+
83+
RUN mkdir -p -m 0755 /jore4/migrations /jore4/scripts
84+
85+
# Download script for reading docker secrets
86+
ADD --chmod=755 https://raw.githubusercontent.com/HSLdevcom/jore4-tools/main/docker/read-secrets.sh /jore4/scripts/read-secrets.sh
87+
88+
# Read in secrets to ensure POSTGRES_USER and/or POSTGRES_DB are set
89+
HEALTHCHECK --interval=5s --timeout=5s --start-period=10s --retries=20 \
90+
CMD bash -c "source /tmp/read-secrets.sh && pg_isready -h localhost -U \${POSTGRES_USER} -d \${POSTGRES_DB}"
91+
92+
93+
# The primary Database image
94+
FROM jore4-postgres AS jore4-postgres-azuredbmock
95+
96+
COPY --chmod=644 migrations/azuredbmock /jore4/migrations/
97+
COPY --chmod=755 replace-placeholders-in-sql-schema-migrations.sh /jore4/scripts/replace-placeholders-in-sql-schema-migrations.sh
98+
COPY --chmod=755 azuredbmock-entrypoint.sh /jore4/scripts/
99+
100+
ENTRYPOINT ["/jore4/scripts/azuredbmock-entrypoint.sh"]
101+
CMD ["postgres"]
102+
103+
104+
# The Map Matching service's image
105+
FROM jore4-postgres AS jore4-postgres-mapmatching
106+
107+
COPY --chmod=644 migrations/mapmatching /docker-entrypoint-initdb.d/
108+
ADD --chmod=644 https://stjore4dev001.blob.core.windows.net/jore4-digiroad/2026-08-04_create_routing_schema_digiroad_r_2026-01_mml_2026-08-04.sql /docker-entrypoint-initdb.d/02-digiroad_r_routing.sql
109+
110+
ENTRYPOINT ["/jore4/scripts/mapmatching-entrypoint.sh"]
111+
CMD ["postgres"]

‎joredb/azuredbmock-entrypoint.sh‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/bin/bash
2+
3+
set -Eeuo pipefail
4+
5+
# read the secrets to environment variables
6+
source /jore4/scripts/read-secrets.sh
7+
8+
# Copy the migrations scripts to the docker-entrypoint-initdb.d folder so that the postgres entrypoint
9+
# executes them (together with the other migrations that might have been mapped as a volume)
10+
mkdir -p /docker-entrypoint-initdb.d
11+
cp /jore4/migrations/* /docker-entrypoint-initdb.d/
12+
13+
# replace placeholders with secrets within the all migration SQL scripts
14+
SECRET_STORE_BASE_PATH="${SECRET_STORE_BASE_PATH:-/run/secrets}"
15+
REPLACE_PLACEHOLDERS_SCRIPT='/jore4/scripts/replace-placeholders-in-sql-schema-migrations.sh'
16+
MIGRATIONS_DIR="/docker-entrypoint-initdb.d/"
17+
"${REPLACE_PLACEHOLDERS_SCRIPT}" "${SECRET_STORE_BASE_PATH}" "${MIGRATIONS_DIR}"
18+
19+
# call the original entrypoint to continue execution
20+
echo 'Start postgres'
21+
exec /usr/local/bin/docker-entrypoint.sh "$@"

‎joredb/mapmatching-entrypoint.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/bin/bash
2+
3+
set -Eeuo pipefail
4+
5+
# Read the secrets to environment variables.
6+
source /tmp/read-secrets.sh
7+
8+
# Call the original entrypoint to continue startup.
9+
echo 'Start postgres'
10+
exec /usr/local/bin/docker-entrypoint.sh "$@"

‎joredb/migrations/azuredbmock/01-create-roles.sql‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
-- These database roles are also created in the azure-infra-jore4aks (Azure
2+
-- DevOps) repository.
3+
CREATE USER xxx_db_auth_username_xxx PASSWORD 'xxx_db_auth_password_xxx';
4+
CREATE USER xxx_db_jore3importer_username_xxx PASSWORD 'xxx_db_jore3importer_password_xxx';
5+
CREATE USER xxx_db_hasura_username_xxx PASSWORD 'xxx_db_hasura_password_xxx';
6+
CREATE USER xxx_db_tiamat_username_xxx PASSWORD 'xxx_db_tiamat_password_xxx';
7+
CREATE USER xxx_db_timetables_api_username_xxx PASSWORD 'xxx_db_timetables_api_password_xxx';
8+
CREATE USER xxx_db_dss_username_xxx PASSWORD 'xxx_db_dss_password_xxx';
9+
10+
-- Make the hasura role a member of jore3importer role because both roles must
11+
-- have ownership of tables and sequences in the default database (network and
12+
-- routes) since both are responsible for populating and truncating tables in
13+
-- the aforementioned database. In particular, sequence reset requires an
14+
-- ownership and cannot be granted as a privilege.
15+
GRANT xxx_db_jore3importer_username_xxx TO xxx_db_hasura_username_xxx;

‎joredb/migrations/azuredbmock/02-create-network-database.sql‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
-- Make the JORE4 admin role the owner of the public schema.
2+
ALTER SCHEMA public OWNER TO CURRENT_USER;
3+
4+
-- Create the extensions used, see https://hasura.io/docs/latest/graphql/core/deployment/postgres-requirements.html
5+
-- Create the extensions in the public schema, since we'd need to give additional privileges ("use schema") to any
6+
-- user who wishes to use these in the future. Also, Hasura would require additional setup to be able to use the
7+
-- extensions from another schema.
8+
CREATE EXTENSION IF NOT EXISTS btree_gist WITH SCHEMA public;
9+
CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public;
10+
CREATE EXTENSION IF NOT EXISTS postgis WITH SCHEMA public;
11+
12+
-- Allow Hasura to connect and create new schemas.
13+
GRANT CONNECT, CREATE ON DATABASE xxx_db_hasura_name_xxx TO xxx_db_hasura_username_xxx;
14+
15+
-- Grant required privileges in the public schema to the Hasura user.
16+
GRANT ALL ON SCHEMA public TO xxx_db_hasura_username_xxx;
17+
GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_hasura_username_xxx;
18+
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_hasura_username_xxx;
19+
20+
-- Grant select permissions on information_schema and pg_catalog to the Hasura
21+
-- user.
22+
GRANT SELECT ON ALL TABLES IN SCHEMA information_schema TO xxx_db_hasura_username_xxx;
23+
GRANT SELECT ON ALL TABLES IN SCHEMA pg_catalog TO xxx_db_hasura_username_xxx;
24+
25+
-- Allow the JORE3-Importer role to connect to the network database.
26+
GRANT CONNECT ON DATABASE xxx_db_hasura_name_xxx TO xxx_db_jore3importer_username_xxx;
27+
28+
-- Grant the JORE3-Importer role to access the public schema and the objects and
29+
-- functions created by extensions. Other schema-specific privileges are granted
30+
-- in Hasura migrations.
31+
GRANT USAGE ON SCHEMA public TO xxx_db_jore3importer_username_xxx;
32+
GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_jore3importer_username_xxx;
33+
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_jore3importer_username_xxx;

‎joredb/migrations/azuredbmock/03-create-auth-database.sql‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
CREATE DATABASE xxx_db_auth_name_xxx;
2+
3+
-- Allow the auth role to connect and create new schemas.
4+
GRANT CONNECT, CREATE ON DATABASE xxx_db_auth_name_xxx TO xxx_db_auth_username_xxx;
5+
6+
\connect xxx_db_auth_name_xxx;
7+
8+
-- Make the JORE4 admin role the owner of the public schema.
9+
ALTER SCHEMA public OWNER TO CURRENT_USER;
10+
11+
-- Grant full schema access to the public schema to the auth role.
12+
GRANT ALL ON SCHEMA public TO xxx_db_auth_username_xxx;

‎joredb/migrations/azuredbmock/04-create-jore3importer-database.sql‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
CREATE DATABASE xxx_db_jore3importer_name_xxx;
2+
3+
-- Allow the jore3importer role to connect and create new schemas.
4+
GRANT CONNECT, CREATE ON DATABASE xxx_db_jore3importer_name_xxx TO xxx_db_jore3importer_username_xxx;
5+
6+
\connect xxx_db_jore3importer_name_xxx;
7+
8+
-- Make the JORE4 admin role the owner of the public schema.
9+
ALTER SCHEMA public OWNER TO CURRENT_USER;
10+
11+
-- Create the extensions that JORE3-Importer needs. In PostgreSQL v15 server,
12+
-- an ordinary user (without admin roles) may not be able to create extensions.
13+
CREATE EXTENSION IF NOT EXISTS btree_gist WITH SCHEMA public;
14+
CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public;
15+
16+
-- Grant privileges in the public schema to the jore3importer role.
17+
GRANT USAGE ON SCHEMA public TO xxx_db_jore3importer_username_xxx;
18+
GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_jore3importer_username_xxx;
19+
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_jore3importer_username_xxx;
20+
21+
-- Grant permission on the pg_catalog schema to conditionally create an
22+
-- extension if the extension is not already created. This must be granted so
23+
-- that the JORE3-Importer role can conditionally create the extension depending
24+
-- on which database is involved (there are several in the importer's test setup).
25+
GRANT SELECT ON TABLE pg_catalog.pg_extension TO xxx_db_jore3importer_username_xxx;

‎joredb/migrations/azuredbmock/05-create-timetables-database.sql‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
CREATE DATABASE xxx_db_timetables_name_xxx;
2+
3+
-- Allow Hasura to connect and create new schemas.
4+
GRANT CONNECT, CREATE ON DATABASE xxx_db_timetables_name_xxx TO xxx_db_hasura_username_xxx;
5+
6+
-- Interval outputs by default are using the sql format ('3 4:05:06'). Here we
7+
-- are switching to ISO 8601 format ('P3DT4H5M6S').
8+
ALTER DATABASE xxx_db_timetables_name_xxx SET intervalstyle = 'iso_8601';
9+
10+
-- Switch database context to be able to add extensions there.
11+
\connect xxx_db_timetables_name_xxx;
12+
13+
-- Make the JORE4 admin role the owner of the public schema.
14+
ALTER SCHEMA public OWNER TO CURRENT_USER;
15+
16+
CREATE EXTENSION IF NOT EXISTS btree_gist WITH SCHEMA public;
17+
CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public;
18+
19+
-- Grant required privileges in the public schema to Hasura.
20+
GRANT ALL ON SCHEMA public TO xxx_db_hasura_username_xxx;
21+
GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_hasura_username_xxx;
22+
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_hasura_username_xxx;
23+
24+
-- Grant select permissions on information_schema and pg_catalog to Hasura.
25+
GRANT SELECT ON ALL TABLES IN SCHEMA information_schema TO xxx_db_hasura_username_xxx;
26+
GRANT SELECT ON ALL TABLES IN SCHEMA pg_catalog TO xxx_db_hasura_username_xxx;
27+
28+
-- Allow the timetables-api role to connect to the timetables database.
29+
GRANT CONNECT ON DATABASE xxx_db_timetables_name_xxx TO xxx_db_timetables_api_username_xxx;
30+
31+
-- Grant the timetables-api role to access the public schema and the objects and
32+
-- functions created by extensions. Other schema-specific privileges are granted
33+
-- in Hasura migrations.
34+
GRANT USAGE ON SCHEMA public TO xxx_db_timetables_api_username_xxx;
35+
GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_timetables_api_username_xxx;
36+
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_timetables_api_username_xxx;

0 commit comments

Comments
 (0)