Improve DB migration

Author: richturner

Dockerfile

@@ -1,6 +1,20 @@
 # -----------------------------------------------------------------------------------------------
 # POSTGIS and TimescaleDB (inc. toolkit for hyperfunctions) image built for aarch64 support
-# using alpine base image.
+# using timescaledev/timescaledb-ha base image with:
+#
+# - OR specific ENV variables and a healthcheck added
+# - PGDATA path set to match old Alpine image (for ease of DB migration)
+# - POSTGRES user UID and GID changed to match old Alpine image (for ease of DB migration)
+# - OR_DISABLE_REINDEX env variable with associated scripts to determine if a REINDEX of the entire DB should be carried
+#   out at first startup with existing DB (checks whether or not $PGDATA/OR_REINDEX_COUNTER.$OR_REINDEX_COUNTER exists).
+#   This is used when a collation change has occurred (glibc version change, muslc <-> glibc) which can break the indexes;
+#   migration can either be manually handled or auto handled depending on OR_DISABLE_REINDEX env variable value.
+#   NOTE THAT A REINDEX CAN TAKE A LONG TIME DEPENDING ON THE SIZE OF THE DB! And startup will be delayed until completed
+#   This functionality is intended to simplify migration for basic users; advanced users with large DBs should take care of this
+#   themselves.
+#
+#
+#
 # 
 # timescale/timescaledb-ha image is ubuntu based and only currently supports amd64; they are
 # working on ARM64 support in timescaledev/timescaledb-ha see:
@@ -25,18 +39,33 @@ RUN usermod -u 70 postgres \
  && (find / -group 1000 -exec chgrp -h postgres {} \; || true) \
  && (find / -user 1000 -exec chown -h postgres {} \; || true)
 
-
 # Set PGDATA to the same location as our old alpine image
 RUN mkdir -p /var/lib/postgresql && mv /home/postgres/pgdata/* /var/lib/postgresql/ && chown -R postgres:postgres /var/lib/postgresql
 
-# Below is copied from https://github.com/timescale/timescaledb-docker-ha/blob/master/Dockerfile
-# to minimise the size of this image
+# Add custom entry point (see file header for details)
+COPY or-entrypoint.sh /
+RUN chmod +x /or-entrypoint.sh
+
+# Add custom initdb script(s)
+COPY docker-entrypoint-initdb.d/ /docker-entrypoint-initdb.d/
+RUN chmod +x /docker-entrypoint-initdb.d/*
+
+
+# Below is mostly copied from https://github.com/timescale/timescaledb-docker-ha/blob/master/Dockerfile (with OR specific entrypoint,
+# workdir and OR env defaults)
+
+
 ## Create a smaller Docker image from the builder image
 FROM scratch
 COPY --from=trimmed / /
 
 ARG PG_MAJOR=14
-ENTRYPOINT ["/docker-entrypoint.sh"]
+
+# Increment this to indicate that a re-index should be carried out on first startup with existing data; REINDEX can still be overidden
+# with OR_DISABLE_REINDEX=true
+ARG OR_REINDEX_COUNTER=1
+
+ENTRYPOINT ["/or-entrypoint.sh"]
 CMD ["postgres"]
 
 ENV PGROOT=/var/lib/postgresql \
@@ -50,17 +79,18 @@ ENV PGROOT=/var/lib/postgresql \
     LC_ALL=C.UTF-8 \
     LANG=C.UTF-8 \
     # When having an interactive psql session, it is useful if the PAGER is disable
-    PAGER=""
+    PAGER="" \
+    # OR ENV DEFAULTS
+    TZ=${TZ:-Europe/Amsterdam} \
+    PGTZ=${PGTZ:-Europe/Amsterdam} \
+    POSTGRES_DB=${POSTGRES_DB:-openremote} \
+    POSTGRES_USER=${POSTGRES_USER:-postgres} \
+    POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} \
+    OR_REINDEX_COUNTER=${OR_REINDEX_COUNTER} \
+    OR_DISABLE_REINDEX=${OR_DISABLE_REINDEX:-false}
 
 WORKDIR /var/lib/postgresql
 EXPOSE 5432 8008 8081
 USER postgres
 
-## ADD OR convenience default ENV values and healthcheck
-ENV TZ ${TZ:-Europe/Amsterdam}
-ENV PGTZ ${PGTZ:-Europe/Amsterdam}
-ENV POSTGRES_DB ${POSTGRES_DB:-openremote}
-ENV POSTGRES_USER ${POSTGRES_USER:-postgres}
-ENV POSTGRES_PASSWORD ${POSTGRES_PASSWORD:-postgres}
-
 HEALTHCHECK --interval=3s --timeout=3s --start-period=2s --retries=30 CMD pg_isready

docker-entrypoint-initdb.d/100_or_create_reindex_file.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+# THIS SCRIPT JUST CREATES THE REINDEX_FILE WHICH WILL PREVENT RE-INDEXING FOR HAPPENING ON A NEWLY INITIALISED DB
+
+
+if [ -n "$OR_REINDEX_COUNTER" ]; then
+    REINDEX_FILE="$PGDATA/OR_REINDEX_COUNTER.$OR_REINDEX_COUNTER"
+    echo "Creating REINDEX_FILE '$REINDEX_FILE'"...
+    touch "$REINDEX_FILE"
+fi

initdb-postgis.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+# THIS FILE IS FOR MIGRATION OF EXISTING DB TO TIMESCALEDB IMAGE AS TIMESCALE INIT SCRIPTS AREN'T RUN WHEN DB
+# ALREADY EXISTS; IT ALSO DOES AN AUTOMATIC REINDEX OF THE DB WHEN OR_REINDEX_COUNTER changes TO SIMPLIFY MIGRATIONS
+
+source /docker-entrypoint.sh
+docker_setup_env
+
+if [ -n "$DATABASE_ALREADY_EXISTS" ]; then
+
+    # Make sure timescaledb library is set to preload (won't work otherwise)
+    echo "Existing postgresql.conf found checking for shared_preload_libraries = 'timescaledb'..."       
+    RESULT=$(cat "$PGDATA/postgresql.conf" | grep "^shared_preload_libraries = 'timescaledb'" || true)
+
+    if [ -n "$RESULT" ]; then
+        echo "Timescale DB library already set to preload"
+    else
+        echo "Adding shared_preload_libraries = 'timescaledb' to postgresql.conf"
+        echo "shared_preload_libraries = 'timescaledb'" >> "$PGDATA/postgresql.conf"
+    fi
+    
+    # Do re-indexing check
+    if [ "$OR_DISABLE_REINDEX" == 'true' ] || [ -z "$OR_REINDEX_COUNTER" ]; then
+        echo "REINDEX check is disabled"
+    else
+        echo "Checking whether REINDEX is required..."
+        REINDEX_FILE="$PGDATA/OR_REINDEX_COUNTER.$OR_REINDEX_COUNTER"
+        if [ -f "$REINDEX_FILE" ]; then
+            echo "REINDEX file '$REINDEX_FILE' already exists so no re-indexing required"
+        else
+            echo "REINDEX file '$REINDEX_FILE' doesn't exist so re-indexing the DB..."
+            docker_temp_server_start "$@"
+            docker_process_sql -c "REINDEX database $POSTGRES_DB;"
+            docker_temp_server_stop
+            echo 'REINDEX completed!'
+            touch "$REINDEX_FILE"
+        fi
+    fi
+fi
+
+exec /docker-entrypoint.sh $@