From eef9b31197f5781d6a2c0a44b270f67d8d059b32 Mon Sep 17 00:00:00 2001 From: Max Rieck <32903793+mrie04@users.noreply.github.com> Date: Tue, 7 Nov 2023 14:51:44 +0100 Subject: [PATCH] add postgres upgrade scripts (#90) --- .gitignore | 2 + docker-compose.yml | 1 + docker-compose_v3.yml | 137 +++++++++++ extras/db-9.5-to-15/Dockerfile | 36 +++ extras/db-9.5-to-15/docker-upgrade | 25 ++ extras/db15/Dockerfile | 12 + .../docker-compose-after-upgrade-postgres.yml | 14 ++ ...docker-compose-before-upgrade-postgres.yml | 21 ++ extras/docker-compose-upgrade-postgres.yml | 11 + extras/upgrade_postgres.sh | 226 ++++++++++++++++++ 10 files changed, 485 insertions(+) create mode 100644 docker-compose_v3.yml create mode 100644 extras/db-9.5-to-15/Dockerfile create mode 100644 extras/db-9.5-to-15/docker-upgrade create mode 100644 extras/db15/Dockerfile create mode 100644 extras/docker-compose-after-upgrade-postgres.yml create mode 100644 extras/docker-compose-before-upgrade-postgres.yml create mode 100644 extras/docker-compose-upgrade-postgres.yml create mode 100644 extras/upgrade_postgres.sh diff --git a/.gitignore b/.gitignore index 6571ec2..e8b6a2d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ **/*tar.gz **/*.pgdump **/*.swp +**/*.out +**/*.txt diff --git a/docker-compose.yml b/docker-compose.yml index 3d57460..f5e8681 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,6 @@ db: build: db + shm_size: 256m restart: always volumes: - ./volumes/db/data:/var/lib/postgresql/data diff --git a/docker-compose_v3.yml b/docker-compose_v3.yml new file mode 100644 index 0000000..8560cba --- /dev/null +++ b/docker-compose_v3.yml @@ -0,0 +1,137 @@ +version: '3' +services: + db: + build: db + shm_size: 256m + restart: always + volumes: + - ./volumes/db/data:/var/lib/postgresql/data + - ./volumes/db/log:/var/log/postgresql + - /etc/localtime:/etc/localtime:ro + - /etc/timezone:/etc/timezone:ro + environment: + - METASFRESH_USERNAME=metasfresh + - METASFRESH_PASSWORD=metasfresh + - METASFRESH_DBNAME=metasfresh + - DB_SYSPASS=System + - POSTGRES_PASSWORD=ip2LmNzXX8p8iXg9lZTWEJ9524kQDbXFudB7LR03T-xK9fLweX3TLMkA2AYcEiaS + logging: + options: + max-size: '30m' + max-file: '3' + driver: json-file + app: + build: app + hostname: app + links: + - db:db + - rabbitmq:rabbitmq + - search:search + expose: + - "8282" + - "8788" + restart: always + volumes: + - ./volumes/app/log:/opt/metasfresh/log:rw + - ./volumes/app/heapdump:/opt/metasfresh/heapdump:rw + - ./volumes/app/external-lib:/opt/metasfresh/external-lib:rw + - /etc/localtime:/etc/localtime:ro + - /etc/timezone:/etc/timezone:ro + environment: + - METASFRESH_HOME=/opt/metasfresh + logging: + options: + max-size: '30m' + max-file: '3' + driver: json-file + webapi: + build: webapi + links: + - app:app + - db:db + - rabbitmq:rabbitmq + - search:search + expose: + - "8789" + # to access the webui-api directly + # (eg. for debugging or connecting your app to the metasfresh api) + # uncomment following port: + #ports: + #- "8080:8080" + restart: always + volumes: + - ./volumes/webapi/log:/opt/metasfresh-webui-api/log:rw + - ./volumes/webapi/heapdump:/opt/metasfresh-webui-api/heapdump:rw + - /etc/localtime:/etc/localtime:ro + - /etc/timezone:/etc/timezone:ro + logging: + options: + max-size: '30m' + max-file: '3' + driver: json-file + webui: + build: webui + links: + - webapi:webapi + ports: + - "80:80" + - "443:443" + restart: always + volumes: + - /etc/localtime:/etc/localtime:ro + - /etc/timezone:/etc/timezone:ro + #uncomment and set to URL where metasfresh will be available from browsers + #environment: + #- WEBAPI_URL=http://example.com:8080 + logging: + options: + max-size: '30m' + max-file: '3' + driver: json-file + rabbitmq: + build: rabbitmq + expose: + - "5672" + restart: always + volumes: + - ./volumes/rabbitmq/log:/var/log/rabbitmq/log + - /etc/localtime:/etc/localtime:ro + - /etc/timezone:/etc/timezone:ro + environment: + RABBITMQ_DEFAULT_USER: "metasfresh" + RABBITMQ_DEFAULT_PASS: "metasfresh" + RABBITMQ_DEFAULT_VHOST: "/" + logging: + options: + max-size: '30m' + max-file: '3' + driver: json-file + search: + build: search + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + cap_add: + - IPC_LOCK + # to access the search api directly + # (e.g. if you did docker-compose up search to have the deachboard with your locally running metasfresh services) + # uncomment following ports: + # ports: + # - "9200:9200" + # - "9300:9300" + volumes: + - ./volumes/search/data:/usr/share/elasticsearch/data + - /etc/localtime:/etc/localtime:ro + - /etc/timezone:/etc/timezone:ro + environment: + - "ES_JAVA_OPTS=-Xms128M -Xmx256m" + restart: always + logging: + options: + max-size: '30m' + max-file: '3' + driver: json-file diff --git a/extras/db-9.5-to-15/Dockerfile b/extras/db-9.5-to-15/Dockerfile new file mode 100644 index 0000000..b9bec3e --- /dev/null +++ b/extras/db-9.5-to-15/Dockerfile @@ -0,0 +1,36 @@ +#thx to https://github.com/tianon/docker-postgres-upgrade/tree/master/9.6-to-15 + +FROM postgres:15.1 + +RUN sed -i 's/$/ 9.5/' /etc/apt/sources.list.d/pgdg.list + +RUN apt-get update && apt-get install -y --no-install-recommends \ + postgresql-9.5=9.5.25-6.pgdg110+1 \ + postgresql-contrib-9.5=9.5.25-6.pgdg110+1 \ + && rm -rf /var/lib/apt/lists/* + +RUN localedef -i de_DE -c -f UTF-8 -A /usr/share/locale/locale.alias de_DE.UTF-8 +RUN localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 + +ENV LANG=de_DE.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_MESSAGES=en_US.UTF-8 + +ENV PGBINOLD /usr/lib/postgresql/9.5/bin +ENV PGBINNEW /usr/lib/postgresql/15/bin + +ENV PGDATAOLD /var/lib/postgresql/9.5/data +ENV PGDATANEW /var/lib/postgresql/15/data + +RUN mkdir -p "$PGDATAOLD" "$PGDATANEW" \ + && chown -R postgres:postgres /var/lib/postgresql + +WORKDIR /var/lib/postgresql + +COPY docker-upgrade /usr/local/bin/ +RUN chmod +x /usr/local/bin/docker-upgrade + +ENTRYPOINT ["/usr/local/bin/docker-upgrade"] + +# recommended: --link +CMD ["pg_upgrade"] \ No newline at end of file diff --git a/extras/db-9.5-to-15/docker-upgrade b/extras/db-9.5-to-15/docker-upgrade new file mode 100644 index 0000000..68e6a1f --- /dev/null +++ b/extras/db-9.5-to-15/docker-upgrade @@ -0,0 +1,25 @@ +#!/bin/bash + +# thx to https://github.com/tianon/docker-postgres-upgrade/tree/master/9.6-to-15 + +set -e + +if [ "$#" -eq 0 -o "${1:0:1}" = '-' ]; then + set -- pg_upgrade "$@" +fi + +if [ "$1" = 'pg_upgrade' -a "$(id -u)" = '0' ]; then + mkdir -p "$PGDATAOLD" "$PGDATANEW" + chmod 700 "$PGDATAOLD" "$PGDATANEW" + chown postgres . + chown -R postgres "$PGDATAOLD" "$PGDATANEW" + exec gosu postgres "$BASH_SOURCE" "$@" +fi + +if [ "$1" = 'pg_upgrade' ]; then + if [ ! -s "$PGDATANEW/PG_VERSION" ]; then + PGDATA="$PGDATANEW" eval "initdb $POSTGRES_INITDB_ARGS" + fi +fi + +exec "$@" \ No newline at end of file diff --git a/extras/db15/Dockerfile b/extras/db15/Dockerfile new file mode 100644 index 0000000..131db59 --- /dev/null +++ b/extras/db15/Dockerfile @@ -0,0 +1,12 @@ +FROM postgres:15.1 + +RUN apt-get update + +RUN localedef -i de_DE -c -f UTF-8 -A /usr/share/locale/locale.alias de_DE.UTF-8 +RUN localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 + +ENV LANG=de_DE.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_MESSAGES=en_US.UTF-8 + +CMD ["postgres"] \ No newline at end of file diff --git a/extras/docker-compose-after-upgrade-postgres.yml b/extras/docker-compose-after-upgrade-postgres.yml new file mode 100644 index 0000000..1088a19 --- /dev/null +++ b/extras/docker-compose-after-upgrade-postgres.yml @@ -0,0 +1,14 @@ +version: '3' +services: + afterdb: + build: db15 + restart: unless-stopped + volumes: + - ./../volumes/newdb/data:/var/lib/postgresql/data + environment: + - POSTGRES_PASSWORD=ip2LmNzXX8p8iXg9lZTWEJ9524kQDbXFudB7LR03T-xK9fLweX3TLMkA2AYcEiaS + healthcheck: + test: [ "CMD-SHELL", "pg_isready -U metasfresh", "-d", "metasfresh" ] + interval: 30s + timeout: 60s + retries: 5 diff --git a/extras/docker-compose-before-upgrade-postgres.yml b/extras/docker-compose-before-upgrade-postgres.yml new file mode 100644 index 0000000..f05d45a --- /dev/null +++ b/extras/docker-compose-before-upgrade-postgres.yml @@ -0,0 +1,21 @@ +version: '3' +services: + beforedb: + build: ./../db + restart: unless-stopped + healthcheck: + test: [ "CMD-SHELL", "pg_isready -U metasfresh", "-d", "metasfresh" ] + interval: 30s + timeout: 60s + retries: 5 + volumes: + - ./../volumes/db/data:/var/lib/postgresql/data + - ./../volumes/db/log:/var/log/postgresql + - /etc/localtime:/etc/localtime:ro + - /etc/timezone:/etc/timezone:ro + environment: + - METASFRESH_USERNAME=metasfresh + - METASFRESH_PASSWORD=metasfresh + - METASFRESH_DBNAME=metasfresh + - DB_SYSPASS=System + - POSTGRES_PASSWORD=ip2LmNzXX8p8iXg9lZTWEJ9524kQDbXFudB7LR03T-xK9fLweX3TLMkA2AYcEiaS \ No newline at end of file diff --git a/extras/docker-compose-upgrade-postgres.yml b/extras/docker-compose-upgrade-postgres.yml new file mode 100644 index 0000000..f1bc825 --- /dev/null +++ b/extras/docker-compose-upgrade-postgres.yml @@ -0,0 +1,11 @@ +version: '3' +services: + upgradedb: + build: db-9.5-to-15 + image: upgradedb + restart: unless-stopped + volumes: + - ./../volumes/newdb/data:/var/lib/postgresql/15/data + - ./../volumes/db/data:/var/lib/postgresql/9.5/data + environment: + - POSTGRES_PASSWORD=ip2LmNzXX8p8iXg9lZTWEJ9524kQDbXFudB7LR03T-xK9fLweX3TLMkA2AYcEiaS diff --git a/extras/upgrade_postgres.sh b/extras/upgrade_postgres.sh new file mode 100644 index 0000000..193aa51 --- /dev/null +++ b/extras/upgrade_postgres.sh @@ -0,0 +1,226 @@ +#!/bin/bash + +#title :upgrade_postgres.sh +#description : +#author :max.rieck@metasfresh.com +#date :2022-11-29 +#usage :nohup bash ./upgrade_postgres.sh +#logs :tail -f nohup.out +# +# Prerequisites: Enough storage space to store a second db-volume +# +# unknown data-types aren't supported anymore, they should be altered before upgrade +# SELECT * +# FROM information_schema.columns +# WHERE data_type ILIKE 'unknown'; +# +# before starting set POSTGRES_PASSWORD in docker-compose-before-upgrade-postgres.yml +# before starting set POSTGRES_PASSWORD in docker-compose-upgrade-postgres.yml +# before starting set POSTGRES_PASSWORD in docker-compose-after-upgrade-postgres.yml +# +# ATTENTION: +# this script generates and executes SQL to alternate Tables with oids +#====================================================================================================== + +start=$(date +%s) + +instance=$(basename "${PWD%/*}") + +echo "stopping metasfresh instance $instance" +docker-compose -p "$instance" stop + +while [ "$(docker inspect -f '{{.State.Running}}' "${instance}"_db_1)" == "true" ] ; do + sleep 2 + echo "waiting for $instance to stop..." +done + +echo "waiting for $instance to stop... done" +echo "" + +echo "docker-compose up -f docker-compose-before-upgrade-postgres.yml -p ${instance}-before-upgrade-postgres" + +if ! docker-compose -f docker-compose-before-upgrade-postgres.yml -p "${instance}"-before-upgrade-postgres up -d; then + end=$(date +%s) + runtime=$((end-start)) + echo "script failed after ${runtime}s" + exit 1 +fi + +echo "starting postgres beforedb for handling tables with oids" + +while [ "$(docker inspect -f '{{.State.Health.Status}}' "${instance}"-before-upgrade-postgres-beforedb-1)" != "healthy" ]; do + echo "waiting for db..." + sleep 5 +done + +echo "waiting for db... done" +echo "" + +echo "alter known oid-tables to without oids" +docker exec -i "${instance}"-before-upgrade-postgres-beforedb-1 psql -U metasfresh -d metasfresh -tAc " +ALTER TABLE de_metas_acct.accounting_docs_to_repost SET WITHOUT OIDS; +ALTER TABLE public.c_invoice_candidate_recompute SET WITHOUT OIDS; +ALTER TABLE public.fact_acct_log SET WITHOUT OIDS; +" + +docker exec -i "${instance}"-before-upgrade-postgres-beforedb-1 psql -U metasfresh -d metasfresh -tAc " +SELECT quote_ident(nspname) || '.' || quote_ident(relname) +FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace +WHERE 1=1 + AND c.relkind = 'r' + AND c.relhasoids = true + AND n.nspname <> 'pg_catalog' +order by n.nspname, c.relname; +" >tables_with_oids.txt + +rm handle_tables_with_oids.sql 2> /dev/null + +while read -ru 3 p; do + nextOID=$(docker exec -i "${instance}"-before-upgrade-postgres-beforedb-1 pg_controldata -D /var/lib/postgresql/data | grep OID) + nextOID=${nextOID#"Latest checkpoint's NextOID:"} + nextOID=${nextOID// /} + + echo " + +ALTER TABLE $p DISABLE TRIGGER ALL; +ALTER TABLE $p ADD newoid bigint NOT NULL; +UPDATE $p SET newoid = oid; +ALTER TABLE $p SET WITHOUT OIDS; +ALTER TABLE $p RENAME newoid TO oid; +CREATE SEQUENCE IF NOT EXISTS oid_seq; +ALTER TABLE $p ALTER oid SET DEFAULT nextval('oid_seq'); +SELECT setval('oid_seq', $nextOID); +ALTER TABLE $p ENABLE TRIGGER ALL; + + " >> handle_tables_with_oids.sql +done 3< tables_with_oids.txt + +oidsql=$(cat handle_tables_with_oids.sql) +docker exec -i "${instance}"-before-upgrade-postgres-beforedb-1 psql -U metasfresh -d metasfresh -c "$oidsql" + +echo "docker stop ${instance}-before-upgrade-postgres-beforedb-1" +docker stop "${instance}"-before-upgrade-postgres-beforedb-1 + +while [ "$(docker inspect -f '{{.State.Running}}' "${instance}"-before-upgrade-postgres-beforedb-1)" == "true" ]; do + sleep 2 + echo "waiting for ${instance}-before-upgrade-postgres-beforedb-1 to stop..." +done + +echo "waiting for ${instance}-before-upgrade-postgres-beforedb-1 to stop... done" + +echo "docker rm ${instance}-before-upgrade-postgres-beforedb-1" +docker rm "${instance}"-before-upgrade-postgres-beforedb-1 + +echo "docker-compose up -f docker-compose-upgrade-postgres.yml -p ${instance}-upgrade-postgres" +docker-compose -f docker-compose-upgrade-postgres.yml -p "${instance}"-upgrade-postgres up + +docker exec -i -u postgres "${instance}"-upgrade-postgres-upgradedb-1 cp ./update_extensions.sql /var/lib/postgresql/15/data/update_extensions.sql + +upgradelogs=$(docker logs "${instance}"-upgrade-postgres-upgradedb-1 --tail 100) +grep -wq "Upgrade Complete" <<< "$upgradelogs" +upgradeComplete=$? + +echo "docker stop ${instance}-upgrade-postgres-upgradedb-1" +docker stop "${instance}"-upgrade-postgres-upgradedb-1 + +while [ "$(docker inspect -f '{{.State.Running}}' "${instance}"-upgrade-postgres-upgradedb-1)" == "true" ]; do + sleep 2 + echo "waiting for ${instance}-upgrade-postgres-upgradedb-1 to stop..." +done +echo "waiting for ${instance}-upgrade-postgres-upgradedb-1 to stop... done" +echo "" + +echo "docker rm ${instance}-upgrade-postgres-upgradedb-1" +docker rm "${instance}"-upgrade-postgres-upgradedb-1 + +if [ $upgradeComplete -ne 0 ]; then + echo "upgrade failed removing newdb volume..." + rm -r ./../volumes/newdb + end=$(date +%s) + runtime=$((end-start)) + echo "script failed after ${runtime}s" + exit 1 +fi + +echo "copying config files..." +cp ./../volumes/db/data/pg_hba.conf ./../volumes/newdb/data/pg_hba.conf +cp ./../volumes/db/data/postgresql.conf ./../volumes/newdb/data/postgresql.conf +cp ./../volumes/db/data/postgresql.auto.conf ./../volumes/newdb/data/postgresql.auto.conf +echo "copying config files... done" +echo "" + +docker-compose -f docker-compose-after-upgrade-postgres.yml -p "${instance}"-after-upgrade-postgres up -d + +echo "starting postgres afterdb for update extensions, vacuumdb and reindex" + +while [ "$(docker inspect -f '{{.State.Health.Status}}' "${instance}"-after-upgrade-postgres-afterdb-1)" != "healthy" ]; do + echo "waiting for db..." + sleep 5 +done + +echo "waiting for db... done" +echo "" + +echo "update extensions" +docker exec -i -u postgres "${instance}"-after-upgrade-postgres-afterdb-1 psql -U metasfresh -d metasfresh -f "/var/lib/postgresql/data/update_extensions.sql" +rm ./../volumes/newdb/data/update_extensions.sql +echo "" + +start_vacuumdb=$(date +%s) +echo "vacuumdb -d metasfresh --analyze-in-stages -v" +docker exec -i -u postgres "${instance}"-after-upgrade-postgres-afterdb-1 vacuumdb -d metasfresh --analyze-in-stages -v +end_vacuumdb=$(date +%s) +runtime_vacuumdb=$((end_vacuumdb-start_vacuumdb)) +echo "vacuumdb took ${runtime_vacuumdb}s" +echo "" + +start_reindex=$(date +%s) +echo "REINDEX each table of metasfresh database" +docker exec -i -u postgres "${instance}"-after-upgrade-postgres-afterdb-1 psql -U metasfresh -d metasfresh -tAc " +SELECT 'REINDEX (VERBOSE, CONCURRENTLY) TABLE ' || quote_ident(nspname) || '.' || quote_ident(relname) || ' /*' || pg_size_pretty(pg_total_relation_size(C.oid)) || '*/;' +FROM pg_class C +LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace) +WHERE nspname <> 'pg_catalog' + AND C.relkind = 'r' + AND nspname !~ '^pg_toast' +ORDER BY pg_total_relation_size(C.oid) ASC; +" > tables_to_reindex.txt + +#ignore table if needed +#grep -v "tablename" tables_to_reindex.txt > tmpfile && mv tmpfile tables_to_reindex.txt + +while read -ru 3 p; do + echo "$p" + docker exec -i -u postgres "${instance}"-after-upgrade-postgres-afterdb-1 psql -U metasfresh -d metasfresh -c "$p" + echo "" +done 3< tables_to_reindex.txt + +end_reindex=$(date +%s) +runtime_reindex=$((end_reindex-start_reindex)) +echo "REINDEX took ${runtime_reindex}s" +echo "" + +echo "docker stop afterdb" +docker stop "${instance}"-after-upgrade-postgres-afterdb-1 + +while [ "$(docker inspect -f '{{.State.Running}}' "${instance}"-after-upgrade-postgres-afterdb-1)" == "true" ]; do + sleep 2 + echo "waiting for ${instance}-after-upgrade-postgres-afterdb-1 to stop..." +done + +echo "waiting for ${instance}-after-upgrade-postgres-afterdb-1 to stop... done" + +echo "docker rm afterdb" +docker rm "${instance}"-after-upgrade-postgres-afterdb-1 + +mv ./../volumes/db ./../volumes/olddb +mv ./../volumes/newdb ./../volumes/db + +echo "" +echo "upgrade done! You can now change db image and restart the instance" +echo "" + +end=$(date +%s) +runtime=$((end-start)) +echo "script finished in ${runtime}s"