From 77dbe1fb9a3f920f9fef7575d9d60ea9cd2c62d0 Mon Sep 17 00:00:00 2001 From: romainsacchi Date: Mon, 2 Nov 2026 20:25:26 +0100 Subject: [PATCH] Deploy curated CLIC catalogue through a gated startup update --- stacks/clic/compose.yaml | 77 +++++++++++++++++++++++++++--- stacks/clic/curated-update.md | 90 +++++++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+), 7 deletions(-) create mode 100644 stacks/clic/curated-update.md diff --git a/stacks/clic/compose.yaml b/stacks/clic/compose.yaml index 0c8ea15..aeb117d 100644 --- a/stacks/clic/compose.yaml +++ b/stacks/clic/compose.yaml @@ -1,9 +1,52 @@ # Included by ../../docker-compose.yaml; use the root Compose entrypoint. services: + # One-time approved catalogue update. Complete the maintenance/backup steps + # in curated-update.md before merging: main is deployed automatically. + clic-curated-update: + image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:curated-v1-20261102@sha256:977313385ae4836c85175ed23ebd7ff84cf7164add67bb64aa3c42a39e7e3917} + container_name: clic-curated-update + restart: "no" + env_file: + - /mount/webhosting_dmz_write/clic/clic.env + entrypoint: ["python", "manage.py", "deploy_curated_catalog_update"] + command: [] + environment: + - DJANGO_SETTINGS_MODULE=config.settings.production + - POSTGRES_HOST=clic-db + - POSTGRES_PORT=5432 + - CLIC_CACHE_BACKEND=redis + - CLIC_REDIS_URL=redis://clic-redis:6379/0 + - CLIC_REQUIRE_SHARED_CACHE=1 + - CLIC_CURATED_UPDATE_ACTOR=${CLIC_CURATED_UPDATE_ACTOR:-romainsacchi} + - CLIC_CURATED_UPDATE_REPORTS=/var/lib/clic/curated-updates + - CLIC_BOOTSTRAP_ADMINS_ENABLED=0 + - BRIGHTPATH_SOURCE_DIR= + - BRIGHTWAY2_DIR=/tmp + - INVENTORY_ARCHIVE_ENVIRONMENT=production + - INVENTORY_ARCHIVE_AUTO_RESTORE=0 + - INVENTORY_ARCHIVE_PUBLISH_MODE=off + - INVENTORY_ARCHIVE_TERMS_REVIEWED=0 + - INVENTORY_ARCHIVE_TOKEN= + - INVENTORY_ARCHIVE_OWNER_ID=86122 + - INVENTORY_ARCHIVE_SERIES_ID=22850223 + - INVENTORY_ARCHIVE_BOOTSTRAP_RECORD_ID= + - INVENTORY_ARCHIVE_BOOTSTRAP_SHA256= + - IMPACT_SCORES_ENABLED=0 + - EMAIL_BACKEND=django.core.mail.backends.locmem.EmailBackend + volumes: + - clic_curated_update_reports:/var/lib/clic/curated-updates + depends_on: + clic-db: + condition: service_healthy + clic-redis: + condition: service_started + networks: + - clic_backend + # Romain Sacchi # CLIC Django platform clic-web: - image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:dab21a5-missing-foreground-2a1fe954da76@sha256:1e13386f3a435681be61900707f11f858faee22171e644912bba7b7df3a19fd2} + image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:curated-v1-20261102@sha256:977313385ae4836c85175ed23ebd7ff84cf7164add67bb64aa3c42a39e7e3917} container_name: clic-web restart: always env_file: @@ -15,6 +58,8 @@ services: - CLIC_CACHE_BACKEND=redis - CLIC_REDIS_URL=redis://clic-redis:6379/0 - CLIC_REQUIRE_SHARED_CACHE=1 + - CLIC_CURATED_UPDATE_REQUIRED=v1 + - BRIGHTPATH_SOURCE_DIR= - INVENTORY_ARCHIVE_ENVIRONMENT=production - INVENTORY_ARCHIVE_AUTO_RESTORE=1 - INVENTORY_ARCHIVE_PUBLISH_MODE=off @@ -28,6 +73,8 @@ services: - /mount/webhosting_dmz_write/clic/media:/app/media - /mount/webhosting_dmz_write/clic/impact_factors:/app/data/impact_factors:ro depends_on: + clic-curated-update: + condition: service_completed_successfully clic-db: condition: service_healthy clic-redis: @@ -48,7 +95,7 @@ services: - clic_backend clic-package-exports: - image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:dab21a5-missing-foreground-2a1fe954da76@sha256:1e13386f3a435681be61900707f11f858faee22171e644912bba7b7df3a19fd2} + image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:curated-v1-20261102@sha256:977313385ae4836c85175ed23ebd7ff84cf7164add67bb64aa3c42a39e7e3917} container_name: clic-package-exports restart: always env_file: @@ -60,15 +107,19 @@ services: - CLIC_CACHE_BACKEND=redis - CLIC_REDIS_URL=redis://clic-redis:6379/0 - CLIC_REQUIRE_SHARED_CACHE=1 + - CLIC_CURATED_UPDATE_REQUIRED=v1 + - BRIGHTPATH_SOURCE_DIR= - INVENTORY_ARCHIVE_AUTO_RESTORE=0 - INVENTORY_ARCHIVE_PUBLISH_MODE=off - INVENTORY_ARCHIVE_TOKEN= volumes: - /mount/webhosting_dmz_write/clic/media:/app/media - /mount/webhosting_dmz_write/clic/impact_factors:/app/data/impact_factors:ro - entrypoint: ["python"] + entrypoint: ["sh", "/app/scripts/curated-worker-entrypoint.sh"] command: ["manage.py", "process_package_exports"] depends_on: + clic-curated-update: + condition: service_completed_successfully clic-web: condition: service_healthy clic-db: @@ -79,7 +130,7 @@ services: - clic_backend clic-brightway-json-exports: - image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:dab21a5-missing-foreground-2a1fe954da76@sha256:1e13386f3a435681be61900707f11f858faee22171e644912bba7b7df3a19fd2} + image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:curated-v1-20261102@sha256:977313385ae4836c85175ed23ebd7ff84cf7164add67bb64aa3c42a39e7e3917} container_name: clic-brightway-json-exports restart: always env_file: @@ -91,13 +142,15 @@ services: - CLIC_CACHE_BACKEND=redis - CLIC_REDIS_URL=redis://clic-redis:6379/0 - CLIC_REQUIRE_SHARED_CACHE=1 + - CLIC_CURATED_UPDATE_REQUIRED=v1 + - BRIGHTPATH_SOURCE_DIR= - INVENTORY_ARCHIVE_AUTO_RESTORE=0 - INVENTORY_ARCHIVE_PUBLISH_MODE=off - INVENTORY_ARCHIVE_TOKEN= volumes: - /mount/webhosting_dmz_write/clic/media:/app/media - /mount/webhosting_dmz_write/clic/impact_factors:/app/data/impact_factors:ro - entrypoint: ["python"] + entrypoint: ["sh", "/app/scripts/curated-worker-entrypoint.sh"] command: [ "manage.py", @@ -107,6 +160,8 @@ services: "${CLIC_API_EXPORT_POLL_SECONDS:-2}", ] depends_on: + clic-curated-update: + condition: service_completed_successfully clic-web: condition: service_healthy clic-db: @@ -117,7 +172,7 @@ services: - clic_backend clic-inventory-archive: - image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:dab21a5-missing-foreground-2a1fe954da76@sha256:1e13386f3a435681be61900707f11f858faee22171e644912bba7b7df3a19fd2} + image: ${CLIC_IMAGE:-gitea.psi.ch/images/clic:curated-v1-20261102@sha256:977313385ae4836c85175ed23ebd7ff84cf7164add67bb64aa3c42a39e7e3917} container_name: clic-inventory-archive restart: always env_file: @@ -129,6 +184,8 @@ services: - CLIC_CACHE_BACKEND=redis - CLIC_REDIS_URL=redis://clic-redis:6379/0 - CLIC_REQUIRE_SHARED_CACHE=1 + - CLIC_CURATED_UPDATE_REQUIRED=v1 + - BRIGHTPATH_SOURCE_DIR= - IMPACT_SCORES_ENABLED=0 - INVENTORY_ARCHIVE_ENVIRONMENT=production - INVENTORY_ARCHIVE_AUTO_RESTORE=0 @@ -141,9 +198,11 @@ services: - INVENTORY_ARCHIVE_METADATA_FILE=/app/config/inventory-archive-metadata.json - INVENTORY_ARCHIVE_REDISTRIBUTION_FILE= - INVENTORY_ARCHIVE_ALERT_EMAIL=romain.sacchi@psi.ch - entrypoint: ["python"] + entrypoint: ["sh", "/app/scripts/curated-worker-entrypoint.sh"] command: ["manage.py", "run_inventory_archive_worker"] depends_on: + clic-curated-update: + condition: service_completed_successfully clic-web: condition: service_healthy healthcheck: @@ -184,3 +243,7 @@ services: - /mount/webhosting_dmz_write/clic/redis:/data networks: - clic_backend + +volumes: + # Private operator receipts; keep outside the public media mount. + clic_curated_update_reports: diff --git a/stacks/clic/curated-update.md b/stacks/clic/curated-update.md new file mode 100644 index 0000000..c8048a1 --- /dev/null +++ b/stacks/clic/curated-update.md @@ -0,0 +1,90 @@ +# CLIC curated catalogue deployment + +This release includes a one-time catalogue update. The image contains the exact +approved archive and operation plan, with BrightPath 1.0.0a14. No separate payload +transfer or manual Django command is required. + +## Coordinate maintenance before merging + +Merges to `main` trigger the shared automatic deployment. Arrange the CLIC +maintenance window and private recovery point **before merging this PR**. +The update service does not stop old containers or make the infrastructure backup. +A normal rolling `compose up` alone is insufficient while old writers remain active. + +1. Record the old image/configuration, place CLIC in maintenance, and drain active + requests. From `/opt/WebHosting_DMZ`, stop the old CLIC writers: + + ```sh + docker compose stop clic-inventory-archive clic-package-exports clic-brightway-json-exports clic-web + ``` + + Pause any external CLIC schedules and manual writers too. Keep PostgreSQL, + Redis and unrelated applications running. +2. Under that freeze, take and verify the usual full private PostgreSQL and + matching media backup. Keep the previous image/configuration for recovery. +3. Check that the deployment `.env` has no stale `CLIC_IMAGE` override; if it does, + set it to the new image reference in `compose.yaml`. The default now pins the + published curated image by digest. `CLIC_CURATED_UPDATE_ACTOR` defaults to the + existing `romainsacchi` editor/admin account; use the actual editor username if + different. No account, role or private rights confirmation is created. +4. Merge during the agreed window and let the existing deployment run. Allow a + longer maintenance window than usual: `clic-curated-update` runs migrations, + checks the actual catalogue and applies the approved changes in one data + transaction. Schema migrations are separate. Any data conflict fails startup + instead of being resolved automatically. No duration on PSI hardware is assumed. +5. Confirm successful updater exit and healthy web/worker services: + + ```sh + docker compose ps -a clic-curated-update clic-web clic-package-exports clic-brightway-json-exports clic-inventory-archive + docker compose logs clic-curated-update + ``` + + Inspect representative dataset pages and authenticated downloads before + reopening CLIC. Graph building and enabled impact-score refresh run through + normal web startup in the background. Resume external schedules deliberately. + +## Update behavior + +The reviewed update adds 169 datasets and 1,738 versions and withdraws 41 datasets +without hard deletion. Existing scientific versions, users, private workflows +and original files remain stored. The updater mounts only a separate private +report volume; it does not mount or rewrite original uploads. Reports contain +internal identifiers and must not be exposed through public media. + +Web, both export workers **and the archive worker** wait for update completion. +Their startup gates also verify the persisted database receipt. Subsequent runs +skip the completed update, allowing later legitimate editorial changes. The +workers retain their existing commands, web-health dependencies and configuration; +the archive worker retains scheduled publication and its existing owner/series. +Only the updater disables mail, impacts, archive publication and auto-restore. +All application processes use the installed BrightPath release. + +The existing last-backup fingerprint is preserved. After the update, the archive +worker detects the changed catalogue through the normal scheduled backup path. + +On failure, retain the writer freeze and inspect logs and private reports before +retrying. A failed shell command does not by itself prove the data transaction +rolled back; the database completion receipt is authoritative. Do not delete the +catalogue, force mismatched revisions, reset archive state or replace the private +database with a public Zenodo ZIP. For wider schema/deployment recovery, restore +the full private database/media recovery point with its matching old image under +the usual staff procedure. + +## Validation + +The exact published Linux/AMD64 image passed a full update on a fresh local +PostgreSQL restore with synthetic private records: 3,916 original scientific +versions and 16 private fixture rows were preserved, 3,505 public datasets and +5,597 public versions exported, and current catalogue heads had no missing +foreground dependencies. Repeat execution changed no managed-model rows. Missing +completion blocked web startup; the updated web container passed live/ready +health checks and built a graph of 3,505 datasets. Focused deployment/startup +checks: 48 passed; earlier separate browser suite: 73 passed. + +The complete CLIC verification suite remains non-green: 7,363 passed, 48 failed, +79 errors and eight skipped, with exactly the same 127 historical audit failure +names as before this deployment change. These mostly involve older runtime pins, +missing historical inputs and receipt hashes; they were not hidden or waived. +The actual private PSI catalogue has not been rehearsed locally; its state is +checked conservatively by the deployment preflight, and infrastructure backup +and reopening checks remain with PSI staff.