Adding time to release snapshot.

all-apps/docker-compose.yaml

@@ -1,7 +1,7 @@
 version: '3'
 services:
   lb:
-    image: docker.io/caddy:2.10.2-alpine
+    image: docker.io/caddy:2.11.4-alpine
     volumes:
 #      - /app/lb:/etc/caddy
       - ./lb/:/etc/caddy

all-apps/ghost/docker-compose.yaml

@@ -36,7 +36,7 @@ services:
       - lb
 
   ghost_db:
-    image: mysql:8.0.44@sha256:f37951fc3753a6a22d6c7bf6978c5e5fefcf6f31814d98c582524f98eae52b21
+    image: mysql:9.7
     restart: always
     expose:
       - "3306"
@@ -59,7 +59,7 @@ services:
       - ghost_network
 
   ghost_traffic-analytics:
-    image: ghost/traffic-analytics:1.0.20@sha256:a72573d89457e778b00e9061422516d2d266d79a72a0fc02005ba6466e391859
+    image: ghost/traffic-analytics:1.0
     restart: always
     expose:
       - "3000"

all-apps/instance-control/hooks/hooks.json.tmpl

@@ -20,6 +20,27 @@
             },
         "execute-command": "/etc/webhook/queue-restic-snapshot.sh"
     },
+    {
+        "id": "queue-restic-snapshot-no-restart",
+        "pass-environment-to-command": [
+            {"source": "payload", "name": "version"},
+            {"source": "payload", "name": "path"},
+            {"source": "payload", "name": "tag"},
+            {"source": "payload", "name": "request_id"}
+        ],
+        "trigger-rule":
+            {
+                "match": {
+                    "type": "payload-hmac-sha256",
+                    "secret": "$INSTANCE_CONTROL_WEBHOOKS_SECRET",
+                    "parameter": {
+                        "source": "header",
+                        "name": "X-Nassella-Signature"
+                    }
+                }
+            },
+        "execute-command": "/etc/webhook/queue-restic-snapshot-no-restart.sh"
+    },
     {
         "id": "restic-snapshot-status",
         "include-command-output-in-response": true,

all-apps/instance-control/hooks/queue-restic-snapshot-no-restart.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+# TODO the systemd unit should actually do this
+# touch /maintenance/maintenance.on
+# rm /maintenance/maintenance.on
+
+# for instance-control docker compose setup:
+# make a directory in /tmp for these pipes and mount that as a volume
+# into the container
+
+# TODO read 'version' arg from request and make sure it
+# matches the version of this script
+
+# use a named pipe
+printf "%s\t%s\t%s\t%s\n" "$HOOK_tag" "$HOOK_request_id" "$HOOK_path" "false" > /tmp/restic/snapshot_trigger_pipe

all-apps/instance-control/hooks/queue-restic-snapshot.sh

@@ -12,4 +12,4 @@
 # matches the version of this script
 
 # use a named pipe
-printf "%s\t%s\t%s\n" "$HOOK_tag" "$HOOK_request_id" "$HOOK_path" > /tmp/restic/snapshot_trigger_pipe
+printf "%s\t%s\t%s\t%s\n" "$HOOK_tag" "$HOOK_request_id" "$HOOK_path" "true" > /tmp/restic/snapshot_trigger_pipe

all-apps/nassella/docker-compose.yaml

@@ -26,7 +26,7 @@ secrets:
 
 services:
   nassella_lldap_db:
-    image: postgres:17.6-trixie
+    image: postgres:18-trixie
     environment:
       - POSTGRES_DB_FILE=/run/secrets/nassella_lldap_postgres_db
       - POSTGRES_USER_FILE=/run/secrets/nassella_lldap_postgres_user
@@ -34,7 +34,7 @@ services:
     shm_size: 128mb
     restart: always
     volumes:
-      - /nassella/nassella/lldap-var-lib-postgresql-data:/var/lib/postgresql/data
+      - /nassella/nassella/lldap-var-lib-postgresql:/var/lib/postgresql
     networks:
       - nassella_internal
     healthcheck:
@@ -64,7 +64,7 @@ services:
       - nassella_lldap_postgres_user
 
   nassella_authelia_db:
-    image: postgres:17.6-trixie
+    image: postgres:18-trixie
     environment:
       - POSTGRES_DB_FILE=/run/secrets/nassella_authelia_postgres_db
       - POSTGRES_USER_FILE=/run/secrets/nassella_authelia_postgres_user
@@ -72,7 +72,7 @@ services:
     shm_size: 128mb
     restart: always
     volumes:
-      - /nassella/nassella/authelia-var-lib-postgresql-data:/var/lib/postgresql/data
+      - /nassella/nassella/authelia-var-lib-postgresql:/var/lib/postgresql
     networks:
       - nassella_internal
     healthcheck:
@@ -104,13 +104,13 @@ services:
       disable: true
 
   nassella_db:
-    image: postgres:17.6-trixie
+    image: postgres:18-trixie
     env_file:
       - ./nassella/nassella.env
     shm_size: 128mb
     restart: always
     volumes:
-      - /nassella/nassella/var-lib-postgresql-data:/var/lib/postgresql/data
+      - /nassella/nassella/var-lib-postgresql:/var/lib/postgresql
     networks:
       - nassella_internal
     healthcheck:

all-apps/nextcloud/docker-compose.yaml

@@ -12,13 +12,13 @@ secrets:
 
 services:
   nextcloud_db:
-    image: postgres:17.6-trixie
+    image: postgres:18-trixie
     env_file:
       - ./nextcloud/nextcloud.env
     shm_size: 128mb
     restart: always
     volumes:
-      - /nassella/nextcloud/var-lib-postgresql-data:/var/lib/postgresql/data
+      - /nassella/nextcloud/var-lib-postgresql:/var/lib/postgresql
     networks:
       - nextcloud_internal
     healthcheck:
@@ -32,7 +32,7 @@ services:
       - nextcloud_postgres_password
       - nextcloud_postgres_user
   nextcloud_redis:
-    image: redis:8.2.1-bookworm
+    image: redis:8-trixie
     env_file:
       - ./nextcloud/nextcloud.env
     command: bash -c 'redis-server --requirepass "$$(cat /run/secrets/nextcloud_redis_password)"'
@@ -48,7 +48,7 @@ services:
     networks:
       - nextcloud_internal
   nextcloud:
-    image: nextcloud:31.0.8-apache
+    image: nextcloud:34-apache
     depends_on:
       nextcloud_redis:
         condition: service_healthy

cl.yaml

@@ -129,10 +129,10 @@ storage:
           AllowUsers core
     ### docker-compose sysext
     ### https://flatcar.github.io/sysext-bakery/docker_compose/
-    - path: /opt/extensions/docker-compose/docker-compose-2.34.0-x86-64.raw
+    - path: /opt/extensions/docker-compose/docker-compose-5.1.4-x86-64.raw
       mode: 0644
       contents:
-        source: https://extensions.flatcar.org/extensions/docker-compose-2.34.0-x86-64.raw
+        source: https://extensions.flatcar.org/extensions/docker-compose-5.1.4-x86-64.raw
     - path: /etc/sysupdate.docker-compose.d/docker-compose.conf
       contents:
         source: https://extensions.flatcar.org/extensions/docker-compose.conf
@@ -140,6 +140,6 @@ storage:
       contents:
         source: https://extensions.flatcar.org/extensions/noop.conf
   links:
-    - target: /opt/extensions/docker-compose/docker-compose-2.34.0-x86-64.raw
+    - target: /opt/extensions/docker-compose/docker-compose-5.1.4-x86-64.raw
       path: /etc/extensions/docker-compose.raw
       hard: false

config/production.tfvars.tmpl

@@ -2,6 +2,8 @@ server_type = "s-2vcpu-2gb" # the digital ocean server type to deploy
 
 do_token = "" # token from "API" settings on DigitalOcean
 
+digitalocean_volume_size = # size in GB of the app storage volume
+
 cloudflare_api_token = "" # corresponding API token should allow modifying DNS settings for the Nassella configured domain
 cloudflare_zone_id = "" # corresponding zone ID for API token for the Nassella configured domain
 cloudflare_account_id = "" # corresponding account ID for API token

main.tf

@@ -79,6 +79,11 @@ variable "subdomains" {
   description = "Subdomains to setup"
 }
 
+variable "digitalocean_volume_size" {
+  type = number
+  description = "Size in GB of the app storage digitalocean volume"
+}
+
 provider "digitalocean" {
   token = var.do_token
 }
@@ -124,7 +129,7 @@ resource "cloudflare_dns_record" "subdomains" {
 resource "digitalocean_volume" "machine" {
   region    = var.datacenter
   name      = "${var.cluster_name}"
-  size                    = 60
+  size                    = var.digitalocean_volume_size
   initial_filesystem_type = "ext4"
   initial_filesystem_label = "appstorage"
   description             = "persistent storage for docker apps"
@@ -137,7 +142,6 @@ resource "digitalocean_droplet" "machine" {
   size      = var.server_type
   ssh_keys  = [digitalocean_ssh_key.first.fingerprint]
   user_data = file("ignition.json")
-  graceful_shutdown = true
   lifecycle {
     create_before_destroy = true
   }

make-caddyfile.sh

@@ -66,13 +66,15 @@ for config_string in ${APP_CONFIGS[@]}; do
     fulldomain="$subdomain.$ROOT_DOMAIN"
 
     echo "$fulldomain {"
-    # config for maintenance mode
-    echo "@maintenanceModeActive file /maintenance/maintenance.on {"
-    echo "  root /"
-    echo "}"
-    echo "handle @maintenanceModeActive {"
-    echo "  respond \"We are performing a maintenance, come back later\" 503"
-    echo "}"
+    if [ "$app" != "instance-control" ] && [ "$app" != "dozzle" ]; then
+        # config for maintenance mode
+        echo "@maintenanceModeActive file /maintenance/maintenance.on {"
+        echo "  root /"
+        echo "}"
+        echo "handle @maintenanceModeActive {"
+        echo "  respond \"We are performing a maintenance, come back later\" 503"
+        echo "}"
+    fi
 
     echo $body
     echo "}"

restic-snapshot.sh

@@ -25,6 +25,7 @@ while read -u 3 msg; do
     tag=$1
     request_id=$2
     path=$3 # TODO not currently used
+    restart=$4 # if we should restart the databases after the snapshot or stay in maintenance mode
 
     # update status for webhooks
     printf "%s\n" "running" > "/tmp/restic/snapshot_status_$request_id"
@@ -60,27 +61,29 @@ while read -u 3 msg; do
     # perform the snapshot
     docker run --rm --volume /nassella:/nassella --volume /restic-password:/restic-password -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} -i restic/restic:0.18.0 backup --verbose --repo s3:${BACKBLAZE_BUCKET_URL} --password-file /restic-password --tag "$tag" "/nassella"
 
-    # restart databases
-    if [ $ghost_db_running = true ]; then
-        docker start app-ghost_db-1
-    fi
-    if [ $nassella_lldap_db_running = true ]; then
-        docker start app-nassella_lldap_db-1
-    fi
-    if [ $nassella_authelia_db_running = true ]; then
-        docker start app-nassella_authelia_db-1
-    fi
-    if [ $nassella_db_running = true ]; then
-        docker start app-nassella_db-1
-    fi
-    if [ $nextcloud_db_running = true ]; then
-        docker start app-nextcloud_db-1
-    fi
-    if [ $nextcloud_redis_running = true ]; then
-        docker start app-nextcloud_redis-1
-    fi
+    if [ $restart = "true" ]; then
+        # restart databases
+        if [ $ghost_db_running = true ]; then
+            docker start app-ghost_db-1
+        fi
+        if [ $nassella_lldap_db_running = true ]; then
+            docker start app-nassella_lldap_db-1
+        fi
+        if [ $nassella_authelia_db_running = true ]; then
+            docker start app-nassella_authelia_db-1
+        fi
+        if [ $nassella_db_running = true ]; then
+            docker start app-nassella_db-1
+        fi
+        if [ $nextcloud_db_running = true ]; then
+            docker start app-nextcloud_db-1
+        fi
+        if [ $nextcloud_redis_running = true ]; then
+            docker start app-nextcloud_redis-1
+        fi
 
-    rm /app/maintenance/maintenance.on
+        rm /app/maintenance/maintenance.on
+    fi
 
     # update status for webhooks
     printf "%s\n" "complete" > "/tmp/restic/snapshot_status_$request_id"

src/Dockerfile

@@ -30,7 +30,7 @@ RUN chicken-install srfi-1 srfi-13 srfi-18 srfi-19 srfi-158 srfi-194 \
         sxml-transforms schematra \
         uri-common http-client medea intarweb \
         sql-null openssl postgresql crypto-tools \
-        hmac sha2 string-hexadecimal
+        hmac sha2 string-utils
 
 WORKDIR /var
 RUN mkdir nassella

src/db-init.sql

@@ -26,6 +26,7 @@ create table user_service_configs(
        digitalocean_api_token_enc varchar(255),
        digitalocean_region varchar(255),
        digitalocean_size varchar(255),
+       digitalocean_volume_size integer,
        backblaze_application_key_enc varchar(255),
        backblaze_key_id_enc varchar(255),
        backblaze_bucket_url_enc varchar(255)
@@ -75,6 +76,7 @@ create table deployments(
        terraform_machine_destroy deployment_status not null default 'queued',
        terraform_ip_create deployment_status not null default 'queued',
        terraform_ip_destroy deployment_status not null default 'queued',
+       instance_backup deployment_status not null default 'queued',
        log_enc text
        );
 
@@ -92,5 +94,9 @@ create unique index user_terraform_state_user_id_instance_id_idx on user_terrafo
 
 create table migrations(
        id bigserial primary key,
-       migration_id integer not null unique,
+       migration_id integer not null unique
        );
+
+insert into migrations(migration_id) values(0);
+insert into migrations(migration_id) values(1);
+insert into migrations(migration_id) values(2);

src/db.scm

@@ -230,6 +230,7 @@ returning users.user_id;"
     (digitalocean-api-token . ("digitalocean_api_token_enc" #t))
     (digitalocean-region . ("digitalocean_region" #f))
     (digitalocean-size . ("digitalocean_size" #f))
+    (digitalocean-volume-size . ("digitalocean_volume_size" #f))
     (backblaze-application-key . ("backblaze_application_key_enc" #t))
     (backblaze-key-id . ("backblaze_key_id_enc" #t))
     (backblaze-bucket-url . ("backblaze_bucket_url_enc" #t))))
@@ -416,6 +417,11 @@ returning users.user_id;"
     (custom-image . "terraform_custom_image")
     (machine-create . "terraform_machine_create")
     (machine-destroy . "terraform_machine_destroy")
+    (ip-create . "terraform_ip_create")
+    (ip-destroy . "terraform_ip_destroy")
+    (volume-create . "terraform_volume_create")
+    (volume-destroy . "terraform_volume_destroy")
+    (instance-backup . "instance_backup")
     (status . "status")
     (id . "id")
     (instance-id . "instance_id")))
@@ -621,13 +627,15 @@ returning users.user_id;"
 ;; The "up" file is called to run the migration and the "down" file is called to
 ;; "undo" the migration.
 (define *migrations*
-  '((0 . "adding-instance-control-app")))
+  '((0 . "adding-instance-control-app")
+    (1 . "adding-service-config-digitalocean-volume-size")
+    (2 . "adding-deployments-instance-backup")))
 
 (define (run-pending-migrations conn)
   (let* ((migration-ids (sort (map car *migrations*) <))
          (migration-rows (query conn "select migration_id from migrations;"))
          (applied-migration-ids (if (> (row-count migration-rows) 0)
-                                    (row-values migration-rows)
+                                    (column-values migration-rows)
                                     '())))
     (for-each
      (lambda (id)

src/migrations/0-adding-instance-control-app-down.sql

@@ -0,0 +1 @@
+alter table user_selected_apps drop column instance_control_version;

src/migrations/0-adding-instance-control-app-up.sql

@@ -0,0 +1 @@
+alter table user_selected_apps add instance_control_version varchar(100);

src/migrations/1-adding-service-config-digitalocean-volume-size-down.sql

@@ -0,0 +1 @@
+alter table user_service_configs drop column digitalocean_volume_size;

src/migrations/1-adding-service-config-digitalocean-volume-size-up.sql

@@ -0,0 +1 @@
+alter table user_service_configs add digitalocean_volume_size integer;

src/migrations/2-adding-deployments-instance-backup-down.sql

@@ -0,0 +1 @@
+alter table deployments drop column instance_backup;

src/migrations/2-adding-deployments-instance-backup-up.sql

@@ -0,0 +1 @@
+alter table deployments add instance_backup deployment_status not null default 'queued';

src/nassella.scm

@@ -13,6 +13,7 @@
         (chicken file)
         (chicken condition)
         (chicken sort)
+        (chicken random)
 
         (rename srfi-1 (delete srfi1:delete))
         srfi-13
@@ -722,23 +723,31 @@ h1, h2, h3, h4, h5, h6 {
   (with-output-to-file (string-append dir "/terraform.tfstate.backup") (lambda () (write-string state-backup))))
 
 (define (parse-deployment-log log)
-  (define (search complete in-progress)
-    (cond ((irregex-search complete log)
+  (define (search complete in-progress failed)
+    (cond ((irregex-search failed log)
+           'failed)
+          ((irregex-search complete log)
            'complete)
           ((irregex-search in-progress log)
            'in-progress)
           (else 'queued)))
-  `((generate-configs . ,(search "terraform apply" "NASSELLA_CONFIG: start"))
+  `((generate-configs . ,(search "terraform apply" "NASSELLA_CONFIG: start" "Failed to install provider"))
     ;; TODO this didn't seem to work right when upgrading the flatcar image
     ;; log: [0mdigitalocean_custom_image.flatcar: Creating...
 ;; digitalocean_custom_image.flatcar: Still creating... [00m10s elapsed]
 ;; digitalocean_custom_image.flatcar: Still creating... [00m20s elapsed]
 ;; digitalocean_custom_image.flatcar: Still creating... [00m30s elapsed]
 ;; digitalocean_custom_image.flatcar: Still creating... [00m40s elapsed]
-    (custom-image . ,(search "custom_image.flatcar: Modifications complete" "custom_image.flatcar: Modifying"))
-    (machine-create . ,(search "droplet.machine: Creation complete" "droplet.machine: Creating..."))
+    (custom-image . ,(search '(or "custom_image.flatcar: Modifications complete" "custom_image.flatcar: Creation complete")
+                             '(or "custom_image.flatcar: Modifying" "custom_image.flatcar: Creating")
+                             "XXX - nothing"))
+    (machine-create . ,(search "droplet.machine: Creation complete" "droplet.machine: Creating..." "XXX - nothing"))
     (machine-destroy . ,(search "droplet.machine: Destruction complete"
-                                '(: "droplet.machine (deposed object " (* alphanum) "): Destroying...")))))
+                                '(: "droplet.machine (deposed object " (* alphanum) "): Destroying...") "XXX - nothing"))
+    (ip-destroy . ,(search "reserved_ip_assignment.machine: Destruction complete" "reserved_ip_assignment.machine: Destroying..." "XXX - nothing"))
+    (ip-create . ,(search "reserved_ip_assignment.machine: Creation complete" "reserved_ip_assignment.machine: Creating..." "Error Assigning reserved IP"))
+    (volume-create . ,(search "volume_attachment.machine: Creation complete" "volume_attachment.machine: Creating..." "XXX - nothing"))
+    (volume-destroy . ,(search "volume_attachment.machine: Destruction complete" "volume_attachment.machine: Destroying..." "XXX - nothing"))))
 
 (define (write-config-entry name value)
   (display name)
@@ -1317,7 +1326,9 @@ chmod -R 777 /opt/keys")))
           (method POST))
        (VStack
         (Fieldset
-         (@ (title "Size"))
+         (@ (title "Instance Properties"))
+         (Field (@ (name "volume-size") (label ("Volume Size in GB (For persistent application storage)"))
+                   (value ,(alist-ref 'digitalocean-volume-size config eq? "60"))))
          (Field (@ (element select) (name "size") (input-style ((max-width "100%"))))
                 ,@(map (lambda (s) `(option (@ (value ,(alist-ref 'slug s))
                                                ,@(if (equal? (alist-ref 'slug s) "s-2vcpu-2gb") `((selected "selected")) '()))
@@ -1337,7 +1348,8 @@ chmod -R 777 /opt/keys")))
        db
        (session-user-id)
        instance-id
-       `((digitalocean-size . ,(alist-ref 'size (current-params)))))))
+       `((digitalocean-size . ,(alist-ref 'size (current-params)))
+         (digitalocean-volume-size . ,(alist-ref 'volume-size (current-params)))))))
    (redirect (conc "/config/wizard/review/" instance-id))))
 
 (get/widgets
@@ -1374,6 +1386,7 @@ chmod -R 777 /opt/keys")))
           (li "Size: " ,(alist-ref 'digitalocean-size service-config)))
       (form
        (@ (action ,(conc "/config/wizard/review-submit/" instance-id)) (method POST))
+       (input (@ (type "hidden") (value ,(alist-ref 'force (current-params))) (name "force")))
        (VStack
         (Form-Nav (@ (back-to ,(conc "/config/wizard/machine2/" instance-id)) (submit-button "Launch")))))))))
 
@@ -1385,7 +1398,8 @@ chmod -R 777 /opt/keys")))
                    (with-db/transaction
                     (lambda (db)
                       (get-most-recent-deployment-status db (session-user-id) instance-id)))))))
-    (when (not (or (eq? status 'queued) (eq? status 'in-progress)))
+    (when (or (not (or (eq? status 'queued) (eq? status 'in-progress)))
+              (equal? (alist-ref 'force (current-params)) "true"))
       (let* ((instance-id (alist-ref "id" (current-params) equal?))
              (restic-snapshot-id (alist-ref 'restic-snapshot-id (current-params)))
              (results
@@ -1465,21 +1479,65 @@ chmod -R 777 /opt/keys")))
                    (write-config-entry (car e) (cdr e)))
                  `(("server_type" . ,(alist-ref 'digitalocean-size service-config))
                    ("do_token" . ,(alist-ref 'digitalocean-api-token service-config))
+                   ("digitalocean_volume_size" . ,(alist-ref 'digitalocean-volume-size service-config))
                    ("cloudflare_api_token" . ,(alist-ref 'cloudflare-api-token service-config))
                    ("cloudflare_zone_id" . ,(alist-ref 'cloudflare-zone-id service-config))
                    ("cloudflare_account_id" . ,(alist-ref 'cloudflare-account-id service-config))
                    ("cluster_name" . "nassella")
                    ("datacenter" . ,(alist-ref 'digitalocean-region service-config))
                    ;; (source <(curl -sSfL https://stable.release.flatcar-linux.net/amd64-usr/current/version.txt); echo "${FLATCAR_VERSION_ID}")
-                   ("flatcar_stable_version" . "4593.2.1")))
+                   ("flatcar_stable_version" . "4593.2.3")))
             ;; remove the newline that generating the ssh key adds
             (display "ssh_keys=[\"") (display (string-drop-right ssh-pub-key 1)) (print "\"]"))))
       (let* ((instance-id (alist-ref "id" (current-params) equal?))
              (user-id (session-user-id))
+             (app-config
+              (with-db/transaction
+               (lambda (db)
+                 (get-user-app-config db (session-user-id) instance-id))))
              (deployment-id (with-db/transaction (lambda (db) (create-deployment db user-id instance-id))))
-             (dir (deployment-directory user-id instance-id)))
+             (dir (deployment-directory user-id instance-id))
+             (backup-request-id (conc (truncate (time->seconds (current-time))) "-" (pseudo-random-integer 10000))))
+        (with-db/transaction
+         (lambda (db)
+           (update-deployment-progress db deployment-id '((instance-backup . in-progress)))))
+        (handle-exceptions
+            exn
+          (with-db/transaction
+           (lambda (db)
+             (update-deployment-progress db deployment-id '((instance-backup . failed)))))
+          (send-instance-control-command
+           (alist-ref 'root-domain app-config)
+           (alist-ref 'subdomain (alist-ref 'instance-control (alist-ref 'config app-config)))
+           "queue-restic-snapshot-no-restart"
+           (alist-ref 'webhooks-secret (alist-ref 'instance-control (alist-ref 'config app-config)))
+           `((path . "/")
+             (tag . "automated_pre_instance_update")
+             ;; effectively a guid, we just want something unique
+             (request_id . ,backup-request-id)
+             (version . 0))))
         (thread-start!
          (lambda ()
+           (let ((start-time (time->seconds (current-time))))
+             (let loop ()
+               (thread-sleep! 1)
+               (let* ((status-result
+                       (handle-exceptions
+                           exn
+                         '((status . "error"))
+                         (send-instance-control-command
+                          (alist-ref 'root-domain app-config)
+                          (alist-ref 'subdomain (alist-ref 'instance-control (alist-ref 'config app-config)))
+                          "restic-snapshot-status"
+                          (alist-ref 'webhooks-secret (alist-ref 'instance-control (alist-ref 'config app-config)))
+                          `((request_id . ,backup-request-id)
+                            (version . 0)))))
+                      (complete (string=? (alist-ref 'status status-result) "complete")))
+                 (if (or complete (> (- (time->seconds (current-time)) start-time) 120))
+                     (with-db/transaction
+                      (lambda (db)
+                        (update-deployment-progress db deployment-id `((instance-backup . ,(or (and complete 'complete) 'failed))))))
+                     (loop)))))
            (change-directory dir)
            (let ((pid (process-run "make apply > make-out 2>&1")))
              (with-db/transaction (lambda (db) (update-deployment-in-progress db deployment-id pid)))
@@ -1511,15 +1569,9 @@ chmod -R 777 /opt/keys")))
                        (with-db/transaction
                         (lambda (db)
                           (update-deployment-progress db deployment-id progress)
-                          ;; TODO THIS DOESN'T WORK RIGHT FOR TERRAFORM OP FAILURES
-                          ;; like the random digital ocean error saying the IP can't be
-                          ;; updated because another operation is in progress.
-                          ;; it still registers as "success".
-                          ;; probably need to also write stderr to a file and read/store/parse that?
-                          ;; Should we parse make-out for string "Apply complete!" ?
                           (update-deployment-status
                            db user-id deployment-id
-                           (if exit-normal 'complete 'failed)
+                           (if (= status 0) 'complete 'failed)
                            (with-input-from-file (string-append dir "/make-out") read-string))
                           (update-user-terraform-state db user-id instance-id
                                                        (if (eof-object? tf-state) "" tf-state)
@@ -1541,7 +1593,7 @@ chmod -R 777 /opt/keys")))
               (lambda (db)
                 `((status . ,(get-most-recent-deployment-status db (session-user-id) instance-id))
                   (progress . ,(get-most-recent-deployment-progress db (session-user-id) instance-id))))))
-        (output (with-input-from-file (string-append (deployment-directory (session-user-id) instance-id) "/make-out") read-string))
+        (output (handle-exceptions exn "" (with-input-from-file (string-append (deployment-directory (session-user-id) instance-id) "/make-out") read-string)))
         (progress (alist-ref 'progress res))
         (status (alist-ref 'status res)))
    `(App
@@ -1553,9 +1605,14 @@ chmod -R 777 /opt/keys")))
            ((in-progress) "Deployment in progress")
            ((complete) "Deployment complete!")
            ((failed) "Deployment failed")))
-       (ul (li "generate configs: " ,(progress-status->text (alist-ref 'generate-configs progress)))
+       (ul (li "perform backup: " ,(progress-status->text (alist-ref 'instance-backup progress)))
+           (li "generate configs: " ,(progress-status->text (alist-ref 'generate-configs progress)))
            (li "custom flatcar image: " ,(progress-status->text (alist-ref 'custom-image progress)))
-           (li "machine create: " ,(progress-status->text (alist-ref 'machine-create progress)))
+           (li "application volume disconnect: " ,(progress-status->text (alist-ref 'volume-destroy progress)))
+           (li "instance create: " ,(progress-status->text (alist-ref 'machine-create progress)))
+           (li "instance mapped ip disconnect: " ,(progress-status->text (alist-ref 'ip-destroy progress)))
+           (li "instance mapped ip connect: " ,(progress-status->text (alist-ref 'ip-create progress)))
+           (li "application volume connect: " ,(progress-status->text (alist-ref 'volume-create progress)))
            (li "cleanup previous machine: " ,(progress-status->text (alist-ref 'machine-destroy progress))))
        (div
         (a (@ (href "/dashboard")) "Dashboard")
@@ -1732,7 +1789,7 @@ chmod -R 777 /opt/keys")))
                    ("cluster_name" . "nassella")
                    ("datacenter" . ,(alist-ref 'digitalocean-region service-config))
                    ;; (source <(curl -sSfL https://stable.release.flatcar-linux.net/amd64-usr/current/version.txt); echo "${FLATCAR_VERSION_ID}")
-                   ("flatcar_stable_version" . "4593.2.1")))
+                   ("Flatcar_stable_version" . "4593.2.3")))
             ;; remove the newline that generating the ssh key adds
             (display "ssh_keys=[\"") (display (string-drop-right ssh-pub-key 1)) (print "\"]")))
          ;; TODO need a new table to track destroying?
@@ -1824,6 +1881,7 @@ chmod -R 777 /opt/keys")))
             `((ul (li "generate configs: " ,(progress-status->text (alist-ref 'generate-configs progress)))
                    (li "custom flatcar image: " ,(progress-status->text (alist-ref 'custom-image progress)))
                    (li "machine create: " ,(progress-status->text (alist-ref 'machine-create progress)))
+                   (li "attaching : " ,(progress-status->text (alist-ref 'machine-create progress)))
                    (li "cleanup previous machine: " ,(progress-status->text (alist-ref 'machine-destroy progress))))
               (div
                (a (@ (href "/dashboard")) "Dashboard")
@@ -1935,18 +1993,65 @@ chmod -R 777 /opt/keys")))
          (Form-Nav (@ (back-to ,(conc "/backups/" instance-id)) (submit-button "Create"))))))))))
 
 (post "/backups/:instance_id/create-submit"
- (let ((instance-id (alist-ref "instance_id" (current-params) equal?))
-       (app-config (with-db/transaction
-                    (lambda (db)
-                      (get-user-app-config db (session-user-id) instance-id)))))
-   ;; TODO make requests to instance control
-   ;; get the root domain and subdomain for instance control
-   ;; then call subdomain.rootdomain/hooks/queue-restic-snapshot
-   ;; content-type application/json
-   ;; data: 'path "/" 'tag tag 'request_id (generate-one?) 'version 0
-   ;; then run through hmac ((hmac "instance-control-secret-key" sha256-primitive) data)
-   ;; then make a new page to redirect the user to that polls for status page using the request id
-   (redirect (conc "/config/wizard/review/" instance-id))))
+ (let* ((instance-id (alist-ref "instance_id" (current-params) equal?))
+        (app-config (with-db/transaction
+                     (lambda (db)
+                       (get-user-app-config db (session-user-id) instance-id))))
+        (request-id (conc (truncate (time->seconds (current-time))) "-" (pseudo-random-integer 10000))))
+   (send-instance-control-command
+    (alist-ref 'root-domain app-config)
+    (alist-ref 'subdomain (alist-ref 'instance-control (alist-ref 'config app-config)))
+    "queue-restic-snapshot"
+    (alist-ref 'webhooks-secret (alist-ref 'instance-control (alist-ref 'config app-config)))
+    `((path . "/") ;; unused for now but we do want root until we support backing up individual apps
+      (tag . ,(alist-ref 'tag (current-params)))
+      ;; effectively a guid, we just want something unique
+      (request_id . ,request-id)
+      (version . 0)))
+   (redirect (conc "/backups/" instance-id "/create-success/" request-id))))
+
+(get/widgets
+ ("/backups/:instance_id/create-success/:request_id"
+  (let* ((instance-id (alist-ref "instance_id" (current-params) equal?))
+         (request-id (alist-ref "request_id" (current-params) equal?))
+         (app-config (with-db/transaction
+                     (lambda (db)
+                       (get-user-app-config db (session-user-id) instance-id))))
+         (status-result
+          (send-instance-control-command
+           (alist-ref 'root-domain app-config)
+           (alist-ref 'subdomain (alist-ref 'instance-control (alist-ref 'config app-config)))
+           "restic-snapshot-status"
+           (alist-ref 'webhooks-secret (alist-ref 'instance-control (alist-ref 'config app-config)))
+           `((request_id . ,request-id)
+             (version . 0))))
+         (complete (string=? (alist-ref 'status status-result) "complete")))
+    (if complete
+        '()
+        '((meta (@ (http-equiv "refresh") (content "5")))))))
+ (let* ((instance-id (alist-ref "instance_id" (current-params) equal?))
+        (request-id (alist-ref "request_id" (current-params) equal?))
+        (app-config (with-db/transaction
+                     (lambda (db)
+                       (get-user-app-config db (session-user-id) instance-id))))
+        (status-result
+         (send-instance-control-command
+          (alist-ref 'root-domain app-config)
+          (alist-ref 'subdomain (alist-ref 'instance-control (alist-ref 'config app-config)))
+          "restic-snapshot-status"
+          (alist-ref 'webhooks-secret (alist-ref 'instance-control (alist-ref 'config app-config)))
+          `((request_id . ,request-id)
+            (version . 0))))
+        (complete (string=? (alist-ref 'status status-result) "complete")))
+   `(App
+     (Main-Container
+      (VStack
+       (h1 ,(if complete "Snapshot Complete" "Snapshot In Progress..."))
+       (div
+        (a (@ (href "/dashboard")) "Dashboard")
+        ,@(if complete
+              '()
+              " (snapshot will continue in the background if you leave this page)")))))))
 
 (schematra-install)