wip

Makefile

@@ -111,11 +111,11 @@ preapply: app/.dirstamp
 
 apply: announce-start restic-init ignition.json $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars
 	echo "NASSELLA_CONFIG: end"
-	terraform init
+	terraform init --upgrade
 	bash -c "terraform apply -auto-approve -input=false -var-file=<(cat $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars)"
 
 destroy: ignition.json $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars
-	terraform init
+	terraform init --upgrade
 	bash -c "terraform destroy -auto-approve -input=false -var-file=<(cat $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars)"
 
 .PHONY: restic-init

all-apps/authelia/4/Caddyfile

@@ -14,8 +14,17 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with Nassella. If not, see <https://www.gnu.org/licenses/>.
 
-{$authelia_SUBDOMAIN}.{$ROOT_DOMAIN} {
-  import maintenance_intercept
+#{$authelia_SUBDOMAIN}.{$ROOT_DOMAIN} {
+#  route {
+#    @authelia path /authelia /authelia/*
+#    handle @authelia {
+#      reverse_proxy authelia:9091
+#    }
+#    forward_auth authelia:9091 {
+#      uri /api/authz/forward-auth
+#      copy_headers Remote-User Remote-Groups Remote-Email Remote-Name
+#    }
+#    reverse_proxy authelia:9091
+#  }
+#}
 
-  reverse_proxy authelia:9091
-}

all-apps/authelia/4/Makefile

@@ -18,7 +18,8 @@ IMAGE_DEPS := $(IMAGE_DEPS) \
 app/authelia/postgres_db \
 app/authelia/postgres_user \
 app/authelia/postgres_password \
-app/authelia/authelia-config/configuration.yml
+app/authelia/authelia-config/configuration.yml \
+app/authelia/authelia-config/users.yml
 
 app/authelia/postgres_db: $(apps_config)
 	bash -c 'source ./$(apps_config); printf "%s\n" "$$AUTHELIA_POSTGRES_DB" > $@'
@@ -29,3 +30,6 @@ app/authelia/postgres_password: $(apps_config)
 app/authelia/authelia-config/configuration.yml: $(apps_config) app/authelia/authelia-config/configuration.yml.tmpl \
 app/authelia/make-authelia-config.sh
 	./app/authelia/make-authelia-config.sh $(apps_config)
+app/authelia/authelia-config/users.yml: $(apps_config) app/authelia/authelia-config/users.yml.tmpl \
+app/authelia/make-authelia-users-config.sh
+	./app/authelia/make-authelia-users-config.sh $(apps_config)

all-apps/authelia/4/authelia-config/configuration.yml.tmpl

@@ -35,58 +35,45 @@ identity_validation:
   reset_password:
     jwt_secret: '$AUTHELIA_JWT_SECRET'
 
-# lldap service account user should instead
-# use an account with lldap_password_manager group
-# since that can't be used to change an admin password
 authentication_backend:
-  ldap:
-    address: 'ldap://lldap:3890'
-    implementation: 'lldap'
-    timeout: '5s'
-    pooling:
-      enable: false
-      count: 5
-      retries: 2
-      timeout: '10 seconds'
-    base_dn: 'DC=nassella,DC=org'
-#    additional_users_dn: 'OU=users'
-#    additional_groups_dn: 'OU=groups'
-#    group_search_mode: 'filter'
-#    permit_referrals: false
-    permit_unauthenticated_bind: false
-    permit_feature_detection_failure: false
-    user: 'uid=admin,ou=people,dc=nassella,dc=org'
-    password: '$LLDAP_ADMIN_PASSWORD'
-    # attributes:
-    #   distinguished_name: 'distinguishedName'
-    #   username: 'uid'
-    #   display_name: 'displayName'
-    #   family_name: 'sn'
-    #   given_name: 'givenName'
-    #   middle_name: 'middleName'
-    #   nickname: ''
-    #   gender: ''
-    #   birthdate: ''
-    #   website: 'wWWHomePage'
-    #   profile: ''
-    #   picture: ''
-    #   zoneinfo: ''
-    #   locale: ''
-    #   phone_number: 'telephoneNumber'
-    #   phone_extension: ''
-    #   street_address: 'streetAddress'
-    #   locality: 'l'
-    #   region: 'st'
-    #   postal_code: 'postalCode'
-    #   country: 'c'
-    #   mail: 'mail'
-    #   member_of: 'memberOf'
-    #   group_name: 'cn'
-    #   extra:
-    #     extra_example:
-    #       name: ''
-    #       multi_valued: false
-    #       value_type: 'string'
+  disable_reset_password: true
+  file:
+    path: '/config/users.yml'
+    watch: false
+    search:
+      email: false
+      case_insensitive: false
+    extra_attributes:
+      extra_example:
+        multi_valued: false
+        value_type: 'string'
+    password:
+      algorithm: 'argon2'
+      argon2:
+        variant: 'argon2id'
+        iterations: 3
+        memory: 65536
+        parallelism: 4
+        key_length: 32
+        salt_length: 16
+      scrypt:
+        variant: 'scrypt'
+        iterations: 16
+        block_size: 8
+        parallelism: 1
+        key_length: 32
+        salt_length: 16
+      pbkdf2:
+        variant: 'sha512'
+        iterations: 310000
+        salt_length: 16
+      sha2crypt:
+        variant: 'sha512'
+        iterations: 50000
+        salt_length: 16
+      bcrypt:
+        variant: 'standard'
+        cost: 12
 
 access_control:
   default_policy: 'deny'
@@ -102,11 +89,12 @@ session:
   secret: '$AUTHELIA_SESSION_SECRET'
 
   cookies:
-    - name: 'authelia_session'
-      domain: '$ROOT_DOMAIN'  # Should match whatever your root protected domain is
-      authelia_url: 'https://$AUTHELIA_FULL_DOMAIN'
+    - name: 'dozzle_session'
+      domain: '$DOZZLE_FULL_DOMAIN'  # Should match whatever your root protected domain is
+      authelia_url: 'https://$DOZZLE_FULL_DOMAIN/authelia'
       expiration: '1 hour'  # 1 hour
       inactivity: '5 minutes'  # 5 minutes
+      default_redirection_url: 'https://$DOZZLE_FULL_DOMAIN/'
 
 regulation:
   max_retries: 3

all-apps/authelia/4/authelia-config/users.yml.tmpl

@@ -0,0 +1,7 @@
+# yaml-language-server: $schema=https://www.authelia.com/schemas/latest/json-schema/user-database.json
+users:
+  admin:
+    disabled: false
+    displayname: 'Admin'
+    password: '$HOST_ADMIN_PASSWORD'
+    email: '$HOST_ADMIN_USER'

all-apps/authelia/4/docker-compose.yaml

@@ -50,13 +50,11 @@ services:
     image: 'authelia/authelia'
     volumes:
       - ./authelia/authelia-config/configuration.yml:/config/configuration.yml:ro
+      - ./authelia/authelia-config/users.yml:/config/users.yml:ro
     networks:
       - lb
       - authelia_internal_db
-      - lldap_internal
     depends_on:
-      lldap:
-        condition: service_healthy
       authelia_db:
         condition: service_healthy
     restart: 'unless-stopped'
@@ -66,9 +64,6 @@ services:
 
 networks:
   lb:
-  lldap_internal:
-    driver: bridge
-    internal: true
   authelia_internal_db:
     driver: bridge
     internal: true

all-apps/authelia/4/make-authelia-users-config.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Copyright 2025-2026 Thomas Hintz
+
+# This file is part of Nassella.
+
+# Nassella is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+
+# Nassella is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with Nassella. If not, see <https://www.gnu.org/licenses/>.
+
+set -e
+set -a # export everything in the config for later use by envsubst
+
+. $1 # source the apps.config file with then env vars
+
+envsubst < app/authelia/authelia-config/users.yml.tmpl > app/authelia/authelia-config/users.yml

all-apps/dozzle/20/Caddyfile

@@ -15,9 +15,15 @@
 # along with Nassella. If not, see <https://www.gnu.org/licenses/>.
 
 {$dozzle_SUBDOMAIN}.{$ROOT_DOMAIN} {
+  route {
+    @authelia path /authelia /authelia/*
+    handle @authelia {
+      reverse_proxy authelia:9091
+    }
     forward_auth authelia:9091 {
       uri /api/authz/forward-auth
       copy_headers Remote-User Remote-Groups Remote-Email Remote-Name
     }
     reverse_proxy http://dozzle:8080
   }
+}

all-apps/nassella/b0.0.1/authelia-config/configuration.yml.tmpl

@@ -84,7 +84,7 @@ access_control:
       policy: 'two_factor'
 
 session:
-  secret: 'insecure_session_secret'
+  secret: 'insecure_session_secret' # TODO fix
 
   cookies:
     - name: 'authelia_session'

main.tf

@@ -2,23 +2,19 @@ terraform {
   required_providers {
     digitalocean = {
       source  = "digitalocean/digitalocean"
-      version = "2.62.0"
+      version = "2.96.0"
     }
     ct = {
       source  = "poseidon/ct"
-      version = "0.13.0"
+      version = "0.14.0"
     }
     null = {
       source  = "hashicorp/null"
-      version = "3.2.4"
-    }
-    time = {
-      source  = "hashicorp/time"
-      version = "0.13.1"
+      version = "3.3.0"
     }
     cloudflare = {
       source = "cloudflare/cloudflare"
-      version = "5.8.2"
+      version = "5.22.0"
     }
   }
 }
@@ -86,6 +82,10 @@ variable "digitalocean_volume_size" {
 
 provider "digitalocean" {
   token = var.do_token
+
+  http_retry_max      = 10
+  http_retry_wait_min = 2
+  http_retry_wait_max = 60
 }
 
 provider "cloudflare" {
@@ -142,21 +142,11 @@ resource "digitalocean_droplet" "machine" {
   size      = var.server_type
   ssh_keys  = [digitalocean_ssh_key.first.fingerprint]
   user_data = file("ignition.json")
-  lifecycle {
-    create_before_destroy = true
 }
-}
-
-resource "time_sleep" "wait_10_seconds" {
-  create_duration = "10s"
-}
-
 resource "digitalocean_reserved_ip_assignment" "machine" {
+  depends_on = [digitalocean_volume_attachment.machine]
   ip_address = digitalocean_reserved_ip.machine.ip_address
   droplet_id = digitalocean_droplet.machine.id
-  # sometimes digital ocean throws an error for this resource
-  # saying the droplet has a pending event so we delay a few seconds
-  depends_on = [time_sleep.wait_10_seconds]
 }
 
 resource "digitalocean_volume_attachment" "machine" {

src/Dockerfile

@@ -45,6 +45,7 @@ RUN chmod +x nassella-run
 
 FROM quay.io/coreos/butane:release AS butane
 FROM hashicorp/terraform:1.15 AS terraform
+FROM authelia/authelia:latest AS authelia
 
 FROM debian:trixie-slim
 RUN apt-get update && apt-get -y --no-install-recommends install \
@@ -53,6 +54,7 @@ RUN apt-get update && apt-get -y --no-install-recommends install \
 
 COPY --from=butane /usr/local/bin/butane /usr/local/bin/butane
 COPY --from=terraform /bin/terraform /usr/local/bin/terraform
+COPY --from=authelia /app/authelia /usr/local/bin/authelia
 
 COPY --from=buildeggs /usr/local/ /usr/local/
 

src/Makefile

@@ -15,7 +15,7 @@
 # along with Nassella. If not, see <https://www.gnu.org/licenses/>.
 
 dockerall:
-	docker buildx build --platform linux/amd64,linux/arm64 -t nassella/b0.0.1 .
+	docker buildx build --platform linux/arm64 -t nassella/b0.0.1 . # linux/amd64,
 
 dockerlocal:
 	docker buildx build -t nassella/b0.0.1 .

src/db.scm

@@ -23,7 +23,7 @@
      db-init db-clean
 
      create-user delete-user
-     get-user-id-by-username
+     get-user-id-by-username get-user-email-by-id
      create-instance destroy-instance get-user-instances
      get-instance-ssh-pub-key get-instance-ssh-priv-key
      update-instance-ssh-pub-key
@@ -181,6 +181,12 @@ returning users.user_id;"
           (value-at res)
           #f)))
 
+  (define (get-user-email-by-id conn id)
+    (let ((res (query conn "select email from users where user_id=$1;" id)))
+      (if (> (row-count res) 0)
+          (value-at res)
+          #f)))
+
   ;; We also encrypt the ssh pub key not to hide it but to make it
   ;; more difficult for someone to tamper with it which could allow
   ;; an attacker to poison an instance with an ssh key that they have
@@ -417,6 +423,7 @@ where usa.user_id=$1 and usa.instance_id=$2;"
       (volume-create . "terraform_volume_create")
       (volume-destroy . "terraform_volume_destroy")
       (instance-backup . "instance_backup")
+      (instance-up . "instance_up")
       (status . "status")
       (id . "id")
       (instance-id . "instance_id")))
@@ -662,7 +669,8 @@ where user_id=$1;"
       (6 . "adding-lldap-app")
       (7 . "adding-authelia-app")
       (8 . "adding-backblaze-bucket-name-and-id")
-      (9 . "extend-deployment-status-add-ignored")))
+      (9 . "extend-deployment-status-add-ignored")
+      (10 . "adding-deployments-instance-up")))
 
   (define (run-pending-migrations conn)
     (let* ((migration-ids (sort (map car *migrations*) <))
@@ -699,7 +707,7 @@ where user_id=$1;"
               (string-split (with-input-from-file "db-init.sql" read-string) ";"))
              (log-to (debug-log) "table creation finished")
              (log-to (debug-log) "creating test user")
-             (create-user db "me@example.com" "username")
+             (create-user db "me@example.com" "username") ;; TODO should this be removed?
              (log-to (debug-log) "test user creation finished")))
        ;; originally there was no migrations table, so first add it if it doesn't exist
        (if (value-at (query db "SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = 'public' AND tablename  = 'migrations');"))

src/migrations/10-adding-deployments-instance-up-up.sql

@@ -0,0 +1 @@
+alter table deployments add instance_up deployment_status not null default 'queued';

src/nassella.scm

@@ -350,6 +350,9 @@ h1, h2, h3, h4, h5, h6 {
 (define (session-user-id)
   (alist-ref 'user-id (current-params)))
 
+(define (session-username)
+  (alist-ref 'username (current-params)))
+
 (define-syntax get/widgets
   (syntax-rules ()
     ((_ (path) body ...)
@@ -670,6 +673,21 @@ h1, h2, h3, h4, h5, h6 {
     #f
     read-json))
 
+(define (b2-create-bucket api-url account-token account-id bucket-name)
+  (with-input-from-request
+      (make-request method: 'POST
+                    uri: (uri-reference (string-append api-url "/b2api/v4/b2_create_bucket"))
+                    headers: (headers `((authorization #(,account-token raw))
+                                        (content-type application/json))))
+    (lambda ()
+      (write-json
+       `((accountId . ,account-id)
+         (bucketName . ,bucket-name)
+         (bucketType . "allPrivate")
+         (lifecycleRules . #(((daysFromHidingToDeleting . 1)
+                              (fileNamePrefix . "")))))))
+    read-json))
+
 ;; request response looks like
 ;; (((accountId . "xxx") (bucketId . "xxx") (bucketInfo) (bucketName . "xxx") (bucketType . "allPrivate") (corsRules) (defaultServerSideEncryption (isClientAuthorizedToRead . #t) (value (algorithm . null) (mode . null))) (fileLockConfiguration (isClientAuthorizedToRead . #t) (value (defaultRetention (mode . null) (period . null)) (isFileLockEnabled . #f))) (lifecycleRules ((daysFromHidingToDeleting . 1) (daysFromStartingToCancelingUnfinishedLargeFiles . null) (daysFromUploadingToHiding . null) (fileNamePrefix . ""))) (options "s3") (replicationConfiguration (isClientAuthorizedToRead . #t) (value . null)) (revision . 3)) ((accountId . "yyy") (bucketId . "yyy") (bucketInfo) (bucketName . "yyy") (bucketType . "allPrivate") (corsRules) (defaultServerSideEncryption (isClientAuthorizedToRead . #t) (value (algorithm . null) (mode . null))) (fileLockConfiguration (isClientAuthorizedToRead . #t) (value (defaultRetention (mode . null) (period . null)) (isFileLockEnabled . #f))) (lifecycleRules ((daysFromHidingToDeleting . 1) (daysFromStartingToCancelingUnfinishedLargeFiles . null) (daysFromUploadingToHiding . null) (fileNamePrefix . ""))) (options "s3") (replicationConfiguration (isClientAuthorizedToRead . #t) (value . null)) (revision . 3)))
 (define (b2-list-buckets api-url account-token account-id)
@@ -869,67 +887,37 @@ h1, h2, h3, h4, h5, h6 {
          `((success . #f)
            (result ,data))))))
 
-;; (define (test-backblaze-connection user-id instance-id key-id application-key bucket-url)
-;;   (let* ((password-path (conc "restic-password-" user-id "-" instance-id))
-;;          (restic-password
-;;           (with-db/transaction
-;;            (lambda (db)
-;;              (get-instance-restic-password db user-id instance-id)))))
-;;     (dynamic-wind
-;;       (lambda ()
-;;         (with-output-to-file password-path (lambda () (display restic-password))))
-;;       (lambda ()
-;;         ;; restic retries (indefinitely?) if the connection can't be made so
-;;         ;; we pass it through timeout to ensure it does not hang
-;;         (receive (in-port out-port pid err-port)
-;;             (cond-expand
-;;               (dev
-;;                (process* "timeout" `("--preserve-status" "8s" "docker" "run" "--rm" "--volume"
-;;                                      ,(conc (current-directory) "/" password-path ":/restic-password")
-;;                                      "-e" ,(conc "AWS_ACCESS_KEY_ID=" key-id)
-;;                                      "-e" ,(conc "AWS_SECRET_ACCESS_KEY=" application-key)
-;;                                      "-i" "restic/restic:0.18.0" "cat" "config"
-;;                                      "--repo" ,(conc "s3:" bucket-url)
-;;                                      "--password-file" "/restic-password"
-;;                                      "--json")))
-;;               (else
-;;                (process* "/bin/timeout"
-;;                          `("--preserve-status" "8s" "/bin/restic" "cat" "config"
-;;                            "--repo" ,(conc "s3:" bucket-url)
-;;                            "--password-file" ,(conc (current-directory) "/" password-path)
-;;                            "--json")
-;;                          `(("AWS_ACCESS_KEY_ID" . ,key-id)
-;;                            ("AWS_SECRET_ACCESS_KEY" . ,application-key)))))
-;;           (let ((thread
-;;                  (thread-start!
-;;                   (lambda ()
-;;                     (let loop ((i 0))
-;;                       (thread-sleep! 1)
-;;                       ;; We do a non-blocking wait here so that we don't
-;;                       ;; block the entire web process.
-;;                       (receive (wait-pid exit-normal status) (process-wait pid #t)
-;;                         (if (= wait-pid 0) ;; wait-pid is 0 until the process has finished
-;;                             (if (< i 12)   ;; 12s timeout
-;;                                 (loop (+ i 1))
-;;                                 `((success . #f)
-;;                                   (errors ((message . "timeout trying to connect to backblaze")))))
-;;                             (if exit-normal
-;;                                 (let ((res (with-input-from-port in-port read-json))
-;;                                       (err (with-input-from-port err-port read-json)))
-;;                                   ;; status 10 is what is returned if we can connect but the repo
-;;                                   ;; is not initialized yet
-;;                                   (if (or res (and err (= status 10)))
-;;                                       `((success . #t)
-;;                                         (result . ,res))
-;;                                       `((success . #f)
-;;                                         (errors ((message . "abnormal exit - check Key ID, Application ID, and Bucket URL"))))))
-;;                                 `((success . #f)
-;;                                   (errors ((message . "abnormal exit - check Key ID, Application ID, and Bucket URL"))))
-;;                                 ))))))))
-;;             (thread-join! thread))))
-;;       (lambda ()
-;;         (handle-exceptions exn 'ignore
-;;                            (delete-file password-path))))))
+(define *instance-up-timeout* 1) ;; seconds
+(define *instance-up-retry-time* 120) ;; 120s, 2 minutes
+(define *instance-up-retries* (/ *instance-up-retry-time* *instance-up-timeout*))
+(define (instance-up? path)
+  (let loop ((i 0))
+    (let ((thread
+           (thread-start!
+            (lambda ()
+              (condition-case
+                  (receive (data uri response) (with-input-from-request path #f read-string)
+                    (if (eq? (response-code response) 401)
+                        #t
+                        (begin (thread-sleep! *instance-up-timeout*) ;; we depend on this taking at least as long as the timeout if the instance is still booting
+                               #f)))
+                (e (exn client-error)
+                   (if (eq? (response-code (get-condition-property e 'client-error 'response)) 401)
+                       #t
+                       (begin (thread-sleep! *instance-up-timeout*) ;; we depend on this taking at least as long as the timeout if the instance is still booting
+                              #f)))
+                (e (exn net)
+                   (thread-sleep! *instance-up-timeout*)
+                   #f)
+                (e ()
+                   (thread-sleep! *instance-up-timeout*)
+                   #f))))))
+      (if (thread-join! thread *instance-up-timeout* #f)
+          #t
+          (if (< i *instance-up-retries*)
+              (loop (+ i 1))
+              #f)))))
+
 
 (define (deployment-directory user-id instance-id)
   (string-append "deploy-" (number->string user-id) "-" (->string instance-id)))
@@ -1077,6 +1065,34 @@ chmod -R 777 /opt/keys")))
                              "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789")
                             64)))
 
+(define (create-authelia-password-hash password)
+  ;; docker run --rm -it authelia/authelia:latest authelia crypto hash generate argon2
+  (receive (in-port out-port pid err-port)
+      (cond-expand
+        (dev
+         (process* "docker" `("run" "--rm" "authelia/authelia:latest" "authelia" "crypto" "hash" "generate" "argon2" "--password" ,password)))
+        (else
+         (process* "/usr/local/bin/authelia" `("crypto" "hash" "generate" "argon2" "--password" ,password))))
+    (let ((thread
+           (thread-start!
+            (lambda ()
+              (let loop ()
+                (thread-sleep! 1)
+                ;; We do a non-blocking wait here so that we don't
+                ;; block the entire web process.
+                (receive (wait-pid exit-normal status) (process-wait pid #t)
+                  (if (= wait-pid 0) ;; wait-pid is 0 until the process has finished
+                      (loop)
+                      (if exit-normal
+                          (begin
+                            (with-input-from-port err-port read-string) ;; left here for debugging and to clear ports
+                            ;; the output is prefixed with "Digest: " and suffixed with a newline
+                            ;; so we clean it up
+                            (string-drop (string-trim-right (with-input-from-port in-port read-string)) 8))
+                          (begin (log-to (debug-log) "authelia: docker command error")
+                                 (error "Authelia for generating password hash had abnormal exit"))))))))))
+      (thread-join! thread))))
+
 (define (generate-postgres-password)
   (generator->string (gtake (make-random-char-generator
                              "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789")
@@ -1188,6 +1204,231 @@ chmod -R 777 /opt/keys")))
         (write-json data))
       read-json)))
 
+(define (perform-review-submit instance-id)
+  (let* ((status (string->symbol
+                  (->string
+                   (with-db/transaction
+                    (lambda (db)
+                      (get-most-recent-deployment-status db (session-user-id) instance-id)))))))
+    (when (or (not (or (eq? status 'queued) (eq? status 'in-progress)))
+              (equal? (alist-ref 'force (current-params)) "true"))
+      (let* ((restic-snapshot-id (alist-ref 'restic-snapshot-id (current-params)))
+             (results
+              (with-db/transaction
+               (lambda (db)
+                 `((selected-apps . ,(get-user-selected-apps db (session-user-id) instance-id))
+                   (app-config . ,(get-user-app-config db (session-user-id) instance-id))
+                   (service-config . ,(get-user-service-config db (session-user-id) instance-id))
+                   (terraform-state . ,(get-user-terraform-state db (session-user-id) instance-id))
+                   (ssh-pub-key . ,(get-instance-ssh-pub-key db (session-user-id) instance-id))
+                   (restic-password . ,(get-instance-restic-password db (session-user-id) instance-id))))))
+             (selected-apps (alist-ref 'selected-apps results))
+             (app-config (alist-ref 'app-config results))
+             (config (alist-ref 'config app-config))
+             (root-domain (alist-ref 'root-domain app-config))
+             (service-config (alist-ref 'service-config results))
+             (terraform-state (alist-ref 'terraform-state results))
+             (ssh-pub-key (alist-ref 'ssh-pub-key results))
+             (restic-password (alist-ref 'restic-password results))
+             (dir (deployment-directory (session-user-id) instance-id)))
+        (setup-deploy-files dir (alist-ref 'state terraform-state) (alist-ref 'backup terraform-state))
+        (with-output-to-file (string-append dir "/config/apps.config")
+          (lambda ()
+            (map (lambda (e)
+                   (write-config-entry (car e) (cdr e)))
+                 `(("ROOT_DOMAIN" . ,root-domain)
+                   ("APP_CONFIGS" . ,(string-intersperse
+                                      (map (lambda (app-version)
+                                             (conc (if (eq? (car app-version) 'log-viewer) 'dozzle (car app-version))
+                                                   ","
+                                                   (or (alist-ref 'subdomain (alist-ref (car app-version) config))
+                                                       "nassella-ignore") ;; a hack to get authelia to work for now
+                                                   ","
+                                                   (cdr app-version)))
+                                           selected-apps)
+                                      " "))
+                   ("HOST_ADMIN_USER" . ,(alist-ref 'user (alist-ref 'log-viewer config)))
+                   ("HOST_ADMIN_PASSWORD" . ,(create-authelia-password-hash (alist-ref 'password (alist-ref 'log-viewer config))))
+                   ("NEXTCLOUD_ADMIN_USER" . ,(alist-ref 'admin-user (alist-ref 'nextcloud config)))
+                   ("NEXTCLOUD_ADMIN_PASSWORD" . ,(alist-ref 'admin-password (alist-ref 'nextcloud config)))
+                   ("NEXTCLOUD_POSTGRES_DB" . "nextcloud")
+                   ("NEXTCLOUD_POSTGRES_USER" . "nextcloud")
+                   ("NEXTCLOUD_POSTGRES_PASSWORD" . ,(alist-ref 'postgres-password (alist-ref 'nextcloud config)))
+                   ("NEXTCLOUD_REDIS_PASSWORD" . ,(alist-ref 'redis-password (alist-ref 'nextcloud config)))
+                   ("GHOST_DATABASE_ROOT_PASSWORD" . ,(alist-ref 'postgres-root-password (alist-ref 'ghost config)))
+                   ("GHOST_DATABASE_PASSWORD" . ,(alist-ref 'postgres-password (alist-ref 'ghost config)))
+                   ("NASSELLA_LLDAP_SUBDOMAIN" . ,(alist-ref 'lldap-subdomain (alist-ref 'nassella config)))
+                   ("NASSELLA_POSTGRES_DB" . "nassella")
+                   ("NASSELLA_POSTGRES_USER" . "nassella")
+                   ("NASSELLA_POSTGRES_PASSWORD" . ,(alist-ref 'postgres-password (alist-ref 'nassella config)))
+                   ("NASSELLA_AUTHELIA_POSTGRES_DB" . "authelia")
+                   ("NASSELLA_AUTHELIA_POSTGRES_USER" . "authelia")
+                   ("NASSELLA_AUTHELIA_POSTGRES_PASSWORD" . ,(alist-ref 'authelia-postgres-password (alist-ref 'nassella config)))
+                   ("NASSELLA_LLDAP_POSTGRES_DB" . "lldap")
+                   ("NASSELLA_LLDAP_POSTGRES_USER" . "lldap")
+                   ("NASSELLA_LLDAP_POSTGRES_PASSWORD" . ,(alist-ref 'lldap-postgres-password (alist-ref 'nassella config)))
+                   ("NASSELLA_LLDAP_JWT_SECRET" . ,(alist-ref 'lldap-jwt-secret (alist-ref 'nassella config)))
+                   ("NASSELLA_LLDAP_KEY_SEED" . ,(alist-ref 'lldap-key-seed (alist-ref 'nassella config)))
+                   ("NASSELLA_LLDAP_ADMIN_PASSWORD" . ,(alist-ref 'lldap-admin-password (alist-ref 'nassella config)))
+                   ("NASSELLA_STRIPE_API_KEY" . ,(alist-ref 'stripe-api-key (alist-ref 'nassella config)))
+                   ("NASSELLA_AUTHELIA_JWT_SECRET" . ,(alist-ref 'authelia-jwt-secret (alist-ref 'nassella config)))
+                   ("NASSELLA_AUTHELIA_KEY_SEED" . ,(alist-ref 'authelia-key-seed (alist-ref 'nassella config)))
+                   ("WORDPRESS_DB_PASSWORD" . ,(alist-ref 'db-password (alist-ref 'wordpress config)))
+                   ("WORDPRESS_DB_ROOT_PASSWORD" . ,(alist-ref 'db-root-password (alist-ref 'wordpress config)))
+                   ("LLDAP_POSTGRES_DB" . "lldap")
+                   ("LLDAP_POSTGRES_USER" . "lldap")
+                   ("LLDAP_POSTGRES_PASSWORD" . ,(alist-ref 'db-password (alist-ref 'lldap config)))
+                   ("LLDAP_JWT_SECRET" . ,(alist-ref 'jwt-secret (alist-ref 'lldap config)))
+                   ("LLDAP_KEY_SEED" . ,(alist-ref 'key-seed (alist-ref 'lldap config)))
+                   ("LLDAP_ADMIN_PASSWORD" . ,(alist-ref 'admin-password (alist-ref 'lldap config)))
+                   ("LLDAP_USER_EMAIL" . ,(alist-ref 'user-email (alist-ref 'lldap config)))
+                   ("AUTHELIA_POSTGRES_DB" . "authelia")
+                   ("AUTHELIA_POSTGRES_USER" . "authelia")
+                   ("AUTHELIA_POSTGRES_PASSWORD" . ,(alist-ref 'db-password (alist-ref 'authelia config)))
+                   ("AUTHELIA_JWT_SECRET" . ,(alist-ref 'jwt-secret (alist-ref 'authelia config)))
+                   ("AUTHELIA_SESSION_SECRET" . ,(alist-ref 'session-secret (alist-ref 'authelia config)))
+                   ("AUTHELIA_ENCRYPTION_KEY" . ,(alist-ref 'encryption-key (alist-ref 'authelia config)))
+                   ("SMTP_HOST" . ,(alist-ref 'smtp-host (alist-ref 'all-apps config)))
+                   ("SMTP_PORT" . ,(alist-ref 'smtp-port (alist-ref 'all-apps config)))
+                   ("SMTP_AUTH_USER" . ,(alist-ref 'smtp-auth-user (alist-ref 'all-apps config)))
+                   ("SMTP_AUTH_PASSWORD" . ,(alist-ref 'smtp-auth-password (alist-ref 'all-apps config)))
+                   ("SMTP_FROM" . ,(alist-ref 'smtp-from (alist-ref 'all-apps config)))
+                   ("BACKBLAZE_KEY_ID" . ,(alist-ref 'backblaze-key-id service-config))
+                   ("BACKBLAZE_APPLICATION_KEY" . ,(alist-ref 'backblaze-application-key service-config))
+                   ("BACKBLAZE_BUCKET_URL" . ,(alist-ref 'backblaze-bucket-url service-config))
+                   ("RESTIC_PASSWORD" . ,restic-password)
+                   ("INSTANCE_CONTROL_WEBHOOKS_SECRET" . ,(alist-ref 'webhooks-secret (alist-ref 'instance-control config)))
+                   ,@(if (and restic-snapshot-id (not (string=? restic-snapshot-id ""))) `(("RESTIC_SNAPSHOT_ID" . ,restic-snapshot-id)) '())))))
+        (with-output-to-file (string-append dir "/config/production.tfvars")
+          (lambda ()
+            (map (lambda (e)
+                   (write-terraform-config-entry (car e) (cdr e)))
+                 `(("server_type" . ,(alist-ref 'digitalocean-size service-config))
+                   ("do_token" . ,(alist-ref 'digitalocean-api-token service-config))
+                   ("digitalocean_volume_size" . ,(alist-ref 'digitalocean-volume-size service-config))
+                   ("cloudflare_api_token" . ,(alist-ref 'cloudflare-api-token service-config))
+                   ("cloudflare_zone_id" . ,(alist-ref 'cloudflare-zone-id service-config))
+                   ("cloudflare_account_id" . ,(alist-ref 'cloudflare-account-id service-config))
+                   ("cluster_name" . ,(string-append "nassella" (string-delete #\. root-domain))) ;; TODO update to: (import srfi-14) (string-filter char-set:letter+digit root-domain)
+                   ("datacenter" . ,(alist-ref 'digitalocean-region service-config))
+                   ;; (source <(curl -sSfL https://stable.release.flatcar-linux.net/amd64-usr/current/version.txt); echo "${FLATCAR_VERSION_ID}")
+                   ("flatcar_stable_version" . "4593.2.4")))
+            ;; remove the newline that generating the ssh key adds
+            (display "ssh_keys=[\"") (display (string-drop-right ssh-pub-key 1)) (print "\"]"))))
+      (let* ((user-id (session-user-id))
+             (app-config
+              (with-db/transaction
+               (lambda (db)
+                 (get-user-app-config db (session-user-id) instance-id))))
+             (first-deployment? (not (with-db/transaction (lambda (db) (get-most-recent-deployment-status db user-id instance-id)))))
+             (deployment-id (with-db/transaction (lambda (db) (create-deployment db user-id instance-id))))
+             (dir (deployment-directory user-id instance-id))
+             (backup-request-id (conc (truncate (time->seconds (current-time))) "-" (pseudo-random-integer 10000))))
+        (if first-deployment?
+            (with-db/transaction
+             (lambda (db)
+               (update-deployment-progress db deployment-id '((instance-backup . ignored)
+                                                              (machine-destroy . ignored)
+                                                              (ip-destroy . ignored)
+                                                              (volume-destroy . ignored)))))
+            (begin
+              (with-db/transaction
+               (lambda (db)
+                 (update-deployment-progress db deployment-id '((instance-backup . in-progress)))))
+              (handle-exceptions
+                  exn
+                (with-db/transaction
+                 (lambda (db)
+                   (update-deployment-progress db deployment-id '((instance-backup . failed)))))
+                (send-instance-control-command
+                 (alist-ref 'root-domain app-config)
+                 (alist-ref 'subdomain (alist-ref 'instance-control (alist-ref 'config app-config)))
+                 "queue-restic-snapshot-no-restart"
+                 (alist-ref 'webhooks-secret (alist-ref 'instance-control (alist-ref 'config app-config)))
+                 `((path . "/")
+                   (tag . "automated_pre_instance_update")
+                   ;; effectively a guid, we just want something unique
+                   (request_id . ,backup-request-id)
+                   (version . 0))))))
+        (thread-start!
+         (lambda ()
+           (when (not first-deployment?)
+             (let ((start-time (time->seconds (current-time))))
+               (let loop ()
+                 (thread-sleep! 1)
+                 (let* ((status-result
+                         (handle-exceptions
+                             exn
+                           '((status . "error"))
+                           (send-instance-control-command
+                            (alist-ref 'root-domain app-config)
+                            (alist-ref 'subdomain (alist-ref 'instance-control (alist-ref 'config app-config)))
+                            "restic-snapshot-status"
+                            (alist-ref 'webhooks-secret (alist-ref 'instance-control (alist-ref 'config app-config)))
+                            `((request_id . ,backup-request-id)
+                              (version . 0)))))
+                        (complete (string=? (alist-ref 'status status-result) "complete")))
+                   (if (or complete (> (- (time->seconds (current-time)) start-time) 120))
+                       (with-db/transaction
+                        (lambda (db)
+                          (update-deployment-progress db deployment-id `((instance-backup . ,(or (and complete 'complete) 'failed))))))
+                       (loop))))))
+           (change-directory dir)
+           (let ((pid (process-run "make preapply && make apply > make-out 2>&1")))
+             (with-db/transaction (lambda (db) (update-deployment-in-progress db deployment-id pid)))
+             (change-directory "../")
+             (let loop ()
+               (thread-sleep! 5)
+               (receive (pid exit-normal status) (process-wait pid #t)
+                 (if (= pid 0) ;; process is still running
+                     (begin (let ((progress (parse-deployment-log
+                                             (with-input-from-file
+                                                 (string-append (deployment-directory user-id instance-id) "/make-out")
+                                               read-string)))
+                                  (tf-state (with-input-from-file (string-append dir "/terraform.tfstate") read-string))
+                                  (tf-state-backup (with-input-from-file (string-append dir "/terraform.tfstate.backup") read-string)))
+                              (with-db/transaction
+                               (lambda (db)
+                                 (update-deployment-progress db deployment-id progress)
+                                 (when (file-exists? (string-append dir "/terraform.tfstate"))
+                                   (update-user-terraform-state db user-id instance-id
+                                                                (if (eof-object? tf-state) "" tf-state)
+                                                                (if (eof-object? tf-state-backup) "" tf-state-backup))))))
+                            (loop))
+                     (let ((progress (parse-deployment-log
+                                      (with-input-from-file
+                                          (string-append (deployment-directory user-id instance-id) "/make-out")
+                                        read-string)))
+                           (tf-state (with-input-from-file (string-append dir "/terraform.tfstate") read-string))
+                           (tf-state-backup (with-input-from-file (string-append dir "/terraform.tfstate.backup") read-string)))
+                       (with-db/transaction
+                        (lambda (db)
+                          (update-deployment-progress db deployment-id progress)
+                          (update-user-terraform-state db user-id instance-id
+                                                       (if (eof-object? tf-state) "" tf-state)
+                                                       (if (eof-object? tf-state-backup) "" tf-state-backup))
+                          (if (= status 0)
+                              (update-deployment-progress db deployment-id `((instance-up . in-progress)))
+                              (update-deployment-status
+                               db user-id deployment-id
+                               'failed
+                               (with-input-from-file (string-append dir "/make-out") read-string)))))
+                       (when (= status 0)
+                         (let ((instance-up (instance-up? (string-append "https://"
+                                                                         (alist-ref 'subdomain (alist-ref 'log-viewer (alist-ref 'config app-config)))
+                                                                         "."
+                                                                         (alist-ref 'root-domain app-config)))))
+                           (with-db/transaction
+                            (lambda (db)
+                              (update-deployment-progress db deployment-id `((instance-up . ,(if instance-up 'complete 'failed))))
+                              (update-deployment-status
+                               db user-id deployment-id
+                               (if instance-up
+                                   'complete
+                                   'failed)
+                               (with-input-from-file (string-append dir "/make-out") read-string))))))))))))))))
+  (redirect (conc "/config/wizard/success/" instance-id)))
+
 (with-schematra-app app
   (lambda ()
 
@@ -1221,6 +1462,122 @@ chmod -R 777 /opt/keys")))
                                          (generate-restic-password))))))
    (redirect (conc "/config/wizard/services/" instance-id))))
 
+
+;; TODOs
+;; backblaze: create bucket then generate a new key restricted to that bucket and only use that key
+;; postmark:
+;;   - use API to create an email server for instance
+;;   - use API to generate SMTP settings for instance
+;; nassella:
+;;   - generate instance name and verify it does not already exist
+;;     - store that instance name in a new db column
+;;     - update dashboard to use instance name
+;;   - (later) detect closest digitalocean region to user
+;;   - (later) ensure can select the correct digitalocean size
+'x
+(post "/config/wizard/create-instance-full"
+ (let* ((ssh-keys (generate-ssh-key (session-user-id)))
+        (instance-id (with-db/transaction
+                      (lambda (db)
+                        (create-instance db (session-user-id) (first ssh-keys) (second ssh-keys)
+                                         (generate-restic-password)))))
+        (instance-name "red1")
+        (root-domain "nassellatest.com")
+        (backblaze-bucket-name (string-append "nassella-instance-" instance-name))
+        (backblaze-application-key "K004v8YQPbm/MdmyoffQBH8Cof9Wt7M") ;; TODO need to generate a restricted key with access to only the new bucket and use that one so we don't leak our key
+        (backblaze-key-id "004da8e20ea1d5e000000000f")
+        (backblaze-auth (b2-authorize-account backblaze-key-id backblaze-application-key))
+        (backblaze-bucket-response
+         (b2-create-bucket (alist-ref 'apiUrl (alist-ref 'storageApi (alist-ref 'apiInfo backblaze-auth)))
+                           (alist-ref 'authorizationToken backblaze-auth)
+                           (alist-ref 'accountId backblaze-auth)
+                           backblaze-bucket-name))
+        (backblaze-s3-url (alist-ref 's3ApiUrl (alist-ref 'storageApi (alist-ref 'apiInfo backblaze-auth)))))
+   (with-db/transaction
+    (lambda (db)
+      (update-user-service-config
+       db
+       (session-user-id)
+       instance-id
+       `((cloudflare-api-token . "cfut_ykuHdXr5P8Nm3SFCM2uQTErxKZ24pZzb98wZ7cUC76189f76")
+         (cloudflare-account-id . "f6542e441e20c201803398433cb923a8")
+         (cloudflare-zone-id . "e126d94dc5cdd32503749b55a9d64faf")
+         (digitalocean-api-token . "dop_v1_44a6f12f1bab78c9c3a4ba4edc7d91c3c98b4ddd0e08f3c80e0689431feb88d0")
+         (backblaze-application-key . ,backblaze-application-key)
+         (backblaze-key-id . ,backblaze-key-id)
+         (backblaze-bucket-name . ,backblaze-bucket-name)
+         (backblaze-bucket-id . ,(alist-ref 'bucketId backblaze-bucket-response))
+         (backblaze-bucket-url . ,(string-append backblaze-s3-url "/" backblaze-bucket-name))
+         (digitalocean-region . "sfo3") ;; TODO detect closest region to user
+         (digitalocean-size . "s-2vcpu-2gb") ;; TODO make sure size is available, align to nassella plan?
+         (digitalocean-volume-size . "60") ;; TODO align to plan?
+         ))
+      (update-root-domain db (session-user-id) instance-id root-domain)
+      (update-user-selected-apps
+       db
+       (session-user-id)
+       instance-id
+       (filter (lambda (app/version) (cdr app/version)) ;; filter out unused apps
+               `((wg-easy . #f)
+                 (nextcloud . "34")
+                 (ghost . #f)
+                 (nassella . #f)
+                 (wordpress . #f)
+                 (lldap . #f)
+                 (authelia . "4")
+                 (instance-control . "b0.0.1")
+                 (log-viewer . "20"))))
+      (update-user-app-config
+       db
+       (session-user-id)
+       instance-id
+       `((nextcloud . ((subdomain . ,(string-append "nextcloud." instance-name))
+                       (admin-user . ,(session-username))
+                       (admin-password . "Foodisgood4!") ;; TODO
+                       (postgres-password . ,(generate-postgres-password))
+                       (redis-password . ,(generate-redis-password))))
+         (authelia . ((subdomain . #f)
+                      (db-password . ,(generate-postgres-password))
+                      (jwt-secret . ,(generate-jwt-secret))
+                      (session-secret . ,(generate-authelia-key-seed))
+                      (encryption-key . ,(generate-authelia-key-seed))))
+         (log-viewer . ((subdomain . ,(string-append "logs." instance-name))
+                        (user . ,(session-username))
+                        (password . "Foodisgood4!"))) ;; TODO
+         (all-apps . ((smtp-host . "smtp.postmarkapp.com")
+                      (smtp-port . "587")
+                      (smtp-auth-user . "PM-T-outbound-L0Ps1dQBMdx2u20RZ9-BsF") ;; TODO
+                      (smtp-auth-password . "alb0YXkEOmp0htTm4-kV_OGLjQbjb7NRwLN2") ;; TODO
+                      (smtp-from . ,(string-append "admin@" root-domain))))
+         (instance-control . ((subdomain . ,(string-append "nassella-instance-control." instance-name))
+                              (webhooks-secret . ,(generate-jwt-secret))))
+         ;; unused but rest of system expects to be set
+         (ghost . ((subdomain . #f)
+                   (postgres-root-password . ,(generate-postgres-password))
+                   (postgres-password . ,(generate-postgres-password))))
+         (wg-easy . ((subdomain . #f)))
+         (nassella . ((subdomain . #f)
+                      (postgres-password . ,(generate-postgres-password))
+                      (authelia-postgres-password . ,(generate-postgres-password))
+                      (lldap-postgres-password . ,(generate-postgres-password))
+                      (lldap-jwt-secret . ,(generate-jwt-secret))
+                      (lldap-key-seed . ,(generate-key-seed))
+                      (lldap-subdomain . #f)
+                      (lldap-admin-password . "")
+                      (stripe-api-key . "")
+                      (authelia-jwt-secret . ,(generate-jwt-secret))
+                      (authelia-key-seed . ,(generate-authelia-key-seed))))
+         (wordpress . ((subdomain . #f)
+                       (db-password . ,(generate-postgres-password))
+                       (db-root-password . ,(generate-postgres-password))))
+         (lldap . ((subdomain . #f)
+                   (db-password . ,(generate-postgres-password))
+                   (jwt-secret . ,(generate-jwt-secret))
+                   (key-seed . ,(generate-key-seed))
+                   (admin-password . "")
+                   (user-email . "")))))))
+   (perform-review-submit instance-id)))
+
 (get/widgets
  ("/config/wizard/services/:id")
  (let* ((instance-id (alist-ref "id" (current-params) equal?))
@@ -1401,8 +1758,9 @@ chmod -R 777 /opt/keys")))
               '()))
          (Field (@ (name "wordpress") (type "checkbox") (label ("Wordpress")) (checked ,(member 'wordpress (alist-ref 'selected-apps results)))))
          (Field (@ (name "log-viewer") (type "checkbox") (label ("Log Viewer")) (checked #t) (disabled "disabled")))
-         (Field (@ (name "lldap") (type "checkbox") (label ("Admin LLDAP")) (checked #t) (disabled "disabled")))
-         (Field (@ (name "authelia") (type "checkbox") (label ("Admin Authelia")) (checked #t) (disabled "disabled"))))
+         ;; (Field (@ (name "lldap") (type "checkbox") (label ("Admin LLDAP")) (checked #t) (disabled "disabled")))
+         ;; (Field (@ (name "authelia") (type "checkbox") (label ("Admin Authelia")) (checked #t) (disabled "disabled")))
+         )
         ;; TODO add config for when automatic upgrades are scheduled for?
         ;; TODO add config for server timezone?
         (Form-Nav (@ (back-to ,(conc "/config/wizard/services-success/" instance-id))))))))))
@@ -1424,7 +1782,7 @@ chmod -R 777 /opt/keys")))
                   (ghost . ,(or (and (alist-ref 'ghost (current-params)) "6") #f))
                   (nassella . ,(or (and (alist-ref 'nassella (current-params)) "b0.0.1") #f))
                   (wordpress . ,(or (and (alist-ref 'wordpress (current-params)) "8.4") #f))
-                  (lldap . "0.6.3")
+                  (lldap . #f) ;; "0.6.3"
                   (authelia . "4")
                   (instance-control . "b0.0.1")
                   (log-viewer . "20"))))
@@ -1500,31 +1858,30 @@ chmod -R 777 /opt/keys")))
                  (Field (@ (name "wordpress-subdomain") (label ("Subdomain")) (required #t)
                            (value ,(alist-ref 'subdomain (alist-ref 'wordpress app-config eq? '()) eq? "wordpress"))))))
               '())
-        ,@(if (member 'lldap selected-apps)
-              `((Fieldset
-                 (@ (title "Admin LLDAP"))
-                 (Field (@ (name "lldap-subdomain") (label ("Subdomain")) (required #t)
-                           (value ,(alist-ref 'subdomain (alist-ref 'lldap app-config eq? '()) eq? "lldap"))))
-                 (Field (@ (name "lldap-user-email") (label ("Admin Email Address")) (type "email") (required #t)
-                           (value ,(alist-ref 'user-email (alist-ref 'lldap app-config eq? '()) eq? ""))))
-                 (Field (@ (name "lldap-admin-password") (label ("Admin Password")) (type "password") (required #t)
-                           (value ,(alist-ref 'admin-password (alist-ref 'lldap app-config eq? '()) eq? ""))))))
-              '())
-        ,@(if (member 'authelia selected-apps)
-              `((Fieldset
-                 (@ (title "Admin Authelia"))
-                 (Field (@ (name "authelia-subdomain") (label ("Subdomain")) (required #t)
-                           (value ,(alist-ref 'subdomain (alist-ref 'authelia app-config eq? '()) eq? "authelia"))))))
-              '())
+        ;; ,@(if (member 'lldap selected-apps)
+        ;;       `((Fieldset
+        ;;          (@ (title "Admin LLDAP"))
+        ;;          (Field (@ (name "lldap-subdomain") (label ("Subdomain")) (required #t)
+        ;;                    (value ,(alist-ref 'subdomain (alist-ref 'lldap app-config eq? '()) eq? "lldap"))))
+        ;;          (Field (@ (name "lldap-user-email") (label ("Admin Email Address")) (type "email") (required #t)
+        ;;                    (value ,(alist-ref 'user-email (alist-ref 'lldap app-config eq? '()) eq? ""))))
+        ;;          (Field (@ (name "lldap-admin-password") (label ("Admin Password")) (type "password") (required #t)
+        ;;                    (value ,(alist-ref 'admin-password (alist-ref 'lldap app-config eq? '()) eq? ""))))))
+        ;;       '())
+        ;; ,@(if (member 'authelia selected-apps)
+        ;;       `((Fieldset
+        ;;          (@ (title "Admin Authelia"))
+        ;;          (Field (@ (name "authelia-subdomain") (label ("Subdomain")) (required #t)
+        ;;                    (value ,(alist-ref 'subdomain (alist-ref 'authelia app-config eq? '()) eq? "authelia"))))))
+        ;;       '())
         (Fieldset
          (@ (title "Log Viewer"))
          (Field (@ (name "log-viewer-subdomain") (label ("Subdomain")) (required #t)
                    (value ,(alist-ref 'subdomain (alist-ref 'log-viewer app-config eq? '()) eq? "logs"))))
-         ;; (Field (@ (name "log-viewer-user") (label ("Username")) (required #t)
-         ;;           (value ,(alist-ref 'user (alist-ref 'log-viewer app-config eq? '()) eq? ""))))
-         ;; (Field (@ (name "log-viewer-password") (label ("Password")) (type "password") (required #t)
-         ;;           (value ,(alist-ref 'password (alist-ref 'log-viewer app-config eq? '()) eq? ""))))
-         )
+         (Field (@ (name "log-viewer-user") (label ("Admin Email")) (required #t)
+                   (value ,(alist-ref 'user (alist-ref 'log-viewer app-config eq? '()) eq? ""))))
+         (Field (@ (name "log-viewer-password") (label ("Admin Password")) (type "password") (required #t)
+                   (value ,(alist-ref 'password (alist-ref 'log-viewer app-config eq? '()) eq? "")))))
         ,@(if (or (member 'nextcloud selected-apps) (member 'ghost selected-apps) (member 'nassella selected-apps) (member 'authelia selected-apps))
               `((Fieldset
                  (@ (title "All Apps - Email - SMTP"))
@@ -1662,8 +2019,8 @@ chmod -R 777 /opt/keys")))
                                                                            (alist-ref 'authelia config eq? '()))
                                                                 (generate-authelia-key-seed)))))
                             (log-viewer . ((subdomain . ,(alist-ref 'log-viewer-subdomain (current-params)))
-                                           (user . "unused") ;; TODO can we remove?
-                                           (password . "unused"))) ;; TODO can we remove?
+                                           (user . ,(alist-ref 'log-viewer-user (current-params)))
+                                           (password . ,(alist-ref 'log-viewer-password (current-params)))))
                             (all-apps . ((smtp-host . ,(alist-ref 'smtp-host (current-params)))
                                          (smtp-port . ,(alist-ref 'smtp-port (current-params)))
                                          (smtp-auth-user . ,(alist-ref 'smtp-auth-user (current-params)))
@@ -1798,215 +2155,7 @@ chmod -R 777 /opt/keys")))
         (Form-Nav (@ (back-to ,(conc "/config/wizard/machine2/" instance-id)) (submit-button "Launch")))))))))
 
 (post "/config/wizard/review-submit/:id"
-  (let* ((instance-id (alist-ref "id" (current-params) equal?))
-         (status (string->symbol
-                  (->string
-                   (with-db/transaction
-                    (lambda (db)
-                      (get-most-recent-deployment-status db (session-user-id) instance-id)))))))
-    (when (or (not (or (eq? status 'queued) (eq? status 'in-progress)))
-              (equal? (alist-ref 'force (current-params)) "true"))
-      (let* ((instance-id (alist-ref "id" (current-params) equal?))
-             (restic-snapshot-id (alist-ref 'restic-snapshot-id (current-params)))
-             (results
-              (with-db/transaction
-               (lambda (db)
-                 `((selected-apps . ,(get-user-selected-apps db (session-user-id) instance-id))
-                   (app-config . ,(get-user-app-config db (session-user-id) instance-id))
-                   (service-config . ,(get-user-service-config db (session-user-id) instance-id))
-                   (terraform-state . ,(get-user-terraform-state db (session-user-id) instance-id))
-                   (ssh-pub-key . ,(get-instance-ssh-pub-key db (session-user-id) instance-id))
-                   (restic-password . ,(get-instance-restic-password db (session-user-id) instance-id))))))
-             (selected-apps (alist-ref 'selected-apps results))
-             (app-config (alist-ref 'app-config results))
-             (config (alist-ref 'config app-config))
-             (root-domain (alist-ref 'root-domain app-config))
-             (service-config (alist-ref 'service-config results))
-             (terraform-state (alist-ref 'terraform-state results))
-             (ssh-pub-key (alist-ref 'ssh-pub-key results))
-             (restic-password (alist-ref 'restic-password results))
-             (dir (deployment-directory (session-user-id) instance-id)))
-        (setup-deploy-files dir (alist-ref 'state terraform-state) (alist-ref 'backup terraform-state))
-        (with-output-to-file (string-append dir "/config/apps.config")
-          (lambda ()
-            (map (lambda (e)
-                   (write-config-entry (car e) (cdr e)))
-                 `(("ROOT_DOMAIN" . ,root-domain)
-                   ("APP_CONFIGS" . ,(string-intersperse
-                                      (map (lambda (app-version)
-                                             (conc (if (eq? (car app-version) 'log-viewer) 'dozzle (car app-version))
-                                                   ","
-                                                   (alist-ref 'subdomain (alist-ref (car app-version) config))
-                                                   ","
-                                                   (cdr app-version)))
-                                           selected-apps)
-                                      " "))
-                   ("HOST_ADMIN_USER" . ,(alist-ref 'user (alist-ref 'log-viewer config)))
-                   ("HOST_ADMIN_PASSWORD" . ,(alist-ref 'password (alist-ref 'log-viewer config)))
-                   ("NEXTCLOUD_ADMIN_USER" . ,(alist-ref 'admin-user (alist-ref 'nextcloud config)))
-                   ("NEXTCLOUD_ADMIN_PASSWORD" . ,(alist-ref 'admin-password (alist-ref 'nextcloud config)))
-                   ("NEXTCLOUD_POSTGRES_DB" . "nextcloud")
-                   ("NEXTCLOUD_POSTGRES_USER" . "nextcloud")
-                   ("NEXTCLOUD_POSTGRES_PASSWORD" . ,(alist-ref 'postgres-password (alist-ref 'nextcloud config)))
-                   ("NEXTCLOUD_REDIS_PASSWORD" . ,(alist-ref 'redis-password (alist-ref 'nextcloud config)))
-                   ("GHOST_DATABASE_ROOT_PASSWORD" . ,(alist-ref 'postgres-root-password (alist-ref 'ghost config)))
-                   ("GHOST_DATABASE_PASSWORD" . ,(alist-ref 'postgres-password (alist-ref 'ghost config)))
-                   ("NASSELLA_LLDAP_SUBDOMAIN" . ,(alist-ref 'lldap-subdomain (alist-ref 'nassella config)))
-                   ("NASSELLA_POSTGRES_DB" . "nassella")
-                   ("NASSELLA_POSTGRES_USER" . "nassella")
-                   ("NASSELLA_POSTGRES_PASSWORD" . ,(alist-ref 'postgres-password (alist-ref 'nassella config)))
-                   ("NASSELLA_AUTHELIA_POSTGRES_DB" . "authelia")
-                   ("NASSELLA_AUTHELIA_POSTGRES_USER" . "authelia")
-                   ("NASSELLA_AUTHELIA_POSTGRES_PASSWORD" . ,(alist-ref 'authelia-postgres-password (alist-ref 'nassella config)))
-                   ("NASSELLA_LLDAP_POSTGRES_DB" . "lldap")
-                   ("NASSELLA_LLDAP_POSTGRES_USER" . "lldap")
-                   ("NASSELLA_LLDAP_POSTGRES_PASSWORD" . ,(alist-ref 'lldap-postgres-password (alist-ref 'nassella config)))
-                   ("NASSELLA_LLDAP_JWT_SECRET" . ,(alist-ref 'lldap-jwt-secret (alist-ref 'nassella config)))
-                   ("NASSELLA_LLDAP_KEY_SEED" . ,(alist-ref 'lldap-key-seed (alist-ref 'nassella config)))
-                   ("NASSELLA_LLDAP_ADMIN_PASSWORD" . ,(alist-ref 'lldap-admin-password (alist-ref 'nassella config)))
-                   ("NASSELLA_STRIPE_API_KEY" . ,(alist-ref 'stripe-api-key (alist-ref 'nassella config)))
-                   ("NASSELLA_AUTHELIA_JWT_SECRET" . ,(alist-ref 'authelia-jwt-secret (alist-ref 'nassella config)))
-                   ("NASSELLA_AUTHELIA_KEY_SEED" . ,(alist-ref 'authelia-key-seed (alist-ref 'nassella config)))
-                   ("WORDPRESS_DB_PASSWORD" . ,(alist-ref 'db-password (alist-ref 'wordpress config)))
-                   ("WORDPRESS_DB_ROOT_PASSWORD" . ,(alist-ref 'db-root-password (alist-ref 'wordpress config)))
-                   ("LLDAP_POSTGRES_DB" . "lldap")
-                   ("LLDAP_POSTGRES_USER" . "lldap")
-                   ("LLDAP_POSTGRES_PASSWORD" . ,(alist-ref 'db-password (alist-ref 'lldap config)))
-                   ("LLDAP_JWT_SECRET" . ,(alist-ref 'jwt-secret (alist-ref 'lldap config)))
-                   ("LLDAP_KEY_SEED" . ,(alist-ref 'key-seed (alist-ref 'lldap config)))
-                   ("LLDAP_ADMIN_PASSWORD" . ,(alist-ref 'admin-password (alist-ref 'lldap config)))
-                   ("LLDAP_USER_EMAIL" . ,(alist-ref 'user-email (alist-ref 'lldap config)))
-                   ("AUTHELIA_POSTGRES_DB" . "authelia")
-                   ("AUTHELIA_POSTGRES_USER" . "authelia")
-                   ("AUTHELIA_POSTGRES_PASSWORD" . ,(alist-ref 'db-password (alist-ref 'authelia config)))
-                   ("AUTHELIA_JWT_SECRET" . ,(alist-ref 'jwt-secret (alist-ref 'authelia config)))
-                   ("AUTHELIA_SESSION_SECRET" . ,(alist-ref 'session-secret (alist-ref 'authelia config)))
-                   ("AUTHELIA_ENCRYPTION_KEY" . ,(alist-ref 'encryption-key (alist-ref 'authelia config)))
-                   ("SMTP_HOST" . ,(alist-ref 'smtp-host (alist-ref 'all-apps config)))
-                   ("SMTP_PORT" . ,(alist-ref 'smtp-port (alist-ref 'all-apps config)))
-                   ("SMTP_AUTH_USER" . ,(alist-ref 'smtp-auth-user (alist-ref 'all-apps config)))
-                   ("SMTP_AUTH_PASSWORD" . ,(alist-ref 'smtp-auth-password (alist-ref 'all-apps config)))
-                   ("SMTP_FROM" . ,(alist-ref 'smtp-from (alist-ref 'all-apps config)))
-                   ("BACKBLAZE_KEY_ID" . ,(alist-ref 'backblaze-key-id service-config))
-                   ("BACKBLAZE_APPLICATION_KEY" . ,(alist-ref 'backblaze-application-key service-config))
-                   ("BACKBLAZE_BUCKET_URL" . ,(alist-ref 'backblaze-bucket-url service-config))
-                   ("RESTIC_PASSWORD" . ,restic-password)
-                   ("INSTANCE_CONTROL_WEBHOOKS_SECRET" . ,(alist-ref 'webhooks-secret (alist-ref 'instance-control config)))
-                   ,@(if (and restic-snapshot-id (not (string=? restic-snapshot-id ""))) `(("RESTIC_SNAPSHOT_ID" . ,restic-snapshot-id)) '())))))
-        (with-output-to-file (string-append dir "/config/production.tfvars")
-          (lambda ()
-            (map (lambda (e)
-                   (write-terraform-config-entry (car e) (cdr e)))
-                 `(("server_type" . ,(alist-ref 'digitalocean-size service-config))
-                   ("do_token" . ,(alist-ref 'digitalocean-api-token service-config))
-                   ("digitalocean_volume_size" . ,(alist-ref 'digitalocean-volume-size service-config))
-                   ("cloudflare_api_token" . ,(alist-ref 'cloudflare-api-token service-config))
-                   ("cloudflare_zone_id" . ,(alist-ref 'cloudflare-zone-id service-config))
-                   ("cloudflare_account_id" . ,(alist-ref 'cloudflare-account-id service-config))
-                   ("cluster_name" . ,(string-append "nassella" (string-delete #\. root-domain))) ;; TODO update to: (import srfi-14) (string-filter char-set:letter+digit root-domain)
-                   ("datacenter" . ,(alist-ref 'digitalocean-region service-config))
-                   ;; (source <(curl -sSfL https://stable.release.flatcar-linux.net/amd64-usr/current/version.txt); echo "${FLATCAR_VERSION_ID}")
-                   ("flatcar_stable_version" . "4593.2.4")))
-            ;; remove the newline that generating the ssh key adds
-            (display "ssh_keys=[\"") (display (string-drop-right ssh-pub-key 1)) (print "\"]"))))
-      (let* ((instance-id (alist-ref "id" (current-params) equal?))
-             (user-id (session-user-id))
-             (app-config
-              (with-db/transaction
-               (lambda (db)
-                 (get-user-app-config db (session-user-id) instance-id))))
-             (first-deployment? (not (with-db/transaction (lambda (db) (get-most-recent-deployment-status db user-id instance-id)))))
-             (deployment-id (with-db/transaction (lambda (db) (create-deployment db user-id instance-id))))
-             (dir (deployment-directory user-id instance-id))
-             (backup-request-id (conc (truncate (time->seconds (current-time))) "-" (pseudo-random-integer 10000))))
-        (if first-deployment?
-            (with-db/transaction
-               (lambda (db)
-                 (update-deployment-progress db deployment-id '((instance-backup . ignored)
-                                                                (machine-destroy . ignored)
-                                                                (ip-destroy . ignored)
-                                                                (volume-destroy . ignored)))))
-            (begin
-              (with-db/transaction
-               (lambda (db)
-                 (update-deployment-progress db deployment-id '((instance-backup . in-progress)))))
-              (handle-exceptions
-                  exn
-                (with-db/transaction
-                 (lambda (db)
-                   (update-deployment-progress db deployment-id '((instance-backup . failed)))))
-                (send-instance-control-command
-                 (alist-ref 'root-domain app-config)
-                 (alist-ref 'subdomain (alist-ref 'instance-control (alist-ref 'config app-config)))
-                 "queue-restic-snapshot-no-restart"
-                 (alist-ref 'webhooks-secret (alist-ref 'instance-control (alist-ref 'config app-config)))
-                 `((path . "/")
-                   (tag . "automated_pre_instance_update")
-                   ;; effectively a guid, we just want something unique
-                   (request_id . ,backup-request-id)
-                   (version . 0))))))
-        (thread-start!
-         (lambda ()
-           (when (not first-deployment?)
-             (let ((start-time (time->seconds (current-time))))
-               (let loop ()
-                 (thread-sleep! 1)
-                 (let* ((status-result
-                         (handle-exceptions
-                             exn
-                           '((status . "error"))
-                           (send-instance-control-command
-                            (alist-ref 'root-domain app-config)
-                            (alist-ref 'subdomain (alist-ref 'instance-control (alist-ref 'config app-config)))
-                            "restic-snapshot-status"
-                            (alist-ref 'webhooks-secret (alist-ref 'instance-control (alist-ref 'config app-config)))
-                            `((request_id . ,backup-request-id)
-                              (version . 0)))))
-                        (complete (string=? (alist-ref 'status status-result) "complete")))
-                   (if (or complete (> (- (time->seconds (current-time)) start-time) 120))
-                       (with-db/transaction
-                        (lambda (db)
-                          (update-deployment-progress db deployment-id `((instance-backup . ,(or (and complete 'complete) 'failed))))))
-                       (loop))))))
-           (change-directory dir)
-           (let ((pid (process-run "make preapply && make apply > make-out 2>&1")))
-             (with-db/transaction (lambda (db) (update-deployment-in-progress db deployment-id pid)))
-             (change-directory "../")
-             (let loop ()
-               (thread-sleep! 5)
-               (receive (pid exit-normal status) (process-wait pid #t)
-                 (if (= pid 0) ;; process is still running
-                     (begin (let ((progress (parse-deployment-log
-                                             (with-input-from-file
-                                                 (string-append (deployment-directory user-id instance-id) "/make-out")
-                                               read-string)))
-                                  (tf-state (with-input-from-file (string-append dir "/terraform.tfstate") read-string))
-                                  (tf-state-backup (with-input-from-file (string-append dir "/terraform.tfstate.backup") read-string)))
-                              (with-db/transaction
-                               (lambda (db)
-                                 (update-deployment-progress db deployment-id progress)
-                                 (when (file-exists? (string-append dir "/terraform.tfstate"))
-                                   (update-user-terraform-state db user-id instance-id
-                                                                (if (eof-object? tf-state) "" tf-state)
-                                                                (if (eof-object? tf-state-backup) "" tf-state-backup))))))
-                            (loop))
-                     (let ((progress (parse-deployment-log
-                                      (with-input-from-file
-                                          (string-append (deployment-directory user-id instance-id) "/make-out")
-                                        read-string)))
-                           (tf-state (with-input-from-file (string-append dir "/terraform.tfstate") read-string))
-                           (tf-state-backup (with-input-from-file (string-append dir "/terraform.tfstate.backup") read-string)))
-                       (with-db/transaction
-                        (lambda (db)
-                          (update-deployment-progress db deployment-id progress)
-                          (update-deployment-status
-                           db user-id deployment-id
-                           (if (= status 0) 'complete 'failed)
-                           (with-input-from-file (string-append dir "/make-out") read-string))
-                          (update-user-terraform-state db user-id instance-id
-                                                       (if (eof-object? tf-state) "" tf-state)
-                                                       (if (eof-object? tf-state-backup) "" tf-state-backup))))))))))))))
-      (redirect (conc "/config/wizard/success/" (alist-ref "id" (current-params) equal?))))
+  (perform-review-submit (alist-ref "id" (current-params) equal?)))
 
 (get/widgets
  ("/config/wizard/success/:id"
@@ -2057,7 +2206,8 @@ chmod -R 777 /opt/keys")))
                     (ip-destroy . "Instance Mapped IP Disconnect")
                     (ip-create . "Instance Mapped IP Connect")
                     (volume-create . "App Volume Connect")
-                    (machine-destroy . "Cleanup Previous Instance"))))
+                    (machine-destroy . "Cleanup Previous Instance")
+                    (instance-up . "Instance Starting Up"))))
        (form
         (@ (action "/dashboard") (method GET))
         ,@(if (or (eq? (string->symbol status) 'complete) (eq? (string->symbol status) 'failed))
@@ -2081,7 +2231,7 @@ chmod -R 777 /opt/keys")))
 (define (instance-status->pill-type status)
   (case (string->symbol status)
     ((queued) 'info)
-    ((in-progress 'warning))
+    ((in-progress) 'warning)
     ((complete) 'safe)
     ((failed) 'alert)
     (else 'alert)))
@@ -2099,7 +2249,8 @@ chmod -R 777 /opt/keys")))
        (@ (element main))
        (h1 (@ (style ((font-size ,($ 'font.size.xxl))))) "Nassella - Instances")
        (form
-        (@ (action "/config/wizard/create-instance")
+        (@ ;; (action "/config/wizard/create-instance")
+           (action "/config/wizard/create-instance-full")
            (method POST))
         (Button "Setup New Instance"))
        (ul (@ (style ((list-style "none")
@@ -2213,6 +2364,7 @@ chmod -R 777 /opt/keys")))
         (@ (title "Delete Instance"))
         (Field (@ (name "instance-domain") (label ("Type the domain name of the instance to confirm.")) (value "")))
         (Field (@ (name "delete-backups") (label ("Delete backups for this instance")) (type "checkbox"))))
+       (input (@ (type "hidden") (value ,(alist-ref 'force (current-params))) (name "force")))
        (Form-Nav (@ (back-to "/dashboard") (submit-button "Destroy"))))))))
 
 (post "/destroy-submit/:id"