Properly manage deploys via unique folders and db.

.gitignore

@@ -26,4 +26,6 @@ generated.tfvars
 restic-env
 restic-password
 ignition.json
-app
+app
+nassella-latest.tar
+src/deploy*/*

Makefile

@@ -62,7 +62,7 @@ plan: ignition.json $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars
 	bash -c "terraform plan -var-file=<(cat $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars)"
 
 apply: ignition.json $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars
-	bash -c "terraform apply -var-file=<(cat $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars)"
+	bash -c "terraform apply -auto-approve -var-file=<(cat $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars)"
 
 destroy: ignition.json $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars
 	bash -c "terraform destroy -var-file=<(cat $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars)"
@@ -76,6 +76,13 @@ restic-init: $(apps_config) restic-password
 restic-snapshots: $(apps_config) restic-password
 	./restic-snapshots.sh $(apps_config)
 
+.PHONY: archive
+archive:
+	tar -cf nassella-latest.tar all-apps cl.yaml init-restic.sh main.tf make-caddyfile.sh Makefile \
+		make-generated.sh make-nextcloud-env.sh make-restic-generated.sh make-restic-password.sh restic-snapshots.sh \
+		.terraform .terraform.lock.hcl
+	cp nassella-latest.tar src/
+
 ## to help me remember the command to run to test the config locally
 testlocalhost:
 	curl -k --resolve localhost:443:146.190.12.129 https://localhost

src/db-init.sql

@@ -63,3 +63,12 @@ create table deployments(
        );
 
 create index deployments_user_id_idx on deployments (user_id);
+
+create table user_terraform_state(
+       id bigserial primary key,
+       user_id integer unique not null references users on delete cascade,
+       state_enc text,
+       state_backup_enc text
+       );
+
+create unique index user_terraform_state_user_id_idx on user_terraform_state (user_id);

src/db.scm

@@ -14,6 +14,7 @@
      update-deployment-status get-deployment-status
      get-most-recent-deployment-status
      update-deployment-in-progress
+     update-user-terraform-state get-user-terraform-state
      )
 
 (import scheme
@@ -140,7 +141,8 @@ returning users.user_id;"
                      user-iv))))
         (query conn "insert into user_service_configs(user_id) values ($1);" user-id)
         (query conn "insert into user_selected_apps(user_id) values ($1);" user-id)
-        (query conn "insert into user_app_configs(user_id) values ($1);" user-id)))))
+        (query conn "insert into user_app_configs(user_id) values ($1);" user-id)
+        (query conn "insert into user_terraform_state(user_id) values ($1);" user-id)))))
 
 (define *user-service-configs-column-map*
   '((cloudflare-api-token . ("cloudflare_api_token_enc" #t))
@@ -299,16 +301,20 @@ returning users.user_id;"
 (define (create-deployment conn user-id)
   (value-at
    (query conn
-          "insert into deployments(user_id) values($1) returning deployments.id;"
+          "insert into deployments(user_id, started) values($1, now()) returning deployments.id;"
           user-id)))
 (define (update-deployment-in-progress conn deployment-id pid)
   (query conn
          "update deployments set status=$1, pid=$2 where id=$3;"
          (alist-ref 'in-progress *deployment-status*) pid deployment-id))
 
-(define (update-deployment-status conn deployment-id status)
+(define (update-deployment-status conn user-id deployment-id status log)
-  (query conn "update deployments set status=$1 where id=$2;"
+  (receive (user-key user-iv auth-user-id)
-         (alist-ref status *deployment-status*) deployment-id))
+      (get-decrypted-user-key-and-iv conn user-id)
+    (query conn "update deployments set status=$1, log_enc=$2, finished=now() where id=$3;"
+           (alist-ref status *deployment-status*)
+           (user-encrypt-for-db log user-key user-iv user-id)
+           deployment-id)))
 
 (define (get-deployment-status conn deployment-id)
   (value-at (query conn "select status from deployments where id=$1;" deployment-id)))
@@ -316,6 +322,35 @@ returning users.user_id;"
 (define (get-most-recent-deployment-status conn user-id)
   (value-at (query conn "select status from deployments where user_id=$1 order by id DESC limit 1;" user-id)))
 
+(define (update-user-terraform-state conn user-id state backup)
+  (receive (user-key user-iv auth-user-id)
+      (get-decrypted-user-key-and-iv conn user-id)
+    (query conn
+           "update user_terraform_state set state_enc=$1, state_backup_enc=$2 where user_id=$3;"
+           (user-encrypt-for-db state user-key user-iv user-id)
+           (user-encrypt-for-db backup user-key user-iv user-id)
+           user-id)))
+
+(define (get-user-terraform-state conn user-id)
+  (receive (user-key user-iv auth-user-id)
+      (get-decrypted-user-key-and-iv conn user-id)
+    (let ((res (row-alist (query conn
+                                 "select state_enc, state_backup_enc from user_terraform_state where user_id=$1;"
+                                 user-id))))
+      `((state . ,(if (sql-null? (alist-ref 'config_enc res))
+                       ""
+                       (user-decrypt-from-db (alist-ref 'state_enc res) user-key user-iv user-id)))
+        (backup . ,(if (sql-null? (alist-ref 'config_enc res))
+                       ""
+                       (user-decrypt-from-db (alist-ref 'state_backup_enc res) user-key user-iv user-id)))))))
+
+;; (with-db/transaction
+;;  (lambda (db)
+;;    (update-user-terraform-state db 7
+;;                                 (with-input-from-file "src/deploy-bak/terraform.tfstate" read-string)
+;;                                 (with-input-from-file "src/deploy-bak/terraform.tfstate.backup" read-string))))
+;; (with-db/transaction (lambda (db) (get-user-terraform-state db 7)))
+
 ;; (with-db/transaction (lambda (db) (create-deployment db 7)))
 ;; (with-db/transaction (lambda (db) (get-deployment-status db 1)))
 ;; (with-db/transaction (lambda (db) (update-deployment-in-progress db 1 123)))

src/nassella.scm

@@ -7,6 +7,7 @@
         (chicken process)
         (chicken process-context)
         (chicken irregex)
+        (chicken file)
 
         (rename srfi-1 (delete srfi1:delete))
         srfi-18
@@ -757,6 +758,19 @@ h1, h2, h3, h4, h5, h6 {
        (VStack
         (Form-Nav (@ (back-to ,(conc "/config/wizard/machine2")) (submit-button "Launch")))))))))
 
+(define (deployment-directory user-id)
+  (string-append "deploy-" (number->string user-id)))
+
+(define (setup-deploy-files dir state state-backup)
+  (when (directory-exists? dir)
+    (delete-directory dir #t))
+  (create-directory dir)
+  (process-wait (process-run (string-append "tar -xf nassella-latest.tar -C " dir)))
+  (create-directory (string-append dir "/config"))
+  (copy-file "../config/ssh-keys" (string-append dir "/config/ssh-keys")) ;; TODO remove
+  (with-output-to-file (string-append dir "/terraform.tfstate") (lambda () (write-string state)))
+  (with-output-to-file (string-append dir "/terraform.tfstate.backup") (lambda () (write-string state-backup))))
+
 (define (write-config-entry name value)
   (display name)
   (display "=\"")
@@ -772,13 +786,17 @@ h1, h2, h3, h4, h5, h6 {
                                  (filter cdr
                                          (get-user-selected-apps db (session-get "user-id")))))
               (app-config . ,(get-user-app-config db (session-get "user-id")))
-              (service-config . ,(get-user-service-config db (session-get "user-id")))))))
+              (service-config . ,(get-user-service-config db (session-get "user-id")))
+              (terraform-state . ,(get-user-terraform-state db (session-get "user-id")))))))
         (selected-apps (cons 'log-viewer (alist-ref 'selected-apps results)))
         (app-config (alist-ref 'app-config results))
         (config (alist-ref 'config app-config))
         (root-domain (alist-ref 'root-domain app-config))
-        (service-config (alist-ref 'service-config results)))
+        (service-config (alist-ref 'service-config results))
-   (with-output-to-file "deploy/config/apps.config"
+        (terraform-state (alist-ref 'terraform-state results))
+        (dir (deployment-directory (session-get "user-id"))))
+   (setup-deploy-files dir (alist-ref 'state terraform-state) (alist-ref 'backup terraform-state))
+   (with-output-to-file (string-append dir "/config/apps.config")
      (lambda ()
        (map (lambda (e)
               (write-config-entry (car e) (cdr e)))
@@ -796,13 +814,13 @@ h1, h2, h3, h4, h5, h6 {
               ("NEXTCLOUD_ADMIN_PASSWORD" . ,(alist-ref 'admin-password (alist-ref 'nextcloud config)))
               ("NEXTCLOUD_POSTGRES_DB" . "nextcloud")
               ("NEXTCLOUD_POSTGRES_USER" . "nextcloud")
-              ("NEXTCLOUD_POSTGRES_PASSWORD" . "dbpassword")
+              ("NEXTCLOUD_POSTGRES_PASSWORD" . "dbpassword") ;; TODO generate
-              ("NEXTCLOUD_REDIS_PASSWORD" . "redispassword")
+              ("NEXTCLOUD_REDIS_PASSWORD" . "redispassword") ;; TODO generate
               ("BACKBLAZE_KEY_ID" . ,(alist-ref 'backblaze-key-id service-config))
               ("BACKBLAZE_APPLICATION_KEY" . ,(alist-ref 'backblaze-application-key service-config))
               ("BACKBLAZE_BUCKET_URL" . ,(alist-ref 'backblaze-bucket-url service-config))
-              ("RESTIC_PASSWORD" . "foodisgood")))))
+              ("RESTIC_PASSWORD" . "foodisgood"))))) ;; TODO generate or get from user
-   (with-output-to-file "deploy/config/production.tfvars"
+   (with-output-to-file (string-append dir "/config/production.tfvars")
      (lambda ()
        (map (lambda (e)
               (write-config-entry (car e) (cdr e)))
@@ -814,12 +832,13 @@ h1, h2, h3, h4, h5, h6 {
               ("cluster_name" . "mycluster")
               ("datacenter" . ,(alist-ref 'digitalocean-region service-config))
               ("flatcar_stable_version" . "4230.2.3")))
-       (display "ssh_keys=[\"") (display (with-input-from-file "deploy/config/ssh-keys" read-string)) (print "\"]"))))
+       (display "ssh_keys=[\"") (display (with-input-from-file (string-append dir "/config/ssh-keys") read-string)) (print "\"]"))))
  (let* ((user-id (session-get "user-id"))
-        (deployment-id (with-db/transaction (lambda (db) (create-deployment db user-id)))))
+        (deployment-id (with-db/transaction (lambda (db) (create-deployment db user-id))))
+        (dir (deployment-directory user-id)))
    (thread-start!
     (lambda ()
-      (change-directory "deploy")
+      (change-directory dir)
       (let ((pid (process-run "make apply > make-out")))
         (with-db/transaction (lambda (db) (update-deployment-in-progress db deployment-id pid)))
         (change-directory "../")
@@ -830,15 +849,25 @@ h1, h2, h3, h4, h5, h6 {
                 (loop)
                 (with-db/transaction
                  (lambda (db)
+                   ;; TODO THIS DOESN'T WORK RIGHT FOR TERRAFORM OP FAILURES
+                   ;; like the random digital ocean error saying the IP can't be
+                   ;; updated because another operation is in progress.
+                   ;; it still registers as "success".
+                   ;; probably need to also write stderr to a file and read/store/parse that?
+                   ;; Should we parse make-out for string "Apply complete!" ?
                    (update-deployment-status
-                    db deployment-id
+                    db user-id deployment-id
-                    (if exit-normal 'complete 'failed)))))))))))
+                    (if exit-normal 'complete 'failed)
+                    (with-input-from-file (string-append dir "/make-out") read-string))
+                   (update-user-terraform-state db user-id
+                                                (with-input-from-file (string-append dir "/terraform.tfstate") read-string)
+                                                (with-input-from-file (string-append dir "/terraform.tfstate.backup") read-string)))))))))))
   (schematra:redirect "/config/wizard/success"))
 
 (get
  ("/config/wizard/success")
  (let ((status (with-db/transaction (lambda (db) (get-most-recent-deployment-status db (session-get "user-id")))))
-       (output (with-input-from-file "deploy/make-out" (lambda () (read-string)))))
+       (output (with-input-from-file (string-append (deployment-directory (session-get "user-id")) "/make-out") read-string)))
    `(VStack
      (h1
       ,(case (string->symbol status)
@@ -877,16 +906,6 @@ h1, h2, h3, h4, h5, h6 {
                      "in progress")
                     (else "queued"))))
      (pre ,output)
-     ;; ,@(intersperse
-     ;;    (with-input-from-file "deploy/make-out"
-     ;;      (lambda ()
-     ;;        (letrec ((loop (lambda (out)
-     ;;                         (let ((v (read-line)))
-     ;;                           (if (eq? v #!eof)
-     ;;                               out
-     ;;                               (loop (cons v out)))))))
-     ;;          (reverse (loop '())))))
-     ;;    `(br))
      )))
 
 (schematra:schematra-install)