diff --git a/.gitignore b/.gitignore index b26a5c9..30d52b7 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,6 @@ generated.tfvars restic-env restic-password ignition.json -app \ No newline at end of file +app +nassella-latest.tar +src/deploy*/* \ No newline at end of file diff --git a/Makefile b/Makefile index 5631f2f..5864fe6 100644 --- a/Makefile +++ b/Makefile @@ -62,7 +62,7 @@ plan: ignition.json $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars bash -c "terraform plan -var-file=<(cat $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars)" apply: ignition.json $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars - bash -c "terraform apply -var-file=<(cat $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars)" + bash -c "terraform apply -auto-approve -var-file=<(cat $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars)" destroy: ignition.json $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars bash -c "terraform destroy -var-file=<(cat $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars)" @@ -76,6 +76,13 @@ restic-init: $(apps_config) restic-password restic-snapshots: $(apps_config) restic-password ./restic-snapshots.sh $(apps_config) +.PHONY: archive +archive: + tar -cf nassella-latest.tar all-apps cl.yaml init-restic.sh main.tf make-caddyfile.sh Makefile \ + make-generated.sh make-nextcloud-env.sh make-restic-generated.sh make-restic-password.sh restic-snapshots.sh \ + .terraform .terraform.lock.hcl + cp nassella-latest.tar src/ + ## to help me remember the command to run to test the config locally testlocalhost: curl -k --resolve localhost:443:146.190.12.129 https://localhost diff --git a/src/db-init.sql b/src/db-init.sql index 7ef7fe1..9869079 100644 --- a/src/db-init.sql +++ b/src/db-init.sql @@ -63,3 +63,12 @@ create table deployments( ); create index deployments_user_id_idx on deployments (user_id); + +create table user_terraform_state( + id bigserial primary key, + user_id integer unique not null references users on delete cascade, + state_enc text, + state_backup_enc text + ); + +create unique index user_terraform_state_user_id_idx on user_terraform_state (user_id); diff --git a/src/db.scm b/src/db.scm index 1532ba5..1304449 100644 --- a/src/db.scm +++ b/src/db.scm @@ -14,6 +14,7 @@ update-deployment-status get-deployment-status get-most-recent-deployment-status update-deployment-in-progress + update-user-terraform-state get-user-terraform-state ) (import scheme @@ -140,7 +141,8 @@ returning users.user_id;" user-iv)))) (query conn "insert into user_service_configs(user_id) values ($1);" user-id) (query conn "insert into user_selected_apps(user_id) values ($1);" user-id) - (query conn "insert into user_app_configs(user_id) values ($1);" user-id))))) + (query conn "insert into user_app_configs(user_id) values ($1);" user-id) + (query conn "insert into user_terraform_state(user_id) values ($1);" user-id))))) (define *user-service-configs-column-map* '((cloudflare-api-token . ("cloudflare_api_token_enc" #t)) @@ -299,16 +301,20 @@ returning users.user_id;" (define (create-deployment conn user-id) (value-at (query conn - "insert into deployments(user_id) values($1) returning deployments.id;" + "insert into deployments(user_id, started) values($1, now()) returning deployments.id;" user-id))) (define (update-deployment-in-progress conn deployment-id pid) (query conn "update deployments set status=$1, pid=$2 where id=$3;" (alist-ref 'in-progress *deployment-status*) pid deployment-id)) -(define (update-deployment-status conn deployment-id status) - (query conn "update deployments set status=$1 where id=$2;" - (alist-ref status *deployment-status*) deployment-id)) +(define (update-deployment-status conn user-id deployment-id status log) + (receive (user-key user-iv auth-user-id) + (get-decrypted-user-key-and-iv conn user-id) + (query conn "update deployments set status=$1, log_enc=$2, finished=now() where id=$3;" + (alist-ref status *deployment-status*) + (user-encrypt-for-db log user-key user-iv user-id) + deployment-id))) (define (get-deployment-status conn deployment-id) (value-at (query conn "select status from deployments where id=$1;" deployment-id))) @@ -316,6 +322,35 @@ returning users.user_id;" (define (get-most-recent-deployment-status conn user-id) (value-at (query conn "select status from deployments where user_id=$1 order by id DESC limit 1;" user-id))) +(define (update-user-terraform-state conn user-id state backup) + (receive (user-key user-iv auth-user-id) + (get-decrypted-user-key-and-iv conn user-id) + (query conn + "update user_terraform_state set state_enc=$1, state_backup_enc=$2 where user_id=$3;" + (user-encrypt-for-db state user-key user-iv user-id) + (user-encrypt-for-db backup user-key user-iv user-id) + user-id))) + +(define (get-user-terraform-state conn user-id) + (receive (user-key user-iv auth-user-id) + (get-decrypted-user-key-and-iv conn user-id) + (let ((res (row-alist (query conn + "select state_enc, state_backup_enc from user_terraform_state where user_id=$1;" + user-id)))) + `((state . ,(if (sql-null? (alist-ref 'config_enc res)) + "" + (user-decrypt-from-db (alist-ref 'state_enc res) user-key user-iv user-id))) + (backup . ,(if (sql-null? (alist-ref 'config_enc res)) + "" + (user-decrypt-from-db (alist-ref 'state_backup_enc res) user-key user-iv user-id))))))) + +;; (with-db/transaction +;; (lambda (db) +;; (update-user-terraform-state db 7 +;; (with-input-from-file "src/deploy-bak/terraform.tfstate" read-string) +;; (with-input-from-file "src/deploy-bak/terraform.tfstate.backup" read-string)))) +;; (with-db/transaction (lambda (db) (get-user-terraform-state db 7))) + ;; (with-db/transaction (lambda (db) (create-deployment db 7))) ;; (with-db/transaction (lambda (db) (get-deployment-status db 1))) ;; (with-db/transaction (lambda (db) (update-deployment-in-progress db 1 123))) diff --git a/src/nassella.scm b/src/nassella.scm index 0e0a7e1..a6a90ab 100644 --- a/src/nassella.scm +++ b/src/nassella.scm @@ -7,6 +7,7 @@ (chicken process) (chicken process-context) (chicken irregex) + (chicken file) (rename srfi-1 (delete srfi1:delete)) srfi-18 @@ -757,6 +758,19 @@ h1, h2, h3, h4, h5, h6 { (VStack (Form-Nav (@ (back-to ,(conc "/config/wizard/machine2")) (submit-button "Launch"))))))))) +(define (deployment-directory user-id) + (string-append "deploy-" (number->string user-id))) + +(define (setup-deploy-files dir state state-backup) + (when (directory-exists? dir) + (delete-directory dir #t)) + (create-directory dir) + (process-wait (process-run (string-append "tar -xf nassella-latest.tar -C " dir))) + (create-directory (string-append dir "/config")) + (copy-file "../config/ssh-keys" (string-append dir "/config/ssh-keys")) ;; TODO remove + (with-output-to-file (string-append dir "/terraform.tfstate") (lambda () (write-string state))) + (with-output-to-file (string-append dir "/terraform.tfstate.backup") (lambda () (write-string state-backup)))) + (define (write-config-entry name value) (display name) (display "=\"") @@ -772,13 +786,17 @@ h1, h2, h3, h4, h5, h6 { (filter cdr (get-user-selected-apps db (session-get "user-id"))))) (app-config . ,(get-user-app-config db (session-get "user-id"))) - (service-config . ,(get-user-service-config db (session-get "user-id"))))))) + (service-config . ,(get-user-service-config db (session-get "user-id"))) + (terraform-state . ,(get-user-terraform-state db (session-get "user-id"))))))) (selected-apps (cons 'log-viewer (alist-ref 'selected-apps results))) (app-config (alist-ref 'app-config results)) (config (alist-ref 'config app-config)) (root-domain (alist-ref 'root-domain app-config)) - (service-config (alist-ref 'service-config results))) - (with-output-to-file "deploy/config/apps.config" + (service-config (alist-ref 'service-config results)) + (terraform-state (alist-ref 'terraform-state results)) + (dir (deployment-directory (session-get "user-id")))) + (setup-deploy-files dir (alist-ref 'state terraform-state) (alist-ref 'backup terraform-state)) + (with-output-to-file (string-append dir "/config/apps.config") (lambda () (map (lambda (e) (write-config-entry (car e) (cdr e))) @@ -796,13 +814,13 @@ h1, h2, h3, h4, h5, h6 { ("NEXTCLOUD_ADMIN_PASSWORD" . ,(alist-ref 'admin-password (alist-ref 'nextcloud config))) ("NEXTCLOUD_POSTGRES_DB" . "nextcloud") ("NEXTCLOUD_POSTGRES_USER" . "nextcloud") - ("NEXTCLOUD_POSTGRES_PASSWORD" . "dbpassword") - ("NEXTCLOUD_REDIS_PASSWORD" . "redispassword") + ("NEXTCLOUD_POSTGRES_PASSWORD" . "dbpassword") ;; TODO generate + ("NEXTCLOUD_REDIS_PASSWORD" . "redispassword") ;; TODO generate ("BACKBLAZE_KEY_ID" . ,(alist-ref 'backblaze-key-id service-config)) ("BACKBLAZE_APPLICATION_KEY" . ,(alist-ref 'backblaze-application-key service-config)) ("BACKBLAZE_BUCKET_URL" . ,(alist-ref 'backblaze-bucket-url service-config)) - ("RESTIC_PASSWORD" . "foodisgood"))))) - (with-output-to-file "deploy/config/production.tfvars" + ("RESTIC_PASSWORD" . "foodisgood"))))) ;; TODO generate or get from user + (with-output-to-file (string-append dir "/config/production.tfvars") (lambda () (map (lambda (e) (write-config-entry (car e) (cdr e))) @@ -814,12 +832,13 @@ h1, h2, h3, h4, h5, h6 { ("cluster_name" . "mycluster") ("datacenter" . ,(alist-ref 'digitalocean-region service-config)) ("flatcar_stable_version" . "4230.2.3"))) - (display "ssh_keys=[\"") (display (with-input-from-file "deploy/config/ssh-keys" read-string)) (print "\"]")))) + (display "ssh_keys=[\"") (display (with-input-from-file (string-append dir "/config/ssh-keys") read-string)) (print "\"]")))) (let* ((user-id (session-get "user-id")) - (deployment-id (with-db/transaction (lambda (db) (create-deployment db user-id))))) + (deployment-id (with-db/transaction (lambda (db) (create-deployment db user-id)))) + (dir (deployment-directory user-id))) (thread-start! (lambda () - (change-directory "deploy") + (change-directory dir) (let ((pid (process-run "make apply > make-out"))) (with-db/transaction (lambda (db) (update-deployment-in-progress db deployment-id pid))) (change-directory "../") @@ -830,15 +849,25 @@ h1, h2, h3, h4, h5, h6 { (loop) (with-db/transaction (lambda (db) + ;; TODO THIS DOESN'T WORK RIGHT FOR TERRAFORM OP FAILURES + ;; like the random digital ocean error saying the IP can't be + ;; updated because another operation is in progress. + ;; it still registers as "success". + ;; probably need to also write stderr to a file and read/store/parse that? + ;; Should we parse make-out for string "Apply complete!" ? (update-deployment-status - db deployment-id - (if exit-normal 'complete 'failed))))))))))) + db user-id deployment-id + (if exit-normal 'complete 'failed) + (with-input-from-file (string-append dir "/make-out") read-string)) + (update-user-terraform-state db user-id + (with-input-from-file (string-append dir "/terraform.tfstate") read-string) + (with-input-from-file (string-append dir "/terraform.tfstate.backup") read-string))))))))))) (schematra:redirect "/config/wizard/success")) (get ("/config/wizard/success") (let ((status (with-db/transaction (lambda (db) (get-most-recent-deployment-status db (session-get "user-id"))))) - (output (with-input-from-file "deploy/make-out" (lambda () (read-string))))) + (output (with-input-from-file (string-append (deployment-directory (session-get "user-id")) "/make-out") read-string))) `(VStack (h1 ,(case (string->symbol status) @@ -877,16 +906,6 @@ h1, h2, h3, h4, h5, h6 { "in progress") (else "queued")))) (pre ,output) - ;; ,@(intersperse - ;; (with-input-from-file "deploy/make-out" - ;; (lambda () - ;; (letrec ((loop (lambda (out) - ;; (let ((v (read-line))) - ;; (if (eq? v #!eof) - ;; out - ;; (loop (cons v out))))))) - ;; (reverse (loop '()))))) - ;; `(br)) ))) (schematra:schematra-install)