nassella
/
app
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Properly manage deploys via unique folders and db.

Browse Source
main
Thomas Hintz 2 weeks ago
parent 5d256e5cf8
commit b781ddb5d7
5 changed files with 102 additions and 30 deletions
Show Stats Download Patch File Download Diff File

2
.gitignore vendored
Unescape Escape View File

@ -27,3 +27,5 @@ restic-env
restic-password
ignition.json
app
nassella-latest.tar
src/deploy*/*

9
Makefile
Unescape Escape View File

@ -62,7 +62,7 @@ plan: ignition.json $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars
bash -c "terraform plan -var-file=<(cat $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars)"
apply: ignition.json $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars
bash -c "terraform apply -var-file=<(cat $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars)"
bash -c "terraform apply -auto-approve -var-file=<(cat $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars)"
destroy: ignition.json $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars
bash -c "terraform destroy -var-file=<(cat $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars)"
@ -76,6 +76,13 @@ restic-init: $(apps_config) restic-password
restic-snapshots: $(apps_config) restic-password
./restic-snapshots.sh $(apps_config)
.PHONY: archive
archive:
tar -cf nassella-latest.tar all-apps cl.yaml init-restic.sh main.tf make-caddyfile.sh Makefile \
make-generated.sh make-nextcloud-env.sh make-restic-generated.sh make-restic-password.sh restic-snapshots.sh \
.terraform .terraform.lock.hcl
cp nassella-latest.tar src/
## to help me remember the command to run to test the config locally
testlocalhost:
curl -k --resolve localhost:443:146.190.12.129 https://localhost

9
src/db-init.sql
Unescape Escape View File

@ -63,3 +63,12 @@ create table deployments(
);
create index deployments_user_id_idx on deployments (user_id);
create table user_terraform_state(
id bigserial primary key,
user_id integer unique not null references users on delete cascade,
state_enc text,
state_backup_enc text
);
create unique index user_terraform_state_user_id_idx on user_terraform_state (user_id);

45
src/db.scm
Unescape Escape View File

@ -14,6 +14,7 @@
update-deployment-status get-deployment-status
get-most-recent-deployment-status
update-deployment-in-progress
update-user-terraform-state get-user-terraform-state
)
(import scheme
@ -140,7 +141,8 @@ returning users.user_id;"
user-iv))))
(query conn "insert into user_service_configs(user_id) values ($1);" user-id)
(query conn "insert into user_selected_apps(user_id) values ($1);" user-id)
(query conn "insert into user_app_configs(user_id) values ($1);" user-id)))))
(query conn "insert into user_app_configs(user_id) values ($1);" user-id)
(query conn "insert into user_terraform_state(user_id) values ($1);" user-id)))))
(define *user-service-configs-column-map*
'((cloudflare-api-token . ("cloudflare_api_token_enc" #t))
@ -299,16 +301,20 @@ returning users.user_id;"
(define (create-deployment conn user-id)
(value-at
(query conn
"insert into deployments(user_id) values($1) returning deployments.id;"
"insert into deployments(user_id, started) values($1, now()) returning deployments.id;"
user-id)))
(define (update-deployment-in-progress conn deployment-id pid)
(query conn
"update deployments set status=$1, pid=$2 where id=$3;"
(alist-ref 'in-progress *deployment-status*) pid deployment-id))
(define (update-deployment-status conn deployment-id status)
(query conn "update deployments set status=$1 where id=$2;"
(alist-ref status *deployment-status*) deployment-id))
(define (update-deployment-status conn user-id deployment-id status log)
(receive (user-key user-iv auth-user-id)
(get-decrypted-user-key-and-iv conn user-id)
(query conn "update deployments set status=$1, log_enc=$2, finished=now() where id=$3;"
(alist-ref status *deployment-status*)
(user-encrypt-for-db log user-key user-iv user-id)
deployment-id)))
(define (get-deployment-status conn deployment-id)
(value-at (query conn "select status from deployments where id=$1;" deployment-id)))
@ -316,6 +322,35 @@ returning users.user_id;"
(define (get-most-recent-deployment-status conn user-id)
(value-at (query conn "select status from deployments where user_id=$1 order by id DESC limit 1;" user-id)))
(define (update-user-terraform-state conn user-id state backup)
(receive (user-key user-iv auth-user-id)
(get-decrypted-user-key-and-iv conn user-id)
(query conn
"update user_terraform_state set state_enc=$1, state_backup_enc=$2 where user_id=$3;"
(user-encrypt-for-db state user-key user-iv user-id)
(user-encrypt-for-db backup user-key user-iv user-id)
user-id)))
(define (get-user-terraform-state conn user-id)
(receive (user-key user-iv auth-user-id)
(get-decrypted-user-key-and-iv conn user-id)
(let ((res (row-alist (query conn
"select state_enc, state_backup_enc from user_terraform_state where user_id=$1;"
user-id))))
`((state . ,(if (sql-null? (alist-ref 'config_enc res))
""
(user-decrypt-from-db (alist-ref 'state_enc res) user-key user-iv user-id)))
(backup . ,(if (sql-null? (alist-ref 'config_enc res))
""
(user-decrypt-from-db (alist-ref 'state_backup_enc res) user-key user-iv user-id)))))))
;; (with-db/transaction
;; (lambda (db)
;; (update-user-terraform-state db 7
;; (with-input-from-file "src/deploy-bak/terraform.tfstate" read-string)
;; (with-input-from-file "src/deploy-bak/terraform.tfstate.backup" read-string))))
;; (with-db/transaction (lambda (db) (get-user-terraform-state db 7)))
;; (with-db/transaction (lambda (db) (create-deployment db 7)))
;; (with-db/transaction (lambda (db) (get-deployment-status db 1)))
;; (with-db/transaction (lambda (db) (update-deployment-in-progress db 1 123)))

65
src/nassella.scm
Unescape Escape View File

@ -7,6 +7,7 @@
(chicken process)
(chicken process-context)
(chicken irregex)
(chicken file)
(rename srfi-1 (delete srfi1:delete))
srfi-18
@ -757,6 +758,19 @@ h1, h2, h3, h4, h5, h6 {
(VStack
(Form-Nav (@ (back-to ,(conc "/config/wizard/machine2")) (submit-button "Launch")))))))))
(define (deployment-directory user-id)
(string-append "deploy-" (number->string user-id)))
(define (setup-deploy-files dir state state-backup)
(when (directory-exists? dir)
(delete-directory dir #t))
(create-directory dir)
(process-wait (process-run (string-append "tar -xf nassella-latest.tar -C " dir)))
(create-directory (string-append dir "/config"))
(copy-file "../config/ssh-keys" (string-append dir "/config/ssh-keys")) ;; TODO remove
(with-output-to-file (string-append dir "/terraform.tfstate") (lambda () (write-string state)))
(with-output-to-file (string-append dir "/terraform.tfstate.backup") (lambda () (write-string state-backup))))
(define (write-config-entry name value)
(display name)
(display "=\"")
@ -772,13 +786,17 @@ h1, h2, h3, h4, h5, h6 {
(filter cdr
(get-user-selected-apps db (session-get "user-id")))))
(app-config . ,(get-user-app-config db (session-get "user-id")))
(service-config . ,(get-user-service-config db (session-get "user-id")))))))
(service-config . ,(get-user-service-config db (session-get "user-id")))
(terraform-state . ,(get-user-terraform-state db (session-get "user-id")))))))
(selected-apps (cons 'log-viewer (alist-ref 'selected-apps results)))
(app-config (alist-ref 'app-config results))
(config (alist-ref 'config app-config))
(root-domain (alist-ref 'root-domain app-config))
(service-config (alist-ref 'service-config results)))
(with-output-to-file "deploy/config/apps.config"
(service-config (alist-ref 'service-config results))
(terraform-state (alist-ref 'terraform-state results))
(dir (deployment-directory (session-get "user-id"))))
(setup-deploy-files dir (alist-ref 'state terraform-state) (alist-ref 'backup terraform-state))
(with-output-to-file (string-append dir "/config/apps.config")
(lambda ()
(map (lambda (e)
(write-config-entry (car e) (cdr e)))
@ -796,13 +814,13 @@ h1, h2, h3, h4, h5, h6 {
("NEXTCLOUD_ADMIN_PASSWORD" . ,(alist-ref 'admin-password (alist-ref 'nextcloud config)))
("NEXTCLOUD_POSTGRES_DB" . "nextcloud")
("NEXTCLOUD_POSTGRES_USER" . "nextcloud")
("NEXTCLOUD_POSTGRES_PASSWORD" . "dbpassword")
("NEXTCLOUD_REDIS_PASSWORD" . "redispassword")
("NEXTCLOUD_POSTGRES_PASSWORD" . "dbpassword") ;; TODO generate
("NEXTCLOUD_REDIS_PASSWORD" . "redispassword") ;; TODO generate
("BACKBLAZE_KEY_ID" . ,(alist-ref 'backblaze-key-id service-config))
("BACKBLAZE_APPLICATION_KEY" . ,(alist-ref 'backblaze-application-key service-config))
("BACKBLAZE_BUCKET_URL" . ,(alist-ref 'backblaze-bucket-url service-config))
("RESTIC_PASSWORD" . "foodisgood")))))
(with-output-to-file "deploy/config/production.tfvars"
("RESTIC_PASSWORD" . "foodisgood"))))) ;; TODO generate or get from user
(with-output-to-file (string-append dir "/config/production.tfvars")
(lambda ()
(map (lambda (e)
(write-config-entry (car e) (cdr e)))
@ -814,12 +832,13 @@ h1, h2, h3, h4, h5, h6 {
("cluster_name" . "mycluster")
("datacenter" . ,(alist-ref 'digitalocean-region service-config))
("flatcar_stable_version" . "4230.2.3")))
(display "ssh_keys=[\"") (display (with-input-from-file "deploy/config/ssh-keys" read-string)) (print "\"]"))))
(display "ssh_keys=[\"") (display (with-input-from-file (string-append dir "/config/ssh-keys") read-string)) (print "\"]"))))
(let* ((user-id (session-get "user-id"))
(deployment-id (with-db/transaction (lambda (db) (create-deployment db user-id)))))
(deployment-id (with-db/transaction (lambda (db) (create-deployment db user-id))))
(dir (deployment-directory user-id)))
(thread-start!
(lambda ()
(change-directory "deploy")
(change-directory dir)
(let ((pid (process-run "make apply > make-out")))
(with-db/transaction (lambda (db) (update-deployment-in-progress db deployment-id pid)))
(change-directory "../")
@ -830,15 +849,25 @@ h1, h2, h3, h4, h5, h6 {
(loop)
(with-db/transaction
(lambda (db)
;; TODO THIS DOESN'T WORK RIGHT FOR TERRAFORM OP FAILURES
;; like the random digital ocean error saying the IP can't be
;; updated because another operation is in progress.
;; it still registers as "success".
;; probably need to also write stderr to a file and read/store/parse that?
;; Should we parse make-out for string "Apply complete!" ?
(update-deployment-status
db deployment-id
(if exit-normal 'complete 'failed)))))))))))
db user-id deployment-id
(if exit-normal 'complete 'failed)
(with-input-from-file (string-append dir "/make-out") read-string))
(update-user-terraform-state db user-id
(with-input-from-file (string-append dir "/terraform.tfstate") read-string)
(with-input-from-file (string-append dir "/terraform.tfstate.backup") read-string)))))))))))
(schematra:redirect "/config/wizard/success"))
(get
("/config/wizard/success")
(let ((status (with-db/transaction (lambda (db) (get-most-recent-deployment-status db (session-get "user-id")))))
(output (with-input-from-file "deploy/make-out" (lambda () (read-string)))))
(output (with-input-from-file (string-append (deployment-directory (session-get "user-id")) "/make-out") read-string)))
`(VStack
(h1
,(case (string->symbol status)
@ -877,16 +906,6 @@ h1, h2, h3, h4, h5, h6 {
"in progress")
(else "queued"))))
(pre ,output)
;; ,@(intersperse
;; (with-input-from-file "deploy/make-out"
;; (lambda ()
;; (letrec ((loop (lambda (out)
;; (let ((v (read-line)))
;; (if (eq? v #!eof)
;; out
;; (loop (cons v out)))))))
;; (reverse (loop '())))))
;; `(br))
)))
(schematra:schematra-install)

Write Preview
Loading…
Cancel
Save