Working nassella running on nassella!!

.gitignore

@@ -6,17 +6,32 @@
 flatcar/flatcar_production_qemu_image.img
 flatcar/flatcar_production_qemu_image.img.fresh
 
-ignition.json
-
-production.tfvars
-
 terraform.tfstate
 terraform.tfstate.backup
 
-app
+config/apps.config
+config/production.tfvars
+config/ssh-keys
 
-apps.config
+# custom chicken eggs for dockerfile
+src/scss
+src/html-widgets
+src/schematra-session
 
 # generated files
+all-apps/.env
 all-apps/lb/Caddyfile
-generated.tfvars
+all-apps/nextcloud/nextcloud.env
+all-apps/nextcloud/nextcloud_admin_user
+all-apps/nextcloud/nextcloud_admin_password
+all-apps/nextcloud/postgres_db
+all-apps/nextcloud/postgres_user
+all-apps/nextcloud/postgres_password
+all-apps/nextcloud/redis_password
+generated.tfvars
+restic-env
+restic-password
+ignition.json
+app
+nassella-latest.tar
+src/deploy*/*

Makefile

@@ -1,25 +1,116 @@
-TERRAFORM_ENV=production
+TERRAFORM_ENV := production
+
+config_dir := ./config/
+apps_config := $(config_dir)apps.config
 
 # .dirstamp plus && $@ is like make magic to get this rule
 # to only run if the contents of all-apps changes
-app/.dirstamp: all-apps/app.service all-apps/docker-compose.yaml $(wildcard all-apps/lb/*) $(wildcard all-apps/Nextcloud/*) $(wildcard all-apps/wg-easy/*)
+app/.dirstamp: all-apps/app.service all-apps/docker-compose.yaml \
+$(wildcard all-apps/lb/*) \
+$(wildcard all-apps/nextcloud/*) \
+$(wildcard all-apps/wg-easy/*) \
+$(wildcard all-apps/ghost/*) \
+$(wildcard all-apps/nassella/*) \
+$(wildcard all-apps/dozzle/*)
+
 	rm -Rf app/
-	cp -a all-apps app && touch $@
+	mkdir app/
+	cp all-apps/app.service app/
+	cp all-apps/docker-compose.yaml app/
+	cp all-apps/.env app/
+	./copy-apps.sh $(apps_config) && touch $@
 
-all-apps/lb/Caddyfile: apps.config make-caddyfile.sh
-	./make-caddyfile.sh > all-apps/lb/Caddyfile
+# compose .env files
+# (compose only supports one .env file at the root by default)
+all-apps/.env: all-apps/ghost/.compose-env
+	find all-apps/ -name ".compose-env" -exec cat > all-apps/.env {} +
 
-ignition.json: cl.yaml app/.dirstamp all-apps/lb/Caddyfile
-	cat cl.yaml | sudo docker run --rm --volume /home/tjhintz/.ssh/id_rsa.pub:/pwd/ssh-keys --volume ${PWD}:/pwd --workdir /pwd -i quay.io/coreos/butane:latest -d /pwd > ignition.json
+# Caddy / lb
+all-apps/lb/Caddyfile: $(apps_config) make-caddyfile.sh
+	mkdir -p all-apps/lb
+	./make-caddyfile.sh $(apps_config) > all-apps/lb/Caddyfile
 
-generated.tfvars: apps.config make-generated.sh
-	./make-generated.sh > generated.tfvars
+# Nextcloud
+all-apps/nextcloud/nextcloud_admin_user: $(apps_config)
+	bash -c 'source $(apps_config); printf "%s\n" "$$NEXTCLOUD_ADMIN_USER" > $@'
+all-apps/nextcloud/nextcloud_admin_password: $(apps_config)
+	bash -c 'source $(apps_config); printf "%s\n" "$$NEXTCLOUD_ADMIN_PASSWORD" > $@'
+all-apps/nextcloud/postgres_db: $(apps_config)
+	bash -c 'source ./$(apps_config); printf "%s\n" "$$NEXTCLOUD_POSTGRES_DB" > $@'
+all-apps/nextcloud/postgres_user: $(apps_config)
+	bash -c 'source ./$(apps_config); printf "%s\n" "$$NEXTCLOUD_POSTGRES_USER" > $@'
+all-apps/nextcloud/postgres_password: $(apps_config)
+	bash -c 'source ./$(apps_config); printf "%s\n" "$$NEXTCLOUD_POSTGRES_PASSWORD" > $@'
+all-apps/nextcloud/redis_password: $(apps_config)
+	bash -c 'source ./$(apps_config); printf "%s\n" "$$NEXTCLOUD_REDIS_PASSWORD" > $@'
+all-apps/nextcloud/nextcloud.env: $(apps_config) all-apps/nextcloud/nextcloud.env.tmpl make-nextcloud-env.sh
+	./make-nextcloud-env.sh $(apps_config)
 
-plan: ignition.json $(TERRAFORM_ENV).tfvars generated.tfvars
-	bash -c "terraform plan -var-file=<(cat $(TERRAFORM_ENV).tfvars generated.tfvars)"
+# Ghost
+all-apps/ghost/.compose-env: $(apps_config) all-apps/ghost/.compose.env.tmpl make-ghost-env.sh
+	./make-ghost-env.sh $(apps_config)
 
-apply: ignition.json $(TERRAFORM_ENV).tfvars generated.tfvars
-	bash -c "terraform apply -var-file=<(cat $(TERRAFORM_ENV).tfvars generated.tfvars)"
+# Backups / Restic / Backblaze
+restic-env: $(apps_config) make-restic-generated.sh
+	./make-restic-generated.sh $(apps_config) > restic-env
+restic-password: $(apps_config) make-restic-password.sh
+	./make-restic-password.sh $(apps_config) > restic-password
+
+ignition.json: cl.yaml app/.dirstamp \
+all-apps/lb/Caddyfile \
+all-apps/nextcloud/nextcloud_admin_user \
+all-apps/nextcloud/nextcloud_admin_password \
+all-apps/nextcloud/postgres_db \
+all-apps/nextcloud/postgres_user \
+all-apps/nextcloud/postgres_password \
+all-apps/nextcloud/redis_password \
+all-apps/nextcloud/nextcloud.env \
+all-apps/nassella/postgres_db \
+all-apps/nassella/postgres_user \
+all-apps/nassella/postgres_password \
+all-apps/nassella/nassella.env \
+all-apps/ghost/.compose-env \
+restic-env \
+restic-password \
+all-apps/.env \
+$(config_dir)ssh-keys
+	cat cl.yaml | docker run --rm --volume $(config_dir)/ssh-keys:/pwd/ssh-keys --volume ${PWD}:/pwd --workdir /pwd -i quay.io/coreos/butane:latest -d /pwd > ignition.json
+
+generated.tfvars: $(apps_config) make-generated.sh
+	./make-generated.sh $(apps_config) > generated.tfvars
+
+plan: ignition.json $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars
+	terraform init
+	bash -c "terraform plan -var-file=<(cat $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars)"
+
+.PHONY: announce-start
+announce-start:
+	echo "NASSELLA_CONFIG: start"
+
+apply: announce-start ignition.json $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars
+	echo "NASSELLA_CONFIG: end"
+	terraform init
+	bash -c "terraform apply -auto-approve -input=false -var-file=<(cat $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars)"
+
+destroy: ignition.json $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars
+	terraform init
+	bash -c "terraform destroy -auto-approve -input=false -var-file=<(cat $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars)"
+
+.PHONY: restic-init
+restic-init: $(apps_config) restic-password
+	./init-restic.sh $(apps_config)
+
+## just an easy way to see snapshots that have been taken
+.PHONY: restic-snapshots
+restic-snapshots: $(apps_config) restic-password
+	./restic-snapshots.sh $(apps_config)
+
+.PHONY: archive
+archive:
+	tar -cf nassella-latest.tar all-apps cl.yaml init-restic.sh main.tf make-caddyfile.sh Makefile \
+		make-generated.sh make-nextcloud-env.sh make-ghost-env.sh make-restic-generated.sh make-restic-password.sh restic-snapshots.sh copy-apps.sh \
+		.terraform.lock.hcl
+	cp nassella-latest.tar src/
 
 ## to help me remember the command to run to test the config locally
 testlocalhost:
@@ -30,3 +121,4 @@ flatcarbuild: ignition.json
 
 flatcarrun:
 	./flatcar/flatcar_production_qemu.sh -i ignition.json
+

README.org

@@ -0,0 +1,268 @@
+* Development Notes
+
+- Currently, only tested on Linux (Debian). Everything should
+  theoretically also work on Mac but some commands may need updating
+  and it has not been tested, but multi-platform support, except for
+  Windows, was kept in mind during development.
+
+* Project Goal
+
+To make deploying, managing, and updating self-hosted app instances
+easy.
+
+* Short Note On Generative AI
+
+This project does NOT use any code or documentation generated by an
+LLM and no code or text generated by an LLM is acceptable as a
+contribution.
+
+* Supported Services
+
+Currently, only a limited set of external services can be used for
+hosting, DNS, and backups. This can be easily extended later with
+modifications to the Terraform config. As of now, you will need to
+have accounts with these providers:
+
+- DigitalOcean
+- Cloudflare (and have DNS available for a domain)
+- Backblaze B2
+
+* Architecture
+
+The software stack is composed of a "base" command line interface that
+can be used to deploy and manage a single instance with a
+multi-instance, multi-user webapp that invokes the "base" as
+needed. The "base" can be run separately from the webapp. The webapp
+automatically generates the configs the "base" needs to run.
+
+The Makefile at the root of this source tree is the point of interface
+for everything and all commands are run via make.
+
+** "Base" Terraform Layer
+
+The project is designed so that if you want to just manage a single
+instance without the complexity of running a webapp you can easily do
+so. This is both so that individual users can take advantage of this
+but also so that when developing the Terraform and Docker Compose
+setup it can be done and tested without needing to deal with the web
+app as well.
+
+The "base" layer is made up of the following: Flatcar Linux, Docker
+Compose, Terraform, and a Makefile with a set of BASH scripts.
+
+*** Flatcar Linux
+
+The deployed instance runs on Flatcar Linux. Flatcar is a "read only"
+Linux distribution designed to only run containers and nothing
+else. Flatcar is used because it provides a high-level of security and
+the OS itself auto-updates on a two-week schedule. Also, being "read
+only" it is much more difficult for an external attacker to attack and
+much harder for a user that does not know what they are doing to "mess
+up".
+
+*** Docker Compose
+
+Each individual supported web app (like NextCloud, Ghost, etc) runs
+via Docker and is configured via Docker Compose. (The docker compose
+files are all in the "all-apps" directory in this source tree).
+
+The Flatcar Linux config contains a systemd unit (service file) that
+runs "docker compose". The Makefile copies all selected apps' docker
+compose files from all-apps/ to app/. The systemd unit runs all the
+docker compose files in the app/ directory. (The app/ directory is
+what actually gets copied to the Flatcar linux install, not the
+all-apps/ directory.)
+
+The docker compose setup is specific and needs further documentation
+here (to cover things like the shared load-balancer network setup and
+how persistent storage is handled).
+
+*** Terraform
+
+Terraform is used to actually manage the deployed instances. Currently
+it is a static terraform config controlled only via terraform
+variables (see config/production.tfvars.tmpl). The terraform commands
+are run via the Makefile.
+
+** Webapp
+
+The webapp is used both to provide a more "user-friendly" interface
+for setting up and managing instances as well as to provide a
+multi-user and multi-instance service. Internally, to manage an
+instance, the webapp generates the configs and invokes the same
+commands used when running the "base" CLI version by itself.
+
+* Setup "Base" CLI Terraform For Deploying Individual Instance
+
+NOTE: some of this may be outdated. It has not been tested on its own
+outside of running via the webapp for a bit. It does work when run via
+the webapp and all of the services still need to be setup as
+detailed (the data can be input via the webapp instead of only via the
+config files).
+
+** Dependencies
+- [[https://developer.hashicorp.com/terraform/install][terraform]]
+- [[https://www.docker.com/][docker]]
+- bash
+- GNU Make
+
+** Services
+
+*** [[https://www.digitalocean.com/][DigitalOcean]]
+
+- Create a DigitalOcean account and sign in to it
+
+- [[https://cloud.digitalocean.com/account/api/tokens][Click "API" on sidebar]]
+
+- Click "generate a new token"
+  - enter a name (can be anything, just for you to remember)
+  - set expiration as you desire
+  - set the "scope" to "Full Access"
+  - save the generated token for placing in production.tfvars -> do_token
+
+*** [[https://www.cloudflare.com/][Cloudflare]]
+
+- Create a CloudFlare account and sign into it
+
+- Either register a new domain or if you already have a domain not being used you can continue with that
+
+- On sidebar to to "Manage Account" -> "Account API Tokens"
+  - Click "Create Token"
+  - Under "templates" click "edit zone dns"
+  - Under "Zone Resources", in the box labelled "Select..." select the domain you want to use
+  - Click "continue to summary"
+  - Click "create token"
+  - Copy the token for use later on for the "cloudflare_api_token" in config/production.tfvars
+
+- Click "Account Home" to go back to the top level
+
+- Click on the domain you are using
+  - This will show the "Overview"
+  - Scroll down until you see the API heading and copy the "Zone ID" and "Account ID"
+
+    These will be used later on in config/production.tfvars for cloudflare_zone_id and cloudflare_account_id
+
+*** [[https://backblaze.com][Backblaze]]
+
+This is used automated for "off-site" backups / snapshots.
+
+- Create a Backblaze B2 account and sign in to it
+
+- Click "create a bucket"
+  - Give it a unique name (recommended something like [my-domain-com]-app-backups) but replace my-domain-com with your domain
+  - Files in bucket should be set to "Private"
+  - Leave "Default Encryption" as "disabled" (restic will encrypt the data)
+  - Leave "Object Lock" as disabled
+
+- Click on "Lifecycle Settings" under the newly created bucket
+  - Change to "Keep only the last version of the file"
+  - Click "Update Bucket"
+
+- Under the bucket details copy "Endpoint" for use later on in config/apps.config BACKBLAZE_BUCKET_URL
+
+- Click "Application Keys"
+  - Click "Add a new application key"
+  - "Name" can be whatever you want to remember it is a key for the backups for your apps
+  - Change "Allow access to buckets" to only the bucket you created in the previous step
+  - Leave "Type of Access" set to "Read and Write"
+  - Leave other options in their default values
+  - Click "Create new key"
+  - Copy/save the key for later use in config/apps.config BACKBLAZE_APPLICATION_KEY and the "keyID" for BACKBLAZE_KEY_ID
+
+** Configuration
+*** apps.config
+
+- ~cp config/apps.config.tmpl config/apps.config~
+
+- then edit ~config/apps.config~ and fill in all variables
+
+*** production.tfvars
+
+- ~cp config/production.tfvars.tmpl config/production.tfvars~
+
+- then edit ~config/production.tfvars~ and fill in all variables
+
+*** ssh keys
+
+- ~touch config/ssh-keys~
+
+- if you want to add your ssh key(s) for debugging then paste the pub ID in to the file
+
+*** initializing the "off-site" Restic backups
+
+- ~make restic-init~
+
+** Deploy
+
+- ~make apply~
+
+** You're done!
+
+It will take a few minutes to deploy the server, start it, and pull
+all the docker images. But after that you should be able to visit your
+site and the apps running on its subdomains!
+
+* Setup Webapp
+** Dependencies
+
+- [[https://code.call-cc.org/][CHICKEN Scheme 5.3+]]
+- docker
+- GNU Make
+
+The webapp is written in Lisp (CHICKEN Scheme) and connects to a
+PostgreSQL database. It also depends on being able to run some docker
+commands. It has only been tested on Linux. Running the commands on
+other platforms may need work. A Makefile command is provided for
+running the Postgres database via docker so Postgres is not a listed
+as a direct dependency.
+
+** CHICKEN Scheme Libraries
+
+These will need to be installed via the ~chicken-install~ command.
+
+~postgresql sql-null srfi-1 srfi-13 srfi-18 srfi-158 srfi-194 openssl crypto-tools sxml-transforms schematra schematra-body-parser schematra-session uri-common http-client medea intarweb~
+
+** html-widgets
+
+This is a CHICKEN Scheme library that also needs to be installed but
+it is not available via the ~chicken-install~ repository as I wrote it
+for this project and I have not published it externally yet. You can
+get the project here: [[https://code.thintz.com/tjhintz/html-widgets][code.thintz.com/tjhintz/html-widgets]]
+
+After downloading the project, you can install it by ~cd~ to the
+directory it is in and then running ~chicken-install~.
+
+** Architecture
+
+The webapp was designed to be very simple. Pages and handlers are
+based on the [[https://schematra.com/][Schematra]] CHICKEN Scheme library (Chiccup is NOT
+used). Currently, there are no external dependencies other than
+docker, which is used to run postgres in development as well as
+generate ssh keys for users in production. There is no JavaScript or
+styling libraries. The webapp is built as HTML pages with forms.
+
+The entry point for the webapp is ~src/nassella.scm~.
+
+*** HTML Widgets
+
+The core of page markup is created with the html-widgets library which
+provides the ~define-widget~ function. This would be similar to a
+"component" in something like React. It allows defining properties of
+a widget (component) as well as a name and default values. It also
+handles style transforms, detailed in the next section.
+
+*** Styling
+
+All styling is done via ~style~ attributes on HTML elements. This is
+handled internally by the html-widgets library. The html-widgets
+library will extract all of the styles into a CSS stylesheet and
+replace the ~style~ attributes with a corresponding, auto-generated,
+CSS class name attribute. Nothing is needed to define or manage styles
+other than to set them directly via an HTML ~style~ attribute.
+
+Shared styling values, like colors, are defined in the *style-tokens*
+global variable. It is a tree of style tokens (similar to
+[[https://css-tricks.com/what-are-design-tokens/][Design Tokens]]). ~nassella.scm~ also includes some functions to make it
+easy for widgets to fetch style values. The most common being ~$~
+which allows getting a style token value based on a dotted symbol or a
+symbol list ~($ 'color.primary.contrast)~ or ~($ '(color primary contrast)~

all-apps/docker-compose.yaml

@@ -1,12 +1,12 @@
 version: '3'
 services:
   lb:
-    image: docker.io/caddy:2
+    image: docker.io/caddy:2.10.2-alpine
     volumes:
 #      - /app/lb:/etc/caddy
       - ./lb/:/etc/caddy
-      - config:/config
-      - data:/data
+      - /nassella/lb/config:/config
+      - /nassella/lb/data:/data
     networks:
       - lb
     restart: unless-stopped
@@ -20,6 +20,4 @@ services:
       - lb
 networks:
   lb:
-volumes:
-  config:
-  data:
+

all-apps/dozzle/docker-compose.yaml

@@ -0,0 +1,10 @@
+services:
+  dozzle:
+    container_name: dozzle
+    image: amir20/dozzle:latest
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    networks:
+      - lb
+networks:
+  lb:

all-apps/ghost/.compose.env.tmpl

@@ -0,0 +1,61 @@
+# Use the below flags to enable the Analytics or ActivityPub containers as well
+# COMPOSE_PROFILES=analytics,activitypub
+
+# Ghost domain
+# Custom public domain Ghost will run on
+# GHOST_DOMAIN=www.nassella.cc
+
+# Ghost Admin domain
+# If you have Ghost Admin setup on a separate domain uncomment the line below and add the domain
+# You also need to uncomment the corresponding block in your Caddyfile
+# ADMIN_DOMAIN=
+
+# Database settings
+# All database settings must not be changed once the database is initialised
+# GHOST_DATABASE_ROOT_PASSWORD=reallysecurerootpassword
+# DATABASE_USER=optionalusername
+# GHOST_DATABASE_PASSWORD=ghostpassword
+
+# ActivityPub
+# If you'd prefer to self-host ActivityPub yourself uncomment the line below
+# ACTIVITYPUB_TARGET=activitypub:8080
+
+# Tinybird configuration
+# If you want to run Analytics, paste the output from `docker compose run --rm tinybird-login get-tokens` below
+# TINYBIRD_API_URL=https://api.tinybird.co
+# TINYBIRD_TRACKER_TOKEN=p.eyJxxxxx
+# TINYBIRD_ADMIN_TOKEN=p.eyJxxxxx
+# TINYBIRD_WORKSPACE_ID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+
+# Ghost configuration (https://ghost.org/docs/config/)
+
+# SMTP Email (https://ghost.org/docs/config/#mail)
+# Transactional email is required for logins, account creation (staff invites), password resets and other features
+# This is not related to bulk mail / newsletter sending
+mail__transport=SMTP
+# mail__options__host=
+# mail__options__port=
+mail__options__secure=true
+# mail__options__auth__user=
+# mail__options__auth__pass=
+# mail__from=""
+
+# Advanced customizations
+
+# Force Ghost version
+# You should only do this if you need to pin a specific version
+# The update commands won't work
+# GHOST_VERSION=6-alpine
+
+# Port Ghost should listen on
+# You should only need to edit this if you want to host
+# multiple sites on the same server
+# GHOST_PORT=2368
+
+# Data locations
+# Location to store uploaded data
+# GHOST_UPLOAD_LOCATION=./data/ghost
+
+# Location for database data
+# GHOST_MYSQL_DATA_LOCATION=./data/mysql
+# NEWLINE REQUIRED AT END OF FILE

all-apps/ghost/docker-compose.yaml

@@ -0,0 +1,185 @@
+services:
+  ghost:
+    image: ghost:${GHOST_VERSION:-6-alpine}
+    restart: always
+    # This is required to import current config when migrating
+    environment:
+      NODE_ENV: production
+      url: https://${GHOST_DOMAIN:?GHOST_DOMAIN environment variable is required}
+      admin__url: ${GHOST_ADMIN_DOMAIN:+https://${GHOST_ADMIN_DOMAIN}}
+      database__client: mysql
+      database__connection__host: ghost_db
+      database__connection__user: ${GHOST_DATABASE_USER:-ghost}
+      database__connection__password: ${GHOST_DATABASE_PASSWORD:?GHOST_DATABASE_PASSWORD environment variable is required}
+      database__connection__database: ghost
+      tinybird__tracker__endpoint: https://${GHOST_DOMAIN:?GHOST_DOMAIN environment variable is required}/.ghost/analytics/api/v1/page_hit
+      tinybird__adminToken: ${GHOST_TINYBIRD_ADMIN_TOKEN:-}
+      tinybird__workspaceId: ${GHOST_TINYBIRD_WORKSPACE_ID:-}
+      tinybird__tracker__datasource: analytics_events
+      tinybird__stats__endpoint: ${GHOST_TINYBIRD_API_URL:-https://api.tinybird.co}
+    volumes:
+      - /nassella/ghost/var-lib-ghost-content:/var/lib/ghost/content
+    depends_on:
+      ghost_db:
+        condition: service_healthy
+      ghost_tinybird-sync:
+        condition: service_completed_successfully
+        required: false
+      ghost_tinybird-deploy:
+        condition: service_completed_successfully
+        required: false
+      ghost_activitypub:
+        condition: service_started
+        required: false
+    networks:
+      - ghost_network
+      - lb
+
+  ghost_db:
+    image: mysql:8.0.44@sha256:f37951fc3753a6a22d6c7bf6978c5e5fefcf6f31814d98c582524f98eae52b21
+    restart: always
+    expose:
+      - "3306"
+    environment:
+      MYSQL_ROOT_PASSWORD: ${GHOST_DATABASE_ROOT_PASSWORD:?GHOST_DATABASE_ROOT_PASSWORD environment variable is required}
+      MYSQL_USER: ${GHOST_DATABASE_USER:-ghost}
+      MYSQL_PASSWORD: ${GHOST_DATABASE_PASSWORD:?GHOST_DATABASE_PASSWORD environment variable is required}
+      MYSQL_DATABASE: ghost
+      MYSQL_MULTIPLE_DATABASES: activitypub
+    volumes:
+      - /nassella/ghost/var-lib-mysql:/var/lib/mysql
+      - ./mysql-init:/docker-entrypoint-initdb.d
+    healthcheck:
+      test: mysqladmin ping -p$$GHOST_MYSQL_ROOT_PASSWORD -h 127.0.0.1
+      interval: 1s
+      start_period: 30s
+      start_interval: 10s
+      retries: 120
+    networks:
+      - ghost_network
+
+  ghost_traffic-analytics:
+    image: ghost/traffic-analytics:1.0.20@sha256:a72573d89457e778b00e9061422516d2d266d79a72a0fc02005ba6466e391859
+    restart: always
+    expose:
+      - "3000"
+    volumes:
+      - traffic_analytics_data:/data
+    environment:
+      NODE_ENV: production
+      PROXY_TARGET: ${GHOST_TINYBIRD_API_URL:-https://api.tinybird.co}/v0/events
+      SALT_STORE_TYPE: ${GHOST_SALT_STORE_TYPE:-file}
+      SALT_STORE_FILE_PATH: /data/salts.json
+      TINYBIRD_TRACKER_TOKEN: ${GHOST_TINYBIRD_TRACKER_TOKEN:-}
+      LOG_LEVEL: debug
+    profiles: [analytics]
+    networks:
+      - ghost_network
+
+  ghost_activitypub:
+    image: ghcr.io/tryghost/activitypub:1.1.0@sha256:39c212fe23603b182d68e67d555c6b9b04b1e57459dfc0bef26d6e4980eb04d1
+    restart: always
+    expose:
+      - "8080"
+    volumes:
+      - /nassella/ghost/var-lib-ghost-content:/opt/activitypub/content
+    environment:
+      # See https://github.com/TryGhost/ActivityPub/blob/main/docs/env-vars.md
+      NODE_ENV: production
+      MYSQL_HOST: ghost_db
+      MYSQL_USER: ${GHOST_DATABASE_USER:-ghost}
+      MYSQL_PASSWORD: ${GHOST_DATABASE_PASSWORD:?GHOST_DATABASE_PASSWORD environment variable is required}
+      MYSQL_DATABASE: activitypub
+      LOCAL_STORAGE_PATH: /opt/activitypub/content/images/activitypub
+      LOCAL_STORAGE_HOSTING_URL: https://${GHOST_DOMAIN}/content/images/activitypub
+    depends_on:
+      ghost_db:
+        condition: service_healthy
+      ghost_activitypub-migrate:
+        condition: service_completed_successfully
+    profiles: [activitypub]
+    networks:
+      - ghost_network
+
+  # Supporting Services
+
+  ghost_tinybird-login:
+    build:
+      context: ./tinybird
+      dockerfile: Dockerfile
+    working_dir: /home/tinybird
+    command: /usr/local/bin/tinybird-login
+    volumes:
+      - tinybird_home:/home/tinybird
+      - tinybird_files:/data/tinybird
+    profiles: [analytics]
+    networks:
+      - ghost_network
+    tty: false
+    restart: no
+
+  ghost_tinybird-sync:
+    # Do not alter this without updating the Ghost container as well
+    image: ghost:${GHOST_VERSION:-6-alpine}
+    command: >
+      sh -c "
+        if [ -d /var/lib/ghost/current/core/server/data/tinybird ]; then
+          rm -rf /data/tinybird/*;
+          cp -rf /var/lib/ghost/current/core/server/data/tinybird/* /data/tinybird/;
+          echo 'Tinybird files synced into shared volume.';
+        else
+          echo 'Tinybird source directory not found.';
+        fi
+      "
+    volumes:
+      - tinybird_files:/data/tinybird
+    depends_on:
+      ghost_tinybird-login:
+        condition: service_completed_successfully
+    networks:
+      - ghost_network
+    profiles: [analytics]
+    restart: no
+
+  ghost_tinybird-deploy:
+    build:
+      context: ./tinybird
+      dockerfile: Dockerfile
+    working_dir: /data/tinybird
+    command: >
+      sh -c "
+        tb-wrapper --cloud deploy
+      "
+    volumes:
+      - tinybird_home:/home/tinybird
+      - tinybird_files:/data/tinybird
+    depends_on:
+      ghost_tinybird-sync:
+        condition: service_completed_successfully
+    profiles: [analytics]
+    networks:
+      - ghost_network
+    tty: true
+
+  ghost_activitypub-migrate:
+    image: ghcr.io/tryghost/activitypub-migrations:1.1.0@sha256:b3ab20f55d66eb79090130ff91b57fe93f8a4254b446c2c7fa4507535f503662
+    environment:
+      MYSQL_DB: mysql://${GHOST_DATABASE_USER:-ghost}:${GHOST_DATABASE_PASSWORD:?GHOST_DATABASE_PASSWORD environment variable is required}@tcp(ghost_db:3306)/activitypub
+    networks:
+      - ghost_network
+    depends_on:
+      ghost_db:
+        condition: service_healthy
+    profiles: [activitypub]
+    restart: no
+
+volumes:
+  tinybird_files:
+  tinybird_home:
+  traffic_analytics_data:
+
+networks:
+  lb:
+  ghost_network:
+    driver: bridge
+    internal: true

all-apps/nassella/docker-compose.yaml

@@ -0,0 +1,51 @@
+version: '3'
+
+secrets:
+  nassella_postgres_db:
+    file: ./nassella/postgres_db
+  nassella_postgres_password:
+    file: ./nassella/postgres_password
+  nassella_postgres_user:
+    file: ./nassella/postgres_user
+
+services:
+  nassella_db:
+    image: postgres:17.6-trixie
+    env_file:
+      - ./nassella/nassella.env
+    shm_size: 128mb
+    restart: always
+    volumes:
+      - /nassella/nassella/var-lib-postgresql-data:/var/lib/postgresql/data
+    networks:
+      - nassella_internal
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -d `cat $$POSTGRES_DB_FILE` -U `cat $$POSTGRES_USER_FILE`"]
+      start_period: 15s
+      interval: 30s
+      retries: 3
+      timeout: 5s
+    secrets:
+      - nassella_postgres_db
+      - nassella_postgres_password
+      - nassella_postgres_user
+  nassella:
+    image: nassella/b0.0.1
+    depends_on:
+      nassella_db:
+        condition: service_healthy
+    env_file:
+      - ./nassella/nassella.env
+    secrets:
+      - nassella_postgres_db
+      - nassella_postgres_password
+      - nassella_postgres_user
+    networks:
+      - lb
+      - nassella_internal
+    restart: unless-stopped
+networks:
+  lb:
+  nassella_internal:
+    driver: bridge
+    internal: true

all-apps/nassella/nassella.env

@@ -0,0 +1,4 @@
+POSTGRES_HOST=nassella_db
+POSTGRES_DB_FILE=/run/secrets/nassella_postgres_db
+POSTGRES_USER_FILE=/run/secrets/nassella_postgres_user
+POSTGRES_PASSWORD_FILE=/run/secrets/nassella_postgres_password

all-apps/nassella/postgres_db

@@ -0,0 +1 @@
+nassella

all-apps/nassella/postgres_password

@@ -0,0 +1 @@
+password

all-apps/nassella/postgres_user

@@ -0,0 +1 @@
+nassella

all-apps/nextcloud/docker-compose.yaml

@@ -1,50 +1,74 @@
 version: '3'
 
+secrets:
+  nextcloud_postgres_db:
+    file: ./nextcloud/postgres_db
+  nextcloud_postgres_password:
+    file: ./nextcloud/postgres_password
+  nextcloud_postgres_user:
+    file: ./nextcloud/postgres_user
+  nextcloud_redis_password:
+    file: ./nextcloud/redis_password
+
 services:
-  db:
-    image: postgres
+  nextcloud_db:
+    image: postgres:17.6-trixie
+    env_file:
+      - ./nextcloud/nextcloud.env
     shm_size: 128mb
     restart: always
     volumes:
-      - db:/var/lib/postgresql/data
-    environment:
-      - POSTGRES_DB=nextcloud
-      - POSTGRES_USER=nextcloud
-      - POSTGRES_PASSWORD=password
+      - /nassella/nextcloud/var-lib-postgresql-data:/var/lib/postgresql/data
     networks:
-      - internal
-  redis:
-    image: redis:alpine
+      - nextcloud_internal
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -d `cat $$POSTGRES_DB_FILE` -U `cat $$POSTGRES_USER_FILE`"]
+      start_period: 15s
+      interval: 30s
+      retries: 3
+      timeout: 5s
+    secrets:
+      - nextcloud_postgres_db
+      - nextcloud_postgres_password
+      - nextcloud_postgres_user
+  nextcloud_redis:
+    image: redis:8.2.1-bookworm
+    env_file:
+      - ./nextcloud/nextcloud.env
+    command: bash -c 'redis-server --requirepass "$$(cat /run/secrets/nextcloud_redis_password)"'
+    secrets:
+      - nextcloud_redis_password
     restart: always
+    healthcheck:
+      test: ["CMD-SHELL", "redis-cli --no-auth-warning -a \"$$(cat /run/secrets/nextcloud_redis_password)\" ping | grep PONG"]
+      start_period: 10s
+      interval: 30s
+      retries: 3
+      timeout: 3s
     networks:
-      - internal
+      - nextcloud_internal
   nextcloud:
-    image: nextcloud
-    environment:
-      - POSTGRES_HOST=db
-      - POSTGRES_DB=nextcloud
-      - POSTGRES_USER=nextcloud
-      - POSTGRES_PASSWORD=password
-      - NEXTCLOUD_ADMIN_PASSWORD=password
-      - NEXTCLOUD_ADMIN_USER=admin
-      - REDIS_HOST=redis
-      - NEXTCLOUD_TRUSTED_DOMAINS=nextcloud1.nassella.cc
-    ports:
-      - "8080:80"
+    image: nextcloud:31.0.8-apache
     depends_on:
-      - redis
-      - db
+      nextcloud_redis:
+        condition: service_healthy
+      nextcloud_db:
+        condition: service_healthy
+    env_file:
+      - ./nextcloud/nextcloud.env
+    secrets:
+      - nextcloud_postgres_db
+      - nextcloud_postgres_password
+      - nextcloud_postgres_user
+      - nextcloud_redis_password
     networks:
       - lb
-      - internal
+      - nextcloud_internal
     volumes:
-      - nextcloud:/var/www
+      - /nassella/nextcloud/var-www-html:/var/www/html
     restart: unless-stopped
 networks:
   lb:
-  internal:
+  nextcloud_internal:
     driver: bridge
     internal: true
-volumes:
-  db:
-  nextcloud:

all-apps/nextcloud/nextcloud.env.tmpl

@@ -0,0 +1,16 @@
+NEXTCLOUD_TRUSTED_DOMAINS=${DOMAIN}
+
+# reverse proxy config
+OVERWRITEHOST=${DOMAIN}
+OVERWRITECLIURL=https://${DOMAIN}
+OVERWRITEPROTOCOL=https
+TRUSTED_PROXIES=172.16.0.0/24 # trust the local lb
+PHP_MEMORY_LIMIT=1G
+PHP_UPLOAD_LIMIT=10G
+POSTGRES_HOST=nextcloud_db
+POSTGRES_DB_FILE=/run/secrets/nextcloud_postgres_db
+POSTGRES_USER_FILE=/run/secrets/nextcloud_postgres_user
+POSTGRES_PASSWORD_FILE=/run/secrets/nextcloud_postgres_password
+# redis
+REDIS_HOST=nextcloud_redis
+REDIS_HOST_PASSWORD_FILE=/run/secrets/nextcloud_redis_password

all-apps/wg-easy/docker-compose.yaml

@@ -12,7 +12,7 @@ services:
         ipv4_address: 10.42.42.42
 #        ipv6_address: fdcc:ad94:bacf:61a3::2a
     volumes:
-      - etc_wireguard:/etc/wireguard
+      - /nassella/wg-easy/etc-wireguard:/etc/wireguard
       - /lib/modules:/lib/modules:ro
     restart: unless-stopped
     cap_add:
@@ -33,5 +33,3 @@ networks:
       config:
         - subnet: 10.42.42.0/24
         - subnet: fdcc:ad94:bacf:61a3::/64
-volumes:
-  etc_wireguard:

apps.config.tmpl

@@ -1,2 +0,0 @@
-ROOT_DOMAIN=example.com
-APP_CONFIGS="nextcloud,nextcloud wg-easy,wg-easy"

cl.yaml

@@ -5,14 +5,16 @@ passwd:
     - name: core
       ssh_authorized_keys_local:
         - /ssh-keys
+    - name: nextcloud
+      uid: 1001
 systemd:
   units:
-    - name: var-lib-docker-volumes.mount
+    - name: nassella.mount
       enabled: true
       contents: |
         [Mount]
         What=/dev/disk/by-partlabel/appstorage
-        Where=/var/lib/docker/volumes
+        Where=/nassella
         Type=ext4
 
         [Install]
@@ -20,6 +22,30 @@ systemd:
     - name: app.service
       enabled: true
       contents_local: app/app.service
+    - name: restic-backup.service
+      contents: |
+        [Unit]
+        Description=Backs up application data
+        Conflicts=app.service
+
+        [Service]
+        Type=oneshot
+        EnvironmentFile=/restic-env
+        ExecStart=/usr/bin/bash -c "docker run --rm --volume /nassella:/nassella --volume /restic-password:/restic-password -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} -i restic/restic:0.18.0 backup --verbose --repo s3:${BACKBLAZE_BUCKET_URL} --password-file /restic-password /nassella"
+        ExecStopPost=systemctl start app.service
+
+    - name: restic-backup.timer
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Run restic-backup.service at 3am PT
+
+        [Timer]
+        OnCalendar=*-*-* 10:00:00
+
+        [Install]
+        WantedBy=multi-user.target
+
     ### docker-compose sysext
     ### https://flatcar.github.io/sysext-bakery/docker_compose/
     - name: systemd-sysupdate.timer
@@ -29,8 +55,8 @@ systemd:
         - name: 10-wait-docker.conf
           contents: |
             [Unit]
-            After=var-lib-docker-volumes.mount
-            Requires=var-lib-docker-volumes.mount
+            After=nassella.mount
+            Requires=nassella.mount
     - name: systemd-sysupdate.service
       dropins:
         - name: docker-compose.conf
@@ -44,7 +70,11 @@ systemd:
 # device: /dev/disk/by-label/appstorage
 storage:
   disks:
+    # TODO I think this can be changed back to
+    # device: /dev/disk/by-label/appstorage
+    # I think it didn't work before becase the partition number was 0 (now correctly set to 1)
     - device: /dev/sda
+#    - device: /dev/disk/by-label/appstorage
       wipe_table: false
       partitions:
         - label: appstorage
@@ -57,6 +87,12 @@ storage:
     - path: /app
       local: app
   files:
+    - path: /restic-password
+      contents:
+        local: restic-password
+    - path: /restic-env
+      contents:
+        local: restic-env
     ### docker-compose sysext
     ### https://flatcar.github.io/sysext-bakery/docker_compose/
     - path: /opt/extensions/docker-compose/docker-compose-2.34.0-x86-64.raw

config/apps.config.tmpl

@@ -0,0 +1,19 @@
+ROOT_DOMAIN=  # example.com :: the root of the domain that all apps should be subdomains of
+APP_CONFIGS="nextcloud,nextcloud wg-easy,wg-easy ghost,ghost" # apps to deploy and their corresponding sub-domain (app,sub-domain)
+NEXTCLOUD_ADMIN_USER=admin # admin user for nextcloud, can be whatever you want
+NEXTCLOUD_ADMIN_PASSWORD=  # the password for the nextcloud admin user
+NEXTCLOUD_POSTGRES_DB=nextcloud # recommended to leave as 'nextcloud'. The postgres db nextcloud uses
+NEXTCLOUD_POSTGRES_USER=nextcloud # recommended to leave as 'nextcloud'. The postgres user nextcloud uses
+NEXTCLOUD_POSTGRES_PASSWORD= # should be a secure, randomly generated, postgres compatible password, stored in the config so it isn't lost on re-deployment but otherwise unneeded
+NEXTCLOUD_REDIS_PASSWORD= # should be a secure, randomly generated, redis compatible password, stored in the config so it isn't lost on re-deployment but otherwise unneeded
+GHOST_DATABASE_ROOT_PASSWORD=
+GHOST_DATABASE_PASSWORD=
+SMTP_HOST=
+SMTP_PORT=
+SMTP_AUTH_USER=
+SMTP_AUTH_PASSWORD=
+SMTP_FROM=
+BACKBLAZE_KEY_ID= # the key ID for a application key created on backblaze that has permissions for the bucket in BACKBLAZE_BUCKET_URL
+BACKBLAZE_APPLICATION_KEY= # the application key for the application key created on backblaze
+BACKBLAZE_BUCKET_URL= # the full URL for the backblaze bucket, found on the backblaze UI for the bucket
+RESTIC_PASSWORD= # should be a secure, randomly generated, restic compatible password. Used for making encrypted backups of the application data

config/production.tfvars.tmpl

@@ -0,0 +1,12 @@
+server_type = "s-2vcpu-2gb" # the digital ocean server type to deploy
+
+do_token = "" # token from "API" settings on DigitalOcean
+
+cloudflare_api_token = "" # corresponding API token should allow modifying DNS settings for the Nassella configured domain
+cloudflare_zone_id = "" # corresponding zone ID for API token for the Nassella configured domain
+cloudflare_account_id = "" # corresponding account ID for API token
+
+cluster_name           = "mycluster" # currently only used as the name of the machine on DigitalOcean
+datacenter             = "sfo3" # datacenter to deploy the droplet to
+ssh_keys               = [""] # unused
+flatcar_stable_version = "4459.2.1" # (source <(curl -sSfL https://stable.release.flatcar-linux.net/amd64-usr/current/version.txt); echo "${FLATCAR_VERSION_ID}")

copy-apps.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# this script copys over the docker configs
+# for in-use apps
+
+# it depends on apps.config which should define:
+# ROOT_DOMAIN - the root domain for all apps
+# APP_CONFIGS - app-subdomain pairs, configured via a comma, like:
+#   app1,subdomain1 app2,subdomain2 app3,subdomain3
+# full example:
+#   ROOT_DOMAIN=nassella.cc
+#   APP_CONFIGS="app1,subdomain1 app2,subdomain2 app3,subdomain3"
+
+set -e
+
+. $1 # source the apps.config file with then env vars
+
+read -r -a APP_CONFIGS <<< "$APP_CONFIGS"
+APP_CONFIGS+=('lb,root')
+
+
+for config_string in ${APP_CONFIGS[@]}; do
+    IFS=','
+    read -r -a config <<< "$config_string"
+
+    app=${config[0]}
+    cp -a all-apps/$app app/
+done
+
+
+

flatcar/app/Makefile

@@ -1,2 +0,0 @@
-run:
-	sudo docker-compose -f docker-compose.yaml $(find . -mindepth 2 -maxdepth 2 -type f -name docker-compose.yaml -exec echo -f {} \;) up

flatcar/app/app.service

@@ -1,13 +0,0 @@
-[Unit]
-Description=Main App
-After=docker.service
-Requires=docker.service
-[Service]
-TimeoutStartSec=0
-ExecStart=/bin/bash -c '/usr/bin/docker compose -f /app/docker-compose.yaml $(find /app -mindepth 2 -maxdepth 2 -type f -name docker-compose.yaml -exec echo -f {} \;) up'
-ExecStop=/bin/bash -c '/usr/bin/docker compose -f /app/docker-compose.yaml $(find /app -mindepth 2 -maxdepth 2 -type f -name docker-compose.yaml -exec echo -f {} \;) stop'
-
-Restart=always
-RestartSec=5s
-[Install]
-WantedBy=multi-user.target

flatcar/app/docker-compose.yaml

@@ -1,25 +0,0 @@
-version: '3'
-services:
-  lb:
-    image: docker.io/caddy:2
-    volumes:
-#      - /app/lb:/etc/caddy
-      - ./lb/:/etc/caddy
-      - config:/config
-      - data:/data
-    networks:
-      - lb
-    restart: unless-stopped
-    ports:
-      - "443:443"
-      - "80:80"
-  nginx:
-    image: nginx
-    restart: unless-stopped
-    networks:
-      - lb
-networks:
-  lb:
-volumes:
-  config:
-  data:

flatcar/app/lb/Caddyfile

@@ -1,17 +0,0 @@
-wg-easy1.nassella.cc {
-  reverse_proxy http://wg-easy:80
-
-  # tls internal
-  # x
-  # log
-}
-
-nextcloud1.nassella.cc {
-  reverse_proxy http://nextcloud:80
-  # tls internal
-}
-
-root.nassella.cc {
-  reverse_proxy http://nginx:80
-  # tls internal
-}

flatcar/app/nextcloud/docker-compose.yaml

@@ -1,50 +0,0 @@
-version: '3'
-
-services:
-  db:
-    image: postgres
-    shm_size: 128mb
-    restart: always
-    volumes:
-      - db:/var/lib/postgresql/data
-    environment:
-      - POSTGRES_DB=nextcloud
-      - POSTGRES_USER=nextcloud
-      - POSTGRES_PASSWORD=password
-    networks:
-      - internal
-  redis:
-    image: redis:alpine
-    restart: always
-    networks:
-      - internal
-  nextcloud:
-    image: nextcloud
-    environment:
-      - POSTGRES_HOST=db
-      - POSTGRES_DB=nextcloud
-      - POSTGRES_USER=nextcloud
-      - POSTGRES_PASSWORD=password
-      - NEXTCLOUD_ADMIN_PASSWORD=password
-      - NEXTCLOUD_ADMIN_USER=admin
-      - REDIS_HOST=redis
-      - NEXTCLOUD_TRUSTED_DOMAINS=nextcloud1.nassella.cc
-    ports:
-      - "8080:80"
-    depends_on:
-      - redis
-      - db
-    networks:
-      - lb
-      - internal
-    volumes:
-      - nextcloud:/var/www
-    restart: unless-stopped
-networks:
-  lb:
-  internal:
-    driver: bridge
-    internal: true
-volumes:
-  db:
-  nextcloud:

flatcar/app/wg-easy/docker-compose.yaml

@@ -1,37 +0,0 @@
-version: '3'
-services:
-  wg-easy:
-    image: ghcr.io/wg-easy/wg-easy:15
-    environment:
-      - PORT=80
-    ports:
-      - "51820:51820/udp"
-    networks:
-      lb:
-      wg:
-        ipv4_address: 10.42.42.42
-#        ipv6_address: fdcc:ad94:bacf:61a3::2a
-    volumes:
-      - etc_wireguard:/etc/wireguard
-      - /lib/modules:/lib/modules:ro
-    restart: unless-stopped
-    cap_add:
-      - NET_ADMIN
-      - SYS_MODULE
-    sysctls:
-      - net.ipv4.ip_forward=1
-      - net.ipv4.conf.all.src_valid_mark=1
-      - net.ipv6.conf.all.disable_ipv6=0
-      - net.ipv6.conf.all.forwarding=1
-      - net.ipv6.conf.default.forwarding=1
-networks:
-  lb:
-  wg:
-    driver: bridge
-    ipam:
-      driver: default
-      config:
-        - subnet: 10.42.42.0/24
-        - subnet: fdcc:ad94:bacf:61a3::/64
-volumes:
-  etc_wireguard:

init-restic.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+. $1 # source the apps.config file with then env vars
+
+mkdir -p emptydir
+docker run --rm --volume $PWD/emptydir:/nassella --volume $PWD/restic-password:/restic-password -e AWS_ACCESS_KEY_ID="$BACKBLAZE_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$BACKBLAZE_APPLICATION_KEY" -i restic/restic:0.18.0 init --repo s3:$BACKBLAZE_BUCKET_URL --password-file /restic-password
+rm -Rf emptydir

main.tf

@@ -104,7 +104,7 @@ resource "digitalocean_reserved_ip" "machine" {
 
 resource "cloudflare_dns_record" "root" {
   zone_id = var.cloudflare_zone_id
-  name    = "@"
+  name    = "_nassella-instance"
   content   = digitalocean_reserved_ip.machine.ip_address
   type    = "A"
   proxied = false
@@ -115,7 +115,7 @@ resource "cloudflare_dns_record" "subdomains" {
   for_each  = toset(var.subdomains)
   zone_id = var.cloudflare_zone_id
   name    = each.key
-  content   = var.domain
+  content   = "_nassella-instance.${var.domain}"
   type    = "CNAME"
   proxied = false
   ttl = 300
@@ -137,6 +137,7 @@ resource "digitalocean_droplet" "machine" {
   size      = var.server_type
   ssh_keys  = [digitalocean_ssh_key.first.fingerprint]
   user_data = file("ignition.json")
+  graceful_shutdown = true
   lifecycle {
     create_before_destroy = true
   }

make-caddyfile.sh

@@ -13,7 +13,10 @@
 
 set -e
 
-. apps.config
+. $1 # source the apps.config file with then env vars
+
+host_admin_password_encoded=`echo "$HOST_ADMIN_PASSWORD" | docker run --rm -i caddy:2 caddy hash-password`
+
 
 read -r -a APP_CONFIGS <<< "$APP_CONFIGS"
 APP_CONFIGS+=('lb,root')
@@ -22,6 +25,15 @@ APP_CONFIGS+=('lb,root')
 declare -A bodys
 bodys["nextcloud"]="  reverse_proxy http://nextcloud:80"
 bodys["wg-easy"]="  reverse_proxy http://wg-easy:80"
+bodys["ghost"]="  reverse_proxy http://ghost:2368"
+bodys["nassella"]="  reverse_proxy http://nassella:8080"
+bodys["dozzle"]=$(cat <<EOF
+  basic_auth {
+    $HOST_ADMIN_USER $host_admin_password_encoded
+  }
+  reverse_proxy http://dozzle:8080
+EOF
+)
 bodys["lb"]="  reverse_proxy http://nginx:80"
 
 for config_string in ${APP_CONFIGS[@]}; do

make-generated.sh

@@ -13,7 +13,7 @@
 
 set -e
 
-. apps.config
+. $1 # source the apps.config file with then env vars
 
 read -r -a APP_CONFIGS <<< "$APP_CONFIGS"
 APP_CONFIGS+=('lb,root')

make-ghost-env.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e
+
+. $1 # source the apps.config file with then env vars
+
+read -r -a APP_CONFIGS <<< "$APP_CONFIGS"
+
+nextcloud_subdomain=
+
+for config_string in ${APP_CONFIGS[@]}; do
+    IFS=','
+    read -r -a config <<< "$config_string"
+
+    app=${config[0]}
+    subdomain=${config[1]}
+
+    if [ "$app" = "ghost" ]; then
+       ghost_subdomain="$subdomain"
+    fi
+done
+
+# write compose env file
+echo "GHOST_DOMAIN=\"$ghost_subdomain.$ROOT_DOMAIN\"" > all-apps/ghost/.compose-env
+echo "GHOST_DATABASE_ROOT_PASSWORD=\"$GHOST_DATABASE_ROOT_PASSWORD\"" >> all-apps/ghost/.compose-env
+echo "GHOST_DATABASE_PASSWORD=\"$GHOST_DATABASE_PASSWORD\"" >> all-apps/ghost/.compose-env
+echo "mail__options__host=\"$SMTP_HOST\"" >> all-apps/ghost/.compose-env
+echo "mail__options__port=\"$SMTP_PORT\"" >> all-apps/ghost/.compose-env
+echo "mail__options__auth__user=\"$SMTP_AUTH_USER\"" >> all-apps/ghost/.compose-env
+echo "mail__options__auth__pass=\"$SMTP_AUTH_PASSWORD\"" >> all-apps/ghost/.compose-env
+echo "mail__from=\"$SMTP_FROM\"" >> all-apps/ghost/.compose-env
+cat all-apps/ghost/.compose.env.tmpl >> all-apps/ghost/.compose-env

make-nextcloud-env.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -e
+
+. $1 # source the apps.config file with then env vars
+
+read -r -a APP_CONFIGS <<< "$APP_CONFIGS"
+
+nextcloud_subdomain=
+
+for config_string in ${APP_CONFIGS[@]}; do
+    IFS=','
+    read -r -a config <<< "$config_string"
+
+    app=${config[0]}
+    subdomain=${config[1]}
+
+    if [ "$app" = "nextcloud" ]; then
+       nextcloud_subdomain="$subdomain"
+    fi
+done
+
+# write container env file
+echo "DOMAIN=\"$nextcloud_subdomain.$ROOT_DOMAIN\"" > all-apps/nextcloud/nextcloud.env
+cat all-apps/nextcloud/nextcloud.env.tmpl >> all-apps/nextcloud/nextcloud.env
+
+# write secrets
+echo "$NEXTCLOUD_POSTGRES_DB" > all-apps/nextcloud/nextcloud_postgres_db
+echo "$NEXTCLOUD_POSTGRES_USER" > all-apps/nextcloud/nextcloud_postgres_user
+echo "$NEXTCLOUD_POSTGRES_PASSWORD" > all-apps/nextcloud/nextcloud_postgres_password
+echo "$NEXTCLOUD_REDIS_PASSWORD" > all-apps/nextcloud/nextcloud_redis_password

make-restic-generated.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+. $1 # source the apps.config file with then env vars
+
+echo "AWS_ACCESS_KEY_ID=\"$BACKBLAZE_KEY_ID\""
+echo "AWS_SECRET_ACCESS_KEY=\"$BACKBLAZE_APPLICATION_KEY\""
+echo "BACKBLAZE_BUCKET_URL=\"$BACKBLAZE_BUCKET_URL\""

make-restic-password.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -e
+
+. $1 # source the apps.config file with then env vars
+
+echo "$RESTIC_PASSWORD"

restic-snapshots.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -e
+
+. $1 # source the apps.config file with then env vars
+
+docker run --rm --volume $PWD/restic-password:/restic-password -e AWS_ACCESS_KEY_ID="$BACKBLAZE_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$BACKBLAZE_APPLICATION_KEY" -i restic/restic:0.18.0 snapshots --repo s3:$BACKBLAZE_BUCKET_URL --password-file /restic-password

src/Dockerfile

@@ -0,0 +1,73 @@
+# based on https://github.com/scheme-containers/monorepo/blob/master/implementations/chicken/5/Dockerfile
+FROM debian:trixie-slim AS build
+RUN apt-get update && apt-get -y --no-install-recommends install \
+      build-essential \
+ && rm -rf /var/lib/apt/lists/*
+WORKDIR /build
+COPY checksum checksum
+ADD https://code.call-cc.org/releases/5.4.0/chicken-5.4.0.tar.gz chicken.tar.gz
+RUN sha256sum chicken.tar.gz && sha256sum -c checksum
+RUN mkdir chicken && tar -C chicken --strip-components 1 -xf chicken.tar.gz
+WORKDIR /build/chicken
+RUN make
+RUN make install
+
+FROM debian:trixie-slim AS buildeggs
+RUN apt-get update && apt-get -y --no-install-recommends install \
+      gcc libc-dev libpq-dev \
+ && rm -rf /var/lib/apt/lists/*
+COPY --from=build /usr/local/ /usr/local/
+
+COPY scss /var/scss
+COPY html-widgets /var/html-widgets
+COPY schematra-session /var/schematra-session
+WORKDIR /var/scss
+RUN chicken-install
+WORKDIR /var/html-widgets
+RUN chicken-install
+WORKDIR /var/
+
+RUN chicken-install srfi-1 srfi-13 srfi-18 srfi-158 srfi-194 \
+        sxml-transforms schematra \
+        uri-common http-client medea intarweb \
+        sql-null openssl postgresql crypto-tools
+
+# Egg is currently broken should be able to move back to regular install after it is fixed
+WORKDIR /var/schematra-session
+RUN chicken-install
+
+WORKDIR /var
+RUN mkdir nassella
+WORKDIR /var/nassella
+COPY mocks.scm mocks.scm
+COPY db.scm db.scm
+COPY nassella.scm nassella.scm
+COPY run.scm run.scm
+
+RUN csc -O3 mocks.scm -J
+RUN csc -O3 db.scm -J
+RUN csc -O3 nassella.scm -J
+RUN csc -O3 -o nassella-run run.scm
+RUN chmod +x nassella-run
+
+FROM debian:trixie-slim
+RUN apt-get update && apt-get -y --no-install-recommends install \
+ libpq-dev \
+ && rm -rf /var/lib/apt/lists/*
+COPY --from=buildeggs /usr/local/ /usr/local/
+
+WORKDIR /var
+COPY --from=buildeggs /var/nassella/mocks /var
+COPY --from=buildeggs /var/nassella/db /var
+COPY --from=buildeggs /var/nassella/nassella /var
+COPY --from=buildeggs /var/nassella/nassella-run /var
+
+COPY nassella-latest.tar nassella-latest.tar
+COPY root-key root-key
+COPY db-init.sql db-init.sql
+COPY db-clean.sql db-clean.sql
+
+# ENTRYPOINT ["ls"]
+# CMD ["/usr/local/lib/chicken/11"]
+ENTRYPOINT ["./nassella-run"]
+CMD ["-:a50"]

src/Makefile

@@ -0,0 +1,14 @@
+dockerall:
+	docker buildx build --platform linux/amd64,linux/arm64 -t nassella/b0.0.1 .
+
+dockerlocal:
+	docker buildx build -t nassella/b0.0.1 .
+
+dockerpush:
+	docker push nassella/b0.0.1
+
+local:
+	docker run -p 8080:8080 --net=host --rm nassella/b0.0.1
+
+localclean:
+	docker run -p 8080:8080 --net=host --rm nassella/b0.0.1 --clean

src/checksum

@@ -0,0 +1 @@
+3c5d4aa61c1167bf6d9bf9eaf891da7630ba9f5f3c15bf09515a7039bfcdec5f  chicken.tar.gz

src/compose.yaml

@@ -0,0 +1,17 @@
+services:
+  db:
+    image: postgres
+    restart: always
+    environment:
+      POSTGRES_USER: nassella
+      POSTGRES_PASSWORD: password
+      POSTGRES_DB: nassella
+    volumes:
+      - /home/tjhintz/nassella-db:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready", "-U", "nassella"]
+      interval: 1s
+      timeout: 5s
+      retries: 10
+    ports:
+      - "5432:5432"

src/db-clean.sql

@@ -0,0 +1,22 @@
+drop index user_terraform_state_user_id_instance_id_idx;
+drop table user_terraform_state;
+
+drop index deployments_user_id_instance_id_idx;
+drop table deployments;
+
+drop type deployment_status;
+
+drop index user_app_configs_user_id_instance_id_idx;
+drop table user_app_configs;
+
+drop index user_selected_apps_user_id_instance_id_idx;
+drop table user_selected_apps;
+
+drop index user_service_configs_user_id_instance_id_idx;
+drop table user_service_configs;
+
+drop index instances_user_id_instance_id_idx;
+drop table instances;
+
+drop index users_auth_user_id_idx;
+drop table users;

src/db-init.sql

@@ -0,0 +1,91 @@
+create table users(
+       user_id bigserial primary key,
+       auth_user_id int unique not null,
+       email varchar(255) not null,
+       username varchar(255) not null unique,
+       key_key varchar(255),
+       key_iv varchar(255)
+);
+create unique index users_auth_user_id_idx on users (auth_user_id);
+
+create table instances(
+       instance_id bigserial primary key,
+       user_id integer not null references users on delete cascade,
+       ssh_key_priv_enc text,
+       ssh_key_pub_enc text,
+       restic_password_enc varchar(255)
+);
+create unique index instances_user_id_instance_id_idx on instances (instance_id, user_id);
+
+create table user_service_configs(
+       id bigserial primary key,
+       user_id integer not null references users on delete cascade,
+       instance_id integer not null references instances on delete cascade,
+       cloudflare_api_token_enc varchar(255),
+       cloudflare_account_id_enc varchar(255),
+       cloudflare_zone_id_enc varchar(255),
+       digitalocean_api_token_enc varchar(255),
+       digitalocean_region varchar(255),
+       digitalocean_size varchar(255),
+       backblaze_application_key_enc varchar(255),
+       backblaze_key_id_enc varchar(255),
+       backblaze_bucket_url_enc varchar(255)
+       );
+create unique index user_service_configs_user_id_instance_id_idx on user_service_configs (user_id, instance_id);
+
+create table user_selected_apps(
+       id bigserial primary key,
+       user_id integer not null references users on delete cascade,
+       instance_id integer not null references instances on delete cascade,
+       wg_easy_version varchar(100),
+       nextcloud_version varchar(100),
+       nassella_version varchar(100),
+       log_viewer_version varchar(100),
+       ghost_version varchar(100)
+       );
+create unique index user_selected_apps_user_id_instance_id_idx on user_selected_apps (user_id, instance_id);
+
+create table user_app_configs(
+       id bigserial primary key,
+       user_id integer not null references users on delete cascade,
+       instance_id integer not null references instances on delete cascade,
+       root_domain varchar(100),
+       config_enc text
+       );
+
+create unique index user_app_configs_user_id_instance_id_idx on user_app_configs (user_id, instance_id);
+
+
+create type deployment_status as enum ('queued', 'in-progress', 'complete', 'failed');
+
+create table deployments(
+       id bigserial primary key,
+       user_id integer not null references users on delete cascade,
+       instance_id integer not null references instances on delete cascade,
+       started timestamptz,
+       finished timestamptz,
+       status deployment_status not null default 'queued',
+       pid integer,
+       generate_configs deployment_status not null default 'queued',
+       terraform_custom_image deployment_status not null default 'queued',
+       terraform_dns deployment_status not null default 'queued',
+       terraform_volume_create deployment_status not null default 'queued',
+       terraform_volume_destroy deployment_status not null default 'queued',
+       terraform_machine_create deployment_status not null default 'queued',
+       terraform_machine_destroy deployment_status not null default 'queued',
+       terraform_ip_create deployment_status not null default 'queued',
+       terraform_ip_destroy deployment_status not null default 'queued',
+       log_enc text
+       );
+
+create index deployments_user_id_instance_id_idx on deployments (user_id, instance_id);
+
+create table user_terraform_state(
+       id bigserial primary key,
+       user_id integer not null references users on delete cascade,
+       instance_id integer not null references instances on delete cascade,
+       state_enc text,
+       state_backup_enc text
+       );
+
+create unique index user_terraform_state_user_id_instance_id_idx on user_terraform_state (user_id, instance_id);

src/db.scm

@@ -0,0 +1,626 @@
+(module nassella-db
+    (;; parameters
+     connection-spec
+
+     ;;functions
+     with-db with-db/transaction
+     db-init db-clean
+
+     create-user delete-user
+     create-instance get-user-instances
+     get-instance-ssh-pub-key get-instance-ssh-priv-key
+     update-instance-ssh-pub-key
+     get-instance-restic-password
+     update-user-service-config get-user-service-config
+     update-user-selected-apps get-user-selected-apps
+     update-user-app-config get-user-app-config
+     update-root-domain
+     create-deployment
+     update-deployment-status get-deployment-status
+     get-most-recent-deployment-status
+     update-deployment-in-progress
+     update-deployment-progress get-deployment-progress
+     get-most-recent-deployment-progress
+     update-user-terraform-state get-user-terraform-state
+     get-dashboard
+     )
+
+(import scheme
+        (chicken base)
+        (chicken blob)
+        (chicken file)
+        (chicken string)
+        (chicken port)
+        (chicken io)
+        postgresql
+        sql-null
+        srfi-1
+        srfi-13
+        (openssl cipher)
+        (openssl random)
+        crypto-tools
+        spiffy)
+
+(define connection-spec (make-parameter '((dbname . "nassella") (user . "nassella") (password . "password")
+                                          ;; (host . "127.0.0.1")
+                                          (host . "nassella_db")
+                                          )))
+(define db-connection (make-parameter #f))
+
+(define (with-db proc)
+  (if (db-connection)
+      (begin (db-connection)
+             (proc (db-connection)))
+      (let ((conn #f))
+        (dynamic-wind
+          (lambda ()
+            (set! conn (connect (connection-spec))))
+          (lambda () (proc conn))
+          (lambda ()
+            (when (and (connection? conn) (connected? conn))
+              (disconnect conn)))))))
+
+(define (with-db/transaction proc)
+  (with-db
+   (lambda (conn)
+     (with-transaction conn
+       (lambda () (proc conn))))))
+
+;; (with-db (lambda (db) (row-values (query db "select * from users;"))))
+
+(define aes-256-gcm (cipher-by-name "aes-256-gcm"))
+(define tag-length 16)
+
+(define (generate-param accessor)
+  (random-bytes (accessor aes-256-gcm)))
+
+(define (generate-key) (generate-param cipher-key-length))
+(define (generate-iv) (generate-param cipher-iv-length))
+
+(define (encrypt message key iv #!optional auth-data)
+  (string-encrypt-and-digest aes-256-gcm message key iv
+                             tag-length: tag-length
+                             auth-data: auth-data))
+
+(define (decrypt message tag key iv #!optional auth-data)
+  (string-decrypt-and-verify aes-256-gcm message tag key iv
+                             auth-data: auth-data))
+
+(define *root-key-file* "root-key")
+(define (generate-root-key) (generate-key))
+(define (save-root-key)
+  (with-output-to-file *root-key-file* (lambda () (write (blob->hexstring/uppercase (generate-root-key))))))
+(define (load-root-key)
+  (hexstring->blob (with-input-from-file *root-key-file* read)))
+
+(define *root-key-iv* (hexstring->blob "1EBBCF6B50C68593C559EF93"))
+(define (ensure-root-key)
+  (when (not (file-exists? *root-key-file*))
+    (save-root-key))
+  (load-root-key))
+(define *root-key-key* (ensure-root-key))
+
+(define (get-user-key-and-iv conn user-id)
+  (row-alist (query conn "select auth_user_id, key_key, key_iv from users where user_id=$1;" user-id)))
+
+(define (get-decrypted-user-key-and-iv conn user-id)
+  (let* ((auth-user-id-and-user-key-and-iv (get-user-key-and-iv conn user-id))
+         (raw-user-key-and-tag (alist-ref 'key_key auth-user-id-and-user-key-and-iv))
+         (raw-user-key (hexstring->blob (string-drop-right raw-user-key-and-tag (* tag-length 2))))
+         (raw-user-tag (hexstring->blob (string-take-right raw-user-key-and-tag (* tag-length 2))))
+         (user-key (decrypt (blob->string raw-user-key) (blob->string raw-user-tag) *root-key-key* *root-key-iv*
+                            (string->blob (number->string (alist-ref 'auth_user_id auth-user-id-and-user-key-and-iv)))))
+         (user-iv (alist-ref 'key_iv auth-user-id-and-user-key-and-iv))
+         (auth-user-id (alist-ref 'auth_user_id auth-user-id-and-user-key-and-iv)))
+    (values (hexstring->blob user-key) (hexstring->blob user-iv) auth-user-id)))
+
+(define (user-encrypt message user-key user-iv user-id)
+  (encrypt message user-key user-iv (string->blob (->string user-id))))
+
+(define (user-encrypt-for-db message user-key user-iv user-id)
+  (receive (message tag)
+      (user-encrypt message user-key user-iv user-id)
+    (string-append (blob->hexstring/uppercase (string->blob message))
+                   (blob->hexstring/uppercase (string->blob tag)))))
+
+(define (user-decrypt message tag user-key user-iv user-id)
+  (decrypt message tag user-key user-iv (string->blob (->string user-id))))
+
+(define (user-decrypt-from-db message-and-tag user-key user-iv user-id)
+  (let ((raw-message (hexstring->blob (string-drop-right message-and-tag (* tag-length 2))))
+        (raw-tag (hexstring->blob (string-take-right message-and-tag (* tag-length 2)))))
+    (user-decrypt (blob->string raw-message) (blob->string raw-tag) user-key user-iv user-id)))
+
+(define (create-user conn auth-user-id email username)
+  (let ((user-key (blob->hexstring/uppercase (generate-key)))
+        (user-iv (blob->hexstring/uppercase (generate-iv))))
+    (receive (enc-user-key tag)
+        (encrypt user-key *root-key-key* *root-key-iv* (string->blob (number->string auth-user-id)))
+      (let ((user-id
+             (value-at
+              (query conn
+                     "insert into users(auth_user_id, email, username, key_key, key_iv) values ($1, $2, $3, $4, $5)
+returning users.user_id;"
+                     auth-user-id email username
+                     (string-append (blob->hexstring/uppercase (string->blob enc-user-key))
+                                    (blob->hexstring/uppercase (string->blob tag)))
+                     user-iv))))
+        user-id))))
+
+(define (delete-user conn user-id)
+  (query conn "delete from users where user_id=$1;" user-id))
+
+;; We also encrypt the ssh pub key not to hide it but to make it
+;; more difficult for someone to tamper with it which could allow
+;; an attacker to poison an instance with an ssh key that they have
+;; access to
+(define (create-instance conn user-id ssh-key-priv ssh-key-pub restic-password)
+  (receive (user-key user-iv auth-user-id)
+      (get-decrypted-user-key-and-iv conn user-id)
+    (let ((instance-id
+           (value-at
+            (query conn
+                   "insert into instances(user_id, ssh_key_priv_enc, ssh_key_pub_enc, restic_password_enc) values ($1, $2, $3, $4) returning instances.instance_id;"
+                   user-id
+                   (user-encrypt-for-db ssh-key-priv user-key user-iv user-id)
+                   (user-encrypt-for-db ssh-key-pub user-key user-iv user-id)
+                   (user-encrypt-for-db restic-password user-key user-iv user-id)))))
+      (query conn "insert into user_service_configs(user_id, instance_id) values ($1, $2);" user-id instance-id)
+      (query conn "insert into user_selected_apps(user_id, instance_id) values ($1, $2);" user-id instance-id)
+      (query conn "insert into user_app_configs(user_id, instance_id) values ($1, $2);" user-id instance-id)
+      (query conn "insert into user_terraform_state(user_id, instance_id) values ($1, $2);" user-id instance-id)
+      instance-id)))
+
+(define (get-instance-ssh-priv-key conn user-id instance-id)
+  (receive (user-key user-iv auth-user-id)
+      (get-decrypted-user-key-and-iv conn user-id)
+    (user-decrypt-from-db
+     (value-at (query conn "select ssh_key_priv_enc from instances where user_id=$1 and instance_id=$2;"
+                      user-id instance-id))
+     user-key user-iv user-id)))
+
+(define (get-instance-ssh-pub-key conn user-id instance-id)
+  (receive (user-key user-iv auth-user-id)
+      (get-decrypted-user-key-and-iv conn user-id)
+    (user-decrypt-from-db
+     (value-at (query conn "select ssh_key_pub_enc from instances where user_id=$1 and instance_id=$2;"
+                      user-id instance-id))
+     user-key user-iv user-id)))
+
+(define (update-instance-ssh-pub-key conn user-id instance-id ssh-pub-key)
+  (receive (user-key user-iv auth-user-id)
+      (get-decrypted-user-key-and-iv conn user-id)
+    (query conn "update instances set ssh_key_pub_enc=$3 where user_id=$1 and instance_id=$2;"
+           user-id instance-id
+           (user-encrypt-for-db ssh-pub-key user-key user-iv user-id))))
+
+(define (get-instance-restic-password conn user-id instance-id)
+  (receive (user-key user-iv auth-user-id)
+      (get-decrypted-user-key-and-iv conn user-id)
+    (user-decrypt-from-db
+     (value-at (query conn "select restic_password_enc from instances where user_id=$1 and instance_id=$2;"
+                      user-id instance-id))
+     user-key user-iv user-id)))
+
+(define (get-user-instances conn user-id)
+  (column-values (query conn "select instance_id from instances where user_id=$1;" user-id)))
+
+(define *user-service-configs-column-map*
+  '((cloudflare-api-token . ("cloudflare_api_token_enc" #t))
+    (cloudflare-account-id . ("cloudflare_account_id_enc" #t))
+    (cloudflare-zone-id . ("cloudflare_zone_id_enc" #t))
+    (digitalocean-api-token . ("digitalocean_api_token_enc" #t))
+    (digitalocean-region . ("digitalocean_region" #f))
+    (digitalocean-size . ("digitalocean_size" #f))
+    (backblaze-application-key . ("backblaze_application_key_enc" #t))
+    (backblaze-key-id . ("backblaze_key_id_enc" #t))
+    (backblaze-bucket-url . ("backblaze_bucket_url_enc" #t))))
+
+(define *user-service-configs-reverse-column-map*
+  (map (lambda (config)
+         `(,(string->symbol (cadr config)) . (,(car config) ,(caddr config))))
+       *user-service-configs-column-map*))
+
+(define (update-user-service-config conn user-id instance-id update-alist)
+  (let ((valid-keys (map car *user-service-configs-column-map*)))
+    (for-each (lambda (update)
+                (if (not (memq (car update) valid-keys))
+                    (error (string-append "Not a valid update key: " (->string (car update))))))
+              update-alist))
+  (receive (user-key user-iv auth-user-id)
+      (get-decrypted-user-key-and-iv conn user-id)
+    (query* conn
+            (string-append
+             "update user_service_configs set "
+             (string-intersperse
+              (map-in-order (lambda (update i)
+                              (conc (car (alist-ref (car update) *user-service-configs-column-map*))
+                                    "=$" i))
+                            update-alist
+                            (iota (length update-alist) 3))
+              ", ")
+             " where user_id=$1 and instance_id=$2;")
+            `(,user-id
+              ,instance-id
+              ,@(map-in-order (lambda (update)
+                                (if (cadr (alist-ref (car update) *user-service-configs-column-map*))
+                                    (user-encrypt-for-db (cdr update) user-key user-iv user-id)
+                                    (cdr update)))
+                              update-alist)))))
+
+(define (get-user-service-config conn user-id instance-id)
+  (receive (user-key user-iv auth-user-id)
+      (get-decrypted-user-key-and-iv conn user-id)
+    (let ((res (row-alist
+                (query conn
+                       (string-append
+                        "select "
+                        (string-intersperse
+                         (map-in-order (lambda (update)
+                                         (car (alist-ref (car update) *user-service-configs-column-map*)))
+                                       *user-service-configs-column-map*)
+                         ", ")
+                        " from user_service_configs where user_id=$1 and instance_id=$2;")
+                       user-id instance-id))))
+      (map (lambda (item)
+             (let* ((key (car item))
+                    (value (cdr item))
+                    (config (alist-ref key *user-service-configs-reverse-column-map*)))
+               `(,(car config) . ,(if (sql-null? value)
+                                      ""
+                                      (if (cadr config)
+                                          (user-decrypt-from-db value user-key user-iv user-id)
+                                          value)))))
+           res))))
+
+(define *user-selected-apps-column-map*
+  '((wg-easy . "wg_easy_version")
+    (nextcloud . "nextcloud_version")
+    (ghost . "ghost_version")
+    (nassella . "nassella_version")
+    (log-viewer . "log_viewer_version")))
+
+(define *user-selected-apps-reverse-column-map*
+  (map (lambda (config)
+         `(,(string->symbol (cdr config)) . ,(car config)))
+       *user-selected-apps-column-map*))
+
+(define (update-user-selected-apps conn user-id instance-id app-alist)
+  (let ((valid-keys (map car *user-selected-apps-column-map*)))
+    (for-each (lambda (app)
+                (if (not (memq (car app) valid-keys))
+                    (error (string-append "Not a valid app key: " (->string (car app))))))
+              app-alist))
+  (query* conn
+          (string-append
+           "update user_selected_apps set "
+           (string-intersperse
+            (map-in-order (lambda (app i)
+                            (conc (alist-ref (car app) *user-selected-apps-column-map*)
+                                  "=$" i))
+                          app-alist
+                          (iota (length app-alist) 3))
+            ", ")
+           " where user_id=$1 and instance_id=$2;")
+          `(,user-id
+            ,instance-id
+            ,@(map-in-order cdr app-alist))))
+
+(define (get-user-selected-apps conn user-id instance-id)
+  (let ((res (row-alist
+              (query conn
+                     (string-append
+                      "select "
+                      (string-intersperse
+                       (map-in-order cdr *user-selected-apps-column-map*)
+                       ", ")
+                      " from user_selected_apps where user_id=$1 and instance_id=$2;")
+                     user-id instance-id))))
+    (map (lambda (item)
+           (let* ((key (car item))
+                  (value (cdr item))
+                  (config (alist-ref key *user-selected-apps-reverse-column-map*)))
+             `(,config . ,(if (sql-null? value)
+                                    #f
+                                    value))))
+         res)))
+
+(define (update-user-app-config conn user-id instance-id config)
+  (receive (user-key user-iv auth-user-id)
+      (get-decrypted-user-key-and-iv conn user-id)
+    (query conn
+           "update user_app_configs set config_enc=$1 where user_id=$2 and instance_id=$3;"
+           (user-encrypt-for-db
+            (with-output-to-string
+              (lambda ()
+                (write config)))
+            user-key user-iv user-id)
+           user-id instance-id)))
+
+(define (update-root-domain conn user-id instance-id root-domain)
+  (query conn
+         "update user_app_configs set root_domain=$1 where user_id=$2 and instance_id=$3;"
+         root-domain
+         user-id
+         instance-id))
+
+(define (get-user-app-config conn user-id instance-id)
+  (receive (user-key user-iv auth-user-id)
+      (get-decrypted-user-key-and-iv conn user-id)
+    (let ((res (row-alist (query conn
+                                 "select root_domain, config_enc from user_app_configs where user_id=$1 and instance_id=$2;"
+                                 user-id instance-id))))
+      `((root-domain . ,(if (sql-null? (alist-ref 'root_domain res))
+                            #f
+                            (alist-ref 'root_domain res)))
+        (config . ,(if (sql-null? (alist-ref 'config_enc res))
+                       '()
+                       (with-input-from-string
+                           (user-decrypt-from-db (alist-ref 'config_enc res) user-key user-iv user-id)
+                         read)))))))
+
+(define *deployment-status*
+  '((queued . "queued")
+    (in-progress . "in-progress")
+    (complete . "complete")
+    (failed . "failed")))
+(define (create-deployment conn user-id instance-id)
+  (value-at
+   (query conn
+          "insert into deployments(user_id, instance_id, started) values($1, $2, now()) returning deployments.id;"
+          user-id instance-id)))
+(define (update-deployment-in-progress conn deployment-id pid)
+  (query conn
+         "update deployments set status=$1, pid=$2 where id=$3;"
+         (alist-ref 'in-progress *deployment-status*) pid deployment-id))
+
+(define (update-deployment-status conn user-id deployment-id status log)
+  (receive (user-key user-iv auth-user-id)
+      (get-decrypted-user-key-and-iv conn user-id)
+    (query conn "update deployments set status=$1, log_enc=$2, finished=now() where id=$3;"
+           (alist-ref status *deployment-status*)
+           (user-encrypt-for-db log user-key user-iv user-id)
+           deployment-id)))
+
+(define (get-deployment-status conn deployment-id)
+  (value-at (query conn "select status from deployments where id=$1;" deployment-id)))
+
+(define (get-most-recent-deployment-status conn user-id instance-id)
+  (value-at (query conn "select status from deployments where user_id=$1 and instance_id=$2 order by id DESC limit 1;" user-id instance-id)))
+
+(define *deployments-column-map*
+  '((generate-configs . "generate_configs")
+    (custom-image . "terraform_custom_image")
+    (machine-create . "terraform_machine_create")
+    (machine-destroy . "terraform_machine_destroy")
+    (status . "status")
+    (id . "id")
+    (instance-id . "instance_id")))
+
+(define *deployments-reverse-column-map*
+  (map (lambda (config)
+         `(,(string->symbol (cdr config)) . ,(car config)))
+       *deployments-column-map*))
+
+(define (update-deployment-progress conn deployment-id progress-alist)
+  (let ((valid-keys (map car *deployments-column-map*)))
+    (for-each (lambda (progress)
+                (if (not (memq (car progress) valid-keys))
+                    (error (string-append "Not a valid progress key: " (->string (car progress))))))
+              progress-alist))
+  (query* conn
+          (string-append
+           "update deployments set "
+           (string-intersperse
+            (map-in-order (lambda (progress i)
+                            (conc (alist-ref (car progress) *deployments-column-map*)
+                                  "=$" i))
+                          progress-alist
+                          (iota (length progress-alist) 2))
+            ", ")
+           " where id=$1;")
+          (cons deployment-id
+                (map-in-order (lambda (progress) (alist-ref (cdr progress) *deployment-status*)) progress-alist))))
+
+(define (get-deployment-progress conn deployment-id)
+  (let ((res (row-alist
+              (query conn
+                     (string-append
+                      "select "
+                      (string-intersperse
+                       (map-in-order cdr *deployments-column-map*)
+                       ", ")
+                      " from deployments where id=$1;")
+                     deployment-id))))
+    (map (lambda (item)
+           (let* ((key (car item))
+                  (value (cdr item))
+                  (config (alist-ref key *deployments-reverse-column-map*)))
+             `(,config . ,(if (sql-null? value)
+                                    #f
+                                    (string->symbol value)))))
+         res)))
+
+(define (get-most-recent-deployment-progress conn user-id instance-id)
+  (let ((res (row-alist
+              (query conn
+                     (string-append
+                      "select "
+                      (string-intersperse
+                       (map-in-order cdr *deployments-column-map*)
+                       ", ")
+                      " from deployments where user_id=$1 and instance_id=$2 order by id DESC limit 1;")
+                     user-id instance-id))))
+    (map (lambda (item)
+           (let* ((key (car item))
+                  (value (cdr item))
+                  (config (alist-ref key *deployments-reverse-column-map*)))
+             `(,config . ,(if (sql-null? value)
+                                    #f
+                                    (if (string? value)
+                                        (string->symbol value)
+                                        value)))))
+         res)))
+
+(define (get-dashboard conn user-id)
+  (receive (user-key user-iv auth-user-id)
+      (get-decrypted-user-key-and-iv conn user-id)
+    (let ((res
+           (query conn
+                  (string-append
+                   "select "
+                   (string-intersperse
+                    (map-in-order (lambda (d) (string-append "d." (cdr d))) *deployments-column-map*)
+                    ", ")
+                   ", uac.root_domain, uac.config_enc, uac.instance_id, "
+                   "usa.wg_easy_version, usa.nextcloud_version, usa.log_viewer_version, usa.ghost_version, usa.nassella_version "
+                   "from instances as i "
+                   "join (select instance_id, max(id) as id from deployments group by instance_id) d2 "
+                   "on d2.instance_id = i.instance_id "
+                   "join deployments d on d.id = d2.id "
+                   "join user_app_configs uac on uac.user_id = d.user_id and uac.instance_id = d.instance_id "
+                   "join user_selected_apps usa on usa.instance_id = uac.instance_id "
+                   "where i.user_id=$1;")
+                  user-id)))
+      (map
+       (lambda (row-num)
+         (map (lambda (item)
+                (let* ((key (car item))
+                       (value (cdr item))
+                       (config (alist-ref key `((root_domain . root-domain)
+                                                (config_enc . config)
+                                                (instance_id . instance-id)
+                                                (wg_easy_version . wg-easy)
+                                                (nextcloud_version . nextcloud)
+                                                (ghost_version . ghost)
+                                                (nassella_version . nassella)
+                                                (log_viewer_version . log-viewer)
+                                                ,@*deployments-reverse-column-map*))))
+                  `(,config . ,(if (sql-null? value)
+                                   #f
+                                   (if (and (string? value) (member config *deployments-column-map*))
+                                       (string->symbol value)
+                                       (if (eq? key 'config_enc)
+                                           (with-input-from-string
+                                               (user-decrypt-from-db value user-key user-iv user-id)
+                                             read)
+                                           value))))))
+              (row-alist res row-num)))
+       (iota (row-count res))))))
+
+(define (update-user-terraform-state conn user-id instance-id state backup)
+  (receive (user-key user-iv auth-user-id)
+      (get-decrypted-user-key-and-iv conn user-id)
+    (query conn
+           "update user_terraform_state set state_enc=$1, state_backup_enc=$2 where user_id=$3 and instance_id=$4;"
+           (user-encrypt-for-db state user-key user-iv user-id)
+           (user-encrypt-for-db backup user-key user-iv user-id)
+           user-id
+           instance-id)))
+
+(define (get-user-terraform-state conn user-id instance-id)
+  (receive (user-key user-iv auth-user-id)
+      (get-decrypted-user-key-and-iv conn user-id)
+    (let ((res (row-alist (query conn
+                                 "select state_enc, state_backup_enc from user_terraform_state where user_id=$1 and instance_id=$2;"
+                                 user-id instance-id))))
+      `((state . ,(if (or (sql-null? (alist-ref 'config_enc res))
+                          (sql-null? (alist-ref 'state_enc res)))
+                       ""
+                       (user-decrypt-from-db (alist-ref 'state_enc res) user-key user-iv user-id)))
+        (backup . ,(if (or (sql-null? (alist-ref 'config_enc res))
+                           (sql-null? (alist-ref 'state_backup_enc res)))
+                       ""
+                       (user-decrypt-from-db (alist-ref 'state_backup_enc res) user-key user-iv user-id)))))))
+
+(debug-log (current-error-port))
+
+(define (db-init)
+  (with-db/transaction
+   (lambda (db)
+     (if (value-at (query db "SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = 'public' AND tablename  = 'users');"))
+         (begin
+           (log-to (debug-log) "database already initialized")
+           #t)
+         (begin
+           (log-to (debug-log) "tables not found in db. Creating...")
+           (for-each
+            (lambda (statement)
+              (query db (conc statement ";")))
+            (string-split (with-input-from-file "db-init.sql" read-string) ";"))
+           (log-to (debug-log) "table creation finished")
+           (log-to (debug-log) "creating test user")
+           (create-user db 1 "me@example.com" "username")
+           (log-to (debug-log) "test user creation finished"))))))
+
+(define (db-clean)
+  (with-db/transaction
+   (lambda (db)
+     (if (value-at (query db "SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = 'public' AND tablename  = 'users');"))
+         (begin
+           (log-to (debug-log) "cleaning database")
+           (for-each
+            (lambda (statement)
+              (query db (conc statement ";")))
+            (string-split (with-input-from-file "db-clean.sql" read-string) ";"))
+           (log-to (debug-log) "database cleaning complete"))
+         (begin
+           (log-to (debug-log) "tables not found, not cleaning")
+           #t)))))
+
+;; (with-db/transaction (lambda (db) (get-user-deployments db 1)))
+;; (with-db/transaction (lambda (db) (get-most-recent-deployment-progress db 7)))
+;; (with-db/transaction (lambda (db) (get-deployment-progress db 14)))
+;; (with-db/transaction (lambda (db) (update-deployment-progress db 14 '((generate-configs . complete) (custom-image . in-progress) (machine-create . queued)))))
+;; (with-db/transaction
+;;  (lambda (db)
+;;    (update-user-terraform-state db 1 22
+;;                                 (with-input-from-file "src/deploy-7/terraform.tfstate" read-string)
+;;                                 (with-input-from-file "src/deploy-7/terraform.tfstate.backup" read-string))))
+;; (with-db/transaction (lambda (db) (get-user-terraform-state db 7)))
+
+;; (with-db/transaction (lambda (db) (create-deployment db 7)))
+;; (with-db/transaction (lambda (db) (get-deployment-status db 1)))
+;; (with-db/transaction (lambda (db) (update-deployment-in-progress db 1 123)))
+;; (with-db/transaction (lambda (db) (update-deployment-status db 1 'complete)))
+;; (with-db/transaction (lambda (db) (get-most-recent-deployment-status db 7)))
+
+;; (with-db/transaction (lambda (db) (create-user db 1 "t@thintz.com" "thecombjelly")))
+;; (with-db/transaction (lambda (db) (create-instance db 2)))
+
+;; (let ((user-id 7))
+;;   (with-db/transaction
+;;    (lambda (db)
+;;      (receive (user-key user-iv auth-user-id)
+;;          (get-decrypted-user-key-and-iv db user-id)
+;;        (receive (message tag)
+;;            (user-encrypt "hello!" user-key user-iv user-id)
+;;          (user-decrypt message tag user-key user-iv user-id))))))
+
+;; (with-db/transaction
+;;  (lambda (db)
+;;    (update-user-service-config db 7 '((cloudflare-api-token . ")
+;;                                       (digitalocean-region . "sfo3")))))
+
+;; (with-db/transaction
+;;  (lambda (db)
+;;    (get-user-service-config db 7)))
+
+;; (with-db/transaction
+;;  (lambda (db)
+;;    (update-user-selected-apps db 7 '((wg-easy . "0.1")
+;;                                      (nextcloud . "1.3")))))
+
+;; (with-db/transaction
+;;  (lambda (db)
+;;    (get-user-selected-apps db 7)))
+
+;; (with-db/transaction
+;;  (lambda (db)
+;;    (update-user-app-config db 7 "domain.com" '())))
+
+;; (with-db/transaction
+;;  (lambda (db)
+;;    (get-user-app-config db 7)))
+
+)

src/run.scm

@@ -0,0 +1,17 @@
+(include "nassella")
+(import (chicken process-context)
+        spiffy
+        schematra)
+
+(debug-log (current-error-port))
+
+(with-schematra-app
+ app
+ (lambda ()
+   (log-to (debug-log) "starting server")
+   (log-to (debug-log) "initializing db")
+   (if (member "--clean" (command-line-arguments) equal?)
+       (db-clean))
+   (db-init)
+   (log-to (debug-log) "db initialization complete")
+   (start-server)))

template.tfvars

@@ -1,15 +0,0 @@
-domain = ""
-subdomains = ["wg-easy"]
-
-server_type = "s-2vcpu-2gb"
-
-do_token = "" # token from "API" settings on DigitalOcean
-
-cloudflare_api_token = ""
-cloudflare_zone_id = ""
-cloudflare_account_id = ""
-
-cluster_name           = "mycluster"
-datacenter             = "sfo3"
-ssh_keys               = [""] # paste contents of id_rsa.pub
-flatcar_stable_version = "4230.2.1"