Compare commits

..

27 Commits

Author SHA1 Message Date
265a682b52 Working nassella running on nassella!! 2026-02-23 09:09:58 -08:00
35b2635b62 Allow cleaning the db. 2026-02-21 10:55:42 -08:00
9d5b8b9f6c Improving docker setup & terraform init. 2026-02-21 08:39:32 -08:00
b93933f4e9 Working dockerfile. 2026-02-20 10:58:57 -08:00
4338a3e891 Don't use root domain for instance. 2026-02-20 08:13:06 -08:00
8587ac0f2c testing gitea 2026-02-20 07:58:13 -08:00
83f68db2d7 wip instance "destroy" action 2026-02-09 08:30:39 -08:00
78f509d946 Fixing readme markup 2026-01-20 06:57:23 -08:00
db380666db Updating readme 2026-01-20 06:56:33 -08:00
c6d4e59867 Updating readme. 2026-01-20 06:39:51 -08:00
701a4fc55d Actually removing app dir. 2026-01-20 06:32:11 -08:00
d8b1f275dc Trying to remove app folder. 2026-01-20 06:31:43 -08:00
661c314ae4 Updating readme. 2026-01-20 06:25:21 -08:00
9bec27b991 Updating readme 2026-01-20 05:56:31 -08:00
757e244688 Only copy docker configs for selected apps. 2026-01-18 08:20:56 -08:00
b285ad3980 Backblaze, db bugfixes and connection testing. 2026-01-18 07:50:31 -08:00
103beca17d Properly gather, save, and generate Ghost config. 2025-12-08 11:32:30 -08:00
c23eef3403 Improving nextcloud env vars 2025-12-08 10:11:29 -08:00
73d6d28c69 Improving ghost & nextcloud compose configs. 2025-12-08 06:58:02 -08:00
179373f04a Adding ghost and fixing compose .env setup. 2025-12-07 10:38:36 -08:00
284b4c37f4 Refactor to support multiple instances. 2025-11-30 20:13:51 -08:00
e372f2157b Adding tests. 2025-11-30 11:36:19 -08:00
5ca856b1ff Initial sketch of dashboard. 2025-11-15 12:34:29 -08:00
908938dd41 Improving deployment progress status handling. 2025-11-12 13:22:25 -08:00
b781ddb5d7 Properly manage deploys via unique folders and db. 2025-11-12 07:31:23 -08:00
5d256e5cf8 Improving deployment process. 2025-11-12 05:42:25 -08:00
fb9c3f8daf Postgres db integration. 2025-11-10 13:13:59 -08:00
33 changed files with 6075 additions and 349 deletions

10
.gitignore vendored
View File

@@ -13,7 +13,13 @@ config/apps.config
config/production.tfvars
config/ssh-keys
# custom chicken eggs for dockerfile
src/scss
src/html-widgets
src/schematra-session
# generated files
all-apps/.env
all-apps/lb/Caddyfile
all-apps/nextcloud/nextcloud.env
all-apps/nextcloud/nextcloud_admin_user
@@ -26,4 +32,6 @@ generated.tfvars
restic-env
restic-password
ignition.json
app
app
nassella-latest.tar
src/deploy*/*

View File

@@ -9,10 +9,21 @@ app/.dirstamp: all-apps/app.service all-apps/docker-compose.yaml \
$(wildcard all-apps/lb/*) \
$(wildcard all-apps/nextcloud/*) \
$(wildcard all-apps/wg-easy/*) \
$(wildcard all-apps/ghost/*) \
$(wildcard all-apps/nassella/*) \
$(wildcard all-apps/dozzle/*)
rm -Rf app/
cp -a all-apps app && touch $@
mkdir app/
cp all-apps/app.service app/
cp all-apps/docker-compose.yaml app/
cp all-apps/.env app/
./copy-apps.sh $(apps_config) && touch $@
# compose .env files
# (compose only supports one .env file at the root by default)
all-apps/.env: all-apps/ghost/.compose-env
find all-apps/ -name ".compose-env" -exec cat > all-apps/.env {} +
# Caddy / lb
all-apps/lb/Caddyfile: $(apps_config) make-caddyfile.sh
@@ -32,9 +43,13 @@ all-apps/nextcloud/postgres_password: $(apps_config)
bash -c 'source ./$(apps_config); printf "%s\n" "$$NEXTCLOUD_POSTGRES_PASSWORD" > $@'
all-apps/nextcloud/redis_password: $(apps_config)
bash -c 'source ./$(apps_config); printf "%s\n" "$$NEXTCLOUD_REDIS_PASSWORD" > $@'
all-apps/nextcloud/nextcloud.env: $(apps_config) make-nextcloud-env.sh
all-apps/nextcloud/nextcloud.env: $(apps_config) all-apps/nextcloud/nextcloud.env.tmpl make-nextcloud-env.sh
./make-nextcloud-env.sh $(apps_config)
# Ghost
all-apps/ghost/.compose-env: $(apps_config) all-apps/ghost/.compose.env.tmpl make-ghost-env.sh
./make-ghost-env.sh $(apps_config)
# Backups / Restic / Backblaze
restic-env: $(apps_config) make-restic-generated.sh
./make-restic-generated.sh $(apps_config) > restic-env
@@ -50,8 +65,14 @@ all-apps/nextcloud/postgres_user \
all-apps/nextcloud/postgres_password \
all-apps/nextcloud/redis_password \
all-apps/nextcloud/nextcloud.env \
all-apps/nassella/postgres_db \
all-apps/nassella/postgres_user \
all-apps/nassella/postgres_password \
all-apps/nassella/nassella.env \
all-apps/ghost/.compose-env \
restic-env \
restic-password \
all-apps/.env \
$(config_dir)ssh-keys
cat cl.yaml | docker run --rm --volume $(config_dir)/ssh-keys:/pwd/ssh-keys --volume ${PWD}:/pwd --workdir /pwd -i quay.io/coreos/butane:latest -d /pwd > ignition.json
@@ -59,13 +80,21 @@ generated.tfvars: $(apps_config) make-generated.sh
./make-generated.sh $(apps_config) > generated.tfvars
plan: ignition.json $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars
terraform init
bash -c "terraform plan -var-file=<(cat $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars)"
apply: ignition.json $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars
bash -c "terraform apply -var-file=<(cat $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars)"
.PHONY: announce-start
announce-start:
echo "NASSELLA_CONFIG: start"
apply: announce-start ignition.json $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars
echo "NASSELLA_CONFIG: end"
terraform init
bash -c "terraform apply -auto-approve -input=false -var-file=<(cat $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars)"
destroy: ignition.json $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars
bash -c "terraform destroy -var-file=<(cat $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars)"
terraform init
bash -c "terraform destroy -auto-approve -input=false -var-file=<(cat $(config_dir)$(TERRAFORM_ENV).tfvars generated.tfvars)"
.PHONY: restic-init
restic-init: $(apps_config) restic-password
@@ -76,6 +105,13 @@ restic-init: $(apps_config) restic-password
restic-snapshots: $(apps_config) restic-password
./restic-snapshots.sh $(apps_config)
.PHONY: archive
archive:
tar -cf nassella-latest.tar all-apps cl.yaml init-restic.sh main.tf make-caddyfile.sh Makefile \
make-generated.sh make-nextcloud-env.sh make-ghost-env.sh make-restic-generated.sh make-restic-password.sh restic-snapshots.sh copy-apps.sh \
.terraform.lock.hcl
cp nassella-latest.tar src/
## to help me remember the command to run to test the config locally
testlocalhost:
curl -k --resolve localhost:443:146.190.12.129 https://localhost

View File

@@ -1,8 +1,110 @@
* Setup
* Development Notes
- Currently, only tested on Linux (Debian). Everything should
theoretically also work on Mac but some commands may need updating
and it has not been tested, but multi-platform support, except for
Windows, was kept in mind during development.
* Project Goal
To make deploying, managing, and updating self-hosted app instances
easy.
* Short Note On Generative AI
This project does NOT use any code or documentation generated by an
LLM and no code or text generated by an LLM is acceptable as a
contribution.
* Supported Services
Currently, only a limited set of external services can be used for
hosting, DNS, and backups. This can be easily extended later with
modifications to the Terraform config. As of now, you will need to
have accounts with these providers:
- DigitalOcean
- Cloudflare (and have DNS available for a domain)
- Backblaze B2
* Architecture
The software stack is composed of a "base" command line interface that
can be used to deploy and manage a single instance with a
multi-instance, multi-user webapp that invokes the "base" as
needed. The "base" can be run separately from the webapp. The webapp
automatically generates the configs the "base" needs to run.
The Makefile at the root of this source tree is the point of interface
for everything and all commands are run via make.
** "Base" Terraform Layer
The project is designed so that if you want to just manage a single
instance without the complexity of running a webapp you can easily do
so. This is both so that individual users can take advantage of this
but also so that when developing the Terraform and Docker Compose
setup it can be done and tested without needing to deal with the web
app as well.
The "base" layer is made up of the following: Flatcar Linux, Docker
Compose, Terraform, and a Makefile with a set of BASH scripts.
*** Flatcar Linux
The deployed instance runs on Flatcar Linux. Flatcar is a "read only"
Linux distribution designed to only run containers and nothing
else. Flatcar is used because it provides a high-level of security and
the OS itself auto-updates on a two-week schedule. Also, being "read
only" it is much more difficult for an external attacker to attack and
much harder for a user that does not know what they are doing to "mess
up".
*** Docker Compose
Each individual supported web app (like NextCloud, Ghost, etc) runs
via Docker and is configured via Docker Compose. (The docker compose
files are all in the "all-apps" directory in this source tree).
The Flatcar Linux config contains a systemd unit (service file) that
runs "docker compose". The Makefile copies all selected apps' docker
compose files from all-apps/ to app/. The systemd unit runs all the
docker compose files in the app/ directory. (The app/ directory is
what actually gets copied to the Flatcar linux install, not the
all-apps/ directory.)
The docker compose setup is specific and needs further documentation
here (to cover things like the shared load-balancer network setup and
how persistent storage is handled).
*** Terraform
Terraform is used to actually manage the deployed instances. Currently
it is a static terraform config controlled only via terraform
variables (see config/production.tfvars.tmpl). The terraform commands
are run via the Makefile.
** Webapp
The webapp is used both to provide a more "user-friendly" interface
for setting up and managing instances as well as to provide a
multi-user and multi-instance service. Internally, to manage an
instance, the webapp generates the configs and invokes the same
commands used when running the "base" CLI version by itself.
* Setup "Base" CLI Terraform For Deploying Individual Instance
NOTE: some of this may be outdated. It has not been tested on its own
outside of running via the webapp for a bit. It does work when run via
the webapp and all of the services still need to be setup as
detailed (the data can be input via the webapp instead of only via the
config files).
** Dependencies
- [[https://developer.hashicorp.com/terraform/install][terraform]]
- [[https://www.docker.com/][docker]]
- bash
- GNU Make
** Services
@@ -90,12 +192,77 @@ This is used automated for "off-site" backups / snapshots.
- ~make restic-init~
* Deploy
** Deploy
- ~make apply~
* You're done!
** You're done!
It will take a few minutes to deploy the server, start it, and pull
all the docker images. But after that you should be able to visit your
site and the apps running on its subdomains!
* Setup Webapp
** Dependencies
- [[https://code.call-cc.org/][CHICKEN Scheme 5.3+]]
- docker
- GNU Make
The webapp is written in Lisp (CHICKEN Scheme) and connects to a
PostgreSQL database. It also depends on being able to run some docker
commands. It has only been tested on Linux. Running the commands on
other platforms may need work. A Makefile command is provided for
running the Postgres database via docker so Postgres is not a listed
as a direct dependency.
** CHICKEN Scheme Libraries
These will need to be installed via the ~chicken-install~ command.
~postgresql sql-null srfi-1 srfi-13 srfi-18 srfi-158 srfi-194 openssl crypto-tools sxml-transforms schematra schematra-body-parser schematra-session uri-common http-client medea intarweb~
** html-widgets
This is a CHICKEN Scheme library that also needs to be installed but
it is not available via the ~chicken-install~ repository as I wrote it
for this project and I have not published it externally yet. You can
get the project here: [[https://code.thintz.com/tjhintz/html-widgets][code.thintz.com/tjhintz/html-widgets]]
After downloading the project, you can install it by ~cd~ to the
directory it is in and then running ~chicken-install~.
** Architecture
The webapp was designed to be very simple. Pages and handlers are
based on the [[https://schematra.com/][Schematra]] CHICKEN Scheme library (Chiccup is NOT
used). Currently, there are no external dependencies other than
docker, which is used to run postgres in development as well as
generate ssh keys for users in production. There is no JavaScript or
styling libraries. The webapp is built as HTML pages with forms.
The entry point for the webapp is ~src/nassella.scm~.
*** HTML Widgets
The core of page markup is created with the html-widgets library which
provides the ~define-widget~ function. This would be similar to a
"component" in something like React. It allows defining properties of
a widget (component) as well as a name and default values. It also
handles style transforms, detailed in the next section.
*** Styling
All styling is done via ~style~ attributes on HTML elements. This is
handled internally by the html-widgets library. The html-widgets
library will extract all of the styles into a CSS stylesheet and
replace the ~style~ attributes with a corresponding, auto-generated,
CSS class name attribute. Nothing is needed to define or manage styles
other than to set them directly via an HTML ~style~ attribute.
Shared styling values, like colors, are defined in the *style-tokens*
global variable. It is a tree of style tokens (similar to
[[https://css-tricks.com/what-are-design-tokens/][Design Tokens]]). ~nassella.scm~ also includes some functions to make it
easy for widgets to fetch style values. The most common being ~$~
which allows getting a style token value based on a dotted symbol or a
symbol list ~($ 'color.primary.contrast)~ or ~($ '(color primary contrast)~

View File

@@ -20,3 +20,4 @@ services:
- lb
networks:
lb:

View File

@@ -0,0 +1,61 @@
# Use the below flags to enable the Analytics or ActivityPub containers as well
# COMPOSE_PROFILES=analytics,activitypub
# Ghost domain
# Custom public domain Ghost will run on
# GHOST_DOMAIN=www.nassella.cc
# Ghost Admin domain
# If you have Ghost Admin setup on a separate domain uncomment the line below and add the domain
# You also need to uncomment the corresponding block in your Caddyfile
# ADMIN_DOMAIN=
# Database settings
# All database settings must not be changed once the database is initialised
# GHOST_DATABASE_ROOT_PASSWORD=reallysecurerootpassword
# DATABASE_USER=optionalusername
# GHOST_DATABASE_PASSWORD=ghostpassword
# ActivityPub
# If you'd prefer to self-host ActivityPub yourself uncomment the line below
# ACTIVITYPUB_TARGET=activitypub:8080
# Tinybird configuration
# If you want to run Analytics, paste the output from `docker compose run --rm tinybird-login get-tokens` below
# TINYBIRD_API_URL=https://api.tinybird.co
# TINYBIRD_TRACKER_TOKEN=p.eyJxxxxx
# TINYBIRD_ADMIN_TOKEN=p.eyJxxxxx
# TINYBIRD_WORKSPACE_ID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
# Ghost configuration (https://ghost.org/docs/config/)
# SMTP Email (https://ghost.org/docs/config/#mail)
# Transactional email is required for logins, account creation (staff invites), password resets and other features
# This is not related to bulk mail / newsletter sending
mail__transport=SMTP
# mail__options__host=
# mail__options__port=
mail__options__secure=true
# mail__options__auth__user=
# mail__options__auth__pass=
# mail__from=""
# Advanced customizations
# Force Ghost version
# You should only do this if you need to pin a specific version
# The update commands won't work
# GHOST_VERSION=6-alpine
# Port Ghost should listen on
# You should only need to edit this if you want to host
# multiple sites on the same server
# GHOST_PORT=2368
# Data locations
# Location to store uploaded data
# GHOST_UPLOAD_LOCATION=./data/ghost
# Location for database data
# GHOST_MYSQL_DATA_LOCATION=./data/mysql
# NEWLINE REQUIRED AT END OF FILE

View File

@@ -0,0 +1,185 @@
services:
ghost:
image: ghost:${GHOST_VERSION:-6-alpine}
restart: always
# This is required to import current config when migrating
environment:
NODE_ENV: production
url: https://${GHOST_DOMAIN:?GHOST_DOMAIN environment variable is required}
admin__url: ${GHOST_ADMIN_DOMAIN:+https://${GHOST_ADMIN_DOMAIN}}
database__client: mysql
database__connection__host: ghost_db
database__connection__user: ${GHOST_DATABASE_USER:-ghost}
database__connection__password: ${GHOST_DATABASE_PASSWORD:?GHOST_DATABASE_PASSWORD environment variable is required}
database__connection__database: ghost
tinybird__tracker__endpoint: https://${GHOST_DOMAIN:?GHOST_DOMAIN environment variable is required}/.ghost/analytics/api/v1/page_hit
tinybird__adminToken: ${GHOST_TINYBIRD_ADMIN_TOKEN:-}
tinybird__workspaceId: ${GHOST_TINYBIRD_WORKSPACE_ID:-}
tinybird__tracker__datasource: analytics_events
tinybird__stats__endpoint: ${GHOST_TINYBIRD_API_URL:-https://api.tinybird.co}
volumes:
- /nassella/ghost/var-lib-ghost-content:/var/lib/ghost/content
depends_on:
ghost_db:
condition: service_healthy
ghost_tinybird-sync:
condition: service_completed_successfully
required: false
ghost_tinybird-deploy:
condition: service_completed_successfully
required: false
ghost_activitypub:
condition: service_started
required: false
networks:
- ghost_network
- lb
ghost_db:
image: mysql:8.0.44@sha256:f37951fc3753a6a22d6c7bf6978c5e5fefcf6f31814d98c582524f98eae52b21
restart: always
expose:
- "3306"
environment:
MYSQL_ROOT_PASSWORD: ${GHOST_DATABASE_ROOT_PASSWORD:?GHOST_DATABASE_ROOT_PASSWORD environment variable is required}
MYSQL_USER: ${GHOST_DATABASE_USER:-ghost}
MYSQL_PASSWORD: ${GHOST_DATABASE_PASSWORD:?GHOST_DATABASE_PASSWORD environment variable is required}
MYSQL_DATABASE: ghost
MYSQL_MULTIPLE_DATABASES: activitypub
volumes:
- /nassella/ghost/var-lib-mysql:/var/lib/mysql
- ./mysql-init:/docker-entrypoint-initdb.d
healthcheck:
test: mysqladmin ping -p$$GHOST_MYSQL_ROOT_PASSWORD -h 127.0.0.1
interval: 1s
start_period: 30s
start_interval: 10s
retries: 120
networks:
- ghost_network
ghost_traffic-analytics:
image: ghost/traffic-analytics:1.0.20@sha256:a72573d89457e778b00e9061422516d2d266d79a72a0fc02005ba6466e391859
restart: always
expose:
- "3000"
volumes:
- traffic_analytics_data:/data
environment:
NODE_ENV: production
PROXY_TARGET: ${GHOST_TINYBIRD_API_URL:-https://api.tinybird.co}/v0/events
SALT_STORE_TYPE: ${GHOST_SALT_STORE_TYPE:-file}
SALT_STORE_FILE_PATH: /data/salts.json
TINYBIRD_TRACKER_TOKEN: ${GHOST_TINYBIRD_TRACKER_TOKEN:-}
LOG_LEVEL: debug
profiles: [analytics]
networks:
- ghost_network
ghost_activitypub:
image: ghcr.io/tryghost/activitypub:1.1.0@sha256:39c212fe23603b182d68e67d555c6b9b04b1e57459dfc0bef26d6e4980eb04d1
restart: always
expose:
- "8080"
volumes:
- /nassella/ghost/var-lib-ghost-content:/opt/activitypub/content
environment:
# See https://github.com/TryGhost/ActivityPub/blob/main/docs/env-vars.md
NODE_ENV: production
MYSQL_HOST: ghost_db
MYSQL_USER: ${GHOST_DATABASE_USER:-ghost}
MYSQL_PASSWORD: ${GHOST_DATABASE_PASSWORD:?GHOST_DATABASE_PASSWORD environment variable is required}
MYSQL_DATABASE: activitypub
LOCAL_STORAGE_PATH: /opt/activitypub/content/images/activitypub
LOCAL_STORAGE_HOSTING_URL: https://${GHOST_DOMAIN}/content/images/activitypub
depends_on:
ghost_db:
condition: service_healthy
ghost_activitypub-migrate:
condition: service_completed_successfully
profiles: [activitypub]
networks:
- ghost_network
# Supporting Services
ghost_tinybird-login:
build:
context: ./tinybird
dockerfile: Dockerfile
working_dir: /home/tinybird
command: /usr/local/bin/tinybird-login
volumes:
- tinybird_home:/home/tinybird
- tinybird_files:/data/tinybird
profiles: [analytics]
networks:
- ghost_network
tty: false
restart: no
ghost_tinybird-sync:
# Do not alter this without updating the Ghost container as well
image: ghost:${GHOST_VERSION:-6-alpine}
command: >
sh -c "
if [ -d /var/lib/ghost/current/core/server/data/tinybird ]; then
rm -rf /data/tinybird/*;
cp -rf /var/lib/ghost/current/core/server/data/tinybird/* /data/tinybird/;
echo 'Tinybird files synced into shared volume.';
else
echo 'Tinybird source directory not found.';
fi
"
volumes:
- tinybird_files:/data/tinybird
depends_on:
ghost_tinybird-login:
condition: service_completed_successfully
networks:
- ghost_network
profiles: [analytics]
restart: no
ghost_tinybird-deploy:
build:
context: ./tinybird
dockerfile: Dockerfile
working_dir: /data/tinybird
command: >
sh -c "
tb-wrapper --cloud deploy
"
volumes:
- tinybird_home:/home/tinybird
- tinybird_files:/data/tinybird
depends_on:
ghost_tinybird-sync:
condition: service_completed_successfully
profiles: [analytics]
networks:
- ghost_network
tty: true
ghost_activitypub-migrate:
image: ghcr.io/tryghost/activitypub-migrations:1.1.0@sha256:b3ab20f55d66eb79090130ff91b57fe93f8a4254b446c2c7fa4507535f503662
environment:
MYSQL_DB: mysql://${GHOST_DATABASE_USER:-ghost}:${GHOST_DATABASE_PASSWORD:?GHOST_DATABASE_PASSWORD environment variable is required}@tcp(ghost_db:3306)/activitypub
networks:
- ghost_network
depends_on:
ghost_db:
condition: service_healthy
profiles: [activitypub]
restart: no
volumes:
tinybird_files:
tinybird_home:
traffic_analytics_data:
networks:
lb:
ghost_network:
driver: bridge
internal: true

View File

@@ -0,0 +1,51 @@
version: '3'
secrets:
nassella_postgres_db:
file: ./nassella/postgres_db
nassella_postgres_password:
file: ./nassella/postgres_password
nassella_postgres_user:
file: ./nassella/postgres_user
services:
nassella_db:
image: postgres:17.6-trixie
env_file:
- ./nassella/nassella.env
shm_size: 128mb
restart: always
volumes:
- /nassella/nassella/var-lib-postgresql-data:/var/lib/postgresql/data
networks:
- nassella_internal
healthcheck:
test: ["CMD-SHELL", "pg_isready -d `cat $$POSTGRES_DB_FILE` -U `cat $$POSTGRES_USER_FILE`"]
start_period: 15s
interval: 30s
retries: 3
timeout: 5s
secrets:
- nassella_postgres_db
- nassella_postgres_password
- nassella_postgres_user
nassella:
image: nassella/b0.0.1
depends_on:
nassella_db:
condition: service_healthy
env_file:
- ./nassella/nassella.env
secrets:
- nassella_postgres_db
- nassella_postgres_password
- nassella_postgres_user
networks:
- lb
- nassella_internal
restart: unless-stopped
networks:
lb:
nassella_internal:
driver: bridge
internal: true

View File

@@ -0,0 +1,4 @@
POSTGRES_HOST=nassella_db
POSTGRES_DB_FILE=/run/secrets/nassella_postgres_db
POSTGRES_USER_FILE=/run/secrets/nassella_postgres_user
POSTGRES_PASSWORD_FILE=/run/secrets/nassella_postgres_password

View File

@@ -0,0 +1 @@
nassella

View File

@@ -0,0 +1 @@
password

View File

@@ -0,0 +1 @@
nassella

View File

@@ -1,21 +1,17 @@
version: '3'
secrets:
nextcloud_admin_password:
file: ./nextcloud/nextcloud_admin_password
nextcloud_admin_user:
file: ./nextcloud/nextcloud_admin_user
postgres_db:
nextcloud_postgres_db:
file: ./nextcloud/postgres_db
postgres_password:
nextcloud_postgres_password:
file: ./nextcloud/postgres_password
postgres_user:
nextcloud_postgres_user:
file: ./nextcloud/postgres_user
redis_password:
nextcloud_redis_password:
file: ./nextcloud/redis_password
services:
db:
nextcloud_db:
image: postgres:17.6-trixie
env_file:
- ./nextcloud/nextcloud.env
@@ -23,12 +19,8 @@ services:
restart: always
volumes:
- /nassella/nextcloud/var-lib-postgresql-data:/var/lib/postgresql/data
environment:
- POSTGRES_DB_FILE=/run/secrets/postgres_db
- POSTGRES_USER_FILE=/run/secrets/postgres_user
- POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password
networks:
- internal
- nextcloud_internal
healthcheck:
test: ["CMD-SHELL", "pg_isready -d `cat $$POSTGRES_DB_FILE` -U `cat $$POSTGRES_USER_FILE`"]
start_period: 15s
@@ -36,58 +28,47 @@ services:
retries: 3
timeout: 5s
secrets:
- postgres_db
- postgres_password
- postgres_user
redis:
- nextcloud_postgres_db
- nextcloud_postgres_password
- nextcloud_postgres_user
nextcloud_redis:
image: redis:8.2.1-bookworm
env_file:
- ./nextcloud/nextcloud.env
command: bash -c 'redis-server --requirepass "$$(cat /run/secrets/redis_password)"'
command: bash -c 'redis-server --requirepass "$$(cat /run/secrets/nextcloud_redis_password)"'
secrets:
- redis_password
- nextcloud_redis_password
restart: always
healthcheck:
test: ["CMD-SHELL", "redis-cli --no-auth-warning -a \"$$(cat /run/secrets/redis_password)\" ping | grep PONG"]
test: ["CMD-SHELL", "redis-cli --no-auth-warning -a \"$$(cat /run/secrets/nextcloud_redis_password)\" ping | grep PONG"]
start_period: 10s
interval: 30s
retries: 3
timeout: 3s
networks:
- internal
- nextcloud_internal
nextcloud:
image: nextcloud:31.0.8-apache
environment:
- POSTGRES_HOST=db
- POSTGRES_DB=nextcloud
- POSTGRES_USER=nextcloud
- POSTGRES_PASSWORD=dbpassword
- NEXTCLOUD_ADMIN_PASSWORD=password
- NEXTCLOUD_ADMIN_USER=admin
- REDIS_HOST=redis
- NEXTCLOUD_TRUSTED_DOMAINS=nextcloud.nassella.cc # TODO generate this?
depends_on:
redis:
nextcloud_redis:
condition: service_healthy
db:
nextcloud_db:
condition: service_healthy
env_file:
- ./nextcloud/nextcloud.env
secrets:
- postgres_db
- postgres_password
- postgres_user
- nextcloud_admin_user
- nextcloud_admin_password
- redis_password
- nextcloud_postgres_db
- nextcloud_postgres_password
- nextcloud_postgres_user
- nextcloud_redis_password
networks:
- lb
- internal
- nextcloud_internal
volumes:
- /nassella/nextcloud/var-www-html:/var/www/html
restart: unless-stopped
networks:
lb:
internal:
nextcloud_internal:
driver: bridge
internal: true

View File

@@ -7,13 +7,10 @@ OVERWRITEPROTOCOL=https
TRUSTED_PROXIES=172.16.0.0/24 # trust the local lb
PHP_MEMORY_LIMIT=1G
PHP_UPLOAD_LIMIT=10G
POSTGRES_HOST=db
POSTGRES_DB_FILE=/run/secrets/postgres_db
POSTGRES_USER_FILE=/run/secrets/postgres_user
POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password
# admin user
NEXTCLOUD_ADMIN_PASSWORD_FILE=/run/secrets/nextcloud_admin_password
NEXTCLOUD_ADMIN_USER_FILE=/run/secrets/nextcloud_admin_user
POSTGRES_HOST=nextcloud_db
POSTGRES_DB_FILE=/run/secrets/nextcloud_postgres_db
POSTGRES_USER_FILE=/run/secrets/nextcloud_postgres_user
POSTGRES_PASSWORD_FILE=/run/secrets/nextcloud_postgres_password
# redis
REDIS_HOST=redis
REDIS_HOST_PASSWORD_FILE=/run/secrets/redis_password
REDIS_HOST=nextcloud_redis
REDIS_HOST_PASSWORD_FILE=/run/secrets/nextcloud_redis_password

View File

@@ -31,7 +31,7 @@ systemd:
[Service]
Type=oneshot
EnvironmentFile=/restic-env
ExecStart=/usr/bin/bash -c "docker run --rm --volume /nassella:/nassella --volume /restic-password:/restic-password -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} -i restic/restic:0.18.0 backup --verbose --repo s3:s3.us-west-004.backblazeb2.com/nassella-test-bucket --password-file /restic-password /nassella"
ExecStart=/usr/bin/bash -c "docker run --rm --volume /nassella:/nassella --volume /restic-password:/restic-password -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} -i restic/restic:0.18.0 backup --verbose --repo s3:${BACKBLAZE_BUCKET_URL} --password-file /restic-password /nassella"
ExecStopPost=systemctl start app.service
- name: restic-backup.timer

View File

@@ -1,11 +1,18 @@
ROOT_DOMAIN= # example.com :: the root of the domain that all apps should be subdomains of
APP_CONFIGS="nextcloud,nextcloud wg-easy,wg-easy" # apps to deploy and their corresponding sub-domain (app,sub-domain)
APP_CONFIGS="nextcloud,nextcloud wg-easy,wg-easy ghost,ghost" # apps to deploy and their corresponding sub-domain (app,sub-domain)
NEXTCLOUD_ADMIN_USER=admin # admin user for nextcloud, can be whatever you want
NEXTCLOUD_ADMIN_PASSWORD= # the password for the nextcloud admin user
NEXTCLOUD_POSTGRES_DB=nextcloud # recommended to leave as 'nextcloud'. The postgres db nextcloud uses
NEXTCLOUD_POSTGRES_USER=nextcloud # recommended to leave as 'nextcloud'. The postgres user nextcloud uses
NEXTCLOUD_POSTGRES_PASSWORD= # should be a secure, randomly generated, postgres compatible password, stored in the config so it isn't lost on re-deployment but otherwise unneeded
NEXTCLOUD_REDIS_PASSWORD= # should be a secure, randomly generated, redis compatible password, stored in the config so it isn't lost on re-deployment but otherwise unneeded
GHOST_DATABASE_ROOT_PASSWORD=
GHOST_DATABASE_PASSWORD=
SMTP_HOST=
SMTP_PORT=
SMTP_AUTH_USER=
SMTP_AUTH_PASSWORD=
SMTP_FROM=
BACKBLAZE_KEY_ID= # the key ID for a application key created on backblaze that has permissions for the bucket in BACKBLAZE_BUCKET_URL
BACKBLAZE_APPLICATION_KEY= # the application key for the application key created on backblaze
BACKBLAZE_BUCKET_URL= # the full URL for the backblaze bucket, found on the backblaze UI for the bucket

View File

@@ -9,4 +9,4 @@ cloudflare_account_id = "" # corresponding account ID for API token
cluster_name = "mycluster" # currently only used as the name of the machine on DigitalOcean
datacenter = "sfo3" # datacenter to deploy the droplet to
ssh_keys = [""] # unused
flatcar_stable_version = "4230.2.1" # (source <(curl -sSfL https://stable.release.flatcar-linux.net/amd64-usr/current/version.txt); echo "${FLATCAR_VERSION_ID}")
flatcar_stable_version = "4459.2.1" # (source <(curl -sSfL https://stable.release.flatcar-linux.net/amd64-usr/current/version.txt); echo "${FLATCAR_VERSION_ID}")

31
copy-apps.sh Executable file
View File

@@ -0,0 +1,31 @@
#!/bin/bash
# this script copys over the docker configs
# for in-use apps
# it depends on apps.config which should define:
# ROOT_DOMAIN - the root domain for all apps
# APP_CONFIGS - app-subdomain pairs, configured via a comma, like:
# app1,subdomain1 app2,subdomain2 app3,subdomain3
# full example:
# ROOT_DOMAIN=nassella.cc
# APP_CONFIGS="app1,subdomain1 app2,subdomain2 app3,subdomain3"
set -e
. $1 # source the apps.config file with then env vars
read -r -a APP_CONFIGS <<< "$APP_CONFIGS"
APP_CONFIGS+=('lb,root')
for config_string in ${APP_CONFIGS[@]}; do
IFS=','
read -r -a config <<< "$config_string"
app=${config[0]}
cp -a all-apps/$app app/
done

View File

@@ -104,7 +104,7 @@ resource "digitalocean_reserved_ip" "machine" {
resource "cloudflare_dns_record" "root" {
zone_id = var.cloudflare_zone_id
name = "@"
name = "_nassella-instance"
content = digitalocean_reserved_ip.machine.ip_address
type = "A"
proxied = false
@@ -115,7 +115,7 @@ resource "cloudflare_dns_record" "subdomains" {
for_each = toset(var.subdomains)
zone_id = var.cloudflare_zone_id
name = each.key
content = var.domain
content = "_nassella-instance.${var.domain}"
type = "CNAME"
proxied = false
ttl = 300
@@ -137,6 +137,7 @@ resource "digitalocean_droplet" "machine" {
size = var.server_type
ssh_keys = [digitalocean_ssh_key.first.fingerprint]
user_data = file("ignition.json")
graceful_shutdown = true
lifecycle {
create_before_destroy = true
}

View File

@@ -25,6 +25,8 @@ APP_CONFIGS+=('lb,root')
declare -A bodys
bodys["nextcloud"]=" reverse_proxy http://nextcloud:80"
bodys["wg-easy"]=" reverse_proxy http://wg-easy:80"
bodys["ghost"]=" reverse_proxy http://ghost:2368"
bodys["nassella"]=" reverse_proxy http://nassella:8080"
bodys["dozzle"]=$(cat <<EOF
basic_auth {
$HOST_ADMIN_USER $host_admin_password_encoded

32
make-ghost-env.sh Executable file
View File

@@ -0,0 +1,32 @@
#!/bin/bash
set -e
. $1 # source the apps.config file with then env vars
read -r -a APP_CONFIGS <<< "$APP_CONFIGS"
nextcloud_subdomain=
for config_string in ${APP_CONFIGS[@]}; do
IFS=','
read -r -a config <<< "$config_string"
app=${config[0]}
subdomain=${config[1]}
if [ "$app" = "ghost" ]; then
ghost_subdomain="$subdomain"
fi
done
# write compose env file
echo "GHOST_DOMAIN=\"$ghost_subdomain.$ROOT_DOMAIN\"" > all-apps/ghost/.compose-env
echo "GHOST_DATABASE_ROOT_PASSWORD=\"$GHOST_DATABASE_ROOT_PASSWORD\"" >> all-apps/ghost/.compose-env
echo "GHOST_DATABASE_PASSWORD=\"$GHOST_DATABASE_PASSWORD\"" >> all-apps/ghost/.compose-env
echo "mail__options__host=\"$SMTP_HOST\"" >> all-apps/ghost/.compose-env
echo "mail__options__port=\"$SMTP_PORT\"" >> all-apps/ghost/.compose-env
echo "mail__options__auth__user=\"$SMTP_AUTH_USER\"" >> all-apps/ghost/.compose-env
echo "mail__options__auth__pass=\"$SMTP_AUTH_PASSWORD\"" >> all-apps/ghost/.compose-env
echo "mail__from=\"$SMTP_FROM\"" >> all-apps/ghost/.compose-env
cat all-apps/ghost/.compose.env.tmpl >> all-apps/ghost/.compose-env

View File

@@ -20,5 +20,12 @@ for config_string in ${APP_CONFIGS[@]}; do
fi
done
# write container env file
echo "DOMAIN=\"$nextcloud_subdomain.$ROOT_DOMAIN\"" > all-apps/nextcloud/nextcloud.env
cat all-apps/nextcloud/nextcloud.env.tmpl >> all-apps/nextcloud/nextcloud.env
# write secrets
echo "$NEXTCLOUD_POSTGRES_DB" > all-apps/nextcloud/nextcloud_postgres_db
echo "$NEXTCLOUD_POSTGRES_USER" > all-apps/nextcloud/nextcloud_postgres_user
echo "$NEXTCLOUD_POSTGRES_PASSWORD" > all-apps/nextcloud/nextcloud_postgres_password
echo "$NEXTCLOUD_REDIS_PASSWORD" > all-apps/nextcloud/nextcloud_redis_password

View File

@@ -6,3 +6,4 @@ set -e
echo "AWS_ACCESS_KEY_ID=\"$BACKBLAZE_KEY_ID\""
echo "AWS_SECRET_ACCESS_KEY=\"$BACKBLAZE_APPLICATION_KEY\""
echo "BACKBLAZE_BUCKET_URL=\"$BACKBLAZE_BUCKET_URL\""

73
src/Dockerfile Normal file
View File

@@ -0,0 +1,73 @@
# based on https://github.com/scheme-containers/monorepo/blob/master/implementations/chicken/5/Dockerfile
FROM debian:trixie-slim AS build
RUN apt-get update && apt-get -y --no-install-recommends install \
build-essential \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /build
COPY checksum checksum
ADD https://code.call-cc.org/releases/5.4.0/chicken-5.4.0.tar.gz chicken.tar.gz
RUN sha256sum chicken.tar.gz && sha256sum -c checksum
RUN mkdir chicken && tar -C chicken --strip-components 1 -xf chicken.tar.gz
WORKDIR /build/chicken
RUN make
RUN make install
FROM debian:trixie-slim AS buildeggs
RUN apt-get update && apt-get -y --no-install-recommends install \
gcc libc-dev libpq-dev \
&& rm -rf /var/lib/apt/lists/*
COPY --from=build /usr/local/ /usr/local/
COPY scss /var/scss
COPY html-widgets /var/html-widgets
COPY schematra-session /var/schematra-session
WORKDIR /var/scss
RUN chicken-install
WORKDIR /var/html-widgets
RUN chicken-install
WORKDIR /var/
RUN chicken-install srfi-1 srfi-13 srfi-18 srfi-158 srfi-194 \
sxml-transforms schematra \
uri-common http-client medea intarweb \
sql-null openssl postgresql crypto-tools
# Egg is currently broken should be able to move back to regular install after it is fixed
WORKDIR /var/schematra-session
RUN chicken-install
WORKDIR /var
RUN mkdir nassella
WORKDIR /var/nassella
COPY mocks.scm mocks.scm
COPY db.scm db.scm
COPY nassella.scm nassella.scm
COPY run.scm run.scm
RUN csc -O3 mocks.scm -J
RUN csc -O3 db.scm -J
RUN csc -O3 nassella.scm -J
RUN csc -O3 -o nassella-run run.scm
RUN chmod +x nassella-run
FROM debian:trixie-slim
RUN apt-get update && apt-get -y --no-install-recommends install \
libpq-dev \
&& rm -rf /var/lib/apt/lists/*
COPY --from=buildeggs /usr/local/ /usr/local/
WORKDIR /var
COPY --from=buildeggs /var/nassella/mocks /var
COPY --from=buildeggs /var/nassella/db /var
COPY --from=buildeggs /var/nassella/nassella /var
COPY --from=buildeggs /var/nassella/nassella-run /var
COPY nassella-latest.tar nassella-latest.tar
COPY root-key root-key
COPY db-init.sql db-init.sql
COPY db-clean.sql db-clean.sql
# ENTRYPOINT ["ls"]
# CMD ["/usr/local/lib/chicken/11"]
ENTRYPOINT ["./nassella-run"]
CMD ["-:a50"]

14
src/Makefile Normal file
View File

@@ -0,0 +1,14 @@
dockerall:
docker buildx build --platform linux/amd64,linux/arm64 -t nassella/b0.0.1 .
dockerlocal:
docker buildx build -t nassella/b0.0.1 .
dockerpush:
docker push nassella/b0.0.1
local:
docker run -p 8080:8080 --net=host --rm nassella/b0.0.1
localclean:
docker run -p 8080:8080 --net=host --rm nassella/b0.0.1 --clean

1
src/checksum Normal file
View File

@@ -0,0 +1 @@
3c5d4aa61c1167bf6d9bf9eaf891da7630ba9f5f3c15bf09515a7039bfcdec5f chicken.tar.gz

17
src/compose.yaml Normal file
View File

@@ -0,0 +1,17 @@
services:
db:
image: postgres
restart: always
environment:
POSTGRES_USER: nassella
POSTGRES_PASSWORD: password
POSTGRES_DB: nassella
volumes:
- /home/tjhintz/nassella-db:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready", "-U", "nassella"]
interval: 1s
timeout: 5s
retries: 10
ports:
- "5432:5432"

22
src/db-clean.sql Normal file
View File

@@ -0,0 +1,22 @@
drop index user_terraform_state_user_id_instance_id_idx;
drop table user_terraform_state;
drop index deployments_user_id_instance_id_idx;
drop table deployments;
drop type deployment_status;
drop index user_app_configs_user_id_instance_id_idx;
drop table user_app_configs;
drop index user_selected_apps_user_id_instance_id_idx;
drop table user_selected_apps;
drop index user_service_configs_user_id_instance_id_idx;
drop table user_service_configs;
drop index instances_user_id_instance_id_idx;
drop table instances;
drop index users_auth_user_id_idx;
drop table users;

91
src/db-init.sql Normal file
View File

@@ -0,0 +1,91 @@
create table users(
user_id bigserial primary key,
auth_user_id int unique not null,
email varchar(255) not null,
username varchar(255) not null unique,
key_key varchar(255),
key_iv varchar(255)
);
create unique index users_auth_user_id_idx on users (auth_user_id);
create table instances(
instance_id bigserial primary key,
user_id integer not null references users on delete cascade,
ssh_key_priv_enc text,
ssh_key_pub_enc text,
restic_password_enc varchar(255)
);
create unique index instances_user_id_instance_id_idx on instances (instance_id, user_id);
create table user_service_configs(
id bigserial primary key,
user_id integer not null references users on delete cascade,
instance_id integer not null references instances on delete cascade,
cloudflare_api_token_enc varchar(255),
cloudflare_account_id_enc varchar(255),
cloudflare_zone_id_enc varchar(255),
digitalocean_api_token_enc varchar(255),
digitalocean_region varchar(255),
digitalocean_size varchar(255),
backblaze_application_key_enc varchar(255),
backblaze_key_id_enc varchar(255),
backblaze_bucket_url_enc varchar(255)
);
create unique index user_service_configs_user_id_instance_id_idx on user_service_configs (user_id, instance_id);
create table user_selected_apps(
id bigserial primary key,
user_id integer not null references users on delete cascade,
instance_id integer not null references instances on delete cascade,
wg_easy_version varchar(100),
nextcloud_version varchar(100),
nassella_version varchar(100),
log_viewer_version varchar(100),
ghost_version varchar(100)
);
create unique index user_selected_apps_user_id_instance_id_idx on user_selected_apps (user_id, instance_id);
create table user_app_configs(
id bigserial primary key,
user_id integer not null references users on delete cascade,
instance_id integer not null references instances on delete cascade,
root_domain varchar(100),
config_enc text
);
create unique index user_app_configs_user_id_instance_id_idx on user_app_configs (user_id, instance_id);
create type deployment_status as enum ('queued', 'in-progress', 'complete', 'failed');
create table deployments(
id bigserial primary key,
user_id integer not null references users on delete cascade,
instance_id integer not null references instances on delete cascade,
started timestamptz,
finished timestamptz,
status deployment_status not null default 'queued',
pid integer,
generate_configs deployment_status not null default 'queued',
terraform_custom_image deployment_status not null default 'queued',
terraform_dns deployment_status not null default 'queued',
terraform_volume_create deployment_status not null default 'queued',
terraform_volume_destroy deployment_status not null default 'queued',
terraform_machine_create deployment_status not null default 'queued',
terraform_machine_destroy deployment_status not null default 'queued',
terraform_ip_create deployment_status not null default 'queued',
terraform_ip_destroy deployment_status not null default 'queued',
log_enc text
);
create index deployments_user_id_instance_id_idx on deployments (user_id, instance_id);
create table user_terraform_state(
id bigserial primary key,
user_id integer not null references users on delete cascade,
instance_id integer not null references instances on delete cascade,
state_enc text,
state_backup_enc text
);
create unique index user_terraform_state_user_id_instance_id_idx on user_terraform_state (user_id, instance_id);

626
src/db.scm Normal file
View File

@@ -0,0 +1,626 @@
(module nassella-db
(;; parameters
connection-spec
;;functions
with-db with-db/transaction
db-init db-clean
create-user delete-user
create-instance get-user-instances
get-instance-ssh-pub-key get-instance-ssh-priv-key
update-instance-ssh-pub-key
get-instance-restic-password
update-user-service-config get-user-service-config
update-user-selected-apps get-user-selected-apps
update-user-app-config get-user-app-config
update-root-domain
create-deployment
update-deployment-status get-deployment-status
get-most-recent-deployment-status
update-deployment-in-progress
update-deployment-progress get-deployment-progress
get-most-recent-deployment-progress
update-user-terraform-state get-user-terraform-state
get-dashboard
)
(import scheme
(chicken base)
(chicken blob)
(chicken file)
(chicken string)
(chicken port)
(chicken io)
postgresql
sql-null
srfi-1
srfi-13
(openssl cipher)
(openssl random)
crypto-tools
spiffy)
(define connection-spec (make-parameter '((dbname . "nassella") (user . "nassella") (password . "password")
;; (host . "127.0.0.1")
(host . "nassella_db")
)))
(define db-connection (make-parameter #f))
(define (with-db proc)
(if (db-connection)
(begin (db-connection)
(proc (db-connection)))
(let ((conn #f))
(dynamic-wind
(lambda ()
(set! conn (connect (connection-spec))))
(lambda () (proc conn))
(lambda ()
(when (and (connection? conn) (connected? conn))
(disconnect conn)))))))
(define (with-db/transaction proc)
(with-db
(lambda (conn)
(with-transaction conn
(lambda () (proc conn))))))
;; (with-db (lambda (db) (row-values (query db "select * from users;"))))
(define aes-256-gcm (cipher-by-name "aes-256-gcm"))
(define tag-length 16)
(define (generate-param accessor)
(random-bytes (accessor aes-256-gcm)))
(define (generate-key) (generate-param cipher-key-length))
(define (generate-iv) (generate-param cipher-iv-length))
(define (encrypt message key iv #!optional auth-data)
(string-encrypt-and-digest aes-256-gcm message key iv
tag-length: tag-length
auth-data: auth-data))
(define (decrypt message tag key iv #!optional auth-data)
(string-decrypt-and-verify aes-256-gcm message tag key iv
auth-data: auth-data))
(define *root-key-file* "root-key")
(define (generate-root-key) (generate-key))
(define (save-root-key)
(with-output-to-file *root-key-file* (lambda () (write (blob->hexstring/uppercase (generate-root-key))))))
(define (load-root-key)
(hexstring->blob (with-input-from-file *root-key-file* read)))
(define *root-key-iv* (hexstring->blob "1EBBCF6B50C68593C559EF93"))
(define (ensure-root-key)
(when (not (file-exists? *root-key-file*))
(save-root-key))
(load-root-key))
(define *root-key-key* (ensure-root-key))
(define (get-user-key-and-iv conn user-id)
(row-alist (query conn "select auth_user_id, key_key, key_iv from users where user_id=$1;" user-id)))
(define (get-decrypted-user-key-and-iv conn user-id)
(let* ((auth-user-id-and-user-key-and-iv (get-user-key-and-iv conn user-id))
(raw-user-key-and-tag (alist-ref 'key_key auth-user-id-and-user-key-and-iv))
(raw-user-key (hexstring->blob (string-drop-right raw-user-key-and-tag (* tag-length 2))))
(raw-user-tag (hexstring->blob (string-take-right raw-user-key-and-tag (* tag-length 2))))
(user-key (decrypt (blob->string raw-user-key) (blob->string raw-user-tag) *root-key-key* *root-key-iv*
(string->blob (number->string (alist-ref 'auth_user_id auth-user-id-and-user-key-and-iv)))))
(user-iv (alist-ref 'key_iv auth-user-id-and-user-key-and-iv))
(auth-user-id (alist-ref 'auth_user_id auth-user-id-and-user-key-and-iv)))
(values (hexstring->blob user-key) (hexstring->blob user-iv) auth-user-id)))
(define (user-encrypt message user-key user-iv user-id)
(encrypt message user-key user-iv (string->blob (->string user-id))))
(define (user-encrypt-for-db message user-key user-iv user-id)
(receive (message tag)
(user-encrypt message user-key user-iv user-id)
(string-append (blob->hexstring/uppercase (string->blob message))
(blob->hexstring/uppercase (string->blob tag)))))
(define (user-decrypt message tag user-key user-iv user-id)
(decrypt message tag user-key user-iv (string->blob (->string user-id))))
(define (user-decrypt-from-db message-and-tag user-key user-iv user-id)
(let ((raw-message (hexstring->blob (string-drop-right message-and-tag (* tag-length 2))))
(raw-tag (hexstring->blob (string-take-right message-and-tag (* tag-length 2)))))
(user-decrypt (blob->string raw-message) (blob->string raw-tag) user-key user-iv user-id)))
(define (create-user conn auth-user-id email username)
(let ((user-key (blob->hexstring/uppercase (generate-key)))
(user-iv (blob->hexstring/uppercase (generate-iv))))
(receive (enc-user-key tag)
(encrypt user-key *root-key-key* *root-key-iv* (string->blob (number->string auth-user-id)))
(let ((user-id
(value-at
(query conn
"insert into users(auth_user_id, email, username, key_key, key_iv) values ($1, $2, $3, $4, $5)
returning users.user_id;"
auth-user-id email username
(string-append (blob->hexstring/uppercase (string->blob enc-user-key))
(blob->hexstring/uppercase (string->blob tag)))
user-iv))))
user-id))))
(define (delete-user conn user-id)
(query conn "delete from users where user_id=$1;" user-id))
;; We also encrypt the ssh pub key not to hide it but to make it
;; more difficult for someone to tamper with it which could allow
;; an attacker to poison an instance with an ssh key that they have
;; access to
(define (create-instance conn user-id ssh-key-priv ssh-key-pub restic-password)
(receive (user-key user-iv auth-user-id)
(get-decrypted-user-key-and-iv conn user-id)
(let ((instance-id
(value-at
(query conn
"insert into instances(user_id, ssh_key_priv_enc, ssh_key_pub_enc, restic_password_enc) values ($1, $2, $3, $4) returning instances.instance_id;"
user-id
(user-encrypt-for-db ssh-key-priv user-key user-iv user-id)
(user-encrypt-for-db ssh-key-pub user-key user-iv user-id)
(user-encrypt-for-db restic-password user-key user-iv user-id)))))
(query conn "insert into user_service_configs(user_id, instance_id) values ($1, $2);" user-id instance-id)
(query conn "insert into user_selected_apps(user_id, instance_id) values ($1, $2);" user-id instance-id)
(query conn "insert into user_app_configs(user_id, instance_id) values ($1, $2);" user-id instance-id)
(query conn "insert into user_terraform_state(user_id, instance_id) values ($1, $2);" user-id instance-id)
instance-id)))
(define (get-instance-ssh-priv-key conn user-id instance-id)
(receive (user-key user-iv auth-user-id)
(get-decrypted-user-key-and-iv conn user-id)
(user-decrypt-from-db
(value-at (query conn "select ssh_key_priv_enc from instances where user_id=$1 and instance_id=$2;"
user-id instance-id))
user-key user-iv user-id)))
(define (get-instance-ssh-pub-key conn user-id instance-id)
(receive (user-key user-iv auth-user-id)
(get-decrypted-user-key-and-iv conn user-id)
(user-decrypt-from-db
(value-at (query conn "select ssh_key_pub_enc from instances where user_id=$1 and instance_id=$2;"
user-id instance-id))
user-key user-iv user-id)))
(define (update-instance-ssh-pub-key conn user-id instance-id ssh-pub-key)
(receive (user-key user-iv auth-user-id)
(get-decrypted-user-key-and-iv conn user-id)
(query conn "update instances set ssh_key_pub_enc=$3 where user_id=$1 and instance_id=$2;"
user-id instance-id
(user-encrypt-for-db ssh-pub-key user-key user-iv user-id))))
(define (get-instance-restic-password conn user-id instance-id)
(receive (user-key user-iv auth-user-id)
(get-decrypted-user-key-and-iv conn user-id)
(user-decrypt-from-db
(value-at (query conn "select restic_password_enc from instances where user_id=$1 and instance_id=$2;"
user-id instance-id))
user-key user-iv user-id)))
(define (get-user-instances conn user-id)
(column-values (query conn "select instance_id from instances where user_id=$1;" user-id)))
(define *user-service-configs-column-map*
'((cloudflare-api-token . ("cloudflare_api_token_enc" #t))
(cloudflare-account-id . ("cloudflare_account_id_enc" #t))
(cloudflare-zone-id . ("cloudflare_zone_id_enc" #t))
(digitalocean-api-token . ("digitalocean_api_token_enc" #t))
(digitalocean-region . ("digitalocean_region" #f))
(digitalocean-size . ("digitalocean_size" #f))
(backblaze-application-key . ("backblaze_application_key_enc" #t))
(backblaze-key-id . ("backblaze_key_id_enc" #t))
(backblaze-bucket-url . ("backblaze_bucket_url_enc" #t))))
(define *user-service-configs-reverse-column-map*
(map (lambda (config)
`(,(string->symbol (cadr config)) . (,(car config) ,(caddr config))))
*user-service-configs-column-map*))
(define (update-user-service-config conn user-id instance-id update-alist)
(let ((valid-keys (map car *user-service-configs-column-map*)))
(for-each (lambda (update)
(if (not (memq (car update) valid-keys))
(error (string-append "Not a valid update key: " (->string (car update))))))
update-alist))
(receive (user-key user-iv auth-user-id)
(get-decrypted-user-key-and-iv conn user-id)
(query* conn
(string-append
"update user_service_configs set "
(string-intersperse
(map-in-order (lambda (update i)
(conc (car (alist-ref (car update) *user-service-configs-column-map*))
"=$" i))
update-alist
(iota (length update-alist) 3))
", ")
" where user_id=$1 and instance_id=$2;")
`(,user-id
,instance-id
,@(map-in-order (lambda (update)
(if (cadr (alist-ref (car update) *user-service-configs-column-map*))
(user-encrypt-for-db (cdr update) user-key user-iv user-id)
(cdr update)))
update-alist)))))
(define (get-user-service-config conn user-id instance-id)
(receive (user-key user-iv auth-user-id)
(get-decrypted-user-key-and-iv conn user-id)
(let ((res (row-alist
(query conn
(string-append
"select "
(string-intersperse
(map-in-order (lambda (update)
(car (alist-ref (car update) *user-service-configs-column-map*)))
*user-service-configs-column-map*)
", ")
" from user_service_configs where user_id=$1 and instance_id=$2;")
user-id instance-id))))
(map (lambda (item)
(let* ((key (car item))
(value (cdr item))
(config (alist-ref key *user-service-configs-reverse-column-map*)))
`(,(car config) . ,(if (sql-null? value)
""
(if (cadr config)
(user-decrypt-from-db value user-key user-iv user-id)
value)))))
res))))
(define *user-selected-apps-column-map*
'((wg-easy . "wg_easy_version")
(nextcloud . "nextcloud_version")
(ghost . "ghost_version")
(nassella . "nassella_version")
(log-viewer . "log_viewer_version")))
(define *user-selected-apps-reverse-column-map*
(map (lambda (config)
`(,(string->symbol (cdr config)) . ,(car config)))
*user-selected-apps-column-map*))
(define (update-user-selected-apps conn user-id instance-id app-alist)
(let ((valid-keys (map car *user-selected-apps-column-map*)))
(for-each (lambda (app)
(if (not (memq (car app) valid-keys))
(error (string-append "Not a valid app key: " (->string (car app))))))
app-alist))
(query* conn
(string-append
"update user_selected_apps set "
(string-intersperse
(map-in-order (lambda (app i)
(conc (alist-ref (car app) *user-selected-apps-column-map*)
"=$" i))
app-alist
(iota (length app-alist) 3))
", ")
" where user_id=$1 and instance_id=$2;")
`(,user-id
,instance-id
,@(map-in-order cdr app-alist))))
(define (get-user-selected-apps conn user-id instance-id)
(let ((res (row-alist
(query conn
(string-append
"select "
(string-intersperse
(map-in-order cdr *user-selected-apps-column-map*)
", ")
" from user_selected_apps where user_id=$1 and instance_id=$2;")
user-id instance-id))))
(map (lambda (item)
(let* ((key (car item))
(value (cdr item))
(config (alist-ref key *user-selected-apps-reverse-column-map*)))
`(,config . ,(if (sql-null? value)
#f
value))))
res)))
(define (update-user-app-config conn user-id instance-id config)
(receive (user-key user-iv auth-user-id)
(get-decrypted-user-key-and-iv conn user-id)
(query conn
"update user_app_configs set config_enc=$1 where user_id=$2 and instance_id=$3;"
(user-encrypt-for-db
(with-output-to-string
(lambda ()
(write config)))
user-key user-iv user-id)
user-id instance-id)))
(define (update-root-domain conn user-id instance-id root-domain)
(query conn
"update user_app_configs set root_domain=$1 where user_id=$2 and instance_id=$3;"
root-domain
user-id
instance-id))
(define (get-user-app-config conn user-id instance-id)
(receive (user-key user-iv auth-user-id)
(get-decrypted-user-key-and-iv conn user-id)
(let ((res (row-alist (query conn
"select root_domain, config_enc from user_app_configs where user_id=$1 and instance_id=$2;"
user-id instance-id))))
`((root-domain . ,(if (sql-null? (alist-ref 'root_domain res))
#f
(alist-ref 'root_domain res)))
(config . ,(if (sql-null? (alist-ref 'config_enc res))
'()
(with-input-from-string
(user-decrypt-from-db (alist-ref 'config_enc res) user-key user-iv user-id)
read)))))))
(define *deployment-status*
'((queued . "queued")
(in-progress . "in-progress")
(complete . "complete")
(failed . "failed")))
(define (create-deployment conn user-id instance-id)
(value-at
(query conn
"insert into deployments(user_id, instance_id, started) values($1, $2, now()) returning deployments.id;"
user-id instance-id)))
(define (update-deployment-in-progress conn deployment-id pid)
(query conn
"update deployments set status=$1, pid=$2 where id=$3;"
(alist-ref 'in-progress *deployment-status*) pid deployment-id))
(define (update-deployment-status conn user-id deployment-id status log)
(receive (user-key user-iv auth-user-id)
(get-decrypted-user-key-and-iv conn user-id)
(query conn "update deployments set status=$1, log_enc=$2, finished=now() where id=$3;"
(alist-ref status *deployment-status*)
(user-encrypt-for-db log user-key user-iv user-id)
deployment-id)))
(define (get-deployment-status conn deployment-id)
(value-at (query conn "select status from deployments where id=$1;" deployment-id)))
(define (get-most-recent-deployment-status conn user-id instance-id)
(value-at (query conn "select status from deployments where user_id=$1 and instance_id=$2 order by id DESC limit 1;" user-id instance-id)))
(define *deployments-column-map*
'((generate-configs . "generate_configs")
(custom-image . "terraform_custom_image")
(machine-create . "terraform_machine_create")
(machine-destroy . "terraform_machine_destroy")
(status . "status")
(id . "id")
(instance-id . "instance_id")))
(define *deployments-reverse-column-map*
(map (lambda (config)
`(,(string->symbol (cdr config)) . ,(car config)))
*deployments-column-map*))
(define (update-deployment-progress conn deployment-id progress-alist)
(let ((valid-keys (map car *deployments-column-map*)))
(for-each (lambda (progress)
(if (not (memq (car progress) valid-keys))
(error (string-append "Not a valid progress key: " (->string (car progress))))))
progress-alist))
(query* conn
(string-append
"update deployments set "
(string-intersperse
(map-in-order (lambda (progress i)
(conc (alist-ref (car progress) *deployments-column-map*)
"=$" i))
progress-alist
(iota (length progress-alist) 2))
", ")
" where id=$1;")
(cons deployment-id
(map-in-order (lambda (progress) (alist-ref (cdr progress) *deployment-status*)) progress-alist))))
(define (get-deployment-progress conn deployment-id)
(let ((res (row-alist
(query conn
(string-append
"select "
(string-intersperse
(map-in-order cdr *deployments-column-map*)
", ")
" from deployments where id=$1;")
deployment-id))))
(map (lambda (item)
(let* ((key (car item))
(value (cdr item))
(config (alist-ref key *deployments-reverse-column-map*)))
`(,config . ,(if (sql-null? value)
#f
(string->symbol value)))))
res)))
(define (get-most-recent-deployment-progress conn user-id instance-id)
(let ((res (row-alist
(query conn
(string-append
"select "
(string-intersperse
(map-in-order cdr *deployments-column-map*)
", ")
" from deployments where user_id=$1 and instance_id=$2 order by id DESC limit 1;")
user-id instance-id))))
(map (lambda (item)
(let* ((key (car item))
(value (cdr item))
(config (alist-ref key *deployments-reverse-column-map*)))
`(,config . ,(if (sql-null? value)
#f
(if (string? value)
(string->symbol value)
value)))))
res)))
(define (get-dashboard conn user-id)
(receive (user-key user-iv auth-user-id)
(get-decrypted-user-key-and-iv conn user-id)
(let ((res
(query conn
(string-append
"select "
(string-intersperse
(map-in-order (lambda (d) (string-append "d." (cdr d))) *deployments-column-map*)
", ")
", uac.root_domain, uac.config_enc, uac.instance_id, "
"usa.wg_easy_version, usa.nextcloud_version, usa.log_viewer_version, usa.ghost_version, usa.nassella_version "
"from instances as i "
"join (select instance_id, max(id) as id from deployments group by instance_id) d2 "
"on d2.instance_id = i.instance_id "
"join deployments d on d.id = d2.id "
"join user_app_configs uac on uac.user_id = d.user_id and uac.instance_id = d.instance_id "
"join user_selected_apps usa on usa.instance_id = uac.instance_id "
"where i.user_id=$1;")
user-id)))
(map
(lambda (row-num)
(map (lambda (item)
(let* ((key (car item))
(value (cdr item))
(config (alist-ref key `((root_domain . root-domain)
(config_enc . config)
(instance_id . instance-id)
(wg_easy_version . wg-easy)
(nextcloud_version . nextcloud)
(ghost_version . ghost)
(nassella_version . nassella)
(log_viewer_version . log-viewer)
,@*deployments-reverse-column-map*))))
`(,config . ,(if (sql-null? value)
#f
(if (and (string? value) (member config *deployments-column-map*))
(string->symbol value)
(if (eq? key 'config_enc)
(with-input-from-string
(user-decrypt-from-db value user-key user-iv user-id)
read)
value))))))
(row-alist res row-num)))
(iota (row-count res))))))
(define (update-user-terraform-state conn user-id instance-id state backup)
(receive (user-key user-iv auth-user-id)
(get-decrypted-user-key-and-iv conn user-id)
(query conn
"update user_terraform_state set state_enc=$1, state_backup_enc=$2 where user_id=$3 and instance_id=$4;"
(user-encrypt-for-db state user-key user-iv user-id)
(user-encrypt-for-db backup user-key user-iv user-id)
user-id
instance-id)))
(define (get-user-terraform-state conn user-id instance-id)
(receive (user-key user-iv auth-user-id)
(get-decrypted-user-key-and-iv conn user-id)
(let ((res (row-alist (query conn
"select state_enc, state_backup_enc from user_terraform_state where user_id=$1 and instance_id=$2;"
user-id instance-id))))
`((state . ,(if (or (sql-null? (alist-ref 'config_enc res))
(sql-null? (alist-ref 'state_enc res)))
""
(user-decrypt-from-db (alist-ref 'state_enc res) user-key user-iv user-id)))
(backup . ,(if (or (sql-null? (alist-ref 'config_enc res))
(sql-null? (alist-ref 'state_backup_enc res)))
""
(user-decrypt-from-db (alist-ref 'state_backup_enc res) user-key user-iv user-id)))))))
(debug-log (current-error-port))
(define (db-init)
(with-db/transaction
(lambda (db)
(if (value-at (query db "SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = 'public' AND tablename = 'users');"))
(begin
(log-to (debug-log) "database already initialized")
#t)
(begin
(log-to (debug-log) "tables not found in db. Creating...")
(for-each
(lambda (statement)
(query db (conc statement ";")))
(string-split (with-input-from-file "db-init.sql" read-string) ";"))
(log-to (debug-log) "table creation finished")
(log-to (debug-log) "creating test user")
(create-user db 1 "me@example.com" "username")
(log-to (debug-log) "test user creation finished"))))))
(define (db-clean)
(with-db/transaction
(lambda (db)
(if (value-at (query db "SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = 'public' AND tablename = 'users');"))
(begin
(log-to (debug-log) "cleaning database")
(for-each
(lambda (statement)
(query db (conc statement ";")))
(string-split (with-input-from-file "db-clean.sql" read-string) ";"))
(log-to (debug-log) "database cleaning complete"))
(begin
(log-to (debug-log) "tables not found, not cleaning")
#t)))))
;; (with-db/transaction (lambda (db) (get-user-deployments db 1)))
;; (with-db/transaction (lambda (db) (get-most-recent-deployment-progress db 7)))
;; (with-db/transaction (lambda (db) (get-deployment-progress db 14)))
;; (with-db/transaction (lambda (db) (update-deployment-progress db 14 '((generate-configs . complete) (custom-image . in-progress) (machine-create . queued)))))
;; (with-db/transaction
;; (lambda (db)
;; (update-user-terraform-state db 1 22
;; (with-input-from-file "src/deploy-7/terraform.tfstate" read-string)
;; (with-input-from-file "src/deploy-7/terraform.tfstate.backup" read-string))))
;; (with-db/transaction (lambda (db) (get-user-terraform-state db 7)))
;; (with-db/transaction (lambda (db) (create-deployment db 7)))
;; (with-db/transaction (lambda (db) (get-deployment-status db 1)))
;; (with-db/transaction (lambda (db) (update-deployment-in-progress db 1 123)))
;; (with-db/transaction (lambda (db) (update-deployment-status db 1 'complete)))
;; (with-db/transaction (lambda (db) (get-most-recent-deployment-status db 7)))
;; (with-db/transaction (lambda (db) (create-user db 1 "t@thintz.com" "thecombjelly")))
;; (with-db/transaction (lambda (db) (create-instance db 2)))
;; (let ((user-id 7))
;; (with-db/transaction
;; (lambda (db)
;; (receive (user-key user-iv auth-user-id)
;; (get-decrypted-user-key-and-iv db user-id)
;; (receive (message tag)
;; (user-encrypt "hello!" user-key user-iv user-id)
;; (user-decrypt message tag user-key user-iv user-id))))))
;; (with-db/transaction
;; (lambda (db)
;; (update-user-service-config db 7 '((cloudflare-api-token . ")
;; (digitalocean-region . "sfo3")))))
;; (with-db/transaction
;; (lambda (db)
;; (get-user-service-config db 7)))
;; (with-db/transaction
;; (lambda (db)
;; (update-user-selected-apps db 7 '((wg-easy . "0.1")
;; (nextcloud . "1.3")))))
;; (with-db/transaction
;; (lambda (db)
;; (get-user-selected-apps db 7)))
;; (with-db/transaction
;; (lambda (db)
;; (update-user-app-config db 7 "domain.com" '())))
;; (with-db/transaction
;; (lambda (db)
;; (get-user-app-config db 7)))
)

3096
src/mocks.scm Normal file

File diff suppressed because one or more lines are too long

File diff suppressed because it is too large Load Diff

17
src/run.scm Normal file
View File

@@ -0,0 +1,17 @@
(include "nassella")
(import (chicken process-context)
spiffy
schematra)
(debug-log (current-error-port))
(with-schematra-app
app
(lambda ()
(log-to (debug-log) "starting server")
(log-to (debug-log) "initializing db")
(if (member "--clean" (command-line-arguments) equal?)
(db-clean))
(db-init)
(log-to (debug-log) "db initialization complete")
(start-server)))

446
src/test.scm Normal file

File diff suppressed because one or more lines are too long