First stage refactor for dynamic app specs (napps.scm).

This commit is contained in:
2026-07-02 19:39:56 -07:00
parent 868e00ac41
commit 04407b16b0
49 changed files with 420 additions and 253 deletions

126
Makefile
View File

@@ -3,93 +3,81 @@ TERRAFORM_ENV := production
config_dir := ./config/
apps_config := $(config_dir)apps.config
ALL_APPS_FILES = $(shell find all-apps/ -type f -name '*')
# .dirstamp plus && $@ is like make magic to get this rule
# to only run if the contents of all-apps changes
app/.dirstamp: all-apps/app.service all-apps/docker-compose.yaml all-apps/.env \
all-apps/restic-snapshot.service \
all-apps/instance-control/webhook_secret \
all-apps/instance-control/hooks/hooks.json \
all-apps/lb/Caddyfile \
$(wildcard all-apps/lb/*) \
$(wildcard all-apps/instance-control/*) \
$(wildcard all-apps/nextcloud/*) \
$(wildcard all-apps/wg-easy/*) \
$(wildcard all-apps/ghost/*) \
$(wildcard all-apps/nassella/*) \
all-apps/nassella/authelia-config/configuration.yml \
all-apps/nassella/lldap-config/lldap_config.toml \
$(wildcard all-apps/dozzle/*)
app/.dirstamp: copy-apps.sh $(apps_config) $(ALL_APPS_FILES)
rm -Rf app/
mkdir app/
cp all-apps/app.service app/
cp all-apps/restic-snapshot.service app/
cp all-apps/docker-compose.yaml app/
cp all-apps/.env app/
./copy-apps.sh $(apps_config) && touch $@
# compose .env files
# (compose only supports one .env file at the root by default)
all-apps/.env: all-apps/ghost/.compose-env
find all-apps/ -name ".compose-env" -exec cat > all-apps/.env {} +
# # compose .env files
# # (compose only supports one .env file at the root by default)
app/.env: app/ghost/.compose-env
find app/ -name ".compose-env" -exec cat > app/.env {} +
# Caddy / lb
all-apps/lb/Caddyfile: $(apps_config) make-caddyfile.sh
mkdir -p all-apps/lb
./make-caddyfile.sh $(apps_config) > all-apps/lb/Caddyfile
app/lb/Caddyfile: $(apps_config) make-caddyfile.sh
mkdir -p app/lb
./make-caddyfile.sh $(apps_config) > app/lb/Caddyfile
# Instance Control
all-apps/instance-control/webhook_secret: $(apps_config)
app/instance-control/webhook_secret: $(apps_config)
bash -c 'source $(apps_config); printf "%s\n" "$$INSTANCE_CONTROL_WEBHOOKS_SECRET" > $@'
all-apps/instance-control/hooks/hooks.json: $(apps_config) make-instance-control-config.sh
app/instance-control/hooks/hooks.json: $(apps_config) make-instance-control-config.sh
./make-instance-control-config.sh $(apps_config)
# Nextcloud
all-apps/nextcloud/nextcloud_admin_user: $(apps_config)
app/nextcloud/nextcloud_admin_user: $(apps_config)
bash -c 'source $(apps_config); printf "%s\n" "$$NEXTCLOUD_ADMIN_USER" > $@'
all-apps/nextcloud/nextcloud_admin_password: $(apps_config)
app/nextcloud/nextcloud_admin_password: $(apps_config)
bash -c 'source $(apps_config); printf "%s\n" "$$NEXTCLOUD_ADMIN_PASSWORD" > $@'
all-apps/nextcloud/postgres_db: $(apps_config)
app/nextcloud/postgres_db: $(apps_config)
bash -c 'source ./$(apps_config); printf "%s\n" "$$NEXTCLOUD_POSTGRES_DB" > $@'
all-apps/nextcloud/postgres_user: $(apps_config)
app/nextcloud/postgres_user: $(apps_config)
bash -c 'source ./$(apps_config); printf "%s\n" "$$NEXTCLOUD_POSTGRES_USER" > $@'
all-apps/nextcloud/postgres_password: $(apps_config)
app/nextcloud/postgres_password: $(apps_config)
bash -c 'source ./$(apps_config); printf "%s\n" "$$NEXTCLOUD_POSTGRES_PASSWORD" > $@'
all-apps/nextcloud/redis_password: $(apps_config)
app/nextcloud/redis_password: $(apps_config)
bash -c 'source ./$(apps_config); printf "%s\n" "$$NEXTCLOUD_REDIS_PASSWORD" > $@'
all-apps/nextcloud/nextcloud.env: $(apps_config) all-apps/nextcloud/nextcloud.env.tmpl make-nextcloud-env.sh
app/nextcloud/nextcloud.env: $(apps_config) app/nextcloud/nextcloud.env.tmpl make-nextcloud-env.sh
./make-nextcloud-env.sh $(apps_config)
# Nassella
all-apps/nassella/postgres_db: $(apps_config)
app/nassella/postgres_db: $(apps_config)
bash -c 'source ./$(apps_config); printf "%s\n" "$$NASSELLA_POSTGRES_DB" > $@'
all-apps/nassella/postgres_user: $(apps_config)
app/nassella/postgres_user: $(apps_config)
bash -c 'source ./$(apps_config); printf "%s\n" "$$NASSELLA_POSTGRES_USER" > $@'
all-apps/nassella/postgres_password: $(apps_config)
app/nassella/postgres_password: $(apps_config)
bash -c 'source ./$(apps_config); printf "%s\n" "$$NASSELLA_POSTGRES_PASSWORD" > $@'
all-apps/nassella/authelia_postgres_db: $(apps_config)
app/nassella/authelia_postgres_db: $(apps_config)
bash -c 'source ./$(apps_config); printf "%s\n" "$$NASSELLA_AUTHELIA_POSTGRES_DB" > $@'
all-apps/nassella/authelia_postgres_user: $(apps_config)
app/nassella/authelia_postgres_user: $(apps_config)
bash -c 'source ./$(apps_config); printf "%s\n" "$$NASSELLA_AUTHELIA_POSTGRES_USER" > $@'
all-apps/nassella/authelia_postgres_password: $(apps_config)
app/nassella/authelia_postgres_password: $(apps_config)
bash -c 'source ./$(apps_config); printf "%s\n" "$$NASSELLA_AUTHELIA_POSTGRES_PASSWORD" > $@'
all-apps/nassella/lldap_postgres_db: $(apps_config)
app/nassella/lldap_postgres_db: $(apps_config)
bash -c 'source ./$(apps_config); printf "%s\n" "$$NASSELLA_LLDAP_POSTGRES_DB" > $@'
all-apps/nassella/lldap_postgres_user: $(apps_config)
app/nassella/lldap_postgres_user: $(apps_config)
bash -c 'source ./$(apps_config); printf "%s\n" "$$NASSELLA_LLDAP_POSTGRES_USER" > $@'
all-apps/nassella/lldap_postgres_password: $(apps_config)
app/nassella/lldap_postgres_password: $(apps_config)
bash -c 'source ./$(apps_config); printf "%s\n" "$$NASSELLA_LLDAP_POSTGRES_PASSWORD" > $@'
all-apps/nassella/lldap_admin_password: $(apps_config)
app/nassella/lldap_admin_password: $(apps_config)
bash -c 'source ./$(apps_config); printf "%s\n" "$$NASSELLA_LLDAP_ADMIN_PASSWORD" > $@'
all-apps/nassella/stripe_api_key: $(apps_config)
app/nassella/stripe_api_key: $(apps_config)
bash -c 'source ./$(apps_config); printf "%s\n" "$$NASSELLA_STRIPE_API_KEY" > $@'
all-apps/nassella/authelia-config/configuration.yml: $(apps_config) all-apps/nassella/authelia-config/configuration.yml.tmpl make-nassella-authelia-config.sh
app/nassella/authelia-config/configuration.yml: $(apps_config) app/nassella/authelia-config/configuration.yml.tmpl make-nassella-authelia-config.sh
./make-nassella-authelia-config.sh $(apps_config)
all-apps/nassella/lldap-config/lldap_config.toml: $(apps_config) all-apps/nassella/lldap-config/lldap_config.toml.tmpl make-nassella-lldap-config.sh
app/nassella/lldap-config/lldap_config.toml: $(apps_config) app/nassella/lldap-config/lldap_config.toml.tmpl make-nassella-lldap-config.sh
./make-nassella-lldap-config.sh $(apps_config)
# Ghost
all-apps/ghost/.compose-env: $(apps_config) all-apps/ghost/.compose.env.tmpl make-ghost-env.sh
app/ghost/.compose-env: $(apps_config) app/ghost/.compose.env.tmpl make-ghost-env.sh
./make-ghost-env.sh $(apps_config)
# Backups / Restic / Backblaze
@@ -99,34 +87,34 @@ restic-password: $(apps_config) make-restic-password.sh
./make-restic-password.sh $(apps_config) > restic-password
ignition.json: cl.yaml app/.dirstamp \
all-apps/lb/Caddyfile \
all-apps/instance-control/webhook_secret \
all-apps/instance-control/hooks/hooks.json \
all-apps/nextcloud/nextcloud_admin_user \
all-apps/nextcloud/nextcloud_admin_password \
all-apps/nextcloud/postgres_db \
all-apps/nextcloud/postgres_user \
all-apps/nextcloud/postgres_password \
all-apps/nextcloud/redis_password \
all-apps/nextcloud/nextcloud.env \
all-apps/nassella/postgres_db \
all-apps/nassella/postgres_user \
all-apps/nassella/postgres_password \
all-apps/nassella/lldap_postgres_db \
all-apps/nassella/lldap_postgres_user \
all-apps/nassella/lldap_postgres_password \
all-apps/nassella/authelia_postgres_db \
all-apps/nassella/authelia_postgres_user \
all-apps/nassella/authelia_postgres_password \
all-apps/nassella/nassella.env \
all-apps/nassella/authelia-config/configuration.yml \
all-apps/nassella/lldap-config/lldap_config.toml \
all-apps/ghost/.compose-env \
app/lb/Caddyfile \
app/instance-control/webhook_secret \
app/instance-control/hooks/hooks.json \
app/nextcloud/nextcloud_admin_user \
app/nextcloud/nextcloud_admin_password \
app/nextcloud/postgres_db \
app/nextcloud/postgres_user \
app/nextcloud/postgres_password \
app/nextcloud/redis_password \
app/nextcloud/nextcloud.env \
app/nassella/postgres_db \
app/nassella/postgres_user \
app/nassella/postgres_password \
app/nassella/lldap_postgres_db \
app/nassella/lldap_postgres_user \
app/nassella/lldap_postgres_password \
app/nassella/authelia_postgres_db \
app/nassella/authelia_postgres_user \
app/nassella/authelia_postgres_password \
app/nassella/nassella.env \
app/nassella/authelia-config/configuration.yml \
app/nassella/lldap-config/lldap_config.toml \
app/ghost/.compose-env \
restic-env \
restic-password \
restic-restore.sh \
restic-snapshot.sh \
all-apps/.env \
app/.env \
$(config_dir)ssh-keys
cat cl.yaml | docker run --rm --volume $(config_dir)/ssh-keys:/pwd/ssh-keys --volume ${PWD}:/pwd --workdir /pwd -i quay.io/coreos/butane:latest -d /pwd > ignition.json

View File

@@ -0,0 +1,20 @@
DOMAIN="nextcloud.nassella.cc"
NEXTCLOUD_TRUSTED_DOMAINS=${DOMAIN}
# reverse proxy config
OVERWRITEHOST=${DOMAIN}
OVERWRITECLIURL=https://${DOMAIN}
OVERWRITEPROTOCOL=https
TRUSTED_PROXIES=172.16.0.0/24 # trust the local lb
PHP_MEMORY_LIMIT=1G
PHP_UPLOAD_LIMIT=10G
POSTGRES_HOST=db
POSTGRES_DB_FILE=/run/secrets/postgres_db
POSTGRES_USER_FILE=/run/secrets/postgres_user
POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password
# admin user
NEXTCLOUD_ADMIN_PASSWORD_FILE=/run/secrets/nextcloud_admin_password
NEXTCLOUD_ADMIN_USER_FILE=/run/secrets/nextcloud_admin_user
# redis
REDIS_HOST=redis
REDIS_HOST_PASSWORD_FILE=/run/secrets/redis_password

View File

@@ -0,0 +1 @@
foodisgood

View File

@@ -0,0 +1 @@
admin

View File

@@ -0,0 +1 @@
nextcloud

View File

@@ -0,0 +1 @@
dbpassword

View File

@@ -0,0 +1 @@
nextcloud

View File

@@ -0,0 +1 @@
redispassword

View File

@@ -0,0 +1,25 @@
((name . "Nextcloud")
(key . nextcloud)
(dir . "nextcloud")
(current-version . "34")
(versions . (("34")))
(website . "https://nextcloud.com")
(docker-secrets . (("NEXTCLOUD_ADMIN_USER" "nextcloud_admin_user")
("NEXTCLOUD_ADMIN_PASSWORD" "nextcloud_admin_password")
("NEXTCLOUD_POSTGRES_DB" "postgres_db")
("NEXTCLOUD_POSTGRES_USER" "postgres_user")
("NEXTCLOUD_POSTGRES_PASSWORD" "postgres_password")
("NEXTCLOUD_REDIS_PASSWORD" "redis_password")))
(scripts . (("make-nextcloud-env.sh"
((deps . ("nextcloud.env.tmpl"))
(output . "nextcloud.env")))))
(apps-config . (("NEXTCLOUD_ADMIN_USER" admin-user)
("NEXTCLOUD_ADMIN_PASSWORD" admin-password)
("NEXTCLOUD_POSTGRES_DB" "nextcloud")
("NEXTCLOUD_POSTGRES_USER" "nextcloud")
("NEXTCLOUD_POSTGRES_PASSWORD" postgres-password)
("NEXTCLOUD_REDIS_PASSWORD" redis-password)))
(general . ((smtp . #t)))
(caddyfile . " reverse_proxy http://nextcloud:80")
(passwd . ((name . "nextcloud")
(uid . "1001"))))

View File

@@ -0,0 +1,7 @@
((name . "wg-easy")
(key . wg-easy)
(dir . "wg-easy")
(current-version . "15")
(versions . (("15")))
(website . "https://wg-easy.github.io/wg-easy/Pre-release/")
(caddyfile . " reverse_proxy http://wg-easy:80"))

View File

@@ -1,5 +1,5 @@
ROOT_DOMAIN= # example.com :: the root of the domain that all apps should be subdomains of
APP_CONFIGS="nextcloud,nextcloud wg-easy,wg-easy ghost,ghost" # apps to deploy and their corresponding sub-domain (app,sub-domain)
APP_CONFIGS="nextcloud,nextcloud,34 wg-easy,wg-easy,15 ghost,ghost,2" # apps to deploy and their corresponding sub-domain and all-apps version (app,sub-domain,version)
NEXTCLOUD_ADMIN_USER=admin # admin user for nextcloud, can be whatever you want
NEXTCLOUD_ADMIN_PASSWORD= # the password for the nextcloud admin user
NEXTCLOUD_POSTGRES_DB=nextcloud # recommended to leave as 'nextcloud'. The postgres db nextcloud uses

View File

@@ -5,18 +5,18 @@
# it depends on apps.config which should define:
# ROOT_DOMAIN - the root domain for all apps
# APP_CONFIGS - app-subdomain pairs, configured via a comma, like:
# app1,subdomain1 app2,subdomain2 app3,subdomain3
# APP_CONFIGS - app-subdomain-version pairs, configured via a comma, like:
# app1,subdomain1,version app2,subdomain2,version app3,subdomain3,version
# full example:
# ROOT_DOMAIN=nassella.cc
# APP_CONFIGS="app1,subdomain1 app2,subdomain2 app3,subdomain3"
# APP_CONFIGS="app1,subdomain1,1 app2,subdomain2,1 app3,subdomain3,1"
set -e
. $1 # source the apps.config file with then env vars
. $1 # source the apps.config file with the env vars
read -r -a APP_CONFIGS <<< "$APP_CONFIGS"
APP_CONFIGS+=('lb,root')
# APP_CONFIGS+=('lb,root,1') # TODO is this used anymore?
for config_string in ${APP_CONFIGS[@]}; do
@@ -24,8 +24,10 @@ for config_string in ${APP_CONFIGS[@]}; do
read -r -a config <<< "$config_string"
app=${config[0]}
cp -a all-apps/$app app/
version=${config[2]}
cp -a all-apps/$app/$version app/$app
done
# compose .env files
# (compose only supports one .env file at the root by default)
find app/ -name ".compose-env" -exec cat > app/.env {} +

View File

@@ -6,7 +6,7 @@ set -e
read -r -a APP_CONFIGS <<< "$APP_CONFIGS"
nextcloud_subdomain=
ghost_subdomain=
for config_string in ${APP_CONFIGS[@]}; do
IFS=','
@@ -21,12 +21,12 @@ for config_string in ${APP_CONFIGS[@]}; do
done
# write compose env file
echo "GHOST_DOMAIN=\"$ghost_subdomain.$ROOT_DOMAIN\"" > all-apps/ghost/.compose-env
echo "GHOST_DATABASE_ROOT_PASSWORD=\"$GHOST_DATABASE_ROOT_PASSWORD\"" >> all-apps/ghost/.compose-env
echo "GHOST_DATABASE_PASSWORD=\"$GHOST_DATABASE_PASSWORD\"" >> all-apps/ghost/.compose-env
echo "mail__options__host=\"$SMTP_HOST\"" >> all-apps/ghost/.compose-env
echo "mail__options__port=\"$SMTP_PORT\"" >> all-apps/ghost/.compose-env
echo "mail__options__auth__user=\"$SMTP_AUTH_USER\"" >> all-apps/ghost/.compose-env
echo "mail__options__auth__pass=\"$SMTP_AUTH_PASSWORD\"" >> all-apps/ghost/.compose-env
echo "mail__from=\"$SMTP_FROM\"" >> all-apps/ghost/.compose-env
cat all-apps/ghost/.compose.env.tmpl >> all-apps/ghost/.compose-env
echo "GHOST_DOMAIN=\"$ghost_subdomain.$ROOT_DOMAIN\"" > app/ghost/.compose-env
echo "GHOST_DATABASE_ROOT_PASSWORD=\"$GHOST_DATABASE_ROOT_PASSWORD\"" >> app/ghost/.compose-env
echo "GHOST_DATABASE_PASSWORD=\"$GHOST_DATABASE_PASSWORD\"" >> app/ghost/.compose-env
echo "mail__options__host=\"$SMTP_HOST\"" >> app/ghost/.compose-env
echo "mail__options__port=\"$SMTP_PORT\"" >> app/ghost/.compose-env
echo "mail__options__auth__user=\"$SMTP_AUTH_USER\"" >> app/ghost/.compose-env
echo "mail__options__auth__pass=\"$SMTP_AUTH_PASSWORD\"" >> app/ghost/.compose-env
echo "mail__from=\"$SMTP_FROM\"" >> app/ghost/.compose-env
cat app/ghost/.compose.env.tmpl >> app/ghost/.compose-env

View File

@@ -6,4 +6,4 @@ set -a # export everything in the config for later use by envsubst
. $1 # source the apps.config file with then env vars
# substitute template with env var to insert webhooks secret
envsubst < all-apps/instance-control/hooks/hooks.json.tmpl > all-apps/instance-control/hooks/hooks.json
envsubst < app/instance-control/hooks/hooks.json.tmpl > app/instance-control/hooks/hooks.json

View File

@@ -23,17 +23,17 @@ done
export NASSELLA_FULL_DOMAIN="$nassella_subdomain.$ROOT_DOMAIN"
export NASSELLA_FULL_LLDAP_DOMAIN="$NASSELLA_LLDAP_SUBDOMAIN.$nassella_subdomain.$ROOT_DOMAIN"
envsubst < all-apps/nassella/authelia-config/configuration.yml.tmpl > all-apps/nassella/authelia-config/configuration.yml
envsubst < app/nassella/authelia-config/configuration.yml.tmpl > app/nassella/authelia-config/configuration.yml
# write secrets
echo "$NASSELLA_POSTGRES_DB" > all-apps/nassella/postgres_db
echo "$NASSELLA_POSTGRES_USER" > all-apps/nassella/postgres_user
echo "$NASSELLA_POSTGRES_PASSWORD" > all-apps/nassella/postgres_password
echo "$NASSELLA_POSTGRES_DB" > app/nassella/postgres_db
echo "$NASSELLA_POSTGRES_USER" > app/nassella/postgres_user
echo "$NASSELLA_POSTGRES_PASSWORD" > app/nassella/postgres_password
echo "$NASSELLA_AUTHELIA_POSTGRES_DB" > all-apps/nassella/authelia_postgres_db
echo "$NASSELLA_AUTHELIA_POSTGRES_USER" > all-apps/nassella/authelia_postgres_user
echo "$NASSELLA_AUTHELIA_POSTGRES_PASSWORD" > all-apps/nassella/authelia_postgres_password
echo "$NASSELLA_AUTHELIA_POSTGRES_DB" > app/nassella/authelia_postgres_db
echo "$NASSELLA_AUTHELIA_POSTGRES_USER" > app/nassella/authelia_postgres_user
echo "$NASSELLA_AUTHELIA_POSTGRES_PASSWORD" > app/nassella/authelia_postgres_password
echo "$NASSELLA_LLDAP_POSTGRES_DB" > all-apps/nassella/lldap_postgres_db
echo "$NASSELLA_LLDAP_POSTGRES_USER" > all-apps/nassella/lldap_postgres_user
echo "$NASSELLA_LLDAP_POSTGRES_PASSWORD" > all-apps/nassella/lldap_postgres_password
echo "$NASSELLA_LLDAP_POSTGRES_DB" > app/nassella/lldap_postgres_db
echo "$NASSELLA_LLDAP_POSTGRES_USER" > app/nassella/lldap_postgres_user
echo "$NASSELLA_LLDAP_POSTGRES_PASSWORD" > app/nassella/lldap_postgres_password

View File

@@ -23,17 +23,17 @@ done
export NASSELLA_FULL_DOMAIN="$nassella_subdomain.$ROOT_DOMAIN"
export NASSELLA_FULL_LLDAP_DOMAIN="$NASSELLA_LLDAP_SUBDOMAIN.$nassella_subdomain.$ROOT_DOMAIN"
envsubst < all-apps/nassella/lldap-config/lldap_config.toml.tmpl > all-apps/nassella/lldap-config/lldap_config.toml
envsubst < app/nassella/lldap-config/lldap_config.toml.tmpl > app/nassella/lldap-config/lldap_config.toml
# write secrets
echo "$NASSELLA_POSTGRES_DB" > all-apps/nassella/postgres_db
echo "$NASSELLA_POSTGRES_USER" > all-apps/nassella/postgres_user
echo "$NASSELLA_POSTGRES_PASSWORD" > all-apps/nassella/postgres_password
echo "$NASSELLA_POSTGRES_DB" > app/nassella/postgres_db
echo "$NASSELLA_POSTGRES_USER" > app/nassella/postgres_user
echo "$NASSELLA_POSTGRES_PASSWORD" > app/nassella/postgres_password
echo "$NASSELLA_AUTHELIA_POSTGRES_DB" > all-apps/nassella/authelia_postgres_db
echo "$NASSELLA_AUTHELIA_POSTGRES_USER" > all-apps/nassella/authelia_postgres_user
echo "$NASSELLA_AUTHELIA_POSTGRES_PASSWORD" > all-apps/nassella/authelia_postgres_password
echo "$NASSELLA_AUTHELIA_POSTGRES_DB" > app/nassella/authelia_postgres_db
echo "$NASSELLA_AUTHELIA_POSTGRES_USER" > app/nassella/authelia_postgres_user
echo "$NASSELLA_AUTHELIA_POSTGRES_PASSWORD" > app/nassella/authelia_postgres_password
echo "$NASSELLA_LLDAP_POSTGRES_DB" > all-apps/nassella/lldap_postgres_db
echo "$NASSELLA_LLDAP_POSTGRES_USER" > all-apps/nassella/lldap_postgres_user
echo "$NASSELLA_LLDAP_POSTGRES_PASSWORD" > all-apps/nassella/lldap_postgres_password
echo "$NASSELLA_LLDAP_POSTGRES_DB" > app/nassella/lldap_postgres_db
echo "$NASSELLA_LLDAP_POSTGRES_USER" > app/nassella/lldap_postgres_user
echo "$NASSELLA_LLDAP_POSTGRES_PASSWORD" > app/nassella/lldap_postgres_password

View File

@@ -21,11 +21,11 @@ for config_string in ${APP_CONFIGS[@]}; do
done
# write container env file
echo "DOMAIN=\"$nextcloud_subdomain.$ROOT_DOMAIN\"" > all-apps/nextcloud/nextcloud.env
cat all-apps/nextcloud/nextcloud.env.tmpl >> all-apps/nextcloud/nextcloud.env
echo "DOMAIN=\"$nextcloud_subdomain.$ROOT_DOMAIN\"" > app/nextcloud/nextcloud.env
cat app/nextcloud/nextcloud.env.tmpl >> app/nextcloud/nextcloud.env
# write secrets
echo "$NEXTCLOUD_POSTGRES_DB" > all-apps/nextcloud/nextcloud_postgres_db
echo "$NEXTCLOUD_POSTGRES_USER" > all-apps/nextcloud/nextcloud_postgres_user
echo "$NEXTCLOUD_POSTGRES_PASSWORD" > all-apps/nextcloud/nextcloud_postgres_password
echo "$NEXTCLOUD_REDIS_PASSWORD" > all-apps/nextcloud/nextcloud_redis_password
echo "$NEXTCLOUD_POSTGRES_DB" > app/nextcloud/nextcloud_postgres_db
echo "$NEXTCLOUD_POSTGRES_USER" > app/nextcloud/nextcloud_postgres_user
echo "$NEXTCLOUD_POSTGRES_PASSWORD" > app/nextcloud/nextcloud_postgres_password
echo "$NEXTCLOUD_REDIS_PASSWORD" > app/nextcloud/nextcloud_redis_password

View File

@@ -33,6 +33,19 @@ create table user_service_configs(
);
create unique index user_service_configs_user_id_instance_id_idx on user_service_configs (user_id, instance_id);
create table apps(
id bigserial primary key,
app_name varchar(255)
);
create unique index apps_app_name_idx on apps (app_name);
insert into apps(app_name) values ('wg-easy');
insert into apps(app_name) values ('nextcloud');
insert into apps(app_name) values ('nassella');
insert into apps(app_name) values ('log-viewer');
insert into apps(app_name) values ('ghost');
insert into apps(app_name) values ('instance-control');
create table user_selected_apps(
id bigserial primary key,
user_id integer not null references users on delete cascade,
@@ -42,9 +55,11 @@ create table user_selected_apps(
nassella_version varchar(100),
log_viewer_version varchar(100),
ghost_version varchar(100),
instance_control_version varchar(100)
instance_control_version varchar(100),
app_id integer not null references apps on delete cascade,
installed_version varchar(100)
);
create unique index user_selected_apps_user_id_instance_id_idx on user_selected_apps (user_id, instance_id);
create unique index user_selected_apps_app_id_user_id_instance_id_idx on user_selected_apps(user_id, app_id, instance_id);
create table user_app_configs(
id bigserial primary key,

View File

@@ -306,45 +306,45 @@ returning users.user_id;"
`(,(string->symbol (cdr config)) . ,(car config)))
*user-selected-apps-column-map*))
;; (define (update-user-selected-apps conn user-id instance-id app-alist)
;; (let ((valid-keys (map car *user-selected-apps-column-map*)))
;; (for-each (lambda (app)
;; (if (not (memq (car app) valid-keys))
;; (error (string-append "Not a valid app key: " (->string (car app))))))
;; app-alist))
;; (query* conn
;; (string-append
;; "update user_selected_apps set "
;; (string-intersperse
;; (map-in-order (lambda (app i)
;; (conc (alist-ref (car app) *user-selected-apps-column-map*)
;; "=$" i))
;; app-alist
;; (iota (length app-alist) 3))
;; ", ")
;; " where user_id=$1 and instance_id=$2;")
;; `(,user-id
;; ,instance-id
;; ,@(map-in-order cdr app-alist))))
(define (update-user-selected-apps conn user-id instance-id app-alist)
(let ((valid-keys (map car *user-selected-apps-column-map*)))
(for-each (lambda (app)
(if (not (memq (car app) valid-keys))
(error (string-append "Not a valid app key: " (->string (car app))))))
(for-each (lambda (app-version)
(query conn "insert into user_selected_apps (app_id, installed_version)
select apps.id, $2 from apps where apps.app_name=$1
on conflict (app_id)
do update set installed_version=$2 where app_id=apps.id
from apps where apps.app_name=$1;"
(car app-version) (cdr app-version)))
app-alist))
(query* conn
(string-append
"update user_selected_apps set "
(string-intersperse
(map-in-order (lambda (app i)
(conc (alist-ref (car app) *user-selected-apps-column-map*)
"=$" i))
app-alist
(iota (length app-alist) 3))
", ")
" where user_id=$1 and instance_id=$2;")
`(,user-id
,instance-id
,@(map-in-order cdr app-alist))))
(define (get-user-selected-apps conn user-id instance-id)
(let ((res (row-alist
(query conn
(string-append
"select "
(string-intersperse
(map-in-order cdr *user-selected-apps-column-map*)
", ")
" from user_selected_apps where user_id=$1 and instance_id=$2;")
user-id instance-id))))
(map (lambda (item)
(let* ((key (car item))
(value (cdr item))
(config (alist-ref key *user-selected-apps-reverse-column-map*)))
`(,config . ,(if (sql-null? value)
#f
value))))
res)))
(row-map*
(lambda (name version)
(cons (string->symbol name) version))
(query conn "select apps.app_name, usa.installed_version from user_selected_apps usa
join apps on apps.id = usa.app_id
where usa.user_id=$1 and usa.instance_id=$2;"
user-id instance-id)))
(define (update-user-app-config conn user-id instance-id config)
(receive (user-key user-iv auth-user-id)
@@ -501,16 +501,33 @@ returning users.user_id;"
(string-intersperse
(map-in-order (lambda (d) (string-append "d." (cdr d))) *deployments-column-map*)
", ")
", uac.root_domain, uac.config_enc, uac.instance_id, "
"usa.wg_easy_version, usa.nextcloud_version, usa.log_viewer_version, usa.ghost_version, usa.nassella_version, usa.instance_control_version "
", uac.root_domain, uac.config_enc, uac.instance_id "
"from instances as i "
"join (select instance_id, max(id) as id from deployments group by instance_id) d2 "
"on d2.instance_id = i.instance_id "
"join deployments d on d.id = d2.id "
"join user_app_configs uac on uac.user_id = d.user_id and uac.instance_id = d.instance_id "
"join user_selected_apps usa on usa.instance_id = uac.instance_id "
"where i.user_id=$1;")
user-id)))
user-id))
(instance-apps
(row-fold*
(lambda (app-name installed-version instance-id lookup)
(alist-update instance-id
`(((app-name . ,(string->symbol app-name))
(installed-version . ,installed-version))
,@(alist-ref instance-id lookup eqv? '()))
lookup))
'()
(query
conn
"select apps.app_name, usa.installed_version, usa.instance_id from user_selected_apps usa
join apps on apps.id=usa.app_id
where user_id=$1;"
user-id))))
(map
(lambda (instance)
(cons `(apps . ,(alist-ref (alist-ref 'instance-id instance) instance-apps))
instance))
(map
(lambda (row-num)
(map (lambda (item)
@@ -519,12 +536,12 @@ returning users.user_id;"
(config (alist-ref key `((root_domain . root-domain)
(config_enc . config)
(instance_id . instance-id)
(wg_easy_version . wg-easy)
(nextcloud_version . nextcloud)
(ghost_version . ghost)
(nassella_version . nassella)
(log_viewer_version . log-viewer)
(instance_control_version . instance-control)
;; (wg_easy_version . wg-easy)
;; (nextcloud_version . nextcloud)
;; (ghost_version . ghost)
;; (nassella_version . nassella)
;; (log_viewer_version . log-viewer)
;; (instance_control_version . instance-control)
,@*deployments-reverse-column-map*))))
`(,config . ,(if (sql-null? value)
#f
@@ -536,7 +553,7 @@ returning users.user_id;"
read)
value))))))
(row-alist res row-num)))
(iota (row-count res))))))
(iota (row-count res)))))))
(define (update-user-terraform-state conn user-id instance-id state backup)
(receive (user-key user-iv auth-user-id)
@@ -629,7 +646,9 @@ returning users.user_id;"
(define *migrations*
'((0 . "adding-instance-control-app")
(1 . "adding-service-config-digitalocean-volume-size")
(2 . "adding-deployments-instance-backup")))
(2 . "adding-deployments-instance-backup")
(3 . "normalizing-apps")
(4 . "fixing-app-normalization")))
(define (run-pending-migrations conn)
(let* ((migration-ids (sort (map car *migrations*) <))

View File

@@ -0,0 +1,11 @@
begin;
delete from apps;
drop index apps_app_name_idx;
drop index user_selected_apps_app_id_user_id_idx;
drop table apps cascade;
alter table user_selected_apps drop column app_id;
alter table user_selected_apps drop column installed_version;
commit;

View File

@@ -0,0 +1,81 @@
begin;
create table apps(
id bigserial primary key,
app_name varchar(255)
);
create unique index apps_app_name_idx on apps (app_name);
insert into apps(app_name) values ('wg-easy');
insert into apps(app_name) values ('nextcloud');
insert into apps(app_name) values ('nassella');
insert into apps(app_name) values ('log-viewer');
insert into apps(app_name) values ('ghost');
insert into apps(app_name) values ('instance-control');
alter table user_selected_apps add app_id integer references apps on delete cascade;
alter table user_selected_apps add installed_version varchar(100);
create index user_selected_apps_app_id_user_id_idx on user_selected_apps(user_id, app_id);
insert into user_selected_apps (user_id, instance_id, app_id)
select usa.user_id, usa.instance_id, apps.id from user_selected_apps usa
join apps on apps.app_name='wg-easy'
where usa.wg_easy_version is not null;
insert into user_selected_apps (user_id, instance_id, app_id)
select usa.user_id, usa.instance_id, apps.id from user_selected_apps usa
join apps on apps.app_name='nextcloud'
where usa.nextcloud_version is not null;
insert into user_selected_apps (user_id, instance_id, app_id)
select usa.user_id, usa.instance_id, apps.id from user_selected_apps usa
join apps on apps.app_name='nassella'
where usa.nassella_version is not null;
insert into user_selected_apps (user_id, instance_id, app_id)
select usa.user_id, usa.instance_id, apps.id from user_selected_apps usa
join apps on apps.app_name='log-viewer'
where usa.log_viewer_version is not null;
insert into user_selected_apps (user_id, instance_id, app_id)
select usa.user_id, usa.instance_id, apps.id from user_selected_apps usa
join apps on apps.app_name='ghost'
where usa.ghost_version is not null;
insert into user_selected_apps (user_id, instance_id, app_id)
select usa.user_id, usa.instance_id, apps.id from user_selected_apps usa
join apps on apps.app_name='instance-control'
where usa.instance_control_version is not null;
update user_selected_apps set installed_version='15' from apps
where user_selected_apps.app_id=apps.id
and apps.app_name='wg-easy';
update user_selected_apps set installed_version='34' from apps
where user_selected_apps.app_id=apps.id
and apps.app_name='nextcloud';
update user_selected_apps set installed_version='6' from apps
where user_selected_apps.app_id=apps.id
and apps.app_name='ghost';
update user_selected_apps set installed_version='b0.0.1' from apps
where user_selected_apps.app_id=apps.id
and apps.app_name='nassella';
update user_selected_apps set installed_version='b0.0.1' from apps
where user_selected_apps.app_id=apps.id
and apps.app_name='instance-control';
update user_selected_apps set installed_version='10.6.6' from apps
where user_selected_apps.app_id=apps.id
and apps.app_name='log-viewer';
delete from user_selected_apps where wg_easy_version is not null;
delete from user_selected_apps where nextcloud_version is not null;
delete from user_selected_apps where nassella_version is not null;
delete from user_selected_apps where log_viewer_version is not null;
delete from user_selected_apps where ghost_version is not null;
delete from user_selected_apps where instance_control_version is not null;
commit;

View File

@@ -0,0 +1,6 @@
begin;
drop index user_selected_apps_app_id_user_id_idx;
create unique index user_selected_apps_app_id_user_id_instance_id_idx on user_selected_apps(user_id, app_id, instance_id);
end;

View File

@@ -1124,9 +1124,9 @@ chmod -R 777 /opt/keys")))
db
(session-user-id)
instance-id
`((wg-easy . ,(or (and (alist-ref 'wg-easy (current-params)) "15.1.0") (sql-null)))
(nextcloud . ,(or (and (alist-ref 'nextcloud (current-params)) "31.0.8") (sql-null)))
(ghost . ,(or (and (alist-ref 'ghost (current-params)) "6.10.0") (sql-null)))
`((wg-easy . ,(or (and (alist-ref 'wg-easy (current-params)) "15") (sql-null)))
(nextcloud . ,(or (and (alist-ref 'nextcloud (current-params)) "34") (sql-null)))
(ghost . ,(or (and (alist-ref 'ghost (current-params)) "6") (sql-null)))
(nassella . ,(or (and (alist-ref 'nassella (current-params)) "b0.0.1") (sql-null)))
(instance-control . "b0.0.1")))
(update-root-domain db
@@ -1364,7 +1364,7 @@ chmod -R 777 /opt/keys")))
(get-user-selected-apps db (session-user-id) instance-id))))
(app-config . ,(get-user-app-config db (session-user-id) instance-id))
(service-config . ,(get-user-service-config db (session-user-id) instance-id))))))
(selected-apps (cons 'log-viewer (alist-ref 'selected-apps results)))
(selected-apps (alist-ref 'selected-apps results))
(app-config (alist-ref 'app-config results))
(config (alist-ref 'config app-config))
(root-domain (alist-ref 'root-domain app-config))
@@ -1405,16 +1405,13 @@ chmod -R 777 /opt/keys")))
(results
(with-db/transaction
(lambda (db)
`((selected-apps . ,(map
car
(filter cdr
(get-user-selected-apps db (session-user-id) instance-id))))
`((selected-apps . ,(get-user-selected-apps db (session-user-id) instance-id))
(app-config . ,(get-user-app-config db (session-user-id) instance-id))
(service-config . ,(get-user-service-config db (session-user-id) instance-id))
(terraform-state . ,(get-user-terraform-state db (session-user-id) instance-id))
(ssh-pub-key . ,(get-instance-ssh-pub-key db (session-user-id) instance-id))
(restic-password . ,(get-instance-restic-password db (session-user-id) instance-id))))))
(selected-apps (cons 'log-viewer (alist-ref 'selected-apps results)))
(selected-apps (alist-ref 'selected-apps results))
(app-config (alist-ref 'app-config results))
(config (alist-ref 'config app-config))
(root-domain (alist-ref 'root-domain app-config))
@@ -1430,10 +1427,12 @@ chmod -R 777 /opt/keys")))
(write-config-entry (car e) (cdr e)))
`(("ROOT_DOMAIN" . ,root-domain)
("APP_CONFIGS" . ,(string-intersperse
(map (lambda (app)
(conc (if (eq? app 'log-viewer) 'dozzle app)
(map (lambda (app-version)
(conc (if (eq? (car app-version) 'log-viewer) 'dozzle (car app-version))
","
(alist-ref 'subdomain (alist-ref app config))))
(alist-ref 'subdomain (alist-ref (car app-version) config))
","
(if (eq? (car app-version) 'log-viewer) "20" (cdr app-version)))) ;; stop doing this weirdly for dozzle haha
selected-apps)
" "))
("HOST_ADMIN_USER" . ,(alist-ref 'user (alist-ref 'log-viewer config)))
@@ -1501,43 +1500,43 @@ chmod -R 777 /opt/keys")))
(with-db/transaction
(lambda (db)
(update-deployment-progress db deployment-id '((instance-backup . in-progress)))))
(handle-exceptions
exn
(with-db/transaction
(lambda (db)
(update-deployment-progress db deployment-id '((instance-backup . failed)))))
(send-instance-control-command
(alist-ref 'root-domain app-config)
(alist-ref 'subdomain (alist-ref 'instance-control (alist-ref 'config app-config)))
"queue-restic-snapshot-no-restart"
(alist-ref 'webhooks-secret (alist-ref 'instance-control (alist-ref 'config app-config)))
`((path . "/")
(tag . "automated_pre_instance_update")
;; effectively a guid, we just want something unique
(request_id . ,backup-request-id)
(version . 0))))
;; (handle-exceptions
;; exn
;; (with-db/transaction
;; (lambda (db)
;; (update-deployment-progress db deployment-id '((instance-backup . failed)))))
;; (send-instance-control-command
;; (alist-ref 'root-domain app-config)
;; (alist-ref 'subdomain (alist-ref 'instance-control (alist-ref 'config app-config)))
;; "queue-restic-snapshot-no-restart"
;; (alist-ref 'webhooks-secret (alist-ref 'instance-control (alist-ref 'config app-config)))
;; `((path . "/")
;; (tag . "automated_pre_instance_update")
;; ;; effectively a guid, we just want something unique
;; (request_id . ,backup-request-id)
;; (version . 0))))
(thread-start!
(lambda ()
(let ((start-time (time->seconds (current-time))))
(let loop ()
(thread-sleep! 1)
(let* ((status-result
(handle-exceptions
exn
'((status . "error"))
(send-instance-control-command
(alist-ref 'root-domain app-config)
(alist-ref 'subdomain (alist-ref 'instance-control (alist-ref 'config app-config)))
"restic-snapshot-status"
(alist-ref 'webhooks-secret (alist-ref 'instance-control (alist-ref 'config app-config)))
`((request_id . ,backup-request-id)
(version . 0)))))
(complete (string=? (alist-ref 'status status-result) "complete")))
(if (or complete (> (- (time->seconds (current-time)) start-time) 120))
(with-db/transaction
(lambda (db)
(update-deployment-progress db deployment-id `((instance-backup . ,(or (and complete 'complete) 'failed))))))
(loop)))))
;; (let ((start-time (time->seconds (current-time))))
;; (let loop ()
;; (thread-sleep! 1)
;; (let* ((status-result
;; (handle-exceptions
;; exn
;; '((status . "error"))
;; (send-instance-control-command
;; (alist-ref 'root-domain app-config)
;; (alist-ref 'subdomain (alist-ref 'instance-control (alist-ref 'config app-config)))
;; "restic-snapshot-status"
;; (alist-ref 'webhooks-secret (alist-ref 'instance-control (alist-ref 'config app-config)))
;; `((request_id . ,backup-request-id)
;; (version . 0)))))
;; (complete (string=? (alist-ref 'status status-result) "complete")))
;; (if (or complete (> (- (time->seconds (current-time)) start-time) 120))
;; (with-db/transaction
;; (lambda (db)
;; (update-deployment-progress db deployment-id `((instance-backup . ,(or (and complete 'complete) 'failed))))))
;; (loop)))))
(change-directory dir)
(let ((pid (process-run "make apply > make-out 2>&1")))
(with-db/transaction (lambda (db) (update-deployment-in-progress db deployment-id pid)))
@@ -1644,29 +1643,17 @@ chmod -R 777 /opt/keys")))
"deployed successfully"
(alist-ref 'status instance)))
(h3 "Apps")
(ul ,@(filter
identity
(map (lambda (app-map)
(let ((app (car app-map))
(doc-url (cdr app-map)))
(if (or (alist-ref app instance)
(eq? app 'log-viewer))
`((li (a (@ (href ,doc-url)) ,app)
" (v" ,(alist-ref app instance eq? "-") ") "
(ul ,@(map (lambda (app-info)
(let ((app (alist-ref 'app-name app-info)))
`((li (a (@ (href "#")) ,app) ;; TODO documentation URL
" (v" ,(alist-ref 'installed-version app-info) ") "
(a (@ (href "https://"
,(alist-ref 'subdomain (alist-ref app config))
"." ,root-domain))
,(alist-ref 'subdomain (alist-ref app config))
"." ,root-domain)))
#f)))
;; TODO update links
'((wg-easy . "https://wg-easy.github.io/wg-easy/Pre-release/")
(nextcloud . "https://nextcloud.com/support/")
(ghost . "https://nextcloud.com/support/")
(nassella . "https://nextcloud.com/support/")
(log-viewer . "https://nextcloud.com/support/")
;; (instance-control . "https://nextcloud.com/support/")
))))
"." ,root-domain)))))
(or (alist-ref 'apps instance) ;; TODO this really should not happen maybe?
'())))
(h3 "Actions")
(ul (li (a (@ (href "/config/wizard/services/"
,(alist-ref 'instance-id instance)))