Updating gitignore.

.gitignore

@@ -1 +1,14 @@
-*~
+*~
+
+.terraform
+.terraform.lock.hcl
+
+flatcar/flatcar_production_qemu_image.img
+flatcar/flatcar_production_qemu_image.img.fresh
+
+ignition.json
+
+production.tfvars
+
+terraform.tfstate
+terraform.tfstate.backup

cl/machine-mynode.yaml.tmpl

@@ -1,18 +0,0 @@
----
-variant: flatcar
-version: 1.1.0
-passwd:
-  users:
-    - name: core
-      ssh_authorized_keys: ${ssh_keys}
-storage:
-  files:
-    - path: /home/core/works
-      mode: 0755
-      contents:
-        inline: |
-          #!/bin/bash
-          set -euo pipefail
-          # This script demonstrates how templating and variable substitution works when using Terraform templates for Container Linux Configs.
-          hostname="$(hostname)"
-          echo My name is ${name} and the hostname is $${hostname} and this is updated, again 5x!

flatcar/Makefile

@@ -2,7 +2,7 @@
 ROOT_DIR:=$(shell dirname "$(realpath $(firstword $(MAKEFILE_LIST)))")
 
 ignition:
-	cat cl.yaml | sudo docker run --rm --volume ${ROOT_DIR}:/pwd --workdir /pwd -i quay.io/coreos/butane:latest -d /pwd > ignition.json
+	cat cl.yaml | sudo docker run --rm --volume /home/tjhintz/.ssh/id_rsa.pub:/pwd/ssh-keys --volume ${ROOT_DIR}:/pwd --workdir /pwd -i quay.io/coreos/butane:latest -d /pwd > ignition.json
 
 build:
 	cp --reflink=auto flatcar_production_qemu_image.img.fresh flatcar_production_qemu_image.img

flatcar/app/Makefile

@@ -0,0 +1,2 @@
+run:
+	sudo docker-compose -f docker-compose.yaml $(find . -mindepth 2 -maxdepth 2 -type f -name docker-compose.yaml -exec echo -f {} \;) up

flatcar/app/app.service

@@ -4,8 +4,8 @@ After=docker.service
 Requires=docker.service
 [Service]
 TimeoutStartSec=0
-ExecStart=/usr/bin/docker compose -f /home/core/docker-compose.yaml up
+ExecStart=/bin/bash -c '/usr/bin/docker compose -f /app/docker-compose.yaml $(find /app -mindepth 2 -maxdepth 2 -type f -name docker-compose.yaml -exec echo -f {} \;) up'
-ExecStop=/usr/bin/docker compose -f /home/core/docker-compose.yaml stop
+ExecStop=/bin/bash -c '/usr/bin/docker compose -f /app/docker-compose.yaml $(find /app -mindepth 2 -maxdepth 2 -type f -name docker-compose.yaml -exec echo -f {} \;) stop'
 
 Restart=always
 RestartSec=5s

flatcar/app/docker-compose.yaml

@@ -3,20 +3,24 @@ services:
   lb:
     image: docker.io/caddy:2
     volumes:
-      - /var/lb/caddy:/etc/caddy
+#      - /app/lb:/etc/caddy
+      - ./lb/:/etc/caddy
+      - config:/config
+      - data:/data
     networks:
       - lb
     restart: unless-stopped
     ports:
       - "443:443"
       - "80:80"
-  thintz-com:
+  # thintz-com:
-    image: docker.io/caddy:2
+  #   image: docker.io/caddy:2
-    volumes:
+  #   volumes:
-      - /var/thintz-com/caddy:/etc/caddy
+  #     #- /app/thintz-com/caddy:/etc/caddy
-    networks:
+  #     - ./thintz-com/:/etc/caddy
-      - lb
+  #   networks:
-    restart: unless-stopped
+  #     - lb
+  #   restart: unless-stopped
   nginx:
     image: nginx
     restart: unless-stopped
@@ -24,3 +28,6 @@ services:
       - lb
 networks:
   lb:
+volumes:
+  config:
+  data:

flatcar/app/lb/Caddyfile

@@ -1,4 +1,17 @@
-localhost
+wg-easy1.nassella.cc {
+  reverse_proxy http://wg-easy:80
 
-reverse_proxy http://nginx
+  # tls internal
-log
+  # x
+  # log
+}
+
+nextcloud1.nassella.cc {
+  reverse_proxy http://nextcloud:80
+  # tls internal
+}
+
+root.nassella.cc {
+  reverse_proxy http://nginx:80
+  # tls internal
+}

flatcar/app/nextcloud/docker-compose.yaml

@@ -0,0 +1,50 @@
+version: '3'
+
+services:
+  db:
+    image: postgres
+    shm_size: 128mb
+    restart: always
+    volumes:
+      - db:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_DB=nextcloud
+      - POSTGRES_USER=nextcloud
+      - POSTGRES_PASSWORD=password
+    networks:
+      - internal
+  redis:
+    image: redis:alpine
+    restart: always
+    networks:
+      - internal
+  nextcloud:
+    image: nextcloud
+    environment:
+      - POSTGRES_HOST=db
+      - POSTGRES_DB=nextcloud
+      - POSTGRES_USER=nextcloud
+      - POSTGRES_PASSWORD=password
+      - NEXTCLOUD_ADMIN_PASSWORD=password
+      - NEXTCLOUD_ADMIN_USER=admin
+      - REDIS_HOST=redis
+      - NEXTCLOUD_TRUSTED_DOMAINS=nextcloud1.nassella.cc
+    ports:
+      - "8080:80"
+    depends_on:
+      - redis
+      - db
+    networks:
+      - lb
+      - internal
+    volumes:
+      - nextcloud:/var/www
+    restart: unless-stopped
+networks:
+  lb:
+  internal:
+    driver: bridge
+    internal: true
+volumes:
+  db:
+  nextcloud:

flatcar/app/wg-easy/docker-compose.yaml

@@ -0,0 +1,37 @@
+version: '3'
+services:
+  wg-easy:
+    image: ghcr.io/wg-easy/wg-easy:15
+    environment:
+      - PORT=80
+    ports:
+      - "51820:51820/udp"
+    networks:
+      lb:
+      wg:
+        ipv4_address: 10.42.42.42
+#        ipv6_address: fdcc:ad94:bacf:61a3::2a
+    volumes:
+      - etc_wireguard:/etc/wireguard
+      - /lib/modules:/lib/modules:ro
+    restart: unless-stopped
+    cap_add:
+      - NET_ADMIN
+      - SYS_MODULE
+    sysctls:
+      - net.ipv4.ip_forward=1
+      - net.ipv4.conf.all.src_valid_mark=1
+      - net.ipv6.conf.all.disable_ipv6=0
+      - net.ipv6.conf.all.forwarding=1
+      - net.ipv6.conf.default.forwarding=1
+networks:
+  lb:
+  wg:
+    driver: bridge
+    ipam:
+      driver: default
+      config:
+        - subnet: 10.42.42.0/24
+        - subnet: fdcc:ad94:bacf:61a3::/64
+volumes:
+  etc_wireguard:

flatcar/cl.yaml

@@ -3,10 +3,20 @@ version: 1.1.0
 passwd:
   users:
     - name: core
-      ssh_authorized_keys:
+      ssh_authorized_keys_local:
-        - ""
+        - /ssh-keys
 systemd:
   units:
+    - name: var-lib-docker-volumes.mount
+      enabled: true
+      contents: |
+        [Mount]
+        What=/dev/disk/by-partlabel/appstorage
+        Where=/var/lib/docker/volumes
+        Type=ext4
+
+        [Install]
+        RequiredBy=local-fs.target
     - name: app.service
       enabled: true
       contents_local: app/app.service
@@ -14,6 +24,13 @@ systemd:
     ### https://flatcar.github.io/sysext-bakery/docker_compose/
     - name: systemd-sysupdate.timer
       enabled: true
+    - name: docker.service
+      dropins:
+        - name: 10-wait-docker.conf
+          contents: |
+            [Unit]
+            After=var-lib-docker-volumes.mount
+            Requires=var-lib-docker-volumes.mount
     - name: systemd-sysupdate.service
       dropins:
         - name: docker-compose.conf
@@ -24,14 +41,22 @@ systemd:
             ExecStartPost=/usr/bin/sh -c "readlink --canonicalize /etc/extensions/docker-compose.raw > /tmp/docker-compose-new"
             ExecStartPost=/usr/bin/sh -c "if ! cmp --silent /tmp/docker-compose /tmp/docker-compose-new; then touch /run/reboot-required; fi"
 
+# device: /dev/disk/by-label/appstorage
 storage:
+  disks:
+    - device: /dev/sda
+      wipe_table: false
+      partitions:
+        - label: appstorage
+          number: 1
+  filesystems:
+    - device: /dev/disk/by-partlabel/appstorage
+      format: ext4
+      wipe_filesystem: false
+  trees:
+    - path: /app
+      local: app
   files:
-    - path: /var/lb/caddy/Caddyfile
-      contents:
-        local: app/lb/Caddyfile
-    - path: /var/thintz-com/caddy/Caddyfile
-      contents:
-        local: app/thintz-com/Caddyfile
     ### docker-compose sysext
     ### https://flatcar.github.io/sysext-bakery/docker_compose/
     - path: /opt/extensions/docker-compose/docker-compose-2.34.0-x86-64.raw
@@ -44,14 +69,6 @@ storage:
     - path: /etc/sysupdate.d/noop.conf
       contents:
         source: https://extensions.flatcar.org/extensions/noop.conf
-    - path: /home/core/docker-compose.yaml
-      mode: 0644
-      user:
-        name: core
-      group:
-        name: core
-      contents:
-        local: app/docker-compose.yaml
   links:
     - target: /opt/extensions/docker-compose/docker-compose-2.34.0-x86-64.raw
       path: /etc/extensions/docker-compose.raw

main.tf

@@ -12,6 +12,14 @@ terraform {
       source  = "hashicorp/null"
       version = "3.2.4"
     }
+    time = {
+      source  = "hashicorp/time"
+      version = "0.13.1"
+    }
+    cloudflare = {
+      source = "cloudflare/cloudflare"
+      version = "5.8.2"
+    }
   }
 }
 
@@ -20,11 +28,6 @@ variable "do_token" {
   type        = string
 }
 
-variable "machines" {
-  type        = list(string)
-  description = "Machine names, corresponding to machine-NAME.yaml.tmpl files"
-}
-
 variable "cluster_name" {
   type        = string
   description = "Cluster name used as prefix for the machine names"
@@ -51,10 +54,39 @@ variable "flatcar_stable_version" {
   description = "The Flatcar Stable release you want to use for the initial installation, e.g., 2605.12.0"
 }
 
+variable "cloudflare_zone_id" {
+  type = string
+  description = "Cloudflare zone ID"
+}
+
+variable "cloudflare_account_id" {
+  type = string
+  description = "Cloudflare account id"
+}
+
+variable "cloudflare_api_token" {
+  type = string
+  description = "Cloudflare api token"
+}
+
+variable "domain" {
+  type = string
+  description = "Root domain to use"
+}
+
+variable "subdomains" {
+  type = list
+  description = "Subdomains to setup"
+}
+
 provider "digitalocean" {
   token = var.do_token
 }
 
+provider "cloudflare" {
+  api_token = var.cloudflare_api_token
+}
+
 resource "digitalocean_ssh_key" "first" {
   name       = var.cluster_name
   public_key = var.ssh_keys.0
@@ -67,49 +99,68 @@ resource "digitalocean_custom_image" "flatcar" {
 }
 
 resource "digitalocean_reserved_ip" "machine" {
-  for_each  = toset(var.machines)
+  region     = digitalocean_droplet.machine.region
-  region     = digitalocean_droplet.machine[each.key].region
+}
+
+resource "cloudflare_dns_record" "root" {
+  zone_id = var.cloudflare_zone_id
+  name    = "@"
+  content   = digitalocean_reserved_ip.machine.ip_address
+  type    = "A"
+  proxied = false
+  ttl = 300
+}
+
+resource "cloudflare_dns_record" "subdomains" {
+  for_each  = toset(var.subdomains)
+  zone_id = var.cloudflare_zone_id
+  name    = each.key
+  content   = var.domain
+  type    = "CNAME"
+  proxied = false
+  ttl = 300
+}
+
+resource "digitalocean_volume" "machine" {
+  region    = var.datacenter
+  name      = "${var.cluster_name}"
+  size                    = 60
+  initial_filesystem_type = "ext4"
+  initial_filesystem_label = "appstorage"
+  description             = "persistent storage for docker apps"
 }
 
 resource "digitalocean_droplet" "machine" {
-  for_each  = toset(var.machines)
+  name      = "${var.cluster_name}"
-  name      = "${var.cluster_name}-${each.key}"
   image     = digitalocean_custom_image.flatcar.id
   region    = var.datacenter
   size      = var.server_type
   ssh_keys  = [digitalocean_ssh_key.first.fingerprint]
-  # user_data = data.ct_config.machine-ignitions[each.key].rendered
   user_data = file("flatcar/ignition.json")
   lifecycle {
     create_before_destroy = true
   }
 }
 
+resource "time_sleep" "wait_10_seconds" {
+  create_duration = "10s"
+}
+
 resource "digitalocean_reserved_ip_assignment" "machine" {
-  for_each  = toset(var.machines)
+  ip_address = digitalocean_reserved_ip.machine.ip_address
-  ip_address = digitalocean_reserved_ip.machine[each.key].ip_address
+  droplet_id = digitalocean_droplet.machine.id
-  droplet_id = digitalocean_droplet.machine[each.key].id
+  # sometimes digital ocean throws an error for this resource
+  # saying the droplet has a pending event so we delay a few seconds
+  depends_on = [time_sleep.wait_10_seconds]
 }
 
-# data "ct_config" "machine-ignitions" {
+resource "digitalocean_volume_attachment" "machine" {
-#   for_each = toset(var.machines)
+  droplet_id = digitalocean_droplet.machine.id
-#   content = templatefile("${path.module}/cl/machine-${each.key}.yaml.tmpl", {
+  volume_id  = digitalocean_volume.machine.id
-#     ssh_keys = jsonencode(var.ssh_keys),
+}
-#     name = each.key
-#   })
-#   strict = true
-# }
 
-output "ip-addresses" {
+output "domain-mappings" {
   value = {
-    for key in var.machines :
+    "${var.domain}" = digitalocean_reserved_ip.machine.ip_address
-    "${var.cluster_name}-${key}" => digitalocean_droplet.machine[key].ipv4_address
-  }
-}
-
-output "reserved-ip-addresses" {
-  value = {
-    for key in var.machines :
-    "${var.cluster_name}-${key}" => digitalocean_reserved_ip.machine[key].ip_address
   }
 }

template.tfvars

@@ -1,7 +1,13 @@
+domain = ""
+subdomains = ["wg-easy"]
+
 do_token = "" # token from "API" settings on DigitalOcean
 
+cloudflare_api_token = ""
+cloudflare_zone_id = ""
+cloudflare_account_id = ""
+
 cluster_name           = "mycluster"
-machines               = ["mynode"]
 datacenter             = "sfo3"
 ssh_keys               = [""] # paste contents of id_rsa.pub
 flatcar_stable_version = "4230.2.1"