From 2721249a892219d25bfc0fe4936b718a55c18f0a Mon Sep 17 00:00:00 2001 From: Thomas Hintz Date: Sat, 25 Jul 2026 10:02:25 -0700 Subject: [PATCH] User selects backblaze bucket instead of providing bucket url --- src/db-init.sql | 4 +- src/db.scm | 7 +- ...adding-backblaze-bucket-name-and-id-up.sql | 2 + src/nassella.scm | 220 +++++++++++------- 4 files changed, 152 insertions(+), 81 deletions(-) create mode 100644 src/migrations/8-adding-backblaze-bucket-name-and-id-up.sql diff --git a/src/db-init.sql b/src/db-init.sql index 6a6cca4..458f49c 100644 --- a/src/db-init.sql +++ b/src/db-init.sql @@ -45,7 +45,9 @@ create table user_service_configs( digitalocean_volume_size integer, backblaze_application_key_enc varchar(255), backblaze_key_id_enc varchar(255), - backblaze_bucket_url_enc varchar(255) + backblaze_bucket_url_enc varchar(255), -- deprecated 072526, replaced by bucket_name and bucket_id + backblaze_bucket_name_enc varchar(255), + backblaze_bucket_id_enc varchar(255) ); create unique index user_service_configs_user_id_instance_id_idx on user_service_configs (user_id, instance_id); diff --git a/src/db.scm b/src/db.scm index 904a209..b94fb2d 100644 --- a/src/db.scm +++ b/src/db.scm @@ -248,7 +248,9 @@ returning users.user_id;" (digitalocean-volume-size . ("digitalocean_volume_size" #f)) (backblaze-application-key . ("backblaze_application_key_enc" #t)) (backblaze-key-id . ("backblaze_key_id_enc" #t)) - (backblaze-bucket-url . ("backblaze_bucket_url_enc" #t)))) + (backblaze-bucket-url . ("backblaze_bucket_url_enc" #t)) + (backblaze-bucket-name . ("backblaze_bucket_name_enc" #t)) + (backblaze-bucket-id . ("backblaze_bucket_id_enc" #t)))) (define *user-service-configs-reverse-column-map* (map (lambda (config) @@ -657,7 +659,8 @@ where user_id=$1;" (4 . "fixing-app-normalization") (5 . "adding-wordpress-app") (6 . "adding-lldap-app") - (7 . "adding-authelia-app"))) + (7 . "adding-authelia-app") + (8 . "adding-backblaze-bucket-name-and-id"))) (define (run-pending-migrations conn) (let* ((migration-ids (sort (map car *migrations*) <)) diff --git a/src/migrations/8-adding-backblaze-bucket-name-and-id-up.sql b/src/migrations/8-adding-backblaze-bucket-name-and-id-up.sql new file mode 100644 index 0000000..aab3557 --- /dev/null +++ b/src/migrations/8-adding-backblaze-bucket-name-and-id-up.sql @@ -0,0 +1,2 @@ +alter table user_service_configs add backblaze_bucket_name_enc varchar(255); +alter table user_service_configs add backblaze_bucket_id_enc varchar(255); diff --git a/src/nassella.scm b/src/nassella.scm index ec4f120..1c3a040 100644 --- a/src/nassella.scm +++ b/src/nassella.scm @@ -670,6 +670,8 @@ h1, h2, h3, h4, h5, h6 { #f read-json)) +;; request response looks like +;; (((accountId . "xxx") (bucketId . "xxx") (bucketInfo) (bucketName . "xxx") (bucketType . "allPrivate") (corsRules) (defaultServerSideEncryption (isClientAuthorizedToRead . #t) (value (algorithm . null) (mode . null))) (fileLockConfiguration (isClientAuthorizedToRead . #t) (value (defaultRetention (mode . null) (period . null)) (isFileLockEnabled . #f))) (lifecycleRules ((daysFromHidingToDeleting . 1) (daysFromStartingToCancelingUnfinishedLargeFiles . null) (daysFromUploadingToHiding . null) (fileNamePrefix . ""))) (options "s3") (replicationConfiguration (isClientAuthorizedToRead . #t) (value . null)) (revision . 3)) ((accountId . "yyy") (bucketId . "yyy") (bucketInfo) (bucketName . "yyy") (bucketType . "allPrivate") (corsRules) (defaultServerSideEncryption (isClientAuthorizedToRead . #t) (value (algorithm . null) (mode . null))) (fileLockConfiguration (isClientAuthorizedToRead . #t) (value (defaultRetention (mode . null) (period . null)) (isFileLockEnabled . #f))) (lifecycleRules ((daysFromHidingToDeleting . 1) (daysFromStartingToCancelingUnfinishedLargeFiles . null) (daysFromUploadingToHiding . null) (fileNamePrefix . ""))) (options "s3") (replicationConfiguration (isClientAuthorizedToRead . #t) (value . null)) (revision . 3))) (define (b2-list-buckets api-url account-token account-id) (with-input-from-request (make-request method: 'POST @@ -681,6 +683,28 @@ h1, h2, h3, h4, h5, h6 { `((accountId . ,account-id)))) read-json)) +;; a convenience method used to retrieve the buckets either +;; from the key details or via b2-list-buckets. +;; depending on the key config and permissions the way you +;; access the list of buckets you can access will vary +(define (b2-account-buckets key-id key) + (let* ((auth (b2-authorize-account key-id key)) + (authorization-token (alist-ref 'authorizationToken auth)) + (account-id (alist-ref 'accountId auth)) + (api-url (alist-ref 'apiUrl (alist-ref 'storageApi (alist-ref 'apiInfo auth)))) + (buckets (alist-ref 'buckets (alist-ref 'allowed (alist-ref 'storageApi (alist-ref 'apiInfo auth))))) + (listed-buckets (and (eq? buckets 'null) ;; if the key does not contain a bucket restriction it should be able to use the api to list buckets + (alist-ref 'buckets (b2-list-buckets api-url authorization-token account-id))))) + (if listed-buckets + (map (lambda (bucket-info) + `((name . ,(alist-ref 'bucketName bucket-info)) + (id . ,(alist-ref 'bucketId bucket-info)))) + listed-buckets) + (map (lambda (bucket-info) + `((name . ,(alist-ref 'name bucket-info)) + (id . ,(alist-ref 'id bucket-info)))) + buckets)))) + (define (b2-authorization-details key-id key bucket-name) (let* ((auth (b2-authorize-account key-id key)) (authorization-token (alist-ref 'authorizationToken auth)) @@ -833,67 +857,79 @@ h1, h2, h3, h4, h5, h6 { `((success . #f) (errors ((message . ,(alist-ref 'message res))))))))) -(define (test-backblaze-connection user-id instance-id key-id application-key bucket-url) - (let* ((password-path (conc "restic-password-" user-id "-" instance-id)) - (restic-password - (with-db/transaction - (lambda (db) - (get-instance-restic-password db user-id instance-id))))) - (dynamic-wind - (lambda () - (with-output-to-file password-path (lambda () (display restic-password)))) - (lambda () - ;; restic retries (indefinitely?) if the connection can't be made so - ;; we pass it through timeout to ensure it does not hang - (receive (in-port out-port pid err-port) - (cond-expand - (dev - (process* "timeout" `("--preserve-status" "8s" "docker" "run" "--rm" "--volume" - ,(conc (current-directory) "/" password-path ":/restic-password") - "-e" ,(conc "AWS_ACCESS_KEY_ID=" key-id) - "-e" ,(conc "AWS_SECRET_ACCESS_KEY=" application-key) - "-i" "restic/restic:0.18.0" "cat" "config" - "--repo" ,(conc "s3:" bucket-url) - "--password-file" "/restic-password" - "--json"))) - (else - (process* "/bin/timeout" - `("--preserve-status" "8s" "/bin/restic" "cat" "config" - "--repo" ,(conc "s3:" bucket-url) - "--password-file" ,(conc (current-directory) "/" password-path) - "--json") - `(("AWS_ACCESS_KEY_ID" . ,key-id) - ("AWS_SECRET_ACCESS_KEY" . ,application-key))))) - (let ((thread - (thread-start! - (lambda () - (let loop ((i 0)) - (thread-sleep! 1) - ;; We do a non-blocking wait here so that we don't - ;; block the entire web process. - (receive (wait-pid exit-normal status) (process-wait pid #t) - (if (= wait-pid 0) ;; wait-pid is 0 until the process has finished - (if (< i 12) ;; 12s timeout - (loop (+ i 1)) - `((success . #f) - (errors ((message . "timeout trying to connect to backblaze"))))) - (if exit-normal - (let ((res (with-input-from-port in-port read-json)) - (err (with-input-from-port err-port read-json))) - ;; status 10 is what is returned if we can connect but the repo - ;; is not initialized yet - (if (or res (and err (= status 10))) - `((success . #t) - (result . ,res)) - `((success . #f) - (errors ((message . "abnormal exit - check Key ID, Application ID, and Bucket URL")))))) - `((success . #f) - (errors ((message . "abnormal exit - check Key ID, Application ID, and Bucket URL")))) - )))))))) - (thread-join! thread)))) - (lambda () - (handle-exceptions exn 'ignore - (delete-file password-path)))))) +(define (test-backblaze-connection key-id application-key) + (handle-exceptions + exn + `((success . #f) + (errors ((message . ,(alist-ref 'code (read-json (get-condition-property exn 'client-error 'body))))))) + (receive (data request-uri response) (b2-authorize-account key-id application-key) + (if (alist-ref 'authorizationToken data) + `((success . #t) + (result ,data)) + `((success . #f) + (result ,data)))))) + +;; (define (test-backblaze-connection user-id instance-id key-id application-key bucket-url) +;; (let* ((password-path (conc "restic-password-" user-id "-" instance-id)) +;; (restic-password +;; (with-db/transaction +;; (lambda (db) +;; (get-instance-restic-password db user-id instance-id))))) +;; (dynamic-wind +;; (lambda () +;; (with-output-to-file password-path (lambda () (display restic-password)))) +;; (lambda () +;; ;; restic retries (indefinitely?) if the connection can't be made so +;; ;; we pass it through timeout to ensure it does not hang +;; (receive (in-port out-port pid err-port) +;; (cond-expand +;; (dev +;; (process* "timeout" `("--preserve-status" "8s" "docker" "run" "--rm" "--volume" +;; ,(conc (current-directory) "/" password-path ":/restic-password") +;; "-e" ,(conc "AWS_ACCESS_KEY_ID=" key-id) +;; "-e" ,(conc "AWS_SECRET_ACCESS_KEY=" application-key) +;; "-i" "restic/restic:0.18.0" "cat" "config" +;; "--repo" ,(conc "s3:" bucket-url) +;; "--password-file" "/restic-password" +;; "--json"))) +;; (else +;; (process* "/bin/timeout" +;; `("--preserve-status" "8s" "/bin/restic" "cat" "config" +;; "--repo" ,(conc "s3:" bucket-url) +;; "--password-file" ,(conc (current-directory) "/" password-path) +;; "--json") +;; `(("AWS_ACCESS_KEY_ID" . ,key-id) +;; ("AWS_SECRET_ACCESS_KEY" . ,application-key))))) +;; (let ((thread +;; (thread-start! +;; (lambda () +;; (let loop ((i 0)) +;; (thread-sleep! 1) +;; ;; We do a non-blocking wait here so that we don't +;; ;; block the entire web process. +;; (receive (wait-pid exit-normal status) (process-wait pid #t) +;; (if (= wait-pid 0) ;; wait-pid is 0 until the process has finished +;; (if (< i 12) ;; 12s timeout +;; (loop (+ i 1)) +;; `((success . #f) +;; (errors ((message . "timeout trying to connect to backblaze"))))) +;; (if exit-normal +;; (let ((res (with-input-from-port in-port read-json)) +;; (err (with-input-from-port err-port read-json))) +;; ;; status 10 is what is returned if we can connect but the repo +;; ;; is not initialized yet +;; (if (or res (and err (= status 10))) +;; `((success . #t) +;; (result . ,res)) +;; `((success . #f) +;; (errors ((message . "abnormal exit - check Key ID, Application ID, and Bucket URL")))))) +;; `((success . #f) +;; (errors ((message . "abnormal exit - check Key ID, Application ID, and Bucket URL")))) +;; )))))))) +;; (thread-join! thread)))) +;; (lambda () +;; (handle-exceptions exn 'ignore +;; (delete-file password-path)))))) (define (deployment-directory user-id instance-id) (string-append "deploy-" (number->string user-id) "-" (->string instance-id))) @@ -1217,9 +1253,7 @@ chmod -R 777 /opt/keys"))) (type "password") (value ,(alist-ref 'backblaze-application-key config)))) (Field (@ (name "backblaze-key-id") (label ("Key ID")) (type "password") - (value ,(alist-ref 'backblaze-key-id config)))) - (Field (@ (name "backblaze-bucket-url") (label ("Bucket URL")) (type "password") - (value ,(alist-ref 'backblaze-bucket-url config))))) + (value ,(alist-ref 'backblaze-key-id config))))) (Form-Nav))))))) (post "/config/wizard/services-submit/:id" @@ -1235,8 +1269,7 @@ chmod -R 777 /opt/keys"))) (cloudflare-zone-id . ,(alist-ref 'cloudflare-zone-id (current-params))) (digitalocean-api-token . ,(alist-ref 'digitalocean-api-token (current-params))) (backblaze-application-key . ,(alist-ref 'backblaze-application-key (current-params))) - (backblaze-key-id . ,(alist-ref 'backblaze-key-id (current-params))) - (backblaze-bucket-url . ,(alist-ref 'backblaze-bucket-url (current-params))))))) + (backblaze-key-id . ,(alist-ref 'backblaze-key-id (current-params))))))) (redirect (conc "/config/wizard/services-success/" instance-id)))) (get/widgets @@ -1250,10 +1283,8 @@ chmod -R 777 /opt/keys"))) (alist-ref 'cloudflare-zone-id service-config) (alist-ref 'cloudflare-account-id service-config))) (digitalocean-result (test-digitalocean-connection (alist-ref 'digitalocean-api-token service-config))) - (backblaze-result (test-backblaze-connection (session-user-id) instance-id - (alist-ref 'backblaze-key-id service-config) - (alist-ref 'backblaze-application-key service-config) - (alist-ref 'backblaze-bucket-url service-config)))) + (backblaze-result (test-backblaze-connection (alist-ref 'backblaze-key-id service-config) + (alist-ref 'backblaze-application-key service-config)))) `(App (Configuration-Wizard (@ (step "Services")) @@ -1315,7 +1346,17 @@ chmod -R 777 /opt/keys"))) (filter cdr (get-user-selected-apps db (session-user-id) instance-id)))) (app-config . ,(get-user-app-config db (session-user-id) instance-id)) - (service-config . ,(get-user-service-config db (session-user-id) instance-id))))))) + (service-config . ,(get-user-service-config db (session-user-id) instance-id)))))) + (backblaze-buckets (b2-account-buckets (alist-ref 'backblaze-key-id (alist-ref 'service-config results)) + (alist-ref 'backblaze-application-key (alist-ref 'service-config results)))) + (backblaze-s3-api-url (alist-ref + 's3ApiUrl + (alist-ref + 'storageApi + (alist-ref + 'apiInfo + (b2-authorize-account (alist-ref 'backblaze-key-id (alist-ref 'service-config results)) + (alist-ref 'backblaze-application-key (alist-ref 'service-config results)))))))) `(App (Configuration-Wizard (@ (step "Apps")) @@ -1327,24 +1368,37 @@ chmod -R 777 /opt/keys"))) (Field (@ (element select) (name "root-domain")) ,@(map (lambda (domain) `(option (@ (value ,domain) + (required #t) ,@(if (equal? domain (alist-ref 'root-domain (alist-ref 'app-config results))) '(selected) '())) ,domain)) (get-cloudflare-domains (alist-ref 'cloudflare-api-token - (alist-ref 'service-config results)))) - )) + (alist-ref 'service-config results)))))) + (Fieldset + (@ (title "Backblaze Bucket")) + (Field (@ (element select) (name "backblaze-bucket-id")) + ,@(map (lambda (bucket) + `(option (@ (value ,(string-append (alist-ref 'id bucket) "|" (alist-ref 'name bucket))) + (required #t) + ,@(if (equal? (alist-ref 'id bucket) + (alist-ref 'backblaze-bucket-id (alist-ref 'service-config results))) + '(selected) + '())) + ,(alist-ref 'name bucket))) + backblaze-buckets))) + (input (@ (type "hidden") (name "backblaze-s3-url") (value ,backblaze-s3-api-url))) (Fieldset (@ (title "Selected Apps")) (Field (@ (name "wg-easy") (type "checkbox") (label ("WG Easy")) (checked ,(member 'wg-easy (alist-ref 'selected-apps results))))) (Field (@ (name "nextcloud") (type "checkbox") (label ("NextCloud")) (checked ,(member 'nextcloud (alist-ref 'selected-apps results))))) (Field (@ (name "ghost") (type "checkbox") (label ("Ghost")) (checked ,(member 'ghost (alist-ref 'selected-apps results))))) ,@(cond-expand - (dev - `((Field (@ (name "nassella") (type "checkbox") (label ("Nassella")) (checked ,(member 'nassella (alist-ref 'selected-apps results))))))) - (else - '())) + (dev + `((Field (@ (name "nassella") (type "checkbox") (label ("Nassella")) (checked ,(member 'nassella (alist-ref 'selected-apps results))))))) + (else + '())) (Field (@ (name "wordpress") (type "checkbox") (label ("Wordpress")) (checked ,(member 'wordpress (alist-ref 'selected-apps results))))) (Field (@ (name "log-viewer") (type "checkbox") (label ("Log Viewer")) (checked #t) (disabled "disabled"))) (Field (@ (name "lldap") (type "checkbox") (label ("Admin LLDAP")) (checked #t) (disabled "disabled"))) @@ -1354,7 +1408,10 @@ chmod -R 777 /opt/keys"))) (Form-Nav (@ (back-to ,(conc "/config/wizard/services-success/" instance-id)))))))))) (post "/config/wizard/apps-submit/:id" - (let ((instance-id (alist-ref "id" (current-params) equal?))) + (let* ((instance-id (alist-ref "id" (current-params) equal?)) + (bucket-values (string-split (alist-ref 'backblaze-bucket-id (current-params)) "|")) + (bucket-name (cadr bucket-values)) + (bucket-id (car bucket-values))) (with-db/transaction (lambda (db) (update-user-selected-apps @@ -1371,6 +1428,13 @@ chmod -R 777 /opt/keys"))) (authelia . "4") (instance-control . "b0.0.1") (log-viewer . "20")))) + (update-user-service-config + db + (session-user-id) + instance-id + `((backblaze-bucket-name . ,bucket-name) + (backblaze-bucket-id . ,bucket-id) + (backblaze-bucket-url . ,(string-append (alist-ref 'backblaze-s3-url (current-params)) "/" bucket-name)))) (update-root-domain db (session-user-id) instance-id