User selects backblaze bucket instead of providing bucket url

src/db-init.sql

@@ -45,7 +45,9 @@ create table user_service_configs(
        digitalocean_volume_size integer,
        backblaze_application_key_enc varchar(255),
        backblaze_key_id_enc varchar(255),
-       backblaze_bucket_url_enc varchar(255)
+       backblaze_bucket_url_enc varchar(255), -- deprecated 072526, replaced by bucket_name and bucket_id
+       backblaze_bucket_name_enc varchar(255),
+       backblaze_bucket_id_enc varchar(255)
        );
 create unique index user_service_configs_user_id_instance_id_idx on user_service_configs (user_id, instance_id);
 

src/db.scm

@@ -248,7 +248,9 @@ returning users.user_id;"
       (digitalocean-volume-size . ("digitalocean_volume_size" #f))
       (backblaze-application-key . ("backblaze_application_key_enc" #t))
       (backblaze-key-id . ("backblaze_key_id_enc" #t))
-      (backblaze-bucket-url . ("backblaze_bucket_url_enc" #t))))
+      (backblaze-bucket-url . ("backblaze_bucket_url_enc" #t))
+      (backblaze-bucket-name . ("backblaze_bucket_name_enc" #t))
+      (backblaze-bucket-id . ("backblaze_bucket_id_enc" #t))))
 
   (define *user-service-configs-reverse-column-map*
     (map (lambda (config)
@@ -657,7 +659,8 @@ where user_id=$1;"
       (4 . "fixing-app-normalization")
       (5 . "adding-wordpress-app")
       (6 . "adding-lldap-app")
-      (7 . "adding-authelia-app")))
+      (7 . "adding-authelia-app")
+      (8 . "adding-backblaze-bucket-name-and-id")))
 
   (define (run-pending-migrations conn)
     (let* ((migration-ids (sort (map car *migrations*) <))

src/migrations/8-adding-backblaze-bucket-name-and-id-up.sql

@@ -0,0 +1,2 @@
+alter table user_service_configs add backblaze_bucket_name_enc varchar(255);
+alter table user_service_configs add backblaze_bucket_id_enc varchar(255);

src/nassella.scm

@@ -670,6 +670,8 @@ h1, h2, h3, h4, h5, h6 {
     #f
     read-json))
 
+;; request response looks like
+;; (((accountId . "xxx") (bucketId . "xxx") (bucketInfo) (bucketName . "xxx") (bucketType . "allPrivate") (corsRules) (defaultServerSideEncryption (isClientAuthorizedToRead . #t) (value (algorithm . null) (mode . null))) (fileLockConfiguration (isClientAuthorizedToRead . #t) (value (defaultRetention (mode . null) (period . null)) (isFileLockEnabled . #f))) (lifecycleRules ((daysFromHidingToDeleting . 1) (daysFromStartingToCancelingUnfinishedLargeFiles . null) (daysFromUploadingToHiding . null) (fileNamePrefix . ""))) (options "s3") (replicationConfiguration (isClientAuthorizedToRead . #t) (value . null)) (revision . 3)) ((accountId . "yyy") (bucketId . "yyy") (bucketInfo) (bucketName . "yyy") (bucketType . "allPrivate") (corsRules) (defaultServerSideEncryption (isClientAuthorizedToRead . #t) (value (algorithm . null) (mode . null))) (fileLockConfiguration (isClientAuthorizedToRead . #t) (value (defaultRetention (mode . null) (period . null)) (isFileLockEnabled . #f))) (lifecycleRules ((daysFromHidingToDeleting . 1) (daysFromStartingToCancelingUnfinishedLargeFiles . null) (daysFromUploadingToHiding . null) (fileNamePrefix . ""))) (options "s3") (replicationConfiguration (isClientAuthorizedToRead . #t) (value . null)) (revision . 3)))
 (define (b2-list-buckets api-url account-token account-id)
   (with-input-from-request
       (make-request method: 'POST
@@ -681,6 +683,28 @@ h1, h2, h3, h4, h5, h6 {
        `((accountId . ,account-id))))
     read-json))
 
+;; a convenience method used to retrieve the buckets either
+;; from the key details or via b2-list-buckets.
+;; depending on the key config and permissions the way you
+;; access the list of buckets you can access will vary
+(define (b2-account-buckets key-id key)
+  (let* ((auth (b2-authorize-account key-id key))
+         (authorization-token (alist-ref 'authorizationToken auth))
+         (account-id (alist-ref 'accountId auth))
+         (api-url (alist-ref 'apiUrl (alist-ref 'storageApi (alist-ref 'apiInfo auth))))
+         (buckets (alist-ref 'buckets (alist-ref 'allowed (alist-ref 'storageApi (alist-ref 'apiInfo auth)))))
+         (listed-buckets (and (eq? buckets 'null) ;; if the key does not contain a bucket restriction it should be able to use the api to list buckets
+                                (alist-ref 'buckets (b2-list-buckets api-url authorization-token account-id)))))
+    (if listed-buckets
+        (map (lambda (bucket-info)
+               `((name . ,(alist-ref 'bucketName bucket-info))
+                 (id . ,(alist-ref 'bucketId bucket-info))))
+             listed-buckets)
+        (map (lambda (bucket-info)
+               `((name . ,(alist-ref 'name bucket-info))
+                 (id . ,(alist-ref 'id bucket-info))))
+             buckets))))
+
 (define (b2-authorization-details key-id key bucket-name)
   (let* ((auth (b2-authorize-account key-id key))
          (authorization-token (alist-ref 'authorizationToken auth))
@@ -833,67 +857,79 @@ h1, h2, h3, h4, h5, h6 {
           `((success . #f)
             (errors ((message . ,(alist-ref 'message res)))))))))
 
-(define (test-backblaze-connection user-id instance-id key-id application-key bucket-url)
+(define (test-backblaze-connection key-id application-key)
-  (let* ((password-path (conc "restic-password-" user-id "-" instance-id))
+  (handle-exceptions
-         (restic-password
+   exn
-          (with-db/transaction
-           (lambda (db)
-             (get-instance-restic-password db user-id instance-id)))))
-    (dynamic-wind
-      (lambda ()
-        (with-output-to-file password-path (lambda () (display restic-password))))
-      (lambda ()
-        ;; restic retries (indefinitely?) if the connection can't be made so
-        ;; we pass it through timeout to ensure it does not hang
-        (receive (in-port out-port pid err-port)
-            (cond-expand
-              (dev
-               (process* "timeout" `("--preserve-status" "8s" "docker" "run" "--rm" "--volume"
-                                     ,(conc (current-directory) "/" password-path ":/restic-password")
-                                     "-e" ,(conc "AWS_ACCESS_KEY_ID=" key-id)
-                                     "-e" ,(conc "AWS_SECRET_ACCESS_KEY=" application-key)
-                                     "-i" "restic/restic:0.18.0" "cat" "config"
-                                     "--repo" ,(conc "s3:" bucket-url)
-                                     "--password-file" "/restic-password"
-                                     "--json")))
-              (else
-               (process* "/bin/timeout"
-                         `("--preserve-status" "8s" "/bin/restic" "cat" "config"
-                           "--repo" ,(conc "s3:" bucket-url)
-                           "--password-file" ,(conc (current-directory) "/" password-path)
-                           "--json")
-                         `(("AWS_ACCESS_KEY_ID" . ,key-id)
-                           ("AWS_SECRET_ACCESS_KEY" . ,application-key)))))
-          (let ((thread
-                 (thread-start!
-                  (lambda ()
-                    (let loop ((i 0))
-                      (thread-sleep! 1)
-                      ;; We do a non-blocking wait here so that we don't
-                      ;; block the entire web process.
-                      (receive (wait-pid exit-normal status) (process-wait pid #t)
-                        (if (= wait-pid 0) ;; wait-pid is 0 until the process has finished
-                            (if (< i 12)   ;; 12s timeout
-                                (loop (+ i 1))
    `((success . #f)
-                                  (errors ((message . "timeout trying to connect to backblaze")))))
+     (errors ((message . ,(alist-ref 'code (read-json (get-condition-property exn 'client-error 'body)))))))
-                            (if exit-normal
+   (receive (data request-uri response) (b2-authorize-account key-id application-key)
-                                (let ((res (with-input-from-port in-port read-json))
+     (if (alist-ref 'authorizationToken data)
-                                      (err (with-input-from-port err-port read-json)))
-                                  ;; status 10 is what is returned if we can connect but the repo
-                                  ;; is not initialized yet
-                                  (if (or res (and err (= status 10)))
          `((success . #t)
-                                        (result . ,res))
+           (result ,data))
          `((success . #f)
-                                        (errors ((message . "abnormal exit - check Key ID, Application ID, and Bucket URL"))))))
+           (result ,data))))))
-                                `((success . #f)
+
-                                  (errors ((message . "abnormal exit - check Key ID, Application ID, and Bucket URL"))))
+;; (define (test-backblaze-connection user-id instance-id key-id application-key bucket-url)
-                                ))))))))
+;;   (let* ((password-path (conc "restic-password-" user-id "-" instance-id))
-            (thread-join! thread))))
+;;          (restic-password
-      (lambda ()
+;;           (with-db/transaction
-        (handle-exceptions exn 'ignore
+;;            (lambda (db)
-                           (delete-file password-path))))))
+;;              (get-instance-restic-password db user-id instance-id)))))
+;;     (dynamic-wind
+;;       (lambda ()
+;;         (with-output-to-file password-path (lambda () (display restic-password))))
+;;       (lambda ()
+;;         ;; restic retries (indefinitely?) if the connection can't be made so
+;;         ;; we pass it through timeout to ensure it does not hang
+;;         (receive (in-port out-port pid err-port)
+;;             (cond-expand
+;;               (dev
+;;                (process* "timeout" `("--preserve-status" "8s" "docker" "run" "--rm" "--volume"
+;;                                      ,(conc (current-directory) "/" password-path ":/restic-password")
+;;                                      "-e" ,(conc "AWS_ACCESS_KEY_ID=" key-id)
+;;                                      "-e" ,(conc "AWS_SECRET_ACCESS_KEY=" application-key)
+;;                                      "-i" "restic/restic:0.18.0" "cat" "config"
+;;                                      "--repo" ,(conc "s3:" bucket-url)
+;;                                      "--password-file" "/restic-password"
+;;                                      "--json")))
+;;               (else
+;;                (process* "/bin/timeout"
+;;                          `("--preserve-status" "8s" "/bin/restic" "cat" "config"
+;;                            "--repo" ,(conc "s3:" bucket-url)
+;;                            "--password-file" ,(conc (current-directory) "/" password-path)
+;;                            "--json")
+;;                          `(("AWS_ACCESS_KEY_ID" . ,key-id)
+;;                            ("AWS_SECRET_ACCESS_KEY" . ,application-key)))))
+;;           (let ((thread
+;;                  (thread-start!
+;;                   (lambda ()
+;;                     (let loop ((i 0))
+;;                       (thread-sleep! 1)
+;;                       ;; We do a non-blocking wait here so that we don't
+;;                       ;; block the entire web process.
+;;                       (receive (wait-pid exit-normal status) (process-wait pid #t)
+;;                         (if (= wait-pid 0) ;; wait-pid is 0 until the process has finished
+;;                             (if (< i 12)   ;; 12s timeout
+;;                                 (loop (+ i 1))
+;;                                 `((success . #f)
+;;                                   (errors ((message . "timeout trying to connect to backblaze")))))
+;;                             (if exit-normal
+;;                                 (let ((res (with-input-from-port in-port read-json))
+;;                                       (err (with-input-from-port err-port read-json)))
+;;                                   ;; status 10 is what is returned if we can connect but the repo
+;;                                   ;; is not initialized yet
+;;                                   (if (or res (and err (= status 10)))
+;;                                       `((success . #t)
+;;                                         (result . ,res))
+;;                                       `((success . #f)
+;;                                         (errors ((message . "abnormal exit - check Key ID, Application ID, and Bucket URL"))))))
+;;                                 `((success . #f)
+;;                                   (errors ((message . "abnormal exit - check Key ID, Application ID, and Bucket URL"))))
+;;                                 ))))))))
+;;             (thread-join! thread))))
+;;       (lambda ()
+;;         (handle-exceptions exn 'ignore
+;;                            (delete-file password-path))))))
 
 (define (deployment-directory user-id instance-id)
   (string-append "deploy-" (number->string user-id) "-" (->string instance-id)))
@@ -1217,9 +1253,7 @@ chmod -R 777 /opt/keys")))
                    (type "password")
                    (value ,(alist-ref 'backblaze-application-key config))))
          (Field (@ (name "backblaze-key-id") (label ("Key ID")) (type "password")
-                   (value ,(alist-ref 'backblaze-key-id config))))
+                   (value ,(alist-ref 'backblaze-key-id config)))))
-         (Field (@ (name "backblaze-bucket-url") (label ("Bucket URL")) (type "password")
-                   (value ,(alist-ref 'backblaze-bucket-url config)))))
         (Form-Nav)))))))
 
 (post "/config/wizard/services-submit/:id"
@@ -1235,8 +1269,7 @@ chmod -R 777 /opt/keys")))
          (cloudflare-zone-id . ,(alist-ref 'cloudflare-zone-id (current-params)))
          (digitalocean-api-token . ,(alist-ref 'digitalocean-api-token (current-params)))
          (backblaze-application-key . ,(alist-ref 'backblaze-application-key (current-params)))
-         (backblaze-key-id . ,(alist-ref 'backblaze-key-id (current-params)))
+         (backblaze-key-id . ,(alist-ref 'backblaze-key-id (current-params)))))))
-         (backblaze-bucket-url . ,(alist-ref 'backblaze-bucket-url (current-params)))))))
    (redirect (conc "/config/wizard/services-success/" instance-id))))
 
 (get/widgets
@@ -1250,10 +1283,8 @@ chmod -R 777 /opt/keys")))
                                                        (alist-ref 'cloudflare-zone-id service-config)
                                                        (alist-ref 'cloudflare-account-id service-config)))
         (digitalocean-result (test-digitalocean-connection (alist-ref 'digitalocean-api-token service-config)))
-        (backblaze-result (test-backblaze-connection (session-user-id) instance-id
+        (backblaze-result (test-backblaze-connection (alist-ref 'backblaze-key-id service-config)
-                                                     (alist-ref 'backblaze-key-id service-config)
+                                                     (alist-ref 'backblaze-application-key service-config))))
-                                                     (alist-ref 'backblaze-application-key service-config)
-                                                     (alist-ref 'backblaze-bucket-url service-config))))
    `(App
      (Configuration-Wizard
       (@ (step "Services"))
@@ -1315,7 +1346,17 @@ chmod -R 777 /opt/keys")))
                                  (filter cdr
                                          (get-user-selected-apps db (session-user-id) instance-id))))
               (app-config . ,(get-user-app-config db (session-user-id) instance-id))
-              (service-config . ,(get-user-service-config db (session-user-id) instance-id)))))))
+              (service-config . ,(get-user-service-config db (session-user-id) instance-id))))))
+        (backblaze-buckets (b2-account-buckets (alist-ref 'backblaze-key-id (alist-ref 'service-config results))
+                                               (alist-ref 'backblaze-application-key (alist-ref 'service-config results))))
+        (backblaze-s3-api-url (alist-ref
+                               's3ApiUrl
+                               (alist-ref
+                                'storageApi
+                                (alist-ref
+                                 'apiInfo
+                                 (b2-authorize-account (alist-ref 'backblaze-key-id (alist-ref 'service-config results))
+                                                       (alist-ref 'backblaze-application-key (alist-ref 'service-config results))))))))
    `(App
      (Configuration-Wizard
       (@ (step "Apps"))
@@ -1327,14 +1368,27 @@ chmod -R 777 /opt/keys")))
          (Field (@ (element select) (name "root-domain"))
                 ,@(map (lambda (domain)
                          `(option (@ (value ,domain)
+                                     (required #t)
                                      ,@(if (equal? domain
                                                    (alist-ref 'root-domain (alist-ref 'app-config results)))
                                            '(selected)
                                            '()))
                                   ,domain))
                        (get-cloudflare-domains (alist-ref 'cloudflare-api-token
-                                                          (alist-ref 'service-config results))))
+                                                          (alist-ref 'service-config results))))))
-                ))
+        (Fieldset
+         (@ (title "Backblaze Bucket"))
+         (Field (@ (element select) (name "backblaze-bucket-id"))
+                ,@(map (lambda (bucket)
+                         `(option (@ (value ,(string-append (alist-ref 'id bucket) "|" (alist-ref 'name bucket)))
+                                     (required #t)
+                                     ,@(if (equal? (alist-ref 'id bucket)
+                                                   (alist-ref 'backblaze-bucket-id (alist-ref 'service-config results)))
+                                           '(selected)
+                                           '()))
+                                  ,(alist-ref 'name bucket)))
+                       backblaze-buckets)))
+        (input (@ (type "hidden") (name "backblaze-s3-url") (value ,backblaze-s3-api-url)))
         (Fieldset
          (@ (title "Selected Apps"))
          (Field (@ (name "wg-easy") (type "checkbox") (label ("WG Easy")) (checked ,(member 'wg-easy (alist-ref 'selected-apps results)))))
@@ -1354,7 +1408,10 @@ chmod -R 777 /opt/keys")))
         (Form-Nav (@ (back-to ,(conc "/config/wizard/services-success/" instance-id))))))))))
 
 (post "/config/wizard/apps-submit/:id"
-  (let ((instance-id (alist-ref "id" (current-params) equal?)))
+  (let* ((instance-id (alist-ref "id" (current-params) equal?))
+         (bucket-values (string-split (alist-ref 'backblaze-bucket-id (current-params)) "|"))
+         (bucket-name (cadr bucket-values))
+         (bucket-id (car bucket-values)))
     (with-db/transaction
      (lambda (db)
        (update-user-selected-apps
@@ -1371,6 +1428,13 @@ chmod -R 777 /opt/keys")))
                   (authelia . "4")
                   (instance-control . "b0.0.1")
                   (log-viewer . "20"))))
+       (update-user-service-config
+        db
+        (session-user-id)
+        instance-id
+        `((backblaze-bucket-name . ,bucket-name)
+          (backblaze-bucket-id . ,bucket-id)
+          (backblaze-bucket-url . ,(string-append (alist-ref 'backblaze-s3-url (current-params)) "/" bucket-name))))
        (update-root-domain db
                            (session-user-id)
                            instance-id