User selects backblaze bucket instead of providing bucket url
This commit is contained in:
@@ -45,7 +45,9 @@ create table user_service_configs(
|
||||
digitalocean_volume_size integer,
|
||||
backblaze_application_key_enc varchar(255),
|
||||
backblaze_key_id_enc varchar(255),
|
||||
backblaze_bucket_url_enc varchar(255)
|
||||
backblaze_bucket_url_enc varchar(255), -- deprecated 072526, replaced by bucket_name and bucket_id
|
||||
backblaze_bucket_name_enc varchar(255),
|
||||
backblaze_bucket_id_enc varchar(255)
|
||||
);
|
||||
create unique index user_service_configs_user_id_instance_id_idx on user_service_configs (user_id, instance_id);
|
||||
|
||||
|
||||
@@ -248,7 +248,9 @@ returning users.user_id;"
|
||||
(digitalocean-volume-size . ("digitalocean_volume_size" #f))
|
||||
(backblaze-application-key . ("backblaze_application_key_enc" #t))
|
||||
(backblaze-key-id . ("backblaze_key_id_enc" #t))
|
||||
(backblaze-bucket-url . ("backblaze_bucket_url_enc" #t))))
|
||||
(backblaze-bucket-url . ("backblaze_bucket_url_enc" #t))
|
||||
(backblaze-bucket-name . ("backblaze_bucket_name_enc" #t))
|
||||
(backblaze-bucket-id . ("backblaze_bucket_id_enc" #t))))
|
||||
|
||||
(define *user-service-configs-reverse-column-map*
|
||||
(map (lambda (config)
|
||||
@@ -657,7 +659,8 @@ where user_id=$1;"
|
||||
(4 . "fixing-app-normalization")
|
||||
(5 . "adding-wordpress-app")
|
||||
(6 . "adding-lldap-app")
|
||||
(7 . "adding-authelia-app")))
|
||||
(7 . "adding-authelia-app")
|
||||
(8 . "adding-backblaze-bucket-name-and-id")))
|
||||
|
||||
(define (run-pending-migrations conn)
|
||||
(let* ((migration-ids (sort (map car *migrations*) <))
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
alter table user_service_configs add backblaze_bucket_name_enc varchar(255);
|
||||
alter table user_service_configs add backblaze_bucket_id_enc varchar(255);
|
||||
220
src/nassella.scm
220
src/nassella.scm
@@ -670,6 +670,8 @@ h1, h2, h3, h4, h5, h6 {
|
||||
#f
|
||||
read-json))
|
||||
|
||||
;; request response looks like
|
||||
;; (((accountId . "xxx") (bucketId . "xxx") (bucketInfo) (bucketName . "xxx") (bucketType . "allPrivate") (corsRules) (defaultServerSideEncryption (isClientAuthorizedToRead . #t) (value (algorithm . null) (mode . null))) (fileLockConfiguration (isClientAuthorizedToRead . #t) (value (defaultRetention (mode . null) (period . null)) (isFileLockEnabled . #f))) (lifecycleRules ((daysFromHidingToDeleting . 1) (daysFromStartingToCancelingUnfinishedLargeFiles . null) (daysFromUploadingToHiding . null) (fileNamePrefix . ""))) (options "s3") (replicationConfiguration (isClientAuthorizedToRead . #t) (value . null)) (revision . 3)) ((accountId . "yyy") (bucketId . "yyy") (bucketInfo) (bucketName . "yyy") (bucketType . "allPrivate") (corsRules) (defaultServerSideEncryption (isClientAuthorizedToRead . #t) (value (algorithm . null) (mode . null))) (fileLockConfiguration (isClientAuthorizedToRead . #t) (value (defaultRetention (mode . null) (period . null)) (isFileLockEnabled . #f))) (lifecycleRules ((daysFromHidingToDeleting . 1) (daysFromStartingToCancelingUnfinishedLargeFiles . null) (daysFromUploadingToHiding . null) (fileNamePrefix . ""))) (options "s3") (replicationConfiguration (isClientAuthorizedToRead . #t) (value . null)) (revision . 3)))
|
||||
(define (b2-list-buckets api-url account-token account-id)
|
||||
(with-input-from-request
|
||||
(make-request method: 'POST
|
||||
@@ -681,6 +683,28 @@ h1, h2, h3, h4, h5, h6 {
|
||||
`((accountId . ,account-id))))
|
||||
read-json))
|
||||
|
||||
;; a convenience method used to retrieve the buckets either
|
||||
;; from the key details or via b2-list-buckets.
|
||||
;; depending on the key config and permissions the way you
|
||||
;; access the list of buckets you can access will vary
|
||||
(define (b2-account-buckets key-id key)
|
||||
(let* ((auth (b2-authorize-account key-id key))
|
||||
(authorization-token (alist-ref 'authorizationToken auth))
|
||||
(account-id (alist-ref 'accountId auth))
|
||||
(api-url (alist-ref 'apiUrl (alist-ref 'storageApi (alist-ref 'apiInfo auth))))
|
||||
(buckets (alist-ref 'buckets (alist-ref 'allowed (alist-ref 'storageApi (alist-ref 'apiInfo auth)))))
|
||||
(listed-buckets (and (eq? buckets 'null) ;; if the key does not contain a bucket restriction it should be able to use the api to list buckets
|
||||
(alist-ref 'buckets (b2-list-buckets api-url authorization-token account-id)))))
|
||||
(if listed-buckets
|
||||
(map (lambda (bucket-info)
|
||||
`((name . ,(alist-ref 'bucketName bucket-info))
|
||||
(id . ,(alist-ref 'bucketId bucket-info))))
|
||||
listed-buckets)
|
||||
(map (lambda (bucket-info)
|
||||
`((name . ,(alist-ref 'name bucket-info))
|
||||
(id . ,(alist-ref 'id bucket-info))))
|
||||
buckets))))
|
||||
|
||||
(define (b2-authorization-details key-id key bucket-name)
|
||||
(let* ((auth (b2-authorize-account key-id key))
|
||||
(authorization-token (alist-ref 'authorizationToken auth))
|
||||
@@ -833,67 +857,79 @@ h1, h2, h3, h4, h5, h6 {
|
||||
`((success . #f)
|
||||
(errors ((message . ,(alist-ref 'message res)))))))))
|
||||
|
||||
(define (test-backblaze-connection user-id instance-id key-id application-key bucket-url)
|
||||
(let* ((password-path (conc "restic-password-" user-id "-" instance-id))
|
||||
(restic-password
|
||||
(with-db/transaction
|
||||
(lambda (db)
|
||||
(get-instance-restic-password db user-id instance-id)))))
|
||||
(dynamic-wind
|
||||
(lambda ()
|
||||
(with-output-to-file password-path (lambda () (display restic-password))))
|
||||
(lambda ()
|
||||
;; restic retries (indefinitely?) if the connection can't be made so
|
||||
;; we pass it through timeout to ensure it does not hang
|
||||
(receive (in-port out-port pid err-port)
|
||||
(cond-expand
|
||||
(dev
|
||||
(process* "timeout" `("--preserve-status" "8s" "docker" "run" "--rm" "--volume"
|
||||
,(conc (current-directory) "/" password-path ":/restic-password")
|
||||
"-e" ,(conc "AWS_ACCESS_KEY_ID=" key-id)
|
||||
"-e" ,(conc "AWS_SECRET_ACCESS_KEY=" application-key)
|
||||
"-i" "restic/restic:0.18.0" "cat" "config"
|
||||
"--repo" ,(conc "s3:" bucket-url)
|
||||
"--password-file" "/restic-password"
|
||||
"--json")))
|
||||
(else
|
||||
(process* "/bin/timeout"
|
||||
`("--preserve-status" "8s" "/bin/restic" "cat" "config"
|
||||
"--repo" ,(conc "s3:" bucket-url)
|
||||
"--password-file" ,(conc (current-directory) "/" password-path)
|
||||
"--json")
|
||||
`(("AWS_ACCESS_KEY_ID" . ,key-id)
|
||||
("AWS_SECRET_ACCESS_KEY" . ,application-key)))))
|
||||
(let ((thread
|
||||
(thread-start!
|
||||
(lambda ()
|
||||
(let loop ((i 0))
|
||||
(thread-sleep! 1)
|
||||
;; We do a non-blocking wait here so that we don't
|
||||
;; block the entire web process.
|
||||
(receive (wait-pid exit-normal status) (process-wait pid #t)
|
||||
(if (= wait-pid 0) ;; wait-pid is 0 until the process has finished
|
||||
(if (< i 12) ;; 12s timeout
|
||||
(loop (+ i 1))
|
||||
`((success . #f)
|
||||
(errors ((message . "timeout trying to connect to backblaze")))))
|
||||
(if exit-normal
|
||||
(let ((res (with-input-from-port in-port read-json))
|
||||
(err (with-input-from-port err-port read-json)))
|
||||
;; status 10 is what is returned if we can connect but the repo
|
||||
;; is not initialized yet
|
||||
(if (or res (and err (= status 10)))
|
||||
`((success . #t)
|
||||
(result . ,res))
|
||||
`((success . #f)
|
||||
(errors ((message . "abnormal exit - check Key ID, Application ID, and Bucket URL"))))))
|
||||
`((success . #f)
|
||||
(errors ((message . "abnormal exit - check Key ID, Application ID, and Bucket URL"))))
|
||||
))))))))
|
||||
(thread-join! thread))))
|
||||
(lambda ()
|
||||
(handle-exceptions exn 'ignore
|
||||
(delete-file password-path))))))
|
||||
(define (test-backblaze-connection key-id application-key)
|
||||
(handle-exceptions
|
||||
exn
|
||||
`((success . #f)
|
||||
(errors ((message . ,(alist-ref 'code (read-json (get-condition-property exn 'client-error 'body)))))))
|
||||
(receive (data request-uri response) (b2-authorize-account key-id application-key)
|
||||
(if (alist-ref 'authorizationToken data)
|
||||
`((success . #t)
|
||||
(result ,data))
|
||||
`((success . #f)
|
||||
(result ,data))))))
|
||||
|
||||
;; (define (test-backblaze-connection user-id instance-id key-id application-key bucket-url)
|
||||
;; (let* ((password-path (conc "restic-password-" user-id "-" instance-id))
|
||||
;; (restic-password
|
||||
;; (with-db/transaction
|
||||
;; (lambda (db)
|
||||
;; (get-instance-restic-password db user-id instance-id)))))
|
||||
;; (dynamic-wind
|
||||
;; (lambda ()
|
||||
;; (with-output-to-file password-path (lambda () (display restic-password))))
|
||||
;; (lambda ()
|
||||
;; ;; restic retries (indefinitely?) if the connection can't be made so
|
||||
;; ;; we pass it through timeout to ensure it does not hang
|
||||
;; (receive (in-port out-port pid err-port)
|
||||
;; (cond-expand
|
||||
;; (dev
|
||||
;; (process* "timeout" `("--preserve-status" "8s" "docker" "run" "--rm" "--volume"
|
||||
;; ,(conc (current-directory) "/" password-path ":/restic-password")
|
||||
;; "-e" ,(conc "AWS_ACCESS_KEY_ID=" key-id)
|
||||
;; "-e" ,(conc "AWS_SECRET_ACCESS_KEY=" application-key)
|
||||
;; "-i" "restic/restic:0.18.0" "cat" "config"
|
||||
;; "--repo" ,(conc "s3:" bucket-url)
|
||||
;; "--password-file" "/restic-password"
|
||||
;; "--json")))
|
||||
;; (else
|
||||
;; (process* "/bin/timeout"
|
||||
;; `("--preserve-status" "8s" "/bin/restic" "cat" "config"
|
||||
;; "--repo" ,(conc "s3:" bucket-url)
|
||||
;; "--password-file" ,(conc (current-directory) "/" password-path)
|
||||
;; "--json")
|
||||
;; `(("AWS_ACCESS_KEY_ID" . ,key-id)
|
||||
;; ("AWS_SECRET_ACCESS_KEY" . ,application-key)))))
|
||||
;; (let ((thread
|
||||
;; (thread-start!
|
||||
;; (lambda ()
|
||||
;; (let loop ((i 0))
|
||||
;; (thread-sleep! 1)
|
||||
;; ;; We do a non-blocking wait here so that we don't
|
||||
;; ;; block the entire web process.
|
||||
;; (receive (wait-pid exit-normal status) (process-wait pid #t)
|
||||
;; (if (= wait-pid 0) ;; wait-pid is 0 until the process has finished
|
||||
;; (if (< i 12) ;; 12s timeout
|
||||
;; (loop (+ i 1))
|
||||
;; `((success . #f)
|
||||
;; (errors ((message . "timeout trying to connect to backblaze")))))
|
||||
;; (if exit-normal
|
||||
;; (let ((res (with-input-from-port in-port read-json))
|
||||
;; (err (with-input-from-port err-port read-json)))
|
||||
;; ;; status 10 is what is returned if we can connect but the repo
|
||||
;; ;; is not initialized yet
|
||||
;; (if (or res (and err (= status 10)))
|
||||
;; `((success . #t)
|
||||
;; (result . ,res))
|
||||
;; `((success . #f)
|
||||
;; (errors ((message . "abnormal exit - check Key ID, Application ID, and Bucket URL"))))))
|
||||
;; `((success . #f)
|
||||
;; (errors ((message . "abnormal exit - check Key ID, Application ID, and Bucket URL"))))
|
||||
;; ))))))))
|
||||
;; (thread-join! thread))))
|
||||
;; (lambda ()
|
||||
;; (handle-exceptions exn 'ignore
|
||||
;; (delete-file password-path))))))
|
||||
|
||||
(define (deployment-directory user-id instance-id)
|
||||
(string-append "deploy-" (number->string user-id) "-" (->string instance-id)))
|
||||
@@ -1217,9 +1253,7 @@ chmod -R 777 /opt/keys")))
|
||||
(type "password")
|
||||
(value ,(alist-ref 'backblaze-application-key config))))
|
||||
(Field (@ (name "backblaze-key-id") (label ("Key ID")) (type "password")
|
||||
(value ,(alist-ref 'backblaze-key-id config))))
|
||||
(Field (@ (name "backblaze-bucket-url") (label ("Bucket URL")) (type "password")
|
||||
(value ,(alist-ref 'backblaze-bucket-url config)))))
|
||||
(value ,(alist-ref 'backblaze-key-id config)))))
|
||||
(Form-Nav)))))))
|
||||
|
||||
(post "/config/wizard/services-submit/:id"
|
||||
@@ -1235,8 +1269,7 @@ chmod -R 777 /opt/keys")))
|
||||
(cloudflare-zone-id . ,(alist-ref 'cloudflare-zone-id (current-params)))
|
||||
(digitalocean-api-token . ,(alist-ref 'digitalocean-api-token (current-params)))
|
||||
(backblaze-application-key . ,(alist-ref 'backblaze-application-key (current-params)))
|
||||
(backblaze-key-id . ,(alist-ref 'backblaze-key-id (current-params)))
|
||||
(backblaze-bucket-url . ,(alist-ref 'backblaze-bucket-url (current-params)))))))
|
||||
(backblaze-key-id . ,(alist-ref 'backblaze-key-id (current-params)))))))
|
||||
(redirect (conc "/config/wizard/services-success/" instance-id))))
|
||||
|
||||
(get/widgets
|
||||
@@ -1250,10 +1283,8 @@ chmod -R 777 /opt/keys")))
|
||||
(alist-ref 'cloudflare-zone-id service-config)
|
||||
(alist-ref 'cloudflare-account-id service-config)))
|
||||
(digitalocean-result (test-digitalocean-connection (alist-ref 'digitalocean-api-token service-config)))
|
||||
(backblaze-result (test-backblaze-connection (session-user-id) instance-id
|
||||
(alist-ref 'backblaze-key-id service-config)
|
||||
(alist-ref 'backblaze-application-key service-config)
|
||||
(alist-ref 'backblaze-bucket-url service-config))))
|
||||
(backblaze-result (test-backblaze-connection (alist-ref 'backblaze-key-id service-config)
|
||||
(alist-ref 'backblaze-application-key service-config))))
|
||||
`(App
|
||||
(Configuration-Wizard
|
||||
(@ (step "Services"))
|
||||
@@ -1315,7 +1346,17 @@ chmod -R 777 /opt/keys")))
|
||||
(filter cdr
|
||||
(get-user-selected-apps db (session-user-id) instance-id))))
|
||||
(app-config . ,(get-user-app-config db (session-user-id) instance-id))
|
||||
(service-config . ,(get-user-service-config db (session-user-id) instance-id)))))))
|
||||
(service-config . ,(get-user-service-config db (session-user-id) instance-id))))))
|
||||
(backblaze-buckets (b2-account-buckets (alist-ref 'backblaze-key-id (alist-ref 'service-config results))
|
||||
(alist-ref 'backblaze-application-key (alist-ref 'service-config results))))
|
||||
(backblaze-s3-api-url (alist-ref
|
||||
's3ApiUrl
|
||||
(alist-ref
|
||||
'storageApi
|
||||
(alist-ref
|
||||
'apiInfo
|
||||
(b2-authorize-account (alist-ref 'backblaze-key-id (alist-ref 'service-config results))
|
||||
(alist-ref 'backblaze-application-key (alist-ref 'service-config results))))))))
|
||||
`(App
|
||||
(Configuration-Wizard
|
||||
(@ (step "Apps"))
|
||||
@@ -1327,24 +1368,37 @@ chmod -R 777 /opt/keys")))
|
||||
(Field (@ (element select) (name "root-domain"))
|
||||
,@(map (lambda (domain)
|
||||
`(option (@ (value ,domain)
|
||||
(required #t)
|
||||
,@(if (equal? domain
|
||||
(alist-ref 'root-domain (alist-ref 'app-config results)))
|
||||
'(selected)
|
||||
'()))
|
||||
,domain))
|
||||
(get-cloudflare-domains (alist-ref 'cloudflare-api-token
|
||||
(alist-ref 'service-config results))))
|
||||
))
|
||||
(alist-ref 'service-config results))))))
|
||||
(Fieldset
|
||||
(@ (title "Backblaze Bucket"))
|
||||
(Field (@ (element select) (name "backblaze-bucket-id"))
|
||||
,@(map (lambda (bucket)
|
||||
`(option (@ (value ,(string-append (alist-ref 'id bucket) "|" (alist-ref 'name bucket)))
|
||||
(required #t)
|
||||
,@(if (equal? (alist-ref 'id bucket)
|
||||
(alist-ref 'backblaze-bucket-id (alist-ref 'service-config results)))
|
||||
'(selected)
|
||||
'()))
|
||||
,(alist-ref 'name bucket)))
|
||||
backblaze-buckets)))
|
||||
(input (@ (type "hidden") (name "backblaze-s3-url") (value ,backblaze-s3-api-url)))
|
||||
(Fieldset
|
||||
(@ (title "Selected Apps"))
|
||||
(Field (@ (name "wg-easy") (type "checkbox") (label ("WG Easy")) (checked ,(member 'wg-easy (alist-ref 'selected-apps results)))))
|
||||
(Field (@ (name "nextcloud") (type "checkbox") (label ("NextCloud")) (checked ,(member 'nextcloud (alist-ref 'selected-apps results)))))
|
||||
(Field (@ (name "ghost") (type "checkbox") (label ("Ghost")) (checked ,(member 'ghost (alist-ref 'selected-apps results)))))
|
||||
,@(cond-expand
|
||||
(dev
|
||||
`((Field (@ (name "nassella") (type "checkbox") (label ("Nassella")) (checked ,(member 'nassella (alist-ref 'selected-apps results)))))))
|
||||
(else
|
||||
'()))
|
||||
(dev
|
||||
`((Field (@ (name "nassella") (type "checkbox") (label ("Nassella")) (checked ,(member 'nassella (alist-ref 'selected-apps results)))))))
|
||||
(else
|
||||
'()))
|
||||
(Field (@ (name "wordpress") (type "checkbox") (label ("Wordpress")) (checked ,(member 'wordpress (alist-ref 'selected-apps results)))))
|
||||
(Field (@ (name "log-viewer") (type "checkbox") (label ("Log Viewer")) (checked #t) (disabled "disabled")))
|
||||
(Field (@ (name "lldap") (type "checkbox") (label ("Admin LLDAP")) (checked #t) (disabled "disabled")))
|
||||
@@ -1354,7 +1408,10 @@ chmod -R 777 /opt/keys")))
|
||||
(Form-Nav (@ (back-to ,(conc "/config/wizard/services-success/" instance-id))))))))))
|
||||
|
||||
(post "/config/wizard/apps-submit/:id"
|
||||
(let ((instance-id (alist-ref "id" (current-params) equal?)))
|
||||
(let* ((instance-id (alist-ref "id" (current-params) equal?))
|
||||
(bucket-values (string-split (alist-ref 'backblaze-bucket-id (current-params)) "|"))
|
||||
(bucket-name (cadr bucket-values))
|
||||
(bucket-id (car bucket-values)))
|
||||
(with-db/transaction
|
||||
(lambda (db)
|
||||
(update-user-selected-apps
|
||||
@@ -1371,6 +1428,13 @@ chmod -R 777 /opt/keys")))
|
||||
(authelia . "4")
|
||||
(instance-control . "b0.0.1")
|
||||
(log-viewer . "20"))))
|
||||
(update-user-service-config
|
||||
db
|
||||
(session-user-id)
|
||||
instance-id
|
||||
`((backblaze-bucket-name . ,bucket-name)
|
||||
(backblaze-bucket-id . ,bucket-id)
|
||||
(backblaze-bucket-url . ,(string-append (alist-ref 'backblaze-s3-url (current-params)) "/" bucket-name))))
|
||||
(update-root-domain db
|
||||
(session-user-id)
|
||||
instance-id
|
||||
|
||||
Reference in New Issue
Block a user