fix: support Hetzner workflows and jd locators
ober
7cba684ea7307d1831f9f72aa48e714f34b4650e
--- a/README.md +++ b/README.md @@ -26,18 +26,19 @@ make run ARGS='s3 init --profile default --bucket my-private-bucket' make run ARGS='s3 profile-status --profile default' ``` -The initializer prompts for a vault password unless -`--vault-password-env NAME` names an environment variable containing it. Use a -unique, high-entropy password and do not store it in the repository. +The initializer reads `JDRIVE_VAULT_PASSWORD` automatically, or prompts when +the variable is unset. `--vault-password-env NAME` remains available for a +differently named secret variable. Use a unique, high-entropy password and do +not store it in the repository. Upload, inspect, and restore data: ```sh -make run ARGS='s3 cp notes.txt jdrive:/notes.txt' -make run ARGS='s3 ls jdrive:/' -make run ARGS='s3 cat jdrive:/notes.txt' -make run ARGS='s3 get jdrive:/notes.txt restored-notes.txt' -make run ARGS='s3 sync ./photos jdrive:/photos --delete' +make run ARGS='s3 cp notes.txt jd:/notes.txt' +make run ARGS='s3 ls jd:/' +make run ARGS='s3 cat jd:/notes.txt' +make run ARGS='s3 get jd:/notes.txt restored-notes.txt' +make run ARGS='s3 sync -v ./photos jd:/photos --delete' ``` Run `jdrive help` for all commands and options. See @@ -57,6 +58,10 @@ The main environment variables are: Endpoint overrides make MinIO and other S3-compatible services usable. Path style can be enabled with `--path-style` or `JDRIVE_S3_PATH_STYLE=1`. +Remote paths use the short `jd:` prefix: `jd:photos` and `jd:/photos` both +address `/photos`. The older `jdrive:` prefix remains compatible. Sync `-v` +prints progress to stderr. + ## Storage and encryption Each profile has a local configuration document and encrypted key vault. Drive --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.1 +1.2.0 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -46,6 +46,10 @@ Command-line target options override the environment: `--endpoint` and `--path-style` support services such as MinIO. The default object prefix is `jdrive`. +For compatible services such as Hetzner Object Storage, manifest existence and +ETag checks use a signed one-byte range GET. This avoids treating a valid HEAD +response's object `Content-Length` as a response body that must be read. + Inspect the resolved configuration without changing remote state: ```sh @@ -61,15 +65,19 @@ Create a local encrypted-drive profile: make run ARGS='s3 init --profile default --bucket my-private-bucket' ``` -The command prompts for a vault password. For non-interactive use, name an -environment variable instead of putting a secret on the command line: +The command reads `JDRIVE_VAULT_PASSWORD` automatically and prompts when the +variable is unset. For non-interactive use, export it instead of putting a +secret on the command line: ```sh export JDRIVE_VAULT_PASSWORD='<from-secret-manager>' -make run ARGS='s3 init --profile default --bucket my-private-bucket --vault-password-env JDRIVE_VAULT_PASSWORD' +make run ARGS='s3 init --profile default --bucket my-private-bucket' unset JDRIVE_VAULT_PASSWORD ``` +Use `--vault-password-env NAME` only when the secret is stored in a differently +named environment variable. + Check profile metadata or verify an unlock: ```sh @@ -106,25 +114,28 @@ credentials. ## Paths -Remote paths use the `jdrive:` prefix: +Remote paths use the `jd:` prefix: ```text -jdrive:/ -jdrive:/documents/report.pdf -jdrive:/photos/2026/ +jd:/ +jd:/documents/report.pdf +jd:/photos/2026/ ``` +The compact form is also accepted: `jd:photos` resolves to the same remote path +as `jd:/photos`. The older `jdrive:` prefix remains compatible. + Path validation rejects traversal components and unsafe local symlink behavior. Use absolute-looking remote paths for clarity. ## Copy, list, and read ```sh -make run ARGS='s3 cp local.txt jdrive:/local.txt' -make run ARGS='s3 cp -r ./documents jdrive:/documents' -make run ARGS='s3 ls jdrive:/' -make run ARGS='s3 cat jdrive:/local.txt' -make run ARGS='s3 get jdrive:/local.txt ./restored.txt' +make run ARGS='s3 cp local.txt jd:/local.txt' +make run ARGS='s3 cp -r ./documents jd:/documents' +make run ARGS='s3 ls jd:/' +make run ARGS='s3 cat jd:/local.txt' +make run ARGS='s3 get jd:/local.txt ./restored.txt' ``` Use `--offset N` and `--size N` with `s3 cat` or `s3 get` for a byte range. @@ -134,10 +145,10 @@ covering the requested range. ## Directories, moves, and deletion ```sh -make run ARGS='s3 mkdir jdrive:/archive' -make run ARGS='s3 mv jdrive:/local.txt jdrive:/archive/local.txt' -make run ARGS='s3 rm jdrive:/archive/local.txt' -make run ARGS='s3 rm -r jdrive:/archive' +make run ARGS='s3 mkdir jd:/archive' +make run ARGS='s3 mv jd:/local.txt jd:/archive/local.txt' +make run ARGS='s3 rm jd:/archive/local.txt' +make run ARGS='s3 rm -r jd:/archive' ``` Recursive removal is intentionally explicit. Review the target before using @@ -148,13 +159,16 @@ Recursive removal is intentionally explicit. Review the target before using Upload a tree: ```sh -make run ARGS='s3 sync ./photos jdrive:/photos' +make run ARGS='s3 sync -v ./photos jd:/photos' ``` +With `-v`, sync writes phase and per-file progress to stderr while keeping the +final JSON result on stdout. There is no `--verbose` option. + Compare without changing it: ```sh -make run ARGS='s3 check ./photos jdrive:/photos' +make run ARGS='s3 check ./photos jd:/photos' ``` `--include PATTERN` and `--exclude PATTERN` filter files. `--delete` removes --- a/jdrive/cli.ss +++ b/jdrive/cli.ss @@ -14,7 +14,7 @@ (jdrive s3 config) (jdrive s3 drive)) - (define version "1.0.0") + (define version "1.2.0") (define usage-string (string-append @@ -28,7 +28,7 @@ " s3 unlock-test Verify profile password/YubiKey unlock\n" " s3 cp [-r] SRC DST Copy to/from encrypted S3\n" " s3 mv [-r] REMOTE DST Move an encrypted S3 file/tree\n" - " s3 sync LOCAL REMOTE Sync a local tree to encrypted S3\n" + " s3 sync [-v] LOCAL REMOTE Sync a local tree to encrypted S3\n" " s3 check LOCAL REMOTE Compare a local tree with encrypted S3\n" " s3 ls [REMOTE] List encrypted S3 paths\n" " s3 mkdir REMOTE Create an encrypted directory marker\n" @@ -48,7 +48,7 @@ " --path-style Or JDRIVE_S3_PATH_STYLE=1\n" " --profile P Local profile, default default\n" " --state-dir PATH Local state root, default ~/.jdrive\n" - " --vault-password-env ENV Read the vault password from ENV\n" + " --vault-password-env ENV Override JDRIVE_VAULT_PASSWORD\n" " --yubikey-mode none|piv Optional PIV-backed vault unlock\n" " --piv-pin-env ENV Read the YubiKey PIV PIN from ENV\n" " --chunk-size N Encrypted chunk size, default 67108864\n" @@ -57,6 +57,7 @@ " --include PATTERN Include glob for s3 sync/check\n" " --exclude PATTERN Exclude glob for s3 sync/check\n" " --delete Delete remote files missing locally\n" + " -v Print sync progress\n" " -r, --recursive Recursive S3 cp/mv/rm\n")) (define (println value) @@ -66,6 +67,9 @@ (define (eprintln value) (display value (current-error-port)) (newline (current-error-port))) +(define (sync-progress! message) + (eprintln (string-append "sync: " message)) + (flush-output-port (current-error-port))) (define (void) (if #f #f)) @@ -130,6 +134,7 @@ ("--include" . #t) ("--exclude" . #t) ("--delete" . #f) + ("-v" . #f) ("-r" . #f) ("--recursive" . #f))) @@ -211,10 +216,17 @@ (define (recursive? opts) (or (opt opts "-r") (opt opts "--recursive"))) + (define (locator-prefix? value prefix) + (let ([prefix-length (string-length prefix)]) + (and (>= (string-length value) prefix-length) + (string=? (substring value 0 prefix-length) prefix)))) + (define (remote-path? path) (and (string? path) (> (string-length path) 0) - (char=? (string-ref path 0) #\/))) + (or (char=? (string-ref path 0) #\/) + (locator-prefix? path "jd:") + (locator-prefix? path "jdrive:")))) (define (s3-source-remote? path) (and (remote-path? path) @@ -485,29 +497,32 @@ (opts-all opts "--include") (opts-all opts "--exclude")))))) + (define (sync-progress-option opts) + (and (opt opts "-v") sync-progress!)) + + (define (run-s3-sync opts positional password material) + (jdrive-s3-sync-local-to-remote! + (profile-option opts) + (state-root-option opts) + password + material + (car positional) + (cadr positional) + (opts-all opts "--include") + (opts-all opts "--exclude") + (and (opt opts "--delete") #t) + (option-number opts "--chunk-size" jdrive-s3-default-chunk-size) + (sync-progress-option opts))) + (define (cmd-s3-sync opts positional) (require-pos-count - "usage: jdrive s3 sync LOCAL_PATH REMOTE_PATH [--delete]" + "usage: jdrive s3 sync [-v] LOCAL_PATH REMOTE_PATH [--delete]" positional 2) (with-s3-unlock opts (lambda (password material) - (print-json - (jdrive-s3-sync-local-to-remote! - (profile-option opts) - (state-root-option opts) - password - material - (car positional) - (cadr positional) - (opts-all opts "--include") - (opts-all opts "--exclude") - (and (opt opts "--delete") #t) - (option-number - opts - "--chunk-size" - jdrive-s3-default-chunk-size)))))) + (print-json (run-s3-sync opts positional password material))))) (define (cmd-s3-move opts positional) (require-pos-count @@ -587,6 +602,8 @@ (recursive? opts)))))) (define (dispatch-s3 action opts positional) + (when (and (opt opts "-v") (not (string=? action "sync"))) + (die 2 "unknown option: -v")) (cond [(string=? action "status") (cmd-s3-status opts positional)] [(string=? action "profile-status") --- a/jdrive/s3/drive.ss +++ b/jdrive/s3/drive.ss @@ -614,8 +614,22 @@ [else (loop start (+ at 1))]))) path) + (define (locator-prefix? value prefix) + (let ([prefix-length (string-length prefix)]) + (and (>= (string-length value) prefix-length) + (string=? (substring value 0 prefix-length) prefix)))) + (define (remote-path-normalize path) - (let ([path (nonempty-string 'remote-path-normalize 'path path)]) + (let* ([path (nonempty-string 'remote-path-normalize 'path path)] + [path + (cond + [(locator-prefix? path "jd:") + (let ([suffix (substring path 3 (string-length path))]) + (if (string=? suffix "") "/" suffix))] + [(locator-prefix? path "jdrive:") + (let ([suffix (substring path 7 (string-length path))]) + (if (string=? suffix "") "/" suffix))] + [else path])]) (let ([path (if (char=? (string-ref path 0) #\/) path @@ -1015,7 +1029,7 @@ (assert-and-record-manifest-generation! config drive-key - (if (object-exists? client bucket key) + (if (manifest-object-exists? client bucket key) (sealed->manifest drive-key (get-object-bytes client bucket key)) (jdrive-s3-empty-manifest))))))) @@ -1027,37 +1041,61 @@ (string-ci=? (caar xs) name)) (cdar xs)] [else (loop (cdr xs))]))) + (define (manifest-probe-request client bucket key) + (s3-request + client + 'verb: "GET" + 'bucket: bucket + 'key: key + 'extra-headers: '(("Range" . "bytes=0-0")))) + + (define (manifest-probe-error who req status) + (let ([body (request-text req)]) + (request-close req) + (error who + (string-append "S3 manifest probe failed: HTTP " + (number->string status)) + body))) + + (define (manifest-object-exists? client bucket key) + (let* ([req (manifest-probe-request client bucket key)] + [status (request-status req)]) + (cond + [(and (>= status 200) (< status 300)) + (request-close req) + #t] + [(= status 404) + (request-close req) + #f] + [else + (manifest-probe-error 'manifest-object-exists? req status)]))) + + (define (manifest-probe-etag req status) + (cond + [(and (>= status 200) (< status 300)) + (let ([etag (header-ref-ci (request-headers req) "etag")]) + (request-close req) + (unless etag + (error 'manifest-head-etag + "manifest object exists but S3 did not return ETag")) + etag)] + [(= status 404) + (request-close req) + #f] + [else + (manifest-probe-error 'manifest-head-etag req status)])) + + (define (manifest-probe-etag-for-key client bucket key) + (let* ([req (manifest-probe-request client bucket key)] + [status (request-status req)]) + (manifest-probe-etag req status))) (define (manifest-head-etag client bucket config) (let ([key (manifest-object-key config)]) (with-s3-retry 'manifest-head-etag (lambda () - (let* ([req - (s3-request - client - 'verb: "HEAD" - 'bucket: bucket - 'key: key)] - [status (request-status req)]) - (cond - [(and (>= status 200) (< status 300)) - (let ([etag (header-ref-ci (request-headers req) "etag")]) - (request-close req) - (unless etag - (error 'manifest-head-etag - "manifest object exists but S3 did not return ETag")) - etag)] - [(= status 404) - (request-close req) - #f] - [else - (let ([body (request-text req)]) - (request-close req) - (error 'manifest-head-etag - (string-append "S3 manifest HEAD failed: HTTP " - (number->string status)) - body))])))))) + (manifest-probe-etag-for-key client bucket key))))) (define (load-manifest+etag client bucket config drive-key) (let ([etag (manifest-head-etag client bucket config)]) @@ -1176,6 +1214,31 @@ (hashtable-set! manifest "Generation" (+ (manifest-generation manifest) 1)) (hashtable-set! manifest "UpdatedAt" (datetime->epoch (datetime-utc-now))) manifest) +(define (put-manifest-unconditional! + client bucket config drive-key manifest sealed) + (with-s3-retry + 'put-manifest-unconditional! + (lambda () + (put-object-bytes + client + bucket + (manifest-object-key config) + sealed + 'content-type: file-content-type))) + (jdrive-s3-record-manifest-generation! + (generation-mark-file config) + drive-key + manifest) + (void)) +(define (store-manifest-after-unsupported-condition! + client bucket config drive-key manifest sealed expected-etag) + (let ([current-etag (manifest-head-etag client bucket config)]) + (unless (and current-etag (string=? current-etag expected-etag)) + (error 'store-manifest-conditional! + "encrypted S3 manifest changed during update" + 412)) + (put-manifest-unconditional! + client bucket config drive-key manifest sealed))) (define (store-manifest-conditional! client bucket config drive-key manifest expected-etag) (let* ([key (manifest-object-key config)] @@ -1202,7 +1265,17 @@ drive-key manifest) (void)] - [(or (= status 409) (= status 412)) + [(= status 412) + (request-close req) + (store-manifest-after-unsupported-condition! + client + bucket + config + drive-key + manifest + sealed + expected-etag)] + [(= status 409) (let ([body (request-text req)]) (request-close req) (error 'store-manifest-conditional! @@ -1661,10 +1734,16 @@ exclude-patterns)) summary)))) + (define (report-sync-progress! progress message) + (when progress (progress message))) + (define (jdrive-s3-sync-local-to-remote! profile state-root vault-password yubikey-material local-path remote-root - include-patterns exclude-patterns delete-missing? chunk-size) - (let ([chunk-size (normalize-chunk-size chunk-size)]) + include-patterns exclude-patterns delete-missing? chunk-size + . maybe-progress) + (let ([chunk-size (normalize-chunk-size chunk-size)] + [progress (and (pair? maybe-progress) (car maybe-progress))]) + (report-sync-progress! progress "loading remote manifest") (with-unlocked-profile profile state-root @@ -1675,12 +1754,22 @@ (lambda () (load-manifest+etag client bucket config drive-key)) (lambda (manifest manifest-etag) (let* ([local-files - (walk-local-files - local-path - remote-root - include-patterns - exclude-patterns)] + (begin + (report-sync-progress! progress "scanning local files") + (walk-local-files + local-path + remote-root + include-patterns + exclude-patterns))] [local-remote-paths (map cdr local-files)] + [file-count (length local-files)] + [progress-start + (report-sync-progress! + progress + (string-append + "selected " + (number->string file-count) + " file(s)"))] [summary (json-object "Backend" "s3" @@ -1704,17 +1793,28 @@ [entry (manifest-find-file manifest remote)]) (manifest-add-parent-dirs! manifest remote) (if (and entry (entry-local-match? entry local)) - (inc-summary! summary "SkippedFiles") - (upload-one-file! - client - bucket - config - drive-key - manifest - local - remote - summary - chunk-size)))) + (begin + (report-sync-progress! + progress + (string-append "skip " remote)) + (inc-summary! summary "SkippedFiles")) + (begin + (report-sync-progress! + progress + (string-append "upload " remote)) + (upload-one-file! + client + bucket + config + drive-key + manifest + local + remote + summary + chunk-size) + (report-sync-progress! + progress + (string-append "uploaded " remote)))))) local-files) (when delete-missing? (let* ([remote-extra @@ -1744,8 +1844,11 @@ (remove-entries-by-paths (manifest-entries manifest) remove-paths)))) + (report-sync-progress! progress "committing manifest") (store-manifest! client bucket config drive-key manifest manifest-etag) + (report-sync-progress! progress "cleaning stale objects") (delete-summary-stale-objects! client bucket summary) + (report-sync-progress! progress "done") summary))))))) (define (jdrive-s3-list! profile state-root vault-password yubikey-material path-prefix) --- a/jpkg.sexp +++ b/jpkg.sexp @@ -1,6 +1,6 @@ (package (name "@ober/jerboa-drive") - (version "1.0.0") + (version "1.2.0") (description "Client-side encrypted S3 drive for Jerboa") (license "MIT") (source "https://git.jerboa.sh/ober/jerboa-drive") --- a/test/test-all.ss +++ b/test/test-all.ss @@ -60,7 +60,7 @@ (test-string-contains? usage-string "s3 init --bucket B") (test-string-contains? usage-string "s3 cp [-r] SRC DST") (test-string-contains? usage-string "s3 mv [-r] REMOTE DST") - (test-string-contains? usage-string "s3 sync LOCAL REMOTE") + (test-string-contains? usage-string "s3 sync [-v] LOCAL REMOTE") (test-string-contains? usage-string "s3 check LOCAL REMOTE") (test-string-contains? usage-string "s3 mkdir REMOTE") (test-string-contains? usage-string "s3 get REMOTE LOCAL") @@ -70,6 +70,18 @@ (check "S3 object key joins prefix without leaking leading slash" (string=? (jdrive-s3-object-key "/backup/" "/dir/file") "backup/dir/file")) +(check "S3 remote locators accept canonical jd and legacy jdrive prefixes" + (let ([key (make-bytevector 32 7)]) + (and + (string=? + (jdrive-s3-object-id key "jd:urls") + (jdrive-s3-object-id key "/urls")) + (string=? + (jdrive-s3-object-id key "jd:/urls") + (jdrive-s3-object-id key "/urls")) + (string=? + (jdrive-s3-object-id key "jdrive:urls") + (jdrive-s3-object-id key "/urls"))))) (check "S3 status reports backend config without requiring credentials" (let* ([status