Add bidirectional encrypted S3 copy
ober
2a8f0c1ab44c53a32eae6d421266eb096c867522
--- a/README.md +++ b/README.md @@ -61,6 +61,9 @@ Daily operations: jdrive s3 mkdir /backups jdrive s3 cp ./notes.txt /backups/notes.txt jdrive s3 cp -r ./backup-dir /backups +jdrive s3 cp /backups/notes.txt ./notes-restored.txt +jdrive s3 cp -r /backups ./restored-backups +jdrive s3 cp -r /backups /archive jdrive s3 sync ./backup-dir /backups --delete jdrive s3 check ./backup-dir /backups jdrive s3 ls /backups @@ -76,7 +79,10 @@ profile was initialized with `--yubikey-mode piv`. Files are stored as independently encrypted chunks by default. Override the 64 MiB default with `--chunk-size N` for testing or provider tuning. `sync` and -`check` support simple `*` wildcards through repeated `--include PATTERN` and +`cp` treats paths beginning with `/` as encrypted remote paths unless the source +is an existing local absolute path. That supports local-to-remote, +remote-to-local, and remote-to-remote encrypted copies. `check` and `sync` +support simple `*` wildcards through repeated `--include PATTERN` and `--exclude PATTERN`. ## Runtime Status @@ -320,7 +326,7 @@ libraries, and a wrapper that sets the runtime library search path. - Prove the encrypted S3 live integration against each target provider. - Add true S3 multipart per encrypted chunk for provider-optimized very large chunks. -- Add full sync/check semantics, include/exclude filters, and conflict policy. +- Add multi-writer conflict policy and provider-side conditional manifest writes. - Finish daemon control socket/background lifecycle and durable operation queue. - Add richer metadata update operations beyond size-changing writes and rename/move. --- a/docs/jerboa-drive-plan.md +++ b/docs/jerboa-drive-plan.md @@ -110,7 +110,9 @@ Completed so far: - opaque encrypted chunk object keys - chunked streaming uploads with streaming SHA-256 - ranged reads across encrypted chunks - - recursive upload + - recursive local-to-remote upload + - recursive remote-to-local copy + - recursive remote-to-remote copy with re-encryption for destination paths - local-to-remote sync - local-vs-remote check - simple `*` include/exclude filters @@ -318,7 +320,7 @@ Path rules: detected as existing local filesystem paths. - For ambiguous `cp`, use existence checks: - local existing path -> upload - - remote `/path` source -> download + - remote `/path` source -> download or remote-to-remote copy Backend config: - `jdrive init` writes `~/.jdrive/profiles/<profile>/config.json`. --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -61,9 +61,9 @@ Implemented today: - Optional encrypted Proton username/password vault protected by a separate password you provide. - S3-compatible encrypted drive profile under `~/.jdrive` with encrypted - manifest, chunked encrypted file objects, recursive upload, sync, check, - list, ranged cat/get, rm, checksum verification, retry wrappers, and optional - built-in YubiKey PIV unlock material. + manifest, chunked encrypted file objects, recursive bidirectional copy, sync, + check, list, ranged cat/get, rm, checksum verification, retry wrappers, and + optional built-in YubiKey PIV unlock material. - Host binary build, host bundle build, and Linux/FreeBSD cross-build targets. Important current limitations: @@ -128,6 +128,9 @@ Copy, list, read, restore, and remove files: jdrive s3 mkdir /backups jdrive s3 cp ./notes.txt /backups/notes.txt jdrive s3 cp -r ./backup-dir /backups +jdrive s3 cp /backups/notes.txt ./notes-restored.txt +jdrive s3 cp -r /backups ./restored-backups +jdrive s3 cp -r /backups /archive jdrive s3 sync ./backup-dir /backups --delete jdrive s3 check ./backup-dir /backups jdrive s3 ls /backups @@ -140,6 +143,10 @@ jdrive s3 rm -r /backups/backup-dir The bucket sees opaque object keys and encrypted bytes. Filenames and directory structure live in the encrypted manifest. +For `s3 cp`, paths beginning with `/` are encrypted remote paths unless the +source is an existing local absolute path. `sync` and `check` are intentionally +local-to-remote tree operations. + Tuning and filters: ```sh --- a/protonstorage/cli.ss +++ b/protonstorage/cli.ss @@ -46,7 +46,7 @@ " mount --mountpoint M Mount decrypted Drive tree\n" " s3 status Show S3 backend configuration\n" " s3 init --bucket B Create encrypted S3 drive profile\n" - " s3 cp [-r] LOCAL REMOTE Copy local data to encrypted S3 drive\n" + " s3 cp [-r] SRC DST Copy to/from encrypted S3 drive\n" " s3 ls [REMOTE] List encrypted S3 drive manifest\n" " doctor Show implementation status\n" "\n" @@ -79,7 +79,7 @@ " s3 init --bucket B Store encrypted S3 drive key in ~/.jdrive\n" " s3 profile-status Show local encrypted S3 profile state\n" " s3 unlock-test Verify S3 profile password/YubiKey unlock\n" - " s3 cp [-r] LOCAL REMOTE Upload a file/tree to encrypted S3\n" + " s3 cp [-r] SRC DST Copy local/remote file/tree through encrypted S3\n" " s3 sync LOCAL REMOTE Sync local tree to encrypted S3\n" " s3 check LOCAL REMOTE Compare local tree to encrypted S3\n" " s3 ls [REMOTE] List encrypted S3 paths\n" @@ -705,6 +705,14 @@ (define (recursive-upload? opts) (or (opt opts "-r") (opt opts "--recursive"))) + (define (s3-source-remote? path) + (and (remote-path? path) + (not (file-exists? path)))) + + (define (s3-destination-remote? source-remote? path) + (and (remote-path? path) + (not (and source-remote? (file-exists? path))))) + (define (remote-folder-exists? session root-context path) (guard (e [#t #f]) (proton-drive-resolve-folder-path session root-context path) @@ -1247,9 +1255,34 @@ password material (car pos))))] - [(or (string=? action "cp") (string=? action "put")) + [(string=? action "cp") + (unless (= (length pos) 2) + (die 2 "usage: jdrive s3 cp [-r] SRC DST")) + (let* ([password (s3-vault-password-from-opts opts)] + [material (s3-yubikey-material-for-profile opts)] + [source (car pos)] + [destination (cadr pos)] + [source-remote? (s3-source-remote? source)] + [destination-remote? + (s3-destination-remote? source-remote? destination)]) + (print-json + (jdrive-s3-copy! + (profile-option opts) + (state-root-option opts) + password + material + source + destination + source-remote? + destination-remote? + (recursive-upload? opts) + (option-number + opts + "--chunk-size" + jdrive-s3-default-chunk-size))))] + [(string=? action "put") (unless (= (length pos) 2) - (die 2 "usage: jdrive s3 cp [-r] LOCAL_PATH REMOTE_PATH")) + (die 2 "usage: jdrive s3 put [-r] LOCAL_PATH REMOTE_PATH")) (let* ([password (s3-vault-password-from-opts opts)] [material (s3-yubikey-material-for-profile opts)]) (print-json --- a/protonstorage/s3/drive.ss +++ b/protonstorage/s3/drive.ss @@ -25,6 +25,7 @@ jdrive-s3-put-local-path! jdrive-s3-check-local-path! jdrive-s3-sync-local-to-remote! + jdrive-s3-copy! jdrive-s3-cat-file-bytes jdrive-s3-cat-file-range-bytes jdrive-s3-get-file! @@ -1369,6 +1370,422 @@ index)))) plaintext)) + (define (destination-for-remote-file remote-path destination) + (if (path-looks-like-directory? destination) + (remote-path-join + (remote-dir-normalize destination) + (path-basename remote-path)) + (remote-path-normalize destination))) + + (define (destination-for-remote-directory remote-path destination) + (if (string=? (remote-dir-normalize remote-path) "/") + (remote-dir-normalize destination) + (remote-path-join + (remote-dir-normalize destination) + (path-basename remote-path)))) + + (define (local-destination-for-remote-file remote-path destination) + (if (or (path-looks-like-directory? destination) + (and (file-exists? destination) (file-directory? destination))) + (path-join destination (path-basename remote-path)) + destination)) + + (define (local-destination-for-remote-directory remote-path destination) + (if (string=? (remote-dir-normalize remote-path) "/") + destination + (path-join destination (path-basename remote-path)))) + + (define (remote-relative-path root path) + (let ([root (remote-dir-normalize root)] + [path (remote-path-normalize path)]) + (cond + [(string=? root "/") (strip-leading-slashes path)] + [(string=? root path) ""] + [else (substring path (+ (string-length root) 1) (string-length path))]))) + + (define (manifest-directory-exists? manifest path) + (string-list-member? (remote-dir-normalize path) (manifest-directories manifest))) + + (define (manifest-has-tree? manifest remote-root) + (let ([root (remote-dir-normalize remote-root)]) + (or (manifest-directory-exists? manifest root) + (let loop ([xs (manifest-entries manifest)]) + (cond + [(null? xs) #f] + [(under-prefix? root (entry-path (car xs))) #t] + [else (loop (cdr xs))]))))) + + (define (entry-file-sha256 entry) + (jmaybe entry "Sha256" "")) + + (define (verify-stream-sha256! who entry ctx path) + (let ([expected (entry-file-sha256 entry)]) + (when (string-present? expected) + (let ([actual (sha256-context-final-base64 ctx)]) + (unless (string=? expected actual) + (error who "decrypted file checksum mismatch" path)))))) + + (define (write-entry-to-port! client bucket drive-key path entry out summary) + (if (entry-chunked? entry) + (let* ([object-id (jdrive-s3-object-id drive-key path)] + [ctx (sha256-context-start)]) + (dynamic-wind + (lambda () (void)) + (lambda () + (let loop ([chunks (entry-chunks entry)]) + (if (null? chunks) + (verify-stream-sha256! + 'write-entry-to-port! + entry + ctx + path) + (let* ([chunk (car chunks)] + [plain + (decrypt-chunk-entry + client + bucket + drive-key + path + object-id + chunk)]) + (digest-update! ctx plain) + (put-bytevector out plain) + (add-summary! summary "CopiedBytes" (bytevector-length plain)) + (inc-summary! summary "CopiedChunks") + (loop (cdr chunks)))))) + (lambda () (free-digest-ctx ctx)))) + (let ([plain (decrypt-single-entry client bucket drive-key path entry)]) + (put-bytevector out plain) + (add-summary! summary "CopiedBytes" (bytevector-length plain)) + (inc-summary! summary "CopiedChunks")))) + + (define (copy-entry-to-local! client bucket drive-key path entry local-path summary) + (ensure-directory! (parent-directory local-path)) + (call-with-port + (open-file-output-port local-path (file-options no-fail) (buffer-mode block)) + (lambda (out) + (write-entry-to-port! client bucket drive-key path entry out summary))) + (inc-summary! summary "CopiedFiles")) + + (define (copy-remote-file-to-local! client bucket drive-key manifest remote-path local-destination summary) + (let* ([path (remote-path-normalize remote-path)] + [entry (manifest-find-file manifest path)]) + (unless entry + (error 'copy-remote-file-to-local! "remote file not found" path)) + (let ([local-path + (local-destination-for-remote-file path local-destination)]) + (copy-entry-to-local! client bucket drive-key path entry local-path summary) + local-path))) + + (define (copy-remote-tree-to-local! client bucket drive-key manifest remote-root local-destination summary) + (let* ([root (remote-dir-normalize remote-root)] + [local-root (local-destination-for-remote-directory root local-destination)]) + (ensure-directory! local-root) + (inc-summary! summary "CreatedDirectories") + (for-each + (lambda (dir) + (when (under-prefix? root dir) + (let ([rel (remote-relative-path root dir)]) + (unless (string=? rel "") + (ensure-directory! (path-join local-root rel)) + (inc-summary! summary "CreatedDirectories"))))) + (manifest-directories manifest)) + (for-each + (lambda (entry) + (let* ([path (entry-path entry)] + [rel (remote-relative-path root path)] + [local-path (path-join local-root rel)]) + (copy-entry-to-local! client bucket drive-key path entry local-path summary))) + (manifest-file-entries-under manifest root '() '())) + local-root)) + + (define (copy-single-entry-remote-to-remote! + client bucket config drive-key manifest source-path dest-path entry summary) + (let* ([plain (decrypt-single-entry client bucket drive-key source-path entry)]) + (call-with-values + (lambda () (jdrive-s3-encrypt-file-bytes drive-key dest-path plain)) + (lambda (object-id sealed sha256) + (let* ([object-key (object-storage-key config object-id)] + [old-entry (manifest-find-file manifest dest-path)]) + (with-s3-retry + 'copy-single-entry-remote-to-remote! + (lambda () + (put-object-bytes + client + bucket + object-key + sealed + 'content-type: file-content-type))) + (jdrive-s3-manifest-add-file + manifest + dest-path + object-key + (bytevector-length plain) + sha256) + (record-stale-object-keys! summary old-entry (list object-key)) + (inc-summary! summary "CopiedFiles") + (inc-summary! summary "CopiedChunks") + (add-summary! summary "CopiedBytes" (bytevector-length plain))))))) + + (define (copy-chunked-entry-remote-to-remote! + client bucket config drive-key manifest source-path dest-path entry summary) + (let* ([object-id (jdrive-s3-object-id drive-key dest-path)] + [old-entry (manifest-find-file manifest dest-path)] + [chunk-size (jmaybe entry "ChunkSize" jdrive-s3-default-chunk-size)] + [ctx (sha256-context-start)]) + (dynamic-wind + (lambda () (void)) + (lambda () + (let loop ([source-chunks (entry-chunks entry)] + [index 0] + [offset 0] + [chunks '()]) + (if (null? source-chunks) + (begin + (verify-stream-sha256! + 'copy-chunked-entry-remote-to-remote! + entry + ctx + source-path) + (jdrive-s3-manifest-add-file + manifest + dest-path + #f + offset + (entry-file-sha256 entry) + (reverse chunks) + chunk-size) + (record-stale-object-keys! + summary + old-entry + (map + (lambda (chunk) + (chunk-field chunk "ObjectKey" "")) + (reverse chunks))) + (inc-summary! summary "CopiedFiles") + (add-summary! summary "CopiedChunks" (length chunks)) + (add-summary! summary "CopiedBytes" offset)) + (let* ([plain + (decrypt-chunk-entry + client + bucket + drive-key + source-path + (jdrive-s3-object-id drive-key source-path) + (car source-chunks))]) + (digest-update! ctx plain) + (call-with-values + (lambda () + (jdrive-s3-encrypt-chunk-bytes + drive-key + dest-path + object-id + index + plain)) + (lambda (sealed chunk-sha256) + (let ([object-key (chunk-storage-key config object-id index)]) + (with-s3-retry + 'copy-chunked-entry-remote-to-remote! + (lambda () + (put-object-bytes + client + bucket + object-key + sealed + 'content-type: file-content-type))) + (loop + (cdr source-chunks) + (+ index 1) + (+ offset (bytevector-length plain)) + (cons + (chunk-entry + index + object-key + (bytevector-length plain) + (bytevector-length sealed) + chunk-sha256) + chunks))))))))) + (lambda () (free-digest-ctx ctx))))) + + (define (copy-entry-remote-to-remote! + client bucket config drive-key manifest source-path dest-path entry summary) + (manifest-add-parent-dirs! manifest dest-path) + (if (entry-chunked? entry) + (copy-chunked-entry-remote-to-remote! + client + bucket + config + drive-key + manifest + source-path + dest-path + entry + summary) + (copy-single-entry-remote-to-remote! + client + bucket + config + drive-key + manifest + source-path + dest-path + entry + summary))) + + (define (copy-remote-tree-to-remote! + client bucket config drive-key manifest source-root destination summary) + (let* ([root (remote-dir-normalize source-root)] + [dest-root (destination-for-remote-directory root destination)]) + (jdrive-s3-manifest-add-directory manifest dest-root) + (inc-summary! summary "CreatedDirectories") + (for-each + (lambda (dir) + (when (under-prefix? root dir) + (let ([rel (remote-relative-path root dir)]) + (unless (string=? rel "") + (jdrive-s3-manifest-add-directory + manifest + (remote-path-join dest-root rel)) + (inc-summary! summary "CreatedDirectories"))))) + (manifest-directories manifest)) + (for-each + (lambda (entry) + (let* ([source-path (entry-path entry)] + [rel (remote-relative-path root source-path)] + [dest-path (remote-path-join dest-root rel)]) + (copy-entry-remote-to-remote! + client + bucket + config + drive-key + manifest + source-path + dest-path + entry + summary))) + (manifest-file-entries-under manifest root '() '())) + dest-root)) + + (define (jdrive-s3-copy! + profile state-root vault-password yubikey-material source destination + source-remote? destination-remote? recursive? chunk-size) + (cond + [(and (not source-remote?) destination-remote?) + (jdrive-s3-put-local-path! + profile + state-root + vault-password + yubikey-material + source + destination + recursive? + chunk-size)] + [(and source-remote? (not destination-remote?)) + (with-unlocked-profile + profile + state-root + vault-password + yubikey-material + (lambda (config vault drive-key client bucket) + (let* ([manifest (load-manifest client bucket config drive-key)] + [path (remote-path-normalize source)] + [entry (manifest-find-file manifest path)] + [summary + (json-object + "Backend" "s3" + "Profile" (jdrive-s3-profile-name profile) + "Source" path + "Destination" destination + "Direction" "remote-to-local" + "Recursive" (if recursive? #t #f) + "CopiedFiles" 0 + "CopiedBytes" 0 + "CopiedChunks" 0 + "CreatedDirectories" 0)]) + (cond + [entry + (copy-remote-file-to-local! + client + bucket + drive-key + manifest + path + destination + summary)] + [(manifest-has-tree? manifest path) + (unless recursive? + (error 'jdrive-s3-copy! + "remote source is a directory; use -r or --recursive" + path)) + (copy-remote-tree-to-local! + client + bucket + drive-key + manifest + path + destination + summary)] + [else (error 'jdrive-s3-copy! "remote source not found" path)]) + summary)))] + [(and source-remote? destination-remote?) + (with-unlocked-profile + profile + state-root + vault-password + yubikey-material + (lambda (config vault drive-key client bucket) + (let* ([manifest (load-manifest client bucket config drive-key)] + [source-path (remote-path-normalize source)] + [entry (manifest-find-file manifest source-path)] + [summary + (json-object + "Backend" "s3" + "Profile" (jdrive-s3-profile-name profile) + "Source" source-path + "Destination" (remote-path-normalize destination) + "Direction" "remote-to-remote" + "Recursive" (if recursive? #t #f) + "CopiedFiles" 0 + "CopiedBytes" 0 + "CopiedChunks" 0 + "RemovedStaleObjects" 0 + "CreatedDirectories" 0)]) + (cond + [entry + (copy-entry-remote-to-remote! + client + bucket + config + drive-key + manifest + source-path + (destination-for-remote-file source-path destination) + entry + summary)] + [(manifest-has-tree? manifest source-path) + (unless recursive? + (error 'jdrive-s3-copy! + "remote source is a directory; use -r or --recursive" + source-path)) + (copy-remote-tree-to-remote! + client + bucket + config + drive-key + manifest + source-path + destination + summary)] + [else (error 'jdrive-s3-copy! "remote source not found" source-path)]) + (store-manifest! client bucket config drive-key manifest) + (delete-summary-stale-objects! client bucket summary) + summary)))] + [else + (error 'jdrive-s3-copy! + "at least one s3 cp argument must be an encrypted remote path" + source + destination)])) + (define (jdrive-s3-cat-file-bytes profile state-root vault-password yubikey-material remote-path) (jdrive-s3-cat-file-range-bytes profile --- a/test/integration-s3.ss +++ b/test/integration-s3.ss @@ -127,6 +127,8 @@ (string-append "/tmp/jdrive-s3-integration-local-" (number->string (get-process-id)))) (define restored (string-append "/tmp/jdrive-s3-integration-restored-" (number->string (get-process-id)) ".txt")) +(define restored-tree + (string-append "/tmp/jdrive-s3-integration-restored-tree-" (number->string (get-process-id)))) (define profile "integration") (ensure-directory! local-root) @@ -190,6 +192,51 @@ restored) (check "get writes restored file" (file-exists? restored)) +(jdrive-s3-copy! + profile state-root vault-password #vu8() + "/a.txt" + "/copy/" + #t + #t + #f + chunk-size) +(check "remote file copy decrypts" + (bytevector=? + (string->utf8 "alpha plaintext") + (jdrive-s3-cat-file-range-bytes + profile state-root vault-password #vu8() + "/copy/a.txt" + 0 + -1))) + +(jdrive-s3-copy! + profile state-root vault-password #vu8() + "/nested" + restored-tree + #t + #f + #t + chunk-size) +(check "remote tree copy restores local file" + (file-exists? (path-join (path-join restored-tree "nested") "b.txt"))) + +(jdrive-s3-copy! + profile state-root vault-password #vu8() + "/nested" + "/mirror" + #t + #t + #t + chunk-size) +(check "remote tree copy decrypts" + (bytevector=? + (string->utf8 "nested plaintext for chunking") + (jdrive-s3-cat-file-range-bytes + profile state-root vault-password #vu8() + "/mirror/nested/b.txt" + 0 + -1))) + (jdrive-s3-remove! profile state-root vault-password #vu8() "/" --- a/test/test-all.ss +++ b/test/test-all.ss @@ -119,7 +119,7 @@ (test-string-contains? usage-string "s3 status") (test-string-contains? usage-string "s3 target --bucket B --key K") (test-string-contains? usage-string "s3 init --bucket B") - (test-string-contains? usage-string "s3 cp [-r] LOCAL REMOTE") + (test-string-contains? usage-string "s3 cp [-r] SRC DST") (test-string-contains? usage-string "s3 sync LOCAL REMOTE") (test-string-contains? usage-string "s3 check LOCAL REMOTE") (test-string-contains? usage-string "s3 mkdir REMOTE")