Add S3 backend foundation for jdrive
ober
d6e342a1c57cc7ba340d08bfa9a2c4fc561b4170
--- a/Makefile +++ b/Makefile @@ -8,26 +8,32 @@ JERBOA_PGP_DIR ?= $(realpath $(CURDIR)/../jerboa-pgp) JERBOA_CRYPTO_DIR ?= $(realpath $(CURDIR)/../jerboa-crypto) JERBOA_FUSE_DIR ?= $(realpath $(CURDIR)/../jerboa-fuse) JERBOA_MAIL_DIR ?= $(realpath $(CURDIR)/../jerboa-mail) -LIBDIRS := $(CURDIR):$(JERBOA_PROTON_BRIDGE_DIR):$(JERBOA_YUBIKEY_DIR):$(JERBOA_YUBIKEY_DIR)/lib:$(JERBOA_PGP_DIR):$(JERBOA_CRYPTO_DIR)/lib:$(JERBOA_FUSE_DIR)/lib:$(JERBOA_MAIL_DIR):$(JERBOA_HTTPS_DIR)/lib:$(JERBOA_SSL_DIR)/lib:$(JERBOA_HOME)/lib +JERBOA_AWS_DIR ?= $(realpath $(CURDIR)/../jerboa-aws) +LIBDIRS := $(CURDIR):$(JERBOA_AWS_DIR)/lib:$(JERBOA_PROTON_BRIDGE_DIR):$(JERBOA_YUBIKEY_DIR):$(JERBOA_YUBIKEY_DIR)/lib:$(JERBOA_PGP_DIR):$(JERBOA_CRYPTO_DIR)/lib:$(JERBOA_FUSE_DIR)/lib:$(JERBOA_MAIL_DIR):$(JERBOA_HTTPS_DIR)/lib:$(JERBOA_SSL_DIR)/lib:$(JERBOA_HOME)/lib XC_LIBDIRS := $(LIBDIRS):$(JERBOA_HOME)/.chez-cross-ta6le/lib:$(JERBOA_HOME)/.chez-cross-tarm64le/lib:$(JERBOA_HOME)/.chez-cross-ta6fb/lib +JERBOA_NATIVE_DIR ?= $(JERBOA_HOME)/jerboa-native-rs/target/release RUN_ENV := JERBOA_HOME=$(JERBOA_HOME) \ JERBOA_SSL_LIB=$(JERBOA_SSL_DIR) \ JERBOA_CRYPTO_LIB=$(JERBOA_CRYPTO_DIR) \ - DYLD_LIBRARY_PATH=$(JERBOA_CRYPTO_DIR):$(JERBOA_FUSE_DIR):$(JERBOA_PROTON_BRIDGE_DIR)/proton-bridge-native/target/release:$${DYLD_LIBRARY_PATH:-} \ - LD_LIBRARY_PATH=$(JERBOA_CRYPTO_DIR):$(JERBOA_FUSE_DIR):$(JERBOA_PROTON_BRIDGE_DIR)/proton-bridge-native/target/release:$${LD_LIBRARY_PATH:-} + DYLD_LIBRARY_PATH=$(JERBOA_NATIVE_DIR):$(JERBOA_CRYPTO_DIR):$(JERBOA_FUSE_DIR):$(JERBOA_PROTON_BRIDGE_DIR)/proton-bridge-native/target/release:$${DYLD_LIBRARY_PATH:-} \ + DYLD_FALLBACK_LIBRARY_PATH=$(JERBOA_NATIVE_DIR):$${DYLD_FALLBACK_LIBRARY_PATH:-} \ + LD_LIBRARY_PATH=$(JERBOA_NATIVE_DIR):$(JERBOA_CRYPTO_DIR):$(JERBOA_FUSE_DIR):$(JERBOA_PROTON_BRIDGE_DIR)/proton-bridge-native/target/release:$${LD_LIBRARY_PATH:-} BINARY_OUTPUT ?= protonstorage-bin LINUX_AMD64_CC ?= x86_64-linux-musl-gcc LINUX_ARM64_CC ?= aarch64-linux-musl-gcc FREEBSD_AMD64_CC ?= $(JERBOA_HOME)/support/cross-cc-freebsd-amd64 HOST_OS := $(shell uname -s | tr '[:upper:]' '[:lower:]') HOST_ARCH := $(shell uname -m) -BUNDLE_OUTPUT ?= dist/protonstorage-$(HOST_OS)-$(HOST_ARCH) +BUNDLE_OUTPUT ?= dist/jerboa-drive-$(HOST_OS)-$(HOST_ARCH) +INSTALL_PREFIX ?= $(HOME)/.local +INSTALL_BINDIR ?= $(INSTALL_PREFIX)/bin +INSTALL_OPTDIR ?= $(INSTALL_PREFIX)/opt/jerboa-drive -.PHONY: help run test integration doctor binary bundle linux linux-amd64 linux-arm64 freebsd freebsd-amd64 clean +.PHONY: help run test integration doctor binary bundle install linux linux-amd64 linux-arm64 freebsd freebsd-amd64 clean .DEFAULT_GOAL := help help: - @echo "jerboa-protonstorage" + @echo "jerboa-drive" @echo "" @echo "Development:" @echo " make run ARGS='help' Run the CLI" @@ -40,6 +46,7 @@ help: @echo " make Show this help" @echo " make binary Build ./$(BINARY_OUTPUT) for this host" @echo " make bundle Package ./$(BUNDLE_OUTPUT).tar.gz" + @echo " make install Install jdrive to $(INSTALL_BINDIR)" @echo " make linux-amd64 Cross-build ./protonstorage-linux-amd64" @echo " make linux-arm64 Cross-build ./protonstorage-linux-arm64" @echo " make freebsd-amd64 Cross-build ./protonstorage-freebsd-amd64" @@ -53,8 +60,11 @@ help: @echo " JERBOA_CRYPTO_DIR = $(JERBOA_CRYPTO_DIR)" @echo " JERBOA_FUSE_DIR = $(JERBOA_FUSE_DIR)" @echo " JERBOA_MAIL_DIR = $(JERBOA_MAIL_DIR)" + @echo " JERBOA_AWS_DIR = $(JERBOA_AWS_DIR)" @echo " JERBOA_HTTPS_DIR = $(JERBOA_HTTPS_DIR)" @echo " JERBOA_SSL_DIR = $(JERBOA_SSL_DIR)" + @echo " JERBOA_NATIVE_DIR = $(JERBOA_NATIVE_DIR)" + @echo " INSTALL_PREFIX = $(INSTALL_PREFIX)" run: $(RUN_ENV) \ @@ -83,6 +93,22 @@ bundle: binary JERBOA_SSL_DIR="$(JERBOA_SSL_DIR)" \ bash support/package-bundle.sh "$(BINARY_OUTPUT)" "$(BUNDLE_OUTPUT)" +install: bundle + mkdir -p "$(INSTALL_BINDIR)" "$(INSTALL_OPTDIR)" + rsync -a --delete "$(BUNDLE_OUTPUT)/" "$(INSTALL_OPTDIR)/" + { \ + echo '#!/usr/bin/env bash'; \ + echo 'set -euo pipefail'; \ + echo 'DIR="$(INSTALL_OPTDIR)"'; \ + echo 'export DYLD_LIBRARY_PATH="$$DIR/lib:$${DYLD_LIBRARY_PATH:-}"'; \ + echo 'export LD_LIBRARY_PATH="$$DIR/lib:$${LD_LIBRARY_PATH:-}"'; \ + echo 'export JERBOA_SSL_LIB="$$DIR/lib"'; \ + echo 'export JERBOA_CRYPTO_LIB="$$DIR/lib"'; \ + echo 'exec "$$DIR/bin/protonstorage" "$$@"'; \ + } > "$(INSTALL_BINDIR)/jdrive" + chmod +x "$(INSTALL_BINDIR)/jdrive" + @echo "Installed jdrive -> $(INSTALL_BINDIR)/jdrive" + linux: linux-amd64 linux-amd64: --- a/README.md +++ b/README.md @@ -1,10 +1,11 @@ -# jerboa-protonstorage +# jerboa-drive -Jerboa-native Proton Drive storage access. +Jerboa-native encrypted remote drive and storage access. This project is intended to become a single deployable Jerboa application for -Proton Drive, with YubiKey-gated access and no rclone or Go helper in the -runtime path. +remote storage, with YubiKey-gated access and no rclone or Go helper in the +runtime path. The Proton Drive backend is currently experimental; the primary +next backend is S3-compatible storage with client-side encryption. ## User Documentation @@ -13,6 +14,23 @@ state layout, authentication, encrypted credential vault setup, listing remote files, reading and writing Drive content, FUSE mounts, configuration variables, and troubleshooting. +Short installed-binary form: + +```sh +jdrive login --username you@example.com +jdrive ls --username you@example.com +jdrive creds store --profile default --username you@example.com +jdrive tree --profile default --max-depth 3 +``` + +The `protonstorage` command and historical `drive ...` prefix still work, but +neither is required. If Proton requires TOTP and you did not pass +`--totp-code`, `jdrive` prompts for the current code. + +After `jdrive creds store`, commands such as `jdrive ls`, `jdrive volumes`, and +`jdrive mount` use the selected profile's encrypted credentials automatically +when you do not pass `--username` or token options. + ## Runtime Status Current runtime code is Scheme/Jerboa only: @@ -108,14 +126,16 @@ make run ARGS='drive tree-decrypted --username you@example.com --max-depth 3' For local CLI use, Proton login credentials can be stored in the selected profile as an encrypted vault protected by a separate vault password. The vault uses `scrypt` and `chacha20-poly1305`; the vault password is still required to -unlock stored credentials. Profiles live under `~/.jproton/profiles/<profile>/` -by default. +unlock stored credentials. After the first successful full login, the encrypted +vault also stores Proton's reusable session tokens and key pass so later +commands can refresh the session without TOTP. Profiles live under +`~/.jproton/profiles/<profile>/` by default. ```sh make run ARGS='drive credentials store --profile default' make run ARGS='drive credentials status --profile default' make run ARGS='drive credentials unlock-test --profile default' -make run ARGS='drive root-children-decrypted --profile default --use-stored-credentials' +make run ARGS='drive root-children-decrypted --profile default' ``` For automation, set `PROTONSTORAGE_VAULT_PASSWORD` or pass new file mode 100755 --- /dev/null +++ b/bin/jdrive @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +DIR="$(cd "$(dirname "$0")/.." && pwd)" +exec "$DIR/bin/protonstorage" "$@" --- a/bin/protonstorage +++ b/bin/protonstorage @@ -10,16 +10,19 @@ JERBOA_PGP_DIR="${JERBOA_PGP_DIR:-"$DIR/../jerboa-pgp"}" JERBOA_CRYPTO_DIR="${JERBOA_CRYPTO_DIR:-"$DIR/../jerboa-crypto"}" JERBOA_FUSE_DIR="${JERBOA_FUSE_DIR:-"$DIR/../jerboa-fuse"}" JERBOA_MAIL_DIR="${JERBOA_MAIL_DIR:-"$DIR/../jerboa-mail"}" +JERBOA_AWS_DIR="${JERBOA_AWS_DIR:-"$DIR/../jerboa-aws"}" JERBOA_HTTPS_DIR="${JERBOA_HTTPS_DIR:-"$DIR/../jerboa-https"}" JERBOA_SSL_DIR="${JERBOA_SSL_DIR:-"$DIR/../jerboa-ssl"}" +JERBOA_NATIVE_DIR="${JERBOA_NATIVE_DIR:-"$JERBOA_HOME/jerboa-native-rs/target/release"}" export JERBOA_HOME export JERBOA_SSL_LIB="$JERBOA_SSL_DIR" export JERBOA_CRYPTO_LIB="$JERBOA_CRYPTO_DIR" PROTON_BRIDGE_NATIVE_DIR="$JERBOA_PROTON_BRIDGE_DIR/proton-bridge-native/target/release" -export DYLD_LIBRARY_PATH="$JERBOA_CRYPTO_DIR:$JERBOA_FUSE_DIR:$PROTON_BRIDGE_NATIVE_DIR:${DYLD_LIBRARY_PATH:-}" -export LD_LIBRARY_PATH="$JERBOA_CRYPTO_DIR:$JERBOA_FUSE_DIR:$PROTON_BRIDGE_NATIVE_DIR:${LD_LIBRARY_PATH:-}" +export DYLD_LIBRARY_PATH="$JERBOA_NATIVE_DIR:$JERBOA_CRYPTO_DIR:$JERBOA_FUSE_DIR:$PROTON_BRIDGE_NATIVE_DIR:${DYLD_LIBRARY_PATH:-}" +export DYLD_FALLBACK_LIBRARY_PATH="$JERBOA_NATIVE_DIR:${DYLD_FALLBACK_LIBRARY_PATH:-}" +export LD_LIBRARY_PATH="$JERBOA_NATIVE_DIR:$JERBOA_CRYPTO_DIR:$JERBOA_FUSE_DIR:$PROTON_BRIDGE_NATIVE_DIR:${LD_LIBRARY_PATH:-}" exec "$SCHEME" -q \ - --libdirs "$DIR:$JERBOA_PROTON_BRIDGE_DIR:$JERBOA_YUBIKEY_DIR:$JERBOA_YUBIKEY_DIR/lib:$JERBOA_PGP_DIR:$JERBOA_CRYPTO_DIR/lib:$JERBOA_FUSE_DIR/lib:$JERBOA_MAIL_DIR:$JERBOA_HTTPS_DIR/lib:$JERBOA_SSL_DIR/lib:$JERBOA_HOME/lib" \ + --libdirs "$DIR:$JERBOA_AWS_DIR/lib:$JERBOA_PROTON_BRIDGE_DIR:$JERBOA_YUBIKEY_DIR:$JERBOA_YUBIKEY_DIR/lib:$JERBOA_PGP_DIR:$JERBOA_CRYPTO_DIR/lib:$JERBOA_FUSE_DIR/lib:$JERBOA_MAIL_DIR:$JERBOA_HTTPS_DIR/lib:$JERBOA_SSL_DIR/lib:$JERBOA_HOME/lib" \ --script "$DIR/main.ss" -- "$@" new file mode 100644 --- /dev/null +++ b/docs/jerboa-drive-plan.md @@ -0,0 +1,372 @@ +# jerboa-drive Implementation Plan + +This plan assumes all implementation is model-written. Human work should be limited +to granting credentials, touching a YubiKey, choosing bucket/provider values, and +reviewing results. + +## Current Starting State + +- Current local repo path: `/Users/user/mine/jerboa-drive`. +- Current sourcehut remote: `git@git.sr.ht:~lisp/jerboa-drive`. +- Sourcehut repo: `~lisp/jerboa-drive`, unlisted. +- Installed binary name remains `jdrive`. +- Current repo has dirty work from Proton Drive work. Do not discard it. +- `~/mine/jerboa-aws` already has basic S3 bucket/object APIs: + - `S3Client` + - `s3-get`, `s3-put`, `s3-delete`, `s3-head` + - `list-objects-v2`, `put-object`, `get-object` +- `jerboa-aws` does not yet have the full S3 feature set needed for a robust + encrypted remote filesystem. + +## Phase 0: Rename Repo and Preserve State + +Status: complete as of the repo rename. Keep this section as audit/history. + +1. Stop any running test upload processes: + - `pgrep -fl 'protonstorage cp|jdrive cp'` + - terminate only matching test processes. +2. Create a local checkpoint before renaming: + - inspect `git status --short` + - commit only if verification requirements can be met; otherwise leave dirty + work intact and document it in the handoff notes. +3. Rename the sourcehut repo with `hut`: + - `hut git update jerboa-protonstorage --name jerboa-drive --visibility unlisted` + - verify with `hut git show jerboa-drive` +4. Rename the local directory: + - from parent dir: `mv ~/mine/jerboa-protonstorage ~/mine/jerboa-drive` +5. Fix the git remote: + - `git remote set-url origin git@git.sr.ht:~lisp/jerboa-drive` + - verify with `git remote -v` +6. Rename project metadata and docs: + - `jerboa-protonstorage` -> `jerboa-drive` + - keep `jdrive` as the user-facing command + - move Proton-specific docs under an experimental/proton section + +Acceptance gate: +- `pwd` is `/Users/user/mine/jerboa-drive`. +- `git remote -v` points at `~lisp/jerboa-drive`. +- `hut git show jerboa-drive` succeeds and shows unlisted visibility. +- `jdrive --help` still works after reinstall. + +## Phase 1: Define Product Boundary + +Primary product: +- A single Jerboa binary, `jdrive`, implementing an encrypted remote filesystem. +- Primary backend: S3-compatible object storage via `~/mine/jerboa-aws`. +- Client-side encryption is mandatory. The bucket must never receive plaintext + filenames, metadata, file contents, or directory structure. +- Unlock requires either: + - local vault passphrase plus YubiKey touch, or + - local vault passphrase plus prompted one-time code. + +Secondary product: +- Proton Drive support remains experimental and should be isolated as a backend, + not the default path. + +Non-goals for the first S3 milestone: +- Cross-device conflict resolution. +- Multi-writer realtime sync. +- Deduplicated global chunk store across independent drives. +- GUI. + +## Phase 2: Extend `jerboa-aws` for Real S3 Drive Use + +Status: in progress. + +Completed so far: +- `std net request` now preserves response bodies as bytevectors through + `request-content` and accepts bytevector request bodies. +- `jerboa-aws` S3 request signing now includes canonical query strings. +- `jerboa-aws` supports virtual-hosted and path-style S3 targets. +- Added `get-object-bytes`, `put-object-bytes`, and `get-object-range`. +- Added multipart helper APIs: + - `create-multipart-upload` + - `upload-part` + - `complete-multipart-upload` + - `abort-multipart-upload` + - `list-parts` +- Added paginated `list-objects-v2/all`. +- Added pure no-container tests for request target construction, query + canonicalization, byte API exports, and multipart completion XML. +- `jerboa-drive` now resolves `~/mine/jerboa-aws` and exposes: + - `jdrive s3 status` + - `jdrive s3 target` + +Remaining: +- Add transient retry/backoff wrappers and apply them to idempotent S3 calls. +- Add optional live S3 integration tests gated by env vars. +- Prove multipart round-trip against a real S3-compatible bucket. + +Add or verify these `jerboa-aws` capabilities before building the drive layer: + +1. Binary-safe S3 object I/O + - `get-object-bytes` + - `put-object-bytes` + - `get-object-range` + - preserve bytevectors, not strings. + +2. S3-compatible endpoint modes + - virtual-hosted style: `bucket.endpoint/key` + - path style: `endpoint/bucket/key` + - configurable HTTPS endpoint for AWS, MinIO, Backblaze B2, Wasabi, R2, etc. + +3. Multipart upload + - `create-multipart-upload` + - `upload-part` + - `complete-multipart-upload` + - `abort-multipart-upload` + - `list-parts` + - retry failed parts without restarting the whole object. + +4. Listing helpers + - paginated `list-objects-v2` iterator + - prefix and delimiter helpers + - object existence and metadata helpers. + +5. Error and retry policy + - classify transient HTTP 408/429/5xx and network errors + - exponential backoff with jitter + - idempotent retries for reads, listings, part uploads, and aborts. + +6. Tests without Docker/Podman + - in-process fake S3 server for request signing and XML behavior + - optional real S3 integration tests gated by env vars: + - `JDRIVE_S3_TEST_BUCKET` + - `AWS_PROFILE` or explicit credentials + +Acceptance gate: +- `jerboa-aws` tests pass. +- `put-object-bytes` and `get-object-bytes` round-trip arbitrary binary data. +- multipart upload round-trips at least a 64 MiB test object. + +## Phase 3: Encrypted Drive Data Model + +Object namespace inside bucket: + +```text +<prefix>/v1/config.json +<prefix>/v1/vault/main.vault +<prefix>/v1/manifests/root.current +<prefix>/v1/manifests/<manifest-id>.enc +<prefix>/v1/chunks/<chunk-id>.enc +<prefix>/v1/tmp/<upload-id>/... +``` + +Plaintext model: +- Drive root contains directory entries. +- Every file has: + - stable file id + - encrypted basename + - size + - mode/type + - mtime/ctime + - chunk list + - plaintext SHA-256 or BLAKE3 if available +- Every chunk has: + - chunk id + - ciphertext object key + - plaintext length + - ciphertext length + - nonce + - authentication tag + - content hash + +Encryption: +- Generate a random 256-bit drive master key at `jdrive init`. +- Derive subkeys by context: + - manifest encryption key + - chunk encryption key + - filename encryption key + - metadata MAC/signing key +- Use AEAD, preferably XChaCha20-Poly1305 if available in `jerboa-crypto`; + otherwise use ChaCha20-Poly1305 with unique random nonces. +- Associated data must include: + - format version + - bucket/prefix id + - object kind + - file id + - chunk index + - plaintext length + - manifest generation id + +Manifest integrity: +- Each manifest is encrypted and authenticated. +- `root.current` points to the latest manifest id. +- Updates write new manifest first, then atomically update `root.current`. +- If S3 conditional writes are unavailable in `jerboa-aws`, emulate cautiously + and document single-writer semantics for v1. + +Acceptance gate: +- Raw bucket listing reveals no filenames or directory structure. +- Corrupting any chunk or manifest causes read failure. +- Copying a directory up and back preserves bytes, names, sizes, and mtimes. + +## Phase 4: Unlock and YubiKey Integration + +Local state directory: + +```text +~/.jdrive/ +~/.jdrive/profiles/<profile>/config.json +~/.jdrive/profiles/<profile>/session.vault +``` + +Do not use XDG paths. + +Unlock modes: + +1. Passphrase-only, for bootstrap and recovery. +2. Passphrase + TOTP code prompt. +3. Passphrase + built-in `jerboa-yubikey` touch flow. +4. Recovery key export/import. + +YubiKey plan: +- Use `~/mine/jerboa-yubikey` directly, not `ykman`. +- First inspect supported features in `jerboa-yubikey`. +- If it only supports FIDO2 assertion, implement unlock using FIDO2 with user + presence. +- If OATH/TOTP is available, support reading a named OATH entry. +- If OATH/TOTP is not available, still prompt for `--totp-code` or interactive + `TOTP code:` so the user can type the code from another machine. +- If FIDO2 hmac-secret or challenge-response support is absent, extend + `jerboa-yubikey` with a minimal API rather than shelling out to external tools. + +Vault format: +- Store encrypted drive master key locally. +- Derive vault wrapping key from: + - passphrase KDF output + - optional YubiKey/FIDO/OATH material + - per-profile random salt +- Use scrypt or Argon2 if present; otherwise extend `jerboa-crypto`. + +Acceptance gate: +- `jdrive unlock` prompts for passphrase and touch/code as configured. +- Wrong passphrase, wrong code, or missing YubiKey fails without revealing key data. +- `jdrive status` shows locked/unlocked state without printing secrets. + +## Phase 5: CLI Shape + +Keep commands short and top-level: + +```text +jdrive init --backend s3 --bucket B --prefix P --region R [--endpoint URL] +jdrive unlock [--profile default] +jdrive lock +jdrive status +jdrive ls [REMOTE] +jdrive tree [REMOTE] +jdrive mkdir REMOTE +jdrive cp [-r] SRC DST +jdrive cat REMOTE +jdrive rm [-r] REMOTE +jdrive mv OLD NEW +jdrive sync LOCAL REMOTE +jdrive mount --mountpoint PATH [--writable] +``` + +Path rules: +- Remote paths start with `/`. +- Local paths do not start with `/` when used as source unless explicitly + detected as existing local filesystem paths. +- For ambiguous `cp`, use existence checks: + - local existing path -> upload + - remote `/path` source -> download + +Backend config: +- `jdrive init` writes `~/.jdrive/profiles/<profile>/config.json`. +- AWS credentials come from `jerboa-aws` resolution first: + - env vars + - AWS profile + - explicit config +- Never store AWS secret keys in plaintext unless the user explicitly opts in; + prefer existing AWS credential files or encrypted local profile vault. + +Acceptance gate: +- New user can run `jdrive init`, `jdrive unlock`, `jdrive cp -r`, `jdrive ls`. +- Help output fits on one screen for common commands. + +## Phase 6: Filesystem Semantics + +V1 file operations: +- create directory +- upload file +- upload directory recursively +- list directory +- read file +- delete file/directory +- rename/move within same drive + +Chunking: +- Default plaintext chunk size: 64 MiB. +- Small files may be stored as one encrypted object. +- Large files use chunk objects and optional S3 multipart internally. +- Upload writes chunks first, then commits a new encrypted manifest. +- Failed uploads leave orphan chunks; add garbage collection command later. + +FUSE: +- Reuse existing `jerboa-fuse` integration from current repo. +- Make FUSE read-only first. +- Add write path after CLI `cp`, `mkdir`, `rm`, and `mv` are stable. + +Acceptance gate: +- `jdrive cp -r localdir /backup` +- `jdrive tree /backup` +- `jdrive cp -r /backup restoredir` +- `diff -r localdir restoredir` + +## Phase 7: Reliability and Recovery + +Add: +- resumable uploads using temp upload journal in `~/.jdrive` +- orphan chunk garbage collection +- manifest history +- `jdrive fsck` +- `jdrive repair --use-manifest <id>` +- `jdrive verify /path` + +Recovery model: +- The encrypted bucket plus exported recovery key is sufficient to restore on a + new machine. +- Without passphrase/YubiKey/recovery key, data is unrecoverable by design. + +Acceptance gate: +- Kill `jdrive cp -r` midway; rerun resumes or safely restarts. +- Corrupt one S3 chunk; `jdrive verify` reports the exact file/chunk. + +## Phase 8: Build, Install, Commit, Push + +Honor repository `AGENT.md`: + +- `make` prints help only. +- Before commit/push, run: + - `make` + - `make binary` + - `make linux-amd64` + - `make linux-arm64` + - `make freebsd-amd64` +- Do not use Docker or Podman. +- `make install` installs `~/.local/bin/jdrive`. + +Commit strategy: +- Commit repo rename separately from S3/encryption work if verification allows. +- Commit `jerboa-aws` extensions in `jerboa-aws`. +- Commit `jerboa-yubikey` extensions in `jerboa-yubikey`. +- Commit `jerboa-drive` app integration in `jerboa-drive`. +- Never revert unrelated dirty changes. + +## Model Handoff Checklist + +For the next model: + +1. Read `AGENT.md` in `jerboa-drive`. +2. Read `~/mine/jerboa-aws/AGENTS.md`. +3. Read `~/mine/jerboa-yubikey` docs/source before designing YubiKey unlock. +4. Inspect dirty git state before editing. +5. Do not write `.sls` user code by hand in Jerboa projects; edit `.ss` sources + and regenerate libraries with the project build. +6. Use `rg` first for code search. +7. Use `apply_patch` for manual edits. +8. Keep `jdrive` as the installed binary name. +9. Keep local state under `~/.jdrive`. +10. Build the S3 encrypted drive as the primary implementation; keep Proton as + experimental unless explicitly removed. --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -1,12 +1,48 @@ -# jerboa-protonstorage User Guide +# jerboa-drive User Guide -This guide covers day-to-day use of `jerboa-protonstorage`: how to run it, +This guide covers day-to-day use of `jerboa-drive`/`jdrive`: how to run it, where it stores local state, how to authenticate, how to list/read/write Proton Drive data, and how to mount the Drive tree through FUSE. The project is a Jerboa-native Proton Drive client. It does not use rclone or a Go helper at runtime. +## Short CLI + +Use the installed binary directly: + +```sh +jdrive login --username you@example.com +jdrive ls --username you@example.com +jdrive creds store --profile default --username you@example.com +jdrive tree --profile default --max-depth 3 +jdrive mount --profile default --mountpoint /tmp/proton --writable +``` + +If Proton requires TOTP and no code was passed, `jdrive` waits at a `TOTP code:` +prompt so you have time to get the current code. + +After `jdrive creds store`, the encrypted credentials in `~/.jproton` are used +automatically whenever you do not pass `--username` or token options. The old +`--use-stored-credentials` flag still works, but it is not required for normal +interactive use. Once a full login succeeds, `jdrive` also saves the rotated +Proton session tokens inside that encrypted vault; later commands refresh that +session and should not need TOTP unless Proton revokes or expires the session. + +`protonstorage` and `drive` are optional compatibility names. These pairs are +equivalent: + +```sh +jdrive ls +protonstorage drive root-children-decrypted + +jdrive tree +protonstorage drive tree-decrypted + +jdrive creds store +protonstorage drive credentials store +``` + ## Current Scope Implemented today: @@ -39,7 +75,7 @@ Important current limitations: Expected source layout: ```text -~/mine/jerboa-protonstorage/ +~/mine/jerboa-drive/ ~/mine/jerboa/ ~/mine/jerboa-proton-bridge/ ~/mine/jerboa-yubikey/ @@ -73,6 +109,7 @@ Equivalent direct form: ```sh ./bin/protonstorage help +./bin/jdrive help ./bin/protonstorage drive status ``` @@ -228,7 +265,14 @@ make run ARGS='drive login --username you@example.com --prompt-pin' TOTP fallback: ```sh -make run ARGS='drive login --username you@example.com --totp-code 123456' +jdrive login --username you@example.com +``` + +If Proton requires TOTP, this prompts for the current code. For automation, +pass it explicitly: + +```sh +jdrive login --username you@example.com --totp-code 123456 ``` or: @@ -260,6 +304,10 @@ material. The credential vault stores your Proton username and Proton password in the selected local profile. The vault is encrypted with `scrypt` and `chacha20-poly1305`, protected by a separate vault password you provide. +After the first successful full login from that vault, it also stores Proton's +current reusable session tokens and the Drive key pass inside the same encrypted +vault. Proton rotates refresh tokens, so `jdrive` rewrites the vault after each +successful session refresh. Default vault path: @@ -289,21 +337,21 @@ make run ARGS='drive credentials unlock-test --profile default' Use stored credentials: ```sh -make run ARGS='drive root-children-decrypted --profile default --use-stored-credentials' +jdrive ls --profile default ``` Noninteractive vault unlock: ```sh export PROTONSTORAGE_VAULT_PASSWORD='your-vault-password' -make run ARGS='drive tree-decrypted --profile default --use-stored-credentials --max-depth 2' +jdrive tree --profile default --max-depth 2 ``` Or choose the vault password variable explicitly: ```sh export MY_VAULT_PASSWORD='your-vault-password' -make run ARGS='drive tree-decrypted --use-stored-credentials --vault-password-env MY_VAULT_PASSWORD' +jdrive tree --vault-password-env MY_VAULT_PASSWORD ``` Store credentials noninteractively: @@ -323,7 +371,8 @@ make run ARGS='drive credentials clear --profile default' Security notes: -- The vault stores the Proton login password, not an OAuth token. +- The vault stores the Proton login password and, after first login, reusable + Proton session tokens plus the Drive key pass. - The vault password is not stored. - The vault password is still required every time stored credentials are used. - Store `~/.jproton` on an encrypted disk if your threat model includes local @@ -347,7 +396,7 @@ Using stored credentials: ```sh export PROTONSTORAGE_VAULT_PASSWORD='your-vault-password' -make run ARGS='drive tree-decrypted --profile default --use-stored-credentials --max-depth 3' +jdrive tree --profile default --max-depth 3 ``` `--max-depth -1` means unlimited depth. The default for recursive listing and @@ -380,7 +429,7 @@ Read from stored credentials: ```sh export PROTONSTORAGE_VAULT_PASSWORD='your-vault-password' -make run ARGS='drive read-file --use-stored-credentials --link FILE_LINK_ID' > file.bin +jdrive cat --link FILE_LINK_ID > file.bin ``` `--size -1` reads the whole file. `--offset 0` starts at the beginning. @@ -481,7 +530,7 @@ Using stored credentials: ```sh export PROTONSTORAGE_VAULT_PASSWORD='your-vault-password' -make run ARGS='drive mount --profile default --use-stored-credentials --mountpoint /tmp/proton-drive --writable' +jdrive mount --profile default --mountpoint /tmp/proton-drive --writable ``` The command remains attached. Press `Ctrl-C` to unmount through the normal FUSE @@ -745,10 +794,9 @@ On the target: ```sh tar -xzf protonstorage-<os>-<arch>.tar.gz -./protonstorage-<os>-<arch>/protonstorage help +./protonstorage-<os>-<arch>/jdrive help ``` Keep `~/.jproton` private. The credential vault is encrypted, but the profile directory also contains operational state such as cache, staging, logs, and lock files as those layers are completed. - --- a/main.ss +++ b/main.ss @@ -15,11 +15,16 @@ (define jerboa-dir (or (getenv "JERBOA_HOME") (string-append home "/mine/jerboa"))) +(define jerboa-aws-dir + (or (getenv "JERBOA_AWS_DIR") + (string-append home "/mine/jerboa-aws"))) (define project-dir (current-directory)) (library-directories (append (list (cons project-dir project-dir) + (cons (string-append jerboa-aws-dir "/lib") + (string-append jerboa-aws-dir "/lib")) (cons (string-append jerboa-dir "/lib") (string-append jerboa-dir "/lib"))) (library-directories))) @@ -27,4 +32,3 @@ (import (protonstorage cli)) (run-cli (command-line-arguments)) - --- a/protonstorage/cli.ss +++ b/protonstorage/cli.ss @@ -22,19 +22,56 @@ (protonstorage drive fs) (protonstorage drive local) (protonstorage drive write) + (protonstorage s3 config) (jerboa-fuse) - (only (proton-bridge api session) proton-session-base-url) + (only (proton-bridge api session) + proton-session-base-url + proton-session-uid + proton-session-access-token + proton-session-refresh-token) (only (std text json) json-object->string)) (define version "0.1.0") (define usage-string (string-append - "usage: jerboa-protonstorage <command> [options]\n" + "usage: jdrive <command> [options]\n" + "\n" + "common commands:\n" + " login --username USER Authenticate with Proton\n" + " ls List decrypted root names\n" + " tree --max-depth N List decrypted tree recursively\n" + " creds store Store encrypted username/password\n" + " mount --mountpoint M Mount decrypted Drive tree\n" + " s3 status Show S3 backend configuration\n" + " doctor Show implementation status\n" + "\n" + "If a profile credential vault exists and no token/username is provided,\n" + "jdrive uses the stored Proton username/password automatically.\n" + "After one full login, it refreshes the encrypted cached session first.\n" + "\n" + "'protonstorage' and the old 'drive ...' prefix still work, but are optional.\n" "\n" "commands:\n" " doctor Show implementation status\n" + " status Show native Drive API surface\n" + " login --username USER Alias for drive login\n" + " ls Alias for drive root-children-decrypted\n" + " tree Alias for drive tree-decrypted\n" + " cat --link L Alias for drive read-file\n" + " cp [-r] LOCAL REMOTE_PATH Copy local file/dir to Proton Drive\n" + " put --parent-path P --path F Alias for drive upload-file\n" + " mkdir --parent-path P --name N Alias for drive create-folder\n" + " trash --parent-path P --name N Alias for drive trash-child\n" + " rename --parent-path P --name OLD --new-name NEW\n" + " Alias for drive rename-child\n" + " move --parent-path P --name N --new-parent-path Q\n" + " Alias for drive move-child\n" + " creds store|status|unlock-test|clear\n" + " Alias for drive credentials ...\n" " drive status Show native Drive API surface\n" + " s3 status Show S3 backend config without secrets\n" + " s3 target --bucket B --key K Show S3 object request target\n" " drive login --username USER Fresh SRP/FIDO2 login probe\n" " drive user Fetch Proton user/quota JSON\n" " drive salts Fetch Proton key salts\n" @@ -60,6 +97,8 @@ " Move one child, optionally with --new-name\n" " drive upload-file --parent-path P --path PATH [--name NAME]\n" " Upload or replace a file under a path\n" + " drive cp [-r] LOCAL REMOTE_PATH\n" + " Upload a local file/tree to Proton Drive\n" " drive upload-root-file --path PATH [--name NAME]\n" " Legacy alias for --parent-path /\n" " drive mount --mountpoint M [--writable]\n" @@ -99,7 +138,7 @@ " --mailbox-password-env ENV For two-password Proton accounts\n" " --pin-env ENV Read FIDO2 PIN from named env var\n" " --prompt-pin Prompt for FIDO2 PIN before YubiKey assertion\n" - " --totp-code CODE TOTP fallback for non-FIDO2 accounts\n" + " --totp-code CODE TOTP fallback; prompted if required\n" " --mountpoint PATH Mount point for drive mount-root commands\n" " --profile NAME Local FUSE/cache profile, default default\n" " --state-dir PATH Local state root, default ~/.jproton\n" @@ -108,12 +147,13 @@ " --foreground Accepted by daemon start; current mode\n" " --force Force local daemon marker cleanup\n" " --vault-password-env ENV Read credential-vault password from env\n" - " --use-stored-credentials Use encrypted profile credentials\n" + " --use-stored-credentials Force encrypted profile credentials\n" " --offset N Byte offset for drive read-file\n" " --size N Byte count for drive read-file\n" " --max-depth N Limit recursive decrypted tree/mount depth\n" " --path PATH Local path for drive upload-root-file\n" " --parent-path PATH Drive folder path, default /\n" + " -r, --recursive Recursively upload a local directory\n" " --name NAME Drive child name for create/upload commands\n" " --new-name NAME New Drive child name for rename-child\n" " --new-parent-path PATH New Drive folder path for move-child\n" @@ -121,7 +161,15 @@ " --modification-time ISO Default current UTC time\n" " --block-size N Default 4194304\n" " --replace-existing-draft Replace an existing failed draft revision\n" - " --base-url URL Default https://drive-api.proton.me\n")) + " --base-url URL Default https://drive-api.proton.me\n" + "\n" + "s3 options/env:\n" + " --bucket B Or JDRIVE_S3_BUCKET\n" + " --prefix P Or JDRIVE_S3_PREFIX, default jdrive\n" + " --endpoint URL Or JDRIVE_S3_ENDPOINT, default s3.amazonaws.com\n" + " --region R Or AWS_REGION/AWS_DEFAULT_REGION\n" + " --aws-profile P Or AWS_PROFILE\n" + " --path-style Or JDRIVE_S3_PATH_STYLE=1\n")) (define (println s) (display s) @@ -187,6 +235,8 @@ ("--hash" . #t) ("--path" . #t) ("--parent-path" . #t) + ("-r" . #f) + ("--recursive" . #f) ("--name" . #t) ("--new-name" . #t) ("--new-parent-path" . #t) @@ -195,6 +245,15 @@ ("--block-size" . #t) ("--replace-existing-draft" . #f))) + (define s3-flags + '(("--bucket" . #t) + ("--prefix" . #t) + ("--endpoint" . #t) + ("--region" . #t) + ("--aws-profile" . #t) + ("--path-style" . #f) + ("--key" . #t))) + (define (split-opts args known-flags) (let loop ([xs args] [opts '()] [pos '()]) (cond @@ -237,6 +296,10 @@ (env-or "PROTONDRIVE_USERNAME" "PROTON_USERNAME") (die 2 "Drive login requires --username or PROTONDRIVE_USERNAME"))) + (define (auth-username-available? opts) + (or (opt opts "--username") + (env-or "PROTONDRIVE_USERNAME" "PROTON_USERNAME"))) + (define (prompt-line prompt) (display prompt (current-error-port)) (let ([line (get-line (current-input-port))]) @@ -277,39 +340,151 @@ (or (opt opts "--use-stored-credentials") (let ([value (env-or "PROTONSTORAGE_USE_STORED_CREDENTIALS" "PROTONDRIVE_USE_STORED_CREDENTIALS")]) - (and value (not (string=? value "0")))))) + (and value (not (string=? value "0")))) + (and (not (auth-username-available? opts)) + (hashtable-ref + (proton-drive-credentials-status + (profile-option opts) + (state-root-option opts)) + "Stored" + #f)))) + + (define stored-credentials-context-cache #f) + + (define (stored-credentials-context-key opts) + (list + (or (profile-option opts) "") + (or (state-root-option opts) ""))) (define (stored-login-credentials opts) - (proton-drive-credentials-load - (profile-option opts) - (state-root-option opts) - (vault-password-from-opts opts))) - - (define (drive-auth-result-from-login opts base-url) - (let* ([stored (and (use-stored-credentials? opts) - (stored-login-credentials opts))] - [username (if stored - (hashtable-ref stored "Username" "") - (auth-username-from-opts opts))] - [password (if stored - (hashtable-ref stored "Password" "") - (auth-password-from-opts opts))] - [mailbox-password (auth-mailbox-password-from-opts opts)] - [pin (auth-pin-from-opts opts)] - [totp (or (opt opts "--totp-code") - (env-or "PROTONDRIVE_TOTP_CODE" "PROTON_TOTP_CODE"))]) - (eprintln "Requesting Proton SRP challenge.") - (eprintln "Submitting Proton SRP proof; touch your YubiKey if prompted.") - (proton-drive-authenticate - username - password - 'base-url: base-url - 'mailbox-password: mailbox-password - 'fido2-pin: pin - 'totp-code: totp))) - - (define (drive-auth-result-from-opts opts) - (let ([uid (or (opt opts "--uid") (getenv "PROTONDRIVE_UID") (getenv "PROTON_UID"))] + (call-with-values + (lambda () (stored-login-credentials/context opts)) + (lambda (creds vault-password) creds))) + + (define (stored-login-credentials/context opts) + (let ([key (stored-credentials-context-key opts)]) + (if (and stored-credentials-context-cache + (equal? key (car stored-credentials-context-cache))) + (values + (cadr stored-credentials-context-cache) + (caddr stored-credentials-context-cache)) + (let* ([vault-password (vault-password-from-opts opts)] + [creds + (proton-drive-credentials-load + (profile-option opts) + (state-root-option opts) + vault-password)]) + (set! stored-credentials-context-cache + (list key creds vault-password)) + (values creds vault-password))))) + + (define (string-present? value) + (and (string? value) (> (string-length value) 0))) + + (define (stored-session-field session key) + (and session (hashtable-ref session key #f))) + + (define (stored-session-usable? session) + (and session + (string-present? (stored-session-field session "UID")) + (string-present? (stored-session-field session "RefreshToken")) + (or (string-present? (stored-session-field session "KeyPass")) + (string-present? (stored-session-field session "SaltedKeyPass"))))) + + (define (derive-salted-key-pass session key-pass) + (and (string-present? key-pass) + (guard (e [#t #f]) + (proton-drive-user-key-pass + key-pass + (proton-drive-get-user session) + (proton-drive-get-key-salts session))))) + + (define (save-auth-session! opts vault-password auth-result) + (let* ([session (proton-drive-auth-result-session auth-result)] + [key-pass (proton-drive-auth-result-key-pass auth-result)] + [salted-key-pass + (or (proton-drive-auth-result-salted-key-pass auth-result) + (derive-salted-key-pass session key-pass))]) + (when (and vault-password + (or (string-present? key-pass) + (string-present? salted-key-pass)) + (string-present? (proton-session-uid session)) + (string-present? (proton-session-access-token session)) + (string-present? (proton-session-refresh-token session))) + (proton-drive-credentials-save-session! + (profile-option opts) + (state-root-option opts) + vault-password + (proton-session-base-url session)