Initial Jerboa Proton Drive implementation
ober
37e67ebb368e30acf16844e6daf56e5ffde0003f
new file mode 100644 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +*.so +*.dylib +*.o +cache/ +tmp/ +mount/ +dist/ +*-main.c +protonstorage-bin +protonstorage-linux-amd64 +protonstorage-linux-arm64 +protonstorage-freebsd-amd64 new file mode 100644 --- /dev/null +++ b/AGENT.md @@ -0,0 +1,40 @@ +# AGENT.md for jerboa-$app Repositories + +These instructions apply to agents working in this repository. + +## Build Contract + +The Makefile must provide these targets: + +- `make` prints the available targets. It must not build, install, test, or mutate artifacts. +- `make binary` builds the application binary for the local host OS and architecture. +- `make linux-amd64` cross-builds the Linux amd64 binary from the current host. +- `make linux-arm64` cross-builds the Linux arm64 binary from the current host. +- `make freebsd-amd64` cross-builds the FreeBSD amd64 binary from the current host. + +The cross-build targets must use the local cross toolchains and Jerboa/Chez +cross-build setup. Never use Docker or Podman for building, testing, smoke +testing, packaging, or verifying these targets. + +## Required Verification + +Before committing or pushing any code, all required build targets must succeed: + +```sh +make +make binary +make linux-amd64 +make linux-arm64 +make freebsd-amd64 +``` + +Do not commit or push if any target fails. If a cross-build cannot run because +the host is missing a required toolchain or sysroot, stop and report the missing +dependency instead of committing or pushing partially verified code. + +## Working Rules + +- Keep changes inside the current repository unless the user explicitly names another path. +- Preserve the target semantics above when editing the Makefile. +- Treat the cross-build targets as release-critical, not optional follow-up work. +- Do not add Dockerfiles, container scripts, Podman invocations, or container-based verification paths. new file mode 100644 --- /dev/null +++ b/Makefile @@ -0,0 +1,129 @@ +JERBOA_HOME ?= $(realpath $(CURDIR)/../jerboa) +SCHEME ?= $(JERBOA_HOME)/.chez/bin/scheme +JERBOA_PROTON_BRIDGE_DIR ?= $(realpath $(CURDIR)/../jerboa-proton-bridge) +JERBOA_HTTPS_DIR ?= $(realpath $(CURDIR)/../jerboa-https) +JERBOA_SSL_DIR ?= $(realpath $(CURDIR)/../jerboa-ssl) +JERBOA_YUBIKEY_DIR ?= $(realpath $(CURDIR)/../jerboa-yubikey) +JERBOA_PGP_DIR ?= $(realpath $(CURDIR)/../jerboa-pgp) +JERBOA_CRYPTO_DIR ?= $(realpath $(CURDIR)/../jerboa-crypto) +JERBOA_FUSE_DIR ?= $(realpath $(CURDIR)/../jerboa-fuse) +JERBOA_MAIL_DIR ?= $(realpath $(CURDIR)/../jerboa-mail) +LIBDIRS := $(CURDIR):$(JERBOA_PROTON_BRIDGE_DIR):$(JERBOA_YUBIKEY_DIR):$(JERBOA_YUBIKEY_DIR)/lib:$(JERBOA_PGP_DIR):$(JERBOA_CRYPTO_DIR)/lib:$(JERBOA_FUSE_DIR)/lib:$(JERBOA_MAIL_DIR):$(JERBOA_HTTPS_DIR)/lib:$(JERBOA_SSL_DIR)/lib:$(JERBOA_HOME)/lib +XC_LIBDIRS := $(LIBDIRS):$(JERBOA_HOME)/.chez-cross-ta6le/lib:$(JERBOA_HOME)/.chez-cross-tarm64le/lib:$(JERBOA_HOME)/.chez-cross-ta6fb/lib +RUN_ENV := JERBOA_HOME=$(JERBOA_HOME) \ + JERBOA_SSL_LIB=$(JERBOA_SSL_DIR) \ + JERBOA_CRYPTO_LIB=$(JERBOA_CRYPTO_DIR) \ + DYLD_LIBRARY_PATH=$(JERBOA_CRYPTO_DIR):$(JERBOA_FUSE_DIR):$(JERBOA_PROTON_BRIDGE_DIR)/proton-bridge-native/target/release:$${DYLD_LIBRARY_PATH:-} \ + LD_LIBRARY_PATH=$(JERBOA_CRYPTO_DIR):$(JERBOA_FUSE_DIR):$(JERBOA_PROTON_BRIDGE_DIR)/proton-bridge-native/target/release:$${LD_LIBRARY_PATH:-} +BINARY_OUTPUT ?= protonstorage-bin +LINUX_AMD64_CC ?= x86_64-linux-musl-gcc +LINUX_ARM64_CC ?= aarch64-linux-musl-gcc +FREEBSD_AMD64_CC ?= $(JERBOA_HOME)/support/cross-cc-freebsd-amd64 +HOST_OS := $(shell uname -s | tr '[:upper:]' '[:lower:]') +HOST_ARCH := $(shell uname -m) +BUNDLE_OUTPUT ?= dist/protonstorage-$(HOST_OS)-$(HOST_ARCH) + +.PHONY: help run test integration doctor binary bundle linux linux-amd64 linux-arm64 freebsd freebsd-amd64 clean +.DEFAULT_GOAL := help + +help: + @echo "jerboa-protonstorage" + @echo "" + @echo "Development:" + @echo " make run ARGS='help' Run the CLI" + @echo " make doctor Check native Jerboa implementation status" + @echo " make test Run smoke tests" + @echo " make integration Run opt-in live Drive test when enabled" + @echo " make clean Remove generated local artifacts" + @echo "" + @echo "Build:" + @echo " make Show this help" + @echo " make binary Build ./$(BINARY_OUTPUT) for this host" + @echo " make bundle Package ./$(BUNDLE_OUTPUT).tar.gz" + @echo " make linux-amd64 Cross-build ./protonstorage-linux-amd64" + @echo " make linux-arm64 Cross-build ./protonstorage-linux-arm64" + @echo " make freebsd-amd64 Cross-build ./protonstorage-freebsd-amd64" + @echo "" + @echo "Environment:" + @echo " JERBOA_HOME = $(JERBOA_HOME)" + @echo " SCHEME = $(SCHEME)" + @echo " JERBOA_PROTON_BRIDGE_DIR = $(JERBOA_PROTON_BRIDGE_DIR)" + @echo " JERBOA_YUBIKEY_DIR = $(JERBOA_YUBIKEY_DIR)" + @echo " JERBOA_PGP_DIR = $(JERBOA_PGP_DIR)" + @echo " JERBOA_CRYPTO_DIR = $(JERBOA_CRYPTO_DIR)" + @echo " JERBOA_FUSE_DIR = $(JERBOA_FUSE_DIR)" + @echo " JERBOA_MAIL_DIR = $(JERBOA_MAIL_DIR)" + @echo " JERBOA_HTTPS_DIR = $(JERBOA_HTTPS_DIR)" + @echo " JERBOA_SSL_DIR = $(JERBOA_SSL_DIR)" + +run: + $(RUN_ENV) \ + $(SCHEME) -q --libdirs $(LIBDIRS) --script main.ss -- $(ARGS) + +doctor: + $(MAKE) run ARGS='doctor' + +test: + $(RUN_ENV) \ + $(SCHEME) -q --libdirs $(LIBDIRS) --script test/test-all.ss + +integration: + $(RUN_ENV) \ + $(SCHEME) -q --libdirs $(LIBDIRS) --script test/integration-drive.ss + +binary: + $(RUN_ENV) BINARY_LIBDIRS='$(LIBDIRS)' \ + JERBOA_CHEZ_PREFIX=$(JERBOA_HOME)/.chez \ + bash $(JERBOA_HOME)/support/build-binary.sh support/binary-entry.ss $(BINARY_OUTPUT) + +bundle: binary + JERBOA_PROTON_BRIDGE_DIR="$(JERBOA_PROTON_BRIDGE_DIR)" \ + JERBOA_CRYPTO_DIR="$(JERBOA_CRYPTO_DIR)" \ + JERBOA_FUSE_DIR="$(JERBOA_FUSE_DIR)" \ + JERBOA_SSL_DIR="$(JERBOA_SSL_DIR)" \ + bash support/package-bundle.sh "$(BINARY_OUTPUT)" "$(BUNDLE_OUTPUT)" + +linux: linux-amd64 + +linux-amd64: + @command -v $(LINUX_AMD64_CC) >/dev/null 2>&1 || { echo "ERROR: $(LINUX_AMD64_CC) not found on PATH."; exit 1; } + @test -d "$(JERBOA_HOME)/.chez-cross-ta6le" || { echo "ERROR: cross Chez prefix missing: $(JERBOA_HOME)/.chez-cross-ta6le"; exit 1; } + @test -f "$(JERBOA_HOME)/build/chez/xc-ta6le/s/xpatch" || { echo "ERROR: xpatch missing: $(JERBOA_HOME)/build/chez/xc-ta6le/s/xpatch"; exit 1; } + $(RUN_ENV) BINARY_LIBDIRS='$(XC_LIBDIRS)' \ + JERBOA_CHEZ_PREFIX=$(JERBOA_HOME)/.chez \ + JERBOA_CROSS_PREFIX=$(JERBOA_HOME)/.chez-cross-ta6le \ + JERBOA_XPATCH=$(JERBOA_HOME)/build/chez/xc-ta6le/s/xpatch \ + TARGET_MACHINE=ta6le CC=$(LINUX_AMD64_CC) \ + bash $(JERBOA_HOME)/support/build-binary.sh support/binary-entry.ss protonstorage-linux-amd64 + +linux-arm64: + @command -v $(LINUX_ARM64_CC) >/dev/null 2>&1 || { echo "ERROR: $(LINUX_ARM64_CC) not found on PATH."; exit 1; } + @test -d "$(JERBOA_HOME)/.chez-cross-tarm64le" || { echo "ERROR: cross Chez prefix missing: $(JERBOA_HOME)/.chez-cross-tarm64le"; exit 1; } + @test -f "$(JERBOA_HOME)/build/chez/xc-tarm64le/s/xpatch" || { echo "ERROR: xpatch missing: $(JERBOA_HOME)/build/chez/xc-tarm64le/s/xpatch"; exit 1; } + $(RUN_ENV) BINARY_LIBDIRS='$(XC_LIBDIRS)' \ + JERBOA_CHEZ_PREFIX=$(JERBOA_HOME)/.chez \ + JERBOA_CROSS_PREFIX=$(JERBOA_HOME)/.chez-cross-tarm64le \ + JERBOA_XPATCH=$(JERBOA_HOME)/build/chez/xc-tarm64le/s/xpatch \ + TARGET_MACHINE=tarm64le CC=$(LINUX_ARM64_CC) \ + bash $(JERBOA_HOME)/support/build-binary.sh support/binary-entry.ss protonstorage-linux-arm64 + +freebsd: freebsd-amd64 + +freebsd-amd64: + @command -v $(firstword $(FREEBSD_AMD64_CC)) >/dev/null 2>&1 || { echo "ERROR: $(FREEBSD_AMD64_CC) not found or not executable."; exit 1; } + @test -d "$(JERBOA_HOME)/.chez-cross-ta6fb" || { echo "ERROR: cross Chez prefix missing: $(JERBOA_HOME)/.chez-cross-ta6fb"; exit 1; } + @test -f "$(JERBOA_HOME)/build/chez/xc-ta6fb/s/xpatch" || { echo "ERROR: xpatch missing: $(JERBOA_HOME)/build/chez/xc-ta6fb/s/xpatch"; exit 1; } + $(RUN_ENV) BINARY_LIBDIRS='$(XC_LIBDIRS)' \ + JERBOA_CHEZ_PREFIX=$(JERBOA_HOME)/.chez \ + JERBOA_CROSS_PREFIX=$(JERBOA_HOME)/.chez-cross-ta6fb \ + JERBOA_XPATCH=$(JERBOA_HOME)/build/chez/xc-ta6fb/s/xpatch \ + TARGET_MACHINE=ta6fb CC='$(FREEBSD_AMD64_CC)' \ + bash $(JERBOA_HOME)/support/build-binary.sh support/binary-entry.ss protonstorage-freebsd-amd64 + +clean: + rm -rf cache tmp mount dist protonstorage-bin protonstorage-linux-amd64 protonstorage-linux-arm64 protonstorage-freebsd-amd64 + rm -f protonstorage-bin-main.c protonstorage-bin.wp.so + rm -f protonstorage-linux-amd64-main.c protonstorage-linux-amd64.wp.so + rm -f protonstorage-linux-arm64-main.c protonstorage-linux-arm64.wp.so + rm -f protonstorage-freebsd-amd64-main.c protonstorage-freebsd-amd64.wp.so + rm -f petite_boot.h scheme_boot.h program_boot.h new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,235 @@ +# jerboa-protonstorage + +Jerboa-native Proton Drive storage access. + +This project is intended to become a single deployable Jerboa application for +Proton Drive, with YubiKey-gated access and no rclone or Go helper in the +runtime path. + +## Runtime Status + +Current runtime code is Scheme/Jerboa only: + +- [main.ss](main.ss) - script entry point. +- [protonstorage/cli.ss](protonstorage/cli.ss) - command dispatch. +- [protonstorage/drive/api.ss](protonstorage/drive/api.ss) - direct Proton + Drive HTTPS API calls through Jerboa HTTPS/JSON, including block upload + reservation and multipart encrypted-block upload. +- [protonstorage/drive/auth.ss](protonstorage/drive/auth.ss) - native SRP, + FIDO2/YubiKey, and TOTP auth flow. +- [protonstorage/drive/discovery.ss](protonstorage/drive/discovery.ss) - + active volume/share/root resolution. +- [protonstorage/drive/crypto.ss](protonstorage/drive/crypto.ss) - Drive + name hashing, key/passphrase decrypt wrappers, raw content-session-key and + block decrypt calls, content-key signature verification, and encrypted-block + SHA-256 verification, plus OpenPGP node-key generation, signed link-name + encryption, node-passphrase encrypt/sign helpers, content-key packet + generation, and content block encryption. +- [protonstorage/drive/client.ss](protonstorage/drive/client.ss) - end-to-end + root unlock, decrypted traversal, file read pipeline, folder-path resolution, + folder creation, file upload, and revision replacement. +- [protonstorage/drive/fs.ss](protonstorage/drive/fs.ss) - recursive FUSE + metadata mapping with reader, mkdir, buffered create/write, unlink/rmdir, + and rename/move callbacks through `jerboa-fuse`. +- [protonstorage/drive/local.ss](protonstorage/drive/local.ss) - local + CLI/FUSE profile paths, cache/staging directory setup, and foreground daemon + marker status. +- [protonstorage/drive/write.ss](protonstorage/drive/write.ss) - create-folder + and create-file request payload assembly from unlocked keys and generated + Drive crypto, encrypted block upload plans, encrypted xattrs, manifest + signatures, and revision commit payloads. + +The cloned Go/rclone/Proton client projects under `~/mine` are references only. +They are used to map endpoint shapes and crypto behavior while porting. + +## Implemented API Surface + +The Jerboa module `(protonstorage drive api)` currently covers: + +- `/core/v4/users` +- `/core/v4/keys/salts` +- `/core/v4/addresses` +- `/core/v4/addresses/{addressID}` +- `/drive/volumes` +- `/drive/shares` +- `/drive/shares/{shareID}` +- `/drive/shares/{shareID}/links/{linkID}` +- `/drive/shares/{shareID}/folders/{linkID}/children` +- `/drive/shares/{shareID}/links/{linkID}/checkAvailableHashes` +- `/drive/shares/{shareID}/links/{linkID}/move` +- `/drive/shares/{shareID}/folders` +- `/drive/shares/{shareID}/files` +- `/drive/shares/{shareID}/files/{linkID}/revisions` +- `/drive/shares/{shareID}/files/{linkID}/revisions/{revisionID}` +- `/drive/shares/{shareID}/folders/{linkID}/trash_multiple` +- `/drive/shares/{shareID}/folders/{linkID}/delete_multiple` +- `/drive/shares/{shareID}/trash` +- `/drive/blocks` + +Inspect the native surface: + +```sh +make run ARGS='drive status' +``` + +Authenticated calls use Proton session material from the environment: + +```sh +export PROTONDRIVE_UID='...' +export PROTONDRIVE_ACCESS_TOKEN='...' +``` + +Examples: + +```sh +make run ARGS='drive user' +make run ARGS='drive volumes' +make run ARGS='drive shares' +make run ARGS='drive link --share SHARE_ID --link LINK_ID' +make run ARGS='drive children --share SHARE_ID --link LINK_ID' +``` + +Fresh native login uses SRP and FIDO2/YubiKey when Proton requests it: + +```sh +export PROTONDRIVE_PASSWORD='...' +make run ARGS='drive login --username you@example.com' +make run ARGS='drive root-children-decrypted --username you@example.com' +make run ARGS='drive tree-decrypted --username you@example.com --max-depth 3' +``` + +For local CLI use, Proton login credentials can be stored in the selected +profile as an encrypted vault protected by a separate vault password. The vault +uses `scrypt` and `chacha20-poly1305`; the vault password is still required to +unlock stored credentials. Profiles live under `~/.jproton/profiles/<profile>/` +by default. + +```sh +make run ARGS='drive credentials store --profile default' +make run ARGS='drive credentials status --profile default' +make run ARGS='drive credentials unlock-test --profile default' +make run ARGS='drive root-children-decrypted --profile default --use-stored-credentials' +``` + +For automation, set `PROTONSTORAGE_VAULT_PASSWORD` or pass +`--vault-password-env ENV_NAME` instead of typing the vault password. + +The default FUSE command mounts a decrypted, read-only Drive tree: + +```sh +mkdir -p /tmp/proton-root +make run ARGS='drive mount-root --username you@example.com --mountpoint /tmp/proton-root' +``` + +The writable mount buffers created file data in memory and commits it through +the native encrypted upload/revision pipeline when the file handle is flushed or +released. It also supports folder creation, unlink/rmdir through Proton trash, +and rename/move through the Proton move endpoint: + +```sh +mkdir -p /tmp/proton-rw +make run ARGS='drive mount-root-write --username you@example.com --mountpoint /tmp/proton-rw' +``` + +The product-level FUSE CLI surface also has a profile-aware mount command and +local state inspection commands. `drive daemon start` currently runs the same +FUSE daemon in the foreground so it can be supervised by a service manager; a +background control socket is tracked as remaining work. + +```sh +make run ARGS='drive cache init --profile default' +make run ARGS='drive cache status --profile default' +make run ARGS='drive daemon status --profile default' +make run ARGS='drive mount --username you@example.com --mountpoint /tmp/proton-root' +make run ARGS='drive mount --username you@example.com --mountpoint /tmp/proton-rw --writable' +make run ARGS='drive daemon start --username you@example.com --mountpoint /tmp/proton-rw --writable --foreground' +``` + +File reads use the same recursive pipeline and write plaintext bytes to stdout: + +```sh +make run ARGS='drive read-file --username you@example.com --link FILE_LINK_ID --offset 0 --size 4096' +``` + +The current read path resolves the root, unlocks the file node key, decrypts +and verifies Proton Drive `ContentKeyPacket` values, selects the active +revision, downloads storage blocks, verifies each encrypted block hash, and +decrypts block packets with the content session key. + +The write-side surface can now generate locked Drive node keys, encrypt signed +link names, encrypt/sign node passphrases, reserve upload blocks, and submit +encrypted block payloads using Proton's `Block` multipart field. It can also +generate `ContentKeyPacket` values and encrypt content blocks with the generated +session key, assemble create-folder/create-file payloads, build encrypted block +metadata with encrypted detached signatures, encrypt revision xattrs, sign +revision manifests, upload blocks, and commit file revisions under decrypted +folder paths. + +Write commands require a fresh native login because Drive key unlock needs +mailbox key material: + +```sh +make run ARGS='drive create-folder --username you@example.com --parent-path / --name docs' +make run ARGS='drive upload-file --username you@example.com --parent-path /docs --path ./notes.txt --mime-type text/plain' +make run ARGS='drive rename-child --username you@example.com --parent-path /docs --name notes.txt --new-name notes-renamed.txt' +make run ARGS='drive move-child --username you@example.com --parent-path /docs --name notes-renamed.txt --new-parent-path /archive' +make run ARGS='drive trash-child --username you@example.com --parent-path /archive --name notes-renamed.txt' +``` + +`drive create-root-folder` and `drive upload-root-file` remain as root-folder +aliases. If a file already has an unfinished draft revision, uploads stop by +default. Pass `--replace-existing-draft` to delete that draft revision and +create a new one for an active file. Cross-folder move and FUSE rename +re-encrypt the moved node passphrase for the destination parent key. + +## Reference Clones + +Relevant reference projects cloned under `~/mine`: + +- `/Users/user/mine/rclone` +- `/Users/user/mine/Proton-API-Bridge` +- `/Users/user/mine/rclone-go-proton-api` +- `/Users/user/mine/go-proton-api` +- `/Users/user/mine/gopenpgp` +- `/Users/user/mine/go-crypto` +- `/Users/user/mine/go-srp` +- `/Users/user/mine/proton-drive-upstream` +- `/Users/user/mine/proton-ios-drive` +- `/Users/user/mine/proton-android-drive` +- `/Users/user/mine/proton-webclients` +- `/Users/user/mine/jerboa-proton-bridge` +- `/Users/user/mine/jerboa-yubikey` +- `/Users/user/mine/jerboa-crypto` +- `/Users/user/mine/jerboa-fuse` + +## Build and Test + +```sh +make +make binary +make linux-amd64 +make linux-arm64 +make freebsd-amd64 +make test +make integration +make doctor +make bundle +``` + +`make integration` skips unless `PROTONDRIVE_INTEGRATION=1` is set. When +enabled, it uses `PROTONDRIVE_USERNAME` and `PROTONDRIVE_PASSWORD`, creates a +disposable folder under `PROTONDRIVE_TEST_PARENT_PATH` or `/`, uploads one small +file, moves it between two disposable folders, and trashes the disposable root +during cleanup. + +`make bundle` builds the host binary and writes +`dist/protonstorage-<os>-<arch>.tar.gz` with the binary, native Jerboa +libraries, and a wrapper that sets the runtime library search path. + +## Remaining Work + +- Finish daemon control socket/background lifecycle and durable operation queue. +- Add richer metadata update operations beyond size-changing writes and + rename/move. +- Collapse the native dynamic library bundle further when Jerboa supports a + fully static bridge path. new file mode 100755 --- /dev/null +++ b/bin/protonstorage @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +DIR="$(cd "$(dirname "$0")/.." && pwd)" +JERBOA_HOME="${JERBOA_HOME:-"$DIR/../jerboa"}" +SCHEME="${SCHEME:-"$JERBOA_HOME/.chez/bin/scheme"}" +JERBOA_PROTON_BRIDGE_DIR="${JERBOA_PROTON_BRIDGE_DIR:-"$DIR/../jerboa-proton-bridge"}" +JERBOA_YUBIKEY_DIR="${JERBOA_YUBIKEY_DIR:-"$DIR/../jerboa-yubikey"}" +JERBOA_PGP_DIR="${JERBOA_PGP_DIR:-"$DIR/../jerboa-pgp"}" +JERBOA_CRYPTO_DIR="${JERBOA_CRYPTO_DIR:-"$DIR/../jerboa-crypto"}" +JERBOA_FUSE_DIR="${JERBOA_FUSE_DIR:-"$DIR/../jerboa-fuse"}" +JERBOA_MAIL_DIR="${JERBOA_MAIL_DIR:-"$DIR/../jerboa-mail"}" +JERBOA_HTTPS_DIR="${JERBOA_HTTPS_DIR:-"$DIR/../jerboa-https"}" +JERBOA_SSL_DIR="${JERBOA_SSL_DIR:-"$DIR/../jerboa-ssl"}" + +export JERBOA_HOME +export JERBOA_SSL_LIB="$JERBOA_SSL_DIR" +export JERBOA_CRYPTO_LIB="$JERBOA_CRYPTO_DIR" +PROTON_BRIDGE_NATIVE_DIR="$JERBOA_PROTON_BRIDGE_DIR/proton-bridge-native/target/release" +export DYLD_LIBRARY_PATH="$JERBOA_CRYPTO_DIR:$JERBOA_FUSE_DIR:$PROTON_BRIDGE_NATIVE_DIR:${DYLD_LIBRARY_PATH:-}" +export LD_LIBRARY_PATH="$JERBOA_CRYPTO_DIR:$JERBOA_FUSE_DIR:$PROTON_BRIDGE_NATIVE_DIR:${LD_LIBRARY_PATH:-}" + +exec "$SCHEME" -q \ + --libdirs "$DIR:$JERBOA_PROTON_BRIDGE_DIR:$JERBOA_YUBIKEY_DIR:$JERBOA_YUBIKEY_DIR/lib:$JERBOA_PGP_DIR:$JERBOA_CRYPTO_DIR/lib:$JERBOA_FUSE_DIR/lib:$JERBOA_MAIL_DIR:$JERBOA_HTTPS_DIR/lib:$JERBOA_SSL_DIR/lib:$JERBOA_HOME/lib" \ + --script "$DIR/main.ss" -- "$@" new file mode 100644 --- /dev/null +++ b/docs/fuse-cli-proton-drive-app-parity-plan.md @@ -0,0 +1,792 @@ +# FUSE CLI Proton Drive App Parity Plan + +This plan targets Proton Drive app-level behavior through a Jerboa-native CLI +and FUSE mount. It does not target a desktop GUI clone. The user-facing product +surface is: + +- A single deployable Jerboa application, eventually fully static when Jerboa + native bridge packaging supports it. +- A CLI for login, mount, sync state, cache control, sharing, versions, trash, + diagnostics, and automation. +- A FUSE filesystem that behaves like a serious cloud-drive client: on-demand + reads, durable writes, offline pinning, conflict handling, shared items, + version access, restore paths, and observable sync state. +- No rclone runtime dependency. +- No Go helper runtime dependency. + +## Product Behavior References + +Official Proton docs describe the app behaviors this plan maps into CLI/FUSE: + +- Desktop apps expose Drive files through Finder/File Explorer and sync in the + background: + https://proton.me/support/proton-drive-windows-app +- On-demand sync exposes placeholders, downloads on access, and supports + online-only, available-on-this-device, and always-available states: + https://proton.me/support/proton-drive-windows-on-demand-sync + https://proton.me/support/proton-drive-macos-on-demand-sync +- Shared with me items have permissions, owner metadata, optional local sync, + and conflict behavior: + https://proton.me/support/shared-with-me-windows +- Trash, restore, and delete behavior is distinct from local OS trash: + https://proton.me/support/proton-drive-delete-restore-synced-files +- Version history supports listing, restoring, deleting, downloading, and + signature details: + https://proton.me/support/version-history +- Share links support password protection, expiration, revocation, and folder + sharing: + https://proton.me/support/password-protect-files-proton-drive + https://proton.me/support/proton-drive-mobile-share-files-folders +- Photos and albums are part of the Drive product surface, but they can be + staged after core filesystem parity: + https://proton.me/support/enable-photo-backup + https://proton.me/support/photo-backup-supported-formats + https://proton.me/support/drive-create-album-add-photos +- Docs and Sheets live in Drive, with real-time collaboration in official + clients. The CLI/FUSE target should preserve and manage those files, not + implement rich collaborative editors: + https://proton.me/drive/docs + https://proton.me/support/drive/sheets + +## Current Baseline + +Already implemented in this repository: + +- Native SRP/FIDO2/TOTP login path. +- Native Drive session setup and root unlock. +- Volume/share/root discovery. +- Decrypted recursive tree traversal. +- File read pipeline with content-session-key decrypt, content signature + verification, encrypted block hash verification, and block decrypt. +- Folder create. +- File upload and revision replacement, including encrypted block upload. +- Trash-based unlink and rmdir. +- Same-folder rename and cross-folder move with node passphrase re-encryption. +- Writable FUSE create/write/mkdir/unlink/rmdir/rename callbacks. +- Profile-aware `drive mount` command. +- Local profile layout and `drive cache status/init`. +- Encrypted profile credential vault using `scrypt` and `chacha20-poly1305`. +- Foreground daemon marker state and `drive daemon status/start/stop` + scaffolding. Background start and the control socket are not implemented yet. +- CLI smoke commands and integration harness. +- Host binary, cross-build targets, and local bundle packaging. + +This is enough for a native remote filesystem core. It is not enough for +Proton Drive app-level behavior. + +## Product Definition + +Parity is reached when a CLI-managed FUSE mount can replace the official +desktop app for filesystem workflows on a trusted workstation: + +- The user can mount Drive, browse, open, edit, create, rename, move, trash, + restore, and share files from the command line and filesystem. +- The mount exposes a stable virtual layout for `My files`, `Shared with me`, + and device/computer synced roots. +- Reads are on-demand and range-aware. +- Writes are durable across process crashes and network outages. +- Offline and online-only states are explicit and controllable from the CLI. +- Changes made locally or remotely converge without silent data loss. +- Conflicts are surfaced, named, and recoverable. +- Version history and trash are first-class CLI workflows. +- Shared files and folders preserve permissions and owner metadata. +- Logs, status, errors, and repair commands are good enough for unattended + deployment. +- Security properties remain consistent with Proton Drive encryption behavior. + +## Non-Goals + +- No GUI desktop application. +- No Finder/File Explorer icon overlays beyond what FUSE/xattrs and optional + helper scripts can expose. +- No rich Proton Docs or Sheets editor. The target is storage, metadata, + sharing, preservation, and safe open/download/upload behavior for these file + types. +- No mobile photo-library integration. Photo import and album management can be + CLI workflows. +- No rclone or Go process at runtime. + +## Target CLI Shape + +Use `protonstorage drive ...` unless the project later promotes a shorter +top-level command. Proposed command groups: + +```sh +protonstorage drive login --username USER +protonstorage drive logout +protonstorage drive session status +protonstorage drive unlock + +protonstorage drive mount --mountpoint PATH +protonstorage drive mount --writable --mountpoint PATH +protonstorage drive daemon start --mountpoint PATH +protonstorage drive daemon stop +protonstorage drive daemon status + +protonstorage drive sync status +protonstorage drive sync pause +protonstorage drive sync resume +protonstorage drive sync now [PATH] +protonstorage drive sync conflicts +protonstorage drive sync resolve --conflict ID --keep local|remote|both + +protonstorage drive cache status +protonstorage drive cache pin PATH +protonstorage drive cache unpin PATH +protonstorage drive cache free PATH +protonstorage drive cache prune --target-free BYTES +protonstorage drive cache verify [PATH] +protonstorage drive cache repair [PATH] + +protonstorage drive trash list +protonstorage drive trash restore ID_OR_PATH +protonstorage drive trash empty [--older-than DURATION] + +protonstorage drive versions list PATH +protonstorage drive versions download PATH --revision REV --output FILE +protonstorage drive versions restore PATH --revision REV +protonstorage drive versions delete PATH --revision REV + +protonstorage drive share list [PATH] +protonstorage drive share link create PATH [--password] [--expires DATE] +protonstorage drive share link update PATH [--password] [--expires DATE] +protonstorage drive share link revoke PATH +protonstorage drive share invite PATH --email EMAIL --role viewer|editor +protonstorage drive share revoke PATH --email EMAIL +protonstorage drive shared-with-me list +protonstorage drive shared-with-me accept ID +protonstorage drive shared-with-me sync ID +protonstorage drive shared-with-me remove ID + +protonstorage drive activity +protonstorage drive quota +protonstorage drive doctor +protonstorage drive logs +protonstorage drive export-diagnostics --output FILE +``` + +## Target Mount Layout + +Initial writable mount can keep the existing root behavior, but product parity +needs a virtual root: + +```text +/My files/ +/Shared with me/ +/Computers/ +/Photos/ +/.protonstorage/ +``` + +`/.protonstorage/` is a control and inspection namespace, not synced user data: + +```text +/.protonstorage/status.json +/.protonstorage/activity.log +/.protonstorage/conflicts/ +/.protonstorage/trash/ +/.protonstorage/versions/ +/.protonstorage/cache/ +``` + +FUSE must reject user writes inside `/.protonstorage/` unless the file is a +documented command/control endpoint. + +## Phase 0: Baseline Hardening + +Goal: make the current native filesystem core predictable under normal command +line use. + +Work items: + +- Normalize all Drive path handling, including repeated slashes, trailing + slashes, root aliases, and names containing shell-sensitive characters. +- Add consistent JSON error envelopes for CLI commands. +- Add structured exit codes for auth, network, crypto, conflict, not-found, + permission, quota, and interrupted states. +- Add command-level timeout and retry settings. +- Add `--json` and `--human` output modes where commands are not already pure + JSON. +- Add durable request IDs and correlation IDs in logs. +- Audit all command help for missing environment variable alternatives. +- Add explicit `drive logout` and token/session teardown behavior. +- Add a read-only `drive quota` command. +- Add capability snapshots to integration output. + +Acceptance criteria: + +- Every implemented command has deterministic help, deterministic error shape, + and tests for missing required arguments. +- `drive status`, `drive doctor`, and `drive quota` can be used in automation. +- Failed write operations do not leave unknown local process state. + +## Phase 1: Daemonized FUSE CLI + +Goal: make the mount lifecycle usable without an app window. + +Work items: + +- Add `drive daemon start`, `stop`, `status`, and `restart`. + `start/status/stop` have initial foreground/profile-marker support. +- Add foreground and background mount modes. + Foreground mode is wired; background mode remains. +- Add PID file, socket path, mountpoint lock, and stale-lock recovery. + PID/mountpoint/mode marker files are wired; socket and stale-lock recovery + remain. +- Add signal handling for clean unmount, write flush, and session cleanup. +- Add health endpoint over a local Unix socket. +- Add machine-readable daemon status with mountpoint, session age, sync state, + queue length, cache usage, and last error. +- Add `drive mount --read-only`, `--writable`, `--foreground`, + `--cache-dir`, `--profile`, and `--log-level`. +- Add mountpoint safety checks so the daemon does not mount over non-empty + directories unless explicitly forced. +- Add automatic unmount on unrecoverable auth failure. + +Acceptance criteria: + +- `daemon start` returns only after the mount is ready. +- `daemon stop` flushes pending writes or reports exactly what remains queued. +- A crashed daemon can be detected and repaired by `daemon status` and + `daemon start`. + +## Phase 2: Durable Local State Store + +Goal: introduce a local metadata and operation store that survives crashes. + +Work items: + +- Define local profile directory layout: + +```text +~/.jproton/profiles/<profile>/ + metadata.db + op-log/ + cache/ + staging/ + locks/ + logs/ +``` + +- Store link metadata keyed by share ID, link ID, revision ID, parent link ID, + content hash, and plaintext path. +- Store decrypted names only in the chosen local profile directory. +- Store cache state: online-only, available, pinned, dirty, uploading, + conflict, tombstoned. +- Store operation queue entries for create, upload, rename, move, trash, + restore, share, version restore, and metadata updates. +- Add schema versioning and migration. +- Add compaction and consistency check. +- Add `cache verify` and `cache repair`. +- Add encrypted local profile option if Jerboa has a suitable local sealed + storage primitive. Otherwise document filesystem permission requirements. + +Acceptance criteria: + +- Restarting the daemon preserves dirty writes and queued operations. +- Metadata can be rebuilt from remote state if the local DB is deleted. +- Cache verification detects missing, truncated, or hash-mismatched content. + +## Phase 3: On-Demand Reads and Cache States + +Goal: match the app model where files appear locally without requiring all +content to be downloaded. + +Work items: + +- Make directory listing use metadata without downloading file content. +- Implement range reads with block-level cache lookup. +- Cache decrypted plaintext blocks or complete files according to policy. +- Support sparse local cache entries where only some ranges are present. +- Add `pin`, `unpin`, and `free` commands. +- Add recursive pinning for folders. +- Add background prefetch for pinned folders. +- Add cache eviction policy based on size, age, access count, and pin state. +- Add `.protonstorage/status.json` and CLI status output for cache states. +- Add xattrs where supported: + - `user.protonstorage.state` + - `user.protonstorage.link_id` + - `user.protonstorage.revision_id` + - `user.protonstorage.share_id` +- Add conservative fallback when xattrs are unavailable. + +Acceptance criteria: + +- A large cloud-only file can be listed instantly and downloaded only when + read. +- `cache pin PATH` makes the content available while offline. +- `cache free PATH` drops local plaintext/cache content without deleting remote + data. +- Cache eviction never removes pinned data. + +## Phase 4: Durable Writes and Sync Queue + +Goal: convert buffered writes into a robust sync engine. + +Work items: + +- Replace purely in-memory write buffers with staging files. +- Add atomic staging lifecycle: opened, dirty, closed, queued, uploading, + committed, failed. +- Support write patterns from common editors: + - create temp file, write, fsync, rename over target + - truncate existing file, rewrite, fsync + - append + - chmod/touch attempts that should map to metadata or no-op safely +- Add local writeback queue with retry, backoff, and resumable state. +- Add upload cancellation and resume. +- Add upload progress visible through `activity` and daemon status. +- Add quota preflight when size is known. +- Add low-disk preflight for staging/cache. +- Add safe handling for interrupted flush/release. +- Add operation dependency ordering, especially create parent before child. +- Add journal replay on startup. + +Acceptance criteria: + +- Killing the daemon during a large upload never loses the local staged content. +- Restarting resumes or safely requeues the upload. +- Editor save patterns do not create broken remote revisions. +- Queue status explains what is pending and why. + +## Phase 5: Remote Change Detection and Reconciliation + +Goal: converge local mount state when changes happen from web, mobile, or +another desktop. + +Work items: + +- Identify Proton Drive event/delta endpoints from current official clients or + reference projects. +- Implement polling or event-based metadata refresh. +- Track remote revision and link state per cached path. +- Detect remote rename, move, trash, restore, and content update. +- Reconcile remote changes into the mounted tree without inode chaos where + possible. +- Add local-vs-remote conflict detection. +- Add conflict naming policy: + - keep remote path intact + - preserve local dirty content + - create a conflict copy with device/profile/timestamp +- Add `sync conflicts` and `sync resolve`. +- Add automatic conflict policy options: manual, keep-local, keep-remote, + keep-both. + +Acceptance criteria: + +- Changes made in the Proton web app appear in the mount without remounting. +- Concurrent edits do not silently overwrite either side. +- Conflict files are recoverable and traceable to source revisions. + +## Phase 6: Trash, Restore, and Permanent Delete + +Goal: make delete behavior match Drive semantics rather than local unlink +semantics only. + +Work items: + +- Add `trash list` with original path, deletion time, size, type, and link ID. +- Add `trash restore`. +- Add `trash empty` and selective permanent delete if the API supports it. +- Represent trash under `/.protonstorage/trash/`. +- Decide and document FUSE `unlink` behavior: + - default: Proton trash + - optional profile policy: hard delete only through explicit CLI +- Handle restoring when original parent no longer exists. +- Handle restoring when name collision exists. +- Add integration tests for trash and restore. + +Acceptance criteria: + +- A file deleted through FUSE can be listed and restored through CLI. +- Restore collision behavior is explicit and tested. +- Permanent delete is never triggered accidentally through ordinary `rm`. + +## Phase 7: Version History + +Goal: expose Drive revision history as a first-class CLI and inspection +feature. + +Work items: + +- Expand revision listing to include author, creation time, size, active flag, + signature verification status, and availability. +- Add `versions download`. +- Add `versions restore`. +- Add `versions delete` if API support is available. +- Expose version metadata under `/.protonstorage/versions/<path>.json` or a + stable ID-based namespace. +- Support duplicate upload policy: + - replace existing file, creating a new version + - keep both + - skip +- Add profile-level default duplicate policy. +- Add tests for restoring an old revision without deleting newer revisions. + +Acceptance criteria: + +- The user can list, download, and restore previous versions from the CLI. +- Duplicate upload behavior is deterministic in batch operations. +- Signature verification status is visible. + +## Phase 8: Sharing and Shared With Me + +Goal: reach practical sharing parity for file and folder workflows. + +Work items: + +- Discover share-link endpoints and payload crypto from official clients and + reference projects. +- Add public link create, update, inspect, and revoke. +- Add password and expiration support for share links. +- Add invite-by-email with viewer/editor role if API support is available. +- Add recipient revoke and role update. +- Add `share list` for user-owned shared items. +- Add `shared-with-me list`, `accept`, `sync`, `unsync`, and `remove`. +- Add `Shared with me` virtual root. +- Preserve owner, shared-by, shared-on, and permission metadata. +- Enforce read-only behavior for viewer permissions in FUSE. +- Handle editor permissions through the normal write queue. +- Add tests for read-only permission rejection. + +Acceptance criteria: + +- The user can create a password-protected expiring link from CLI. +- The user can revoke a link from CLI. +- Shared-with-me items can be mounted and optionally cached offline. +- Viewer-only shared items cannot be modified through FUSE. + +## Phase 9: Metadata Operations + +Goal: close the non-content metadata gaps that app users expect. + +Work items: + +- Determine supported Drive metadata update endpoints. +- Add modification time preservation on upload where supported. +- Add file MIME type update where supported. +- Add folder/file rename already present, but harden metadata refresh after + remote response. +- Add size, media type, and custom Drive attributes where available. +- Decide how local chmod/chown maps: + - likely no-op with success for compatibility, or readonly enforcement for + permission boundaries +- Decide how symlinks are represented: + - reject by default unless Proton Drive has native equivalent +- Add hidden/system file policy. +- Add ignore rules for OS metadata noise: + - `.DS_Store` + - `Thumbs.db` + - editor swap files, optionally configurable + +Acceptance criteria: + +- Common copy tools preserve useful timestamps where Proton supports it. +- Unsupported POSIX metadata does not fail common workflows unnecessarily. +- Noise-file policy is documented and configurable. + +## Phase 10: Multi-Root, Computers, and Selective Sync + +Goal: model the desktop app's device/computer sync concepts without a GUI. + +Work items: + +- Discover account/device/computer root APIs. +- Represent `My files`, `Computers`, and any device roots explicitly. +- Add `sync roots list`. +- Add `sync roots add-local PATH --remote-name NAME`. +- Add `sync roots remove`. +- Add selective sync include/exclude rules. +- Add CLI to rename local device identity. +- Add maximum-device and duplicate-device handling. +- Add policy for mounting only selected roots. + +Acceptance criteria: + +- The user can mount only `My files`, only `Shared with me`, or the virtual + product root. +- Multiple synced local folders can be represented under `Computers`. +- Selective sync rules survive daemon restart. + +## Phase 11: Photos and Albums + +Goal: provide CLI-level support for Drive photo workflows without mobile OS +integration. + +Work items: + +- Discover Photos API shape and encryption behavior. +- Add `photos import PATH_OR_DIR`. +- Add duplicate detection based on content hash and media metadata. +- Add album list/create/delete. +- Add album add/remove photo. +- Add `Photos` virtual root if the API can be represented safely as files. +- Preserve original quality and timestamps. +- Add large media upload progress and retry behavior. +- Add optional EXIF extraction if Jerboa libraries support it, otherwise store + server-side metadata only. + +Acceptance criteria: + +- A directory of photos/videos can be imported idempotently. +- Albums can be listed and managed from CLI. +- Photo files are not accidentally deleted by normal filesystem operations + unless explicitly supported and documented. + +## Phase 12: Docs and Sheets File Handling + +Goal: make Drive-native document types safe in a filesystem workflow. + +Work items: + +- Identify Drive link/file types for Docs and Sheets. +- Decide read behavior: + - expose as metadata stubs + - export to supported formats through CLI if API supports it + - open in browser helper command