Generate SSD sync tree exports from typed Jerboa
ober
90a77a56cdef047873f7c0cca924d5919dd664ca
--- a/templates/ssd-review.ss +++ b/templates/ssd-review.ss @@ -15555,8 +15555,10 @@ (typed-kotlin-file "com/sfb/ssdreview/TruthStoreLocal.kt" (kotlin-imports (android content Context) + (android content ContentResolver) (android content SharedPreferences) (android content res AssetManager) + (android net Uri) (android util Base64) (java io File) (java io BufferedInputStream) @@ -15575,6 +15577,7 @@ (java util zip ZipInputStream) (java util zip ZipOutputStream) (javax net ssl HttpsURLConnection) + (androidx documentfile provider DocumentFile) (org json JSONArray) (org json JSONObject)) (typed-library (com sfb ssdreview) @@ -15592,6 +15595,9 @@ truthStoreTruthIndexEntryLocal truthStoreExportZipLocal truthStoreImportZipLocal + truthStoreExportToTreeLocal + truthStoreEnsureTreeDirLocal + truthStoreWriteTreeFileLocal truthStoreReadResponseTextLocal truthStoreWriteRemoteTruthLocal truthStoreApplyLearnedGuessesLocal @@ -15613,8 +15619,11 @@ (type Any) (type BufferedInputStream) (type BufferedOutputStream) + (type Char) + (type ContentResolver) (type Context) (type DateTimeFormatter) + (type DocumentFile) (type Exception) (type File) (type Instant) @@ -15634,6 +15643,7 @@ (type StorageStats) (type TruthIndexEntry) (type TruthStore) + (type Uri) (type URL) (type JSONArray) (type JSONObject) @@ -15646,6 +15656,9 @@ (extern (truthStoreContext (store : TruthStore)) : Context (kotlin-member-get context)) + (extern (truthStoreContextContentResolver + (context : Context)) : ContentResolver + (kotlin-member-get contentResolver)) (extern (truthStoreContextCacheDir (context : Context)) : File (kotlin-member-get cacheDir)) @@ -15872,6 +15885,8 @@ (extern (truthStoreFileName (file : File)) : String (kotlin-member-get name)) + (extern (truthStoreFileSeparatorChar) : Char + (kotlin-value File separatorChar)) (extern (truthStoreListFiles (file : File)) : (Nullable (Array File)) (kotlin-member-call listFiles)) @@ -15890,6 +15905,26 @@ (extern (truthStoreInputStreamClose (input : InputStream)) : Unit (kotlin-member-call close)) + (extern (truthStoreContentResolverOpenOutputStream + (resolver : ContentResolver) + (uri : Uri)) : (Nullable OutputStream) + (kotlin-member-call openOutputStream)) + (extern (truthStoreDocumentFindFile + (file : DocumentFile) + (name : String)) : (Nullable DocumentFile) + (kotlin-member-call findFile)) + (extern (truthStoreDocumentCreateDirectory + (file : DocumentFile) + (name : String)) : (Nullable DocumentFile) + (kotlin-member-call createDirectory)) + (extern (truthStoreDocumentCreateFile + (file : DocumentFile) + (mime : String) + (name : String)) : (Nullable DocumentFile) + (kotlin-member-call createFile)) + (extern (truthStoreDocumentUri + (file : DocumentFile)) : Uri + (kotlin-member-get uri)) (extern (truthStoreFileToPath (file : File)) : Path (kotlin-member-call toPath)) @@ -15973,6 +16008,8 @@ (kotlin-value MAX_ZIP_TOTAL_BYTES)) (extern (truthStoreMaxZipEntries) : Int32 (kotlin-value MAX_ZIP_ENTRIES)) + (extern (truthStoreMaxZipDepth) : Int32 + (kotlin-value MAX_ZIP_DEPTH)) (extern (truthStoreRatioFloorBytes) : Int (kotlin-value RATIO_FLOOR_BYTES)) (extern (truthStoreMaxCompressionRatio) : Int @@ -16805,6 +16842,200 @@ input stage) (truthStoreDeleteRecursivelyUnit stage)))) + (def (truthStoreEnsureTreeDirLocal + (parent : DocumentFile) + (name : String)) : DocumentFile + (begin + (if (safePathComponentInvalid name) + (throw + (truthStoreIllegalArgumentException + "Unsafe tree directory") + Unit) + (begin)) + (let ((existing + (truthStoreDocumentFindFile parent name))) + (if (documentFilePresent existing) + (let ((existingDir (nullable-get existing))) + (if (documentFileIsDirectory existingDir) + existingDir + (let ((created + (truthStoreDocumentCreateDirectory + parent + name))) + (if (documentFilePresent created) + (nullable-get created) + (throw + (truthStoreIllegalStateException + (unableToCreateSyncFolderMessage name)) + DocumentFile))))) + (let ((created + (truthStoreDocumentCreateDirectory + parent + name))) + (if (documentFilePresent created) + (nullable-get created) + (throw + (truthStoreIllegalStateException + (unableToCreateSyncFolderMessage name)) + DocumentFile))))))) + (def (truthStoreCopyFileToDocumentOutput + (source : File) + (output : OutputStream)) : Unit + (let ((input (truthStoreFileInputStream source))) + (try-finally + (copyBoundedInputToOutput + input + output + (truthStoreMaxZipEntryBytes)) + (truthStoreInputStreamClose input)))) + (def (truthStoreWriteTreeFileToDocument + (store : TruthStore) + (doc : DocumentFile) + (source : File)) : Unit + (let ((output + (truthStoreContentResolverOpenOutputStream + (truthStoreContextContentResolver + (truthStoreContext store)) + (truthStoreDocumentUri doc)))) + (if (outputStreamPresent output) + (let ((stream (nullable-get output))) + (try-finally + (truthStoreCopyFileToDocumentOutput source stream) + (truthStoreOutputStreamClose stream))) + (begin)))) + (def (truthStoreWriteTreeFileLocal + (store : TruthStore) + (parent : DocumentFile) + (relativeName : String) + (source : File)) : Unit + (begin + (truthStoreRequired + (safeLocalRegularFile source) + "Failed requirement.") + (truthStoreRequired + (byteCountWithinLimit + (fileSizeBytes source) + (truthStoreMaxZipEntryBytes)) + "Tree export file exceeds byte limit") + (let ((parts + (trimmedNonBlankPathParts + relativeName + (truthStoreFileSeparatorChar) + #\/))) + (begin + (truthStoreRequired + (safeTreePathParts parts (truthStoreMaxZipDepth)) + "Failed requirement.") + (let ((fileName (treePathLeaf parts))) + (let ((dir + (ensureTreePathDirectories + parent + parts + (lambda ((current : DocumentFile) + (name : String)) + (truthStoreEnsureTreeDirLocal + current + name))))) + (begin + (deleteDocumentFileIfPresent + (truthStoreDocumentFindFile dir fileName)) + (let ((created + (truthStoreDocumentCreateFile + dir + (treeFileMime fileName) + fileName))) + (if (documentFilePresent created) + (truthStoreWriteTreeFileToDocument + store + (nullable-get created) + source) + (throw + (truthStoreIllegalStateException + (unableToCreateFileMessage fileName)) + Unit)))))))))) + (def (truthStoreTreeExportAccount! + (budget : ImportBudget) + (file : File)) : Unit + (begin + (ImportBudget-entries-set! + budget + (+ (ImportBudget-entries budget) (int32 1))) + (ImportBudget-expandedBytes-set! + budget + (+ (ImportBudget-expandedBytes budget) (fileSizeBytes file))) + (if (treeExportQuotaExceeded + (ImportBudget-entries budget) + (ImportBudget-expandedBytes budget) + (truthStoreMaxZipEntries) + (truthStoreMaxZipTotalBytes)) + (throw + (truthStoreIllegalStateException + "Tree export quota exceeded") + Unit) + (begin)))) + (def (truthStoreExportTreeFileLocal + (store : TruthStore) + (targetDir : DocumentFile) + (sourceDir : File) + (budget : ImportBudget) + (file : File)) : Unit + (begin + (truthStoreTreeExportAccount! budget file) + (truthStoreWriteTreeFileLocal + store + targetDir + (truthStorePathToString + (truthStorePathRelativize + (truthStoreFileToPath sourceDir) + (truthStoreFileToPath file))) + file))) + (def (truthStoreExportTreeDirectoryLocal + (store : TruthStore) + (tree : DocumentFile) + (budget : ImportBudget) + (dir : File)) : Unit + (let ((targetDir + (truthStoreEnsureTreeDirLocal + tree + (truthStoreFileName dir)))) + (forEachRegularFileUnder + dir + (lambda ((file : File)) + (truthStoreExportTreeFileLocal + store + targetDir + dir + budget + file))))) + (def (truthStoreExportToTreeLocal + (store : TruthStore) + (tree : DocumentFile)) : Int32 + (begin + (truthStoreStorageStatsLocal store) + (let ((budget + (make-ImportBudget (int32 0) (int 0)))) + (begin + (truthStoreExportTreeDirectoryLocal + store + tree + budget + (truthStoreGroundTruthDir store)) + (truthStoreExportTreeDirectoryLocal + store + tree + budget + (truthStoreLearnedDir store)) + (truthStoreExportTreeDirectoryLocal + store + tree + budget + (truthStoreEventsDir store)) + (truthStoreExportTreeDirectoryLocal + store + tree + budget + (truthStoreReviewsDir store)) + (ImportBudget-entries budget))))) (def (truthStoreSeedBundledTruthLocal (store : TruthStore) (context : Context)) : Int32 @@ -17663,25 +17894,8 @@ " internal fun seedBundledTruth(): Int =" " truthStoreSeedBundledTruthLocal(this, context)" "" - " private fun exportToTree(tree: DocumentFile): Int {" - " storageStats()" - " var count = 0" - " var bytes = 0L" - " forEachFileUntil(listOf(groundTruthDir, learnedDir, eventsDir, reviewsDir)) { dir ->" - " val targetDir = ensureTreeDir(tree, dir.name)" - " forEachRegularFileUnder(dir) { file ->" - " count += 1" - " bytes += fileSizeBytes(file)" - " if (treeExportQuotaExceeded(count, bytes, MAX_ZIP_ENTRIES, MAX_ZIP_TOTAL_BYTES)) {" - " throw IllegalStateException(\"Tree export quota exceeded\")" - " }" - " val relative = dir.toPath().relativize(file.toPath()).toString()" - " writeTreeFile(targetDir, relative, file)" - " }" - " true" - " }" - " return count" - " }" + " private fun exportToTree(tree: DocumentFile): Int =" + " truthStoreExportToTreeLocal(this, tree)" "" " private fun importFromTree(tree: DocumentFile): Int {" " var count = 0" @@ -17746,38 +17960,11 @@ " return count" " }" "" - " private fun ensureTreeDir(parent: DocumentFile, name: String): DocumentFile {" - " if (safePathComponentInvalid(name)) throw IllegalArgumentException(\"Unsafe tree directory\")" - " val existing = parent.findFile(name)" - " if (documentFilePresent(existing)) {" - " val existingDir = checkNotNull(existing)" - " if (documentFileIsDirectory(existingDir)) return existingDir" - " }" - " val created = parent.createDirectory(name)" - " if (!documentFilePresent(created)) throw IllegalStateException(unableToCreateSyncFolderMessage(name))" - " return checkNotNull(created)" - " }" + " private fun ensureTreeDir(parent: DocumentFile, name: String): DocumentFile =" + " truthStoreEnsureTreeDirLocal(parent, name)" "" - " private fun writeTreeFile(parent: DocumentFile, relativeName: String, source: File) {" - " require(safeLocalRegularFile(source))" - " require(byteCountWithinLimit(fileSizeBytes(source), MAX_ZIP_ENTRY_BYTES)) { \"Tree export file exceeds byte limit\" }" - " val parts = trimmedNonBlankPathParts(relativeName, File.separatorChar, '/')" - " require(safeTreePathParts(parts, MAX_ZIP_DEPTH))" - " val fileName = treePathLeaf(parts)" - " val dir = ensureTreePathDirectories(parent, parts) { current, name -> ensureTreeDir(current, name) }" - " deleteDocumentFileIfPresent(dir.findFile(fileName))" - " val mime = treeFileMime(fileName)" - " val createdFile = dir.createFile(mime, fileName)" - " if (!documentFilePresent(createdFile)) throw IllegalStateException(unableToCreateFileMessage(fileName))" - " val doc = checkNotNull(createdFile)" - " val outputStream = context.contentResolver.openOutputStream(doc.uri)" - " if (!outputStreamPresent(outputStream)) return" - " checkNotNull(outputStream).use { output ->" - " source.inputStream().use { input ->" - " LimitedInputStream(input, MAX_ZIP_ENTRY_BYTES).use { it.copyTo(output, 32 * 1024) }" - " }" - " }" - " }" + " private fun writeTreeFile(parent: DocumentFile, relativeName: String, source: File) =" + " truthStoreWriteTreeFileLocal(this, parent, relativeName, source)" "" " private fun mergeJsonLines(file: File, remoteText: String) =" " truthStoreMergeJsonLinesLocal(this, file, remoteText)"