Generate SSD sync tree imports from typed Jerboa
ober
63fad75fdfb35213e5b58613c80c6fd63138c80d
--- a/templates/ssd-review.ss +++ b/templates/ssd-review.ss @@ -15596,6 +15596,8 @@ truthStoreExportZipLocal truthStoreImportZipLocal truthStoreExportToTreeLocal + truthStoreImportFromTreeLocal + truthStoreImportTreeDirLocal truthStoreEnsureTreeDirLocal truthStoreWriteTreeFileLocal truthStoreReadResponseTextLocal @@ -15909,10 +15911,26 @@ (resolver : ContentResolver) (uri : Uri)) : (Nullable OutputStream) (kotlin-member-call openOutputStream)) + (extern (truthStoreContentResolverOpenInputStream + (resolver : ContentResolver) + (uri : Uri)) : (Nullable InputStream) + (kotlin-member-call openInputStream)) + (extern (truthStoreFileMkdirs + (file : File)) : Bool + (kotlin-member-call mkdirs)) + (extern (truthStoreDocumentListFiles + (file : DocumentFile)) : (Array DocumentFile) + (kotlin-member-call listFiles)) (extern (truthStoreDocumentFindFile (file : DocumentFile) (name : String)) : (Nullable DocumentFile) (kotlin-member-call findFile)) + (extern (truthStoreDocumentName + (file : DocumentFile)) : (Nullable String) + (kotlin-member-get name)) + (extern (truthStoreDocumentLastModified + (file : DocumentFile)) : Int + (kotlin-member-call lastModified)) (extern (truthStoreDocumentCreateDirectory (file : DocumentFile) (name : String)) : (Nullable DocumentFile) @@ -17036,6 +17054,244 @@ budget (truthStoreReviewsDir store)) (ImportBudget-entries budget))))) + (def (truthStoreMkdirsUnit + (file : File)) : Unit + (begin + (truthStoreFileMkdirs file) + (begin))) + (def (truthStoreAccountImportTreeEntry! + (budget : ImportBudget)) : Unit + (begin + (ImportBudget-entries-set! + budget + (+ (ImportBudget-entries budget) (int32 1))) + (if (zipEntryCountExceeded + (ImportBudget-entries budget) + (truthStoreMaxZipEntries)) + (throw + (truthStoreIllegalStateException + "Import tree exceeds entry limit") + Unit) + (begin)))) + (def (truthStoreReadDocumentBytesLocal + (store : TruthStore) + (child : DocumentFile) + (remaining : Int)) : (Nullable Bytes) + (let ((input + (truthStoreContentResolverOpenInputStream + (truthStoreContextContentResolver + (truthStoreContext store)) + (truthStoreDocumentUri child)))) + (if (inputStreamPresent input) + (let ((stream (nullable-get input))) + (try-finally + (nullable-some + (readBoundedBytes + stream + (boundedTreeReadLimit + (truthStoreMaxZipEntryBytes) + remaining))) + (truthStoreInputStreamClose stream))) + (nullable-none Bytes)))) + (def (truthStoreImportTreeFileDestination + (store : TruthStore) + (localDir : File) + (name : String)) : File + (truthStoreContainedImportPathLocal + store + (truthStorePathToString + (truthStorePathRelativize + (truthStoreFileToPath (truthStoreRoot store)) + (truthStoreFileToPath + (truthStoreChildFile localDir name)))))) + (def (truthStoreImportTreeFileBytesLocal + (store : TruthStore) + (localDir : File) + (name : String) + (modified : Int) + (bytes : Bytes)) : Int32 + (let ((text (utf8->string bytes)) + (dest + (truthStoreImportTreeFileDestination + store + localDir + name))) + (begin + (truthStoreValidateImportedTextLocal name text) + (if (jsonLinesFileName name) + (begin + (truthStoreMergeJsonLinesLocal store dest text) + (int32 1)) + (if (truthStoreShouldReplaceLocal dest text modified) + (begin + (truthStoreAtomicWriteRaw store dest bytes) + (int32 1)) + (int32 0)))))) + (def (truthStoreImportTreeRegularFileLocal + (store : TruthStore) + (localDir : File) + (budget : ImportBudget) + (child : DocumentFile) + (name : String)) : Int32 + (if (safeLeafInvalid name) + (int32 0) + (let ((remaining + (- (truthStoreMaxZipTotalBytes) + (ImportBudget-expandedBytes budget)))) + (begin + (if (nonPositiveByteCount remaining) + (throw + (truthStoreIllegalStateException + "Import tree exceeds byte limit") + Unit) + (begin)) + (let ((bytes + (truthStoreReadDocumentBytesLocal + store + child + remaining))) + (if (nullable-null? bytes) + (int32 0) + (begin + (ImportBudget-expandedBytes-set! + budget + (+ (ImportBudget-expandedBytes budget) + (byteArraySizeBytes (nullable-get bytes)))) + (truthStoreImportTreeFileBytesLocal + store + localDir + name + (truthStoreDocumentLastModified child) + (nullable-get bytes))))))))) + (def (truthStoreImportTreeDirectoryChildLocal + (store : TruthStore) + (localDir : File) + (depth : Int32) + (budget : ImportBudget) + (child : DocumentFile) + (rawName : (Nullable String))) : Int32 + (if (nullableTextPresent rawName) + (let ((directoryName (nullableTextOrEmpty rawName))) + (begin + (if (safePathComponentInvalid directoryName) + (throw + (truthStoreIllegalStateException + "Unsafe import directory") + Unit) + (begin)) + (truthStoreImportTreeDirLocal + store + child + (truthStoreChildFile localDir directoryName) + (+ depth (int32 1)) + budget))) + (int32 0))) + (def (truthStoreImportTreeRegularChildLocal + (store : TruthStore) + (localDir : File) + (budget : ImportBudget) + (child : DocumentFile) + (rawName : (Nullable String))) : Int32 + (if (nullableTextPresent rawName) + (truthStoreImportTreeRegularFileLocal + store + localDir + budget + child + (nullableTextOrEmpty rawName)) + (int32 0))) + (def (truthStoreImportTreeChildLocal + (store : TruthStore) + (localDir : File) + (depth : Int32) + (budget : ImportBudget) + (child : DocumentFile)) : Int32 + (begin + (truthStoreAccountImportTreeEntry! budget) + (if (documentFileIsDirectory child) + (truthStoreImportTreeDirectoryChildLocal + store + localDir + depth + budget + child + (truthStoreDocumentName child)) + (if (documentFileIsRegular child) + (truthStoreImportTreeRegularChildLocal + store + localDir + budget + child + (truthStoreDocumentName child)) + (int32 0))))) + (def (truthStoreImportTreeDirLocal + (store : TruthStore) + (sourceDir : DocumentFile) + (localDir : File) + (depth : Int32) + (budget : ImportBudget)) : Int32 + (begin + (if (treeDepthExceeded depth (truthStoreMaxZipDepth)) + (throw + (truthStoreIllegalStateException + "Import tree exceeds depth limit") + Unit) + (begin)) + (truthStoreMkdirsUnit localDir) + (let ((children (truthStoreDocumentListFiles sourceDir))) + (for/fold ((count (int32 0))) + ((i (in-range + (int32 0) + (list-size children)))) + (+ count + (truthStoreImportTreeChildLocal + store + localDir + depth + budget + (list-ref children i))))))) + (def (truthStoreImportNamedTreeDirLocal + (store : TruthStore) + (tree : DocumentFile) + (budget : ImportBudget) + (name : String)) : Int32 + (let ((sourceDir (truthStoreDocumentFindFile tree name))) + (if (and (documentFilePresent sourceDir) + (documentFileIsDirectory + (nullable-get sourceDir))) + (truthStoreImportTreeDirLocal + store + (nullable-get sourceDir) + (truthStoreChildFile + (truthStoreRoot store) + name) + (int32 0) + budget) + (int32 0)))) + (def (truthStoreImportFromTreeLocal + (store : TruthStore) + (tree : DocumentFile)) : Int32 + (let ((budget (make-ImportBudget (int32 0) (int 0)))) + (+ (truthStoreImportNamedTreeDirLocal + store + tree + budget + "ground_truth") + (+ (truthStoreImportNamedTreeDirLocal + store + tree + budget + "learned") + (+ (truthStoreImportNamedTreeDirLocal + store + tree + budget + "events") + (truthStoreImportNamedTreeDirLocal + store + tree + budget + "reviews")))))) (def (truthStoreSeedBundledTruthLocal (store : TruthStore) (context : Context)) : Int32 @@ -17897,68 +18153,16 @@ " private fun exportToTree(tree: DocumentFile): Int =" " truthStoreExportToTreeLocal(this, tree)" "" - " private fun importFromTree(tree: DocumentFile): Int {" - " var count = 0" - " val budget = ImportBudget(0, 0L)" - " forEachStringUntil(listOf(\"ground_truth\", \"learned\", \"events\", \"reviews\")) { name ->" - " val sourceDir = tree.findFile(name)" - " val localDir = File(root, name)" - " if (documentFilePresent(sourceDir) && documentFileIsDirectory(checkNotNull(sourceDir))) {" - " count += importTreeDir(checkNotNull(sourceDir), localDir, 0, budget)" - " }" - " true" - " }" - " return count" - " }" + " private fun importFromTree(tree: DocumentFile): Int =" + " truthStoreImportFromTreeLocal(this, tree)" "" " private fun importTreeDir(" " sourceDir: DocumentFile," " localDir: File," " depth: Int," " budget: ImportBudget" - " ): Int {" - " if (treeDepthExceeded(depth, MAX_ZIP_DEPTH)) throw IllegalStateException(\"Import tree exceeds depth limit\")" - " localDir.mkdirs()" - " var count = 0" - " forEachDocumentFile(sourceDir.listFiles().toList()) { child ->" - " budget.entries += 1" - " if (zipEntryCountExceeded(budget.entries, MAX_ZIP_ENTRIES)) throw IllegalStateException(\"Import tree exceeds entry limit\")" - " if (documentFileIsDirectory(child)) {" - " val rawDirectoryName = child.name" - " if (nullableTextPresent(rawDirectoryName)) {" - " val directoryName = nullableTextOrEmpty(rawDirectoryName)" - " if (safePathComponentInvalid(directoryName)) throw IllegalStateException(\"Unsafe import directory\")" - " count += importTreeDir(child, File(localDir, directoryName), depth + 1, budget)" - " }" - " } else if (documentFileIsRegular(child)) {" - " val rawName = child.name" - " if (nullableTextPresent(rawName)) {" - " val name = nullableTextOrEmpty(rawName)" - " if (!safeLeafInvalid(name)) {" - " val remaining = MAX_ZIP_TOTAL_BYTES - budget.expandedBytes" - " if (nonPositiveByteCount(remaining)) throw IllegalStateException(\"Import tree exceeds byte limit\")" - " val childInput = context.contentResolver.openInputStream(child.uri)" - " if (inputStreamPresent(childInput)) {" - " val bytes = checkNotNull(childInput).use {" - " readBoundedBytes(it, boundedTreeReadLimit(MAX_ZIP_ENTRY_BYTES, remaining))" - " }" - " budget.expandedBytes += byteArraySizeBytes(bytes)" - " validateImportedText(name, bytes.decodeToString())" - " val dest = containedImportPath(root.toPath().relativize(File(localDir, name).toPath()).toString())" - " if (jsonLinesFileName(name)) {" - " mergeJsonLines(dest, bytes.decodeToString())" - " count += 1" - " } else if (shouldReplaceLocal(dest, bytes.decodeToString(), child.lastModified())) {" - " atomicWrite(dest, bytes)" - " count += 1" - " }" - " }" - " }" - " }" - " }" - " }" - " return count" - " }" + " ): Int =" + " truthStoreImportTreeDirLocal(this, sourceDir, localDir, depth, budget)" "" " private fun ensureTreeDir(parent: DocumentFile, name: String): DocumentFile =" " truthStoreEnsureTreeDirLocal(parent, name)"