Generate SSD tree import traversal from typed Jerboa
ober
2287362f8ea65c6e5fd3ca8269d8a19abeb0e3b0
--- a/templates/ssd-review.ss +++ b/templates/ssd-review.ss @@ -2091,6 +2091,7 @@ zipEntryPresent zipEntryIsDirectory zipEntryIsFile zipEntryDeclaresExcessiveSize forEachFileArray forEachFileUntil forEachRegularFileUnder + forEachStringUntil countInstalledStagedZipEntries) (type File) (type FileTreeWalk) @@ -2159,6 +2160,16 @@ (invoke action (list-ref files i)) keepGoing)) (begin))) + (def (forEachStringUntil + (items : (List String)) + (action : (-> String Bool))) : Unit + (begin + (for/fold ((keepGoing #t)) + ((i (in-range (int32 0) (list-size items)))) + (if keepGoing + (invoke action (list-ref items i)) + keepGoing)) + (begin))) (extern (pathIsSymbolicLinkRaw (path : Path)) : Bool (kotlin-call Files isSymbolicLink)) (extern (pathParent (path : Path)) : Path @@ -12360,12 +12371,13 @@ " private fun importFromTree(tree: DocumentFile): Int {" " var count = 0" " val budget = ImportBudget(0, 0L)" - " listOf(\"ground_truth\", \"learned\", \"events\", \"reviews\").forEach { name ->" + " forEachStringUntil(listOf(\"ground_truth\", \"learned\", \"events\", \"reviews\")) { name ->" " val sourceDir = tree.findFile(name)" " val localDir = File(root, name)" " if (documentFilePresent(sourceDir) && documentFileIsDirectory(checkNotNull(sourceDir))) {" " count += importTreeDir(checkNotNull(sourceDir), localDir, 0, budget)" " }" + " true" " }" " return count" " }" @@ -12379,36 +12391,40 @@ " if (treeDepthExceeded(depth, MAX_ZIP_DEPTH)) throw IllegalStateException(\"Import tree exceeds depth limit\")" " localDir.mkdirs()" " var count = 0" - " sourceDir.listFiles().forEach { child ->" + " forEachDocumentFile(sourceDir.listFiles().toList()) { child ->" " budget.entries += 1" " if (zipEntryCountExceeded(budget.entries, MAX_ZIP_ENTRIES)) throw IllegalStateException(\"Import tree exceeds entry limit\")" " if (documentFileIsDirectory(child)) {" " val rawDirectoryName = child.name" - " if (!nullableTextPresent(rawDirectoryName)) return@forEach" - " val directoryName = nullableTextOrEmpty(rawDirectoryName)" - " if (safePathComponentInvalid(directoryName)) throw IllegalStateException(\"Unsafe import directory\")" - " count += importTreeDir(child, File(localDir, directoryName), depth + 1, budget)" + " if (nullableTextPresent(rawDirectoryName)) {" + " val directoryName = nullableTextOrEmpty(rawDirectoryName)" + " if (safePathComponentInvalid(directoryName)) throw IllegalStateException(\"Unsafe import directory\")" + " count += importTreeDir(child, File(localDir, directoryName), depth + 1, budget)" + " }" " } else if (documentFileIsRegular(child)) {" " val rawName = child.name" - " if (!nullableTextPresent(rawName)) return@forEach" - " val name = nullableTextOrEmpty(rawName)" - " if (safeLeafInvalid(name)) return@forEach" - " val remaining = MAX_ZIP_TOTAL_BYTES - budget.expandedBytes" - " if (nonPositiveByteCount(remaining)) throw IllegalStateException(\"Import tree exceeds byte limit\")" - " val childInput = context.contentResolver.openInputStream(child.uri)" - " if (!inputStreamPresent(childInput)) return@forEach" - " val bytes = checkNotNull(childInput).use {" - " readBoundedBytes(it, boundedTreeReadLimit(MAX_ZIP_ENTRY_BYTES, remaining))" - " }" - " budget.expandedBytes += byteArraySizeBytes(bytes)" - " validateImportedText(name, bytes.decodeToString())" - " val dest = containedImportPath(root.toPath().relativize(File(localDir, name).toPath()).toString())" - " if (jsonLinesFileName(name)) {" - " mergeJsonLines(dest, bytes.decodeToString())" - " count += 1" - " } else if (shouldReplaceLocal(dest, bytes.decodeToString(), child.lastModified())) {" - " atomicWrite(dest, bytes)" - " count += 1" + " if (nullableTextPresent(rawName)) {" + " val name = nullableTextOrEmpty(rawName)" + " if (!safeLeafInvalid(name)) {" + " val remaining = MAX_ZIP_TOTAL_BYTES - budget.expandedBytes" + " if (nonPositiveByteCount(remaining)) throw IllegalStateException(\"Import tree exceeds byte limit\")" + " val childInput = context.contentResolver.openInputStream(child.uri)" + " if (inputStreamPresent(childInput)) {" + " val bytes = checkNotNull(childInput).use {" + " readBoundedBytes(it, boundedTreeReadLimit(MAX_ZIP_ENTRY_BYTES, remaining))" + " }" + " budget.expandedBytes += byteArraySizeBytes(bytes)" + " validateImportedText(name, bytes.decodeToString())" + " val dest = containedImportPath(root.toPath().relativize(File(localDir, name).toPath()).toString())" + " if (jsonLinesFileName(name)) {" + " mergeJsonLines(dest, bytes.decodeToString())" + " count += 1" + " } else if (shouldReplaceLocal(dest, bytes.decodeToString(), child.lastModified())) {" + " atomicWrite(dest, bytes)" + " count += 1" + " }" + " }" + " }" " }" " }" " }"