Generate SSD remote dump pushes from typed Jerboa
ober
b42df6092aed775d212c0a6646b7070c4be1f300
--- a/templates/ssd-review.ss +++ b/templates/ssd-review.ss @@ -12278,7 +12278,8 @@ (java util zip ZipEntry) (java util zip ZipInputStream)) (typed-library (com sfb ssdreview) - (export readBoundedBytes sha1Hex forEachZipEntry streamZipEntryToOutput) + (export readBoundedBytes sha1Hex forEachZipEntry + copyBoundedInputToOutput streamZipEntryToOutput) (type ByteArrayOutputStream) (type Exception) (type InputStream) @@ -12382,6 +12383,31 @@ (invoke action (nullable-get nextEntry)) (zipInputStreamCloseEntry zip)) (break))))) + (def (copyBoundedInputToOutput + (input : InputStream) + (output : OutputStream) + (limit : Int)) : Unit + (if (boundedReadLimitValid limit) + (let ((buffer (make-bytevector (int32 32768) 0))) + (var ((total (int 0))) + (while #t + (let ((read (inputStreamRead input buffer))) + (begin + (if (streamReadEnded read) (break) (begin)) + (if (streamReadEmpty read) (continue) (begin)) + (set! total (+ total (int read))) + (if (inputCountExceeded total limit) + (throw + (illegalStateException + (inputLimitExceededMessage limit)) + Unit) + (begin)) + (outputStreamWrite + output + buffer + (int32 0) + read)))))) + (throw (illegalArgumentException "Invalid bounded read limit") Unit))) (def (streamZipEntryToOutput (zip : ZipInputStream) (output : OutputStream) @@ -15534,6 +15560,7 @@ (android util Base64) (java io File) (java io InputStream) + (java io OutputStream) (java net URL) (java nio file LinkOption) (java nio file Path) @@ -15566,6 +15593,7 @@ truthStoreShouldReplaceLocal truthStoreSaveTruthSnapshotLocal truthStoreMergeJsonLinesLocal + truthStorePushRemoteDumpLocal truthStorePullRemoteDumpLocal) (type AssetManager) (type Any) @@ -15581,6 +15609,7 @@ (type MessageDigest) (type Number) (type OffsetDateTime) + (type OutputStream) (type Path) (type RemoteConfig) (type SsdSession) @@ -15593,6 +15622,12 @@ (extern (truthStoreRoot (store : TruthStore)) : File (kotlin-member-get root)) + (extern (truthStoreContext + (store : TruthStore)) : Context + (kotlin-member-get context)) + (extern (truthStoreContextCacheDir + (context : Context)) : File + (kotlin-member-get cacheDir)) (extern (truthStoreGroundTruthDir (store : TruthStore)) : File (kotlin-member-get groundTruthDir)) @@ -15627,6 +15662,10 @@ (store : TruthStore) (input : InputStream)) : Int32 (kotlin-member-call importZip)) + (extern (truthStoreExportZipRaw + (store : TruthStore) + (output : OutputStream)) : Unit + (kotlin-member-call exportZip)) (extern (truthStoreOpenPinnedConnectionRaw (store : TruthStore) (relative : String) @@ -15694,6 +15733,9 @@ (extern (truthStoreConnectionInputStream (connection : HttpsURLConnection)) : InputStream (kotlin-member-get inputStream)) + (extern (truthStoreConnectionOutputStream + (connection : HttpsURLConnection)) : OutputStream + (kotlin-member-get outputStream)) (extern (truthStoreConnectionContentEncoding (connection : HttpsURLConnection)) : (Nullable String) (kotlin-member-get contentEncoding)) @@ -15738,6 +15780,12 @@ (extern (truthStoreFileInputStream (file : File)) : InputStream (kotlin-member-call inputStream)) + (extern (truthStoreFileOutputStream + (file : File)) : OutputStream + (kotlin-member-call outputStream)) + (extern (truthStoreOutputStreamClose + (output : OutputStream)) : Unit + (kotlin-member-call close)) (extern (truthStoreFileLastModified (file : File)) : Int (kotlin-member-call lastModified)) @@ -15761,6 +15809,14 @@ (extern (truthStorePathToFile (path : Path)) : File (kotlin-member-call toFile)) + (extern (truthStoreCreateTempFile + (prefix : String) + (suffix : String) + (directory : File)) : File + (kotlin-call File createTempFile)) + (extern (truthStoreFileDelete + (file : File)) : Bool + (kotlin-member-call delete)) (extern (truthStoreNoFollowLinks) : LinkOption (kotlin-value LinkOption NOFOLLOW_LINKS)) (extern (truthStoreIllegalArgumentException @@ -15801,6 +15857,8 @@ (kotlin-value MAX_HTTP_COMPRESSED_BYTES)) (extern (truthStoreMaxRemoteZipBytes) : Int (kotlin-value MAX_REMOTE_ZIP_BYTES)) + (extern (truthStoreMaxZipTotalBytes) : Int + (kotlin-value MAX_ZIP_TOTAL_BYTES)) (extern (truthStoreMaxStorageBytes) : Int (kotlin-value MAX_STORAGE_BYTES)) (extern (truthStoreMaxStorageFiles) : Int32 @@ -16467,6 +16525,79 @@ store file merged))))))) + (def (truthStoreDeleteFileUnit + (file : File)) : Unit + (begin + (truthStoreFileDelete file) + (begin))) + (def (truthStoreExportZipToFileLocal + (store : TruthStore) + (archive : File)) : Unit + (let ((output + (truthStoreFileOutputStream archive))) + (try-finally + (truthStoreExportZipRaw store output) + (truthStoreOutputStreamClose output)))) + (def (truthStoreUploadArchiveLocal + (archive : File) + (connection : HttpsURLConnection)) : Unit + (let ((output + (truthStoreConnectionOutputStream connection))) + (try-finally + (let ((input + (truthStoreFileInputStream archive))) + (try-finally + (copyBoundedInputToOutput + input + output + (truthStoreMaxZipTotalBytes)) + (truthStoreInputStreamClose input))) + (truthStoreOutputStreamClose output)))) + (def (truthStorePushRemoteDumpLocal + (store : TruthStore)) : Int32 + (let ((archive + (truthStoreCreateTempFile + "ssd-upload-" + ".zip" + (truthStoreContextCacheDir + (truthStoreContext store))))) + (try-finally + (begin + (truthStoreExportZipToFileLocal store archive) + (truthStoreRequired + (byteCountWithinLimit + (fileSizeBytes archive) + (truthStoreMaxZipTotalBytes)) + "Upload ZIP exceeds byte limit") + (let ((connection + (truthStoreOpenPinnedConnectionRaw + store + "load.zip" + "POST" + (nullable-some (fileSizeBytes archive)) + (nullable-some "application/zip") + #f))) + (try-finally + (begin + (truthStoreUploadArchiveLocal archive connection) + (let ((code + (truthStoreConnectionResponseCode connection))) + (begin + (if (httpStatusFailed code) + (begin + (closeInputStreamIfPresent + (truthStoreConnectionErrorStream connection)) + (throw + (truthStoreIllegalStateException + (httpStatusMessage code)) + Unit)) + (begin)) + (remoteDumpImportedCount + (jsonObjectFromText + (truthStoreReadResponseTextLocal + connection)))))) + (truthStoreConnectionDisconnect connection)))) + (truthStoreDeleteFileUnit archive)))) (def (truthStorePullRemoteDumpLocal (store : TruthStore)) : Int32 (let ((connection @@ -16547,7 +16678,7 @@ "import javax.net.ssl.X509TrustManager" "import kotlin.concurrent.thread" "" - "class TruthStore(private val context: Context) {" + "class TruthStore(internal val context: Context) {" " internal val root = File(context.filesDir, \"ssd_review\")" " internal val groundTruthDir = File(root, \"ground_truth\")" " internal val learnedDir = File(root, \"learned\")" @@ -16906,35 +17037,8 @@ " fun pullRemoteDump(): Int =" " truthStorePullRemoteDumpLocal(this)" "" - " fun pushRemoteDump(): Int {" - " val archive = File.createTempFile(\"ssd-upload-\", \".zip\", context.cacheDir)" - " try {" - " archive.outputStream().use { exportZip(it) }" - " require(byteCountWithinLimit(fileSizeBytes(archive), MAX_ZIP_TOTAL_BYTES)) { \"Upload ZIP exceeds byte limit\" }" - " val connection = openPinnedConnection(" - " \"load.zip\"," - " \"POST\"," - " outputBytes = fileSizeBytes(archive)," - " contentType = \"application/zip\"" - " )" - " try {" - " connection.outputStream.use { output ->" - " archive.inputStream().use { input -> input.copyTo(output, 32 * 1024) }" - " }" - " val code = connection.responseCode" - " if (httpStatusFailed(code)) {" - " closeInputStreamIfPresent(connection.errorStream)" - " throw IllegalStateException(httpStatusMessage(code))" - " }" - " val response = readResponseText(connection)" - " return remoteDumpImportedCount(jsonObjectFromText(response))" - " } finally {" - " connection.disconnect()" - " }" - " } finally {" - " archive.delete()" - " }" - " }" + " fun pushRemoteDump(): Int =" + " truthStorePushRemoteDumpLocal(this)" "" " fun storageSummary(): String =" " truthStoreStorageSummaryLocal(this)"