Generate SSD stream loops from typed Jerboa
ober
525a0ab8cddad59c7e9b289e6c2bfe385d39dd84
--- a/.build.yml +++ b/.build.yml @@ -5,7 +5,7 @@ packages: - make=4.4.1-r4 sources: # Build dependency: full immutable commit, mirrored in dependencies.lock.json. - - "https://git.sr.ht/~lisp/jerboa#9fc12d13fd151e299d3694922aa9fac9a272ecaa" + - "https://git.sr.ht/~lisp/jerboa#fc3264c8101a7892d13b11b8c223946df95adaf5" # The second source is the build subject selected by the SourceHut submitter. - https://git.sr.ht/~lisp/jerboa-android tasks: @@ -14,6 +14,6 @@ tasks: test "$(apk info -v chez-scheme)" = chez-scheme-10.3.0-r2 test "$(apk info -v git)" = git-2.54.0-r0 test "$(apk info -v make)" = make-4.4.1-r4 - test "$(git -C ../jerboa rev-parse HEAD)" = 9fc12d13fd151e299d3694922aa9fac9a272ecaa - test "$(git -C ../jerboa rev-parse 'HEAD^{tree}')" = b41f749819c50653fe7a7c0254a5efa80fd44d80 + test "$(git -C ../jerboa rev-parse HEAD)" = fc3264c8101a7892d13b11b8c223946df95adaf5 + test "$(git -C ../jerboa rev-parse 'HEAD^{tree}')" = 43991ea0c37ca239fb5742ecf88616295481f5e3 JERBOA="chez --libdirs .:../jerboa/lib --script" make test --- a/dependencies.lock.json +++ b/dependencies.lock.json @@ -11,8 +11,8 @@ "generator_runtime": { "name": "jerboa", "repository": "https://git.sr.ht/~lisp/jerboa", - "commit": "9fc12d13fd151e299d3694922aa9fac9a272ecaa", - "tree": "b41f749819c50653fe7a7c0254a5efa80fd44d80" + "commit": "fc3264c8101a7892d13b11b8c223946df95adaf5", + "tree": "43991ea0c37ca239fb5742ecf88616295481f5e3" }, "assurance_tools": { "osv_scanner": { --- a/scripts/verify-supply-chain.sh +++ b/scripts/verify-supply-chain.sh @@ -3,8 +3,8 @@ set -eu repo=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd -P) lock="$repo/dependencies.lock.json" -jerboa_commit=9fc12d13fd151e299d3694922aa9fac9a272ecaa # gitsafe:ignore -jerboa_tree=b41f749819c50653fe7a7c0254a5efa80fd44d80 # gitsafe:ignore +jerboa_commit=fc3264c8101a7892d13b11b8c223946df95adaf5 # gitsafe:ignore +jerboa_tree=43991ea0c37ca239fb5742ecf88616295481f5e3 # gitsafe:ignore gradle_sha=20f1b1176237254a6fc204d8434196fa11a4cfb387567519c61556e8710aed78 jdk_macos_sha=8fa1eff40bb637a33613b2ccb8b12c70dc3661cc22cf8e784943715769a05336 jdk_linux_sha=d8afc263758141a66e0e3aafc321e783f7016696f4eaea067d340a269037d331 --- a/templates/ssd-review.ss +++ b/templates/ssd-review.ss @@ -9567,6 +9567,163 @@ (def (sdkBefore (sdk : Int32) (api : Int32)) : Bool (< sdk api)))) + (typed-kotlin-file "com/sfb/ssdreview/TruthStoreStreams.kt" + (kotlin-imports (java io ByteArrayOutputStream) + (java io InputStream) + (java io OutputStream) + (java security MessageDigest) + (java util zip ZipEntry) + (java util zip ZipInputStream)) + (typed-library (com sfb ssdreview) + (export readBoundedBytes sha1Hex forEachZipEntry streamZipEntryToOutput) + (type ByteArrayOutputStream) + (type Exception) + (type InputStream) + (type MessageDigest) + (type OutputStream) + (type ZipEntry) + (type ZipInputStream) + (type Int32) + (extern (byteArrayOutputStream (capacity : Int32)) : ByteArrayOutputStream + (kotlin-call ByteArrayOutputStream)) + (extern (byteArrayOutputStreamWrite + (output : ByteArrayOutputStream) + (buffer : Bytes) + (offset : Int32) + (length : Int32)) : Unit + (kotlin-member-call write)) + (extern (byteArrayOutputStreamToByteArray + (output : ByteArrayOutputStream)) : Bytes + (kotlin-member-call toByteArray)) + (extern (inputStreamRead + (input : InputStream) + (buffer : Bytes)) : Int32 + (kotlin-member-call read)) + (extern (outputStreamWrite + (output : OutputStream) + (buffer : Bytes) + (offset : Int32) + (length : Int32)) : Unit + (kotlin-member-call write)) + (extern (zipInputStreamRead + (zip : ZipInputStream) + (buffer : Bytes)) : Int32 + (kotlin-member-call read)) + (extern (zipInputStreamNextEntry + (zip : ZipInputStream)) : (Nullable ZipEntry) + (kotlin-member-get nextEntry)) + (extern (zipInputStreamCloseEntry (zip : ZipInputStream)) : Unit + (kotlin-member-call closeEntry)) + (extern (messageDigestGetInstanceForStreams + (algorithm : String)) : MessageDigest + (kotlin-call MessageDigest getInstance)) + (extern (messageDigestUpdateBytes + (digest : MessageDigest) + (buffer : Bytes) + (offset : Int32) + (length : Int32)) : Unit + (kotlin-member-call update)) + (extern (messageDigestDigestBytes + (digest : MessageDigest)) : Bytes + (kotlin-member-call digest)) + (extern (illegalArgumentException + (message : String)) : Exception + (kotlin-call IllegalArgumentException)) + (extern (illegalStateException + (message : String)) : Exception + (kotlin-call IllegalStateException)) + (def (readBoundedBytes (input : InputStream) (limit : Int)) : Bytes + (if (boundedReadLimitValid limit) + (let ((output + (byteArrayOutputStream + (int32 (boundedReadInitialCapacity limit)))) + (buffer (make-bytevector (int32 32768) 0))) + (var ((total (int 0))) + (begin + (while #t + (let ((read (inputStreamRead input buffer))) + (begin + (if (streamReadEnded read) (break) (begin)) + (if (streamReadEmpty read) (continue) (begin)) + (set! total (+ total (int read))) + (if (inputCountExceeded total limit) + (throw + (illegalStateException + (inputLimitExceededMessage limit)) + Unit) + (begin)) + (byteArrayOutputStreamWrite + output + buffer + (int32 0) + read)))) + (byteArrayOutputStreamToByteArray output)))) + (throw (illegalArgumentException "Invalid bounded read limit") Bytes))) + (def (sha1Hex (input : InputStream)) : String + (let ((digest (messageDigestGetInstanceForStreams "SHA-1")) + (buffer (make-bytevector (int32 65536) 0))) + (begin + (while #t + (let ((read (inputStreamRead input buffer))) + (begin + (if (streamReadEnded read) (break) (begin)) + (messageDigestUpdateBytes digest buffer (int32 0) read)))) + (bytesToLowerHex (messageDigestDigestBytes digest))))) + (def (forEachZipEntry + (zip : ZipInputStream) + (action : (-> ZipEntry Unit))) : Unit + (while #t + (let ((nextEntry (zipInputStreamNextEntry zip))) + (if (zipEntryPresent nextEntry) + (begin + (invoke action (nullable-get nextEntry)) + (zipInputStreamCloseEntry zip)) + (break))))) + (def (streamZipEntryToOutput + (zip : ZipInputStream) + (output : OutputStream) + (budget : ImportBudget) + (maxEntryBytes : Int) + (maxTotalBytes : Int) + (ratioFloorBytes : Int) + (maxCompressionRatio : Int) + (compressedBytes : (-> Int))) : Unit + (let ((buffer (make-bytevector (int32 32768) 0))) + (var ((entryBytes (int 0))) + (while #t + (let ((read (zipInputStreamRead zip buffer))) + (begin + (if (streamReadEnded read) (break) (begin)) + (if (streamReadEmpty read) (continue) (begin)) + (set! entryBytes (+ entryBytes (int read))) + (ImportBudget-expandedBytes-set! + budget + (+ (ImportBudget-expandedBytes budget) (int read))) + (if (zipExpandedBytesExceeded + entryBytes + (ImportBudget-expandedBytes budget) + maxEntryBytes + maxTotalBytes) + (throw + (illegalStateException + "ZIP expanded-byte limit exceeded") + Unit) + (begin)) + (let ((ratioBudget + (compressionRatioBudget + ratioFloorBytes + (invoke compressedBytes) + maxCompressionRatio))) + (if (zipCompressionRatioExceeded + (ImportBudget-expandedBytes budget) + ratioBudget) + (throw + (illegalStateException + "ZIP compression-ratio limit exceeded") + Unit) + (begin))) + (outputStreamWrite output buffer (int32 0) read))))))))) + (typed-kotlin-file "com/sfb/ssdreview/RemoteTruthQuery.kt" (typed-library (com sfb ssdreview) (export remoteTruthQuery) @@ -11859,22 +12016,6 @@ " }" " }" "" - " private fun readBoundedBytes(input: InputStream, limit: Long): ByteArray {" - " require(boundedReadLimitValid(limit))" - " val output = ByteArrayOutputStream(boundedReadInitialCapacity(limit).toInt())" - " val buffer = ByteArray(32 * 1024)" - " var total = 0L" - " while (true) {" - " val read = input.read(buffer)" - " if (streamReadEnded(read)) break" - " if (streamReadEmpty(read)) continue" - " total += read.toLong()" - " if (inputCountExceeded(total, limit)) throw IllegalStateException(inputLimitExceededMessage(limit))" - " output.write(buffer, 0, read)" - " }" - " return output.toByteArray()" - " }" - "" " private fun sha256Hex(bytes: ByteArray): String =" " bytesToLowerHex(MessageDigest.getInstance(\"SHA-256\").digest(bytes))" "" @@ -12155,10 +12296,7 @@ " try {" " val compressed = LimitedInputStream(input, MAX_REMOTE_ZIP_BYTES)" " ZipInputStream(BufferedInputStream(compressed)).use { zip ->" - " while (true) {" - " val nextEntry = zip.nextEntry" - " if (!zipEntryPresent(nextEntry)) break" - " val entry = checkNotNull(nextEntry)" + " forEachZipEntry(zip) { entry ->" " budget.entries += 1" " if (zipEntryCountExceeded(budget.entries, MAX_ZIP_ENTRIES)) throw IllegalStateException(zipEntryCountLimitExceededMessage(budget.entries, MAX_ZIP_ENTRIES))" " val name = entry.name" @@ -12174,7 +12312,6 @@ " validateImportedFile(name, stagedFile)" " staged.add(StagedZipEntry(name, stagedFile, entry.time))" " }" - " zip.closeEntry()" " }" " }" " val current = storageStats()" @@ -12196,24 +12333,16 @@ " compressed: LimitedInputStream," " budget: ImportBudget" " ) {" - " val buffer = ByteArray(32 * 1024)" - " var entryBytes = 0L" " destination.outputStream().use { output ->" - " while (true) {" - " val read = zip.read(buffer)" - " if (streamReadEnded(read)) break" - " if (streamReadEmpty(read)) continue" - " entryBytes += read.toLong()" - " budget.expandedBytes += read.toLong()" - " if (zipExpandedBytesExceeded(entryBytes, budget.expandedBytes, MAX_ZIP_ENTRY_BYTES, MAX_ZIP_TOTAL_BYTES)) {" - " throw IllegalStateException(\"ZIP expanded-byte limit exceeded\")" - " }" - " val ratioBudget = compressionRatioBudget(RATIO_FLOOR_BYTES, compressed.count, MAX_COMPRESSION_RATIO)" - " if (zipCompressionRatioExceeded(budget.expandedBytes, ratioBudget)) {" - " throw IllegalStateException(\"ZIP compression-ratio limit exceeded\")" - " }" - " output.write(buffer, 0, read)" - " }" + " streamZipEntryToOutput(" + " zip," + " output," + " budget," + " MAX_ZIP_ENTRY_BYTES," + " MAX_ZIP_TOTAL_BYTES," + " RATIO_FLOOR_BYTES," + " MAX_COMPRESSION_RATIO" + " ) { compressed.count }" " }" " }" "" @@ -12556,16 +12685,5 @@ "" "}" "" - "fun sha1Hex(input: InputStream): String {" - " val digest = MessageDigest.getInstance(\"SHA-1\")" - " val buffer = ByteArray(1024 * 64)" - " while (true) {" - " val n = input.read(buffer)" - " if (streamReadEnded(n)) break" - " digest.update(buffer, 0, n)" - " }" - " return bytesToLowerHex(digest.digest())" - "}" - "" )) ))