Generate SSD learned example rebuilds from typed Jerboa
ober
cdf504e9151768d6d84f72083a917ce83091ac43
--- a/templates/ssd-review.ss +++ b/templates/ssd-review.ss @@ -15596,6 +15596,7 @@ truthStoreMergeJsonLinesLocal truthStorePushRemoteDumpLocal truthStoreStreamZipEntryLocal + truthStoreRebuildLearnedExamplesLocal truthStoreInstallStagedZipEntryLocal truthStorePullRemoteDumpLocal) (type AssetManager) @@ -15883,6 +15884,8 @@ (kotlin-value MAX_STORAGE_BYTES)) (extern (truthStoreMaxStorageFiles) : Int32 (kotlin-value MAX_STORAGE_FILES)) + (extern (truthStoreMaxLearnedExamples) : Int32 + (kotlin-value MAX_LEARNED_EXAMPLES)) (extern (truthStoreMaxJsonlLines) : Int32 (kotlin-value MAX_JSONL_LINES)) (extern (truthStoreBase64NoWrap) : Int32 @@ -16636,6 +16639,113 @@ (truthStoreMaxCompressionRatio) (lambda () (truthStoreLimitedInputStreamCount compressed))) (truthStoreOutputStreamClose output)))) + (def (truthStoreLearnedFileBefore? + (candidate : File) + (existing : File)) : Bool + (let ((candidateModified (truthStoreFileLastModified candidate)) + (existingModified (truthStoreFileLastModified existing))) + (if (> candidateModified existingModified) + #t + (if (< candidateModified existingModified) + #f + (< (string-compare-to + (truthStoreFileName candidate) + (truthStoreFileName existing)) + (int32 0)))))) + (def (truthStoreLearnedInsertIndex + (sorted : (MutableList File)) + (file : File)) : Int32 + (for/fold ((found (int32 -1))) + ((i (in-range (int32 0) (list-size sorted)))) + (if (>= found (int32 0)) + found + (if (truthStoreLearnedFileBefore? file (list-ref sorted i)) + i + found)))) + (def (truthStoreLearnedInsertSorted + (sorted : (MutableList File)) + (file : File)) : (MutableList File) + (let ((insertAt (truthStoreLearnedInsertIndex sorted file))) + (if (< insertAt (int32 0)) + (begin + (mutable-list-add! sorted file) + sorted) + (let ((out (mutable-list-empty File))) + (begin + (for/fold ((ignored (int32 0))) + ((i (in-range (int32 0) (list-size sorted)))) + (begin + (if (= i insertAt) + (begin + (mutable-list-add! out file) + (mutable-list-add! out (list-ref sorted i))) + (mutable-list-add! out (list-ref sorted i))) + ignored)) + out))))) + (def (truthStoreSortedGroundTruthFiles + (store : TruthStore)) : (MutableList File) + (let ((files + (truthStoreListFiles + (truthStoreGroundTruthDir store)))) + (if (nullable-null? files) + (mutable-list-empty File) + (for/fold ((sorted (mutable-list-empty File))) + ((i (in-range + (int32 0) + (list-size (nullable-get files))))) + (let ((file (list-ref (nullable-get files) i))) + (if (truthJsonFileName (truthStoreFileName file)) + (truthStoreLearnedInsertSorted sorted file) + sorted)))))) + (def (truthStoreTruthFromLocalFile + (store : TruthStore) + (file : File)) : (Nullable JSONObject) + (try + (nullable-some + (jsonObjectFromText + (truthStoreReadLocalText file))) + (catch (error : Exception) + (nullable-none JSONObject)))) + (def (truthStoreAppendLearnedFromFile + (store : TruthStore) + (examples : JSONArray) + (file : File)) : Bool + (let ((truth + (truthStoreTruthFromLocalFile store file))) + (begin + (if (truthPresent truth) + (appendLearnedExamplesFromTruth + examples + (truthStoreMaxLearnedExamples) + (truthStoreFileName file) + (truthJsonObjectOrEmpty truth)) + (begin)) + (not + (learnedExamplesFull + (learnedExamplesCount examples) + (truthStoreMaxLearnedExamples)))))) + (def (truthStoreRebuildLearnedExamplesLocal + (store : TruthStore)) : Unit + (let ((examples (learnedExamplesEmpty)) + (files (truthStoreSortedGroundTruthFiles store))) + (begin + (forEachFileUntil + files + (lambda ((file : File)) + (truthStoreAppendLearnedFromFile + store + examples + file))) + (let ((learned + (learnedExamplesDocument + examples + (truthStoreMaxLearnedExamples)))) + (truthStoreAtomicWriteTextRaw + store + (truthStoreContainedLeaf + (truthStoreLearnedDir store) + (learnedExamplesFileName)) + (truthStoreJsonToPrettyString learned (int32 2))))))) (def (truthStoreReadStagedZipEntryBytesLocal (entry : StagedZipEntry)) : Bytes (let ((input @@ -17122,28 +17232,8 @@ " truthStoreSeedBundledTruthAsync(this, done)" " }" "" - " internal fun rebuildLearnedExamples() {" - " val examples = learnedExamplesEmpty()" - " val files = if (fileListFilesPresent(groundTruthDir)) {" - " checkNotNull(groundTruthDir.listFiles { f -> truthJsonFileName(f.name) })" - " .sortedWith(compareByDescending<File> { it.lastModified() }.thenBy { it.name })" - " } else {" - " emptyList()" - " }" - " forEachFileUntil(files) { file ->" - " val truth = try {" - " jsonObjectFromText(readLocalText(file))" - " } catch (_: Exception) {" - " null" - " }" - " if (truthPresent(truth)) {" - " appendLearnedExamplesFromTruth(examples, MAX_LEARNED_EXAMPLES, file.name, truthJsonObjectOrEmpty(truth))" - " }" - " !learnedExamplesFull(learnedExamplesCount(examples), MAX_LEARNED_EXAMPLES)" - " }" - " val learned = learnedExamplesDocument(examples, MAX_LEARNED_EXAMPLES)" - " atomicWriteText(containedLeaf(learnedDir, learnedExamplesFileName()), learned.toString(2))" - " }" + " internal fun rebuildLearnedExamples() =" + " truthStoreRebuildLearnedExamplesLocal(this)" "" " internal fun seedBundledTruth(): Int =" " truthStoreSeedBundledTruthLocal(this, context)"