Keep SSD learned rebuild from blocking saves
ober
2fd2645a628a1788e336667799f557e89f0ae5a6
--- a/templates/ssd-review.ss +++ b/templates/ssd-review.ss @@ -3658,7 +3658,7 @@ " private const val RATIO_FLOOR_BYTES = 64L * 1024L" " private const val MAX_STORAGE_BYTES = 128L * 1024L * 1024L" " private const val MAX_STORAGE_FILES = 16384" - " private const val MAX_LEARNED_EXAMPLES = 10000" + " private const val MAX_LEARNED_EXAMPLES = 50000" " private const val MAX_JSONL_LINES = 51200" " private const val MAX_SOURCE_KEY_CHARS = 96" " private const val MAX_AUTH_TOKEN_CHARS = 4096" @@ -4451,31 +4451,39 @@ "" " private fun rebuildLearnedExamples() {" " val examples = JSONArray()" - " groundTruthDir.listFiles { f -> f.name.endsWith(\".truth.json\") }" - " ?.sortedBy { it.name }" - " ?.forEach { file ->" - " val truth = JSONObject(readLocalText(file))" - " val groups = truth.optJSONArray(\"groups\") ?: JSONArray()" - " for (i in 0 until groups.length()) {" - " if (examples.length() >= MAX_LEARNED_EXAMPLES) {" - " throw IllegalStateException(\"Learned-example limit exceeded\")" - " }" - " val group = groups.getJSONObject(i)" - " examples.put(JSONObject()" - " .put(\"source_key\", truth.optString(\"source_key\"))" - " .put(\"truth_path\", file.name)" - " .put(\"label\", group.optString(\"label\"))" - " .put(\"box_type_id\", group.optString(\"box_type_id\"))" - " .put(\"firing_arc\", group.optString(\"firing_arc\", group.optString(\"arc\")))" - " .put(\"count\", group.optInt(\"count\"))" - " .put(\"bbox\", group.optJSONArray(\"bbox\") ?: JSONArray())" - " .put(\"features\", groupFeatures(group, imageWidth(truth), imageHeight(truth))))" + " val files = groundTruthDir.listFiles { f -> f.name.endsWith(\".truth.json\") }" + " ?.sortedWith(compareByDescending<File> { it.lastModified() }.thenBy { it.name })" + " ?: emptyList()" + " fileLoop@ for (file in files) {" + " val truth = try {" + " JSONObject(readLocalText(file))" + " } catch (_: Exception) {" + " continue" + " }" + " val groups = truth.optJSONArray(\"groups\") ?: JSONArray()" + " for (i in 0 until groups.length()) {" + " if (examples.length() >= MAX_LEARNED_EXAMPLES) break@fileLoop" + " val group = try {" + " groups.getJSONObject(i)" + " } catch (_: Exception) {" + " continue" " }" + " examples.put(JSONObject()" + " .put(\"source_key\", truth.optString(\"source_key\"))" + " .put(\"truth_path\", file.name)" + " .put(\"label\", group.optString(\"label\"))" + " .put(\"box_type_id\", group.optString(\"box_type_id\"))" + " .put(\"firing_arc\", group.optString(\"firing_arc\", group.optString(\"arc\")))" + " .put(\"count\", group.optInt(\"count\"))" + " .put(\"bbox\", group.optJSONArray(\"bbox\") ?: JSONArray())" + " .put(\"features\", groupFeatures(group, imageWidth(truth), imageHeight(truth))))" " }" + " }" " val learned = JSONObject()" " .put(\"schema_version\", 1)" " .put(\"generated_at\", System.currentTimeMillis())" " .put(\"example_count\", examples.length())" + " .put(\"capped\", examples.length() >= MAX_LEARNED_EXAMPLES)" " .put(\"examples\", examples)" " atomicWriteText(containedLeaf(learnedDir, \"group_examples.json\"), learned.toString(2))" " }"