Speed up SSD truth loading and restore page state
ober
d0ed0023847caf35eb3699e4a720425b749ef5dc
--- a/templates/ssd-review.ss +++ b/templates/ssd-review.ss @@ -344,6 +344,7 @@ " truthStore = TruthStore(this)" " setContentView(buildUi())" " setStatus(\"Open an SSD PDF. ${truthStore.storageSummary()}\")" + " restoreLastPdf()" " truthStore.seedBundledTruthAsync { imported ->" " if (imported > 0) {" " runOnUiThread {" @@ -361,6 +362,11 @@ " }" " }" "" + " override fun onPause() {" + " super.onPause()" + " persistCurrentPdfState(includeViewport = true)" + " }" + "" " private fun buildUi(): View {" " val root = LinearLayout(this).apply {" " orientation = LinearLayout.VERTICAL" @@ -659,6 +665,7 @@ " private fun changePage(delta: Int) {" " if (pdfUri == null || pageCount <= 0) return" " pageIndex = (pageIndex + delta).coerceIn(0, pageCount - 1)" + " persistCurrentPdfState(includeViewport = false)" " renderCurrentPage()" " }" "" @@ -837,6 +844,7 @@ " val renderPdfName = pdfName" " val renderPdfSha1 = pdfSha1" " val renderPageCount = pageCount" + " val restoredViewport = restoredViewportFor(uri, renderPageIndex)" " thread {" " try {" " val bitmap = renderPdfPage(uri, renderPageIndex)" @@ -849,6 +857,7 @@ " reviewView.session = null" " reviewView.selectedGroupId = null" " reviewView.ssdArea = null" + " restoreViewportAfterLayout(restoredViewport)" " setStatus(\"Rendered $renderPdfName page ${renderPageIndex + 1}/$renderPageCount; loading truth\")" " detectCurrent(" " expectedGeneration = renderGeneration," @@ -917,7 +926,7 @@ " return@thread" " }" "" - " val remoteMatches = truthStore.pullRemoteTruth(session)" + " val remoteMatches = 0" " val existingTruth = truthStore.loadTruth(session)" " var ocrGuesses = 0" " var detectorNote = \"\"" @@ -954,14 +963,11 @@ " reviewView.session = session" " reviewView.selectedGroupId = null" " reviewView.ssdArea = session.ssdArea" - " val truthNote = if (existingTruth != null) {" - " if (remoteMatches > 0) \", pulled remote truth\" else \", replayed truth\"" - " } else {" - " if (remoteMatches > 0) \", remote truth unavailable for this render\" else \"\"" - " }" + " val truthNote = if (existingTruth != null) \", replayed truth\" else \"\"" " val ocrNote = if (ocrGuesses > 0) \", $ocrGuesses OCR guesses\" else if (session.ocrWords.isNotEmpty()) \", OCR ${session.ocrWords.size} words\" else \"\"" " val learnedNote = if (learnedGuesses > 0) \", $learnedGuesses learned guesses\" else \"\"" " setStatus(\"${session.cells.size} cells, ${session.groups.size} groups$detectorNote$truthNote$ocrNote$learnedNote\")" + " if (existingTruth == null && !forceDetect) mergeRemoteTruthAsync(session, expectedGeneration, uri, expectedPageIndex)" " }" " } catch (error: Exception) {" " runOnUiThread {" @@ -973,6 +979,33 @@ " }" " }" "" + " private fun mergeRemoteTruthAsync(session: SsdSession, generation: Int, uri: Uri, page: Int) {" + " thread {" + " val matches = truthStore.pullRemoteTruth(session)" + " if (matches <= 0) return@thread" + " val truth = truthStore.loadTruth(session) ?: return@thread" + " val merged = session.copy(" + " cells = mutableListOf()," + " groups = mutableListOf()," + " suppressedGroups = mutableListOf()," + " ocrWords = session.ocrWords.toMutableList()" + " )" + " merged.ssdArea = jsonRect(truth.optJSONArray(\"ssd_area\"))" + " truthStore.applyTruth(merged)" + " merged.pruneGroupsOutsideSsdArea()" + " runOnUiThread {" + " if (!isCurrentLoad(generation, uri, page)) return@runOnUiThread" + " currentSession = merged" + " selectedGroupId = null" + " reviewView.session = merged" + " reviewView.selectedGroupId = null" + " reviewView.ssdArea = merged.ssdArea" + " reviewView.invalidate()" + " setStatus(\"${merged.cells.size} cells, ${merged.groups.size} groups, pulled remote truth\")" + " }" + " }" + " }" + "" " private fun isCurrentLoad(generation: Int, uri: Uri, page: Int): Boolean =" " generation == loadGeneration && pdfUri == uri && pageIndex == page" "" @@ -1331,13 +1364,13 @@ " }" "" " private fun pullTruthDump() {" - " setStatus(\"Pulling full truth dump from service...\")" + " setStatus(\"Pulling app truth and learned model from service...\")" " thread {" " try {" " val imported = truthStore.pullRemoteDump()" " runOnUiThread {" " Toast.makeText(this, \"Truth pull complete\", Toast.LENGTH_SHORT).show()" - " setStatus(\"Pulled full truth: imported $imported files; ${truthStore.storageSummary()}\")" + " setStatus(\"Pulled app truth: imported $imported files; ${truthStore.storageSummary()}\")" " if (pdfUri != null) renderCurrentPage()" " }" " } catch (error: Exception) {" @@ -1611,7 +1644,7 @@ " }" " }" "" - " private fun loadPdf(uri: Uri, flags: Int) {" + " private fun loadPdf(uri: Uri, flags: Int, initialPageIndex: Int = 0) {" " try {" " contentResolver.takePersistableUriPermission(uri, flags and Intent.FLAG_GRANT_READ_URI_PERMISSION)" " } catch (_: Exception) {" @@ -1630,7 +1663,8 @@ " runOnUiThread {" " pdfSha1 = hash" " pageCount = count" - " pageIndex = 0" + " pageIndex = initialPageIndex.coerceIn(0, maxOf(0, count - 1))" + " persistCurrentPdfState(includeViewport = false)" " setStatus(\"Loaded $pdfName ($pageCount pages)\")" " renderCurrentPage()" " }" @@ -1640,6 +1674,62 @@ " }" " }" "" + " private fun restoreLastPdf(): Boolean {" + " val prefs = getPreferences(MODE_PRIVATE)" + " val raw = prefs.getString(PREF_LAST_PDF_URI, null) ?: return false" + " val page = prefs.getInt(PREF_LAST_PAGE_INDEX, 0)" + " return try {" + " loadPdf(Uri.parse(raw), Intent.FLAG_GRANT_READ_URI_PERMISSION, page)" + " true" + " } catch (_: Exception) {" + " false" + " }" + " }" + "" + " private fun persistCurrentPdfState(includeViewport: Boolean) {" + " val uri = pdfUri ?: return" + " val edit = getPreferences(MODE_PRIVATE).edit()" + " .putString(PREF_LAST_PDF_URI, uri.toString())" + " .putString(PREF_LAST_PDF_NAME, pdfName)" + " .putString(PREF_LAST_PDF_SHA1, pdfSha1)" + " .putInt(PREF_LAST_PAGE_INDEX, pageIndex)" + " .putInt(PREF_LAST_PAGE_COUNT, pageCount)" + " val viewport = if (includeViewport) reviewView.captureViewport() else null" + " if (viewport != null) {" + " edit.putInt(PREF_LAST_VIEWPORT_PAGE_INDEX, pageIndex)" + " .putFloat(PREF_LAST_VIEWPORT_SCALE, viewport.scale)" + " .putFloat(PREF_LAST_VIEWPORT_OFFSET_X, viewport.offsetX)" + " .putFloat(PREF_LAST_VIEWPORT_OFFSET_Y, viewport.offsetY)" + " .putInt(PREF_LAST_VIEWPORT_ROTATION, viewport.rotationDegrees)" + " .putInt(PREF_LAST_VIEWPORT_IMAGE_WIDTH, viewport.imageWidth)" + " .putInt(PREF_LAST_VIEWPORT_IMAGE_HEIGHT, viewport.imageHeight)" + " } else {" + " edit.remove(PREF_LAST_VIEWPORT_PAGE_INDEX)" + " .remove(PREF_LAST_VIEWPORT_SCALE)" + " .remove(PREF_LAST_VIEWPORT_OFFSET_X)" + " .remove(PREF_LAST_VIEWPORT_OFFSET_Y)" + " .remove(PREF_LAST_VIEWPORT_ROTATION)" + " .remove(PREF_LAST_VIEWPORT_IMAGE_WIDTH)" + " .remove(PREF_LAST_VIEWPORT_IMAGE_HEIGHT)" + " }" + " edit.apply()" + " }" + "" + " private fun restoredViewportFor(uri: Uri, page: Int): SsdReviewView.ViewportState? {" + " val prefs = getPreferences(MODE_PRIVATE)" + " if (prefs.getString(PREF_LAST_PDF_URI, null) != uri.toString()) return null" + " if (prefs.getInt(PREF_LAST_VIEWPORT_PAGE_INDEX, -1) != page) return null" + " if (!prefs.contains(PREF_LAST_VIEWPORT_SCALE)) return null" + " return SsdReviewView.ViewportState(" + " prefs.getFloat(PREF_LAST_VIEWPORT_SCALE, 1f)," + " prefs.getFloat(PREF_LAST_VIEWPORT_OFFSET_X, 0f)," + " prefs.getFloat(PREF_LAST_VIEWPORT_OFFSET_Y, 0f)," + " prefs.getInt(PREF_LAST_VIEWPORT_ROTATION, 0)," + " prefs.getInt(PREF_LAST_VIEWPORT_IMAGE_WIDTH, 0)," + " prefs.getInt(PREF_LAST_VIEWPORT_IMAGE_HEIGHT, 0)" + " )" + " }" + "" " private fun pdfPageCount(uri: Uri): Int {" " openPdfRenderer(uri).use { renderer -> return renderer.pageCount }" " }" @@ -1790,6 +1880,17 @@ " private const val REQ_READ_STORAGE = 60" " private const val PREF_SSD_PDF_TREE_URI = \"ssd_pdf_tree_uri\"" " private const val PREF_LAST_PDF_URI = \"last_pdf_uri\"" + " private const val PREF_LAST_PDF_NAME = \"last_pdf_name\"" + " private const val PREF_LAST_PDF_SHA1 = \"last_pdf_sha1\"" + " private const val PREF_LAST_PAGE_INDEX = \"last_page_index\"" + " private const val PREF_LAST_PAGE_COUNT = \"last_page_count\"" + " private const val PREF_LAST_VIEWPORT_PAGE_INDEX = \"last_viewport_page_index\"" + " private const val PREF_LAST_VIEWPORT_SCALE = \"last_viewport_scale\"" + " private const val PREF_LAST_VIEWPORT_OFFSET_X = \"last_viewport_offset_x\"" + " private const val PREF_LAST_VIEWPORT_OFFSET_Y = \"last_viewport_offset_y\"" + " private const val PREF_LAST_VIEWPORT_ROTATION = \"last_viewport_rotation\"" + " private const val PREF_LAST_VIEWPORT_IMAGE_WIDTH = \"last_viewport_image_width\"" + " private const val PREF_LAST_VIEWPORT_IMAGE_HEIGHT = \"last_viewport_image_height\"" " private const val MAX_SSD_PDF_SCAN_DEPTH = 8" " private const val MAX_SSD_PDF_CHOICES = 1000" " private const val FAST_TRUTH_GROUP_THRESHOLD = 5" @@ -3363,6 +3464,17 @@ "" " private data class StagedZipEntry(val name: String, val file: File, val modified: Long)" "" + " private data class TruthIndexEntry(" + " val file: File," + " val sourceKey: String," + " val sourceName: String," + " val canonicalSourceName: String," + " val page: Int," + " val dpi: Int," + " val groupCount: Int," + " val truthTime: Long" + " )" + "" " private val root = File(context.filesDir, \"ssd_review\")" " private val groundTruthDir = File(root, \"ground_truth\")" " private val learnedDir = File(root, \"learned\")" @@ -3370,6 +3482,7 @@ " private val reviewsDir = File(root, \"reviews\")" " private val remoteConfig: RemoteConfig? by lazy { loadRemoteConfig() }" " @Volatile private var seedStarted = false" + " @Volatile private var truthIndexCache: List<TruthIndexEntry>? = null" "" " init {" " listOf(root, groundTruthDir, learnedDir, eventsDir, reviewsDir).forEach { it.mkdirs() }" @@ -3388,35 +3501,65 @@ " }" "" " fun loadTruth(session: SsdSession): JSONObject? {" - " var best = loadTruth(session.sourceKey)" + " var best = truthIndexEntry(truthPath(session.sourceKey))" " val wantedName = sourceBaseName(session.sourceName)" " val wantedCanonicalName = canonicalSourceName(session.sourceName)" - " groundTruthDir.listFiles { f -> f.name.endsWith(\".truth.json\") }" - " ?.sortedBy { it.name }" - " ?.forEach { file ->" - " val truth = JSONObject(readLocalText(file))" - " val source = truth.optJSONObject(\"source\") ?: JSONObject()" - " val name = sourceBaseName(source.optString(\"name\", source.optString(\"path\")))" - " val canonicalName = canonicalSourceName(name)" - " val page = source.optInt(\"page\", -1)" - " val dpi = source.optInt(\"dpi\", -1)" + " truthIndexEntries().forEach { entry ->" " if (" - " (name == wantedName || canonicalName == wantedCanonicalName) &&" - " page == session.page &&" - " dpi == session.dpi" + " (entry.sourceName == wantedName || entry.canonicalSourceName == wantedCanonicalName) &&" + " entry.page == session.page &&" + " entry.dpi == session.dpi" " ) {" " val current = best" - " if (current == null ||" - " truthGroupCount(truth) > truthGroupCount(current) ||" - " (truthGroupCount(truth) == truthGroupCount(current) &&" - " (truthTime(truth) ?: 0L) > (truthTime(current) ?: 0L))) {" - " best = truth" + " if (current == null || betterTruthIndex(entry, current)) {" + " best = entry" " }" " }" - " }" - " return best" + " }" + " return best?.let { JSONObject(readLocalText(it.file)) }" + " }" + "" + " @Synchronized" + " private fun invalidateTruthIndex() {" + " truthIndexCache = null" " }" "" + " @Synchronized" + " private fun truthIndexEntries(): List<TruthIndexEntry> {" + " truthIndexCache?.let { return it }" + " val entries = groundTruthDir.listFiles { f -> f.name.endsWith(\".truth.json\") }" + " ?.sortedBy { it.name }" + " ?.mapNotNull { truthIndexEntry(it) }" + " ?: emptyList()" + " truthIndexCache = entries" + " return entries" + " }" + "" + " private fun truthIndexEntry(file: File): TruthIndexEntry? {" + " if (!file.exists() || !file.isFile) return null" + " return try {" + " val truth = JSONObject(readLocalText(file))" + " val source = truth.optJSONObject(\"source\") ?: JSONObject()" + " val sourceName = sourceBaseName(source.optString(\"name\", source.optString(\"path\")))" + " TruthIndexEntry(" + " file = file," + " sourceKey = truth.optString(\"source_key\")," + " sourceName = sourceName," + " canonicalSourceName = canonicalSourceName(sourceName)," + " page = source.optInt(\"page\", -1)," + " dpi = source.optInt(\"dpi\", -1)," + " groupCount = truthGroupCount(truth)," + " truthTime = truthTime(truth) ?: file.lastModified()" + " )" + " } catch (_: Exception) {" + " null" + " }" + " }" + "" + " private fun betterTruthIndex(candidate: TruthIndexEntry, current: TruthIndexEntry): Boolean =" + " candidate.groupCount > current.groupCount ||" + " (candidate.groupCount == current.groupCount && candidate.truthTime > current.truthTime)" + "" " fun saveSession(session: SsdSession): File =" " saveTruthSnapshot(session.toTruthJson())" "" @@ -3426,6 +3569,7 @@ " ?: throw IllegalArgumentException(\"Invalid source key\")" " val truthFile = truthPath(sourceKey)" " atomicWriteText(truthFile, truth.toString(2))" + " invalidateTruthIndex()" " val sessionId = truth.optString(\"session_id\", sourceKey.ifBlank { \"unknown\" })" " val reviewFile = containedLeaf(reviewsDir, \"review-${sha256Hex(sessionId.toByteArray())}.review.json\")" " atomicWriteText(reviewFile, truth.toString(2))" @@ -3872,6 +4016,7 @@ " }" " if (!shouldWrite) return false" " atomicWriteText(dest, text)" + " invalidateTruthIndex()" " return true" " }" "" @@ -3974,6 +4119,7 @@ " require(current.files + staged.size <= MAX_STORAGE_FILES) { \"Storage file-count quota exceeded\" }" " require(current.bytes + budget.expandedBytes <= MAX_STORAGE_BYTES) { \"Storage byte quota exceeded\" }" " staged.forEach { if (installStagedZipEntry(it)) count += 1 }" + " if (count > 0) invalidateTruthIndex()" " } finally {" " stage.deleteRecursively()" " }" @@ -4049,13 +4195,14 @@ " fun syncTree(treeUri: Uri): Pair<Int, Int> {" " val tree = DocumentFile.fromTreeUri(context, treeUri) ?: return 0 to 0" " val imported = importFromTree(tree)" + " if (imported > 0) invalidateTruthIndex()" " rebuildLearnedExamples()" " val exported = exportToTree(tree)" " return imported to exported" " }" "" " fun pullRemoteDump(): Int {" - " val connection = openPinnedConnection(\"dump.zip\", \"GET\")" + " val connection = openPinnedConnection(\"dump.zip?scope=app\", \"GET\")" " try {" " val code = connection.responseCode" " if (code !in 200..299) {"