diff --git a/android/app/src/main/java/tv/orbit/classic/CatalogueShadow.kt b/android/app/src/main/java/tv/orbit/classic/CatalogueShadow.kt index 3222aa31..95f36672 100644 --- a/android/app/src/main/java/tv/orbit/classic/CatalogueShadow.kt +++ b/android/app/src/main/java/tv/orbit/classic/CatalogueShadow.kt @@ -27,10 +27,11 @@ internal object ProviderSwitches { * first raw pages (titles, people and ids outside Orbit replaced, structure kept) as a test fixture. Asked for from adb: * am start -n tv.orbit.classic/.MainActivity --ei orbit_shadow_catalogue * am start -n tv.orbit.classic/.MainActivity --ez orbit_shadow_catalogue_stop true (stop at the next page) - * am start -n tv.orbit.classic/.MainActivity --el orbit_shadow_from --ei orbit_shadow_catalogue 0 (resume) + * am start -n tv.orbit.classic/.MainActivity --ez orbit_shadow_resume true --ei orbit_shadow_catalogue 0 (resume) * It is paced (at least 1.5 s between pages, 5 s while a programme plays, never faster than the server answers), tries a * failed page once more after 30 s, and stops by itself if Orbit closes, playback stalls three times, memory passes 80% - * or the server takes over 20 s a page three times running. The report is rewritten every 25 pages, so a stop keeps it. + * or the server takes over 20 s a page three times running. The report is saved after every page, so its positionReached + * is always the exact last verified position; a resume starts there and nowhere else (see [resumePoint]). * The report lands in Orbit's shared folder, shadow-catalogue/, for adb to pull. Counts and ids only; no titles. */ internal object CatalogueShadow { @@ -40,17 +41,50 @@ internal object CatalogueShadow { /** Set from adb (orbit_shadow_catalogue_stop) to end a running comparison at the next page; the report so far is kept. */ @Volatile var cancel = false - /** Where a resumed comparison starts (orbit_shadow_from), from an earlier report's positionReached. */ - @Volatile var from = 0L + /** Where a resumed comparison is asked to start (orbit_shadow_from); it must equal the last verified position. */ + @Volatile var from: Long? = null + /** Set from adb (orbit_shadow_resume): carry on from the last verified position in the saved report. */ + @Volatile var resume = false + + /** + * Where a resumed run may start, or why it may not (9 November 2026, Paul: a resume must never count unchecked pages). + * The only allowed start is the saved report's positionReached: the end of the last page both readers returned and that + * was compared and saved. Anything else is refused, as is a resume with no report or one that already finished. + */ + fun resumePoint(previous: JSONObject?, asked: Long?): Pair { + if (previous == null) return null to "refused: no saved report to resume from" + val verified = previous.optLong("positionReached", -1L) + if (verified < 0) return null to "refused: the saved report has no verified position" + val total = if (previous.isNull("total")) null else previous.optLong("total") + if (total != null && verified >= total) return null to "refused: the saved run already reached the end ($verified of $total)" + if (asked != null && asked != verified) return null to "refused: asked to start at $asked, but the last verified position is $verified" + return verified to null + } fun runIfPending(session: EmbySampleSession, context: Context) { val pages = pending ?: return pending = null; cancel = false - val startAt = from.also { from = 0L } - val out = File(context.getExternalFilesDir(null), "shadow-catalogue").apply { - // A resumed run keeps the earlier report beside its own; a fresh one starts clean. - if (startAt > 0) File(this, "report.json").takeIf { it.exists() }?.renameTo(File(this, "report-before-$startAt.json")) else deleteRecursively() - mkdirs() } + val asked = from.also { from = null }; val resuming = resume || asked != null; resume = false + val out = File(context.getExternalFilesDir(null), "shadow-catalogue").apply { mkdirs() } + val reportFile = File(out, "report.json") + val previous = if (resuming) runCatching { JSONObject(reportFile.readText()) }.getOrNull() else null + val startAt: Long + if (resuming) { + val (point, refusal) = resumePoint(previous, asked) + if (point == null) { + File(out, "resume-refused.txt").writeText("$refusal\n") + android.util.Log.w("OrbitProvider", "shadow catalogue $refusal"); return + } + startAt = point + // The earlier report stays beside the new one; the new one carries the runs so far. + reportFile.renameTo(File(out, "report-before-$startAt.json")) + } else { out.deleteRecursively(); out.mkdirs(); startAt = 0L } + val earlierRuns = previous?.optJSONArray("runs") ?: JSONArray() + val earlierRecords = previous?.let { it.optLong("recordsVerifiedInAllRuns", it.optLong("records")) } ?: 0L + val earlierMismatches = previous?.let { it.optInt("mismatchesInAllRuns", it.optInt("mismatches")) } ?: 0 + fun withHistory(r: JSONObject): JSONObject = r.put("runs", JSONArray(earlierRuns.toString()).put(JSONObject().put("startedAt", r.opt("startedAt")) + .put("positionReached", r.opt("positionReached")).put("stopped", r.opt("stopped")).put("error", r.opt("error")).put("running", r.opt("running")))) + .put("recordsVerifiedInAllRuns", earlierRecords + r.optLong("records")).put("mismatchesInAllRuns", earlierMismatches + r.optInt("mismatches")) val raw = mutableListOf() val stallsAtStart = PlaybackHealth.stalls val runtime = Runtime.getRuntime() @@ -69,8 +103,8 @@ internal object CatalogueShadow { }, pause = { ms -> Thread.sleep(ms) }, gapMs = { pageMs -> maxOf(if (PlaybackHealth.playing) PLAYING_GAP_MS else IDLE_GAP_MS, pageMs) }, - onProgress = { File(out, "report.json").writeText(it.toString(2)) }) - File(out, "report.json").writeText(report.toString(2)) + onProgress = { writeSafely(reportFile, withHistory(it).toString(2)) }) + writeSafely(reportFile, withHistory(report).toString(2)) raw.forEachIndexed { i, body -> File(out, "page-$i.json").writeText(sanitiseCataloguePage(body).toString()) } android.util.Log.i("OrbitProvider", "shadow catalogue pages=${report.optInt("pages")} same=${report.optBoolean("identical")} mismatches=${report.optInt("mismatches")} stopped=${report.opt("stopped")}") } @@ -84,8 +118,11 @@ internal object CatalogueShadow { /** A failed page is tried once more after this long; a second failure stops the run. */ const val RETRY_AFTER_MS = 30_000L const val MAX_STALLS = 3 - /** The report is written this often while running, so a stop at any point leaves the results so far. */ - const val PROGRESS_EVERY = 25 + /** + * The report is saved (written beside, then swapped in) after every compared page, so its positionReached is always the + * exact last verified position, whatever stops the run. + */ + const val PROGRESS_EVERY = 1 /** * Reads [maxPages] pages from both paths, the same way today's import does (counted first page, CATALOGUE_PAGE_SIZE rows), @@ -102,10 +139,10 @@ internal object CatalogueShadow { slowPageMs: Long = SLOW_PAGE_MS): JSONObject { var start = startAt; var pages = 0; var mismatches = 0; var records = 0; var retries = 0; var slowRun = 0 var todayMs = 0L; var providerMs = 0L; var peakHeap = heapUsed(); val heapBefore = peakHeap - val firstMismatches = JSONArray(); var error: String? = null; var total: Long? = null; var stopped: String? = null + val firstMismatches = JSONArray(); val mismatchPositions = JSONArray(); var error: String? = null; var total: Long? = null; var stopped: String? = null fun report(running: Boolean) = JSONObject().put("pages", pages).put("records", records).put("startedAt", startAt).put("positionReached", start) .put("total", total ?: JSONObject.NULL).put("identical", mismatches == 0 && error == null).put("mismatches", mismatches) - .put("firstMismatches", firstMismatches).put("todayMs", todayMs).put("providerMs", providerMs).put("retries", retries) + .put("firstMismatches", firstMismatches).put("mismatchPositions", mismatchPositions).put("todayMs", todayMs).put("providerMs", providerMs).put("retries", retries) .put("heapBeforeKb", heapBefore / 1024).put("heapPeakKb", peakHeap / 1024).put("error", error ?: JSONObject.NULL) .put("stopped", stopped ?: JSONObject.NULL).put("running", running).put("wroteCatalogue", false) while (pages < maxPages) { @@ -129,6 +166,7 @@ internal object CatalogueShadow { val same = a.rawCount == b.rawCount && a.total == b.total && a.records == b.records if (!same) { mismatches++ + if (mismatchPositions.length() < 5_000) mismatchPositions.put(start) if (firstMismatches.length() < 20) firstMismatches.put(JSONObject().put("position", start).put("rawToday", a.rawCount).put("rawProvider", b.rawCount) .put("totalToday", a.total ?: JSONObject.NULL).put("totalProvider", b.total ?: JSONObject.NULL) .put("ids", JSONArray((a.records.map { it.id } + b.records.map { it.id }).distinct().filter { id -> diff --git a/android/app/src/main/java/tv/orbit/classic/MainActivity.kt b/android/app/src/main/java/tv/orbit/classic/MainActivity.kt index e0d954fe..b8142e0c 100644 --- a/android/app/src/main/java/tv/orbit/classic/MainActivity.kt +++ b/android/app/src/main/java/tv/orbit/classic/MainActivity.kt @@ -75,6 +75,7 @@ class MainActivity : ComponentActivity() { when (intent?.getStringExtra("orbit_avi")) { "on" -> PlayerSwitches.setAvi(this, true); "off" -> PlayerSwitches.setAvi(this, false) } if (intent?.getBooleanExtra("orbit_shadow_catalogue_stop", false) == true) CatalogueShadow.cancel = true if (intent?.hasExtra("orbit_shadow_from") == true) CatalogueShadow.from = intent.getLongExtra("orbit_shadow_from", 0L).coerceAtLeast(0L) + if (intent?.getBooleanExtra("orbit_shadow_resume", false) == true) CatalogueShadow.resume = true if (intent?.hasExtra("orbit_shadow_catalogue") == true) CatalogueShadow.pending = intent.getIntExtra("orbit_shadow_catalogue", 5) } override fun onNewIntent(intent: android.content.Intent) { super.onNewIntent(intent); setIntent(intent); readAdbRequests(intent) } diff --git a/android/app/src/test/java/tv/orbit/classic/ShadowSafetyTest.kt b/android/app/src/test/java/tv/orbit/classic/ShadowSafetyTest.kt index 15ae7352..80706211 100644 --- a/android/app/src/test/java/tv/orbit/classic/ShadowSafetyTest.kt +++ b/android/app/src/test/java/tv/orbit/classic/ShadowSafetyTest.kt @@ -54,6 +54,31 @@ class ShadowSafetyTest { val progress = mutableListOf() CatalogueShadow.compare({ s, _, _ -> CataloguePage(emptyList(), 10, 1_000) }, { s, _, _ -> CataloguePage(emptyList(), 10, 1_000) }, 100, onProgress = { assertTrue(it.getBoolean("running")); progress += it.getInt("pages") }) - assertEquals(listOf(25, 50, 75), progress) + assertEquals((1..99).toList(), progress) // saved after every compared page; the 100th is the final report + } + + @Test fun theSavedPositionIsAlwaysTheLastComparedPage() { + val saved = mutableListOf(); var reads = 0 + // The provider fails for good on the sixth page: five pages were compared, so 1,000 is the last verified position. + val r = CatalogueShadow.compare(reader, { s, n, w -> reads++; if (s >= 1_000) throw java.io.IOException("down") else reader(s, n, w) }, 100, + onProgress = { saved += it.getLong("positionReached") }) + assertEquals(listOf(200L, 400L, 600L, 800L, 1_000L), saved) + assertEquals(1_000L, r.getLong("positionReached")); assertEquals(5, r.getInt("pages")) + } + + @Test fun aResumeStartsOnlyAtTheLastVerifiedPosition() { + fun report(at: Long, total: Long? = 2_000L) = org.json.JSONObject().put("positionReached", at).put("total", total ?: org.json.JSONObject.NULL) + assertEquals(1_000L to null, CatalogueShadow.resumePoint(report(1_000), null)) + assertEquals(1_000L to null, CatalogueShadow.resumePoint(report(1_000), 1_000)) + assertTrue(CatalogueShadow.resumePoint(report(1_000), 1_200).second!!.contains("last verified position is 1000")) // can't skip ahead + assertTrue(CatalogueShadow.resumePoint(report(1_000), 800).second!!.startsWith("refused")) // or go back unasked + assertTrue(CatalogueShadow.resumePoint(null, null).second!!.startsWith("refused")) // nothing to resume + assertTrue(CatalogueShadow.resumePoint(report(2_000), null).second!!.contains("already reached the end")) + assertTrue(CatalogueShadow.resumePoint(org.json.JSONObject(), null).second!!.startsWith("refused")) + } + + @Test fun everyMismatchPositionIsKept() { + val r = CatalogueShadow.compare(reader, { s, n, w -> CataloguePage(emptyList(), page(s).rawCount, total + 1) }, 100) + assertEquals(10, r.getInt("mismatches")); assertEquals(10, r.getJSONArray("mismatchPositions").length()) } } diff --git a/tools/verify.sh b/tools/verify.sh index fb9a2443..09a32987 100755 --- a/tools/verify.sh +++ b/tools/verify.sh @@ -11,6 +11,8 @@ if [ -d "$TOOLING/gradle-cache" ]; then export GRADLE_USER_HOME="$TOOLING/gradle export ANDROID_HOME="${ANDROID_HOME:-/opt/android-sdk}" python3 "$ROOT/tools/test_compare_snapshots.py" 2>&1 | tail -1 # the install checker is tested too cd "$ROOT/android" +# A clean build every time (9 November 2026): incremental builds left stale Compose stability checks in the APK. +./gradlew --no-daemon "${OFFLINE[@]}" -q clean ./gradlew --no-daemon "${OFFLINE[@]}" -q :app:assembleRelease :app:testDebugUnitTest --rerun :app:lintRelease python3 - <<'PY' import glob, re, sys