Shadow comparison: exact cursor saved after every page; a resume can only start there

The report (positionReached, every mismatch position, the runs so far) is saved
safely after each compared page. orbit_shadow_resume picks up from the saved
position; a start anywhere else, a resume without a report, or one after the end
is refused and noted in resume-refused.txt. verify.sh now builds clean: incremental
builds had left stale Compose stability checks in two earlier APKs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-09 15:16:30 +00:00
co-authored by Claude Opus 5.5
parent 1d6800773d
commit be7b4e963e
4 changed files with 82 additions and 16 deletions
@@ -27,10 +27,11 @@ internal object ProviderSwitches {
* first raw pages (titles, people and ids outside Orbit replaced, structure kept) as a test fixture. Asked for from adb:
* am start -n tv.orbit.classic/.MainActivity --ei orbit_shadow_catalogue <pages, 0 = the whole library>
* am start -n tv.orbit.classic/.MainActivity --ez orbit_shadow_catalogue_stop true (stop at the next page)
* am start -n tv.orbit.classic/.MainActivity --el orbit_shadow_from <position> --ei orbit_shadow_catalogue 0 (resume)
* am start -n tv.orbit.classic/.MainActivity --ez orbit_shadow_resume true --ei orbit_shadow_catalogue 0 (resume)
* It is paced (at least 1.5 s between pages, 5 s while a programme plays, never faster than the server answers), tries a
* failed page once more after 30 s, and stops by itself if Orbit closes, playback stalls three times, memory passes 80%
* or the server takes over 20 s a page three times running. The report is rewritten every 25 pages, so a stop keeps it.
* or the server takes over 20 s a page three times running. The report is saved after every page, so its positionReached
* is always the exact last verified position; a resume starts there and nowhere else (see [resumePoint]).
* The report lands in Orbit's shared folder, shadow-catalogue/, for adb to pull. Counts and ids only; no titles.
*/
internal object CatalogueShadow {
@@ -40,17 +41,50 @@ internal object CatalogueShadow {
/** Set from adb (orbit_shadow_catalogue_stop) to end a running comparison at the next page; the report so far is kept. */
@Volatile var cancel = false
/** Where a resumed comparison starts (orbit_shadow_from), from an earlier report's positionReached. */
@Volatile var from = 0L
/** Where a resumed comparison is asked to start (orbit_shadow_from); it must equal the last verified position. */
@Volatile var from: Long? = null
/** Set from adb (orbit_shadow_resume): carry on from the last verified position in the saved report. */
@Volatile var resume = false
/**
* Where a resumed run may start, or why it may not (9 November 2026, Paul: a resume must never count unchecked pages).
* The only allowed start is the saved report's positionReached: the end of the last page both readers returned and that
* was compared and saved. Anything else is refused, as is a resume with no report or one that already finished.
*/
fun resumePoint(previous: JSONObject?, asked: Long?): Pair<Long?, String?> {
if (previous == null) return null to "refused: no saved report to resume from"
val verified = previous.optLong("positionReached", -1L)
if (verified < 0) return null to "refused: the saved report has no verified position"
val total = if (previous.isNull("total")) null else previous.optLong("total")
if (total != null && verified >= total) return null to "refused: the saved run already reached the end ($verified of $total)"
if (asked != null && asked != verified) return null to "refused: asked to start at $asked, but the last verified position is $verified"
return verified to null
}
fun runIfPending(session: EmbySampleSession, context: Context) {
val pages = pending ?: return
pending = null; cancel = false
val startAt = from.also { from = 0L }
val out = File(context.getExternalFilesDir(null), "shadow-catalogue").apply {
// A resumed run keeps the earlier report beside its own; a fresh one starts clean.
if (startAt > 0) File(this, "report.json").takeIf { it.exists() }?.renameTo(File(this, "report-before-$startAt.json")) else deleteRecursively()
mkdirs() }
val asked = from.also { from = null }; val resuming = resume || asked != null; resume = false
val out = File(context.getExternalFilesDir(null), "shadow-catalogue").apply { mkdirs() }
val reportFile = File(out, "report.json")
val previous = if (resuming) runCatching { JSONObject(reportFile.readText()) }.getOrNull() else null
val startAt: Long
if (resuming) {
val (point, refusal) = resumePoint(previous, asked)
if (point == null) {
File(out, "resume-refused.txt").writeText("$refusal\n")
android.util.Log.w("OrbitProvider", "shadow catalogue $refusal"); return
}
startAt = point
// The earlier report stays beside the new one; the new one carries the runs so far.
reportFile.renameTo(File(out, "report-before-$startAt.json"))
} else { out.deleteRecursively(); out.mkdirs(); startAt = 0L }
val earlierRuns = previous?.optJSONArray("runs") ?: JSONArray()
val earlierRecords = previous?.let { it.optLong("recordsVerifiedInAllRuns", it.optLong("records")) } ?: 0L
val earlierMismatches = previous?.let { it.optInt("mismatchesInAllRuns", it.optInt("mismatches")) } ?: 0
fun withHistory(r: JSONObject): JSONObject = r.put("runs", JSONArray(earlierRuns.toString()).put(JSONObject().put("startedAt", r.opt("startedAt"))
.put("positionReached", r.opt("positionReached")).put("stopped", r.opt("stopped")).put("error", r.opt("error")).put("running", r.opt("running"))))
.put("recordsVerifiedInAllRuns", earlierRecords + r.optLong("records")).put("mismatchesInAllRuns", earlierMismatches + r.optInt("mismatches"))
val raw = mutableListOf<String>()
val stallsAtStart = PlaybackHealth.stalls
val runtime = Runtime.getRuntime()
@@ -69,8 +103,8 @@ internal object CatalogueShadow {
},
pause = { ms -> Thread.sleep(ms) },
gapMs = { pageMs -> maxOf(if (PlaybackHealth.playing) PLAYING_GAP_MS else IDLE_GAP_MS, pageMs) },
onProgress = { File(out, "report.json").writeText(it.toString(2)) })
File(out, "report.json").writeText(report.toString(2))
onProgress = { writeSafely(reportFile, withHistory(it).toString(2)) })
writeSafely(reportFile, withHistory(report).toString(2))
raw.forEachIndexed { i, body -> File(out, "page-$i.json").writeText(sanitiseCataloguePage(body).toString()) }
android.util.Log.i("OrbitProvider", "shadow catalogue pages=${report.optInt("pages")} same=${report.optBoolean("identical")} mismatches=${report.optInt("mismatches")} stopped=${report.opt("stopped")}")
}
@@ -84,8 +118,11 @@ internal object CatalogueShadow {
/** A failed page is tried once more after this long; a second failure stops the run. */
const val RETRY_AFTER_MS = 30_000L
const val MAX_STALLS = 3
/** The report is written this often while running, so a stop at any point leaves the results so far. */
const val PROGRESS_EVERY = 25
/**
* The report is saved (written beside, then swapped in) after every compared page, so its positionReached is always the
* exact last verified position, whatever stops the run.
*/
const val PROGRESS_EVERY = 1
/**
* Reads [maxPages] pages from both paths, the same way today's import does (counted first page, CATALOGUE_PAGE_SIZE rows),
@@ -102,10 +139,10 @@ internal object CatalogueShadow {
slowPageMs: Long = SLOW_PAGE_MS): JSONObject {
var start = startAt; var pages = 0; var mismatches = 0; var records = 0; var retries = 0; var slowRun = 0
var todayMs = 0L; var providerMs = 0L; var peakHeap = heapUsed(); val heapBefore = peakHeap
val firstMismatches = JSONArray(); var error: String? = null; var total: Long? = null; var stopped: String? = null
val firstMismatches = JSONArray(); val mismatchPositions = JSONArray(); var error: String? = null; var total: Long? = null; var stopped: String? = null
fun report(running: Boolean) = JSONObject().put("pages", pages).put("records", records).put("startedAt", startAt).put("positionReached", start)
.put("total", total ?: JSONObject.NULL).put("identical", mismatches == 0 && error == null).put("mismatches", mismatches)
.put("firstMismatches", firstMismatches).put("todayMs", todayMs).put("providerMs", providerMs).put("retries", retries)
.put("firstMismatches", firstMismatches).put("mismatchPositions", mismatchPositions).put("todayMs", todayMs).put("providerMs", providerMs).put("retries", retries)
.put("heapBeforeKb", heapBefore / 1024).put("heapPeakKb", peakHeap / 1024).put("error", error ?: JSONObject.NULL)
.put("stopped", stopped ?: JSONObject.NULL).put("running", running).put("wroteCatalogue", false)
while (pages < maxPages) {
@@ -129,6 +166,7 @@ internal object CatalogueShadow {
val same = a.rawCount == b.rawCount && a.total == b.total && a.records == b.records
if (!same) {
mismatches++
if (mismatchPositions.length() < 5_000) mismatchPositions.put(start)
if (firstMismatches.length() < 20) firstMismatches.put(JSONObject().put("position", start).put("rawToday", a.rawCount).put("rawProvider", b.rawCount)
.put("totalToday", a.total ?: JSONObject.NULL).put("totalProvider", b.total ?: JSONObject.NULL)
.put("ids", JSONArray((a.records.map { it.id } + b.records.map { it.id }).distinct().filter { id ->
@@ -75,6 +75,7 @@ class MainActivity : ComponentActivity() {
when (intent?.getStringExtra("orbit_avi")) { "on" -> PlayerSwitches.setAvi(this, true); "off" -> PlayerSwitches.setAvi(this, false) }
if (intent?.getBooleanExtra("orbit_shadow_catalogue_stop", false) == true) CatalogueShadow.cancel = true
if (intent?.hasExtra("orbit_shadow_from") == true) CatalogueShadow.from = intent.getLongExtra("orbit_shadow_from", 0L).coerceAtLeast(0L)
if (intent?.getBooleanExtra("orbit_shadow_resume", false) == true) CatalogueShadow.resume = true
if (intent?.hasExtra("orbit_shadow_catalogue") == true) CatalogueShadow.pending = intent.getIntExtra("orbit_shadow_catalogue", 5)
}
override fun onNewIntent(intent: android.content.Intent) { super.onNewIntent(intent); setIntent(intent); readAdbRequests(intent) }
@@ -54,6 +54,31 @@ class ShadowSafetyTest {
val progress = mutableListOf<Int>()
CatalogueShadow.compare({ s, _, _ -> CataloguePage(emptyList(), 10, 1_000) }, { s, _, _ -> CataloguePage(emptyList(), 10, 1_000) }, 100,
onProgress = { assertTrue(it.getBoolean("running")); progress += it.getInt("pages") })
assertEquals(listOf(25, 50, 75), progress)
assertEquals((1..99).toList(), progress) // saved after every compared page; the 100th is the final report
}
@Test fun theSavedPositionIsAlwaysTheLastComparedPage() {
val saved = mutableListOf<Long>(); var reads = 0
// The provider fails for good on the sixth page: five pages were compared, so 1,000 is the last verified position.
val r = CatalogueShadow.compare(reader, { s, n, w -> reads++; if (s >= 1_000) throw java.io.IOException("down") else reader(s, n, w) }, 100,
onProgress = { saved += it.getLong("positionReached") })
assertEquals(listOf(200L, 400L, 600L, 800L, 1_000L), saved)
assertEquals(1_000L, r.getLong("positionReached")); assertEquals(5, r.getInt("pages"))
}
@Test fun aResumeStartsOnlyAtTheLastVerifiedPosition() {
fun report(at: Long, total: Long? = 2_000L) = org.json.JSONObject().put("positionReached", at).put("total", total ?: org.json.JSONObject.NULL)
assertEquals(1_000L to null, CatalogueShadow.resumePoint(report(1_000), null))
assertEquals(1_000L to null, CatalogueShadow.resumePoint(report(1_000), 1_000))
assertTrue(CatalogueShadow.resumePoint(report(1_000), 1_200).second!!.contains("last verified position is 1000")) // can't skip ahead
assertTrue(CatalogueShadow.resumePoint(report(1_000), 800).second!!.startsWith("refused")) // or go back unasked
assertTrue(CatalogueShadow.resumePoint(null, null).second!!.startsWith("refused")) // nothing to resume
assertTrue(CatalogueShadow.resumePoint(report(2_000), null).second!!.contains("already reached the end"))
assertTrue(CatalogueShadow.resumePoint(org.json.JSONObject(), null).second!!.startsWith("refused"))
}
@Test fun everyMismatchPositionIsKept() {
val r = CatalogueShadow.compare(reader, { s, n, w -> CataloguePage(emptyList(), page(s).rawCount, total + 1) }, 100)
assertEquals(10, r.getInt("mismatches")); assertEquals(10, r.getJSONArray("mismatchPositions").length())
}
}