Shadow comparison: exact cursor saved after every page; a resume can only start there
The report (positionReached, every mismatch position, the runs so far) is saved safely after each compared page. orbit_shadow_resume picks up from the saved position; a start anywhere else, a resume without a report, or one after the end is refused and noted in resume-refused.txt. verify.sh now builds clean: incremental builds had left stale Compose stability checks in two earlier APKs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -27,10 +27,11 @@ internal object ProviderSwitches {
|
||||
* first raw pages (titles, people and ids outside Orbit replaced, structure kept) as a test fixture. Asked for from adb:
|
||||
* am start -n tv.orbit.classic/.MainActivity --ei orbit_shadow_catalogue <pages, 0 = the whole library>
|
||||
* am start -n tv.orbit.classic/.MainActivity --ez orbit_shadow_catalogue_stop true (stop at the next page)
|
||||
* am start -n tv.orbit.classic/.MainActivity --el orbit_shadow_from <position> --ei orbit_shadow_catalogue 0 (resume)
|
||||
* am start -n tv.orbit.classic/.MainActivity --ez orbit_shadow_resume true --ei orbit_shadow_catalogue 0 (resume)
|
||||
* It is paced (at least 1.5 s between pages, 5 s while a programme plays, never faster than the server answers), tries a
|
||||
* failed page once more after 30 s, and stops by itself if Orbit closes, playback stalls three times, memory passes 80%
|
||||
* or the server takes over 20 s a page three times running. The report is rewritten every 25 pages, so a stop keeps it.
|
||||
* or the server takes over 20 s a page three times running. The report is saved after every page, so its positionReached
|
||||
* is always the exact last verified position; a resume starts there and nowhere else (see [resumePoint]).
|
||||
* The report lands in Orbit's shared folder, shadow-catalogue/, for adb to pull. Counts and ids only; no titles.
|
||||
*/
|
||||
internal object CatalogueShadow {
|
||||
@@ -40,17 +41,50 @@ internal object CatalogueShadow {
|
||||
|
||||
/** Set from adb (orbit_shadow_catalogue_stop) to end a running comparison at the next page; the report so far is kept. */
|
||||
@Volatile var cancel = false
|
||||
/** Where a resumed comparison starts (orbit_shadow_from), from an earlier report's positionReached. */
|
||||
@Volatile var from = 0L
|
||||
/** Where a resumed comparison is asked to start (orbit_shadow_from); it must equal the last verified position. */
|
||||
@Volatile var from: Long? = null
|
||||
/** Set from adb (orbit_shadow_resume): carry on from the last verified position in the saved report. */
|
||||
@Volatile var resume = false
|
||||
|
||||
/**
|
||||
* Where a resumed run may start, or why it may not (9 November 2026, Paul: a resume must never count unchecked pages).
|
||||
* The only allowed start is the saved report's positionReached: the end of the last page both readers returned and that
|
||||
* was compared and saved. Anything else is refused, as is a resume with no report or one that already finished.
|
||||
*/
|
||||
fun resumePoint(previous: JSONObject?, asked: Long?): Pair<Long?, String?> {
|
||||
if (previous == null) return null to "refused: no saved report to resume from"
|
||||
val verified = previous.optLong("positionReached", -1L)
|
||||
if (verified < 0) return null to "refused: the saved report has no verified position"
|
||||
val total = if (previous.isNull("total")) null else previous.optLong("total")
|
||||
if (total != null && verified >= total) return null to "refused: the saved run already reached the end ($verified of $total)"
|
||||
if (asked != null && asked != verified) return null to "refused: asked to start at $asked, but the last verified position is $verified"
|
||||
return verified to null
|
||||
}
|
||||
|
||||
fun runIfPending(session: EmbySampleSession, context: Context) {
|
||||
val pages = pending ?: return
|
||||
pending = null; cancel = false
|
||||
val startAt = from.also { from = 0L }
|
||||
val out = File(context.getExternalFilesDir(null), "shadow-catalogue").apply {
|
||||
// A resumed run keeps the earlier report beside its own; a fresh one starts clean.
|
||||
if (startAt > 0) File(this, "report.json").takeIf { it.exists() }?.renameTo(File(this, "report-before-$startAt.json")) else deleteRecursively()
|
||||
mkdirs() }
|
||||
val asked = from.also { from = null }; val resuming = resume || asked != null; resume = false
|
||||
val out = File(context.getExternalFilesDir(null), "shadow-catalogue").apply { mkdirs() }
|
||||
val reportFile = File(out, "report.json")
|
||||
val previous = if (resuming) runCatching { JSONObject(reportFile.readText()) }.getOrNull() else null
|
||||
val startAt: Long
|
||||
if (resuming) {
|
||||
val (point, refusal) = resumePoint(previous, asked)
|
||||
if (point == null) {
|
||||
File(out, "resume-refused.txt").writeText("$refusal\n")
|
||||
android.util.Log.w("OrbitProvider", "shadow catalogue $refusal"); return
|
||||
}
|
||||
startAt = point
|
||||
// The earlier report stays beside the new one; the new one carries the runs so far.
|
||||
reportFile.renameTo(File(out, "report-before-$startAt.json"))
|
||||
} else { out.deleteRecursively(); out.mkdirs(); startAt = 0L }
|
||||
val earlierRuns = previous?.optJSONArray("runs") ?: JSONArray()
|
||||
val earlierRecords = previous?.let { it.optLong("recordsVerifiedInAllRuns", it.optLong("records")) } ?: 0L
|
||||
val earlierMismatches = previous?.let { it.optInt("mismatchesInAllRuns", it.optInt("mismatches")) } ?: 0
|
||||
fun withHistory(r: JSONObject): JSONObject = r.put("runs", JSONArray(earlierRuns.toString()).put(JSONObject().put("startedAt", r.opt("startedAt"))
|
||||
.put("positionReached", r.opt("positionReached")).put("stopped", r.opt("stopped")).put("error", r.opt("error")).put("running", r.opt("running"))))
|
||||
.put("recordsVerifiedInAllRuns", earlierRecords + r.optLong("records")).put("mismatchesInAllRuns", earlierMismatches + r.optInt("mismatches"))
|
||||
val raw = mutableListOf<String>()
|
||||
val stallsAtStart = PlaybackHealth.stalls
|
||||
val runtime = Runtime.getRuntime()
|
||||
@@ -69,8 +103,8 @@ internal object CatalogueShadow {
|
||||
},
|
||||
pause = { ms -> Thread.sleep(ms) },
|
||||
gapMs = { pageMs -> maxOf(if (PlaybackHealth.playing) PLAYING_GAP_MS else IDLE_GAP_MS, pageMs) },
|
||||
onProgress = { File(out, "report.json").writeText(it.toString(2)) })
|
||||
File(out, "report.json").writeText(report.toString(2))
|
||||
onProgress = { writeSafely(reportFile, withHistory(it).toString(2)) })
|
||||
writeSafely(reportFile, withHistory(report).toString(2))
|
||||
raw.forEachIndexed { i, body -> File(out, "page-$i.json").writeText(sanitiseCataloguePage(body).toString()) }
|
||||
android.util.Log.i("OrbitProvider", "shadow catalogue pages=${report.optInt("pages")} same=${report.optBoolean("identical")} mismatches=${report.optInt("mismatches")} stopped=${report.opt("stopped")}")
|
||||
}
|
||||
@@ -84,8 +118,11 @@ internal object CatalogueShadow {
|
||||
/** A failed page is tried once more after this long; a second failure stops the run. */
|
||||
const val RETRY_AFTER_MS = 30_000L
|
||||
const val MAX_STALLS = 3
|
||||
/** The report is written this often while running, so a stop at any point leaves the results so far. */
|
||||
const val PROGRESS_EVERY = 25
|
||||
/**
|
||||
* The report is saved (written beside, then swapped in) after every compared page, so its positionReached is always the
|
||||
* exact last verified position, whatever stops the run.
|
||||
*/
|
||||
const val PROGRESS_EVERY = 1
|
||||
|
||||
/**
|
||||
* Reads [maxPages] pages from both paths, the same way today's import does (counted first page, CATALOGUE_PAGE_SIZE rows),
|
||||
@@ -102,10 +139,10 @@ internal object CatalogueShadow {
|
||||
slowPageMs: Long = SLOW_PAGE_MS): JSONObject {
|
||||
var start = startAt; var pages = 0; var mismatches = 0; var records = 0; var retries = 0; var slowRun = 0
|
||||
var todayMs = 0L; var providerMs = 0L; var peakHeap = heapUsed(); val heapBefore = peakHeap
|
||||
val firstMismatches = JSONArray(); var error: String? = null; var total: Long? = null; var stopped: String? = null
|
||||
val firstMismatches = JSONArray(); val mismatchPositions = JSONArray(); var error: String? = null; var total: Long? = null; var stopped: String? = null
|
||||
fun report(running: Boolean) = JSONObject().put("pages", pages).put("records", records).put("startedAt", startAt).put("positionReached", start)
|
||||
.put("total", total ?: JSONObject.NULL).put("identical", mismatches == 0 && error == null).put("mismatches", mismatches)
|
||||
.put("firstMismatches", firstMismatches).put("todayMs", todayMs).put("providerMs", providerMs).put("retries", retries)
|
||||
.put("firstMismatches", firstMismatches).put("mismatchPositions", mismatchPositions).put("todayMs", todayMs).put("providerMs", providerMs).put("retries", retries)
|
||||
.put("heapBeforeKb", heapBefore / 1024).put("heapPeakKb", peakHeap / 1024).put("error", error ?: JSONObject.NULL)
|
||||
.put("stopped", stopped ?: JSONObject.NULL).put("running", running).put("wroteCatalogue", false)
|
||||
while (pages < maxPages) {
|
||||
@@ -129,6 +166,7 @@ internal object CatalogueShadow {
|
||||
val same = a.rawCount == b.rawCount && a.total == b.total && a.records == b.records
|
||||
if (!same) {
|
||||
mismatches++
|
||||
if (mismatchPositions.length() < 5_000) mismatchPositions.put(start)
|
||||
if (firstMismatches.length() < 20) firstMismatches.put(JSONObject().put("position", start).put("rawToday", a.rawCount).put("rawProvider", b.rawCount)
|
||||
.put("totalToday", a.total ?: JSONObject.NULL).put("totalProvider", b.total ?: JSONObject.NULL)
|
||||
.put("ids", JSONArray((a.records.map { it.id } + b.records.map { it.id }).distinct().filter { id ->
|
||||
|
||||
@@ -75,6 +75,7 @@ class MainActivity : ComponentActivity() {
|
||||
when (intent?.getStringExtra("orbit_avi")) { "on" -> PlayerSwitches.setAvi(this, true); "off" -> PlayerSwitches.setAvi(this, false) }
|
||||
if (intent?.getBooleanExtra("orbit_shadow_catalogue_stop", false) == true) CatalogueShadow.cancel = true
|
||||
if (intent?.hasExtra("orbit_shadow_from") == true) CatalogueShadow.from = intent.getLongExtra("orbit_shadow_from", 0L).coerceAtLeast(0L)
|
||||
if (intent?.getBooleanExtra("orbit_shadow_resume", false) == true) CatalogueShadow.resume = true
|
||||
if (intent?.hasExtra("orbit_shadow_catalogue") == true) CatalogueShadow.pending = intent.getIntExtra("orbit_shadow_catalogue", 5)
|
||||
}
|
||||
override fun onNewIntent(intent: android.content.Intent) { super.onNewIntent(intent); setIntent(intent); readAdbRequests(intent) }
|
||||
|
||||
@@ -54,6 +54,31 @@ class ShadowSafetyTest {
|
||||
val progress = mutableListOf<Int>()
|
||||
CatalogueShadow.compare({ s, _, _ -> CataloguePage(emptyList(), 10, 1_000) }, { s, _, _ -> CataloguePage(emptyList(), 10, 1_000) }, 100,
|
||||
onProgress = { assertTrue(it.getBoolean("running")); progress += it.getInt("pages") })
|
||||
assertEquals(listOf(25, 50, 75), progress)
|
||||
assertEquals((1..99).toList(), progress) // saved after every compared page; the 100th is the final report
|
||||
}
|
||||
|
||||
@Test fun theSavedPositionIsAlwaysTheLastComparedPage() {
|
||||
val saved = mutableListOf<Long>(); var reads = 0
|
||||
// The provider fails for good on the sixth page: five pages were compared, so 1,000 is the last verified position.
|
||||
val r = CatalogueShadow.compare(reader, { s, n, w -> reads++; if (s >= 1_000) throw java.io.IOException("down") else reader(s, n, w) }, 100,
|
||||
onProgress = { saved += it.getLong("positionReached") })
|
||||
assertEquals(listOf(200L, 400L, 600L, 800L, 1_000L), saved)
|
||||
assertEquals(1_000L, r.getLong("positionReached")); assertEquals(5, r.getInt("pages"))
|
||||
}
|
||||
|
||||
@Test fun aResumeStartsOnlyAtTheLastVerifiedPosition() {
|
||||
fun report(at: Long, total: Long? = 2_000L) = org.json.JSONObject().put("positionReached", at).put("total", total ?: org.json.JSONObject.NULL)
|
||||
assertEquals(1_000L to null, CatalogueShadow.resumePoint(report(1_000), null))
|
||||
assertEquals(1_000L to null, CatalogueShadow.resumePoint(report(1_000), 1_000))
|
||||
assertTrue(CatalogueShadow.resumePoint(report(1_000), 1_200).second!!.contains("last verified position is 1000")) // can't skip ahead
|
||||
assertTrue(CatalogueShadow.resumePoint(report(1_000), 800).second!!.startsWith("refused")) // or go back unasked
|
||||
assertTrue(CatalogueShadow.resumePoint(null, null).second!!.startsWith("refused")) // nothing to resume
|
||||
assertTrue(CatalogueShadow.resumePoint(report(2_000), null).second!!.contains("already reached the end"))
|
||||
assertTrue(CatalogueShadow.resumePoint(org.json.JSONObject(), null).second!!.startsWith("refused"))
|
||||
}
|
||||
|
||||
@Test fun everyMismatchPositionIsKept() {
|
||||
val r = CatalogueShadow.compare(reader, { s, n, w -> CataloguePage(emptyList(), page(s).rawCount, total + 1) }, 100)
|
||||
assertEquals(10, r.getInt("mismatches")); assertEquals(10, r.getJSONArray("mismatchPositions").length())
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user