From 40ba3fecdfde99f503f863be66aae7ede57768a8 Mon Sep 17 00:00:00 2001 From: QIU Date: Thu, 20 Aug 2026 15:00:13 +0000 Subject: [PATCH] chore(amsat): remove the HTML scraping path superseded by the JSON API The merged upstream AMSAT implementation fetches status data from AMSAT's JSON endpoints (getAmSatCatalog / getAmSatReports), so the fork's HTML scraping path no longer has a caller: - core/data/.../source/AmSatParser.kt (136 lines): parsed the amsat.org status table, deriving state from the page's inline colour codes. - IRemoteSource.getStatusHtml() plus its RemoteSource implementation and the DatabaseRepoTest fake override. Verified zero references repo-wide before removing, and again afterwards. Request / CancellationException imports in RemoteSource remain in use by the other fetchers. compileReleaseKotlin plus core:domain / core:data / feature:map / feature:roaming unit tests stay green. --- .../look4sat/core/data/source/AmSatParser.kt | 136 ------------------ .../look4sat/core/data/source/RemoteSource.kt | 18 --- .../core/data/repository/DatabaseRepoTest.kt | 2 - .../core/domain/source/IRemoteSource.kt | 3 - 4 files changed, 159 deletions(-) delete mode 100644 core/data/src/main/java/com/rtbishop/look4sat/core/data/source/AmSatParser.kt diff --git a/core/data/src/main/java/com/rtbishop/look4sat/core/data/source/AmSatParser.kt b/core/data/src/main/java/com/rtbishop/look4sat/core/data/source/AmSatParser.kt deleted file mode 100644 index 37f02570..00000000 --- a/core/data/src/main/java/com/rtbishop/look4sat/core/data/source/AmSatParser.kt +++ /dev/null @@ -1,136 +0,0 @@ -package com.rtbishop.look4sat.core.data.source - -import com.rtbishop.look4sat.core.domain.model.SatDay -import com.rtbishop.look4sat.core.domain.model.SatReport -import com.rtbishop.look4sat.core.domain.model.SatSlot -import com.rtbishop.look4sat.core.domain.model.SatStatus -import com.rtbishop.look4sat.core.domain.model.SatStatusPage -import java.util.regex.Pattern - -/** - * AMSAT satellite status page parser (https://amsat.org/status/) - * - * Page structure (static HTML, verified 2026-08): - * - Status table: 48 rows in , header = Name + 6 days (each colspan=12) - * Data row has 73 cells: [0]=satellite name, [1..72] = 6 days x 12 two-hour slots (new->old) - * Each cell: ; when reports exist the count carries a - * docTips.show('id') link - * - Report details live in the page's inline JS: - * tips.a885153 = new Array(5,5,120,'status
callsign
grid
date
time span UTC') - * - * Status colors: #648fff=active, #ffb000=telemetry only, #dc267f=not heard, #fe6100=conflicting - */ -object AmSatParser { - - private val STATUS_COLORS = mapOf( - "#648fff" to 0xFF648FFF.toLong(), - "#ffb000" to 0xFFFFB000.toLong(), - "#dc267f" to 0xFFDC267F.toLong(), - "#fe6100" to 0xFFFE6100.toLong() - ) - private val GRAY = 0xFFC0C0C0.toLong() - - private val rowRe = Pattern.compile("(.*?)", Pattern.DOTALL) - private val cellRe = Pattern.compile("(]*>.*?)", Pattern.DOTALL) - private val bgRe = Pattern.compile("bgcolor=\"?(#[0-9a-fA-F]{6}|C0C0C0)\"?") - private val linkRe = Pattern.compile("docTips\\.show\\('(a\\d+)'\\)") - private val tipRe = Pattern.compile( - "tips\\.(a\\d+)\\s*=\\s*new\\s+Array\\(\\s*\\d+,\\s*\\d+,\\s*\\d+,\\s*'([^']*)'" - ) - - /** Parse the full page */ - fun parse(html: String, fetchedAtUtcMs: Long): SatStatusPage { - val reports = parseReports(html) - val statuses = parseStatusTable(html, reports) - return SatStatusPage(fetchedAtUtcMs, statuses, reports) - } - - /** Extract all reports (tips.* JS arrays) */ - fun parseReports(html: String): Map { - val map = mutableMapOf() - val m = tipRe.matcher(html) - while (m.find()) { - val id = m.group(1) - val parts = m.group(2).split("
") - map[id] = SatReport( - id = id, - statusText = parts.getOrElse(0) { "" }.trim(), - call = parts.getOrElse(1) { "" }.trim(), - grid = parts.getOrElse(2) { "" }.trim(), - dateUtc = parts.getOrElse(3) { "" }.trim(), - timeUtc = parts.getOrElse(4) { "" }.trim() - ) - } - return map - } - - /** Parse the status table (48 satellite rows) */ - fun parseStatusTable(html: String, reports: Map): List { - val result = mutableListOf() - val tables = extractTables(html) - for (table in tables) { - val rows = rowRe.matcher(table) - val parsed = mutableListOf() - var rowIndex = 0 - var dayHeaders: List = emptyList() - while (rows.find()) { - val rowHtml = rows.group(1) - val cells = cellRe.matcher(rowHtml) - val cellList = mutableListOf() - while (cells.find()) cellList.add(cells.group(1)) - if (rowIndex == 0) { - // Header: Name + 6 days (each colspan=12) - dayHeaders = cellList.drop(1).take(6).map { stripHtml(it) } - rowIndex++ - continue - } - if (cellList.size < 7) { rowIndex++; continue } - val name = stripHtml(cellList[0]) - if (name.isBlank()) { rowIndex++; continue } - val days = mutableListOf() - for (d in 0 until 6) { - val start = 1 + d * 12 - val end = start + 12 - val slots = (start until end).mapNotNull { i -> - cellList.getOrNull(i)?.let { cell -> - val bg = bgRe.matcher(cell) - val color = if (bg.find()) { - STATUS_COLORS[bg.group(1).lowercase()] ?: GRAY - } else GRAY - val link = linkRe.matcher(cell) - val ids = mutableListOf() - while (link.find()) ids.add(link.group(1)) - val hasReport = ids.isNotEmpty() - val count = if (hasReport) { - stripHtml(cell).trim().toIntOrNull() ?: ids.size - } else 0 - SatSlot( - statusColor = if (hasReport) color else GRAY, - count = count, - reportIds = ids - ) - } - } - days.add(SatDay(dayHeaders.getOrElse(d) { "" }, slots)) - } - parsed.add(SatStatus(name, days)) - rowIndex++ - } - if (parsed.isNotEmpty()) { - result.addAll(parsed) - break - } - } - return result - } - - private fun extractTables(html: String): List { - val result = mutableListOf() - val m = Pattern.compile("", Pattern.DOTALL).matcher(html) - while (m.find()) result.add(m.group()) - return result - } - - private fun stripHtml(s: String): String = - s.replace(Regex("<[^>]+>"), "").trim() -} diff --git a/core/data/src/main/java/com/rtbishop/look4sat/core/data/source/RemoteSource.kt b/core/data/src/main/java/com/rtbishop/look4sat/core/data/source/RemoteSource.kt index e2fa0af6..1ead6069 100644 --- a/core/data/src/main/java/com/rtbishop/look4sat/core/data/source/RemoteSource.kt +++ b/core/data/src/main/java/com/rtbishop/look4sat/core/data/source/RemoteSource.kt @@ -45,24 +45,6 @@ class RemoteSource( } } - override suspend fun getStatusHtml(): String? = withContext(dispatcher) { - try { - val request = Request.Builder() - .url("https://amsat.org/status/") - .header("User-Agent", "Mozilla/5.0 (Linux; Android 13) Look4Sat/4.5") - .build() - httpClient.newCall(request).execute().use { response -> - if (!response.isSuccessful) return@use null - response.body?.string() - } - } catch (exception: CancellationException) { - throw exception - } catch (exception: Exception) { - println("RemoteSource amsat status exception: $exception") - null - } - } - override suspend fun getNetworkStream(url: String): InputStream? = withContext(dispatcher) { try { val networkRequest = Request.Builder().url(url).build() diff --git a/core/data/src/test/java/com/rtbishop/look4sat/core/data/repository/DatabaseRepoTest.kt b/core/data/src/test/java/com/rtbishop/look4sat/core/data/repository/DatabaseRepoTest.kt index 3b76b6ca..f8867d03 100644 --- a/core/data/src/test/java/com/rtbishop/look4sat/core/data/repository/DatabaseRepoTest.kt +++ b/core/data/src/test/java/com/rtbishop/look4sat/core/data/repository/DatabaseRepoTest.kt @@ -126,8 +126,6 @@ private class FakeRemoteSource : IRemoteSource { override suspend fun getNetworkStream(url: String): InputStream? = networkStreams[url]?.invoke() - override suspend fun getStatusHtml(): String? = null - override suspend fun getAmSatCatalog(): String? = null override suspend fun getAmSatReports(hours: Int, limit: Int): String? = null diff --git a/core/domain/src/main/java/com/rtbishop/look4sat/core/domain/source/IRemoteSource.kt b/core/domain/src/main/java/com/rtbishop/look4sat/core/domain/source/IRemoteSource.kt index 8eab563e..e2271594 100644 --- a/core/domain/src/main/java/com/rtbishop/look4sat/core/domain/source/IRemoteSource.kt +++ b/core/domain/src/main/java/com/rtbishop/look4sat/core/domain/source/IRemoteSource.kt @@ -23,9 +23,6 @@ interface IRemoteSource { suspend fun getFileStream(uri: String): InputStream? suspend fun getNetworkStream(url: String): InputStream? - /** Fetch AMSAT status page HTML (with UA; null = failure) */ - suspend fun getStatusHtml(): String? - /** Fetch AMSAT API catalog (JSON string; null on failure) */ suspend fun getAmSatCatalog(): String?
count