chore(amsat): remove the HTML scraping path superseded by the JSON API
The merged upstream AMSAT implementation fetches status data from AMSAT's JSON endpoints (getAmSatCatalog / getAmSatReports), so the fork's HTML scraping path no longer has a caller: - core/data/.../source/AmSatParser.kt (136 lines): parsed the amsat.org status table, deriving state from the page's inline colour codes. - IRemoteSource.getStatusHtml() plus its RemoteSource implementation and the DatabaseRepoTest fake override. Verified zero references repo-wide before removing, and again afterwards. Request / CancellationException imports in RemoteSource remain in use by the other fetchers. compileReleaseKotlin plus core:domain / core:data / feature:map / feature:roaming unit tests stay green.
This commit is contained in:
mckero committed
1 parent
57d6f9d7ed
commit
40ba3fecdf
4 files changed
-159
No files matched your search
@@ -1,136 +0,0 @@
|
|||||||
package com.rtbishop.look4sat.core.data.source
|
|
||||||
|
|
||||||
import com.rtbishop.look4sat.core.domain.model.SatDay
|
|
||||||
import com.rtbishop.look4sat.core.domain.model.SatReport
|
|
||||||
import com.rtbishop.look4sat.core.domain.model.SatSlot
|
|
||||||
import com.rtbishop.look4sat.core.domain.model.SatStatus
|
|
||||||
import com.rtbishop.look4sat.core.domain.model.SatStatusPage
|
|
||||||
import java.util.regex.Pattern
|
|
||||||
|
|
||||||
/**
|
|
||||||
* AMSAT satellite status page parser (https://amsat.org/status/)
|
|
||||||
*
|
|
||||||
* Page structure (static HTML, verified 2026-08):
|
|
||||||
* - Status table: 48 rows in <table>, header = Name + 6 days (each colspan=12)
|
|
||||||
* Data row has 73 cells: [0]=satellite name, [1..72] = 6 days x 12 two-hour slots (new->old)
|
|
||||||
* Each cell: <td width=9 bgcolor="color">count</td>; when reports exist the count carries a
|
|
||||||
* docTips.show('id') link
|
|
||||||
* - Report details live in the page's inline JS:
|
|
||||||
* tips.a885153 = new Array(5,5,120,'status<br>callsign<br>grid<br>date<br>time span UTC')
|
|
||||||
*
|
|
||||||
* Status colors: #648fff=active, #ffb000=telemetry only, #dc267f=not heard, #fe6100=conflicting
|
|
||||||
*/
|
|
||||||
object AmSatParser {
|
|
||||||
|
|
||||||
private val STATUS_COLORS = mapOf(
|
|
||||||
"#648fff" to 0xFF648FFF.toLong(),
|
|
||||||
"#ffb000" to 0xFFFFB000.toLong(),
|
|
||||||
"#dc267f" to 0xFFDC267F.toLong(),
|
|
||||||
"#fe6100" to 0xFFFE6100.toLong()
|
|
||||||
)
|
|
||||||
private val GRAY = 0xFFC0C0C0.toLong()
|
|
||||||
|
|
||||||
private val rowRe = Pattern.compile("<tr>(.*?)</tr>", Pattern.DOTALL)
|
|
||||||
private val cellRe = Pattern.compile("(<t[dh][^>]*>.*?</t[dh]>)", Pattern.DOTALL)
|
|
||||||
private val bgRe = Pattern.compile("bgcolor=\"?(#[0-9a-fA-F]{6}|C0C0C0)\"?")
|
|
||||||
private val linkRe = Pattern.compile("docTips\\.show\\('(a\\d+)'\\)")
|
|
||||||
private val tipRe = Pattern.compile(
|
|
||||||
"tips\\.(a\\d+)\\s*=\\s*new\\s+Array\\(\\s*\\d+,\\s*\\d+,\\s*\\d+,\\s*'([^']*)'"
|
|
||||||
)
|
|
||||||
|
|
||||||
/** Parse the full page */
|
|
||||||
fun parse(html: String, fetchedAtUtcMs: Long): SatStatusPage {
|
|
||||||
val reports = parseReports(html)
|
|
||||||
val statuses = parseStatusTable(html, reports)
|
|
||||||
return SatStatusPage(fetchedAtUtcMs, statuses, reports)
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Extract all reports (tips.* JS arrays) */
|
|
||||||
fun parseReports(html: String): Map<String, SatReport> {
|
|
||||||
val map = mutableMapOf<String, SatReport>()
|
|
||||||
val m = tipRe.matcher(html)
|
|
||||||
while (m.find()) {
|
|
||||||
val id = m.group(1)
|
|
||||||
val parts = m.group(2).split("<br>")
|
|
||||||
map[id] = SatReport(
|
|
||||||
id = id,
|
|
||||||
statusText = parts.getOrElse(0) { "" }.trim(),
|
|
||||||
call = parts.getOrElse(1) { "" }.trim(),
|
|
||||||
grid = parts.getOrElse(2) { "" }.trim(),
|
|
||||||
dateUtc = parts.getOrElse(3) { "" }.trim(),
|
|
||||||
timeUtc = parts.getOrElse(4) { "" }.trim()
|
|
||||||
)
|
|
||||||
}
|
|
||||||
return map
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Parse the status table (48 satellite rows) */
|
|
||||||
fun parseStatusTable(html: String, reports: Map<String, SatReport>): List<SatStatus> {
|
|
||||||
val result = mutableListOf<SatStatus>()
|
|
||||||
val tables = extractTables(html)
|
|
||||||
for (table in tables) {
|
|
||||||
val rows = rowRe.matcher(table)
|
|
||||||
val parsed = mutableListOf<SatStatus>()
|
|
||||||
var rowIndex = 0
|
|
||||||
var dayHeaders: List<String> = emptyList()
|
|
||||||
while (rows.find()) {
|
|
||||||
val rowHtml = rows.group(1)
|
|
||||||
val cells = cellRe.matcher(rowHtml)
|
|
||||||
val cellList = mutableListOf<String>()
|
|
||||||
while (cells.find()) cellList.add(cells.group(1))
|
|
||||||
if (rowIndex == 0) {
|
|
||||||
// Header: Name + 6 days (each colspan=12)
|
|
||||||
dayHeaders = cellList.drop(1).take(6).map { stripHtml(it) }
|
|
||||||
rowIndex++
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if (cellList.size < 7) { rowIndex++; continue }
|
|
||||||
val name = stripHtml(cellList[0])
|
|
||||||
if (name.isBlank()) { rowIndex++; continue }
|
|
||||||
val days = mutableListOf<SatDay>()
|
|
||||||
for (d in 0 until 6) {
|
|
||||||
val start = 1 + d * 12
|
|
||||||
val end = start + 12
|
|
||||||
val slots = (start until end).mapNotNull { i ->
|
|
||||||
cellList.getOrNull(i)?.let { cell ->
|
|
||||||
val bg = bgRe.matcher(cell)
|
|
||||||
val color = if (bg.find()) {
|
|
||||||
STATUS_COLORS[bg.group(1).lowercase()] ?: GRAY
|
|
||||||
} else GRAY
|
|
||||||
val link = linkRe.matcher(cell)
|
|
||||||
val ids = mutableListOf<String>()
|
|
||||||
while (link.find()) ids.add(link.group(1))
|
|
||||||
val hasReport = ids.isNotEmpty()
|
|
||||||
val count = if (hasReport) {
|
|
||||||
stripHtml(cell).trim().toIntOrNull() ?: ids.size
|
|
||||||
} else 0
|
|
||||||
SatSlot(
|
|
||||||
statusColor = if (hasReport) color else GRAY,
|
|
||||||
count = count,
|
|
||||||
reportIds = ids
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
days.add(SatDay(dayHeaders.getOrElse(d) { "" }, slots))
|
|
||||||
}
|
|
||||||
parsed.add(SatStatus(name, days))
|
|
||||||
rowIndex++
|
|
||||||
}
|
|
||||||
if (parsed.isNotEmpty()) {
|
|
||||||
result.addAll(parsed)
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun extractTables(html: String): List<String> {
|
|
||||||
val result = mutableListOf<String>()
|
|
||||||
val m = Pattern.compile("<table.*?</table>", Pattern.DOTALL).matcher(html)
|
|
||||||
while (m.find()) result.add(m.group())
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun stripHtml(s: String): String =
|
|
||||||
s.replace(Regex("<[^>]+>"), "").trim()
|
|
||||||
}
|
|
||||||
@@ -45,24 +45,6 @@ class RemoteSource(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
override suspend fun getStatusHtml(): String? = withContext(dispatcher) {
|
|
||||||
try {
|
|
||||||
val request = Request.Builder()
|
|
||||||
.url("https://amsat.org/status/")
|
|
||||||
.header("User-Agent", "Mozilla/5.0 (Linux; Android 13) Look4Sat/4.5")
|
|
||||||
.build()
|
|
||||||
httpClient.newCall(request).execute().use { response ->
|
|
||||||
if (!response.isSuccessful) return@use null
|
|
||||||
response.body?.string()
|
|
||||||
}
|
|
||||||
} catch (exception: CancellationException) {
|
|
||||||
throw exception
|
|
||||||
} catch (exception: Exception) {
|
|
||||||
println("RemoteSource amsat status exception: $exception")
|
|
||||||
null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
override suspend fun getNetworkStream(url: String): InputStream? = withContext(dispatcher) {
|
override suspend fun getNetworkStream(url: String): InputStream? = withContext(dispatcher) {
|
||||||
try {
|
try {
|
||||||
val networkRequest = Request.Builder().url(url).build()
|
val networkRequest = Request.Builder().url(url).build()
|
||||||
|
|||||||
@@ -126,8 +126,6 @@ private class FakeRemoteSource : IRemoteSource {
|
|||||||
|
|
||||||
override suspend fun getNetworkStream(url: String): InputStream? = networkStreams[url]?.invoke()
|
override suspend fun getNetworkStream(url: String): InputStream? = networkStreams[url]?.invoke()
|
||||||
|
|
||||||
override suspend fun getStatusHtml(): String? = null
|
|
||||||
|
|
||||||
override suspend fun getAmSatCatalog(): String? = null
|
override suspend fun getAmSatCatalog(): String? = null
|
||||||
|
|
||||||
override suspend fun getAmSatReports(hours: Int, limit: Int): String? = null
|
override suspend fun getAmSatReports(hours: Int, limit: Int): String? = null
|
||||||
|
|||||||
@@ -23,9 +23,6 @@ interface IRemoteSource {
|
|||||||
suspend fun getFileStream(uri: String): InputStream?
|
suspend fun getFileStream(uri: String): InputStream?
|
||||||
suspend fun getNetworkStream(url: String): InputStream?
|
suspend fun getNetworkStream(url: String): InputStream?
|
||||||
|
|
||||||
/** Fetch AMSAT status page HTML (with UA; null = failure) */
|
|
||||||
suspend fun getStatusHtml(): String?
|
|
||||||
|
|
||||||
/** Fetch AMSAT API catalog (JSON string; null on failure) */
|
/** Fetch AMSAT API catalog (JSON string; null on failure) */
|
||||||
suspend fun getAmSatCatalog(): String?
|
suspend fun getAmSatCatalog(): String?
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user