From ce8c2a4834bd8c0cfd07f762b0982185561a1432 Mon Sep 17 00:00:00 2001 From: Arty Bishop Date: Mon, 4 May 2026 13:30:09 +0100 Subject: [PATCH] Added zipped data sources handling to DatabaseRepo --- .../core/data/repository/DatabaseRepo.kt | 66 +++++++++---------- .../look4sat/core/domain/source/Sources.kt | 4 +- .../core/domain/utility/DataParser.kt | 5 +- 3 files changed, 35 insertions(+), 40 deletions(-) diff --git a/core/data/src/main/java/com/rtbishop/look4sat/core/data/repository/DatabaseRepo.kt b/core/data/src/main/java/com/rtbishop/look4sat/core/data/repository/DatabaseRepo.kt index 3fbabeea..957d5b1d 100644 --- a/core/data/src/main/java/com/rtbishop/look4sat/core/data/repository/DatabaseRepo.kt +++ b/core/data/src/main/java/com/rtbishop/look4sat/core/data/repository/DatabaseRepo.kt @@ -40,28 +40,22 @@ class DatabaseRepo( private val settingsRepo: ISettingsRepo ) : IDatabaseRepo { - private companion object { - val tleTypes = setOf("Amsat", "R4UAB", "Other") - val zippedTleTypes = setOf("McCants", "Classified") - } + private val customSourceType = "Other" override suspend fun updateTLEFromFile(uri: String) = withContext(dispatcher) { remoteSource.getFileStream(uri)?.let { stream -> - val entries = dataParser.parseTLEStream(stream) + val entries = dataParser.parseTLEStream(unwrapIfZipped(uri, stream)) localSource.insertEntries(entries) - settingsRepo.setSatelliteTypeIds("Other", entries.map { it.catnum }) + settingsRepo.setSatelliteTypeIds(customSourceType, entries.map { it.catnum }) } setUpdateSuccessful(System.currentTimeMillis()) } override suspend fun updateTransceiversFromFile(uri: String) = withContext(dispatcher) { - remoteSource.getFileStream(uri) - ?.let { dataParser.parseJSONStream(it) } - ?.takeIf { it.isNotEmpty() } - ?.let { - localSource.deleteRadios() - localSource.insertRadios(it) - } + remoteSource.getFileStream(uri)?.let { stream -> + val transceivers = dataParser.parseJSONStream(unwrapIfZipped(uri, stream)) + localSource.insertRadios(transceivers) + } setUpdateSuccessful(System.currentTimeMillis()) } @@ -69,28 +63,26 @@ class DatabaseRepo( val dataSourcesSettings = settingsRepo.dataSourcesSettings.value val tleUrls = buildMap { putAll(Sources.satelliteDataUrls) - if (dataSourcesSettings.useCustomTLE) put("Other", dataSourcesSettings.tleUrl) - }.filterValues { it.isNotEmpty() } - - val radioUrls = buildList { - add(Sources.RADIO_DATA_URL) - if (dataSourcesSettings.useCustomTransceivers) add(dataSourcesSettings.transceiversUrl) - } - + if (dataSourcesSettings.useCustomTLE) put(customSourceType, dataSourcesSettings.tleUrl) + }.filterValues { it.isNotBlank() } + val radioUrls = buildMap { + putAll(Sources.transceiversDataUrls) + if (dataSourcesSettings.useCustomTransceivers) put(customSourceType, dataSourcesSettings.transceiversUrl) + }.filterValues { it.isNotBlank() } // launch all network requests concurrently - val tleJobs = tleUrls.map { (type, url) -> async { type to remoteSource.getNetworkStream(url) } } - val radioJobs = radioUrls.map { url -> async { remoteSource.getNetworkStream(url) } } - - // parse satellite data - val importedEntries = tleJobs.awaitAll().flatMap { (type, stream) -> - stream?.let { parseSatelliteStream(type, it) }.orEmpty().also { satellites -> - settingsRepo.setSatelliteTypeIds(type, satellites.map { it.catnum }) + val tleJobs = tleUrls.values.map { url -> async { url to remoteSource.getNetworkStream(url) } } + val radioJobs = radioUrls.values.map { url -> async { url to remoteSource.getNetworkStream(url) } } + // parse fetched data concurrently and associate with types + val importedEntries = tleJobs.awaitAll().flatMap { (url, stream) -> + val type = tleUrls.entries.find { it.value == url }?.key ?: customSourceType + stream?.let { parseSatelliteStream(url, unwrapIfZipped(url, it)) }.orEmpty().also { entries -> + settingsRepo.setSatelliteTypeIds(type, entries.map { it.catnum }) } } - - // parse radio data - val importedRadios = radioJobs.awaitAll().filterNotNull().flatMap { dataParser.parseJSONStream(it) } - + val importedRadios = radioJobs.awaitAll().flatMap { (url, stream) -> + stream?.let { dataParser.parseJSONStream(unwrapIfZipped(url, it)) }.orEmpty() + } + // insert parsed data into the database localSource.insertEntries(importedEntries) localSource.insertRadios(importedRadios) setUpdateSuccessful(System.currentTimeMillis()) @@ -102,10 +94,9 @@ class DatabaseRepo( setUpdateSuccessful(0L) } - private suspend fun parseSatelliteStream(type: String, stream: InputStream): List = when (type) { - in tleTypes -> dataParser.parseTLEStream(stream) - in zippedTleTypes -> dataParser.parseTLEStream(ZipInputStream(stream).apply { nextEntry }) - else -> dataParser.parseCSVStream(stream) + private suspend fun parseSatelliteStream(url: String, stream: InputStream): List = when { + url.contains("FORMAT=csv", ignoreCase = true) -> dataParser.parseCSVStream(stream) + else -> dataParser.parseTLEStream(stream) } private suspend fun setUpdateSuccessful(timestamp: Long) { @@ -113,4 +104,7 @@ class DatabaseRepo( DatabaseState(localSource.getRadiosTotal(), localSource.getEntriesTotal(), timestamp) ) } + + private fun unwrapIfZipped(url: String, stream: InputStream): InputStream = + if (url.endsWith(".zip", ignoreCase = true)) ZipInputStream(stream).apply { nextEntry } else stream } diff --git a/core/domain/src/main/java/com/rtbishop/look4sat/core/domain/source/Sources.kt b/core/domain/src/main/java/com/rtbishop/look4sat/core/domain/source/Sources.kt index f662a2d5..17c91e34 100644 --- a/core/domain/src/main/java/com/rtbishop/look4sat/core/domain/source/Sources.kt +++ b/core/domain/src/main/java/com/rtbishop/look4sat/core/domain/source/Sources.kt @@ -18,7 +18,6 @@ package com.rtbishop.look4sat.core.domain.source object Sources { - const val RADIO_DATA_URL = "https://db.satnogs.org/api/transmitters/?format=json&status=active" val satelliteDataUrls = mapOf( "All" to "https://celestrak.org/NORAD/elements/gp.php?GROUP=active&FORMAT=csv", "Amateur" to "https://celestrak.org/NORAD/elements/gp.php?GROUP=amateur&FORMAT=csv", @@ -49,4 +48,7 @@ object Sources { "R4UAB" to "https://r4uab.ru/satonline.txt", "Other" to "" // key for sats filter ) + val transceiversDataUrls = mapOf( + "SatNOGS" to "https://db.satnogs.org/api/transmitters/?format=json&status=active" + ) } diff --git a/core/domain/src/main/java/com/rtbishop/look4sat/core/domain/utility/DataParser.kt b/core/domain/src/main/java/com/rtbishop/look4sat/core/domain/utility/DataParser.kt index dda7d4c5..9cada20d 100644 --- a/core/domain/src/main/java/com/rtbishop/look4sat/core/domain/utility/DataParser.kt +++ b/core/domain/src/main/java/com/rtbishop/look4sat/core/domain/utility/DataParser.kt @@ -58,8 +58,6 @@ class DataParser(private val dispatcher: CoroutineDispatcher) { }.getOrDefault(emptyList()) } - fun isLeapYear(year: Int): Boolean = (year % 4 == 0 && year % 100 != 0) || year % 400 == 0 - private fun parseCSV(values: List): OrbitalData? = runCatching { val name = values[0] val timestamp = values[2] @@ -106,7 +104,8 @@ class DataParser(private val dispatcher: CoroutineDispatcher) { }.onFailure { println("TLE parsing exception: $it") }.getOrNull() private fun getDayOfYear(year: Int, month: Int, dayOfMonth: Int): Int { - val daysInMonth = intArrayOf(31, if (isLeapYear(year)) 29 else 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31) + val isLeapYear = (year % 4 == 0 && year % 100 != 0) || year % 400 == 0 + val daysInMonth = intArrayOf(31, if (isLeapYear) 29 else 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31) return daysInMonth.take(month - 1).sum() + dayOfMonth } }