Overhauled data source management, added HTTP status (#242)

core/data/src/main/java/com/rtbishop/look4sat/core/data/repository/DatabaseRepo.kt

@@ -65,18 +65,25 @@ class DatabaseRepo(
     override suspend fun updateFromRemote() = withContext(dispatcher) {
         val settings = settingsRepo.dataSourcesSettings.value
         fun normalizeUrl(url: String) = if (url.startsWith("http")) url else "https://$url"
-        val tleUrls = settings.satelliteUrls.filter { it.isNotBlank() }.map(::normalizeUrl)
-        val radioUrls = settings.transceiversUrls.filter { it.isNotBlank() }.map(::normalizeUrl)
-        // launch all network requests concurrently
-        val tleJobs = tleUrls.map { url -> async { url to remoteSource.getNetworkStream(url) } }
-        val radioJobs = radioUrls.map { url -> async { url to remoteSource.getNetworkStream(url) } }
-        // parse fetched data concurrently
-        val importedEntries = tleJobs.awaitAll().flatMap { (url, stream) ->
-            stream?.let { parseSatelliteStream(url, unwrapIfZipped(url, it)) }.orEmpty()
-        }
-        val importedRadios = radioJobs.awaitAll().flatMap { (url, stream) ->
-            stream?.let { dataParser.parseJSONStream(unwrapIfZipped(url, it)) }.orEmpty()
-        }
+        val tleUrls = settings.satelliteUrls.filterIndexed { i, url -> url.isNotBlank() && settings.isSatelliteEnabled(i) }
+        val radioUrls = settings.transceiversUrls.filterIndexed { i, url -> url.isNotBlank() && settings.isTransceiverEnabled(i) }
+        // launch all network requests concurrently, keeping the raw url as key for status reporting
+        val tleJobs = tleUrls.map { url -> async { url to remoteSource.getNetworkStream(normalizeUrl(url)) } }
+        val radioJobs = radioUrls.map { url -> async { url to remoteSource.getNetworkStream(normalizeUrl(url)) } }
+        val tleResults = tleJobs.awaitAll()
+        val radioResults = radioJobs.awaitAll()
+        // report the HTTP status code of every source (200, 404, ...)
+        settingsRepo.updateDataSourcesStatus(
+            (tleResults + radioResults).associate { (url, result) -> url to result.code }
+        )
+        // parse fetched data concurrently, keeping the first occurrence per primary key
+        // so sources listed higher in the dialog take priority over lower ones
+        val importedEntries = tleResults.flatMap { (url, result) ->
+            result.stream?.let { val nUrl = normalizeUrl(url); parseSatelliteStream(nUrl, unwrapIfZipped(nUrl, it)) }.orEmpty()
+        }.distinctBy { it.catnum }
+        val importedRadios = radioResults.flatMap { (url, result) ->
+            result.stream?.let { val nUrl = normalizeUrl(url); dataParser.parseJSONStream(unwrapIfZipped(nUrl, it)) }.orEmpty()
+        }.filter { it.uuid.isNotBlank() }.distinctBy { it.uuid }
         // insert parsed data into the database
         localSource.insertEntries(importedEntries)
         localSource.insertRadios(importedRadios)

core/data/src/main/java/com/rtbishop/look4sat/core/data/repository/SettingsRepo.kt

@@ -94,6 +94,8 @@ class SettingsRepo(
     private val keyRadarCompassOffsetElev = "radarCompassOffsetElev"
     private val keySatelliteUrls = "satelliteUrls"
     private val keyTransceiversUrls = "transceiversUrls"
+    private val keySatelliteEnabled = "satelliteEnabled"
+    private val keyTransceiversEnabled = "transceiversEnabled"
     private val separatorComma = ","
     private val separatorUrl = "\n"
 
@@ -373,21 +375,73 @@ class SettingsRepo(
     override val dataSourcesSettings: StateFlow<DataSourcesSettings> = _dataSourcesSettings
 
     override fun updateDataSourcesSettings(settings: DataSourcesSettings) {
+        // Normalize the enabled lists so they are positionally aligned with the URL lists.
+        // Missing entries default to enabled (true), keeping the persisted "one flag per URL"
+        // invariant intact even when a default empty list is used to construct the model.
+        val normalized = settings.copy(
+            satelliteEnabled = alignFlags(settings.satelliteUrls, settings.satelliteEnabled),
+            transceiversEnabled = alignFlags(settings.transceiversUrls, settings.transceiversEnabled)
+        )
         preferences.edit {
-            putString(keySatelliteUrls, settings.satelliteUrls.joinToString(separatorUrl))
-            putString(keyTransceiversUrls, settings.transceiversUrls.joinToString(separatorUrl))
+            putString(keySatelliteUrls, normalized.satelliteUrls.joinToString(separatorUrl))
+            putString(keyTransceiversUrls, normalized.transceiversUrls.joinToString(separatorUrl))
+            putString(keySatelliteEnabled, normalized.satelliteEnabled.joinToString(separatorComma))
+            putString(keyTransceiversEnabled, normalized.transceiversEnabled.joinToString(separatorComma))
         }
-        _dataSourcesSettings.value = settings
+        _dataSourcesSettings.value = normalized
     }
 
-    private fun getDataSourcesSettings(): DataSourcesSettings = DataSourcesSettings(
-        satelliteUrls = preferences.getString(keySatelliteUrls, null)
-            ?.split(separatorUrl)?.filter { it.isNotBlank() }
-            ?: Sources.satelliteDataUrls,
-        transceiversUrls = preferences.getString(keyTransceiversUrls, null)
-            ?.split(separatorUrl)?.filter { it.isNotBlank() }
-            ?: Sources.transceiversDataUrls
-    )
+    private fun getDataSourcesSettings(): DataSourcesSettings {
+        val (satUrls, satEnabled) = parseSources(
+            preferences.getString(keySatelliteUrls, null),
+            preferences.getString(keySatelliteEnabled, null),
+            Sources.satelliteDataUrls
+        )
+        val (txUrls, txEnabled) = parseSources(
+            preferences.getString(keyTransceiversUrls, null),
+            preferences.getString(keyTransceiversEnabled, null),
+            Sources.transceiversDataUrls
+        )
+        return DataSourcesSettings(
+            satelliteUrls = satUrls,
+            transceiversUrls = txUrls,
+            satelliteEnabled = satEnabled,
+            transceiversEnabled = txEnabled
+        )
+    }
+
+    private fun parseSources(
+        storedUrls: String?,
+        storedEnabled: String?,
+        defaults: List<String>
+    ): Pair<List<String>, List<Boolean>> {
+        if (storedUrls == null) return defaults to defaults.map { true }
+        val urls = storedUrls.split(separatorUrl)
+        val flags = if (storedEnabled.isNullOrEmpty()) emptyList() else storedEnabled.split(separatorComma)
+        val filteredUrls = mutableListOf<String>()
+        val filteredFlags = mutableListOf<Boolean>()
+        urls.forEachIndexed { index, url ->
+            if (url.isNotBlank()) {
+                filteredUrls.add(url)
+                filteredFlags.add(flags.getOrNull(index)?.toBoolean() ?: true)
+            }
+        }
+        return filteredUrls to filteredFlags
+    }
+
+    private fun alignFlags(urls: List<String>, flags: List<Boolean>): List<Boolean> {
+        if (flags.size >= urls.size) return flags.take(urls.size)
+        return flags + List(urls.size - flags.size) { true }
+    }
+    //endregion
+
+    //region # Data sources status
+    private val _dataSourcesStatus = MutableStateFlow<Map<String, Int>>(emptyMap())
+    override val dataSourcesStatus: StateFlow<Map<String, Int>> = _dataSourcesStatus
+
+    override fun updateDataSourcesStatus(status: Map<String, Int>) {
+        _dataSourcesStatus.value = status
+    }
     //endregion
 
     //region # Radio control settings

core/data/src/main/java/com/rtbishop/look4sat/core/data/source/RemoteSource.kt

@@ -20,6 +20,7 @@ package com.rtbishop.look4sat.core.data.source
 import android.content.ContentResolver
 import androidx.core.net.toUri
 import com.rtbishop.look4sat.core.domain.source.IRemoteSource
+import com.rtbishop.look4sat.core.domain.source.NetworkResult
 import kotlinx.coroutines.CoroutineDispatcher
 import kotlinx.coroutines.withContext
 import okhttp3.OkHttpClient
@@ -42,20 +43,26 @@ class RemoteSource(
         }
     }
 
-    override suspend fun getNetworkStream(url: String): InputStream? = withContext(dispatcher) {
+    override suspend fun getNetworkStream(url: String): NetworkResult = withContext(dispatcher) {
         try {
             val networkRequest = Request.Builder().url(url).build()
             val response = httpClient.newCall(networkRequest).execute()
             if (!response.isSuccessful) {
+                val code = response.code
                 response.close()
-                return@withContext null
+                return@withContext NetworkResult(code, null)
             }
             // Return the body stream directly as the caller is responsible for closing it
             // That returns the connection to OkHttp's pool
-            response.body.byteStream().buffered()
+            val body = response.body
+            if (body == null) {
+                response.close()
+                return@withContext NetworkResult(response.code, null)
+            }
+            NetworkResult(response.code, body.byteStream().buffered())
         } catch (exception: Exception) {
             println("RemoteSource network stream exception: $exception")
-            null
+            NetworkResult(NetworkResult.CONNECTION_ERROR, null)
         }
     }
 

core/data/src/test/java/com/rtbishop/look4sat/core/data/repository/DatabaseRepoTest.kt

@@ -31,6 +31,7 @@ import com.rtbishop.look4sat.core.domain.predict.OrbitalObject
 import com.rtbishop.look4sat.core.domain.repository.ISettingsRepo
 import com.rtbishop.look4sat.core.domain.source.ILocalSource
 import com.rtbishop.look4sat.core.domain.source.IRemoteSource
+import com.rtbishop.look4sat.core.domain.source.NetworkResult
 import com.rtbishop.look4sat.core.domain.utility.DataParser
 import kotlinx.coroutines.ExperimentalCoroutinesApi
 import kotlinx.coroutines.flow.MutableStateFlow
@@ -119,7 +120,10 @@ private class FakeRemoteSource : IRemoteSource {
 
     override suspend fun getFileStream(uri: String): InputStream? = fileStreams[uri]?.invoke()
 
-    override suspend fun getNetworkStream(url: String): InputStream? = networkStreams[url]?.invoke()
+    override suspend fun getNetworkStream(url: String): NetworkResult {
+        val stream = networkStreams[url]?.invoke()
+        return if (stream != null) NetworkResult(200, stream) else NetworkResult(404, null)
+    }
 
     override suspend fun getAmSatCatalog(): String? = null
 
@@ -185,6 +189,8 @@ private class FakeSettingsRepo(dataSources: DataSourcesSettings = defaultDataSou
 
     override val dataSourcesSettings: MutableStateFlow<DataSourcesSettings> = MutableStateFlow(dataSources)
 
+    override val dataSourcesStatus: MutableStateFlow<Map<String, Int>> = MutableStateFlow(emptyMap())
+
     override val radioControlSettings: StateFlow<RadioControlSettings> = MutableStateFlow(
         RadioControlSettings(false, RadioControlSettings.MODEL_YAESU_FT817, "", "", "", "", 9600)
     )
@@ -214,6 +220,10 @@ private class FakeSettingsRepo(dataSources: DataSourcesSettings = defaultDataSou
         dataSourcesSettings.value = settings
     }
 
+    override fun updateDataSourcesStatus(status: Map<String, Int>) {
+        dataSourcesStatus.value = status
+    }
+
     override fun updateRadioControlSettings(settings: RadioControlSettings) = Unit
 
     override fun getSatelliteOffset(catnum: Int): String = ""

core/data/src/test/java/com/rtbishop/look4sat/core/data/repository/SelectionRepoTest.kt

@@ -138,6 +138,8 @@ class SelectionRepoTest {
             DataSourcesSettings(emptyList(), emptyList())
         )
 
+        override val dataSourcesStatus: StateFlow<Map<String, Int>> = MutableStateFlow(emptyMap())
+
         override val radioControlSettings: StateFlow<RadioControlSettings> = MutableStateFlow(
             RadioControlSettings(false, RadioControlSettings.MODEL_YAESU_FT817, "", "", "", "", 9600)
         )
@@ -166,6 +168,8 @@ class SelectionRepoTest {
 
         override fun updateDataSourcesSettings(settings: DataSourcesSettings) = Unit
 
+        override fun updateDataSourcesStatus(status: Map<String, Int>) = Unit
+
         override fun updateRadioControlSettings(settings: RadioControlSettings) = Unit
 
         override fun getSatelliteOffset(catnum: Int): String = ""