Fetching satellite data asynchronously, increased build speed

core/src/main/java/com/rtbishop/look4sat/data/DefaultRepository.kt

@@ -20,28 +20,28 @@ package com.rtbishop.look4sat.data
 import com.rtbishop.look4sat.domain.DataParser
 import com.rtbishop.look4sat.domain.DataRepository
 import com.rtbishop.look4sat.domain.model.SatEntry
-import kotlinx.coroutines.CoroutineDispatcher
+import kotlinx.coroutines.Deferred
+import kotlinx.coroutines.async
 import kotlinx.coroutines.coroutineScope
 import kotlinx.coroutines.launch
-import kotlinx.coroutines.withContext
 import java.io.InputStream
 import java.util.zip.ZipInputStream
+import kotlin.system.measureTimeMillis
 
 class DefaultRepository(
     private val dataParser: DataParser,
     private val localSource: LocalDataSource,
-    private val remoteSource: RemoteDataSource,
-    private val repoDispatcher: CoroutineDispatcher
+    private val remoteSource: RemoteDataSource
 ) : DataRepository {
 
     override val defaultSelection = listOf(43700, 25544, 25338, 28654, 33591, 40069, 27607, 24278)
     override val defaultSources = listOf(
-        "https://celestrak.com/NORAD/elements/active.txt",
+        "https://celestrak.com/NORAD/elements/gp.php?GROUP=active&FORMAT=csv",
         "https://amsat.org/tle/current/nasabare.txt",
         "https://www.prismnet.com/~mmccants/tles/classfd.zip",
         "https://www.prismnet.com/~mmccants/tles/inttles.zip"
     )
-    override val transmittersSource = "https://db.satnogs.org/api/transmitters/"
+    override val transmittersSource = "https://db.satnogs.org/api/transmitters/?format=json"
 
     override fun getSatelliteItems() = localSource.getSatelliteItems()
 
@@ -49,36 +49,46 @@ class DefaultRepository(
 
     override suspend fun getTransmitters(catnum: Int) = localSource.getTransmitters(catnum)
 
-    override suspend fun getWebSources() = localSource.getWebSources().also { sources ->
+    override suspend fun getWebSources() = localSource.getSources().also { sources ->
         return if (sources.isNotEmpty()) sources
         else defaultSources
     }
 
-    override suspend fun updateDataFromFile(stream: InputStream) = withContext(repoDispatcher) {
+    override suspend fun updateDataFromFile(stream: InputStream) {
         localSource.updateEntries(importSatellites(stream))
     }
 
     override suspend fun updateDataFromWeb(sources: List<String>) {
         coroutineScope {
-            launch(repoDispatcher) {
-                localSource.updateWebSources(sources)
+            launch {
+                localSource.updateSources(sources)
             }
-            launch(repoDispatcher) {
-                val streams = mutableListOf<InputStream>()
-                val entries = mutableListOf<SatEntry>()
-                sources.forEach { source ->
-                    val fileStream = remoteSource.fetchFileStream(source)
-                    if (source.contains(".zip", true)) {
-                        val zipStream = ZipInputStream(fileStream).apply { nextEntry }
-                        streams.add(zipStream)
-                    } else {
-                        streams.add(fileStream)
+            launch {
+                val updateTimeMillis = measureTimeMillis {
+                    val fetchesMap = mutableMapOf<String, Deferred<InputStream>>()
+                    val streamsMap = mutableMapOf<String, InputStream>()
+                    val streams = mutableListOf<InputStream>()
+                    val entries = mutableListOf<SatEntry>()
+                    sources.forEach { fetchesMap[it] = async { remoteSource.fetchFileStream(it) } }
+                    fetchesMap.forEach { streamsMap[it.key] = it.value.await() }
+                    streamsMap.forEach { stream ->
+                        when {
+                            stream.key.contains("=csv", true) -> {
+                                val tles = dataParser.parseCSVStream(stream.value)
+                                entries.addAll(tles.map { tle -> SatEntry(tle) })
+                            }
+                            stream.key.contains(".zip", true) -> {
+                                streams.add(ZipInputStream(stream.value).apply { nextEntry })
+                            }
+                            else -> streams.add(stream.value)
+                        }
                     }
+                    streams.forEach { stream -> entries.addAll(importSatellites(stream)) }
+                    localSource.updateEntries(entries)
                 }
-                streams.forEach { stream -> entries.addAll(importSatellites(stream)) }
-                localSource.updateEntries(entries)
+                println("Update from web took $updateTimeMillis ms")
             }
-            launch(repoDispatcher) {
+            launch {
                 val jsonStream = remoteSource.fetchFileStream(transmittersSource)
                 val transmitters = dataParser.parseJSONStream(jsonStream)
                 localSource.updateTransmitters(transmitters)
@@ -87,7 +97,7 @@ class DefaultRepository(
     }
 
     override suspend fun updateSelection(catnums: List<Int>, isSelected: Boolean) {
-        localSource.updateSelection(catnums, isSelected)
+        localSource.updateEntriesSelection(catnums, isSelected)
     }
 
     private suspend fun importSatellites(stream: InputStream): List<SatEntry> {

core/src/main/java/com/rtbishop/look4sat/data/LocalDataSource.kt

@@ -29,15 +29,15 @@ interface LocalDataSource {
 
     suspend fun getSelectedSatellites(): List<Satellite>
 
-    suspend fun getTransmitters(catnum: Int): List<Transmitter>
+    suspend fun getSources(): List<String>
 
-    suspend fun getWebSources(): List<String>
+    suspend fun getTransmitters(catnum: Int): List<Transmitter>
 
     suspend fun updateEntries(entries: List<SatEntry>)
 
-    suspend fun updateSelection(catnums: List<Int>, isSelected: Boolean)
+    suspend fun updateEntriesSelection(catnums: List<Int>, isSelected: Boolean)
+
+    suspend fun updateSources(sources: List<String>)
 
     suspend fun updateTransmitters(transmitters: List<Transmitter>)
-
-    suspend fun updateWebSources(sources: List<String>)
 }