Added tweaks to DataParser.kt and DatabaseRepo.kt

core/data/src/main/java/com/rtbishop/look4sat/core/data/repository/DatabaseRepo.kt

@@ -18,7 +18,6 @@
 package com.rtbishop.look4sat.core.data.repository
 
 import com.rtbishop.look4sat.core.domain.model.DatabaseState
-import com.rtbishop.look4sat.core.domain.model.SatRadio
 import com.rtbishop.look4sat.core.domain.predict.OrbitalData
 import com.rtbishop.look4sat.core.domain.repository.IDatabaseRepo
 import com.rtbishop.look4sat.core.domain.repository.ISettingsRepo
@@ -28,7 +27,9 @@ import com.rtbishop.look4sat.core.domain.source.Sources
 import com.rtbishop.look4sat.core.domain.utility.DataParser
 import kotlinx.coroutines.CoroutineDispatcher
 import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
 import kotlinx.coroutines.withContext
+import java.io.InputStream
 import java.util.zip.ZipInputStream
 
 class DatabaseRepo(
@@ -39,78 +40,55 @@ class DatabaseRepo(
     private val settingsRepo: ISettingsRepo
 ) : IDatabaseRepo {
 
+    private companion object {
+        val tleTypes = setOf("Amsat", "R4UAB", "Other")
+        val zippedTleTypes = setOf("McCants", "Classified")
+    }
+
     override suspend fun updateTLEFromFile(uri: String) = withContext(dispatcher) {
-        val importedSatellites = remoteSource.getFileStream(uri)?.let { dataParser.parseTLEStream(it) }
+        remoteSource.getFileStream(uri)?.let {
-        importedSatellites?.let { localSource.insertEntries(it) }
+            localSource.insertEntries(dataParser.parseTLEStream(it))
+        }
         setUpdateSuccessful(System.currentTimeMillis())
     }
 
     override suspend fun updateTransceiversFromFile(uri: String) = withContext(dispatcher) {
-        val radios = remoteSource.getFileStream(uri)?.let { dataParser.parseJSONStream(it) }
+        remoteSource.getFileStream(uri)
-        radios?.let {
+            ?.let { dataParser.parseJSONStream(it) }
-            if(it.isNotEmpty()) {
+            ?.takeIf { it.isNotEmpty() }
+            ?.let {
                 localSource.deleteRadios()
                 localSource.insertRadios(it)
             }
-        }
         setUpdateSuccessful(System.currentTimeMillis())
     }
 
     override suspend fun updateFromRemote() = withContext(dispatcher) {
-        val importedEntries = mutableListOf<OrbitalData>()
+        val dataSourcesSettings = settingsRepo.dataSourcesSettings.value
-        val importedRadios = mutableListOf<SatRadio>()
+        val tleUrls = buildMap {
-        val tleUrls = Sources.satelliteDataUrls.toMutableMap().apply {
+            putAll(Sources.satelliteDataUrls)
-            if (settingsRepo.dataSourcesSettings.value.useCustomTLE) {
+            if (dataSourcesSettings.useCustomTLE) put("Other", dataSourcesSettings.tleUrl)
-                this["Other"] = settingsRepo.dataSourcesSettings.value.tleUrl
+        }.filterValues { it.isNotEmpty() }
-            }
+
-        }
-        val jobsMap = tleUrls
-            .filter { (_, url) -> url.isNotEmpty() }
-            .mapValues { (_, url) -> async { remoteSource.getNetworkStream(url) } }
         val radioUrls = buildList {
             add(Sources.RADIO_DATA_URL)
-            if (settingsRepo.dataSourcesSettings.value.useCustomTransceivers) {
+            if (dataSourcesSettings.useCustomTransceivers) add(dataSourcesSettings.transceiversUrl)
-                add(settingsRepo.dataSourcesSettings.value.transceiversUrl)
-            }
         }
-        val jobRadios = radioUrls.associateWith { url -> async { remoteSource.getNetworkStream(url) } }
-        // parse
-        jobsMap.mapValues { job -> job.value.await() }.forEach { entry ->
-            entry.value?.let { stream ->
-                when (val type = entry.key) {
-                    "Amsat", "R4UAB", "Other" -> {
-                        // parse tle stream
-                        val satellites = dataParser.parseTLEStream(stream)
-                        val catnums = satellites.map { it.catnum }
-                        settingsRepo.setSatelliteTypeIds(type, catnums)
-                        importedEntries.addAll(satellites)
-                    }
 
-                    "McCants", "Classified" -> {
+        // launch all network requests concurrently
-                        // unzip and parse tle stream
+        val tleJobs = tleUrls.map { (type, url) -> async { type to remoteSource.getNetworkStream(url) } }
-                        val unzipped = ZipInputStream(stream).apply { nextEntry }
+        val radioJobs = radioUrls.map { url -> async { remoteSource.getNetworkStream(url) } }
-                        val satellites = dataParser.parseTLEStream(unzipped)
-                        val catnums = satellites.map { it.catnum }
-                        settingsRepo.setSatelliteTypeIds(type, catnums)
-                        importedEntries.addAll(satellites)
-                    }
 
-                    else -> {
+        // parse satellite data
-                        // parse csv stream
+        val importedEntries = tleJobs.awaitAll().flatMap { (type, stream) ->
-                        val satellites = dataParser.parseCSVStream(stream)
+            stream?.let { parseSatelliteStream(type, it) }.orEmpty().also { satellites ->
-                        val catnums = satellites.map { it.catnum }
+                settingsRepo.setSatelliteTypeIds(type, satellites.map { it.catnum })
-                        settingsRepo.setSatelliteTypeIds(type, catnums)
-                        importedEntries.addAll(satellites)
-                    }
-                }
             }
         }
-        jobRadios.values.forEach { job ->
+
-            job.await()?.let {
+        // parse radio data
-                importedRadios.addAll(dataParser.parseJSONStream(it))
+        val importedRadios = radioJobs.awaitAll().filterNotNull().flatMap { dataParser.parseJSONStream(it) }
-            }
+
-        }
-        // insert
         localSource.insertEntries(importedEntries)
         localSource.insertRadios(importedRadios)
         setUpdateSuccessful(System.currentTimeMillis())
@@ -122,9 +100,15 @@ class DatabaseRepo(
         setUpdateSuccessful(0L)
     }
 
-    private suspend fun setUpdateSuccessful(timestamp: Long) = withContext(dispatcher) {
+    private suspend fun parseSatelliteStream(type: String, stream: InputStream): List<OrbitalData> = when (type) {
-        val numberOfRadios = localSource.getRadiosTotal()
+        in tleTypes -> dataParser.parseTLEStream(stream)
-        val numberOfSatellites = localSource.getEntriesTotal()
+        in zippedTleTypes -> dataParser.parseTLEStream(ZipInputStream(stream).apply { nextEntry })
-        settingsRepo.updateDatabaseState(DatabaseState(numberOfRadios, numberOfSatellites, timestamp))
+        else -> dataParser.parseCSVStream(stream)
+    }
+
+    private suspend fun setUpdateSuccessful(timestamp: Long) {
+        settingsRepo.updateDatabaseState(
+            DatabaseState(localSource.getRadiosTotal(), localSource.getEntriesTotal(), timestamp)
+        )
     }
 }

core/domain/src/main/java/com/rtbishop/look4sat/core/domain/utility/DataParser.kt

@@ -29,126 +29,94 @@ import kotlin.math.pow
 class DataParser(private val dispatcher: CoroutineDispatcher) {
 
     suspend fun parseCSVStream(stream: InputStream): List<OrbitalData> = withContext(dispatcher) {
-        val parsedItems = mutableListOf<OrbitalData>()
         stream.bufferedReader().useLines { lines ->
-            lines.forEachIndexed { index, line ->
+            lines.drop(1).mapNotNull { parseCSV(it.split(",")) }.toList()
-                if (index != 0) {
-                    val values = line.split(",")
-                    parseCSV(values)?.let { tle -> parsedItems.add(tle) }
-                }
-            }
         }
-        return@withContext parsedItems
     }
 
     suspend fun parseTLEStream(stream: InputStream): List<OrbitalData> = withContext(dispatcher) {
-        val tleStrings = mutableListOf(String(), String(), String())
+        stream.bufferedReader().readLines()
-        val parsedItems = mutableListOf<OrbitalData>()
+            .chunked(3)
-        var lineIndex = 0
+            .filter { it.size == 3 && it[1].startsWith("1") && it[2].startsWith("2") }
-        stream.bufferedReader().forEachLine { line ->
+            .mapNotNull { parseTLE(it) }
-            tleStrings[lineIndex] = line
-            if (lineIndex < 2) {
-                lineIndex++
-            } else {
-                val isLineOneValid = tleStrings[1].substring(0, 1) == "1"
-                val isLineTwoValid = tleStrings[2].substring(0, 1) == "2"
-                if (!isLineOneValid && !isLineTwoValid) return@forEachLine
-                parseTLE(tleStrings)?.let { tle -> parsedItems.add(tle) }
-                lineIndex = 0
-            }
-        }
-        return@withContext parsedItems
     }
 
     suspend fun parseJSONStream(stream: InputStream): List<SatRadio> = withContext(dispatcher) {
-        val parsedItems = mutableListOf<SatRadio>()
+        runCatching {
-        try {
             val jsonArray = JSONArray(stream.bufferedReader().readText())
-            for (index in 0 until jsonArray.length()) {
+            (0 until jsonArray.length()).mapNotNull { parseJSON(jsonArray.getJSONObject(it)) }
-                val jsonObject = jsonArray.getJSONObject(index)
+        }.getOrDefault(emptyList())
-                parseJSON(jsonObject)?.let { parsedItems.add(it) }
-            }
-            return@withContext parsedItems
-        } catch (_: Exception) {
-            return@withContext parsedItems
-        }
     }
 
-    fun isLeapYear(year: Int): Boolean {
+    fun isLeapYear(year: Int): Boolean = (year % 4 == 0 && year % 100 != 0) || year % 400 == 0
-        return ((year % 4 == 0) && (year % 100 != 0)) || (year % 400 == 0)
-    }
 
-    private fun parseCSV(values: List<String>): OrbitalData? = try {
+    private fun parseCSV(values: List<String>): OrbitalData? = runCatching {
         val name = values[0]
-        val year = values[2].substring(0, 4)
+        val timestamp = values[2]
-        val month = values[2].substring(5, 7)
+        val year = timestamp.substring(0, 4)
-        val dayOfMonth = values[2].substring(8, 10)
+        val month = timestamp.substring(5, 7).toInt()
-        val dayInt = getDayOfYear(year.toInt(), month.toInt(), dayOfMonth.toInt())
+        val dayOfMonth = timestamp.substring(8, 10).toInt()
-        val day = if (dayInt < 10) "00$dayInt" else if (dayInt < 100) "0$dayInt" else "$dayInt"
+        val dayInt = getDayOfYear(year.toInt(), month, dayOfMonth)
-        val hour = values[2].substring(11, 13).toInt() * 3600000 // ms in one hour
+        val day = dayInt.toString().padStart(3, '0')
-        val min = values[2].substring(14, 16).toInt() * 60000 // ms in one minute
+        val hour = timestamp.substring(11, 13).toInt() * 3600000
-        val sec = values[2].substring(17, 19).toInt() * 1000 // ms in one second
+        val min = timestamp.substring(14, 16).toInt() * 60000
-        val ms = values[2].substring(20, 26).toInt() / 1000.0 // microseconds to ms
+        val sec = timestamp.substring(17, 19).toInt() * 1000
-        val frac = ((hour + min + sec + ms) / 86400000.0).toString()
+        val ms = timestamp.substring(20, 26).toInt() / 1000.0
-        val epoch = "${year.substring(2)}$day${frac.substring(1)}".toDouble()
+        val frac = ((hour + min + sec + ms) / 86400000.0).toString().substring(1)
-        val meanmo = values[3].toDouble()
+        val epoch = "${year.substring(2)}$day$frac".toDouble()
-        val eccn = values[4].toDouble()
+        OrbitalData(
-        val incl = values[5].toDouble()
+            name = name,
-        val raan = values[6].toDouble()
+            epoch = epoch,
-        val argper = values[7].toDouble()
+            meanmo = values[3].toDouble(),
-        val meanan = values[8].toDouble()
+            eccn = values[4].toDouble(),
-        val catnum = values[11].toInt()
+            incl = values[5].toDouble(),
-        val bstar = values[14].toDouble()
+            raan = values[6].toDouble(),
-        OrbitalData(name, epoch, meanmo, eccn, incl, raan, argper, meanan, catnum, bstar)
+            argper = values[7].toDouble(),
-    } catch (exception: Exception) {
+            meanan = values[8].toDouble(),
-        println("CSV parsing exception: $exception")
+            catnum = values[11].toInt(),
-        null
+            bstar = values[14].toDouble()
-    }
+        )
+    }.onFailure { println("CSV parsing exception: $it") }.getOrNull()
 
-    private fun parseTLE(tle: List<String>): OrbitalData? = try {
+    private fun parseTLE(tle: List<String>): OrbitalData? = runCatching {
-        val name: String = tle[0].trim()
+        val line1 = tle[1]
-        val epoch: Double = tle[1].substring(18, 32).toDouble()
+        val line2 = tle[2]
-        val meanmo: Double = tle[2].substring(52, 63).toDouble()
+        OrbitalData(
-        val eccn: Double = tle[2].substring(26, 33).toDouble() / 10000000.0
+            name = tle[0].trim(),
-        val incl: Double = tle[2].substring(8, 16).toDouble()
+            epoch = line1.substring(18, 32).toDouble(),
-        val raan: Double = tle[2].substring(17, 25).toDouble()
+            meanmo = line2.substring(52, 63).toDouble(),
-        val argper: Double = tle[2].substring(34, 42).toDouble()
+            eccn = line2.substring(26, 33).toDouble() / 1e7,
-        val meanan: Double = tle[2].substring(43, 51).toDouble()
+            incl = line2.substring(8, 16).toDouble(),
-        val catnum: Int = tle[1].substring(2, 7).trim().toInt()
+            raan = line2.substring(17, 25).toDouble(),
-        val bstar: Double = 1.0e-5 * tle[1].substring(53, 59).toDouble() / 10.0.pow(tle[1].substring(60, 61).toDouble())
+            argper = line2.substring(34, 42).toDouble(),
-        OrbitalData(name, epoch, meanmo, eccn, incl, raan, argper, meanan, catnum, bstar)
+            meanan = line2.substring(43, 51).toDouble(),
-    } catch (exception: Exception) {
+            catnum = line1.substring(2, 7).trim().toInt(),
-        println("TLE parsing exception: $exception")
+            bstar = 1e-5 * line1.substring(53, 59).toDouble() / 10.0.pow(line1.substring(60, 61).toDouble())
-        null
+        )
-    }
+    }.onFailure { println("TLE parsing exception: $it") }.getOrNull()
 
-    private fun parseJSON(json: JSONObject): SatRadio? = try {
+    private fun parseJSON(json: JSONObject): SatRadio? = runCatching {
-        val uuid = json.getString("uuid")
+        SatRadio(
-        val info = json.getString("description")
+            uuid = json.getString("uuid"),
-        val alive = json.getBoolean("alive")
+            info = json.getString("description"),
-        val dlinkLow = if (json.isNull("downlink_low")) null else json.getLong("downlink_low")
+            isAlive = json.getBoolean("alive"),
-        val dlinkHigh = if (json.isNull("downlink_high")) null else json.getLong("downlink_high")
+            downlinkLow = json.optLongOrNull("downlink_low"),
-        val dlinkMode = if (json.isNull("mode")) null else json.getString("mode")
+            downlinkHigh = json.optLongOrNull("downlink_high"),
-        val ulinkLow = if (json.isNull("uplink_low")) null else json.getLong("uplink_low")
+            downlinkMode = json.optStringOrNull("mode"),
-        val ulinkHigh = if (json.isNull("uplink_high")) null else json.getLong("uplink_high")
+            uplinkLow = json.optLongOrNull("uplink_low"),
-        val ulinkMode = if (json.isNull("uplink_mode")) null else json.getString("uplink_mode")
+            uplinkHigh = json.optLongOrNull("uplink_high"),
-        val inverted = json.getBoolean("invert")
+            uplinkMode = json.optStringOrNull("uplink_mode"),
-        val catnum = if (json.isNull("norad_cat_id")) null else json.getInt("norad_cat_id")
+            isInverted = json.getBoolean("invert"),
-        SatRadio(uuid, info, alive, dlinkLow, dlinkHigh, dlinkMode, ulinkLow, ulinkHigh, ulinkMode, inverted, catnum)
+            catnum = json.optIntOrNull("norad_cat_id")
-    } catch (exception: Exception) {
+        )
-        println("JSON parsing exception: $exception")
+    }.onFailure { println("JSON parsing exception: $it") }.getOrNull()
-        null
+
-    }
+    private fun JSONObject.optStringOrNull(key: String): String? = if (isNull(key)) null else getString(key)
+    private fun JSONObject.optLongOrNull(key: String): Long? = if (isNull(key)) null else getLong(key)
+    private fun JSONObject.optIntOrNull(key: String): Int? = if (isNull(key)) null else getInt(key)
 
     private fun getDayOfYear(year: Int, month: Int, dayOfMonth: Int): Int {
-        if (month == 1) return dayOfMonth
+        val daysInMonth = intArrayOf(31, if (isLeapYear(year)) 29 else 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31)
-        val daysArray = arrayOf(31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31)
+        return daysInMonth.take(month - 1).sum() + dayOfMonth
-        var dayOfYear = dayOfMonth
-        // If leap year increment Feb days
-        if (isLeapYear(year)) daysArray[1]++
-        for (i in 0 until month - 1) {
-            dayOfYear += daysArray[i]
-        }
-        return dayOfYear
     }
 }