Skip to content

Commit

Permalink
Merge branch 'osmandapp:master' into hardy_Afa
Browse files Browse the repository at this point in the history
  • Loading branch information
@sonora
sonora authored Sep 25, 2024
2 parents 48a76a8 + 295460a commit a0851fb
Show file tree
Hide file tree
Showing 34 changed files with 682 additions and 136 deletions.
4 changes: 2 additions & 2 deletions OsmAnd-shared/src/androidMain/kotlin/net/osmand/shared/extensions/FileExtensions.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,5 @@ package net.osmand.shared.extensions
import net.osmand.shared.io.KFile
import java.io.File

fun File.kFile(): KFile = KFile(this.absolutePath)
fun KFile.jFile(): File = File(this.absolutePath())
fun File.kFile(): KFile = KFile(this.path)
fun KFile.jFile(): File = File(if (this.isPathEmpty()) "" else this.path())
29 changes: 27 additions & 2 deletions OsmAnd-shared/src/commonMain/kotlin/net/osmand/shared/gpx/GpxDatabase.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ class GpxDatabase {
const val TMP_NAME_COLUMN_COUNT = "itemsCount"
const val UNKNOWN_TIME_THRESHOLD = 10L
val GPX_UPDATE_PARAMETERS_START = "UPDATE $GPX_TABLE_NAME SET "
val GPX_FIND_BY_NAME_AND_DIR =
" WHERE ${FILE_NAME.columnName} = ? AND ${FILE_DIR.columnName} = ?"
val GPX_FIND_BY_NAME_AND_DIR = " WHERE ${FILE_NAME.columnName} = ? AND ${FILE_DIR.columnName} = ?"
val GPX_NAME_AND_DIR = "${FILE_NAME.columnName} = ? AND ${FILE_DIR.columnName} = ?"
val GPX_MIN_CREATE_DATE =
"SELECT MIN(${FILE_CREATION_TIME.columnName}) FROM $GPX_TABLE_NAME WHERE ${FILE_CREATION_TIME.columnName} > $UNKNOWN_TIME_THRESHOLD"
val GPX_MAX_COLUMN_VALUE = "SELECT MAX(%s) FROM $GPX_TABLE_NAME"
Expand Down Expand Up @@ -140,6 +140,31 @@ class GpxDatabase {
}
}

fun remove(files: Collection<KFile>): Boolean {
if (files.isEmpty()) return false

val time = currentTimeMillis()
var db: SQLiteConnection? = null
try {
db = openConnection(false)
db?.let {
val fileDeletionMap = files.groupBy { GpxDbUtils.getTableName(it) }
fileDeletionMap.forEach { (tableName, files) ->
files.chunked(BATCH_SIZE).forEach { batch ->
val deleteConditions = batch.joinToString(separator = " OR ") { "($GPX_NAME_AND_DIR)" }
val args: Array<Any?> = batch.flatMap { listOf(it.name(), GpxDbUtils.getGpxFileDir(it)) }.toTypedArray()
db.execSQL("DELETE FROM $tableName WHERE $deleteConditions", args)
}
}
log.info("Remove gpx files from db count=${files.size} in ${currentTimeMillis() - time} ms")
return true
}
return false
} finally {
db?.close()
}
}

fun add(item: DataItem): Boolean {
var db: SQLiteConnection? = null
try {
Expand Down
65 changes: 41 additions & 24 deletions OsmAnd-shared/src/commonMain/kotlin/net/osmand/shared/gpx/GpxDbHelper.kt
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package net.osmand.shared.gpx

import co.touchlab.stately.collections.ConcurrentMutableList
import co.touchlab.stately.collections.ConcurrentMutableMap
import co.touchlab.stately.concurrency.AtomicInt
import co.touchlab.stately.concurrency.Synchronizable
import co.touchlab.stately.concurrency.synchronize
import kotlinx.coroutines.Dispatchers
Expand All @@ -13,21 +13,21 @@ import kotlinx.coroutines.runBlocking
import net.osmand.shared.api.SQLiteAPI.SQLiteConnection
import net.osmand.shared.data.StringIntPair
import net.osmand.shared.extensions.currentTimeMillis
import net.osmand.shared.gpx.GpxReader.GpxDbReaderCallback
import net.osmand.shared.gpx.GpxReader.GpxReaderAdapter
import net.osmand.shared.io.KFile
import net.osmand.shared.util.LoggerFactory


object GpxDbHelper : GpxDbReaderCallback {
object GpxDbHelper : GpxReaderAdapter {
val log = LoggerFactory.getLogger("GpxDbHelper")

private val database: GpxDatabase by lazy { GpxDatabase() }

private val dirItems = ConcurrentMutableMap<KFile, GpxDirItem>()
private val dataItems = ConcurrentMutableMap<KFile, GpxDataItem>()
private var itemsVersion = AtomicInt(0)

private val readingItems = ConcurrentMutableList<KFile>()
private val readingItemsMap = ConcurrentMutableMap<KFile, GpxDataItem>()
private val readingItemsMap = mutableMapOf<KFile, GpxDataItem>()
private val readingItemsCallbacks = mutableMapOf<KFile, MutableList<GpxDataItemCallback>?>()

private const val READER_TASKS_LIMIT = 4
Expand Down Expand Up @@ -55,7 +55,7 @@ object GpxDbHelper : GpxDbReaderCallback {

private suspend fun loadGpxItems() {
val start = currentTimeMillis()
val items = getItems()
val items = readItems()
val startEx = currentTimeMillis()
val fileExistenceMap = getFileExistenceMap(items)
log.info("Time to getFileExistenceMap ${currentTimeMillis() - startEx} ms, ${items.size} items")
Expand All @@ -70,8 +70,9 @@ object GpxDbHelper : GpxDbReaderCallback {
itemsToRemove.add(file)
}
}
putToCacheBulk(itemsToCache);
removeFromCacheBulk(itemsToRemove);
putToCacheBulk(itemsToCache)
removeFromCacheBulk(itemsToRemove)
database.remove(itemsToRemove)
log.info("Time to loadGpxItems ${currentTimeMillis() - start} ms, ${items.size} items")
}

Expand All @@ -86,7 +87,7 @@ object GpxDbHelper : GpxDbReaderCallback {

private fun loadGpxDirItems() {
val start = currentTimeMillis()
val items = getDirItems()
val items = readDirItems()
items.forEach { item ->
val file = item.file
if (file.exists()) {
Expand All @@ -100,12 +101,15 @@ object GpxDbHelper : GpxDbReaderCallback {

fun isInitialized() = initialized

fun getItemsVersion() = itemsVersion.get()

private fun putToCache(item: DataItem) {
val file = item.file
when (item) {
is GpxDataItem -> dataItems[file] = item
is GpxDirItem -> dirItems[file] = item
}
itemsVersion.incrementAndGet()
}

private fun removeFromCache(file: KFile) {
Expand All @@ -114,14 +118,17 @@ object GpxDbHelper : GpxDbReaderCallback {
} else {
dirItems.remove(file)
}
itemsVersion.incrementAndGet()
}

private fun putToCacheBulk(itemsToCache: Map<KFile, GpxDataItem>) {
dataItems.putAll(itemsToCache)
itemsVersion.incrementAndGet()
}

private fun removeFromCacheBulk(filesToRemove: Set<KFile>) {
dataItems.keys.removeAll(filesToRemove)
itemsVersion.incrementAndGet()
}

fun rename(currentFile: KFile, newFile: KFile): Boolean {
Expand Down Expand Up @@ -167,6 +174,12 @@ object GpxDbHelper : GpxDbReaderCallback {
return res
}

fun remove(files: Collection<KFile>): Boolean {
val res = database.remove(files)
removeFromCacheBulk(files.toSet())
return res
}

fun remove(item: DataItem): Boolean {
val file = item.file
val res = database.remove(file)
Expand All @@ -187,10 +200,13 @@ object GpxDbHelper : GpxDbReaderCallback {
return res
}

fun getItemsBlocking(): List<GpxDataItem> = database.getGpxDataItemsBlocking()
suspend fun getItems(): List<GpxDataItem> = database.getGpxDataItems()
fun getItems() = dataItems.values.toList()

fun getDirItems() = dirItems.values.toList()

fun getDirItems(): List<GpxDirItem> = database.getGpxDirItems()
private suspend fun readItems(): List<GpxDataItem> = database.getGpxDataItems()

private fun readDirItems(): List<GpxDirItem> = database.getGpxDirItems()

fun getStringIntItemsCollection(
columnName: String,
Expand Down Expand Up @@ -229,7 +245,7 @@ object GpxDbHelper : GpxDbReaderCallback {
}

fun getItem(file: KFile, callback: GpxDataItemCallback?): GpxDataItem? {
if (file.path().isEmpty()) {
if (file.isPathEmpty()) {
return null
}
val item = dataItems[file]
Expand All @@ -250,7 +266,7 @@ object GpxDbHelper : GpxDbReaderCallback {

fun getSplitItemsBlocking(): List<GpxDataItem> = runBlocking { getSplitItems() }
suspend fun getSplitItems(): List<GpxDataItem> {
return getItems().filter {
return readItems().filter {
it.getAppearanceParameter<Int>(GpxParameter.SPLIT_TYPE) != 0
}
}
Expand All @@ -274,7 +290,7 @@ object GpxDbHelper : GpxDbReaderCallback {
fun isReading(): Boolean = readerSync.synchronize { readers.isNotEmpty() }

private fun isReading(file: KFile): Boolean =
readerSync.synchronize { readingItems.contains(file) || readers.any { it.file == file } }
readerSync.synchronize { readingItemsMap.contains(file) || readers.any { it.isReading(file) } }

private fun readGpxItem(file: KFile, item: GpxDataItem?, callback: GpxDataItemCallback?) {
readerSync.synchronize {
Expand All @@ -283,7 +299,6 @@ object GpxDbHelper : GpxDbReaderCallback {
}
if (!isReading(file)) {
readingItemsMap[file] = item ?: GpxDataItem(file)
readingItems.add(file)
if (readers.size < READER_TASKS_LIMIT) {
startReading()
}
Expand All @@ -293,21 +308,26 @@ object GpxDbHelper : GpxDbReaderCallback {

private fun startReading() {
readerSync.synchronize {
readers.add(GpxReader(readingItems, readingItemsMap, this).apply { execute() })
log.info(">>>> GpxReader created = ${readers.size}")
readers.add(GpxReader(this).apply { execute() })
}
}

private fun stopReading() {
readerSync.synchronize {
readers.forEach { it.cancel() }
readers.clear()
log.info(">>>> GpxReaders stopped")
}
}

fun getGPXDatabase(): GpxDatabase = database

override fun pullNextFileItem(action: ((Pair<KFile, GpxDataItem>?) -> Unit)?): Pair<KFile, GpxDataItem>? =
readerSync.synchronize {
val result = readingItemsMap.entries.firstOrNull()?.toPair()?.apply { readingItemsMap.remove(first) }
action?.invoke(result)
result
}

override fun onGpxDataItemRead(item: GpxDataItem) {
putGpxDataItemToSmartFolder(item)
}
Expand Down Expand Up @@ -336,20 +356,17 @@ object GpxDbHelper : GpxDbReaderCallback {

override fun onReadingCancelled() {
readerSync.synchronize {
readingItems.clear()
readingItemsMap.clear()
readingItemsCallbacks.clear()
}
}

override fun onReadingFinished(reader: GpxReader, cancelled: Boolean) {
readerSync.synchronize {
if (readingItems.isNotEmpty() && readers.size < READER_TASKS_LIMIT && !cancelled) {
if (readingItemsMap.isNotEmpty() && readers.size < READER_TASKS_LIMIT && !cancelled) {
startReading()
} else {
readers.remove(reader)
}
log.info(">>>> GpxReader onReadingFinished readers=${readers.size}")
readers.remove(reader)
}
}
}
57 changes: 35 additions & 22 deletions OsmAnd-shared/src/commonMain/kotlin/net/osmand/shared/gpx/GpxReader.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,8 @@ import net.osmand.shared.io.KFile
import net.osmand.shared.util.LoggerFactory
import net.osmand.shared.util.PlatformUtil

class GpxReader(
private val readingItems: MutableList<KFile>,
private val readingItemsMap: MutableMap<KFile, GpxDataItem>,
private val listener: GpxDbReaderCallback?
) : KAsyncTask<Unit, GpxDataItem, Unit>(true) {
class GpxReader(private val adapter: GpxReaderAdapter)
: KAsyncTask<Unit, GpxDataItem, Unit>(true) {

companion object {
private val log = LoggerFactory.getLogger("GpxReader")
Expand All @@ -23,7 +20,8 @@ class GpxReader(
}

private val database: GpxDatabase = GpxDbHelper.getGPXDatabase()
var file: KFile? = null
private var currentFile: KFile? = null
private var currentItem: GpxDataItem? = null

override suspend fun doInBackground(vararg params: Unit) {
waitForInitialization()
Expand All @@ -37,45 +35,56 @@ class GpxReader(
}

private fun doReading() {
log.info(">>>> start GpxReader ===== ")
var filesCount = 0
val conn = database.openConnection(false)
if (conn != null) {
try {
file = readingItems.removeFirstOrNull()
var file: KFile?
var item: GpxDataItem?
pullNextFileItem()
file = currentFile
item = currentItem
while (file != null && !isCancelled()) {
var item = readingItemsMap.remove(file)
if (GpxDbUtils.isAnalyseNeeded(item)) {
item = updateGpxDataItem(conn, item, file!!)
item = updateGpxDataItem(conn, item, file)
}
if (item != null) {
listener?.onGpxDataItemRead(item)
adapter.onGpxDataItemRead(item)
publishProgress(item)
}
file = readingItems.removeFirstOrNull()

pullNextFileItem()
file = currentFile
item = currentItem
filesCount++
}
} catch (e: Exception) {
log.error(e.message)
} finally {
conn.close()
log.info(">>>> done GpxReader ===== filesCount=$filesCount")
}
} else {
cancel()
}
}

private fun pullNextFileItem() {
adapter.pullNextFileItem {
currentFile = it?.first
currentItem = it?.second
}
}

override fun onProgressUpdate(vararg values: GpxDataItem) {
listener?.onProgressUpdate(*values)
adapter.onProgressUpdate(*values)
}

override fun onCancelled() {
listener?.onReadingCancelled()
adapter.onReadingCancelled()
}

override fun onPostExecute(result: Unit) {
listener?.onReadingFinished(this, isCancelled())
adapter.onReadingFinished(this, isCancelled())
}

private fun updateGpxDataItem(conn: SQLiteConnection, item: GpxDataItem?, file: KFile): GpxDataItem {
Expand Down Expand Up @@ -134,12 +143,16 @@ class GpxReader(
}
}

fun isReading(): Boolean = readingItems.isNotEmpty() || file != null
fun isReading(): Boolean = isRunning()

fun isReading(file: KFile): Boolean = currentFile == file

interface GpxReaderAdapter {
fun pullNextFileItem(action: ((Pair<KFile, GpxDataItem>?) -> Unit)? = null): Pair<KFile, GpxDataItem>?

interface GpxDbReaderCallback {
fun onGpxDataItemRead(item: GpxDataItem)
fun onProgressUpdate(vararg dataItems: GpxDataItem)
fun onReadingCancelled()
fun onReadingFinished(reader: GpxReader, cancelled: Boolean)
fun onGpxDataItemRead(item: GpxDataItem) {}
fun onProgressUpdate(vararg dataItems: GpxDataItem) {}
fun onReadingCancelled() {}
fun onReadingFinished(reader: GpxReader, cancelled: Boolean) {}
}
}
Loading

0 comments on commit a0851fb

Please sign in to comment.