Madeorsk/SpotiFlyer
1
0
Fork 0
mirror of https://github.com/Shabinder/SpotiFlyer.git synced 2024-11-22 01:04:31 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Caching Fixes

Browse Source
This commit is contained in:
Shabinder Singh 2021-10-10 21:51:14 +05:30
parent 7f279f2602
commit f7e38c2c6e
13 changed files with 127 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
android/build.gradle.kts
View File

@ -124,8 +124,9 @@ dependencies {
implementation(storage.chooser)
with(bundles) {
implementation(androidx.lifecycle)
implementation(ktor)
implementation(mviKotlin)
implementation(androidx.lifecycle)
implementation(accompanist.inset)
}

4
android/src/main/java/com/shabinder/spotiflyer/service/ForegroundService.kt
View File

@ -43,6 +43,7 @@ import com.shabinder.common.models.event.coroutines.failure
import com.shabinder.common.providers.FetchPlatformQueryResult
import com.shabinder.common.translations.Strings
import com.shabinder.spotiflyer.R
import io.ktor.client.HttpClient
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.flow.MutableSharedFlow
@ -62,6 +63,7 @@ class ForegroundService : LifecycleService() {
private val fetcher: FetchPlatformQueryResult by inject()
private val logger: Kermit by inject()
private val dir: FileManager by inject()
private val httpClient: HttpClient by inject()
private var messageList =
java.util.Collections.synchronizedList(MutableList(5) { emptyMessage })
@ -170,7 +172,7 @@ class ForegroundService : LifecycleService() {
trackStatusFlowMap[track.title] = DownloadStatus.Downloading()
// Enqueueing Download
downloadFile(url).collect {
httpClient.downloadFile(url).collect {
when (it) {
is DownloadResult.Error -> {
logger.d(TAG) { it.message }

13
android/src/main/java/com/shabinder/spotiflyer/service/Utils.kt
View File

@ -15,12 +15,19 @@ fun cleanFiles(dir: File) {
if (file.isDirectory) {
cleanFiles(file)
} else if (file.isFile) {
if (file.path.toString().substringAfterLast(".") != "mp3") {
Log.d("Files Cleaning", "Cleaning ${file.path}")
val filePath = file.path.toString()
if (filePath.substringAfterLast(".") != "mp3" || filePath.isTempFile()) {
Log.d("Files Cleaning", "Cleaning $filePath")
file.delete()
}
}
}
}
} catch (e: Exception) { e.printStackTrace() }
} catch (e: Exception) {
e.printStackTrace()
}
}
private fun String.isTempFile(): Boolean {
return substringBeforeLast(".").takeLast(5) == ".temp"
}

4
common/core-components/src/androidMain/kotlin/com/shabinder/common/core_components/file_manager/AndroidFileManager.kt
View File

@ -166,7 +166,7 @@ class AndroidFileManager(
override suspend fun loadImage(url: String, reqWidth: Int, reqHeight: Int): Picture =
withContext(dispatcherIO) {
val cachePath = imageCacheDir() + getNameURL(url)
val cachePath = getImageCachePath(url)
Picture(
image = (loadCachedImage(cachePath, reqWidth, reqHeight) ?: freshImage(
url,
@ -214,7 +214,7 @@ class AndroidFileManager(
// Decode and Cache Full Sized Image in Background
cacheImage(
BitmapFactory.decodeByteArray(input, 0, input.size),
imageCacheDir() + getNameURL(url)
getImageCachePath(url)
)
}
bitmap // return Memory Efficient Bitmap

70
common/core-components/src/commonMain/kotlin/com.shabinder.common.core_components/file_manager/FileManager.kt
View File

@ -25,10 +25,14 @@ import com.shabinder.common.models.DownloadResult
import com.shabinder.common.models.TrackDetails
import com.shabinder.common.models.event.coroutines.SuspendableEvent
import com.shabinder.common.utils.removeIllegalChars
import com.shabinder.common.utils.requireNotNull
import com.shabinder.database.Database
import io.ktor.client.request.*
import io.ktor.client.statement.*
import io.ktor.http.*
import io.ktor.client.HttpClient
import io.ktor.client.request.HttpRequestBuilder
import io.ktor.client.request.get
import io.ktor.client.statement.HttpStatement
import io.ktor.http.contentLength
import io.ktor.http.isSuccess
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.catch
import kotlinx.coroutines.flow.flow
@ -80,12 +84,13 @@ fun FileManager.createDirectories() {
if (!defaultDir().contains("null${fileSeparator()}SpotiFlyer")) {
createDirectory(defaultDir())
createDirectory(imageCacheDir())
createDirectory(defaultDir() + "Tracks/")
createDirectory(defaultDir() + "Albums/")
createDirectory(defaultDir() + "Playlists/")
createDirectory(defaultDir() + "YT_Downloads/")
createDirectory(defaultDir() + "Tracks" + fileSeparator())
createDirectory(defaultDir() + "Albums" + fileSeparator())
createDirectory(defaultDir() + "Playlists" + fileSeparator())
createDirectory(defaultDir() + "YT_Downloads" + fileSeparator())
}
} catch (ignored: Exception) {
}
} catch (ignored: Exception) { }
}
fun FileManager.finalOutputDir(
@ -100,24 +105,50 @@ fun FileManager.finalOutputDir(
removeIllegalChars(subFolder) + this.fileSeparator()
} +
removeIllegalChars(itemName) + extension
/*DIR Specific Operation End*/
fun getNameURL(url: String): String {
return url.substring(url.lastIndexOf('/', url.lastIndexOf('/') - 1) + 1, url.length)
.replace('/', '_')
fun FileManager.getImageCachePath(
url: String
): String = imageCacheDir() + getNameFromURL(url, isImage = true)
/*DIR Specific Operation End*/
private fun getNameFromURL(url: String, isImage: Boolean = false): String {
val startIndex = url.lastIndexOf('/', url.lastIndexOf('/') - 1) + 1
var fileName = if (startIndex != -1)
url.substring(startIndex).replace('/', '_')
else url.substringAfterLast("/")
// Generify File Extensions
if (isImage) {
if (fileName.length - fileName.lastIndexOf(".") > 5) {
fileName += ".jpeg"
} else {
if (fileName.endsWith(".jpg"))
fileName = fileName.substringBeforeLast(".") + ".jpeg"
}
}
suspend fun downloadFile(url: String): Flow<DownloadResult> {
return fileName
}
suspend fun HttpClient.downloadFile(url: String) = downloadFile(url, this)
suspend fun downloadFile(url: String, client: HttpClient? = null): Flow<DownloadResult> {
return flow {
val client = createHttpClient()
val response = client.get<HttpStatement>(url).execute()
val data = ByteArray(response.contentLength()!!.toInt())
val httpClient = client ?: createHttpClient()
val response = httpClient.get<HttpStatement>(url).execute()
// Not all requests return Content Length
val data = kotlin.runCatching {
ByteArray(response.contentLength().requireNotNull().toInt())
}.getOrNull() ?: byteArrayOf()
var offset = 0
do {
// Set Length optimally, after how many kb you want a progress update, now it 0.25mb
val currentRead = response.content.readAvailable(data, offset, 2_50_000)
offset += currentRead
val progress = (offset * 100f / data.size).roundToInt()
val progress = data.size.takeIf { it != 0 }?.let { fileSize ->
(offset * 100f / fileSize).roundToInt()
} ?: 0
emit(DownloadResult.Progress(progress))
} while (currentRead > 0)
if (response.status.isSuccess()) {
@ -125,7 +156,10 @@ suspend fun downloadFile(url: String): Flow<DownloadResult> {
} else {
emit(DownloadResult.Error("File not downloaded"))
}
client.close()
// Close Client if We Created One
if (client == null)
httpClient.close()
}.catch { e ->
e.printStackTrace()
emit(DownloadResult.Error(e.message ?: "File not downloaded"))

5
common/core-components/src/desktopMain/kotlin/com.shabinder.common.core_components/file_manager/DesktopFileManager.kt
View File

@ -178,8 +178,7 @@ class DesktopFileManager(
override fun addToLibrary(path: String) {}
override suspend fun loadImage(url: String, reqWidth: Int, reqHeight: Int): Picture {
val cachePath = imageCacheDir() + getNameURL(url)
var picture: ImageBitmap? = loadCachedImage(cachePath, reqWidth, reqHeight)
var picture: ImageBitmap? = loadCachedImage(getImageCachePath(url), reqWidth, reqHeight)
if (picture == null) picture = freshImage(url, reqWidth, reqHeight)
return Picture(image = picture)
}
@ -208,7 +207,7 @@ class DesktopFileManager(
if (result != null) {
GlobalScope.launch(Dispatchers.IO) { // TODO Refactor
cacheImage(result, imageCacheDir() + getNameURL(url))
cacheImage(result, getImageCachePath(url))
}
result.toImageBitmap()
} else null

2
common/data-models/src/commonMain/kotlin/com/shabinder/common/models/saavn/SaavnSong.kt
View File

@ -20,7 +20,7 @@ data class SaavnSong @OptIn(ExperimentalSerializationApi::class) constructor(
// val explicit_content: Int = 0,
val has_lyrics: Boolean = false,
val id: String,
val image: String,
val image: String = "",
val label: String? = null,
val label_url: String? = null,
val language: String,

4
common/providers/src/commonMain/kotlin/com.shabinder.common.providers/gaana/GaanaProvider.kt
View File

@ -19,6 +19,7 @@ package com.shabinder.common.providers.gaana
import co.touchlab.kermit.Kermit
import com.shabinder.common.core_components.file_manager.FileManager
import com.shabinder.common.core_components.file_manager.finalOutputDir
import com.shabinder.common.core_components.file_manager.getImageCachePath
import com.shabinder.common.models.DownloadStatus
import com.shabinder.common.models.PlatformQueryResult
import com.shabinder.common.models.SpotiFlyerException
@ -124,8 +125,7 @@ class GaanaProvider(
title = it.track_title,
artists = it.artist.map { artist -> artist?.name.toString() },
durationSec = it.duration,
albumArtPath = fileManager.imageCacheDir() + (it.artworkLink.substringBeforeLast('/')
.substringAfterLast('/')) + ".jpeg",
albumArtPath = fileManager.getImageCachePath(it.artworkLink),
albumName = it.album_title,
genre = it.genre?.mapNotNull { genre -> genre?.name } ?: emptyList(),
year = it.release_date,

3
common/providers/src/commonMain/kotlin/com.shabinder.common.providers/saavn/SaavnProvider.kt
View File

@ -3,6 +3,7 @@ package com.shabinder.common.providers.saavn
import co.touchlab.kermit.Kermit
import com.shabinder.common.core_components.file_manager.FileManager
import com.shabinder.common.core_components.file_manager.finalOutputDir
import com.shabinder.common.core_components.file_manager.getImageCachePath
import com.shabinder.common.models.*
import com.shabinder.common.models.event.coroutines.SuspendableEvent
import com.shabinder.common.models.saavn.SaavnSong
@ -68,7 +69,7 @@ class SaavnProvider(
artists = it.artistMap.keys.toMutableSet().apply { addAll(it.singers.split(",")) }.toList(),
durationSec = it.duration.toInt(),
albumName = it.album,
albumArtPath = fileManager.imageCacheDir() + (it.image.substringBeforeLast('/').substringAfterLast('/')) + ".jpeg",
albumArtPath = fileManager.getImageCachePath(it.image),
year = it.year,
comment = it.copyright_text,
trackUrl = it.perma_url,

5
common/providers/src/commonMain/kotlin/com.shabinder.common.providers/sound_cloud/SoundCloudProvider.kt
View File

@ -3,6 +3,7 @@ package com.shabinder.common.providers.sound_cloud
import co.touchlab.kermit.Kermit
import com.shabinder.common.core_components.file_manager.FileManager
import com.shabinder.common.core_components.file_manager.finalOutputDir
import com.shabinder.common.core_components.file_manager.getImageCachePath
import com.shabinder.common.models.AudioFormat
import com.shabinder.common.models.AudioQuality
import com.shabinder.common.models.DownloadStatus
@ -70,9 +71,7 @@ class SoundCloudProvider(
artists = /*it.artists?.map { artist -> artist?.name.toString() } ?:*/ listOf(it.user.username.ifBlank { it.genre }),
albumArtists = /*it.album?.artists?.mapNotNull { artist -> artist?.name } ?:*/ emptyList(),
durationSec = (it.duration / 1000),
albumArtPath = fileManager.imageCacheDir() + (it.artworkUrl.formatArtworkUrl()).substringAfterLast(
'/'
) + ".jpeg",
albumArtPath = fileManager.getImageCachePath(it.artworkUrl.formatArtworkUrl()),
albumName = "", //it.album?.name,
year = runCatching { it.displayDate.substring(0, 4) }.getOrNull(),
comment = it.caption,

5
common/providers/src/commonMain/kotlin/com.shabinder.common.providers/spotify/SpotifyProvider.kt
View File

@ -19,6 +19,7 @@ package com.shabinder.common.providers.spotify
import co.touchlab.kermit.Kermit
import com.shabinder.common.core_components.file_manager.FileManager
import com.shabinder.common.core_components.file_manager.finalOutputDir
import com.shabinder.common.core_components.file_manager.getImageCachePath
import com.shabinder.common.core_components.utils.createHttpClient
import com.shabinder.common.models.*
import com.shabinder.common.models.event.coroutines.SuspendableEvent
@ -201,9 +202,7 @@ class SpotifyProvider(
artists = it.artists?.map { artist -> artist?.name.toString() } ?: listOf(),
albumArtists = it.album?.artists?.mapNotNull { artist -> artist?.name } ?: emptyList(),
durationSec = (it.duration_ms / 1000).toInt(),
albumArtPath = fileManager.imageCacheDir() + (it.album?.images?.firstOrNull()?.url.toString()).substringAfterLast(
'/'
) + ".jpeg",
albumArtPath = fileManager.getImageCachePath(it.album?.images?.firstOrNull()?.url ?: ""),
albumName = it.album?.name,
year = it.album?.release_date,
comment = "Genres:${it.album?.genres?.joinToString()}",

21
common/providers/src/commonMain/kotlin/com.shabinder.common.providers/youtube/YoutubeProvider.kt
View File

@ -19,6 +19,7 @@ package com.shabinder.common.providers.youtube
import co.touchlab.kermit.Kermit
import com.shabinder.common.core_components.file_manager.FileManager
import com.shabinder.common.core_components.file_manager.finalOutputDir
import com.shabinder.common.core_components.file_manager.getImageCachePath
import com.shabinder.common.models.DownloadStatus
import com.shabinder.common.models.PlatformQueryResult
import com.shabinder.common.models.SpotiFlyerException
@ -30,7 +31,7 @@ import io.github.shabinder.YoutubeDownloader
import io.github.shabinder.models.YoutubeVideo
import io.github.shabinder.models.formats.Format
import io.github.shabinder.models.quality.AudioQuality
import io.ktor.client.*
import io.ktor.client.HttpClient
class YoutubeProvider(
private val httpClient: HttpClient,
@ -108,13 +109,14 @@ class YoutubeProvider(
title = name
trackList = videos.map {
val imageURL = "https://i.ytimg.com/vi/${it.videoId}/hqdefault.jpg"
TrackDetails(
title = it.title ?: "N/A",
artists = listOf(it.author ?: "N/A"),
durationSec = it.lengthSeconds,
albumArtPath = fileManager.imageCacheDir() + it.videoId + ".jpeg",
albumArtPath = fileManager.getImageCachePath(imageURL),
source = Source.YouTube,
albumArtURL = "https://i.ytimg.com/vi/${it.videoId}/hqdefault.jpg",
albumArtURL = imageURL,
downloaded = if (fileManager.isPresent(
fileManager.finalOutputDir(
itemName = it.title ?: "N/A",
@ -155,7 +157,7 @@ class YoutubeProvider(
val video = ytDownloader.getVideo(searchId)
coverUrl = "https://i.ytimg.com/vi/$searchId/hqdefault.jpg"
val detail = video.videoDetails
val name = detail.title?.replace(detail.author?.uppercase() ?: "", "", true)
val name = detail.title?.replace(detail.author?.toUpperCase() ?: "", "", true)
?: detail.title ?: ""
// logger.i{ detail.toString() }
trackList = listOf(
@ -163,9 +165,9 @@ class YoutubeProvider(
title = name,
artists = listOf(detail.author ?: "N/A"),
durationSec = detail.lengthSeconds,
albumArtPath = fileManager.imageCacheDir() + "$searchId.jpeg",
albumArtPath = fileManager.getImageCachePath(coverUrl),
source = Source.YouTube,
albumArtURL = "https://i.ytimg.com/vi/$searchId/hqdefault.jpg",
albumArtURL = coverUrl,
downloaded = if (fileManager.isPresent(
fileManager.finalOutputDir(
itemName = name,
@ -179,7 +181,12 @@ class YoutubeProvider(
else {
DownloadStatus.NotDownloaded
},
outputFilePath = fileManager.finalOutputDir(name, folderType, subFolder, fileManager.defaultDir()/*,".m4a"*/),
outputFilePath = fileManager.finalOutputDir(
name,
folderType,
subFolder,
fileManager.defaultDir()/*,".m4a"*/
),
videoID = searchId
)
)

46
common/providers/src/commonMain/kotlin/com.shabinder.common.providers/youtube_music/YoutubeMusic.kt
View File

@ -18,19 +18,33 @@ package com.shabinder.common.providers.youtube_music
import co.touchlab.kermit.Kermit
import com.shabinder.common.core_components.file_manager.FileManager
import com.shabinder.common.models.*
import com.shabinder.common.models.AudioFormat
import com.shabinder.common.models.AudioQuality
import com.shabinder.common.models.SpotiFlyerException
import com.shabinder.common.models.TrackDetails
import com.shabinder.common.models.YoutubeTrack
import com.shabinder.common.models.corsApi
import com.shabinder.common.models.event.coroutines.SuspendableEvent
import com.shabinder.common.models.event.coroutines.flatMap
import com.shabinder.common.models.event.coroutines.flatMapError
import com.shabinder.common.models.event.coroutines.map
import com.shabinder.common.providers.youtube.YoutubeProvider
import com.shabinder.common.providers.youtube.get
import com.shabinder.common.providers.youtube_to_mp3.requests.YoutubeMp3
import io.github.shabinder.fuzzywuzzy.diffutils.FuzzySearch
import io.ktor.client.*
import io.ktor.client.request.*
import io.ktor.http.*
import kotlinx.serialization.json.*
import io.ktor.client.HttpClient
import io.ktor.client.request.headers
import io.ktor.client.request.post
import io.ktor.http.ContentType
import io.ktor.http.contentType
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.buildJsonArray
import kotlinx.serialization.json.buildJsonObject
import kotlinx.serialization.json.contentOrNull
import kotlinx.serialization.json.jsonArray
import kotlinx.serialization.json.jsonObject
import kotlinx.serialization.json.jsonPrimitive
import kotlinx.serialization.json.put
import kotlinx.serialization.json.putJsonObject
import kotlin.collections.set
import kotlin.math.absoluteValue
@ -168,7 +182,7 @@ class YoutubeMusic constructor(
! 4 - Duration (hh:mm:ss)
!
! We blindly gather all the details we get our hands on, then
! cherry pick the details we need based on their index numbers,
! cherry-pick the details we need based on their index numbers,
! we do so only if their Type is 'Song' or 'Video
*/
@ -180,7 +194,7 @@ class YoutubeMusic constructor(
/*
Filter Out dummies here itself
! 'musicResponsiveListItemFlexColumnRenderer' should have more that one
! sub-block, if not its a dummy, why does the YTM response contain dummies?
! sub-block, if not it is a dummy, why does the YTM response contain dummies?
! I have no clue. We skip these.
! Remember that we appended the linkBlock to result, treating that like the
@ -189,7 +203,7 @@ class YoutubeMusic constructor(
*/
for (detailArray in result.subList(0, result.size - 1)) {
for (detail in detailArray.jsonArray) {
if (detail.jsonObject["musicResponsiveListItemFlexColumnRenderer"]?.jsonObject?.size ?: 0 < 2) continue
if ((detail.jsonObject["musicResponsiveListItemFlexColumnRenderer"]?.jsonObject?.size ?: 0) < 2) continue
// if not a dummy, collect All Variables
val details =
@ -262,8 +276,8 @@ class YoutubeMusic constructor(
// most song results on youtube go by $artist - $songName or artist1/artist2
var hasCommonWord = false
val resultName = result.name?.lowercase()?.replace("-", " ")?.replace("/", " ") ?: ""
val trackNameWords = trackName.lowercase().split(" ")
val resultName = result.name?.toLowerCase()?.replace("-", " ")?.replace("/", " ") ?: ""
val trackNameWords = trackName.toLowerCase().split(" ")
for (nameWord in trackNameWords) {
if (nameWord.isNotBlank() && FuzzySearch.partialRatio(
@ -287,8 +301,8 @@ class YoutubeMusic constructor(
if (result.type == "Song") {
for (artist in trackArtists) {
if (FuzzySearch.ratio(
artist.lowercase(),
result.artist?.lowercase() ?: ""
artist.toLowerCase(),
result.artist?.toLowerCase() ?: ""
) > 85
)
artistMatchNumber++
@ -296,8 +310,8 @@ class YoutubeMusic constructor(
} else { // i.e. is a Video
for (artist in trackArtists) {
if (FuzzySearch.partialRatio(
artist.lowercase(),
result.name?.lowercase() ?: ""
artist.toLowerCase(),
result.name?.toLowerCase() ?: ""
) > 85
)
artistMatchNumber++