Update and Fix Manual Yt Extraction

Fixes: https://github.com/Shabinder/SpotiFlyer/issues/654 and related issues

buildSrc/deps.versions.toml

@@ -68,7 +68,7 @@ slf4j-simple = { group = "org.slf4j", name = "slf4j-simple", version.ref = "slf4
 i18n4k-core = { group = "de.comahe.i18n4k", name = "i18n4k-core", version.ref = "i18n4k" }
 i18n4k-gradle-plugin = { group = "de.comahe.i18n4k", name = "i18n4k-gradle-plugin", version.ref = "i18n4k" }
 
-youtube-downloader = { group = "io.github.shabinder", name = "youtube-api-dl", version = "1.3" }
+youtube-downloader = { group = "io.github.shabinder", name = "youtube-api-dl", version = "1.4" }
 fuzzy-wuzzy = { group = "io.github.shabinder", name = "fuzzywuzzy", version = "1.1" }
 mp3agic = { group = "com.mpatric", name = "mp3agic", version = "0.9.0" }
 kermit = { group = "co.touchlab", name = "kermit", version.ref = "kermit" }

common/core-components/src/commonMain/kotlin/com.shabinder.common.core_components/file_manager/FileManager.kt

@@ -136,28 +136,38 @@ suspend fun HttpClient.downloadFile(url: String) = downloadFile(url, this)
 suspend fun downloadFile(url: String, client: HttpClient? = null): Flow<DownloadResult> {
     return flow {
         val httpClient = client ?: createHttpClient()
-        val response = httpClient.get<HttpStatement>(url).execute()
+        httpClient.get<HttpStatement>(url).execute { response ->
-        // Not all requests return Content Length
+            // Not all requests return Content Length
-        val data = kotlin.runCatching {
+            val data = kotlin.runCatching {
-            ByteArray(response.contentLength().requireNotNull().toInt())
+                ByteArray(response.contentLength().requireNotNull().toInt())
-        }.getOrNull() ?: byteArrayOf()
+            }.getOrNull() ?: byteArrayOf()
-        var offset = 0
+            var offset = 0
-        do {
+            val downloadableContent = response.content
-            // Set Length optimally, after how many kb you want a progress update, now it 0.25mb
+
-            val currentRead = response.content.readAvailable(data, offset, 2_50_000)
+            do {
-            offset += currentRead
+                // Set Length optimally, after how many kb you want a progress update, now its 0.25mb
-            val progress = data.size.takeIf { it != 0 }?.let { fileSize ->
+                val currentRead = downloadableContent.readAvailable(data, offset, 2_50_000).also {
-                (offset * 100f / fileSize).roundToInt()
+                    offset += it
-            } ?: 0
+                }
-            emit(DownloadResult.Progress(progress))
+
-        } while (currentRead > 0)
+                // Calculate Download Progress
-        if (response.status.isSuccess()) {
+                val progress = data.size.takeIf { it != 0 }?.let { fileSize ->
-            emit(DownloadResult.Success(data))
+                    (offset * 100f / fileSize).roundToInt()
-        } else {
+                }
-            emit(DownloadResult.Error("File not downloaded"))
+
+                // Emit Progress Update
+                emit(DownloadResult.Progress(progress ?: 0))
+            } while (currentRead > 0)
+
+            // Download Complete
+            if (response.status.isSuccess()) {
+                emit(DownloadResult.Success(data))
+            } else {
+                emit(DownloadResult.Error("File not downloaded"))
+            }
         }
 
-        // Close Client if We Created One
+        // Close Client if We Created One during invocation
         if (client == null)
             httpClient.close()
     }.catch { e ->

common/providers/src/commonMain/kotlin/com.shabinder.common.providers/FetchPlatformQueryResult.kt

@@ -121,22 +121,32 @@ class FetchPlatformQueryResult(
                         youtubeMp3.getMp3DownloadLink(
                             track.videoID.requireNotNull(),
                             preferredQuality
-                        ).let { ytMp3Link ->
+                        ).let { ytMp3LinkRes ->
                             if (
-                                ytMp3Link is SuspendableEvent.Failure
+                                ytMp3LinkRes is SuspendableEvent.Failure
                                 ||
-                                ytMp3Link.component1().isNullOrBlank()
+                                ytMp3LinkRes.component1().isNullOrBlank()
                             ) {
                                 appendPadded(
                                     "Yt1sMp3 Failed for ${track.videoID}:",
-                                    ytMp3Link.component2()?.stackTraceToString()
+                                    ytMp3LinkRes.component2()?.stackTraceToString()
                                         ?: "couldn't fetch link for ${track.videoID} ,trying manual extraction"
                                 )
-                                //appendLine("Trying Local Extraction")
+
-                                null
+                                appendLine("Trying Local Extraction")
+                                runCatching {
+                                    youtubeProvider.fetchVideoM4aLink(
+                                        track.videoID.requireNotNull()
+                                    ).also {
+                                        audioQuality = it.second
+                                        audioFormat = AudioFormat.MP4
+                                    }.first
+                                }.onFailure {
+                                    appendPadded(it.stackTraceToString())
+                                }.getOrNull()
                             } else {
                                 audioFormat = AudioFormat.MP3
-                                ytMp3Link.component1()
+                                ytMp3LinkRes.component1()
                             }
                         }
                     }

common/providers/src/commonMain/kotlin/com.shabinder.common.providers/saavn/SaavnProvider.kt

@@ -47,7 +47,8 @@ class SaavnProvider(
                         coverUrl = it.image.replace("http:", "https:")
                     }
                 }
-                pageLink.contains("featured/", true) -> { // Playlist
+                pageLink.contains("featured/", true)
+                        || pageLink.contains("playlist/", true) -> { // Playlist
                     getPlaylist(fullLink).value.let {
                         folderType = "Playlists"
                         subFolder = removeIllegalChars(it.listname)
@@ -79,7 +80,7 @@ class SaavnProvider(
             albumArtURL = it.image.replace("http:", "https:"),
             lyrics = it.lyrics ?: it.lyrics_snippet,
             source = Source.JioSaavn,
-            audioQuality = if(it.is320Kbps) AudioQuality.KBPS320 else AudioQuality.KBPS160,
+            audioQuality = if (it.is320Kbps) AudioQuality.KBPS320 else AudioQuality.KBPS160,
             outputFilePath = fileManager.finalOutputDir(it.song, type, subFolder, fileManager.defaultDir() /*".m4a"*/)
         )
     }

common/providers/src/commonMain/kotlin/com.shabinder.common.providers/youtube/YoutubeProvider.kt

@@ -28,10 +28,15 @@ import com.shabinder.common.models.event.coroutines.SuspendableEvent
 import com.shabinder.common.models.spotify.Source
 import com.shabinder.common.utils.removeIllegalChars
 import io.github.shabinder.YoutubeDownloader
+import io.github.shabinder.models.Extension
 import io.github.shabinder.models.YoutubeVideo
 import io.github.shabinder.models.formats.Format
 import io.github.shabinder.models.quality.AudioQuality
 import io.ktor.client.HttpClient
+import io.ktor.client.request.get
+import io.ktor.client.statement.HttpStatement
+import io.ktor.http.HttpStatusCode
+import com.shabinder.common.models.AudioQuality as Quality
 
 class YoutubeProvider(
     private val httpClient: HttpClient,
@@ -193,10 +198,43 @@ class YoutubeProvider(
             title = name
         }
     }
+
+    suspend fun fetchVideoM4aLink(videoId: String, retryCount: Int = 3): Pair<String, Quality> {
+        @Suppress("NAME_SHADOWING")
+        var retryCount = retryCount
+        var validM4aLink: String? = null
+        var audioQuality: Quality = Quality.KBPS128
+
+        val ex = SpotiFlyerException.DownloadLinkFetchFailed("Manual Extraction Failed for VideoID: $videoId")
+        while (validM4aLink.isNullOrEmpty() && retryCount > 0) {
+            val m4aLink = ytDownloader.getVideo(videoId).getM4aLink()?.also {
+                audioQuality =
+                    if (it.bitrate > 160_000) Quality.KBPS192 else Quality.KBPS128
+            }?.url
+                ?: throw ex
+
+            if (validateLink(m4aLink)) {
+                validM4aLink = m4aLink
+            }
+            retryCount--
+        }
+
+        if (validM4aLink.isNullOrBlank())
+            throw ex
+
+        return validM4aLink to audioQuality
+    }
+
+    private suspend fun validateLink(link: String): Boolean {
+        var status = HttpStatusCode.BadRequest
+        httpClient.get<HttpStatement>(link).execute { res -> status = res.status }
+        return status == HttpStatusCode.OK
+    }
+
+    private fun YoutubeVideo.getM4aLink(): Format? {
+        return getAudioWithQuality(AudioQuality.high).firstOrNull { it.extension == Extension.M4A }
+            ?: getAudioWithQuality(AudioQuality.medium).firstOrNull { it.extension == Extension.M4A }
+            ?: getAudioWithQuality(AudioQuality.low).firstOrNull { it.extension == Extension.M4A }
+    }
 }
 
-fun YoutubeVideo.get(): Format? {
-    return getAudioWithQuality(AudioQuality.high).getOrNull(0)
-        ?: getAudioWithQuality(AudioQuality.medium).getOrNull(0)
-        ?: getAudioWithQuality(AudioQuality.low).getOrNull(0)
-}