From ee8ab4a6c54e33a1c6c741b367a073d60689aaea Mon Sep 17 00:00:00 2001 From: Shabinder Singh Date: Tue, 12 Oct 2021 17:39:32 +0530 Subject: [PATCH] Update and Fix Manual Yt Extraction Fixes: https://github.com/Shabinder/SpotiFlyer/issues/654 and related issues --- buildSrc/deps.versions.toml | 2 +- .../file_manager/FileManager.kt | 50 +++++++++++-------- .../FetchPlatformQueryResult.kt | 24 ++++++--- .../saavn/SaavnProvider.kt | 5 +- .../youtube/YoutubeProvider.kt | 48 ++++++++++++++++-- 5 files changed, 94 insertions(+), 35 deletions(-) diff --git a/buildSrc/deps.versions.toml b/buildSrc/deps.versions.toml index a3507216..aad933bc 100644 --- a/buildSrc/deps.versions.toml +++ b/buildSrc/deps.versions.toml @@ -68,7 +68,7 @@ slf4j-simple = { group = "org.slf4j", name = "slf4j-simple", version.ref = "slf4 i18n4k-core = { group = "de.comahe.i18n4k", name = "i18n4k-core", version.ref = "i18n4k" } i18n4k-gradle-plugin = { group = "de.comahe.i18n4k", name = "i18n4k-gradle-plugin", version.ref = "i18n4k" } -youtube-downloader = { group = "io.github.shabinder", name = "youtube-api-dl", version = "1.3" } +youtube-downloader = { group = "io.github.shabinder", name = "youtube-api-dl", version = "1.4" } fuzzy-wuzzy = { group = "io.github.shabinder", name = "fuzzywuzzy", version = "1.1" } mp3agic = { group = "com.mpatric", name = "mp3agic", version = "0.9.0" } kermit = { group = "co.touchlab", name = "kermit", version.ref = "kermit" } diff --git a/common/core-components/src/commonMain/kotlin/com.shabinder.common.core_components/file_manager/FileManager.kt b/common/core-components/src/commonMain/kotlin/com.shabinder.common.core_components/file_manager/FileManager.kt index d27662e9..8ace804a 100644 --- a/common/core-components/src/commonMain/kotlin/com.shabinder.common.core_components/file_manager/FileManager.kt +++ b/common/core-components/src/commonMain/kotlin/com.shabinder.common.core_components/file_manager/FileManager.kt @@ -136,28 +136,38 @@ suspend fun HttpClient.downloadFile(url: String) = downloadFile(url, this) suspend fun downloadFile(url: String, client: HttpClient? = null): Flow { return flow { val httpClient = client ?: createHttpClient() - val response = httpClient.get(url).execute() - // Not all requests return Content Length - val data = kotlin.runCatching { - ByteArray(response.contentLength().requireNotNull().toInt()) - }.getOrNull() ?: byteArrayOf() - var offset = 0 - do { - // Set Length optimally, after how many kb you want a progress update, now it 0.25mb - val currentRead = response.content.readAvailable(data, offset, 2_50_000) - offset += currentRead - val progress = data.size.takeIf { it != 0 }?.let { fileSize -> - (offset * 100f / fileSize).roundToInt() - } ?: 0 - emit(DownloadResult.Progress(progress)) - } while (currentRead > 0) - if (response.status.isSuccess()) { - emit(DownloadResult.Success(data)) - } else { - emit(DownloadResult.Error("File not downloaded")) + httpClient.get(url).execute { response -> + // Not all requests return Content Length + val data = kotlin.runCatching { + ByteArray(response.contentLength().requireNotNull().toInt()) + }.getOrNull() ?: byteArrayOf() + var offset = 0 + val downloadableContent = response.content + + do { + // Set Length optimally, after how many kb you want a progress update, now its 0.25mb + val currentRead = downloadableContent.readAvailable(data, offset, 2_50_000).also { + offset += it + } + + // Calculate Download Progress + val progress = data.size.takeIf { it != 0 }?.let { fileSize -> + (offset * 100f / fileSize).roundToInt() + } + + // Emit Progress Update + emit(DownloadResult.Progress(progress ?: 0)) + } while (currentRead > 0) + + // Download Complete + if (response.status.isSuccess()) { + emit(DownloadResult.Success(data)) + } else { + emit(DownloadResult.Error("File not downloaded")) + } } - // Close Client if We Created One + // Close Client if We Created One during invocation if (client == null) httpClient.close() }.catch { e -> diff --git a/common/providers/src/commonMain/kotlin/com.shabinder.common.providers/FetchPlatformQueryResult.kt b/common/providers/src/commonMain/kotlin/com.shabinder.common.providers/FetchPlatformQueryResult.kt index 87658110..35b6c6b3 100644 --- a/common/providers/src/commonMain/kotlin/com.shabinder.common.providers/FetchPlatformQueryResult.kt +++ b/common/providers/src/commonMain/kotlin/com.shabinder.common.providers/FetchPlatformQueryResult.kt @@ -121,22 +121,32 @@ class FetchPlatformQueryResult( youtubeMp3.getMp3DownloadLink( track.videoID.requireNotNull(), preferredQuality - ).let { ytMp3Link -> + ).let { ytMp3LinkRes -> if ( - ytMp3Link is SuspendableEvent.Failure + ytMp3LinkRes is SuspendableEvent.Failure || - ytMp3Link.component1().isNullOrBlank() + ytMp3LinkRes.component1().isNullOrBlank() ) { appendPadded( "Yt1sMp3 Failed for ${track.videoID}:", - ytMp3Link.component2()?.stackTraceToString() + ytMp3LinkRes.component2()?.stackTraceToString() ?: "couldn't fetch link for ${track.videoID} ,trying manual extraction" ) - //appendLine("Trying Local Extraction") - null + + appendLine("Trying Local Extraction") + runCatching { + youtubeProvider.fetchVideoM4aLink( + track.videoID.requireNotNull() + ).also { + audioQuality = it.second + audioFormat = AudioFormat.MP4 + }.first + }.onFailure { + appendPadded(it.stackTraceToString()) + }.getOrNull() } else { audioFormat = AudioFormat.MP3 - ytMp3Link.component1() + ytMp3LinkRes.component1() } } } diff --git a/common/providers/src/commonMain/kotlin/com.shabinder.common.providers/saavn/SaavnProvider.kt b/common/providers/src/commonMain/kotlin/com.shabinder.common.providers/saavn/SaavnProvider.kt index a6c7e836..478886a5 100644 --- a/common/providers/src/commonMain/kotlin/com.shabinder.common.providers/saavn/SaavnProvider.kt +++ b/common/providers/src/commonMain/kotlin/com.shabinder.common.providers/saavn/SaavnProvider.kt @@ -47,7 +47,8 @@ class SaavnProvider( coverUrl = it.image.replace("http:", "https:") } } - pageLink.contains("featured/", true) -> { // Playlist + pageLink.contains("featured/", true) + || pageLink.contains("playlist/", true) -> { // Playlist getPlaylist(fullLink).value.let { folderType = "Playlists" subFolder = removeIllegalChars(it.listname) @@ -79,7 +80,7 @@ class SaavnProvider( albumArtURL = it.image.replace("http:", "https:"), lyrics = it.lyrics ?: it.lyrics_snippet, source = Source.JioSaavn, - audioQuality = if(it.is320Kbps) AudioQuality.KBPS320 else AudioQuality.KBPS160, + audioQuality = if (it.is320Kbps) AudioQuality.KBPS320 else AudioQuality.KBPS160, outputFilePath = fileManager.finalOutputDir(it.song, type, subFolder, fileManager.defaultDir() /*".m4a"*/) ) } diff --git a/common/providers/src/commonMain/kotlin/com.shabinder.common.providers/youtube/YoutubeProvider.kt b/common/providers/src/commonMain/kotlin/com.shabinder.common.providers/youtube/YoutubeProvider.kt index 7e5714c6..38605283 100644 --- a/common/providers/src/commonMain/kotlin/com.shabinder.common.providers/youtube/YoutubeProvider.kt +++ b/common/providers/src/commonMain/kotlin/com.shabinder.common.providers/youtube/YoutubeProvider.kt @@ -28,10 +28,15 @@ import com.shabinder.common.models.event.coroutines.SuspendableEvent import com.shabinder.common.models.spotify.Source import com.shabinder.common.utils.removeIllegalChars import io.github.shabinder.YoutubeDownloader +import io.github.shabinder.models.Extension import io.github.shabinder.models.YoutubeVideo import io.github.shabinder.models.formats.Format import io.github.shabinder.models.quality.AudioQuality import io.ktor.client.HttpClient +import io.ktor.client.request.get +import io.ktor.client.statement.HttpStatement +import io.ktor.http.HttpStatusCode +import com.shabinder.common.models.AudioQuality as Quality class YoutubeProvider( private val httpClient: HttpClient, @@ -193,10 +198,43 @@ class YoutubeProvider( title = name } } + + suspend fun fetchVideoM4aLink(videoId: String, retryCount: Int = 3): Pair { + @Suppress("NAME_SHADOWING") + var retryCount = retryCount + var validM4aLink: String? = null + var audioQuality: Quality = Quality.KBPS128 + + val ex = SpotiFlyerException.DownloadLinkFetchFailed("Manual Extraction Failed for VideoID: $videoId") + while (validM4aLink.isNullOrEmpty() && retryCount > 0) { + val m4aLink = ytDownloader.getVideo(videoId).getM4aLink()?.also { + audioQuality = + if (it.bitrate > 160_000) Quality.KBPS192 else Quality.KBPS128 + }?.url + ?: throw ex + + if (validateLink(m4aLink)) { + validM4aLink = m4aLink + } + retryCount-- + } + + if (validM4aLink.isNullOrBlank()) + throw ex + + return validM4aLink to audioQuality + } + + private suspend fun validateLink(link: String): Boolean { + var status = HttpStatusCode.BadRequest + httpClient.get(link).execute { res -> status = res.status } + return status == HttpStatusCode.OK + } + + private fun YoutubeVideo.getM4aLink(): Format? { + return getAudioWithQuality(AudioQuality.high).firstOrNull { it.extension == Extension.M4A } + ?: getAudioWithQuality(AudioQuality.medium).firstOrNull { it.extension == Extension.M4A } + ?: getAudioWithQuality(AudioQuality.low).firstOrNull { it.extension == Extension.M4A } + } } -fun YoutubeVideo.get(): Format? { - return getAudioWithQuality(AudioQuality.high).getOrNull(0) - ?: getAudioWithQuality(AudioQuality.medium).getOrNull(0) - ?: getAudioWithQuality(AudioQuality.low).getOrNull(0) -}