From ba50bc789dcd3a8ff696e5b4cc07d662dccf7c43 Mon Sep 17 00:00:00 2001 From: shabinder Date: Thu, 2 Sep 2021 23:27:17 +0530 Subject: [PATCH] YT Match Fixes and Testing Utils --- .../multiplatform-setup-test.gradle.kts | 8 +- .../ProvidersModule.kt | 2 +- .../youtube_music/YoutubeMusic.kt | 83 ++++++++++++------- .../providers/TestSpotifyTrackMatching.kt | 34 ++++++++ .../providers/placeholders/FileManager.kt | 43 ++++++++++ .../providers/placeholders/MediaConverter.kt | 14 ++++ .../placeholders/PreferenceManager.kt | 31 +++++++ .../common/providers/utils/CommonUtils.kt | 20 +++++ .../common/providers/utils/SpotifyUtils.kt | 68 +++++++++++++++ maintenance-tasks/build.gradle.kts | 1 + .../src/main/java/utils/TestClass.kt | 4 +- 11 files changed, 274 insertions(+), 34 deletions(-) create mode 100644 common/providers/src/commonTest/kotlin/com/shabinder/common/providers/TestSpotifyTrackMatching.kt create mode 100644 common/providers/src/commonTest/kotlin/com/shabinder/common/providers/placeholders/FileManager.kt create mode 100644 common/providers/src/commonTest/kotlin/com/shabinder/common/providers/placeholders/MediaConverter.kt create mode 100644 common/providers/src/commonTest/kotlin/com/shabinder/common/providers/placeholders/PreferenceManager.kt create mode 100644 common/providers/src/commonTest/kotlin/com/shabinder/common/providers/utils/CommonUtils.kt create mode 100644 common/providers/src/commonTest/kotlin/com/shabinder/common/providers/utils/SpotifyUtils.kt diff --git a/buildSrc/src/main/kotlin/multiplatform-setup-test.gradle.kts b/buildSrc/src/main/kotlin/multiplatform-setup-test.gradle.kts index 0363cb46..d2741101 100644 --- a/buildSrc/src/main/kotlin/multiplatform-setup-test.gradle.kts +++ b/buildSrc/src/main/kotlin/multiplatform-setup-test.gradle.kts @@ -41,19 +41,19 @@ kotlin { sourceSets { named("commonTest") { dependencies { - //implementation(JetBrains.Kotlin.testCommon) - //implementation(JetBrains.Kotlin.testAnnotationsCommon) + implementation(JetBrains.Kotlin.testCommon) + implementation(JetBrains.Kotlin.testAnnotationsCommon) } } named("androidTest") { dependencies { - //implementation(JetBrains.Kotlin.testJunit) + implementation(JetBrains.Kotlin.testJunit) } } named("desktopTest") { dependencies { - //implementation(JetBrains.Kotlin.testJunit) + implementation(JetBrains.Kotlin.testJunit) } } named("jsTest") { diff --git a/common/providers/src/commonMain/kotlin/com.shabinder.common.providers/ProvidersModule.kt b/common/providers/src/commonMain/kotlin/com.shabinder.common.providers/ProvidersModule.kt index 959ab95c..44c67ea3 100644 --- a/common/providers/src/commonMain/kotlin/com.shabinder.common.providers/ProvidersModule.kt +++ b/common/providers/src/commonMain/kotlin/com.shabinder.common.providers/ProvidersModule.kt @@ -17,6 +17,6 @@ fun providersModule(enableNetworkLogs: Boolean) = module { single { SaavnProvider(get(), get(), get()) } single { YoutubeProvider(get(), get(), get()) } single { YoutubeMp3(get(), get()) } - single { YoutubeMusic(get(), get(), get(), get()) } + single { YoutubeMusic(get(), get(), get(), get(), get()) } single { FetchPlatformQueryResult(get(), get(), get(), get(), get(), get(), get(), get(), get()) } } diff --git a/common/providers/src/commonMain/kotlin/com.shabinder.common.providers/youtube_music/YoutubeMusic.kt b/common/providers/src/commonMain/kotlin/com.shabinder.common.providers/youtube_music/YoutubeMusic.kt index efe0308f..1ff59d24 100644 --- a/common/providers/src/commonMain/kotlin/com.shabinder.common.providers/youtube_music/YoutubeMusic.kt +++ b/common/providers/src/commonMain/kotlin/com.shabinder.common.providers/youtube_music/YoutubeMusic.kt @@ -17,6 +17,7 @@ package com.shabinder.common.providers.youtube_music import co.touchlab.kermit.Kermit +import com.shabinder.common.core_components.file_manager.FileManager import com.shabinder.common.models.* import com.shabinder.common.models.event.coroutines.SuspendableEvent import com.shabinder.common.models.event.coroutines.flatMap @@ -37,7 +38,8 @@ class YoutubeMusic constructor( private val logger: Kermit, private val httpClient: HttpClient, private val youtubeProvider: YoutubeProvider, - private val youtubeMp3: YoutubeMp3 + private val youtubeMp3: YoutubeMp3, + private val fileManager: FileManager, ) { companion object { const val apiKey = "AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30" @@ -47,12 +49,14 @@ class YoutubeMusic constructor( // Get Downloadable Link suspend fun findMp3SongDownloadURLYT( trackDetails: TrackDetails, - preferredQuality: AudioQuality + preferredQuality: AudioQuality = fileManager.preferenceManager.audioQuality ): SuspendableEvent { return getYTIDBestMatch(trackDetails).flatMap { videoID -> // As YT compress Audio hence there is no benefit of quality for more than 192 val optimalQuality = - if ((preferredQuality.kbps.toIntOrNull() ?: 0) > 192) AudioQuality.KBPS192 else preferredQuality + if ((preferredQuality.kbps.toIntOrNull() + ?: 0) > 192 + ) AudioQuality.KBPS192 else preferredQuality // 1 Try getting Link from Yt1s youtubeMp3.getMp3DownloadLink(videoID, optimalQuality).flatMapError { // 2 if Yt1s failed , Extract Manually @@ -76,7 +80,8 @@ class YoutubeMusic constructor( trackName = trackDetails.title, trackArtists = trackDetails.artists, trackDurationSec = trackDetails.durationSec - ).keys.firstOrNull() ?: throw SpotiFlyerException.NoMatchFound(trackDetails.title) + ).also { logger.d("YT-M Matches:") { it.entries.joinToString("\n") { "${it.key} --- ${it.value}" } } }.keys.firstOrNull() + ?: throw SpotiFlyerException.NoMatchFound(trackDetails.title) } private suspend fun getYTTracks(query: String): SuspendableEvent, Throwable> = @@ -85,6 +90,10 @@ class YoutubeMusic constructor( val responseObj = Json.parseToJsonElement(youtubeResponseData) // logger.i { "Youtube Music Response Received" } val contentBlocks = responseObj.jsonObject["contents"] + ?.jsonObject?.get("tabbedSearchResultsRenderer") + ?.jsonObject?.get("tabs")?.jsonArray?.get(0) + ?.jsonObject?.get("tabRenderer") + ?.jsonObject?.get("content") ?.jsonObject?.get("sectionListRenderer") ?.jsonObject?.get("contents")?.jsonArray @@ -180,9 +189,10 @@ class YoutubeMusic constructor( if (detail.jsonObject["musicResponsiveListItemFlexColumnRenderer"]?.jsonObject?.size ?: 0 < 2) continue // if not a dummy, collect All Variables - val details = detail.jsonObject["musicResponsiveListItemFlexColumnRenderer"] - ?.jsonObject?.get("text") - ?.jsonObject?.get("runs")?.jsonArray ?: listOf() + val details = + detail.jsonObject["musicResponsiveListItemFlexColumnRenderer"] + ?.jsonObject?.get("text") + ?.jsonObject?.get("runs")?.jsonArray ?: listOf() for (d in details) { d.jsonObject["text"]?.jsonPrimitive?.contentOrNull?.let { @@ -198,7 +208,11 @@ class YoutubeMusic constructor( ! Filter Out non-Song/Video results and incomplete results here itself ! From what we know about detail order, note that [1] - indicate result type */ - if (availableDetails.size == 5 && availableDetails[1] in listOf("Song", "Video")) { + if (availableDetails.size == 5 && availableDetails[1] in listOf( + "Song", + "Video" + ) + ) { // skip if result is in hours instead of minutes (no song is that long) if (availableDetails[4].split(':').size != 2) continue @@ -228,7 +242,7 @@ class YoutubeMusic constructor( youtubeTracks } - private fun sortByBestMatch( + fun sortByBestMatch( ytTracks: List, trackName: String, trackArtists: List, @@ -249,12 +263,16 @@ class YoutubeMusic constructor( val trackNameWords = trackName.lowercase().split(" ") for (nameWord in trackNameWords) { - if (nameWord.isNotBlank() && FuzzySearch.partialRatio(nameWord, resultName) > 85) hasCommonWord = true + if (nameWord.isNotBlank() && FuzzySearch.partialRatio( + nameWord, + resultName + ) > 85 + ) hasCommonWord = true } // Skip this Result if No Word is Common in Name if (!hasCommonWord) { - // log("YT Api Removing", result.toString()) + logger.d("YT Api Removing No common Word") { result.toString() } continue } @@ -265,18 +283,26 @@ class YoutubeMusic constructor( if (result.type == "Song") { for (artist in trackArtists) { - if (FuzzySearch.ratio(artist.lowercase(), result.artist?.lowercase() ?: "") > 85) + if (FuzzySearch.ratio( + artist.lowercase(), + result.artist?.lowercase() ?: "" + ) > 85 + ) artistMatchNumber++ } } else { // i.e. is a Video for (artist in trackArtists) { - if (FuzzySearch.partialRatio(artist.lowercase(), result.name?.lowercase() ?: "") > 85) + if (FuzzySearch.partialRatio( + artist.lowercase(), + result.name?.lowercase() ?: "" + ) > 85 + ) artistMatchNumber++ } } if (artistMatchNumber == 0F) { - // logger.d{ "YT Api Removing: $result" } + logger.d { "YT Api Removing Artist Match 0: $result" } continue } @@ -302,21 +328,22 @@ class YoutubeMusic constructor( } } - private suspend fun getYoutubeMusicResponse(query: String): SuspendableEvent = SuspendableEvent { - httpClient.post("${corsApi}https://music.youtube.com/youtubei/v1/search?alt=json&key=$apiKey") { - contentType(ContentType.Application.Json) - headers { - append("referer", "https://music.youtube.com/search") - } - body = buildJsonObject { - putJsonObject("context") { - putJsonObject("client") { - put("clientName", "WEB_REMIX") - put("clientVersion", "0.1") - } + private suspend fun getYoutubeMusicResponse(query: String): SuspendableEvent = + SuspendableEvent { + httpClient.post("${corsApi}https://music.youtube.com/youtubei/v1/search?alt=json&key=$apiKey") { + contentType(ContentType.Application.Json) + headers { + append("referer", "https://music.youtube.com/search") + } + body = buildJsonObject { + putJsonObject("context") { + putJsonObject("client") { + put("clientName", "WEB_REMIX") + put("clientVersion", "0.1") + } + } + put("query", query) } - put("query", query) } } - } } diff --git a/common/providers/src/commonTest/kotlin/com/shabinder/common/providers/TestSpotifyTrackMatching.kt b/common/providers/src/commonTest/kotlin/com/shabinder/common/providers/TestSpotifyTrackMatching.kt new file mode 100644 index 00000000..d32cbcb0 --- /dev/null +++ b/common/providers/src/commonTest/kotlin/com/shabinder/common/providers/TestSpotifyTrackMatching.kt @@ -0,0 +1,34 @@ +package com.shabinder.common.providers + +import com.shabinder.common.models.TrackDetails +import com.shabinder.common.providers.utils.CommonUtils +import com.shabinder.common.providers.utils.SpotifyUtils +import com.shabinder.common.providers.utils.SpotifyUtils.toTrackDetailsList +import io.github.shabinder.runBlocking +import kotlin.test.Test + +class TestSpotifyTrackMatching { + + companion object { + const val SPOTIFY_TRACK_ID = "58f4twRnbZOOVUhMUpplJ4" + const val SPOTIFY_TRACK_LINK = "https://open.spotify.com/track/$SPOTIFY_TRACK_ID?si=e45de595053e4ee2" + const val EXPECTED_YT_VIDEO_ID = "VNs_cCtdbPc" + } + + private val spotifyToken: String? +// get() = null + get() = "BQB41HqrLcrh5eRYaL97GvaH6tRe-1EktQ8VGTWUQuFnYVWBEoTcF7T_8ogqVn1GHl9HCcMiQ0HBT-ybC74" + + @Test + fun matchVideo() = runBlocking { + val spotifyRequests = SpotifyUtils.getSpotifyRequests(spotifyToken) + + val trackDetails: TrackDetails = spotifyRequests.getTrack(SPOTIFY_TRACK_ID).toTrackDetailsList() + println("TRACK_DETAILS: $trackDetails") + +// val matched = CommonUtils.youtubeMusic.getYTTracks(CommonUtils.getYTQueryString(trackDetails)) +// println("YT-MATCHES: \n ${matched.component1()?.joinToString("\n")} \n") + val ytMatch = CommonUtils.youtubeMusic.findMp3SongDownloadURLYT(trackDetails) + println("YT MATCH: $ytMatch") + } +} \ No newline at end of file diff --git a/common/providers/src/commonTest/kotlin/com/shabinder/common/providers/placeholders/FileManager.kt b/common/providers/src/commonTest/kotlin/com/shabinder/common/providers/placeholders/FileManager.kt new file mode 100644 index 00000000..87ea6b8b --- /dev/null +++ b/common/providers/src/commonTest/kotlin/com/shabinder/common/providers/placeholders/FileManager.kt @@ -0,0 +1,43 @@ +package com.shabinder.common.providers.placeholders + +import co.touchlab.kermit.Kermit +import com.shabinder.common.core_components.file_manager.FileManager +import com.shabinder.common.core_components.picture.Picture +import com.shabinder.common.database.getLogger +import com.shabinder.common.models.TrackDetails +import com.shabinder.common.models.event.coroutines.SuspendableEvent +import com.shabinder.database.Database + +val FileManagerPlaceholder = object : FileManager { + override val logger: Kermit = Kermit(getLogger()) + override val preferenceManager = PreferenceManagerPlaceholder + override val mediaConverter = MediaConverterPlaceholder + + override val db: Database? = null + + override fun isPresent(path: String): Boolean = false + + override fun fileSeparator(): String = "/" + + override fun defaultDir(): String = "/" + + override fun imageCacheDir(): String = "/" + + override fun createDirectory(dirPath: String) {} + + override suspend fun cacheImage(image: Any, path: String) {} + + override suspend fun loadImage(url: String, reqWidth: Int, reqHeight: Int): Picture { + TODO("Not yet implemented") + } + + override suspend fun clearCache() {} + + override suspend fun saveFileWithMetadata( + mp3ByteArray: ByteArray, + trackDetails: TrackDetails, + postProcess: (track: TrackDetails) -> Unit + ): SuspendableEvent = SuspendableEvent.success("") + + override fun addToLibrary(path: String) {} +} \ No newline at end of file diff --git a/common/providers/src/commonTest/kotlin/com/shabinder/common/providers/placeholders/MediaConverter.kt b/common/providers/src/commonTest/kotlin/com/shabinder/common/providers/placeholders/MediaConverter.kt new file mode 100644 index 00000000..2ef2b45b --- /dev/null +++ b/common/providers/src/commonTest/kotlin/com/shabinder/common/providers/placeholders/MediaConverter.kt @@ -0,0 +1,14 @@ +package com.shabinder.common.providers.placeholders + +import com.shabinder.common.core_components.media_converter.MediaConverter +import com.shabinder.common.models.AudioQuality +import com.shabinder.common.models.event.coroutines.SuspendableEvent + +val MediaConverterPlaceholder = object : MediaConverter() { + override suspend fun convertAudioFile( + inputFilePath: String, + outputFilePath: String, + audioQuality: AudioQuality, + progressCallbacks: (Long) -> Unit + ): SuspendableEvent = SuspendableEvent.success("") +} \ No newline at end of file diff --git a/common/providers/src/commonTest/kotlin/com/shabinder/common/providers/placeholders/PreferenceManager.kt b/common/providers/src/commonTest/kotlin/com/shabinder/common/providers/placeholders/PreferenceManager.kt new file mode 100644 index 00000000..70232c6f --- /dev/null +++ b/common/providers/src/commonTest/kotlin/com/shabinder/common/providers/placeholders/PreferenceManager.kt @@ -0,0 +1,31 @@ +package com.shabinder.common.providers.placeholders + +import com.russhwolf.settings.Settings +import com.shabinder.common.core_components.preference_manager.PreferenceManager + +private val settings = object : Settings { + override val keys: Set = setOf() + override val size: Int = 0 + override fun clear() {} + override fun getBoolean(key: String, defaultValue: Boolean): Boolean = false + override fun getBooleanOrNull(key: String): Boolean? = null + override fun getDouble(key: String, defaultValue: Double): Double = 0.0 + override fun getDoubleOrNull(key: String): Double? = null + override fun getFloat(key: String, defaultValue: Float): Float = 0f + override fun getFloatOrNull(key: String): Float? = null + override fun getInt(key: String, defaultValue: Int): Int = 0 + override fun getIntOrNull(key: String): Int? = null + override fun getLong(key: String, defaultValue: Long): Long = 0L + override fun getLongOrNull(key: String): Long? = null + override fun getString(key: String, defaultValue: String): String = "" + override fun getStringOrNull(key: String): String? = null + override fun hasKey(key: String): Boolean = false + override fun putBoolean(key: String, value: Boolean) {} + override fun putDouble(key: String, value: Double) {} + override fun putFloat(key: String, value: Float) {} + override fun putInt(key: String, value: Int) {} + override fun putLong(key: String, value: Long) {} + override fun putString(key: String, value: String) {} + override fun remove(key: String) {} +} +val PreferenceManagerPlaceholder = PreferenceManager(settings) diff --git a/common/providers/src/commonTest/kotlin/com/shabinder/common/providers/utils/CommonUtils.kt b/common/providers/src/commonTest/kotlin/com/shabinder/common/providers/utils/CommonUtils.kt new file mode 100644 index 00000000..53d253d0 --- /dev/null +++ b/common/providers/src/commonTest/kotlin/com/shabinder/common/providers/utils/CommonUtils.kt @@ -0,0 +1,20 @@ +package com.shabinder.common.providers.utils + +import co.touchlab.kermit.Kermit +import com.shabinder.common.core_components.utils.createHttpClient +import com.shabinder.common.database.getLogger +import com.shabinder.common.models.TrackDetails +import com.shabinder.common.providers.placeholders.FileManagerPlaceholder +import com.shabinder.common.providers.youtube.YoutubeProvider +import com.shabinder.common.providers.youtube_music.YoutubeMusic +import com.shabinder.common.providers.youtube_to_mp3.requests.YoutubeMp3 + +object CommonUtils { + val httpClient by lazy { createHttpClient() } + val logger by lazy { Kermit(getLogger()) } + val youtubeProvider by lazy { YoutubeProvider(httpClient, logger, FileManagerPlaceholder) } + val youtubeMp3 = YoutubeMp3(httpClient, logger) + val youtubeMusic = YoutubeMusic(logger, httpClient, youtubeProvider, youtubeMp3, FileManagerPlaceholder) + + fun getYTQueryString(trackDetails: TrackDetails) = "${trackDetails.title} - ${trackDetails.artists.joinToString(",")}" +} \ No newline at end of file diff --git a/common/providers/src/commonTest/kotlin/com/shabinder/common/providers/utils/SpotifyUtils.kt b/common/providers/src/commonTest/kotlin/com/shabinder/common/providers/utils/SpotifyUtils.kt new file mode 100644 index 00000000..b272c1a1 --- /dev/null +++ b/common/providers/src/commonTest/kotlin/com/shabinder/common/providers/utils/SpotifyUtils.kt @@ -0,0 +1,68 @@ +package com.shabinder.common.providers.utils + +import com.shabinder.common.core_components.file_manager.finalOutputDir +import com.shabinder.common.models.DownloadStatus +import com.shabinder.common.models.NativeAtomicReference +import com.shabinder.common.models.SpotiFlyerException +import com.shabinder.common.models.TrackDetails +import com.shabinder.common.models.spotify.Source +import com.shabinder.common.models.spotify.Track +import com.shabinder.common.providers.spotify.requests.SpotifyRequests +import com.shabinder.common.providers.spotify.requests.authenticateSpotify +import com.shabinder.common.utils.globalJson +import io.ktor.client.* +import io.ktor.client.features.* +import io.ktor.client.features.json.* +import io.ktor.client.features.json.serializer.* +import io.ktor.client.request.* + +object SpotifyUtils { + + suspend fun getSpotifyRequests(spotifyToken: String? = null): SpotifyRequests { + val spotifyClient = getSpotifyClient(spotifyToken) + return object : SpotifyRequests { + override val httpClientRef: NativeAtomicReference = NativeAtomicReference(spotifyClient) + override suspend fun authenticateSpotifyClient(override: Boolean) { httpClientRef.value = getSpotifyClient(spotifyToken) } + } + } + + suspend fun getSpotifyClient(spotifyToken: String? = null): HttpClient { + val token = spotifyToken ?: authenticateSpotify().component1()?.access_token + return if (token == null) { + println("Spotify Auth Failed: Please Check your Network Connection") + throw SpotiFlyerException.NoInternetException() + } else { + println("Spotify Token: $token") + HttpClient { + defaultRequest { + header("Authorization", "Bearer $token") + } + install(JsonFeature) { + serializer = KotlinxSerializer(globalJson) + } + } + } + } + + fun Track.toTrackDetailsList(type: String = "Track", subFolder: String = "SpotifyFolder") = let { + TrackDetails( + title = it.name.toString(), + trackNumber = it.track_number, + genre = it.album?.genres?.filterNotNull() ?: emptyList(), + artists = it.artists?.map { artist -> artist?.name.toString() } ?: listOf(), + albumArtists = it.album?.artists?.mapNotNull { artist -> artist?.name } ?: emptyList(), + durationSec = (it.duration_ms / 1000).toInt(), + albumArtPath = (it.album?.images?.firstOrNull()?.url.toString()).substringAfterLast( + '/' + ) + ".jpeg", + albumName = it.album?.name, + year = it.album?.release_date, + comment = "Genres:${it.album?.genres?.joinToString()}", + trackUrl = it.href, + downloaded = DownloadStatus.NotDownloaded, + source = Source.Spotify, + albumArtURL = it.album?.images?.firstOrNull()?.url.toString(), + outputFilePath = "" + ) + } +} \ No newline at end of file diff --git a/maintenance-tasks/build.gradle.kts b/maintenance-tasks/build.gradle.kts index e48e44da..b0395302 100644 --- a/maintenance-tasks/build.gradle.kts +++ b/maintenance-tasks/build.gradle.kts @@ -25,6 +25,7 @@ dependencies { implementation(Ktor.clientLogging) implementation(Ktor.clientSerialization) implementation(Serialization.json) + // testDeps testImplementation("org.jetbrains.kotlin:kotlin-test-junit:1.5.21") } diff --git a/maintenance-tasks/src/main/java/utils/TestClass.kt b/maintenance-tasks/src/main/java/utils/TestClass.kt index 3ffb46b9..4283c707 100644 --- a/maintenance-tasks/src/main/java/utils/TestClass.kt +++ b/maintenance-tasks/src/main/java/utils/TestClass.kt @@ -3,4 +3,6 @@ package utils import kotlinx.coroutines.runBlocking // Test Class- at development Time -fun main(): Unit = runBlocking {} +fun main(): Unit = runBlocking { + +}