From 6e20cc3b0a7a4b8c573daa672fc7f7b6db2f71a2 Mon Sep 17 00:00:00 2001 From: shabinder Date: Sat, 30 Jan 2021 22:20:04 +0530 Subject: [PATCH] YoutubeMusic.kt Ported to KMP --- .../kotlin/com/shabinder/common/DI.kt | 2 + .../com/shabinder/common/YoutubeMusic.kt | 20 -- .../common/providers}/YoutubeMusic.kt | 89 +++---- .../com/shabinder/common/YoutubeMusic.kt | 252 ------------------ fuzzywuzzy/app/build.gradle | 16 +- 5 files changed, 55 insertions(+), 324 deletions(-) delete mode 100644 common/dependency-injection/src/commonMain/kotlin/com/shabinder/common/YoutubeMusic.kt rename common/dependency-injection/src/{androidMain/kotlin/com/shabinder/common => commonMain/kotlin/com/shabinder/common/providers}/YoutubeMusic.kt (77%) delete mode 100644 common/dependency-injection/src/desktopMain/kotlin/com/shabinder/common/YoutubeMusic.kt diff --git a/common/dependency-injection/src/commonMain/kotlin/com/shabinder/common/DI.kt b/common/dependency-injection/src/commonMain/kotlin/com/shabinder/common/DI.kt index 7a9d8c8c..b10375b2 100644 --- a/common/dependency-injection/src/commonMain/kotlin/com/shabinder/common/DI.kt +++ b/common/dependency-injection/src/commonMain/kotlin/com/shabinder/common/DI.kt @@ -5,6 +5,7 @@ import com.shabinder.common.database.createDb import com.shabinder.common.database.getLogger import com.shabinder.common.providers.GaanaProvider import com.shabinder.common.providers.SpotifyProvider +import com.shabinder.common.providers.YoutubeMusic import io.ktor.client.* import io.ktor.client.features.json.* import io.ktor.client.features.json.serializer.* @@ -24,6 +25,7 @@ fun commonModule(enableNetworkLogs: Boolean) = module { single { Dir() } single { createDb() } single { Kermit(getLogger()) } + single { YoutubeMusic(get(),get()) } single { SpotifyProvider(get(),get(),get(),get()) } single { GaanaProvider(get(),get(),get(),get()) } single { YoutubeProvider(get(),get(),get(),get()) } diff --git a/common/dependency-injection/src/commonMain/kotlin/com/shabinder/common/YoutubeMusic.kt b/common/dependency-injection/src/commonMain/kotlin/com/shabinder/common/YoutubeMusic.kt deleted file mode 100644 index e549ea5f..00000000 --- a/common/dependency-injection/src/commonMain/kotlin/com/shabinder/common/YoutubeMusic.kt +++ /dev/null @@ -1,20 +0,0 @@ -package com.shabinder.common - -import co.touchlab.kermit.Logger -import io.ktor.client.* - -expect class YoutubeMusic( - logger: Logger, - httpClient: HttpClient -) { - fun getYTTracks(response: String): List - fun sortByBestMatch( - ytTracks: List, - trackName: String, - trackArtists: List, - trackDurationSec: Int - ): Map - - suspend fun getYoutubeMusicResponse(query: String): String - -} \ No newline at end of file diff --git a/common/dependency-injection/src/androidMain/kotlin/com/shabinder/common/YoutubeMusic.kt b/common/dependency-injection/src/commonMain/kotlin/com/shabinder/common/providers/YoutubeMusic.kt similarity index 77% rename from common/dependency-injection/src/androidMain/kotlin/com/shabinder/common/YoutubeMusic.kt rename to common/dependency-injection/src/commonMain/kotlin/com/shabinder/common/providers/YoutubeMusic.kt index 7c2ef5ef..b33dcc46 100644 --- a/common/dependency-injection/src/androidMain/kotlin/com/shabinder/common/YoutubeMusic.kt +++ b/common/dependency-injection/src/commonMain/kotlin/com/shabinder/common/providers/YoutubeMusic.kt @@ -1,33 +1,30 @@ -package com.shabinder.common +package com.shabinder.common.providers -import android.annotation.SuppressLint import co.touchlab.kermit.Logger -import com.beust.klaxon.JsonArray -import com.beust.klaxon.JsonObject -import com.beust.klaxon.Parser +import com.shabinder.common.YoutubeTrack import com.willowtreeapps.fuzzywuzzy.diffutils.FuzzySearch import io.ktor.client.* import io.ktor.client.request.* import io.ktor.http.* -import kotlinx.serialization.json.buildJsonObject -import kotlinx.serialization.json.put -import kotlinx.serialization.json.putJsonObject +import kotlinx.serialization.json.* import kotlin.math.absoluteValue private const val apiKey = "AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30" -actual class YoutubeMusic actual constructor( +class YoutubeMusic constructor( private val logger: Logger, private val httpClient:HttpClient, ) { - private val tag = "YTMUSIC" - actual fun getYTTracks(response: String):List{ + private val tag = "YT Music" + suspend fun getYTTracks(query: String):List{ val youtubeTracks = mutableListOf() - val stringBuilder: StringBuilder = StringBuilder(response) - val responseObj: JsonObject = Parser.default().parse(stringBuilder) as JsonObject - val contentBlocks = responseObj.obj("contents")?.obj("sectionListRenderer")?.array("contents") - val resultBlocks = mutableListOf>() + val responseObj = Json.parseToJsonElement(getYoutubeMusicResponse(query)) + val contentBlocks = responseObj.jsonObject["contents"] + ?.jsonObject?.get("sectionListRenderer") + ?.jsonObject?.get("contents")?.jsonArray + + val resultBlocks = mutableListOf() if (contentBlocks != null) { for (cBlock in contentBlocks){ /** @@ -36,11 +33,12 @@ actual class YoutubeMusic actual constructor( *results for xyz, search for abc instead') we have no use for them, the for *loop below if throw a keyError if we don't ignore them */ - if(cBlock.containsKey("itemSectionRenderer")){ + if(cBlock.jsonObject.containsKey("itemSectionRenderer")){ continue } - for(contents in cBlock.obj("musicShelfRenderer")?.array("contents") ?: listOf()){ + for(contents in cBlock.jsonObject["musicShelfRenderer"]?.jsonObject?.get("contents")?.jsonArray + ?: listOf()){ /** * apparently content Blocks without an 'overlay' field don't have linkBlocks * I have no clue what they are and why there even exist @@ -51,21 +49,24 @@ actual class YoutubeMusic actual constructor( TODO check and correct }*/ - val result = contents.obj("musicResponsiveListItemRenderer") - ?.array("flexColumns") + val result = contents.jsonObject["musicResponsiveListItemRenderer"] + ?.jsonObject?.get("flexColumns")?.jsonArray //Add the linkBlock - val linkBlock = contents.obj("musicResponsiveListItemRenderer") - ?.obj("overlay") - ?.obj("musicItemThumbnailOverlayRenderer") - ?.obj("content") - ?.obj("musicPlayButtonRenderer") - ?.obj("playNavigationEndpoint") + val linkBlock = contents.jsonObject["musicResponsiveListItemRenderer"] + ?.jsonObject?.get("overlay") + ?.jsonObject?.get("musicItemThumbnailOverlayRenderer") + ?.jsonObject?.get("content") + ?.jsonObject?.get("musicPlayButtonRenderer") + ?.jsonObject?.get("playNavigationEndpoint") // detailsBlock is always a list, so we just append the linkBlock to it // instead of carrying along all the other junk from "musicResponsiveListItemRenderer" - linkBlock?.let { result?.add(it) } - result?.let { resultBlocks.add(it) } + val finalResult = buildJsonArray { + result?.let { add(it) } + linkBlock?.let { add(it) } + } + resultBlocks.add(finalResult) } } @@ -89,7 +90,7 @@ actual class YoutubeMusic actual constructor( ! 4 - Duration (hh:mm:ss) ! ! We blindly gather all the details we get our hands on, then - ! cherrypick the details we need based on their index numbers, + ! cherry pick the details we need based on their index numbers, ! we do so only if their Type is 'Song' or 'Video */ @@ -109,18 +110,17 @@ actual class YoutubeMusic actual constructor( ! result[:-1] ,i.e., skip last element in array ' */ for(detail in result.subList(0,result.size-1)){ - if(detail.obj("musicResponsiveListItemFlexColumnRenderer")?.size!! < 2) continue + if(detail.jsonObject["musicResponsiveListItemFlexColumnRenderer"]?.jsonObject?.size?:0 < 2) continue // if not a dummy, collect All Variables - val details = detail.obj("musicResponsiveListItemFlexColumnRenderer") - ?.obj("text") - ?.array("runs") ?: listOf() + val details = detail.jsonObject["musicResponsiveListItemFlexColumnRenderer"] + ?.jsonObject?.get("text") + ?.jsonObject?.get("runs")?.jsonArray ?: listOf() + for (d in details){ - d["text"]?.let { - if(it.toString() != " • "){ - availableDetails.add( - it.toString() - ) + d.jsonObject["text"]?.jsonPrimitive?.contentOrNull?.let { + if(it != " • "){ + availableDetails.add(it) } } } @@ -143,7 +143,7 @@ actual class YoutubeMusic actual constructor( ! reference the dict keys by index */ - val videoId:String? = result.last().obj("watchEndpoint")?.get("videoId") as String? + val videoId:String? = result.last().jsonObject["watchEndpoint"]?.jsonObject?.get("videoId")?.jsonPrimitive?.content val ytTrack = YoutubeTrack( name = availableDetails[0], type = availableDetails[1], @@ -152,7 +152,6 @@ actual class YoutubeMusic actual constructor( videoId = videoId ) youtubeTracks.add(ytTrack) - } } } @@ -160,11 +159,11 @@ actual class YoutubeMusic actual constructor( return youtubeTracks } - @SuppressLint("DefaultLocale") - actual fun sortByBestMatch(ytTracks:List, - trackName:String, - trackArtists:List, - trackDurationSec:Int, + fun sortByBestMatch( + ytTracks:List, + trackName:String, + trackArtists:List, + trackDurationSec:Int, ):Map{ /* * "linksWithMatchValue" is map with Youtube VideoID and its rating/match with 100 as Max Value @@ -233,7 +232,7 @@ actual class YoutubeMusic actual constructor( return linksWithMatchValue.toList().sortedByDescending { it.second }.toMap() } - actual suspend fun getYoutubeMusicResponse(query: String):String{ + private suspend fun getYoutubeMusicResponse(query: String):String{ return httpClient.post("https://music.youtube.com/youtubei/v1/search?alt=json&key=$apiKey"){ contentType(ContentType.Application.Json) headers{ diff --git a/common/dependency-injection/src/desktopMain/kotlin/com/shabinder/common/YoutubeMusic.kt b/common/dependency-injection/src/desktopMain/kotlin/com/shabinder/common/YoutubeMusic.kt deleted file mode 100644 index 5562359d..00000000 --- a/common/dependency-injection/src/desktopMain/kotlin/com/shabinder/common/YoutubeMusic.kt +++ /dev/null @@ -1,252 +0,0 @@ -package com.shabinder.common - -import co.touchlab.kermit.Logger -import com.beust.klaxon.JsonArray -import com.beust.klaxon.JsonObject -import com.beust.klaxon.Parser -import io.ktor.client.* -import io.ktor.client.request.* -import io.ktor.http.* -import kotlinx.serialization.json.buildJsonObject -import kotlinx.serialization.json.put -import kotlinx.serialization.json.putJsonObject -import me.xdrop.fuzzywuzzy.FuzzySearch -import kotlin.math.absoluteValue - -private const val apiKey = "AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30" - -actual class YoutubeMusic actual constructor( - private val logger: Logger, - private val httpClient:HttpClient, -) { - private val tag = "YTMUSIC" - actual fun getYTTracks(response: String):List{ - val youtubeTracks = mutableListOf() - - val stringBuilder: StringBuilder = StringBuilder(response) - val responseObj: JsonObject = Parser.default().parse(stringBuilder) as JsonObject - val contentBlocks = responseObj.obj("contents")?.obj("sectionListRenderer")?.array("contents") - val resultBlocks = mutableListOf>() - if (contentBlocks != null) { - for (cBlock in contentBlocks){ - /** - *Ignore user-suggestion - *The 'itemSectionRenderer' field is for user notices (stuff like - 'showing - *results for xyz, search for abc instead') we have no use for them, the for - *loop below if throw a keyError if we don't ignore them - */ - if(cBlock.containsKey("itemSectionRenderer")){ - continue - } - - for(contents in cBlock.obj("musicShelfRenderer")?.array("contents") ?: listOf()){ - /** - * apparently content Blocks without an 'overlay' field don't have linkBlocks - * I have no clue what they are and why there even exist - * - if(!contents.containsKey("overlay")){ - println(contents) - continue - TODO check and correct - }*/ - - val result = contents.obj("musicResponsiveListItemRenderer") - ?.array("flexColumns") - - //Add the linkBlock - val linkBlock = contents.obj("musicResponsiveListItemRenderer") - ?.obj("overlay") - ?.obj("musicItemThumbnailOverlayRenderer") - ?.obj("content") - ?.obj("musicPlayButtonRenderer") - ?.obj("playNavigationEndpoint") - - // detailsBlock is always a list, so we just append the linkBlock to it - // instead of carrying along all the other junk from "musicResponsiveListItemRenderer" - linkBlock?.let { result?.add(it) } - result?.let { resultBlocks.add(it) } - } - } - - /* We only need results that are Songs or Videos, so we filter out the rest, since - ! Songs and Videos are supplied with different details, extracting all details from - ! both is just carrying on redundant data, so we also have to selectively extract - ! relevant details. What you need to know to understand how we do that here: - ! - ! Songs details are ALWAYS in the following order: - ! 0 - Name - ! 1 - Type (Song) - ! 2 - com.shabinder.spotiflyer.models.gaana.Artist - ! 3 - Album - ! 4 - Duration (mm:ss) - ! - ! Video details are ALWAYS in the following order: - ! 0 - Name - ! 1 - Type (Video) - ! 2 - Channel - ! 3 - Viewers - ! 4 - Duration (hh:mm:ss) - ! - ! We blindly gather all the details we get our hands on, then - ! cherrypick the details we need based on their index numbers, - ! we do so only if their Type is 'Song' or 'Video - */ - - for(result in resultBlocks){ - - // Blindly gather available details - val availableDetails = mutableListOf() - - /* - Filter Out dummies here itself - ! 'musicResponsiveListItemFlexColumnRenderer' should have more that one - ! sub-block, if not its a dummy, why does the YTM response contain dummies? - ! I have no clue. We skip these. - - ! Remember that we appended the linkBlock to result, treating that like the - ! other constituents of a result block will lead to errors, hence the 'in - ! result[:-1] ,i.e., skip last element in array ' - */ - for(detail in result.subList(0,result.size-1)){ - if(detail.obj("musicResponsiveListItemFlexColumnRenderer")?.size!! < 2) continue - - // if not a dummy, collect All Variables - val details = detail.obj("musicResponsiveListItemFlexColumnRenderer") - ?.obj("text") - ?.array("runs") ?: listOf() - for (d in details){ - d["text"]?.let { - if(it.toString() != " • "){ - availableDetails.add( - it.toString() - ) - } - } - } - } -// log("YT Music details",availableDetails.toString()) - /* - ! Filter Out non-Song/Video results and incomplete results here itself - ! From what we know about detail order, note that [1] - indicate result type - */ - if ( availableDetails.size == 5 && availableDetails[1] in listOf("Song","Video") ){ - - // skip if result is in hours instead of minutes (no song is that long) - if(availableDetails[4].split(':').size != 2) continue - - /* - ! grab Video ID - ! this is nested as [playlistEndpoint/watchEndpoint][videoId/playlistId/...] - ! so hardcoding the dict keys for data look up is an ardours process, since - ! the sub-block pattern is fixed even though the key isn't, we just - ! reference the dict keys by index - */ - - val videoId:String? = result.last().obj("watchEndpoint")?.get("videoId") as String? - val ytTrack = YoutubeTrack( - name = availableDetails[0], - type = availableDetails[1], - artist = availableDetails[2], - duration = availableDetails[4], - videoId = videoId - ) - youtubeTracks.add(ytTrack) - - } - } - } - logger.i(youtubeTracks.joinToString(" abc \n"),tag) - return youtubeTracks - } - - actual fun sortByBestMatch(ytTracks:List, - trackName:String, - trackArtists:List, - trackDurationSec:Int, - ):Map{ - /* - * "linksWithMatchValue" is map with Youtube VideoID and its rating/match with 100 as Max Value - **/ - val linksWithMatchValue = mutableMapOf() - - for (result in ytTracks){ - - // LoweCasing Name to match Properly - // most song results on youtube go by $artist - $songName or artist1/artist2 - var hasCommonWord = false - - val resultName = result.name?.toLowerCase()?.replace("-"," ")?.replace("/"," ") ?: "" - val trackNameWords = trackName.toLowerCase().split(" ") - - for (nameWord in trackNameWords){ - if (nameWord.isNotBlank() && FuzzySearch.partialRatio(nameWord,resultName) > 85) hasCommonWord = true - } - - // Skip this Result if No Word is Common in Name - if (!hasCommonWord) { - //log("YT Api Removing", result.toString()) - continue - } - - - // Find artist match - // Will Be Using Fuzzy Search Because YT Spelling might be mucked up - // match = (no of artist names in result) / (no. of artist names on spotify) * 100 - var artistMatchNumber = 0 - - if(result.type == "Song"){ - for (artist in trackArtists){ - if(FuzzySearch.ratio(artist.toLowerCase(),result.artist?.toLowerCase()) > 85) - artistMatchNumber++ - } - }else{//i.e. is a Video - for (artist in trackArtists) { - if(FuzzySearch.partialRatio(artist.toLowerCase(),result.name?.toLowerCase()) > 85) - artistMatchNumber++ - } - } - - if(artistMatchNumber == 0) { - //log("YT Api Removing", result.toString()) - continue - } - - val artistMatch = (artistMatchNumber / trackArtists.size ) * 100 - - // Duration Match - /*! time match = 100 - (delta(duration)**2 / original duration * 100) - ! difference in song duration (delta) is usually of the magnitude of a few - ! seconds, we need to amplify the delta if it is to have any meaningful impact - ! wen we calculate the avg match value*/ - val difference = result.duration?.split(":")?.get(0)?.toInt()?.times(60) - ?.plus(result.duration?.split(":")?.get(1)?.toInt()?:0) - ?.minus(trackDurationSec)?.absoluteValue ?: 0 - val nonMatchValue :Float= ((difference*difference).toFloat()/trackDurationSec.toFloat()) - val durationMatch = 100 - (nonMatchValue*100) - - val avgMatch = (artistMatch + durationMatch)/2 - linksWithMatchValue[result.videoId.toString()] = avgMatch.toInt() - } - //log("YT Api Result", "$trackName - $linksWithMatchValue") - return linksWithMatchValue.toList().sortedByDescending { it.second }.toMap() - } - - actual suspend fun getYoutubeMusicResponse(query: String):String{ - return httpClient.post("https://music.youtube.com/youtubei/v1/search?alt=json&key=$apiKey"){ - contentType(ContentType.Application.Json) - headers{ - //append("Content-Type"," application/json") - append("Referer"," https://music.youtube.com/search") - } - body = buildJsonObject { - putJsonObject("context"){ - putJsonObject("client"){ - put("clientName" ,"WEB_REMIX") - put("clientVersion" ,"0.1") - } - } - put("query",query) - } - } - } -} \ No newline at end of file diff --git a/fuzzywuzzy/app/build.gradle b/fuzzywuzzy/app/build.gradle index 73cae3fd..a80ed937 100644 --- a/fuzzywuzzy/app/build.gradle +++ b/fuzzywuzzy/app/build.gradle @@ -24,15 +24,17 @@ kotlin { } } +/* + wasm32("wasm") iosArm64("ios") iosX64("iosSim") macosX64("macos") mingwX64("win") - wasm32("wasm") linuxArm32Hfp("linArm32") linuxMips32("linMips32") linuxMipsel32("linMipsel32") linuxX64("lin64") +*/ sourceSets { commonMain { @@ -81,16 +83,16 @@ kotlin { implementation kotlin("stdlib-js") } } - nativeMain { + /*nativeMain { kotlin.srcDir('src/nativeMain/kotlin') - } + }*/ - iosSimMain.dependsOn iosMain - iosSimTest.dependsOn iosTest + //iosSimMain.dependsOn iosMain + //iosSimTest.dependsOn iosTest - configure([targets.ios, targets.iosSim, targets.macos, targets.win, targets.linArm32, targets.linMips32, targets.linMipsel32, targets.lin64]) { + /*configure([targets.ios, targets.iosSim, targets.macos, targets.win, targets.linArm32, targets.linMips32, targets.linMipsel32, targets.lin64]) { compilations.main.source(sourceSets.nativeMain) - } + }*/ } }