Madeorsk/SpotiFlyer
1
0
Fork 0
mirror of https://github.com/Shabinder/SpotiFlyer.git synced 2024-11-25 02:14:32 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

YT Match Fixes and Testing Utils

Browse Source
This commit is contained in:
shabinder 2021-09-02 23:27:17 +05:30
parent b98c4c13d1
commit ba50bc789d
11 changed files with 274 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
buildSrc/src/main/kotlin/multiplatform-setup-test.gradle.kts
View File

@ -41,19 +41,19 @@ kotlin {
sourceSets {
named("commonTest") {
dependencies {
//implementation(JetBrains.Kotlin.testCommon)
//implementation(JetBrains.Kotlin.testAnnotationsCommon)
implementation(JetBrains.Kotlin.testCommon)
implementation(JetBrains.Kotlin.testAnnotationsCommon)
}
}
named("androidTest") {
dependencies {
//implementation(JetBrains.Kotlin.testJunit)
implementation(JetBrains.Kotlin.testJunit)
}
}
named("desktopTest") {
dependencies {
//implementation(JetBrains.Kotlin.testJunit)
implementation(JetBrains.Kotlin.testJunit)
}
}
named("jsTest") {

2
common/providers/src/commonMain/kotlin/com.shabinder.common.providers/ProvidersModule.kt
View File

@ -17,6 +17,6 @@ fun providersModule(enableNetworkLogs: Boolean) = module {
single { SaavnProvider(get(), get(), get()) }
single { YoutubeProvider(get(), get(), get()) }
single { YoutubeMp3(get(), get()) }
single { YoutubeMusic(get(), get(), get(), get()) }
single { YoutubeMusic(get(), get(), get(), get(), get()) }
single { FetchPlatformQueryResult(get(), get(), get(), get(), get(), get(), get(), get(), get()) }
}

53
common/providers/src/commonMain/kotlin/com.shabinder.common.providers/youtube_music/YoutubeMusic.kt
View File

@ -17,6 +17,7 @@
package com.shabinder.common.providers.youtube_music
import co.touchlab.kermit.Kermit
import com.shabinder.common.core_components.file_manager.FileManager
import com.shabinder.common.models.*
import com.shabinder.common.models.event.coroutines.SuspendableEvent
import com.shabinder.common.models.event.coroutines.flatMap
@ -37,7 +38,8 @@ class YoutubeMusic constructor(
private val logger: Kermit,
private val httpClient: HttpClient,
private val youtubeProvider: YoutubeProvider,
private val youtubeMp3: YoutubeMp3
private val youtubeMp3: YoutubeMp3,
private val fileManager: FileManager,
) {
companion object {
const val apiKey = "AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30"
@ -47,12 +49,14 @@ class YoutubeMusic constructor(
// Get Downloadable Link
suspend fun findMp3SongDownloadURLYT(
trackDetails: TrackDetails,
preferredQuality: AudioQuality
preferredQuality: AudioQuality = fileManager.preferenceManager.audioQuality
): SuspendableEvent<String, Throwable> {
return getYTIDBestMatch(trackDetails).flatMap { videoID ->
// As YT compress Audio hence there is no benefit of quality for more than 192
val optimalQuality =
if ((preferredQuality.kbps.toIntOrNull() ?: 0) > 192) AudioQuality.KBPS192 else preferredQuality
if ((preferredQuality.kbps.toIntOrNull()
?: 0) > 192
) AudioQuality.KBPS192 else preferredQuality
// 1 Try getting Link from Yt1s
youtubeMp3.getMp3DownloadLink(videoID, optimalQuality).flatMapError {
// 2 if Yt1s failed , Extract Manually
@ -76,7 +80,8 @@ class YoutubeMusic constructor(
trackName = trackDetails.title,
trackArtists = trackDetails.artists,
trackDurationSec = trackDetails.durationSec
).keys.firstOrNull() ?: throw SpotiFlyerException.NoMatchFound(trackDetails.title)
).also { logger.d("YT-M Matches:") { it.entries.joinToString("\n") { "${it.key} --- ${it.value}" } } }.keys.firstOrNull()
?: throw SpotiFlyerException.NoMatchFound(trackDetails.title)
}
private suspend fun getYTTracks(query: String): SuspendableEvent<List<YoutubeTrack>, Throwable> =
@ -85,6 +90,10 @@ class YoutubeMusic constructor(
val responseObj = Json.parseToJsonElement(youtubeResponseData)
// logger.i { "Youtube Music Response Received" }
val contentBlocks = responseObj.jsonObject["contents"]
?.jsonObject?.get("tabbedSearchResultsRenderer")
?.jsonObject?.get("tabs")?.jsonArray?.get(0)
?.jsonObject?.get("tabRenderer")
?.jsonObject?.get("content")
?.jsonObject?.get("sectionListRenderer")
?.jsonObject?.get("contents")?.jsonArray
@ -180,7 +189,8 @@ class YoutubeMusic constructor(
if (detail.jsonObject["musicResponsiveListItemFlexColumnRenderer"]?.jsonObject?.size ?: 0 < 2) continue
// if not a dummy, collect All Variables
val details = detail.jsonObject["musicResponsiveListItemFlexColumnRenderer"]
val details =
detail.jsonObject["musicResponsiveListItemFlexColumnRenderer"]
?.jsonObject?.get("text")
?.jsonObject?.get("runs")?.jsonArray ?: listOf()
@ -198,7 +208,11 @@ class YoutubeMusic constructor(
! Filter Out non-Song/Video results and incomplete results here itself
! From what we know about detail order, note that [1] - indicate result type
*/
if (availableDetails.size == 5 && availableDetails[1] in listOf("Song", "Video")) {
if (availableDetails.size == 5 && availableDetails[1] in listOf(
"Song",
"Video"
)
) {
// skip if result is in hours instead of minutes (no song is that long)
if (availableDetails[4].split(':').size != 2) continue
@ -228,7 +242,7 @@ class YoutubeMusic constructor(
youtubeTracks
}
private fun sortByBestMatch(
fun sortByBestMatch(
ytTracks: List<YoutubeTrack>,
trackName: String,
trackArtists: List<String>,
@ -249,12 +263,16 @@ class YoutubeMusic constructor(
val trackNameWords = trackName.lowercase().split(" ")
for (nameWord in trackNameWords) {
if (nameWord.isNotBlank() && FuzzySearch.partialRatio(nameWord, resultName) > 85) hasCommonWord = true
if (nameWord.isNotBlank() && FuzzySearch.partialRatio(
nameWord,
resultName
) > 85
) hasCommonWord = true
}
// Skip this Result if No Word is Common in Name
if (!hasCommonWord) {
// log("YT Api Removing", result.toString())
logger.d("YT Api Removing No common Word") { result.toString() }
continue
}
@ -265,18 +283,26 @@ class YoutubeMusic constructor(
if (result.type == "Song") {
for (artist in trackArtists) {
if (FuzzySearch.ratio(artist.lowercase(), result.artist?.lowercase() ?: "") > 85)
if (FuzzySearch.ratio(
artist.lowercase(),
result.artist?.lowercase() ?: ""
) > 85
)
artistMatchNumber++
}
} else { // i.e. is a Video
for (artist in trackArtists) {
if (FuzzySearch.partialRatio(artist.lowercase(), result.name?.lowercase() ?: "") > 85)
if (FuzzySearch.partialRatio(
artist.lowercase(),
result.name?.lowercase() ?: ""
) > 85
)
artistMatchNumber++
}
}
if (artistMatchNumber == 0F) {
// logger.d{ "YT Api Removing: $result" }
logger.d { "YT Api Removing Artist Match 0: $result" }
continue
}
@ -302,7 +328,8 @@ class YoutubeMusic constructor(
}
}
private suspend fun getYoutubeMusicResponse(query: String): SuspendableEvent<String, Throwable> = SuspendableEvent {
private suspend fun getYoutubeMusicResponse(query: String): SuspendableEvent<String, Throwable> =
SuspendableEvent {
httpClient.post("${corsApi}https://music.youtube.com/youtubei/v1/search?alt=json&key=$apiKey") {
contentType(ContentType.Application.Json)
headers {

34
common/providers/src/commonTest/kotlin/com/shabinder/common/providers/TestSpotifyTrackMatching.kt Normal file
View File

@ -0,0 +1,34 @@
package com.shabinder.common.providers
import com.shabinder.common.models.TrackDetails
import com.shabinder.common.providers.utils.CommonUtils
import com.shabinder.common.providers.utils.SpotifyUtils
import com.shabinder.common.providers.utils.SpotifyUtils.toTrackDetailsList
import io.github.shabinder.runBlocking
import kotlin.test.Test
class TestSpotifyTrackMatching {
companion object {
const val SPOTIFY_TRACK_ID = "58f4twRnbZOOVUhMUpplJ4"
const val SPOTIFY_TRACK_LINK = "https://open.spotify.com/track/$SPOTIFY_TRACK_ID?si=e45de595053e4ee2"
const val EXPECTED_YT_VIDEO_ID = "VNs_cCtdbPc"
}
private val spotifyToken: String?
// get() = null
get() = "BQB41HqrLcrh5eRYaL97GvaH6tRe-1EktQ8VGTWUQuFnYVWBEoTcF7T_8ogqVn1GHl9HCcMiQ0HBT-ybC74"
@Test
fun matchVideo() = runBlocking {
val spotifyRequests = SpotifyUtils.getSpotifyRequests(spotifyToken)
val trackDetails: TrackDetails = spotifyRequests.getTrack(SPOTIFY_TRACK_ID).toTrackDetailsList()
println("TRACK_DETAILS: $trackDetails")
// val matched = CommonUtils.youtubeMusic.getYTTracks(CommonUtils.getYTQueryString(trackDetails))
// println("YT-MATCHES: \n ${matched.component1()?.joinToString("\n")} \n")
val ytMatch = CommonUtils.youtubeMusic.findMp3SongDownloadURLYT(trackDetails)
println("YT MATCH: $ytMatch")
}
}

43
common/providers/src/commonTest/kotlin/com/shabinder/common/providers/placeholders/FileManager.kt Normal file
View File

@ -0,0 +1,43 @@
package com.shabinder.common.providers.placeholders
import co.touchlab.kermit.Kermit
import com.shabinder.common.core_components.file_manager.FileManager
import com.shabinder.common.core_components.picture.Picture
import com.shabinder.common.database.getLogger
import com.shabinder.common.models.TrackDetails
import com.shabinder.common.models.event.coroutines.SuspendableEvent
import com.shabinder.database.Database
val FileManagerPlaceholder = object : FileManager {
override val logger: Kermit = Kermit(getLogger())
override val preferenceManager = PreferenceManagerPlaceholder
override val mediaConverter = MediaConverterPlaceholder
override val db: Database? = null
override fun isPresent(path: String): Boolean = false
override fun fileSeparator(): String = "/"
override fun defaultDir(): String = "/"
override fun imageCacheDir(): String = "/"
override fun createDirectory(dirPath: String) {}
override suspend fun cacheImage(image: Any, path: String) {}
override suspend fun loadImage(url: String, reqWidth: Int, reqHeight: Int): Picture {
TODO("Not yet implemented")
}
override suspend fun clearCache() {}
override suspend fun saveFileWithMetadata(
mp3ByteArray: ByteArray,
trackDetails: TrackDetails,
postProcess: (track: TrackDetails) -> Unit
): SuspendableEvent<String, Throwable> = SuspendableEvent.success("")
override fun addToLibrary(path: String) {}
}

14
common/providers/src/commonTest/kotlin/com/shabinder/common/providers/placeholders/MediaConverter.kt Normal file
View File

@ -0,0 +1,14 @@
package com.shabinder.common.providers.placeholders
import com.shabinder.common.core_components.media_converter.MediaConverter
import com.shabinder.common.models.AudioQuality
import com.shabinder.common.models.event.coroutines.SuspendableEvent
val MediaConverterPlaceholder = object : MediaConverter() {
override suspend fun convertAudioFile(
inputFilePath: String,
outputFilePath: String,
audioQuality: AudioQuality,
progressCallbacks: (Long) -> Unit
): SuspendableEvent<String, Throwable> = SuspendableEvent.success("")
}

31
common/providers/src/commonTest/kotlin/com/shabinder/common/providers/placeholders/PreferenceManager.kt Normal file
View File

@ -0,0 +1,31 @@
package com.shabinder.common.providers.placeholders
import com.russhwolf.settings.Settings
import com.shabinder.common.core_components.preference_manager.PreferenceManager
private val settings = object : Settings {
override val keys: Set<String> = setOf()
override val size: Int = 0
override fun clear() {}
override fun getBoolean(key: String, defaultValue: Boolean): Boolean = false
override fun getBooleanOrNull(key: String): Boolean? = null
override fun getDouble(key: String, defaultValue: Double): Double = 0.0
override fun getDoubleOrNull(key: String): Double? = null
override fun getFloat(key: String, defaultValue: Float): Float = 0f
override fun getFloatOrNull(key: String): Float? = null
override fun getInt(key: String, defaultValue: Int): Int = 0
override fun getIntOrNull(key: String): Int? = null
override fun getLong(key: String, defaultValue: Long): Long = 0L
override fun getLongOrNull(key: String): Long? = null
override fun getString(key: String, defaultValue: String): String = ""
override fun getStringOrNull(key: String): String? = null
override fun hasKey(key: String): Boolean = false
override fun putBoolean(key: String, value: Boolean) {}
override fun putDouble(key: String, value: Double) {}
override fun putFloat(key: String, value: Float) {}
override fun putInt(key: String, value: Int) {}
override fun putLong(key: String, value: Long) {}
override fun putString(key: String, value: String) {}
override fun remove(key: String) {}
}
val PreferenceManagerPlaceholder = PreferenceManager(settings)

20
common/providers/src/commonTest/kotlin/com/shabinder/common/providers/utils/CommonUtils.kt Normal file
View File

@ -0,0 +1,20 @@
package com.shabinder.common.providers.utils
import co.touchlab.kermit.Kermit
import com.shabinder.common.core_components.utils.createHttpClient
import com.shabinder.common.database.getLogger
import com.shabinder.common.models.TrackDetails
import com.shabinder.common.providers.placeholders.FileManagerPlaceholder
import com.shabinder.common.providers.youtube.YoutubeProvider
import com.shabinder.common.providers.youtube_music.YoutubeMusic
import com.shabinder.common.providers.youtube_to_mp3.requests.YoutubeMp3
object CommonUtils {
val httpClient by lazy { createHttpClient() }
val logger by lazy { Kermit(getLogger()) }
val youtubeProvider by lazy { YoutubeProvider(httpClient, logger, FileManagerPlaceholder) }
val youtubeMp3 = YoutubeMp3(httpClient, logger)
val youtubeMusic = YoutubeMusic(logger, httpClient, youtubeProvider, youtubeMp3, FileManagerPlaceholder)
fun getYTQueryString(trackDetails: TrackDetails) = "${trackDetails.title} - ${trackDetails.artists.joinToString(",")}"
}

68
common/providers/src/commonTest/kotlin/com/shabinder/common/providers/utils/SpotifyUtils.kt Normal file
View File

@ -0,0 +1,68 @@
package com.shabinder.common.providers.utils
import com.shabinder.common.core_components.file_manager.finalOutputDir
import com.shabinder.common.models.DownloadStatus
import com.shabinder.common.models.NativeAtomicReference
import com.shabinder.common.models.SpotiFlyerException
import com.shabinder.common.models.TrackDetails
import com.shabinder.common.models.spotify.Source
import com.shabinder.common.models.spotify.Track
import com.shabinder.common.providers.spotify.requests.SpotifyRequests
import com.shabinder.common.providers.spotify.requests.authenticateSpotify
import com.shabinder.common.utils.globalJson
import io.ktor.client.*
import io.ktor.client.features.*
import io.ktor.client.features.json.*
import io.ktor.client.features.json.serializer.*
import io.ktor.client.request.*
object SpotifyUtils {
suspend fun getSpotifyRequests(spotifyToken: String? = null): SpotifyRequests {
val spotifyClient = getSpotifyClient(spotifyToken)
return object : SpotifyRequests {
override val httpClientRef: NativeAtomicReference<HttpClient> = NativeAtomicReference(spotifyClient)
override suspend fun authenticateSpotifyClient(override: Boolean) { httpClientRef.value = getSpotifyClient(spotifyToken) }
}
}
suspend fun getSpotifyClient(spotifyToken: String? = null): HttpClient {
val token = spotifyToken ?: authenticateSpotify().component1()?.access_token
return if (token == null) {
println("Spotify Auth Failed: Please Check your Network Connection")
throw SpotiFlyerException.NoInternetException()
} else {
println("Spotify Token: $token")
HttpClient {
defaultRequest {
header("Authorization", "Bearer $token")
}
install(JsonFeature) {
serializer = KotlinxSerializer(globalJson)
}
}
}
}
fun Track.toTrackDetailsList(type: String = "Track", subFolder: String = "SpotifyFolder") = let {
TrackDetails(
title = it.name.toString(),
trackNumber = it.track_number,
genre = it.album?.genres?.filterNotNull() ?: emptyList(),
artists = it.artists?.map { artist -> artist?.name.toString() } ?: listOf(),
albumArtists = it.album?.artists?.mapNotNull { artist -> artist?.name } ?: emptyList(),
durationSec = (it.duration_ms / 1000).toInt(),
albumArtPath = (it.album?.images?.firstOrNull()?.url.toString()).substringAfterLast(
'/'
) + ".jpeg",
albumName = it.album?.name,
year = it.album?.release_date,
comment = "Genres:${it.album?.genres?.joinToString()}",
trackUrl = it.href,
downloaded = DownloadStatus.NotDownloaded,
source = Source.Spotify,
albumArtURL = it.album?.images?.firstOrNull()?.url.toString(),
outputFilePath = ""
)
}
}

1
maintenance-tasks/build.gradle.kts
View File

@ -25,6 +25,7 @@ dependencies {
implementation(Ktor.clientLogging)
implementation(Ktor.clientSerialization)
implementation(Serialization.json)
// testDeps
testImplementation("org.jetbrains.kotlin:kotlin-test-junit:1.5.21")
}

4
maintenance-tasks/src/main/java/utils/TestClass.kt
View File

@ -3,4 +3,6 @@ package utils
import kotlinx.coroutines.runBlocking
// Test Class- at development Time
fun main(): Unit = runBlocking {}
fun main(): Unit = runBlocking {
}