Madeorsk/SpotiFlyer
1
0
Fork 0
mirror of https://github.com/Shabinder/SpotiFlyer.git synced 2024-11-22 09:04:32 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

320Kbps and JioSaavn Support

Browse Source
This commit is contained in:
shabinder 2021-05-26 04:02:21 +05:30
parent ec8f77d121
commit 3913bfa4b1
17 changed files with 514 additions and 393 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
android/src/main/AndroidManifest.xml
View File

@ -71,11 +71,6 @@
</intent-filter>
</activity>
<meta-data
android:name="com.razorpay.ApiKey"
android:value="rzp_live_3ZQeoFYOxjmXye"
/>
<service android:name="com.shabinder.common.di.worker.ForegroundService"/>
</application>
</manifest>

24
common/data-models/src/commonMain/kotlin/com/shabinder/common/models/AudioQuality.kt Normal file
View File

@ -0,0 +1,24 @@
package com.shabinder.common.models
enum class AudioQuality(val kbps: String) {
KBPS128("128"),
KBPS160("160"),
KBPS192("192"),
KBPS224("224"),
KBPS256("256"),
KBPS320("320");
companion object {
fun getQuality(kbps: String): AudioQuality {
return when (kbps) {
"128" -> KBPS128
"160" -> KBPS160
"192" -> KBPS192
"224" -> KBPS224
"256" -> KBPS256
"320" -> KBPS320
else -> KBPS160 // Use 160 as baseline
}
}
}
}

4
common/data-models/src/commonMain/kotlin/com/shabinder/common/models/saavn/SaavnSong.kt
View File

@ -7,7 +7,7 @@ import kotlinx.serialization.json.JsonNames
@Serializable
data class SaavnSong @OptIn(ExperimentalSerializationApi::class) constructor(
@JsonNames("320kbps") val is320kbps: Boolean = false,
@JsonNames("320kbps") val is320Kbps: Boolean,
val album: String,
val album_url: String? = null,
val albumid: String? = null,
@ -23,8 +23,8 @@ data class SaavnSong @OptIn(ExperimentalSerializationApi::class) constructor(
val label: String? = null,
val label_url: String? = null,
val language: String,
val lyrics: String? = null,
val lyrics_snippet: String? = null,
val lyrics: String? = null,
val media_preview_url: String? = null,
val media_url: String? = null, // Downloadable M4A Link
val music: String,

29
common/dependency-injection/src/androidMain/kotlin/com/shabinder/common/di/worker/ForegroundService.kt
View File

@ -37,13 +37,11 @@ import com.shabinder.common.di.Dir
import com.shabinder.common.di.FetchPlatformQueryResult
import com.shabinder.common.di.R
import com.shabinder.common.di.downloadFile
import com.shabinder.common.di.providers.get
import com.shabinder.common.di.utils.ParallelExecutor
import com.shabinder.common.models.DownloadResult
import com.shabinder.common.models.DownloadStatus
import com.shabinder.common.models.Status
import com.shabinder.common.models.TrackDetails
import io.github.shabinder.models.formats.Format
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
@ -163,40 +161,19 @@ class ForegroundService : Service(), CoroutineScope {
trackList.forEach {
launch(Dispatchers.IO) {
downloadService.execute {
if (!it.videoID.isNullOrBlank()) { // Video ID already known!
downloadTrack(it.videoID!!, it)
val url = fetcher.findMp3DownloadLink(it)
if (!url.isNullOrBlank()) { // Successfully Grabbed Mp3 URL
enqueueDownload(url, it)
} else {
val searchQuery = "${it.title} - ${it.artists.joinToString(",")}"
val videoID = fetcher.youtubeMusic.getYTIDBestMatch(searchQuery, it)
logger.d("Service VideoID") { videoID ?: "Not Found" }
if (videoID.isNullOrBlank()) {
sendTrackBroadcast(Status.FAILED.name, it)
failed++
updateNotification()
allTracksStatus[it.title] = DownloadStatus.Failed
} else { // Found Youtube Video ID
downloadTrack(videoID, it)
}
}
}
}
}
}
private suspend fun downloadTrack(videoID: String, track: TrackDetails) {
try {
val url = fetcher.youtubeMp3.getMp3DownloadLink(videoID)
if (url == null) {
val audioData: Format = ytDownloader?.getVideo(videoID)?.get() ?: throw Exception("Java YT Dependency Error")
val ytUrl = audioData.url!! // We Will catch NPE
enqueueDownload(ytUrl, track)
} else enqueueDownload(url, track)
} catch (e: Exception) {
logger.d("Service YT Error") { e.message.toString() }
sendTrackBroadcast(Status.FAILED.name, track)
allTracksStatus[track.title] = DownloadStatus.Failed
}
}
private suspend fun enqueueDownload(url: String, track: TrackDetails) {
// Initiating Download

8
common/dependency-injection/src/commonMain/kotlin/com/shabinder/common/di/DI.kt
View File

@ -20,6 +20,7 @@ import co.touchlab.kermit.Kermit
import com.russhwolf.settings.Settings
import com.shabinder.common.database.databaseModule
import com.shabinder.common.database.getLogger
import com.shabinder.common.di.audioToMp3.AudioToMp3
import com.shabinder.common.di.providers.GaanaProvider
import com.shabinder.common.di.providers.SaavnProvider
import com.shabinder.common.di.providers.SpotifyProvider
@ -56,13 +57,14 @@ fun commonModule(enableNetworkLogs: Boolean) = module {
single { Settings() }
single { Kermit(getLogger()) }
single { TokenStore(get(), get()) }
single { YoutubeMusic(get(), get()) }
single { AudioToMp3(get(), get()) }
single { SpotifyProvider(get(), get(), get()) }
single { GaanaProvider(get(), get(), get()) }
single { SaavnProvider(get(), get(), get()) }
single { SaavnProvider(get(), get(), get(), get()) }
single { YoutubeProvider(get(), get(), get()) }
single { YoutubeMp3(get(), get(), get()) }
single { FetchPlatformQueryResult(get(), get(), get(), get(), get(), get(), get()) }
single { YoutubeMusic(get(), get(), get(), get(), get()) }
single { FetchPlatformQueryResult(get(), get(), get(), get(), get(), get(), get(), get()) }
}
@ThreadLocal

43
common/dependency-injection/src/commonMain/kotlin/com/shabinder/common/di/FetchPlatformQueryResult.kt
View File

@ -17,23 +17,28 @@
package com.shabinder.common.di
import com.shabinder.common.database.DownloadRecordDatabaseQueries
import com.shabinder.common.di.audioToMp3.AudioToMp3
import com.shabinder.common.di.providers.GaanaProvider
import com.shabinder.common.di.providers.SaavnProvider
import com.shabinder.common.di.providers.SpotifyProvider
import com.shabinder.common.di.providers.YoutubeMp3
import com.shabinder.common.di.providers.YoutubeMusic
import com.shabinder.common.di.providers.YoutubeProvider
import com.shabinder.common.di.providers.get
import com.shabinder.common.models.PlatformQueryResult
import com.shabinder.common.models.TrackDetails
import com.shabinder.common.models.spotify.Source
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.launch
class FetchPlatformQueryResult(
val gaanaProvider: GaanaProvider,
private val gaanaProvider: GaanaProvider,
val spotifyProvider: SpotifyProvider,
val youtubeProvider: YoutubeProvider,
val saavnProvider: SaavnProvider,
private val saavnProvider: SaavnProvider,
val youtubeMusic: YoutubeMusic,
val youtubeMp3: YoutubeMp3,
val audioToMp3: AudioToMp3,
val dir: Dir
) {
private val db: DownloadRecordDatabaseQueries?
@ -69,6 +74,40 @@ class FetchPlatformQueryResult(
}
return result
}
// 1) Try Finding on JioSaavn (better quality upto 320KBPS)
// 2) If Not found try finding on Youtube Music
suspend fun findMp3DownloadLink(
track: TrackDetails
): String? =
if (track.videoID != null) {
// We Already have VideoID
when (track.source) {
Source.JioSaavn -> {
saavnProvider.getSongFromID(track.videoID!!).media_url?.let { m4aLink ->
audioToMp3.convertToMp3(m4aLink)
}
}
Source.YouTube -> {
youtubeMp3.getMp3DownloadLink(track.videoID!!)
?: youtubeProvider.ytDownloader?.getVideo(track.videoID!!)?.get()?.url?.let { m4aLink ->
audioToMp3.convertToMp3(m4aLink)
}
}
else -> {
null/* Do Nothing, We should never reach here for now*/
}
}
} else {
// First Try Getting A Link From JioSaavn
saavnProvider.findSongDownloadURL(
trackName = track.title,
trackArtists = track.artists
)
// Lets Try Fetching Now From Youtube Music
?: youtubeMusic.findSongDownloadURL(track)
}
private fun addToDatabaseAsync(link: String, result: PlatformQueryResult) {
GlobalScope.launch(dispatcherIO) {
db?.add(

117
common/dependency-injection/src/commonMain/kotlin/com/shabinder/common/di/audioToMp3/AudioToMp3.kt Normal file
View File

@ -0,0 +1,117 @@
package com.shabinder.common.di.audioToMp3
import co.touchlab.kermit.Kermit
import com.shabinder.common.models.AudioQuality
import io.ktor.client.HttpClient
import io.ktor.client.request.forms.formData
import io.ktor.client.request.forms.submitFormWithBinaryData
import io.ktor.client.request.get
import io.ktor.client.request.header
import io.ktor.client.request.headers
import io.ktor.client.statement.HttpStatement
import io.ktor.http.isSuccess
import kotlinx.coroutines.delay
interface AudioToMp3 {
val client: HttpClient
val logger: Kermit
companion object {
operator fun invoke(
client: HttpClient,
logger: Kermit
): AudioToMp3 {
return object : AudioToMp3 {
override val client: HttpClient = client
override val logger: Kermit = logger
}
}
}
suspend fun convertToMp3(
URL: String,
audioQuality: AudioQuality = AudioQuality.getQuality(URL.substringBeforeLast(".").takeLast(3)),
): String? {
val activeHost = getHost() // ex - https://hostveryfast.onlineconverter.com/file/send
val jobLink = convertRequest(URL, activeHost, audioQuality) // ex - https://www.onlineconverter.com/convert/309a0f2bbaeb5687b04f96b6d65b47bfdd
// (jobStatus.contains("d")) == COMPLETION
var jobStatus: String
var retryCount = 40 // Set it to optimal level
do {
jobStatus = try {
client.get(
"${activeHost.removeSuffix("send")}${jobLink.substringAfterLast("/")}"
)
} catch (e: Exception) {
e.printStackTrace()
""
}
retryCount--
logger.i("Job Status") { jobStatus }
if (!jobStatus.contains("d")) delay(400) // Add Delay , to give Server Time to process audio
} while (!jobStatus.contains("d", true) && retryCount != 0)
return if (jobStatus.equals("d", true)) {
// Return MP3 Download Link
"${activeHost.removeSuffix("send")}${jobLink.substringAfterLast("/")}/download"
} else null
}
/*
* Response Link Ex : `https://www.onlineconverter.com/convert/11affb6d88d31861fe5bcd33da7b10a26c`
* - to start the conversion
* */
private suspend fun convertRequest(
URL: String,
host: String? = null,
audioQuality: AudioQuality = AudioQuality.KBPS160,
): String {
val activeHost = host ?: getHost()
val res = client.submitFormWithBinaryData<String>(
url = activeHost,
formData = formData {
append("class", "audio")
append("from", "audio")
append("to", "mp3")
append("source", "url")
append("url", URL.replace("https:", "http:"))
append("audio_quality", audioQuality.kbps)
}
) {
headers {
header("Host", activeHost.getHostDomain().also { logger.i("AudioToMp3 Host") { it } })
header("Origin", "https://www.onlineconverter.com")
header("Referer", "https://www.onlineconverter.com/")
}
}.run {
logger.d { this }
dropLast(3) // last 3 are useless unicode char
}
val job = client.get<HttpStatement>(res) {
headers {
header("Host", "www.onlineconverter.com")
}
}.execute()
logger.i("Schedule Conversion Job") { job.status.isSuccess().toString() }
return res
}
// Active Host free to process conversion
// ex - https://hostveryfast.onlineconverter.com/file/send
private suspend fun getHost(): String {
return client.get<String>("https://www.onlineconverter.com/get/host") {
headers {
header("Host", "www.onlineconverter.com")
}
}.also { logger.i("Active Host") { it } }
}
// Extract full Domain from URL
// ex - hostveryfast.onlineconverter.com
private fun String.getHostDomain(): String {
return this.removePrefix("https://").substringBeforeLast(".") + "." + this.substringAfterLast(".").substringBefore("/")
}
}

66
common/dependency-injection/src/commonMain/kotlin/com/shabinder/common/di/providers/SaavnProvider.kt
View File

@ -2,21 +2,21 @@ package com.shabinder.common.di.providers
import co.touchlab.kermit.Kermit
import com.shabinder.common.di.Dir
import com.shabinder.common.di.audioToMp3.AudioToMp3
import com.shabinder.common.di.finalOutputDir
import com.shabinder.common.di.saavn.JioSaavnRequests
import com.shabinder.common.di.utils.removeIllegalChars
import com.shabinder.common.models.DownloadStatus
import com.shabinder.common.models.PlatformQueryResult
import com.shabinder.common.models.TrackDetails
import com.shabinder.common.models.saavn.SaavnSearchResult
import com.shabinder.common.models.saavn.SaavnSong
import com.shabinder.common.models.spotify.Source
import io.github.shabinder.fuzzywuzzy.diffutils.FuzzySearch
import io.ktor.client.HttpClient
class SaavnProvider(
override val httpClient: HttpClient,
private val logger: Kermit,
override val logger: Kermit,
override val audioToMp3: AudioToMp3,
private val dir: Dir,
) : JioSaavnRequests {
@ -87,66 +87,6 @@ class SaavnProvider(
)
}
private fun sortByBestMatch(
tracks: List<SaavnSearchResult>,
trackName: String,
trackArtists: List<String>,
): Map<String, Float> {
/*
* "linksWithMatchValue" is map with Saavn VideoID and its rating/match with 100 as Max Value
**/
val linksWithMatchValue = mutableMapOf<String, Float>()
for (result in tracks) {
var hasCommonWord = false
val resultName = result.title.toLowerCase().replace("/", " ")
val trackNameWords = trackName.toLowerCase().split(" ")
for (nameWord in trackNameWords) {
if (nameWord.isNotBlank() && FuzzySearch.partialRatio(nameWord, resultName) > 85) hasCommonWord = true
}
// Skip this Result if No Word is Common in Name
if (!hasCommonWord) {
// log("Saavn Removing", result.toString())
continue
}
// Find artist match
// Will Be Using Fuzzy Search Because YT Spelling might be mucked up
// match = (no of artist names in result) / (no. of artist names on spotify) * 100
var artistMatchNumber = 0F
// String Containing All Artist Names from JioSaavn Search Result
val artistListString = mutableSetOf<String>().apply {
result.more_info?.singers?.split(",")?.let { addAll(it) }
result.more_info?.primary_artists?.toLowerCase()?.split(",")?.let { addAll(it) }
}.joinToString(" , ")
for (artist in trackArtists) {
if (FuzzySearch.partialRatio(artist.toLowerCase(), artistListString) > 85)
artistMatchNumber++
}
if (artistMatchNumber == 0F) {
// logger.d{ "Saavn Removing: $result" }
continue
}
val artistMatch: Float = (artistMatchNumber / trackArtists.size.toFloat()) * 100F
val nameMatch: Float = FuzzySearch.partialRatio(resultName, trackName).toFloat() / 100F
val avgMatch = (artistMatch + nameMatch) / 2
linksWithMatchValue[result.id] = avgMatch
}
return linksWithMatchValue.toList().sortedByDescending { it.second }.toMap().also {
logger.d("Saavn Search") { "Match Found for $trackName - ${!it.isNullOrEmpty()}" }
}
}
private fun SaavnSong.updateStatusIfPresent(folderType: String, subFolder: String): DownloadStatus {
return if (dir.isPresent(
dir.finalOutputDir(

24
common/dependency-injection/src/commonMain/kotlin/com/shabinder/common/di/providers/YoutubeMusic.kt
View File

@ -17,6 +17,7 @@
package com.shabinder.common.di.providers
import co.touchlab.kermit.Kermit
import com.shabinder.common.di.audioToMp3.AudioToMp3
import com.shabinder.common.di.gaana.corsApi
import com.shabinder.common.models.TrackDetails
import com.shabinder.common.models.YoutubeTrack
@ -37,9 +38,13 @@ import kotlinx.serialization.json.jsonPrimitive
import kotlinx.serialization.json.put
import kotlinx.serialization.json.putJsonObject
import kotlin.math.absoluteValue
class YoutubeMusic constructor(
private val logger: Kermit,
private val httpClient: HttpClient,
private val youtubeMp3: YoutubeMp3,
private val youtubeProvider: YoutubeProvider,
private val audioToMp3: AudioToMp3
) {
companion object {
@ -47,10 +52,25 @@ class YoutubeMusic constructor(
const val tag = "YT Music"
}
suspend fun getYTIDBestMatch(query: String, trackDetails: TrackDetails): String? {
suspend fun findSongDownloadURL(
trackDetails: TrackDetails
): String? {
val bestMatchVideoID = getYTIDBestMatch(trackDetails)
return bestMatchVideoID?.let { videoID ->
youtubeMp3.getMp3DownloadLink(videoID) ?: youtubeProvider.ytDownloader?.getVideo(videoID)?.get()?.url?.let { m4aLink ->
audioToMp3.convertToMp3(
m4aLink
)
}
}
}
suspend fun getYTIDBestMatch(
trackDetails: TrackDetails
): String? {
return try {
sortByBestMatch(
getYTTracks(query),
getYTTracks("${trackDetails.title} - ${trackDetails.artists.joinToString(",")}"),
trackName = trackDetails.title,
trackArtists = trackDetails.artists,
trackDurationSec = trackDetails.durationSec

86
common/dependency-injection/src/commonMain/kotlin/com/shabinder/common/di/saavn/JioSaavnRequests.kt
View File

@ -1,10 +1,13 @@
package com.shabinder.common.di.saavn
import co.touchlab.kermit.Kermit
import com.shabinder.common.di.audioToMp3.AudioToMp3
import com.shabinder.common.di.globalJson
import com.shabinder.common.models.saavn.SaavnAlbum
import com.shabinder.common.models.saavn.SaavnPlaylist
import com.shabinder.common.models.saavn.SaavnSearchResult
import com.shabinder.common.models.saavn.SaavnSong
import io.github.shabinder.fuzzywuzzy.diffutils.FuzzySearch
import io.github.shabinder.utils.getBoolean
import io.github.shabinder.utils.getJsonArray
import io.github.shabinder.utils.getJsonObject
@ -24,11 +27,26 @@ import kotlinx.serialization.json.put
interface JioSaavnRequests {
val audioToMp3: AudioToMp3
val httpClient: HttpClient
val logger: Kermit
suspend fun findSongDownloadURL(
trackName: String,
trackArtists: List<String>,
): String? {
val songs = searchForSong(trackName)
val bestMatches = sortByBestMatch(songs, trackName, trackArtists)
val m4aLink: String? = bestMatches.keys.firstOrNull()?.let {
getSongFromID(it).media_url
}
val mp3Link = m4aLink?.let { audioToMp3.convertToMp3(it) }
return mp3Link
}
suspend fun searchForSong(
query: String,
includeLyrics: Boolean = true
includeLyrics: Boolean = false
): List<SaavnSearchResult> {
/*if (query.startsWith("http") && query.contains("saavn.com")) {
return listOf(getSong(query))
@ -58,6 +76,14 @@ interface JioSaavnRequests {
.formatData(fetchLyrics)
return globalJson.decodeFromJsonElement(SaavnSong.serializer(), data)
}
suspend fun getSongFromID(
ID: String,
fetchLyrics: Boolean = false
): SaavnSong {
val data = ((globalJson.parseToJsonElement(httpClient.get(song_details_base_url + ID)) as JsonObject)[ID] as JsonObject)
.formatData(fetchLyrics)
return globalJson.decodeFromJsonElement(SaavnSong.serializer(), data)
}
private suspend fun getSongID(
URL: String,
@ -198,6 +224,64 @@ interface JioSaavnRequests {
}
}
fun sortByBestMatch(
tracks: List<SaavnSearchResult>,
trackName: String,
trackArtists: List<String>,
): Map<String, Float> {
/*
* "linksWithMatchValue" is map with Saavn VideoID and its rating/match with 100 as Max Value
**/
val linksWithMatchValue = mutableMapOf<String, Float>()
for (result in tracks) {
var hasCommonWord = false
val resultName = result.title.toLowerCase().replace("/", " ")
val trackNameWords = trackName.toLowerCase().split(" ")
for (nameWord in trackNameWords) {
if (nameWord.isNotBlank() && FuzzySearch.partialRatio(nameWord, resultName) > 85) hasCommonWord = true
}
// Skip this Result if No Word is Common in Name
if (!hasCommonWord) {
logger.i("Saavn Removing Common Word") { result.toString() }
continue
}
// Find artist match
// Will Be Using Fuzzy Search Because YT Spelling might be mucked up
// match = (no of artist names in result) / (no. of artist names on spotify) * 100
var artistMatchNumber = 0
// String Containing All Artist Names from JioSaavn Search Result
val artistListString = mutableSetOf<String>().apply {
result.more_info?.singers?.split(",")?.let { addAll(it) }
result.more_info?.primary_artists?.toLowerCase()?.split(",")?.let { addAll(it) }
}.joinToString(" , ")
for (artist in trackArtists) {
if (FuzzySearch.partialRatio(artist.toLowerCase(), artistListString) > 85)
artistMatchNumber++
}
if (artistMatchNumber == 0) {
logger.i("Artist Match Saavn Removing") { result.toString() }
continue
}
val artistMatch: Float = (artistMatchNumber.toFloat() / trackArtists.size) * 100
val nameMatch: Float = FuzzySearch.partialRatio(resultName, trackName).toFloat() / 100
val avgMatch = (artistMatch + nameMatch) / 2
linksWithMatchValue[result.id] = avgMatch
}
return linksWithMatchValue.toList().sortedByDescending { it.second }.toMap().also {
logger.i { "Match Found for $trackName - ${!it.isNullOrEmpty()}" }
}
}
companion object {
// EndPoints
const val search_base_url = "https://www.jiosaavn.com/api.php?__call=autocomplete.get&_format=json&_marker=0&cc=in&includeMetaTags=1&query="

59
common/dependency-injection/src/desktopMain/kotlin/com/shabinder/common/di/DesktopActual.kt
View File

@ -16,14 +16,11 @@
package com.shabinder.common.di
import com.shabinder.common.di.providers.YoutubeMp3
import com.shabinder.common.di.providers.get
import com.shabinder.common.di.utils.ParallelExecutor
import com.shabinder.common.models.AllPlatforms
import com.shabinder.common.models.DownloadResult
import com.shabinder.common.models.DownloadStatus
import com.shabinder.common.models.TrackDetails
import io.github.shabinder.YoutubeDownloader
import kotlinx.coroutines.CoroutineDispatcher
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.flow.MutableSharedFlow
@ -45,47 +42,11 @@ actual suspend fun downloadTracks(
fetcher: FetchPlatformQueryResult,
dir: Dir
) {
list.forEach {
list.forEach { trackDetails ->
DownloadScope.execute { // Send Download to Pool.
if (!it.videoID.isNullOrBlank()) { // Video ID already known!
downloadTrack(it.videoID!!, it, dir::saveFileWithMetadata, fetcher.youtubeMp3)
} else {
val searchQuery = "${it.title} - ${it.artists.joinToString(",")}"
val videoId = fetcher.youtubeMusic.getYTIDBestMatch(searchQuery, it)
if (videoId.isNullOrBlank()) {
DownloadProgressFlow.emit(
DownloadProgressFlow.replayCache.getOrElse(
0
) { hashMapOf() }.apply { set(it.title, DownloadStatus.Failed) }
)
} else { // Found Youtube Video ID
downloadTrack(videoId, it, dir::saveFileWithMetadata, fetcher.youtubeMp3)
}
}
}
}
}
private val ytDownloader = YoutubeDownloader()
suspend fun downloadTrack(
videoID: String,
trackDetails: TrackDetails,
saveFileWithMetaData: suspend (mp3ByteArray: ByteArray, trackDetails: TrackDetails, postProcess: (TrackDetails) -> Unit) -> Unit,
youtubeMp3: YoutubeMp3
) {
try {
val link = youtubeMp3.getMp3DownloadLink(videoID) ?: ytDownloader.getVideo(videoID).get()?.url
if (link == null) {
DownloadProgressFlow.emit(
DownloadProgressFlow.replayCache.getOrElse(
0
) { hashMapOf() }.apply { set(trackDetails.title, DownloadStatus.Failed) }
)
return
}
downloadFile(link).collect {
val url = fetcher.findMp3DownloadLink(trackDetails)
if (!url.isNullOrBlank()) { // Successfully Grabbed Mp3 URL
downloadFile(url).collect {
when (it) {
is DownloadResult.Error -> {
DownloadProgressFlow.emit(
@ -102,7 +63,7 @@ suspend fun downloadTrack(
)
}
is DownloadResult.Success -> { // Todo clear map
saveFileWithMetaData(it.byteArray, trackDetails) {}
dir.saveFileWithMetadata(it.byteArray, trackDetails) {}
DownloadProgressFlow.emit(
DownloadProgressFlow.replayCache.getOrElse(
0
@ -111,7 +72,13 @@ suspend fun downloadTrack(
}
}
}
} catch (e: java.lang.Exception) {
e.printStackTrace()
} else {
DownloadProgressFlow.emit(
DownloadProgressFlow.replayCache.getOrElse(
0
) { hashMapOf() }.apply { set(trackDetails.title, DownloadStatus.Failed) }
)
}
}
}
}

64
common/dependency-injection/src/iosMain/kotlin/com.shabinder.common.di/IOSActual.kt
View File

@ -1,6 +1,5 @@
package com.shabinder.common.di
import com.shabinder.common.di.providers.get
import com.shabinder.common.di.utils.ParallelExecutor
import com.shabinder.common.models.AllPlatforms
import com.shabinder.common.models.DownloadResult
@ -28,45 +27,9 @@ actual suspend fun downloadTracks(
dir.logger.i { "Downloading ${list.size} Tracks" }
for (track in list) {
Downloader.execute {
if (!track.videoID.isNullOrBlank()) { // Video ID already known!
dir.logger.i { "VideoID: ${track.title} -> ${track.videoID}" }
downloadTrack(track.videoID!!, track, dir::saveFileWithMetadata, fetcher)
} else {
val searchQuery = "${track.title} - ${track.artists.joinToString(",")}"
val videoId = fetcher.youtubeMusic.getYTIDBestMatch(searchQuery, track)
dir.logger.i { "VideoID: ${track.title} -> $videoId" }
if (videoId.isNullOrBlank()) {
DownloadProgressFlow.emit(
DownloadProgressFlow.replayCache.getOrElse(
0
) { hashMapOf() }.apply { set(track.title, DownloadStatus.Failed) }
)
} else { // Found Youtube Video ID
downloadTrack(videoId, track, dir::saveFileWithMetadata, fetcher)
}
}
}
}
}
@SharedImmutable
val DownloadProgressFlow: MutableSharedFlow<HashMap<String, DownloadStatus>> = MutableSharedFlow(1)
suspend fun downloadTrack(
videoID: String,
trackDetails: TrackDetails,
saveFileWithMetaData: suspend (mp3ByteArray: ByteArray, trackDetails: TrackDetails, postProcess: (TrackDetails) -> Unit) -> Unit,
fetcher: FetchPlatformQueryResult
) {
try {
var link = fetcher.youtubeMp3.getMp3DownloadLink(videoID)
fetcher.dir.logger.i { "LINK: $videoID -> $link" }
if (link == null) {
link = fetcher.youtubeProvider.ytDownloader?.getVideo(videoID)?.get()?.url ?: return
}
fetcher.dir.logger.i { "LINK: $videoID -> $link" }
downloadFile(link).collect {
val url = fetcher.findMp3DownloadLink(track)
if (!url.isNullOrBlank()) { // Successfully Grabbed Mp3 URL
downloadFile(url).collect {
fetcher.dir.logger.d { it.toString() }
/*Construct a `NEW Map` from frozen Map to Modify for Native Platforms*/
val map: MutableMap<String, DownloadStatus> = when (it) {
@ -74,22 +37,22 @@ suspend fun downloadTrack(
DownloadProgressFlow.replayCache.getOrElse(
0
) { hashMapOf() }.toMutableMap().apply {
set(trackDetails.title, DownloadStatus.Failed)
set(track.title, DownloadStatus.Failed)
}
}
is DownloadResult.Progress -> {
DownloadProgressFlow.replayCache.getOrElse(
0
) { hashMapOf() }.toMutableMap().apply {
set(trackDetails.title, DownloadStatus.Downloading(it.progress))
set(track.title, DownloadStatus.Downloading(it.progress))
}
}
is DownloadResult.Success -> { // Todo clear map
saveFileWithMetaData(it.byteArray, trackDetails, methods.value::writeMp3Tags)
dir.saveFileWithMetadata(it.byteArray, track, methods.value::writeMp3Tags)
DownloadProgressFlow.replayCache.getOrElse(
0
) { hashMapOf() }.toMutableMap().apply {
set(trackDetails.title, DownloadStatus.Downloaded)
set(track.title, DownloadStatus.Downloaded)
}
}
else -> { mutableMapOf() }
@ -98,7 +61,16 @@ suspend fun downloadTrack(
map as HashMap<String, DownloadStatus>
)
}
} catch (e: Exception) {
e.printStackTrace()
} else {
DownloadProgressFlow.emit(
DownloadProgressFlow.replayCache.getOrElse(
0
) { hashMapOf() }.apply { set(track.title, DownloadStatus.Failed) }
)
}
}
}
}
@SharedImmutable
val DownloadProgressFlow: MutableSharedFlow<HashMap<String, DownloadStatus>> = MutableSharedFlow(1)

36
common/dependency-injection/src/jsMain/kotlin/com/shabinder/common/di/WebActual.kt
View File

@ -42,34 +42,11 @@ actual suspend fun downloadTracks(
fetcher: FetchPlatformQueryResult,
dir: Dir
) {
list.forEach {
list.forEach { track ->
withContext(dispatcherIO) {
allTracksStatus[it.title] = DownloadStatus.Queued
if (!it.videoID.isNullOrBlank()) { // Video ID already known!
downloadTrack(it.videoID!!, it, fetcher, dir)
} else {
val searchQuery = "${it.title} - ${it.artists.joinToString(",")}"
val videoID = fetcher.youtubeMusic.getYTIDBestMatch(searchQuery, it)
println(videoID + " : " + it.title)
if (videoID.isNullOrBlank()) {
allTracksStatus[it.title] = DownloadStatus.Failed
DownloadProgressFlow.emit(allTracksStatus)
} else { // Found Youtube Video ID
downloadTrack(videoID, it, fetcher, dir)
}
}
DownloadProgressFlow.emit(allTracksStatus)
}
}
}
suspend fun downloadTrack(videoID: String, track: TrackDetails, fetcher: FetchPlatformQueryResult, dir: Dir) {
val url = fetcher.youtubeMp3.getMp3DownloadLink(videoID)
if (url == null) {
allTracksStatus[track.title] = DownloadStatus.Failed
DownloadProgressFlow.emit(allTracksStatus)
println("No URL to Download")
} else {
allTracksStatus[track.title] = DownloadStatus.Queued
val url = fetcher.findMp3DownloadLink(track)
if (!url.isNullOrBlank()) { // Successfully Grabbed Mp3 URL
downloadFile(url).collect {
when (it) {
is DownloadResult.Success -> {
@ -87,5 +64,10 @@ suspend fun downloadTrack(videoID: String, track: TrackDetails, fetcher: FetchPl
}
DownloadProgressFlow.emit(allTracksStatus)
}
} else {
allTracksStatus[track.title] = DownloadStatus.Failed
DownloadProgressFlow.emit(allTracksStatus)
}
}
}
}

25
maintenance-tasks/src/main/java/audio_conversion/AudioQuality.kt Normal file
View File

@ -0,0 +1,25 @@
package audio_conversion
@Suppress("EnumEntryName")
enum class AudioQuality(val kbps: String) {
`128KBPS`("128"),
`160KBPS`("160"),
`192KBPS`("192"),
`224KBPS`("224"),
`256KBPS`("256"),
`320KBPS`("320");
companion object {
fun getQuality(kbps: String): AudioQuality {
return when (kbps) {
"128" -> `128KBPS`
"160" -> `160KBPS`
"192" -> `192KBPS`
"224" -> `224KBPS`
"256" -> `256KBPS`
"320" -> `320KBPS`
else -> `160KBPS`
}
}
}
}

45
maintenance-tasks/src/main/java/audio_conversion/AudioToMp3.kt
View File

@ -9,35 +9,24 @@ import io.ktor.client.request.headers
import io.ktor.client.statement.HttpStatement
import io.ktor.http.isSuccess
import kotlinx.coroutines.delay
import kotlinx.serialization.json.Json
import utils.debug
interface AudioToMp3 {
@Suppress("EnumEntryName")
enum class Quality(val kbps: String) {
`128KBPS`("128"),
`160KBPS`("160"),
`192KBPS`("192"),
`224KBPS`("224"),
`256KBPS`("256"),
`320KBPS`("320"),
}
object AudioToMp3 {
suspend fun convertToMp3(
URL: String,
quality: Quality = kotlin.runCatching { Quality.valueOf(URL.substringBeforeLast(".").takeLast(3)) }.getOrNull() ?: Quality.`160KBPS`,
audioQuality: AudioQuality = AudioQuality.getQuality(URL.substringBeforeLast(".").takeLast(3)),
): String? {
val activeHost = getHost() // ex - https://hostveryfast.onlineconverter.com/file/send
val jobLink = convertRequest(URL, activeHost, quality) // ex - https://www.onlineconverter.com/convert/309a0f2bbaeb5687b04f96b6d65b47bfdd
val jobLink = convertRequest(URL, activeHost, audioQuality) // ex - https://www.onlineconverter.com/convert/309a0f2bbaeb5687b04f96b6d65b47bfdd
// (jobStatus = "d") == COMPLETION
// (jobStatus.contains("d")) == COMPLETION
var jobStatus: String
var retryCount = 20
var retryCount = 40 // Set it to optimal level
do {
jobStatus = try {
client.get<String>(
client.get(
"${activeHost.removeSuffix("send")}${jobLink.substringAfterLast("/")}"
)
} catch (e: Exception) {
@ -46,7 +35,7 @@ interface AudioToMp3 {
}
retryCount--
debug("Job Status", jobStatus)
if (!jobStatus.contains("d")) delay(200) // Add Delay , to give Server Time to process audio
if (!jobStatus.contains("d")) delay(400) // Add Delay , to give Server Time to process audio
} while (!jobStatus.contains("d", true) && retryCount != 0)
return if (jobStatus.equals("d", true)) {
@ -62,7 +51,7 @@ interface AudioToMp3 {
private suspend fun convertRequest(
URL: String,
host: String? = null,
quality: Quality = Quality.`320KBPS`,
audioQuality: AudioQuality = AudioQuality.`320KBPS`,
): String {
val activeHost = host ?: getHost()
val res = client.submitFormWithBinaryData<String>(
@ -73,11 +62,11 @@ interface AudioToMp3 {
append("to", "mp3")
append("source", "url")
append("url", URL.replace("https:", "http:"))
append("audio_quality", quality.kbps)
append("audio_quality", audioQuality.kbps)
}
) {
headers {
header("Host", activeHost.getHostURL().also { debug(it) })
header("Host", activeHost.getHostDomain().also { debug(it) })
header("Origin", "https://www.onlineconverter.com")
header("Referer", "https://www.onlineconverter.com/")
}
@ -95,6 +84,8 @@ interface AudioToMp3 {
return res
}
// Active Host free to process conversion
// ex - https://hostveryfast.onlineconverter.com/file/send
private suspend fun getHost(): String {
return client.get<String>("https://www.onlineconverter.com/get/host") {
headers {
@ -102,15 +93,9 @@ interface AudioToMp3 {
}
}.also { debug("Active Host", it) }
}
private fun String.getHostURL(): String {
// Extract full Domain from URL
// ex - hostveryfast.onlineconverter.com
private fun String.getHostDomain(): String {
return this.removePrefix("https://").substringBeforeLast(".") + "." + this.substringAfterLast(".").substringBefore("/")
}
companion object {
val serializer = Json {
ignoreUnknownKeys = true
isLenient = true
}
}
}

76
maintenance-tasks/src/main/java/jiosaavn/JioSaavnRequests.kt
View File

@ -1,6 +1,8 @@
package jiosaavn
import analytics_html_img.client
import audio_conversion.AudioToMp3
import io.github.shabinder.fuzzywuzzy.diffutils.FuzzySearch
import io.ktor.client.request.forms.FormDataContent
import io.ktor.client.request.get
import io.ktor.http.Parameters
@ -16,6 +18,7 @@ import kotlinx.serialization.json.buildJsonArray
import kotlinx.serialization.json.buildJsonObject
import kotlinx.serialization.json.jsonPrimitive
import kotlinx.serialization.json.put
import utils.debug
val serializer = Json {
ignoreUnknownKeys = true
@ -24,9 +27,22 @@ val serializer = Json {
interface JioSaavnRequests {
suspend fun findSongDownloadURL(
trackName: String,
trackArtists: List<String>,
): String? {
val songs = searchForSong(trackName)
val bestMatches = sortByBestMatch(songs, trackName, trackArtists)
val m4aLink = bestMatches.keys.firstOrNull()?.let {
getSongFromID(it).media_url
}
val mp3Link = m4aLink?.let { AudioToMp3.convertToMp3(it) }
return mp3Link
}
suspend fun searchForSong(
query: String,
includeLyrics: Boolean = true
includeLyrics: Boolean = false
): List<SaavnSearchResult> {
/*if (query.startsWith("http") && query.contains("saavn.com")) {
return listOf(getSong(query))
@ -204,6 +220,64 @@ interface JioSaavnRequests {
}
}
fun sortByBestMatch(
tracks: List<SaavnSearchResult>,
trackName: String,
trackArtists: List<String>,
): Map<String, Float> {
/*
* "linksWithMatchValue" is map with Saavn VideoID and its rating/match with 100 as Max Value
**/
val linksWithMatchValue = mutableMapOf<String, Float>()
for (result in tracks) {
var hasCommonWord = false
val resultName = result.title.toLowerCase().replace("/", " ")
val trackNameWords = trackName.toLowerCase().split(" ")
for (nameWord in trackNameWords) {
if (nameWord.isNotBlank() && FuzzySearch.partialRatio(nameWord, resultName) > 85) hasCommonWord = true
}
// Skip this Result if No Word is Common in Name
if (!hasCommonWord) {
debug("Saavn Removing Common Word: ", result.toString())
continue
}
// Find artist match
// Will Be Using Fuzzy Search Because YT Spelling might be mucked up
// match = (no of artist names in result) / (no. of artist names on spotify) * 100
var artistMatchNumber = 0
// String Containing All Artist Names from JioSaavn Search Result
val artistListString = mutableSetOf<String>().apply {
result.more_info?.singers?.split(",")?.let { addAll(it) }
result.more_info?.primary_artists?.toLowerCase()?.split(",")?.let { addAll(it) }
}.joinToString(" , ")
for (artist in trackArtists) {
if (FuzzySearch.partialRatio(artist.toLowerCase(), artistListString) > 85)
artistMatchNumber++
}
if (artistMatchNumber == 0) {
debug("Artist Match Saavn Removing: $result")
continue
}
val artistMatch: Float = (artistMatchNumber.toFloat() / trackArtists.size) * 100
val nameMatch: Float = FuzzySearch.partialRatio(resultName, trackName).toFloat() / 100
val avgMatch = (artistMatch + nameMatch) / 2
linksWithMatchValue[result.id] = avgMatch
}
return linksWithMatchValue.toList().sortedByDescending { it.second }.toMap().also {
debug("Match Found for $trackName - ${!it.isNullOrEmpty()}")
}
}
companion object {
// EndPoints
const val search_base_url = "https://www.jiosaavn.com/api.php?__call=autocomplete.get&_format=json&_marker=0&cc=in&includeMetaTags=1&query="

82
maintenance-tasks/src/main/java/utils/TestClass.kt
View File

@ -1,89 +1,7 @@
package utils
import audio_conversion.AudioToMp3
import io.github.shabinder.fuzzywuzzy.diffutils.FuzzySearch
import jiosaavn.models.SaavnSearchResult
import kotlinx.coroutines.runBlocking
// Test Class- at development Time
fun main(): Unit = runBlocking {
/*val jioSaavnClient = object : JioSaavnRequests {}
val resp = jioSaavnClient.searchForSong(
query = "Ye Faasla"
)
println(resp.joinToString("\n"))
val matches = sortByBestMatch(
tracks = resp,
trackName = "Ye Faasla",
trackArtists = listOf("Shaan", "Hardy")
)
debug(matches.toString())
val link = matches.keys.firstOrNull()?.let {
jioSaavnClient.getSongFromID(it).media_url
}
debug(link.toString())*/
val link = "https://aac.saavncdn.com/787/956c23404206e8f4822827eff5da61a0_320.mp4"
val audioConverter = object : AudioToMp3 {}
val mp3Link = audioConverter.convertToMp3(link.toString())
debug(mp3Link.toString())
}
private fun sortByBestMatch(
tracks: List<SaavnSearchResult>,
trackName: String,
trackArtists: List<String>,
): Map<String, Float> {
/*
* "linksWithMatchValue" is map with Saavn VideoID and its rating/match with 100 as Max Value
**/
val linksWithMatchValue = mutableMapOf<String, Float>()
for (result in tracks) {
var hasCommonWord = false
val resultName = result.title.toLowerCase().replace("/", " ")
val trackNameWords = trackName.toLowerCase().split(" ")
for (nameWord in trackNameWords) {
if (nameWord.isNotBlank() && FuzzySearch.partialRatio(nameWord, resultName) > 85) hasCommonWord = true
}
// Skip this Result if No Word is Common in Name
if (!hasCommonWord) {
debug("Saavn Removing Common Word: ", result.toString())
continue
}
// Find artist match
// Will Be Using Fuzzy Search Because YT Spelling might be mucked up
// match = (no of artist names in result) / (no. of artist names on spotify) * 100
var artistMatchNumber = 0
// String Containing All Artist Names from JioSaavn Search Result
val artistListString = mutableSetOf<String>().apply {
result.more_info?.singers?.split(",")?.let { addAll(it) }
result.more_info?.primary_artists?.toLowerCase()?.split(",")?.let { addAll(it) }
}.joinToString(" , ")
for (artist in trackArtists) {
if (FuzzySearch.partialRatio(artist.toLowerCase(), artistListString) > 85)
artistMatchNumber++
}
if (artistMatchNumber == 0) {
debug("Artist Match Saavn Removing: $result")
continue
}
val artistMatch: Float = (artistMatchNumber.toFloat() / trackArtists.size) * 100
val nameMatch: Float = FuzzySearch.partialRatio(resultName, trackName).toFloat() / 100
val avgMatch = (artistMatch + nameMatch) / 2
linksWithMatchValue[result.id] = avgMatch
}
return linksWithMatchValue.toList().sortedByDescending { it.second }.toMap().also {
debug("Match Found for $trackName - ${!it.isNullOrEmpty()}")
}
}