import groovy.json.JsonSlurper import groovy.json.StringEscapeUtils import org.serviio.library.metadata.MediaFileType import org.serviio.library.online.* import java.util.regex.Matcher //it.x/add 07.02.2025 import java.util.regex.Pattern //it.x/add 07.02.2025 /** * YouTube.com content URL extractor plugin. * * It uses YouTube API v3 for retrieving playlists. * * @see http://en.wikipedia.org/wiki/Youtube#Quality_and_codecs * * @author Petr Nejedly * @modified drJeckyll * @modified Pavlo Kudlay * tested on serviio 1.9.1 * 30.07.2021 guest changes by it.x: "/get_video_info" issue; lines <106..133>; tested on v.2.1.0-0038 * 07.02.2025 guest changes by it.x: bypass with YTDLP.ONLINE; tested on v.2.1 (nas/linux), v.2.3 (win) */ class YouTube extends WebResourceUrlExtractor { final VALID_RESOURCE_URL = '^https?://www.googleapis.com/youtube/.*$' final PLAYER_REGEX = '"assets":.+?"js":\\s*"([^"]+)"' final user_agent = "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)" /* Listed in order of quality */ final availableFormats = ['37', '46', '22', '45', '35', '34', '18', '44', '43', '6', '5'] String getExtractorName() { return getClass().getName() } boolean extractorMatches(URL feedUrl) { return feedUrl ==~ VALID_RESOURCE_URL } public int getVersion() { return 5; } static class FormatDetails { public String itag public String url public Date expiry FormatDetails(String itag, String url, Date expiry) { this.itag = itag this.url = url this.expiry = expiry } } @Override WebResourceContainer extractItems(URL resourceUrl, int maxItemsToRetrieve) { final api_key = "AIzaSyCbFQoCv41IVcHal6bbsDuKuSqGVE243q0" // this is Serviio API key, don't use it for anything else please, they are easy to get if (maxItemsToRetrieve == -1) maxItemsToRetrieve = 50 // Handle channel name urls if (resourceUrl.toString().contains("channels")) { def channelUrl = new URL(resourceUrl.toString() + "&part=contentDetails" + "&key=" + api_key) def channeljson = new JsonSlurper().parseText(openURL(channelUrl, user_agent)) def uploadPlaylist = channeljson.items[0].contentDetails.relatedPlaylists.uploads resourceUrl = new URL("https://www.googleapis.com/youtube/v3/playlistItems?playlistId=$uploadPlaylist") } def apiUrl = new URL(resourceUrl.toString() + "&part=snippet" + "&maxResults=" + maxItemsToRetrieve + "&type=video" + "&key=" + api_key) def json = new JsonSlurper().parseText(openURL(apiUrl, user_agent)) def items = [] // for long playlists it may take a while to update def i = 0 boolean done = false while (json.pageInfo.totalResults > i && !done) { json.items.each() { i++ if (it.snippet.title != "Deleted video" && it.snippet.title != "Private video") { items.add(new WebResourceItem(title: it.snippet.title, additionalInfo: ['videoId': resourceUrl.toString().contains("videos") ? it.id : resourceUrl.toString().contains("search") ? it.id.videoId : it.snippet.resourceId.videoId, 'thumb' : it.snippet.thumbnails.high.url])) } } done = items.size() >= maxItemsToRetrieve if (json.nextPageToken != null && !done) { // repeat with supplied token apiUrl = new URL(resourceUrl.toString() + "&part=snippet" + "&maxResults=" + maxItemsToRetrieve + "&type=video" + "&key=" + api_key + "&pageToken=" + json.nextPageToken) json = new JsonSlurper().parseText(openURL(apiUrl, user_agent)) } } def containerThumbnailUrl = items?.find { it -> it.additionalInfo['thumb'] != null }?.additionalInfo['thumb'] return new WebResourceContainer(items: items, thumbnailUrl: containerThumbnailUrl) } @Override protected ContentURLContainer extractUrl(WebResourceItem wrItem, PreferredQuality requestedQuality) { def contentUrl def expiryDate def expiresImmediately def cacheKey def videoId = wrItem.additionalInfo['videoId'] def thumbnailUrl = wrItem.additionalInfo['thumb'] /////////////// for (elType in ['&el=embedded', '&el=detailpage', '&el=vevo', '']) { // ver.1 blocked by Google ca. 01.05.2021 def videoInfoUrl = "https://www.youtube.com/get_video_info?&video_id=$videoId$elType&ps=default&eurl=&gl=US&hl=en" // ver.2 blocked by Google 19.06.2021 def videoInfoUrl = "https://www.youtube.com/get_video_info?html5=1&video_id=$videoId$elType&ps=default&eurl=&gl=US&hl=en" // ver.3 blocked by Google 30.07.2021 def videoInfoUrl = "https://www.youtube.com/get_video_info?video_id=$videoId&eurl=https%3A%2F%2Fyoutube.googleapis.com%2Fv%2F$videoId&html5=1&c=TVHTML5&cver=6.20180913$elType&ps=default&eurl=&gl=US&hl=en" ///////////////////////////////it.x/add 07.02.2025 HTTP GET block START def TmpHttpGet2 = new URL("https://ytdlp.online/stream?command=--get-url%20https://www.youtube.com/watch?v=$videoId%20-f%2018").openConnection(); TmpHttpGet2.setRequestMethod("GET") TmpHttpGet2.setDoOutput(true) TmpHttpGet2.setRequestProperty("Accept", "text/event-stream") TmpHttpGet2.setRequestProperty("Accept-Encoding", "gzip, deflate, br, zstd") TmpHttpGet2.setRequestProperty("Accept-Language", "aen-US,en;q=0.5") TmpHttpGet2.setRequestProperty("Cache-Control", "no-cache") TmpHttpGet2.setRequestProperty("Connection", "keep-alive") TmpHttpGet2.setRequestProperty("DNT", "1") TmpHttpGet2.setRequestProperty("Host", "ytdlp.online") TmpHttpGet2.setRequestProperty("Pragma", "no-cache") TmpHttpGet2.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:134.0) Gecko/20100101 Firefox/134.0") TmpHttpGet2.setRequestProperty("Info1", "I am a frindly Serviio Bot and I dont want to harm You :)") TmpHttpGet2.setRequestProperty("Info2", "https://forum.serviio.org/viewtopic.php?f=20&t=3276&hilit=you+tube&start=580#p133877") def YTDLP_response = TmpHttpGet2.getInputStream().getText() def YTDLP_links = [] Pattern urlPattern = Pattern.compile("\\b(https?)://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]",Pattern.CASE_INSENSITIVE); Matcher matcher = urlPattern.matcher( YTDLP_response); while (matcher.find()) { YTDLP_links.add(matcher.group()) } def YTDLP_link1 = YTDLP_links[1] ///////////////////////////////it.x/add 07.02.2025 HTTP GET block END def videoInfoUrl = "https://youtubei.googleapis.com/youtubei/v1/player?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" def TmpHttpPost = new URL("https://youtubei.googleapis.com/youtubei/v1/player?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8").openConnection(); def TmpHttpBody ='{"context": {"client": {"hl": "en", "clientName": "ANDROID", "clientVersion": "21.02.35", "mainAppWebInfo": {"graftUrl": "/watch?v='+"$videoId"+'"}}}, "videoId": "'+"$videoId"+'"}' TmpHttpPost.setRequestMethod("POST") TmpHttpPost.setDoOutput(true) TmpHttpPost.setRequestProperty("Content-Type", "application/json") TmpHttpPost.getOutputStream().write(TmpHttpBody.getBytes("UTF-8")) def videoInfoWebPage = TmpHttpPost.getInputStream().getText() def jsonSlurper2 = new JsonSlurper() def parameters = jsonSlurper2.parseText(videoInfoWebPage) def playerResponse = parameters['responseContext'] if(playerResponse) { def streamingData = parameters['streamingData'] if(!streamingData) { continue } List streamingFormats = [] List formats = streamingData['formats'] List adaptiveFormats = streamingData['adaptiveFormats'] if(formats && !formats.empty) { streamingFormats.addAll(formats) } if(adaptiveFormats && !adaptiveFormats.empty) { streamingFormats.addAll(adaptiveFormats) } String videoPageHtml = openURL(new URL("https://www.youtube.com/embed/$videoId"), user_agent) Map allFormatUrls = [:] streamingFormats.each { it -> processStreamData2(videoId, it, videoPageHtml, allFormatUrls) } Map formatUrlMap = new LinkedHashMap() if(allFormatUrls.isEmpty()) { // signature probably failed } else { FormatDetails selectedFormat = null if (requestedQuality == PreferredQuality.HIGH) { // best quality, get the first from the list sortAvailableFormatUrls2(availableFormats, allFormatUrls, formatUrlMap) selectedFormat = formatUrlMap.entrySet().toList().head().getValue() } else if (requestedQuality == PreferredQuality.MEDIUM) { // work with subset of available formats, starting at the position of format 35 and then take the best quality from there sortAvailableFormatUrls2(availableFormats.getAt(4..availableFormats.size - 1), allFormatUrls, formatUrlMap) selectedFormat = formatUrlMap.entrySet().toList().head().getValue() } else { // worst quality, take the last url sortAvailableFormatUrls2(availableFormats, allFormatUrls, formatUrlMap) selectedFormat = formatUrlMap.entrySet().toList().last().getValue() } if (selectedFormat != null) { //org contentUrl = selectedFormat.url //it.x/rem 07.02.2025 contentUrl = YTDLP_link1 //it.x/add 07.02.2025 cacheKey = getCacheKey(videoId, selectedFormat.itag) if (selectedFormat.expiry) { expiresImmediately = false expiryDate = selectedFormat.expiry } else { expiresImmediately = true } } } break } } if(contentUrl) { return new ContentURLContainer(fileType: MediaFileType.VIDEO, contentUrl: contentUrl, thumbnailUrl: thumbnailUrl, expiresOn: expiryDate, expiresImmediately: expiresImmediately, cacheKey: cacheKey) } else { // could not work out the stream url return null } } def addParameter(parameterString, parameters, separator) { def values = parameterString.split(separator) if (values.length == 2) { parameters.put(values[0], values[1]) } } def processStreamData2(String videoId, Map format, String videoPageHtml, Map streamFormats) { def url = format['url'] String streamUrl = null if(url) { streamUrl = URLDecoder.decode(url, 'UTF-8') } else { def cipher = format['cipher'] if (!cipher) { return } def urlData = parseQueryString(cipher) url = urlData['url'] if (!url) return streamUrl = URLDecoder.decode(url, 'UTF-8') String signature = urlData['sig'] String s = urlData['s'] if (signature) { streamUrl = streamUrl + "&signature=" + signature } else if (s) { String playerUrlJson = getPlayerUrl(videoPageHtml, videoId, s) if(playerUrlJson) { } else { log("Ignoring videos with encrypted signatures") return } } } String qs = streamUrl.substring(streamUrl.indexOf('?') + 1) Map urlQsData = parseQueryString(qs) Date expiryDate = null if(urlQsData['expire']) { expiryDate = new Date(Long.parseLong(urlQsData['expire']) * 1000) } streamFormats.put(format['itag'].toString(), new FormatDetails(format['itag'].toString(), streamUrl, expiryDate)) } def parseQueryString(String qs) { Map params = [:] qs.split('&').each { item2 -> addParameter(item2, params, '=') } params } def getPlayerUrl(String videoPageHtml, String videoId, String exampleSignature) { def playerMatcher = videoPageHtml =~ PLAYER_REGEX def playerUrl = playerMatcher[0][1] if(playerUrl) { String playerPath = StringEscapeUtils.unescapeJavaScript(playerUrl) String fullPlayerUrl = null if(playerPath.startsWith('//')) { fullPlayerUrl = "https:$playerPath" } else if(!playerPath.startsWith("http")) { fullPlayerUrl = "https://www.youtube.com$playerPath" } if(fullPlayerUrl) { // todo cache the calculateSignture(fullPlayerUrl, videoId, exampleSignature) } } return null } def calculateSignture(String playerUrl, String videoId, String exampleSignature) { def playerIdMatcher = playerUrl =~ '.*?-([a-zA-Z0-9_-]+)(?:\\/watch_as3|\\/html5player(?:-new)?|(?:\\/[a-z]{2,3}_[A-Z]{2})?\\/base)?\\.([a-z]+)$' String playerId = playerIdMatcher[0][1] String playerExt = playerIdMatcher[0][2] if(playerExt == "js") { // String playerSource = openURL(new URL(playerUrl), user_agent) //todo cache this // ScriptEngine engine = new ScriptEngineManager().getEngineByName("nashorn"); // engine.eval("") //todo can we implement signatures? https://github.com/ytdl-org/youtube-dl/blob/c3cfea906869e8358652e382679a5996c2aec73e/youtube_dl/extractor/youtube.py } else { // swf supported? } return null } String getCacheKey(String videoId, String qualityId) { "youtube_${videoId}_${qualityId}" } def sortAvailableFormatUrls2(List formatIds, Map sourceMap, Map targetMap) { formatIds.each { formatId -> if (sourceMap.containsKey(formatId)) { targetMap.put(formatId, sourceMap.get(formatId)) } } } static void main(args) { // this is just to test YouTube extractor = new YouTube() assert extractor.extractorMatches(new URL("https://www.googleapis.com/youtube/v3/playlistItems?playlistId=YOUR_PLAYLIST_ID_HERE")) assert !extractor.extractorMatches(new URL("http://google.com/feeds/api/standardfeeds/top_rated?time=today")) WebResourceContainer container1 = extractor.extractItems(new URL("https://www.googleapis.com/youtube/v3/playlistItems?playlistId=PLx6bGx4zt6EmUH0nP0Vbny7qbGABlrxnr"), 10) println container1 ContentURLContainer result1 = extractor.extractUrl(container1.getItems()[2], PreferredQuality.MEDIUM) println result1 println "" WebResourceContainer container2 = extractor.extractItems(new URL("https://www.googleapis.com/youtube/v3/videos?chart=mostPopular"), 10) println container2 ContentURLContainer result2 = extractor.extractUrl(container2.getItems()[2], PreferredQuality.MEDIUM) println result2 println "" WebResourceContainer container3 = extractor.extractItems(new URL("https://www.googleapis.com/youtube/v3/channels?forUsername=NFL"), 10) println container3 ContentURLContainer result3 = extractor.extractUrl(container3.getItems()[2], PreferredQuality.MEDIUM) println result3 println "" WebResourceContainer container4 = extractor.extractItems(new URL("https://www.googleapis.com/youtube/v3/search?q=crazy"), 10) println container4 container4.getItems().forEach { it -> println(extractor.extractUrl(it, PreferredQuality.MEDIUM)) } // live hls // println "" // WebResourceContainer container5 = extractor.extractItems( new URL("https://www.googleapis.com/youtube/v3/videos?id=sw4hmqVPe0E"), 10) // println container5 // ContentURLContainer result5 = extractor.extractUrl(container5.getItems()[0], PreferredQuality.MEDIUM) // println result5 } }