From 258ef03aaa64b6488e562ea3fa042032ce8ffbb3 Mon Sep 17 00:00:00 2001 From: absidue <48293849+absidue@users.noreply.github.com> Date: Sat, 26 Feb 2022 21:50:51 +0100 Subject: [PATCH 1/6] Fix error when playing legacy streams caused by the content length missing --- play-dl/Request/index.ts | 37 ++++++++++++++++++++++++++++++++++--- play-dl/YouTube/stream.ts | 12 ++++++++++-- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/play-dl/Request/index.ts b/play-dl/Request/index.ts index fccbe15..5e91149 100644 --- a/play-dl/Request/index.ts +++ b/play-dl/Request/index.ts @@ -136,9 +136,8 @@ export function request_resolve_redirect(url: string): Promise { resolve(url); } else if (statusCode < 400) { const resolved = await request_resolve_redirect(res.headers.location as string).catch((err) => err); - - if (res instanceof Error) { - reject(res); + if (resolved instanceof Error) { + reject(resolved); return; } @@ -149,6 +148,38 @@ export function request_resolve_redirect(url: string): Promise { }); } +export function request_content_length(url: string): Promise { + return new Promise(async (resolve, reject) => { + let res = await https_getter(url, { method: 'HEAD' }).catch((err: Error) => err); + if (res instanceof Error) { + reject(res); + return; + } + const statusCode = Number(res.statusCode); + if (statusCode < 300) { + resolve(Number(res.headers['content-length'])); + } else if (statusCode < 400) { + const newURL = await request_resolve_redirect(res.headers.location as string).catch((err) => err); + if (newURL instanceof Error) { + reject(newURL); + return; + } + + const res2 = await request_content_length(newURL).catch((err) => err); + if (res2 instanceof Error) { + reject(res2); + return; + } + + resolve(res2); + } else { + reject( + new Error(`Failed to get content length with error: ${res.statusCode}, ${res.statusMessage}, ${url}`) + ); + } + }); +} + /** * Main module that play-dl uses for making a https request * @param req_url URL to make https request to diff --git a/play-dl/YouTube/stream.ts b/play-dl/YouTube/stream.ts index f87efd5..b9fdfd0 100644 --- a/play-dl/YouTube/stream.ts +++ b/play-dl/YouTube/stream.ts @@ -1,4 +1,4 @@ -import { request_stream } from '../Request'; +import { request_content_length, request_stream } from '../Request'; import { LiveStream, Stream } from './classes/LiveStream'; import { SeekStream } from './classes/SeekStream'; import { InfoData, StreamInfoData } from './utils/constants'; @@ -104,11 +104,19 @@ export async function stream_from_info( ); } else if (options.seek) throw new Error('Can not seek with discordPlayerCompatibility set to true.'); } + + let contentLength; + if (final[0].contentLength) { + contentLength = Number(final[0].contentLength); + } else { + contentLength = await request_content_length(final[0].url); + } + return new Stream( final[0].url, type, info.video_details.durationInSec, - Number(final[0].contentLength), + contentLength, info.video_details.url, options ); From 24f7855c4f5d578d50d6d1619bd72cfce501cc32 Mon Sep 17 00:00:00 2001 From: absidue <48293849+absidue@users.noreply.github.com> Date: Sat, 26 Feb 2022 23:06:02 +0100 Subject: [PATCH 2/6] Fix empty search results caused by adverts --- play-dl/YouTube/utils/parser.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/play-dl/YouTube/utils/parser.ts b/play-dl/YouTube/utils/parser.ts index a203953..8515b4f 100644 --- a/play-dl/YouTube/utils/parser.ts +++ b/play-dl/YouTube/utils/parser.ts @@ -45,11 +45,12 @@ export function ParseSearchResult(html: string, options?: ParseSearchInterface): const json_data = JSON.parse(data); const results = []; const details = - json_data.contents.twoColumnSearchResultsRenderer.primaryContents.sectionListRenderer.contents[0] - .itemSectionRenderer.contents; + json_data.contents.twoColumnSearchResultsRenderer.primaryContents.sectionListRenderer.contents.flatMap( + (s: any) => s.itemSectionRenderer?.contents + ); for (const detail of details) { if (hasLimit && results.length === options.limit) break; - if (!detail.videoRenderer && !detail.channelRenderer && !detail.playlistRenderer) continue; + if (!detail || (!detail.videoRenderer && !detail.channelRenderer && !detail.playlistRenderer)) continue; switch (options.type) { case 'video': { const parsed = parseVideo(detail); From 89b843a07f67ca41f8bd81810130697ad1d5a484 Mon Sep 17 00:00:00 2001 From: absidue <48293849+absidue@users.noreply.github.com> Date: Sat, 26 Feb 2022 23:17:08 +0100 Subject: [PATCH 3/6] Allow youtu.be playlist URLs to be validated correctly --- play-dl/YouTube/utils/extractor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/play-dl/YouTube/utils/extractor.ts b/play-dl/YouTube/utils/extractor.ts index 5a57968..14236c7 100644 --- a/play-dl/YouTube/utils/extractor.ts +++ b/play-dl/YouTube/utils/extractor.ts @@ -22,7 +22,7 @@ const DEFAULT_API_KEY = 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'; const video_pattern = /^((?:https?:)?\/\/)?(?:(?:www|m|music)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w\-]+\?v=|shorts\/|embed\/|v\/)?)([\w\-]+)(\S+)?$/; const playlist_pattern = - /^((?:https?:)?\/\/)?(?:(?:www|m|music)\.)?(youtube\.com)\/(?:(playlist|watch))(.*)?((\?|\&)list=)(PL|UU|LL|RD|OL)[a-zA-Z\d_-]{10,}(.*)?$/; + /^((?:https?:)?\/\/)?(?:(?:www|m|music)\.)?((?:youtube\.com|youtu.be))\/(?:(playlist|watch))?(.*)?((\?|\&)list=)(PL|UU|LL|RD|OL)[a-zA-Z\d_-]{10,}(.*)?$/; /** * Validate YouTube URL or ID. * From d1d0b7256ad0549b4c56da9fba7bde7371241672 Mon Sep 17 00:00:00 2001 From: absidue <48293849+absidue@users.noreply.github.com> Date: Sat, 26 Feb 2022 23:35:14 +0100 Subject: [PATCH 4/6] Try validating without playlist parameter if playlist validation fails --- play-dl/YouTube/utils/extractor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/play-dl/YouTube/utils/extractor.ts b/play-dl/YouTube/utils/extractor.ts index 14236c7..1984d65 100644 --- a/play-dl/YouTube/utils/extractor.ts +++ b/play-dl/YouTube/utils/extractor.ts @@ -60,7 +60,7 @@ export function yt_validate(url: string): 'playlist' | 'video' | 'search' | fals else return 'search'; } } else { - if (!url.match(playlist_pattern)) return false; + if (!url.match(playlist_pattern)) return yt_validate(url.replace(/(\?|\&)list=[a-zA-Z\d_-]+/, '')); else return 'playlist'; } } From 771b15e09f41e8c6b5153a005005db1ddd1a1601 Mon Sep 17 00:00:00 2001 From: absidue <48293849+absidue@users.noreply.github.com> Date: Sun, 27 Feb 2022 11:28:45 +0100 Subject: [PATCH 5/6] Add permalink property to SoundCloudTrack --- play-dl/SoundCloud/classes.ts | 6 ++++++ play-dl/SoundCloud/constants.ts | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/play-dl/SoundCloud/classes.ts b/play-dl/SoundCloud/classes.ts index 0fcff6e..ae1254f 100644 --- a/play-dl/SoundCloud/classes.ts +++ b/play-dl/SoundCloud/classes.ts @@ -104,6 +104,10 @@ export class SoundCloudTrack { * SoundCloud Track url */ url: string; + /** + * User friendly SoundCloud track URL + */ + permalink: string; /** * SoundCloud Track fetched status */ @@ -150,6 +154,7 @@ export class SoundCloudTrack { this.name = data.title; this.id = data.id; this.url = data.uri; + this.permalink = data.permalink_url; this.fetched = true; this.type = 'track'; this.durationInSec = Math.round(Number(data.duration) / 1000); @@ -187,6 +192,7 @@ export class SoundCloudTrack { name: this.name, id: this.id, url: this.url, + permalink: this.permalink, fetched: this.fetched, durationInMs: this.durationInMs, durationInSec: this.durationInSec, diff --git a/play-dl/SoundCloud/constants.ts b/play-dl/SoundCloud/constants.ts index 04f1c35..08656aa 100644 --- a/play-dl/SoundCloud/constants.ts +++ b/play-dl/SoundCloud/constants.ts @@ -13,6 +13,10 @@ export interface SoundTrackJSON { * SoundCloud Track url */ url: string; + /** + * User friendly SoundCloud track URL + */ + permalink: string; /** * SoundCloud Track fetched status */ From 026fa34fdb4cae3516634347677f4ce7e437eba4 Mon Sep 17 00:00:00 2001 From: absidue <48293849+absidue@users.noreply.github.com> Date: Sun, 27 Feb 2022 12:28:09 +0100 Subject: [PATCH 6/6] Fix legacy streams being closed early because they look idle --- play-dl/YouTube/utils/extractor.ts | 44 ++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/play-dl/YouTube/utils/extractor.ts b/play-dl/YouTube/utils/extractor.ts index 1984d65..39ac7f3 100644 --- a/play-dl/YouTube/utils/extractor.ts +++ b/play-dl/YouTube/utils/extractor.ts @@ -274,6 +274,13 @@ export async function video_basic_info(url: string, options: InfoOptions = {}): if (!upcoming) { format.push(...(player_response.streamingData.formats ?? [])); format.push(...(player_response.streamingData.adaptiveFormats ?? [])); + + // get the formats for the android player for legacy videos + // fixes the stream being closed because not enough data + // arrived in time for ffmpeg to be able to extract audio data + if (parseAudioFormats(format).length === 0 && !options.htmldata) { + format = await getAndroidFormats(vid.videoId, cookieJar, body); + } } const LiveStreamData = { isLive: video_details.live, @@ -373,6 +380,13 @@ export async function video_stream_info(url: string, options: InfoOptions = {}): if (!upcoming) { format.push(...(player_response.streamingData.formats ?? [])); format.push(...(player_response.streamingData.adaptiveFormats ?? [])); + + // get the formats for the android player for legacy videos + // fixes the stream being closed because not enough data + // arrived in time for ffmpeg to be able to extract audio data + if (parseAudioFormats(format).length === 0 && !options.htmldata) { + format = await getAndroidFormats(player_response.videoDetails.videoId, cookieJar, body); + } } const LiveStreamData = { @@ -639,6 +653,36 @@ async function acceptViewerDiscretion( return { streamingData }; } +async function getAndroidFormats(videoId: string, cookieJar: { [key: string]: string }, body: string): Promise { + const apiKey = + body.split('INNERTUBE_API_KEY":"')[1]?.split('"')[0] ?? + body.split('innertubeApiKey":"')[1]?.split('"')[0] ?? + DEFAULT_API_KEY; + + const response = await request(`https://www.youtube.com/youtubei/v1/player?key=${apiKey}`, { + method: 'POST', + body: JSON.stringify({ + context: { + client: { + clientName: 'ANDROID', + clientVersion: '16.49', + hl: 'en', + timeZone: 'UTC', + utcOffsetMinutes: 0 + } + }, + videoId: videoId, + playbackContext: { contentPlaybackContext: { html5Preference: 'HTML5_PREF_WANTS' } }, + contentCheckOk: true, + racyCheckOk: true + }), + cookies: true, + cookieJar + }); + + return JSON.parse(response).streamingData.formats; +} + function getWatchPlaylist(response: any, body: any, url: string): YouTubePlayList { const playlist_details = response.contents.twoColumnWatchNextResults.playlist.playlist;