From 61cddbce5e40a4c8b1d38228757d2ca45c8cff52 Mon Sep 17 00:00:00 2001 From: absidue <48293849+absidue@users.noreply.github.com> Date: Mon, 27 Dec 2021 17:15:48 +0100 Subject: [PATCH] Add support for videos with a viewer discretion advisory --- play-dl/Request/index.ts | 26 ++++- play-dl/YouTube/classes/Video.ts | 12 +- play-dl/YouTube/utils/extractor.ts | 174 ++++++++++++++++++++++++++--- 3 files changed, 190 insertions(+), 22 deletions(-) diff --git a/play-dl/Request/index.ts b/play-dl/Request/index.ts index 44881b0..fccbe15 100644 --- a/play-dl/Request/index.ts +++ b/play-dl/Request/index.ts @@ -9,6 +9,7 @@ interface RequestOpts extends RequestOptions { body?: string; method?: 'GET' | 'POST' | 'HEAD'; cookies?: boolean; + cookieJar?: { [key: string]: string }; } /** @@ -33,7 +34,6 @@ export function request_stream(req_url: string, options: RequestOpts = { method: /** * Makes a request and follows redirects if necessary * @param req_url URL to make https request to - * @param cookies_added Whether cookies were added or not * @param options Request options for https request * @returns A promise with the final response object */ @@ -69,6 +69,18 @@ export function request(req_url: string, options: RequestOpts = { method: 'GET' cookies_added = true; } } + if (options.cookieJar) { + const cookies = []; + for (const cookie of Object.entries(options.cookieJar)) { + cookies.push(cookie.join('=')); + } + + if (cookies.length !== 0) { + if (!options.headers) options.headers = {}; + const existingCookies = cookies_added ? `; ${options.headers.cookie}` : ''; + Object.assign(options.headers, { cookie: `${cookies.join('; ')}${existingCookies}` }); + } + } if (options.headers) { options.headers = { ...options.headers, @@ -81,8 +93,16 @@ export function request(req_url: string, options: RequestOpts = { method: 'GET' reject(res); return; } - if (res.headers && res.headers['set-cookie'] && cookies_added) { - cookieHeaders(res.headers['set-cookie']); + if (res.headers && res.headers['set-cookie']) { + if (options.cookieJar) { + for (const cookie of res.headers['set-cookie']) { + const parts = cookie.split(';')[0].trim().split('='); + options.cookieJar[parts.shift() as string] = parts.join('='); + } + } + if (cookies_added) { + cookieHeaders(res.headers['set-cookie']); + } } const data: string[] = []; let decoder: BrotliDecompress | Gunzip | Deflate | undefined = undefined; diff --git a/play-dl/YouTube/classes/Video.ts b/play-dl/YouTube/classes/Video.ts index 4121be8..9ac9418 100644 --- a/play-dl/YouTube/classes/Video.ts +++ b/play-dl/YouTube/classes/Video.ts @@ -62,6 +62,10 @@ interface VideoOptions { * YouTube Video tags */ tags: string[]; + /** + * `true` if the video has been identified by the YouTube community as inappropriate or offensive to some audiences and viewer discretion is advised + */ + discretionAdvised: boolean; } /** * Class for YouTube Video url @@ -127,6 +131,10 @@ export class YouTubeVideo { * YouTube Video tags */ tags: string[]; + /** + * `true` if the video has been identified by the YouTube community as inappropriate or offensive to some audiences and viewer discretion is advised + */ + discretionAdvised: boolean; /** * Constructor for YouTube Video Class * @param data JSON parsed data. @@ -153,6 +161,7 @@ export class YouTubeVideo { this.live = !!data.live; this.private = !!data.private; this.tags = data.tags || []; + this.discretionAdvised = !!data.discretionAdvised; } /** * Converts class to title name of video. @@ -180,7 +189,8 @@ export class YouTubeVideo { tags: this.tags, likes: this.likes, live: this.live, - private: this.private + private: this.private, + discretionAdvised: this.discretionAdvised }; } } diff --git a/play-dl/YouTube/utils/extractor.ts b/play-dl/YouTube/utils/extractor.ts index 24a5ca1..c6aabdf 100644 --- a/play-dl/YouTube/utils/extractor.ts +++ b/play-dl/YouTube/utils/extractor.ts @@ -3,6 +3,7 @@ import { format_decipher } from './cipher'; import { YouTubeVideo } from '../classes/Video'; import { YouTubePlayList } from '../classes/Playlist'; import { InfoData, StreamInfoData } from './constants'; +import { URLSearchParams } from 'node:url'; interface InfoOptions { htmldata?: boolean; @@ -104,6 +105,7 @@ export function extractID(url: string): string { export async function video_basic_info(url: string, options: InfoOptions = {}): Promise { if (typeof url !== 'string') throw new Error('url parameter is not a URL string or a string of HTML'); let body: string; + const cookieJar = {}; if (options.htmldata) { body = url; } else { @@ -114,7 +116,8 @@ export async function video_basic_info(url: string, options: InfoOptions = {}): headers: { 'accept-language': options.language || 'en-US;q=0.9' }, - cookies: true + cookies: true, + cookieJar }); } if (body.indexOf('Our systems have detected unusual traffic from your computer network.') !== -1) @@ -131,13 +134,35 @@ export async function video_basic_info(url: string, options: InfoOptions = {}): if (!initial_data) throw new Error('Initial Response Data is undefined.'); const player_response = JSON.parse(player_data); const initial_response = JSON.parse(initial_data); - if (player_response.playabilityStatus.status !== 'OK') - throw new Error( - `While getting info from url\n${ - player_response.playabilityStatus.errorScreen.playerErrorMessageRenderer?.reason.simpleText ?? - player_response.playabilityStatus.errorScreen.playerKavRenderer?.reason.simpleText - }` - ); + const vid = player_response.videoDetails; + + let discretionAdvised = false; + if (player_response.playabilityStatus.status !== 'OK') { + if (player_response.playabilityStatus.status === 'CONTENT_CHECK_REQUIRED') { + if (options.htmldata) + throw new Error( + `Accepting the viewer discretion is not supported when using htmldata, video: ${vid.videoId}` + ); + discretionAdvised = true; + const cookies = + initial_response.topbar.desktopTopbarRenderer.interstitial.consentBumpV2Renderer.agreeButton + .buttonRenderer.command.saveConsentAction; + Object.assign(cookieJar, { + VISITOR_INFO1_LIVE: cookies.visitorCookie, + CONSENT: cookies.consentCookie + }); + + const updatedValues = await acceptViewerDiscretion(vid.videoId, cookieJar, body, true); + player_response.streamingData = updatedValues.streamingData; + initial_response.contents.twoColumnWatchNextResults.secondaryResults = updatedValues.relatedVideos; + } else + throw new Error( + `While getting info from url\n${ + player_response.playabilityStatus.errorScreen.playerErrorMessageRenderer?.reason.simpleText ?? + player_response.playabilityStatus.errorScreen.playerKavRenderer?.reason.simpleText + }` + ); + } const ownerInfo = initial_response.contents.twoColumnWatchNextResults.results?.results?.contents[1]?.videoSecondaryInfoRenderer ?.owner?.videoOwnerRenderer; @@ -150,7 +175,6 @@ export async function video_basic_info(url: string, options: InfoOptions = {}): related.push(`https://www.youtube.com/watch?v=${res.compactVideoRenderer.videoId}`); } ); - const vid = player_response.videoDetails; const microformat = player_response.microformat.playerMicroformatRenderer; const video_details = new YouTubeVideo({ id: vid.videoId, @@ -179,7 +203,8 @@ export async function video_basic_info(url: string, options: InfoOptions = {}): ?.toggleButtonRenderer.defaultText.accessibility?.accessibilityData.label.replace(/\D+/g, '') ?? 0 ), live: vid.isLiveContent, - private: vid.isPrivate + private: vid.isPrivate, + discretionAdvised }); const format = player_response.streamingData.formats ?? []; format.push(...(player_response.streamingData.adaptiveFormats ?? [])); @@ -210,6 +235,7 @@ export async function video_basic_info(url: string, options: InfoOptions = {}): export async function video_stream_info(url: string, options: InfoOptions = {}): Promise { if (typeof url !== 'string') throw new Error('url parameter is not a URL string or a string of HTML'); let body: string; + const cookieJar = {}; if (options.htmldata) { body = url; } else { @@ -218,7 +244,8 @@ export async function video_stream_info(url: string, options: InfoOptions = {}): const new_url = `https://www.youtube.com/watch?v=${video_id}&has_verified=1`; body = await request(new_url, { headers: { 'accept-language': 'en-US,en;q=0.9' }, - cookies: true + cookies: true, + cookieJar }); } if (body.indexOf('Our systems have detected unusual traffic from your computer network.') !== -1) @@ -229,13 +256,42 @@ export async function video_stream_info(url: string, options: InfoOptions = {}): .split(/;\s*(var|const|let)\s/)[0]; if (!player_data) throw new Error('Initial Player Response Data is undefined.'); const player_response = JSON.parse(player_data); - if (player_response.playabilityStatus.status !== 'OK') - throw new Error( - `While getting info from url\n${ - player_response.playabilityStatus.errorScreen.playerErrorMessageRenderer?.reason.simpleText ?? - player_response.playabilityStatus.errorScreen.playerKavRenderer?.reason.simpleText - }` - ); + if (player_response.playabilityStatus.status !== 'OK') { + if (player_response.playabilityStatus.status === 'CONTENT_CHECK_REQUIRED') { + if (options.htmldata) + throw new Error( + `Accepting the viewer discretion is not supported when using htmldata, video: ${player_response.videoDetails.videoId}` + ); + + const initial_data = body + .split('var ytInitialData = ')?.[1] + ?.split(';')[0] + .split(/;\s*(var|const|let)\s/)[0]; + if (!initial_data) throw new Error('Initial Response Data is undefined.'); + + const cookies = + JSON.parse(initial_data).topbar.desktopTopbarRenderer.interstitial.consentBumpV2Renderer.agreeButton + .buttonRenderer.command.saveConsentAction; + Object.assign(cookieJar, { + VISITOR_INFO1_LIVE: cookies.visitorCookie, + CONSENT: cookies.consentCookie + }); + + const updatedValues = await acceptViewerDiscretion( + player_response.videoDetails.videoId, + cookieJar, + body, + false + ); + player_response.streamingData = updatedValues.streamingData; + } else + throw new Error( + `While getting info from url\n${ + player_response.playabilityStatus.errorScreen.playerErrorMessageRenderer?.reason.simpleText ?? + player_response.playabilityStatus.errorScreen.playerKavRenderer?.reason.simpleText + }` + ); + } const html5player = `https://www.youtube.com${body.split('"jsUrl":"')[1].split('"')[0]}`; const duration = Number(player_response.videoDetails.lengthSeconds); const video_details = { @@ -412,6 +468,88 @@ export function getContinuationToken(data: any): string { .continuationEndpoint?.continuationCommand?.token; } +async function acceptViewerDiscretion( + videoId: string, + cookieJar: { [key: string]: string }, + body: string, + extractRelated: boolean +): Promise<{ streamingData: any; relatedVideos?: any }> { + const apiKey = + body.split('INNERTUBE_API_KEY":"')[1]?.split('"')[0] ?? + body.split('innertubeApiKey":"')[1]?.split('"')[0] ?? + DEFAULT_API_KEY; + const sessionToken = + body.split('"XSRF_TOKEN":"')[1]?.split('"')[0].replaceAll('\\u003d', '=') ?? + body.split('"xsrf_token":"')[1]?.split('"')[0].replaceAll('\\u003d', '='); + if (!sessionToken) + throw new Error(`Unable to extract XSRF_TOKEN to accept the viewer discretion popup for video: ${videoId}.`); + + const verificationResponse = await request(`https://www.youtube.com/youtubei/v1/verify_age?key=${apiKey}`, { + method: 'POST', + body: JSON.stringify({ + context: { + client: { + utcOffsetMinutes: 0, + gl: 'US', + hl: 'en', + clientName: 'WEB', + clientVersion: + body.split('"INNERTUBE_CONTEXT_CLIENT_VERSION":"')[1]?.split('"')[0] ?? + body.split('"innertube_context_client_version":"')[1]?.split('"')[0] ?? + '' + }, + user: {}, + request: {} + }, + nextEndpoint: { + urlEndpoint: { + url: `watch?v=${videoId}` + } + }, + setControvercy: true + }), + cookieJar + }); + + const endpoint = JSON.parse(verificationResponse).actions[0].navigateAction.endpoint; + + const videoPage = await request(`https://www.youtube.com/${endpoint.urlEndpoint.url}&pbj=1`, { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded' + }, + body: new URLSearchParams([ + ['command', JSON.stringify(endpoint)], + ['session_token', sessionToken] + ]).toString(), + cookieJar + }); + + if (videoPage.includes('

Something went wrong

')) + throw new Error(`Unable to accept the viewer discretion popup for video: ${videoId}`); + + const videoPageData = JSON.parse(videoPage); + + if (videoPageData[2].playerResponse.playabilityStatus.status !== 'OK') + throw new Error( + `While getting info from url after trying to accept the discretion popup for video ${videoId}\n${ + videoPageData[2].playerResponse.playabilityStatus.errorScreen.playerErrorMessageRenderer?.reason + .simpleText ?? + videoPageData[2].playerResponse.playabilityStatus.errorScreen.playerKavRenderer?.reason.simpleText + }` + ); + + const streamingData = videoPageData[2].playerResponse.streamingData; + + if (extractRelated) + return { + streamingData, + relatedVideos: videoPageData[3].response.contents.twoColumnWatchNextResults.secondaryResults + }; + + return { streamingData }; +} + function getWatchPlaylist(response: any, body: any): YouTubePlayList { const playlist_details = response.contents.twoColumnWatchNextResults.playlist.playlist;