From 04770cc9291dd2a4018a89bf7eca4bcf50f5eba8 Mon Sep 17 00:00:00 2001 From: killer069 <65385476+killer069@users.noreply.github.com> Date: Fri, 13 Aug 2021 13:16:34 +0530 Subject: [PATCH] YouTube : Completed --- node-youtube-dl/YouTube/classes/Channel.ts | 6 +- node-youtube-dl/YouTube/classes/Playlist.ts | 38 ++++- node-youtube-dl/YouTube/classes/Thumbnail.ts | 4 +- node-youtube-dl/YouTube/classes/Video.ts | 11 +- node-youtube-dl/YouTube/index.ts | 4 +- node-youtube-dl/YouTube/search.ts | 25 +++- node-youtube-dl/YouTube/utils/extractor.ts | 146 ++++++++++++++++++- node-youtube-dl/YouTube/utils/index.ts | 2 +- node-youtube-dl/YouTube/utils/parser.ts | 51 ++----- node-youtube-dl/index.ts | 4 +- 10 files changed, 215 insertions(+), 76 deletions(-) diff --git a/node-youtube-dl/YouTube/classes/Channel.ts b/node-youtube-dl/YouTube/classes/Channel.ts index c0fa6bc..d1bafa3 100644 --- a/node-youtube-dl/YouTube/classes/Channel.ts +++ b/node-youtube-dl/YouTube/classes/Channel.ts @@ -6,10 +6,10 @@ export interface ChannelIconInterface { export class Channel { name?: string; - verified!: boolean; + verified?: boolean; id?: string; url?: string; - icon!: ChannelIconInterface; + icon?: ChannelIconInterface; subscribers?: string; constructor(data: any) { @@ -36,7 +36,7 @@ export class Channel { */ iconURL(options = { size: 0 }): string | undefined{ if (typeof options.size !== "number" || options.size < 0) throw new Error("invalid icon size"); - if (!this.icon.url) return undefined; + if (!this.icon?.url) return undefined; const def = this.icon.url.split("=s")[1].split("-c")[0]; return this.icon.url.replace(`=s${def}-c`, `=s${options.size}-c`); } diff --git a/node-youtube-dl/YouTube/classes/Playlist.ts b/node-youtube-dl/YouTube/classes/Playlist.ts index f321981..46a48e1 100644 --- a/node-youtube-dl/YouTube/classes/Playlist.ts +++ b/node-youtube-dl/YouTube/classes/Playlist.ts @@ -1,41 +1,47 @@ -import { getContinuationToken, getPlaylistVideos } from "../utils/parser"; +import { getPlaylistVideos, getContinuationToken } from "../utils/extractor"; import { url_get } from "../utils/request"; import { Thumbnail } from "./Thumbnail"; import { Channel } from "./Channel"; import { Video } from "./Video"; +import fs from 'fs' const BASE_API = "https://www.youtube.com/youtubei/v1/browse?key="; export class PlayList{ id?: string; title?: string; - videoCount!: number; + videoCount?: number; lastUpdate?: string; views?: number; url?: string; link?: string; channel?: Channel; thumbnail?: Thumbnail; - videos!: []; + videos?: []; + private fetched_videos : Map private _continuation: { api?: string; token?: string; clientVersion?: string } = {}; + private __count : number constructor(data : any, searchResult : Boolean = false){ if (!data) throw new Error(`Cannot instantiate the ${this.constructor.name} class without data!`); - + this.__count = 0 + this.fetched_videos = new Map() if(searchResult) this.__patchSearch(data) else this.__patch(data) } private __patch(data:any){ this.id = data.id || undefined; + this.url = data.url || undefined; this.title = data.title || undefined; this.videoCount = data.videoCount || 0; this.lastUpdate = data.lastUpdate || undefined; this.views = data.views || 0; - this.url = data.url || undefined; this.link = data.link || undefined; this.channel = data.author || undefined; this.thumbnail = data.thumbnail || undefined; this.videos = data.videos || []; + this.__count ++ + this.fetched_videos.set(`page${this.__count}`, this.videos as Video[]) this._continuation.api = data.continuation?.api ?? undefined; this._continuation.token = data.continuation?.token ?? undefined; this._continuation.clientVersion = data.continuation?.clientVersion ?? ""; @@ -43,12 +49,12 @@ export class PlayList{ private __patchSearch(data: any){ this.id = data.id || undefined; + this.url = this.id ? `https://www.youtube.com/playlist?list=${this.id}` : undefined; this.title = data.title || undefined; this.thumbnail = data.thumbnail || undefined; this.channel = data.channel || undefined; this.videos = []; this.videoCount = data.videos || 0; - this.url = this.id ? `https://www.youtube.com/playlist?list=${this.id}` : undefined; this.link = undefined; this.lastUpdate = undefined; this.views = 0; @@ -79,8 +85,8 @@ export class PlayList{ if(!contents) return [] let playlist_videos = getPlaylistVideos(contents, limit) + this.fetched_videos.set(`page${this.__count}`, playlist_videos) this._continuation.token = getContinuationToken(contents) - return playlist_videos } @@ -90,7 +96,8 @@ export class PlayList{ if (max < 1) max = Infinity; while (typeof this._continuation.token === "string" && this._continuation.token.length) { - if (this.videos.length >= max) break; + if (this.videos?.length as number >= max) break; + this.__count++ const res = await this.next(); if (!res.length) break; } @@ -102,6 +109,21 @@ export class PlayList{ return "playlist"; } + page(number : number): Video[]{ + if(!number) throw new Error('Given Page number is not provided') + if(!this.fetched_videos.has(`page${number}`)) throw new Error('Given Page number is invalid') + return this.fetched_videos.get(`page${number}`) as Video[] + } + + get total_pages(){ + return this.fetched_videos.size + } + + get total_videos(){ + let page_number: number = this.total_pages + return (page_number - 1) * 100 + (this.fetched_videos.get(`page${page_number}`) as Video[]).length + } + toJSON() { return { id: this.id, diff --git a/node-youtube-dl/YouTube/classes/Thumbnail.ts b/node-youtube-dl/YouTube/classes/Thumbnail.ts index 2bead19..4ea8db5 100644 --- a/node-youtube-dl/YouTube/classes/Thumbnail.ts +++ b/node-youtube-dl/YouTube/classes/Thumbnail.ts @@ -2,8 +2,8 @@ type ThumbnailType = "default" | "hqdefault" | "mqdefault" | "sddefault" | "maxr export class Thumbnail { id?: string; - width!: number; - height!: number; + width?: number; + height?: number; url?: string; constructor(data: any) { diff --git a/node-youtube-dl/YouTube/classes/Video.ts b/node-youtube-dl/YouTube/classes/Video.ts index 2295683..bb1969e 100644 --- a/node-youtube-dl/YouTube/classes/Video.ts +++ b/node-youtube-dl/YouTube/classes/Video.ts @@ -12,8 +12,8 @@ interface VideoOptions { views: number; thumbnail?: { id: string | undefined; - width: number; - height: number; + width: number | undefined ; + height: number | undefined; url: string | undefined; }; channel?: { @@ -34,6 +34,7 @@ interface VideoOptions { export class Video { id?: string; + url? : string; title?: string; description?: string; durationFormatted: string; @@ -53,6 +54,7 @@ export class Video { if(!data) throw new Error(`Can not initiate ${this.constructor.name} without data`) this.id = data.id || undefined; + this.url = `https://www.youtube.com/watch?v=${this.id}` this.title = data.title || undefined; this.description = data.description || undefined; this.durationFormatted = data.duration_raw || "0:00"; @@ -68,11 +70,6 @@ export class Video { this.tags = data.tags || []; } - get url(){ - if(!this.id) return undefined - else return `https://www.youtube.com/watch?v=${this.id}`; - } - get type(): "video" { return "video"; } diff --git a/node-youtube-dl/YouTube/index.ts b/node-youtube-dl/YouTube/index.ts index 128af5e..ea8738c 100644 --- a/node-youtube-dl/YouTube/index.ts +++ b/node-youtube-dl/YouTube/index.ts @@ -1 +1,3 @@ -export { search } from './search' \ No newline at end of file +export { search } from './search' + +export * from './utils' \ No newline at end of file diff --git a/node-youtube-dl/YouTube/search.ts b/node-youtube-dl/YouTube/search.ts index 76d7c7f..0d597d7 100644 --- a/node-youtube-dl/YouTube/search.ts +++ b/node-youtube-dl/YouTube/search.ts @@ -6,8 +6,29 @@ import { Channel } from "./classes/Channel"; import { PlayList } from "./classes/Playlist"; -export async function search(url:string, options? : ParseSearchInterface): Promise<(Video | Channel | PlayList)[]> { +enum SearchType { + Video = 'EgIQAQ%253D%253D', + PlayList = 'EgIQAw%253D%253D', + Channel = 'EgIQAg%253D%253D', +} + +export async function search(search :string, options? : ParseSearchInterface): Promise<(Video | Channel | PlayList)[]> { + let url = 'https://www.youtube.com/results?search_query=' + search.replaceAll(' ', '+') + if(!url.match('&sp=')){ + url += '&sp=' + switch(options?.type){ + case 'channel': + url += SearchType.Channel + break + case 'playlist': + url += SearchType.PlayList + break + case 'video': + url += SearchType.Video + break + } + } let body = await url_get(url) - let data = ParseSearchResult(body) + let data = ParseSearchResult(body, options) return data } \ No newline at end of file diff --git a/node-youtube-dl/YouTube/utils/extractor.ts b/node-youtube-dl/YouTube/utils/extractor.ts index 56e79e1..8ea2078 100644 --- a/node-youtube-dl/YouTube/utils/extractor.ts +++ b/node-youtube-dl/YouTube/utils/extractor.ts @@ -1,10 +1,24 @@ import { url_get } from './request' import { format_decipher, js_tokens } from './cipher' +import { Video } from '../classes/Video' +import { RequestInit } from 'node-fetch' +import { PlayList } from '../classes/Playlist' +import fs from 'fs' +const DEFAULT_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"; +const youtube_url = /https:\/\/www.youtube.com\//g +const video_pattern = /^((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w\-]+\?v=|embed\/|v\/)?)([\w\-]+)(\S+)?$/; -export async function yt_initial_data(url : string){ +export interface PlaylistOptions { + limit?: number; + requestOptions?: RequestInit; +} + +export async function video_basic_info(url : string){ + if(!url.match(youtube_url) || !url.match(video_pattern)) throw new Error('This is not a YouTube URL') let body = await url_get(url) let player_response = JSON.parse(body.split("var ytInitialPlayerResponse = ")[1].split(";")[0]) + if(player_response.playabilityStatus.status === 'ERROR') throw new Error(`While getting info from url \n ${player_response.playabilityStatus.reason}`) let response = JSON.parse(body.split("var ytInitialData = ")[1].split(";")[0]) let html5player = 'https://www.youtube.com' + body.split('"jsUrl":"')[1].split('"')[0] let format = [] @@ -12,14 +26,18 @@ export async function yt_initial_data(url : string){ format.push(...player_response.streamingData.adaptiveFormats) let vid = player_response.videoDetails let microformat = player_response.microformat.playerMicroformatRenderer - let video_details = { + let video_details = new Video ({ id : vid.videoId, url : 'https://www.youtube.com/watch?v=' + vid.videoId, title : vid.title, description : vid.shortDescription, duration : vid.lengthSeconds, uploadedDate : microformat.publishDate, - thumbnail : `https://i.ytimg.com/vi/${vid.videoId}/maxresdefault.jpg`, + thumbnail : { + width : vid.thumbnail.thumbnails[vid.thumbnail.thumbnails.length - 1].width, + height : vid.thumbnail.thumbnails[vid.thumbnail.thumbnails.length - 1].height, + url : `https://i.ytimg.com/vi/${vid.videoId}/maxresdefault.jpg` + }, channel : { name : vid.author, id : vid.channelId, @@ -30,19 +48,18 @@ export async function yt_initial_data(url : string){ averageRating : vid.averageRating, live : vid.isLiveContent, private : vid.isPrivate - } - let final = { + }) + return { player_response, response, html5player, format, video_details } - return final } -export async function yt_deciphered_data(url : string) { - let data = await yt_initial_data(url) +export async function video_info(url : string) { + let data = await video_basic_info(url) if(data.format[0].signatureCipher || data.format[0].cipher){ data.format = await format_decipher(data.format, data.html5player) return data @@ -51,3 +68,116 @@ export async function yt_deciphered_data(url : string) { return data } } + +export async function playlist_info(url : string , options? : PlaylistOptions) { + if (!options) options = { limit: 100, requestOptions: {} }; + if(!options.limit) options.limit = 100 + if (!url || typeof url !== "string") throw new Error(`Expected playlist url, received ${typeof url}!`); + if(url.search('(\\?|\\&)list\\=') === -1) throw new Error('This is not a PlayList URL') + + let Playlist_id = url.split('list=')[1].split('&')[0] + let new_url = `https://www.youtube.com/playlist?list=${Playlist_id}` + + let body = await url_get(new_url) + let response = JSON.parse(body.split("var ytInitialData = ")[1].split(";")[0]) + if(response.alerts && response.alerts[0].alertRenderer.type === 'ERROR') throw new Error(`While parsing playlist url\n ${response.alerts[0].alertRenderer.text.runs[0].text}`) + + let rawJSON = `${body.split('{"playlistVideoListRenderer":{"contents":')[1].split('}],"playlistId"')[0]}}]`; + let parsed = JSON.parse(rawJSON); + let playlistDetails = JSON.parse(body.split('{"playlistSidebarRenderer":')[1].split("}};")[0]).items; + + let API_KEY = body.split('INNERTUBE_API_KEY":"')[1]?.split('"')[0] ?? body.split('innertubeApiKey":"')[1]?.split('"')[0] ?? DEFAULT_API_KEY; + let videos = getPlaylistVideos(parsed, options.limit); + + let data = playlistDetails[0].playlistSidebarPrimaryInfoRenderer; + if (!data.title.runs || !data.title.runs.length) return undefined; + + let author = playlistDetails[1]?.playlistSidebarSecondaryInfoRenderer.videoOwner; + let views = data.stats.length === 3 ? data.stats[1].simpleText.replace(/[^0-9]/g, "") : 0; + let lastUpdate = data.stats.find((x: any) => "runs" in x && x["runs"].find((y: any) => y.text.toLowerCase().includes("last update")))?.runs.pop()?.text ?? null; + let videosCount = data.stats[0].runs[0].text.replace(/[^0-9]/g, "") || 0; + + let res = new PlayList({ + continuation: { + api: API_KEY, + token: getContinuationToken(parsed), + clientVersion: body.split('"INNERTUBE_CONTEXT_CLIENT_VERSION":"')[1]?.split('"')[0] ?? body.split('"innertube_context_client_version":"')[1]?.split('"')[0] ?? "" + }, + id: data.title.runs[0].navigationEndpoint.watchEndpoint.playlistId, + title: data.title.runs[0].text, + videoCount: parseInt(videosCount) || 0, + lastUpdate: lastUpdate, + views: parseInt(views) || 0, + videos: videos, + url: `https://www.youtube.com/playlist?list=${data.title.runs[0].navigationEndpoint.watchEndpoint.playlistId}`, + link: `https://www.youtube.com${data.title.runs[0].navigationEndpoint.commandMetadata.webCommandMetadata.url}`, + author: author + ? { + name: author.videoOwnerRenderer.title.runs[0].text, + id: author.videoOwnerRenderer.title.runs[0].navigationEndpoint.browseEndpoint.browseId, + url: `https://www.youtube.com${author.videoOwnerRenderer.navigationEndpoint.commandMetadata.webCommandMetadata.url || author.videoOwnerRenderer.navigationEndpoint.browseEndpoint.canonicalBaseUrl}`, + icon: author.videoOwnerRenderer.thumbnail.thumbnails.length ? author.videoOwnerRenderer.thumbnail.thumbnails[author.videoOwnerRenderer.thumbnail.thumbnails.length - 1].url : null + } + : {}, + thumbnail: data.thumbnailRenderer.playlistVideoThumbnailRenderer?.thumbnail.thumbnails.length ? data.thumbnailRenderer.playlistVideoThumbnailRenderer.thumbnail.thumbnails[data.thumbnailRenderer.playlistVideoThumbnailRenderer.thumbnail.thumbnails.length - 1].url : null + }); + return res; +} + +export function getPlaylistVideos(data:any, limit : number = Infinity) : Video[] { + const videos = []; + + for (let i = 0; i < data.length; i++) { + if (limit === videos.length) break; + const info = data[i].playlistVideoRenderer; + if (!info || !info.shortBylineText) continue; + + videos.push( + new Video({ + id: info.videoId, + index: parseInt(info.index?.simpleText) || 0, + duration: parseDuration(info.lengthText?.simpleText) || 0, + duration_raw: info.lengthText?.simpleText ?? "0:00", + thumbnail: { + id: info.videoId, + url: info.thumbnail.thumbnails[info.thumbnail.thumbnails.length - 1].url, + height: info.thumbnail.thumbnails[info.thumbnail.thumbnails.length - 1].height, + width: info.thumbnail.thumbnails[info.thumbnail.thumbnails.length - 1].width + }, + title: info.title.runs[0].text, + channel: { + id: info.shortBylineText.runs[0].navigationEndpoint.browseEndpoint.browseId || undefined, + name: info.shortBylineText.runs[0].text || undefined, + url: `https://www.youtube.com${info.shortBylineText.runs[0].navigationEndpoint.browseEndpoint.canonicalBaseUrl || info.shortBylineText.runs[0].navigationEndpoint.commandMetadata.webCommandMetadata.url}`, + icon: undefined + } + }) + ); + } + return videos +} + +function parseDuration(duration: string): number { + duration ??= "0:00"; + const args = duration.split(":"); + let dur = 0; + + switch (args.length) { + case 3: + dur = parseInt(args[0]) * 60 * 60 + parseInt(args[1]) * 60 + parseInt(args[2]); + break; + case 2: + dur = parseInt(args[0]) * 60 + parseInt(args[1]); + break; + default: + dur = parseInt(args[0]); + } + + return dur; +} + + +export function getContinuationToken(data:any): string { + const continuationToken = data.find((x: any) => Object.keys(x)[0] === "continuationItemRenderer")?.continuationItemRenderer.continuationEndpoint?.continuationCommand?.token; + return continuationToken; +} \ No newline at end of file diff --git a/node-youtube-dl/YouTube/utils/index.ts b/node-youtube-dl/YouTube/utils/index.ts index f09bebc..52ec51e 100644 --- a/node-youtube-dl/YouTube/utils/index.ts +++ b/node-youtube-dl/YouTube/utils/index.ts @@ -1 +1 @@ -export { yt_initial_data, yt_deciphered_data } from './extractor' \ No newline at end of file +export { video_basic_info, video_info, playlist_info } from './extractor' \ No newline at end of file diff --git a/node-youtube-dl/YouTube/utils/parser.ts b/node-youtube-dl/YouTube/utils/parser.ts index 5fc1dbf..ab8c702 100644 --- a/node-youtube-dl/YouTube/utils/parser.ts +++ b/node-youtube-dl/YouTube/utils/parser.ts @@ -2,6 +2,7 @@ import { Video } from "../classes/Video"; import { PlayList } from "../classes/Playlist"; import { Channel } from "../classes/Channel"; import { RequestInit } from "node-fetch"; +import fs from 'fs' export interface ParseSearchInterface { type?: "video" | "playlist" | "channel" | "all"; @@ -31,7 +32,7 @@ export function ParseSearchResult(html :string, options? : ParseSearchInterface) details = JSON.parse(html.split('{"itemSectionRenderer":{"contents":')[html.split('{"itemSectionRenderer":{"contents":').length - 1].split(',"continuations":[{')[0]); fetched = true; } catch { - /* do nothing */ + /* Do nothing*/ } if (!fetched) { @@ -76,40 +77,7 @@ export function ParseSearchResult(html :string, options? : ParseSearchInterface) return results as (Video | Channel | PlayList)[]; } -export function getPlaylistVideos(data:any, limit : number = Infinity) : Video[] { - const videos = []; - - for (let i = 0; i < data.length; i++) { - if (limit === videos.length) break; - const info = data[i].playlistVideoRenderer; - if (!info || !info.shortBylineText) continue; - - videos.push( - new Video({ - id: info.videoId, - index: parseInt(info.index?.simpleText) || 0, - duration: parseDuration(info.lengthText?.simpleText) || 0, - duration_raw: info.lengthText?.simpleText ?? "0:00", - thumbnail: { - id: info.videoId, - url: info.thumbnail.thumbnails[info.thumbnail.thumbnails.length - 1].url, - height: info.thumbnail.thumbnails[info.thumbnail.thumbnails.length - 1].height, - width: info.thumbnail.thumbnails[info.thumbnail.thumbnails.length - 1].width - }, - title: info.title.runs[0].text, - channel: { - id: info.shortBylineText.runs[0].navigationEndpoint.browseEndpoint.browseId || undefined, - name: info.shortBylineText.runs[0].text || undefined, - url: `https://www.youtube.com${info.shortBylineText.runs[0].navigationEndpoint.browseEndpoint.canonicalBaseUrl || info.shortBylineText.runs[0].navigationEndpoint.commandMetadata.webCommandMetadata.url}`, - icon: undefined - } - }) - ); - } - return videos -} - -export function parseDuration(duration: string): number { +function parseDuration(duration: string): number { duration ??= "0:00"; const args = duration.split(":"); let dur = 0; @@ -128,11 +96,6 @@ export function parseDuration(duration: string): number { return dur; } -export function getContinuationToken(data:any): string { - const continuationToken = data.find((x: any) => Object.keys(x)[0] === "continuationItemRenderer")?.continuationItemRenderer.continuationEndpoint?.continuationCommand?.token; - return continuationToken; -} - export function parseChannel(data?: any): Channel | void { if (!data || !data.channelRenderer) return; const badge = data.channelRenderer.ownerBadges && data.channelRenderer.ownerBadges[0]; @@ -140,10 +103,14 @@ export function parseChannel(data?: any): Channel | void { let res = new Channel({ id: data.channelRenderer.channelId, name: data.channelRenderer.title.simpleText, - icon: data.channelRenderer.thumbnail.thumbnails[data.channelRenderer.thumbnail.thumbnails.length - 1], + icon: { + url : data.channelRenderer.thumbnail.thumbnails[data.channelRenderer.thumbnail.thumbnails.length - 1].url.replace('//', 'https://'), + width : data.channelRenderer.thumbnail.thumbnails[data.channelRenderer.thumbnail.thumbnails.length - 1].width, + height: data.channelRenderer.thumbnail.thumbnails[data.channelRenderer.thumbnail.thumbnails.length - 1].height + }, url: url, verified: Boolean(badge?.metadataBadgeRenderer?.style?.toLowerCase().includes("verified")), - subscribers: data.channelRenderer.subscriberCountText.simpleText + subscribers: (data.channelRenderer.subscriberCountText?.simpleText) ? data.channelRenderer.subscriberCountText.simpleText : '0 subscribers' }); return res; diff --git a/node-youtube-dl/index.ts b/node-youtube-dl/index.ts index 18cc5d6..4b7e624 100644 --- a/node-youtube-dl/index.ts +++ b/node-youtube-dl/index.ts @@ -1,8 +1,8 @@ -import { search } from "./YouTube/"; +import { playlist_info } from "./YouTube"; let main = async() => { let time_start = Date.now() - await search('https://www.youtube.com/results?search_query=Hello+Neghibour') + let playlist = await playlist_info('https://www.youtube.com/watch?v=bM7SZ5SBzyY&list=PLzkuLC6Yvumv_Rd5apfPRWEcjf9b1JRnq') let time_end = Date.now() console.log(`Time Taken : ${(time_end - time_start)/1000} seconds`) }