YouTube : Completed

node-youtube-dl/YouTube/classes/Channel.ts

@@ -6,10 +6,10 @@ export interface ChannelIconInterface {
 
 export class Channel {
     name?: string;
-    verified!: boolean;
+    verified?: boolean;
     id?: string;
     url?: string;
-    icon!: ChannelIconInterface;
+    icon?: ChannelIconInterface;
     subscribers?: string;
 
     constructor(data: any) {
@@ -36,7 +36,7 @@ export class Channel {
      */
     iconURL(options = { size: 0 }): string | undefined{
         if (typeof options.size !== "number" || options.size < 0) throw new Error("invalid icon size");
-        if (!this.icon.url) return undefined;
+        if (!this.icon?.url) return undefined;
         const def = this.icon.url.split("=s")[1].split("-c")[0];
         return this.icon.url.replace(`=s${def}-c`, `=s${options.size}-c`);
     }

node-youtube-dl/YouTube/classes/Playlist.ts

@@ -1,41 +1,47 @@
-import { getContinuationToken, getPlaylistVideos } from "../utils/parser";
+import { getPlaylistVideos, getContinuationToken } from "../utils/extractor";
 import { url_get } from "../utils/request";
 import { Thumbnail } from "./Thumbnail";
 import { Channel } from "./Channel";
 import { Video } from "./Video";
+import fs from 'fs'
 const BASE_API = "https://www.youtube.com/youtubei/v1/browse?key=";
 
 export class PlayList{
     id?: string;
     title?: string;
-    videoCount!: number;
+    videoCount?: number;
     lastUpdate?: string;
     views?: number;
     url?: string;
     link?: string;
     channel?: Channel;
     thumbnail?: Thumbnail;
-    videos!: [];
+    videos?: [];
+    private fetched_videos : Map<string, Video[]>
     private _continuation: { api?: string; token?: string; clientVersion?: string } = {};
+    private __count : number
 
     constructor(data : any, searchResult : Boolean = false){
         if (!data) throw new Error(`Cannot instantiate the ${this.constructor.name} class without data!`);
-
+        this.__count = 0
+        this.fetched_videos = new Map()
         if(searchResult) this.__patchSearch(data)
         else this.__patch(data)
     }
 
     private __patch(data:any){
         this.id = data.id || undefined;
+        this.url = data.url || undefined;
         this.title = data.title || undefined;
         this.videoCount = data.videoCount || 0;
         this.lastUpdate = data.lastUpdate || undefined;
         this.views = data.views || 0;
-        this.url = data.url || undefined;
         this.link = data.link || undefined;
         this.channel = data.author || undefined;
         this.thumbnail = data.thumbnail || undefined;
         this.videos = data.videos || [];
+        this.__count ++
+        this.fetched_videos.set(`page${this.__count}`, this.videos as Video[])
         this._continuation.api = data.continuation?.api ?? undefined;
         this._continuation.token = data.continuation?.token ?? undefined;
         this._continuation.clientVersion = data.continuation?.clientVersion ?? "<important data>";
@@ -43,12 +49,12 @@ export class PlayList{
 
     private __patchSearch(data: any){
         this.id = data.id || undefined;
+        this.url = this.id ? `https://www.youtube.com/playlist?list=${this.id}` : undefined;
         this.title = data.title || undefined;
         this.thumbnail = data.thumbnail || undefined;
         this.channel = data.channel || undefined;
         this.videos = [];
         this.videoCount = data.videos || 0;
-        this.url = this.id ? `https://www.youtube.com/playlist?list=${this.id}` : undefined;
         this.link = undefined;
         this.lastUpdate = undefined;
         this.views = 0;
@@ -79,8 +85,8 @@ export class PlayList{
         if(!contents) return []
 
         let playlist_videos = getPlaylistVideos(contents, limit)
+        this.fetched_videos.set(`page${this.__count}`, playlist_videos)
         this._continuation.token = getContinuationToken(contents)
-
         return playlist_videos
     }
 
@@ -90,7 +96,8 @@ export class PlayList{
         if (max < 1) max = Infinity;
 
         while (typeof this._continuation.token === "string" && this._continuation.token.length) {
-            if (this.videos.length >= max) break;
+            if (this.videos?.length as number >= max) break;
+            this.__count++
             const res = await this.next();
             if (!res.length) break;
         }
@@ -102,6 +109,21 @@ export class PlayList{
         return "playlist";
     }
 
+    page(number : number): Video[]{
+        if(!number) throw new Error('Given Page number is not provided')
+        if(!this.fetched_videos.has(`page${number}`)) throw new Error('Given Page number is invalid')
+        return this.fetched_videos.get(`page${number}`) as Video[]
+    }
+
+    get total_pages(){
+        return this.fetched_videos.size
+    }
+
+    get total_videos(){
+        let page_number: number = this.total_pages
+        return (page_number - 1) * 100 + (this.fetched_videos.get(`page${page_number}`) as Video[]).length
+    }
+
     toJSON() {
         return {
             id: this.id,

node-youtube-dl/YouTube/classes/Thumbnail.ts

@@ -2,8 +2,8 @@ type ThumbnailType = "default" | "hqdefault" | "mqdefault" | "sddefault" | "maxr
 
 export class Thumbnail {
     id?: string;
-    width!: number;
+    width?: number;
-    height!: number;
+    height?: number;
     url?: string;
 
     constructor(data: any) {

node-youtube-dl/YouTube/classes/Video.ts

@@ -12,8 +12,8 @@ interface VideoOptions {
     views: number;
     thumbnail?: {
         id: string | undefined;
-        width: number;
+        width: number | undefined ;
-        height: number;
+        height: number | undefined;
         url: string | undefined;
     };
     channel?: {
@@ -34,6 +34,7 @@ interface VideoOptions {
 
 export class Video {
     id?: string;
+    url? : string;
     title?: string;
     description?: string;
     durationFormatted: string;
@@ -53,6 +54,7 @@ export class Video {
         if(!data) throw new Error(`Can not initiate ${this.constructor.name} without data`)
         
         this.id = data.id || undefined;
+        this.url = `https://www.youtube.com/watch?v=${this.id}`
         this.title = data.title || undefined;
         this.description = data.description || undefined;
         this.durationFormatted = data.duration_raw || "0:00";
@@ -68,11 +70,6 @@ export class Video {
         this.tags = data.tags || [];
     }
 
-    get url(){
-        if(!this.id) return undefined
-        else return `https://www.youtube.com/watch?v=${this.id}`;
-    }
-
     get type(): "video" {
         return "video";
     }

node-youtube-dl/YouTube/index.ts

@@ -1 +1,3 @@
 export { search } from './search'
+
+export * from './utils'

node-youtube-dl/YouTube/search.ts

@@ -6,8 +6,29 @@ import { Channel } from "./classes/Channel";
 import { PlayList } from "./classes/Playlist";
 
 
-export async function search(url:string, options? : ParseSearchInterface): Promise<(Video | Channel | PlayList)[]> {
+enum SearchType {
+    Video = 'EgIQAQ%253D%253D',
+    PlayList = 'EgIQAw%253D%253D',
+    Channel = 'EgIQAg%253D%253D',
+}
+
+export async function search(search :string, options? : ParseSearchInterface): Promise<(Video | Channel | PlayList)[]> {
+    let url = 'https://www.youtube.com/results?search_query=' + search.replaceAll(' ', '+')
+    if(!url.match('&sp=')){
+        url += '&sp='
+        switch(options?.type){
+            case 'channel':
+                url += SearchType.Channel
+                break
+            case 'playlist':
+                url += SearchType.PlayList
+                break
+            case 'video':
+                url += SearchType.Video
+                break
+        }
+    }
     let body = await url_get(url)
-    let data = ParseSearchResult(body)
+    let data = ParseSearchResult(body, options)
     return data
 }

node-youtube-dl/YouTube/utils/extractor.ts

@@ -1,10 +1,24 @@
 import { url_get } from './request'
 import { format_decipher, js_tokens } from './cipher'
+import { Video } from '../classes/Video'
+import { RequestInit } from 'node-fetch'
+import { PlayList } from '../classes/Playlist'
+import fs from 'fs'
 
+const DEFAULT_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
+const youtube_url = /https:\/\/www.youtube.com\//g
+const video_pattern = /^((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w\-]+\?v=|embed\/|v\/)?)([\w\-]+)(\S+)?$/;
 
-export async function yt_initial_data(url : string){
+export interface PlaylistOptions {
+    limit?: number;
+    requestOptions?: RequestInit;
+}
+
+export async function video_basic_info(url : string){
+        if(!url.match(youtube_url) || !url.match(video_pattern)) throw new Error('This is not a YouTube URL')
         let body = await url_get(url)
         let player_response = JSON.parse(body.split("var ytInitialPlayerResponse = ")[1].split(";</script>")[0])
+        if(player_response.playabilityStatus.status === 'ERROR') throw new Error(`While getting info from url \n  ${player_response.playabilityStatus.reason}`)
         let response = JSON.parse(body.split("var ytInitialData = ")[1].split(";</script>")[0])
         let html5player =  'https://www.youtube.com' + body.split('"jsUrl":"')[1].split('"')[0]
         let format = []
@@ -12,14 +26,18 @@ export async function yt_initial_data(url : string){
         format.push(...player_response.streamingData.adaptiveFormats)
         let vid = player_response.videoDetails
         let microformat = player_response.microformat.playerMicroformatRenderer
-        let video_details = {
+        let video_details = new Video ({
             id : vid.videoId,
             url : 'https://www.youtube.com/watch?v=' + vid.videoId,
             title : vid.title,
             description : vid.shortDescription,
             duration : vid.lengthSeconds,
             uploadedDate : microformat.publishDate,
-            thumbnail : `https://i.ytimg.com/vi/${vid.videoId}/maxresdefault.jpg`,
+            thumbnail : {
+                width : vid.thumbnail.thumbnails[vid.thumbnail.thumbnails.length - 1].width,
+                height : vid.thumbnail.thumbnails[vid.thumbnail.thumbnails.length - 1].height,
+                url : `https://i.ytimg.com/vi/${vid.videoId}/maxresdefault.jpg`
+            },
             channel : {
                 name : vid.author,
                 id : vid.channelId,
@@ -30,19 +48,18 @@ export async function yt_initial_data(url : string){
             averageRating : vid.averageRating,
             live : vid.isLiveContent,
             private : vid.isPrivate
-        }
+        })
-        let final = {
+        return {
             player_response,
             response,
             html5player,
             format,
             video_details
         }
-        return final
 }
 
-export async function yt_deciphered_data(url : string) {
+export async function video_info(url : string) {
-    let data = await yt_initial_data(url)
+    let data = await video_basic_info(url)
     if(data.format[0].signatureCipher || data.format[0].cipher){
         data.format = await format_decipher(data.format, data.html5player)
         return data
@@ -51,3 +68,116 @@ export async function yt_deciphered_data(url : string) {
         return data
     }
 }
+
+export async function playlist_info(url : string , options? : PlaylistOptions) {
+    if (!options) options = { limit: 100, requestOptions: {} };
+    if(!options.limit) options.limit = 100
+    if (!url || typeof url !== "string") throw new Error(`Expected playlist url, received ${typeof url}!`);
+    if(url.search('(\\?|\\&)list\\=') === -1) throw new Error('This is not a PlayList URL')
+
+    let Playlist_id = url.split('list=')[1].split('&')[0]
+    let new_url = `https://www.youtube.com/playlist?list=${Playlist_id}`
+    
+    let body = await url_get(new_url)
+    let response = JSON.parse(body.split("var ytInitialData = ")[1].split(";</script>")[0])
+    if(response.alerts && response.alerts[0].alertRenderer.type === 'ERROR') throw new Error(`While parsing playlist url\n   ${response.alerts[0].alertRenderer.text.runs[0].text}`)
+
+    let rawJSON = `${body.split('{"playlistVideoListRenderer":{"contents":')[1].split('}],"playlistId"')[0]}}]`;
+    let parsed = JSON.parse(rawJSON);
+    let playlistDetails = JSON.parse(body.split('{"playlistSidebarRenderer":')[1].split("}};</script>")[0]).items;
+
+    let API_KEY = body.split('INNERTUBE_API_KEY":"')[1]?.split('"')[0] ?? body.split('innertubeApiKey":"')[1]?.split('"')[0] ?? DEFAULT_API_KEY;
+    let videos = getPlaylistVideos(parsed, options.limit);
+
+    let data = playlistDetails[0].playlistSidebarPrimaryInfoRenderer;
+    if (!data.title.runs || !data.title.runs.length) return undefined;
+
+    let author = playlistDetails[1]?.playlistSidebarSecondaryInfoRenderer.videoOwner;
+    let views = data.stats.length === 3 ? data.stats[1].simpleText.replace(/[^0-9]/g, "") : 0;
+    let lastUpdate = data.stats.find((x: any) => "runs" in x && x["runs"].find((y: any) => y.text.toLowerCase().includes("last update")))?.runs.pop()?.text ?? null;
+    let videosCount = data.stats[0].runs[0].text.replace(/[^0-9]/g, "") || 0;
+
+    let res = new PlayList({
+        continuation: {
+            api: API_KEY,
+            token: getContinuationToken(parsed),
+            clientVersion: body.split('"INNERTUBE_CONTEXT_CLIENT_VERSION":"')[1]?.split('"')[0] ?? body.split('"innertube_context_client_version":"')[1]?.split('"')[0] ?? "<some version>"
+        },
+        id: data.title.runs[0].navigationEndpoint.watchEndpoint.playlistId,
+        title: data.title.runs[0].text,
+        videoCount: parseInt(videosCount) || 0,
+        lastUpdate: lastUpdate,
+        views: parseInt(views) || 0,
+        videos: videos,
+        url: `https://www.youtube.com/playlist?list=${data.title.runs[0].navigationEndpoint.watchEndpoint.playlistId}`,
+        link: `https://www.youtube.com${data.title.runs[0].navigationEndpoint.commandMetadata.webCommandMetadata.url}`,
+        author: author
+            ? {
+                  name: author.videoOwnerRenderer.title.runs[0].text,
+                  id: author.videoOwnerRenderer.title.runs[0].navigationEndpoint.browseEndpoint.browseId,
+                  url: `https://www.youtube.com${author.videoOwnerRenderer.navigationEndpoint.commandMetadata.webCommandMetadata.url || author.videoOwnerRenderer.navigationEndpoint.browseEndpoint.canonicalBaseUrl}`,
+                  icon: author.videoOwnerRenderer.thumbnail.thumbnails.length ? author.videoOwnerRenderer.thumbnail.thumbnails[author.videoOwnerRenderer.thumbnail.thumbnails.length - 1].url : null
+              }
+            : {},
+        thumbnail: data.thumbnailRenderer.playlistVideoThumbnailRenderer?.thumbnail.thumbnails.length ? data.thumbnailRenderer.playlistVideoThumbnailRenderer.thumbnail.thumbnails[data.thumbnailRenderer.playlistVideoThumbnailRenderer.thumbnail.thumbnails.length - 1].url : null
+    });
+    return res;
+}
+
+export function getPlaylistVideos(data:any, limit : number = Infinity) : Video[] {
+    const videos = [];
+
+    for (let i = 0; i < data.length; i++) {
+        if (limit === videos.length) break;
+        const info = data[i].playlistVideoRenderer;
+        if (!info || !info.shortBylineText) continue;
+
+        videos.push(
+            new Video({
+                id: info.videoId,
+                index: parseInt(info.index?.simpleText) || 0,
+                duration: parseDuration(info.lengthText?.simpleText) || 0,
+                duration_raw: info.lengthText?.simpleText ?? "0:00",
+                thumbnail: {
+                    id: info.videoId,
+                    url: info.thumbnail.thumbnails[info.thumbnail.thumbnails.length - 1].url,
+                    height: info.thumbnail.thumbnails[info.thumbnail.thumbnails.length - 1].height,
+                    width: info.thumbnail.thumbnails[info.thumbnail.thumbnails.length - 1].width
+                },
+                title: info.title.runs[0].text,
+                channel: {
+                    id: info.shortBylineText.runs[0].navigationEndpoint.browseEndpoint.browseId || undefined,
+                    name: info.shortBylineText.runs[0].text || undefined,
+                    url: `https://www.youtube.com${info.shortBylineText.runs[0].navigationEndpoint.browseEndpoint.canonicalBaseUrl || info.shortBylineText.runs[0].navigationEndpoint.commandMetadata.webCommandMetadata.url}`,
+                    icon: undefined
+                }
+            })
+        );
+    }
+    return videos   
+}
+
+function parseDuration(duration: string): number {
+    duration ??= "0:00";
+    const args = duration.split(":");
+    let dur = 0;
+
+    switch (args.length) {
+        case 3:
+            dur = parseInt(args[0]) * 60 * 60 + parseInt(args[1]) * 60 + parseInt(args[2]);
+            break;
+        case 2:
+            dur = parseInt(args[0]) * 60 + parseInt(args[1]);
+            break;
+        default:
+            dur = parseInt(args[0]);
+    }
+
+    return dur;
+}
+
+
+export function getContinuationToken(data:any): string {
+    const continuationToken = data.find((x: any) => Object.keys(x)[0] === "continuationItemRenderer")?.continuationItemRenderer.continuationEndpoint?.continuationCommand?.token;
+    return continuationToken;
+}

node-youtube-dl/YouTube/utils/index.ts

@@ -1 +1 @@
-export { yt_initial_data, yt_deciphered_data } from './extractor'
+export { video_basic_info, video_info, playlist_info } from './extractor'

node-youtube-dl/YouTube/utils/parser.ts

@@ -2,6 +2,7 @@ import { Video } from "../classes/Video";
 import { PlayList } from "../classes/Playlist";
 import { Channel } from "../classes/Channel";
 import { RequestInit } from "node-fetch";
+import fs from 'fs'
 
 export interface ParseSearchInterface {
     type?: "video" | "playlist" | "channel" | "all";
@@ -31,7 +32,7 @@ export function ParseSearchResult(html :string, options? : ParseSearchInterface)
         details = JSON.parse(html.split('{"itemSectionRenderer":{"contents":')[html.split('{"itemSectionRenderer":{"contents":').length - 1].split(',"continuations":[{')[0]);
         fetched = true;
     } catch {
-        /* do nothing */
+        /* Do nothing*/
     }
 
     if (!fetched) {
@@ -76,40 +77,7 @@ export function ParseSearchResult(html :string, options? : ParseSearchInterface)
 return results as (Video | Channel | PlayList)[];
 }
 
-export function getPlaylistVideos(data:any, limit : number = Infinity) : Video[] {
+function parseDuration(duration: string): number {
-    const videos = [];
-
-    for (let i = 0; i < data.length; i++) {
-        if (limit === videos.length) break;
-        const info = data[i].playlistVideoRenderer;
-        if (!info || !info.shortBylineText) continue;
-
-        videos.push(
-            new Video({
-                id: info.videoId,
-                index: parseInt(info.index?.simpleText) || 0,
-                duration: parseDuration(info.lengthText?.simpleText) || 0,
-                duration_raw: info.lengthText?.simpleText ?? "0:00",
-                thumbnail: {
-                    id: info.videoId,
-                    url: info.thumbnail.thumbnails[info.thumbnail.thumbnails.length - 1].url,
-                    height: info.thumbnail.thumbnails[info.thumbnail.thumbnails.length - 1].height,
-                    width: info.thumbnail.thumbnails[info.thumbnail.thumbnails.length - 1].width
-                },
-                title: info.title.runs[0].text,
-                channel: {
-                    id: info.shortBylineText.runs[0].navigationEndpoint.browseEndpoint.browseId || undefined,
-                    name: info.shortBylineText.runs[0].text || undefined,
-                    url: `https://www.youtube.com${info.shortBylineText.runs[0].navigationEndpoint.browseEndpoint.canonicalBaseUrl || info.shortBylineText.runs[0].navigationEndpoint.commandMetadata.webCommandMetadata.url}`,
-                    icon: undefined
-                }
-            })
-        );
-    }
-    return videos   
-}
-
-export function parseDuration(duration: string): number {
     duration ??= "0:00";
     const args = duration.split(":");
     let dur = 0;
@@ -128,11 +96,6 @@ export function parseDuration(duration: string): number {
     return dur;
 }
 
-export function getContinuationToken(data:any): string {
-    const continuationToken = data.find((x: any) => Object.keys(x)[0] === "continuationItemRenderer")?.continuationItemRenderer.continuationEndpoint?.continuationCommand?.token;
-    return continuationToken;
-}
-
 export function parseChannel(data?: any): Channel | void {
     if (!data || !data.channelRenderer) return;
     const badge = data.channelRenderer.ownerBadges && data.channelRenderer.ownerBadges[0];
@@ -140,10 +103,14 @@ export function parseChannel(data?: any): Channel | void {
     let res = new Channel({
         id: data.channelRenderer.channelId,
         name: data.channelRenderer.title.simpleText,
-        icon: data.channelRenderer.thumbnail.thumbnails[data.channelRenderer.thumbnail.thumbnails.length - 1],
+        icon: {
+            url : data.channelRenderer.thumbnail.thumbnails[data.channelRenderer.thumbnail.thumbnails.length - 1].url.replace('//', 'https://'),
+            width : data.channelRenderer.thumbnail.thumbnails[data.channelRenderer.thumbnail.thumbnails.length - 1].width,
+            height: data.channelRenderer.thumbnail.thumbnails[data.channelRenderer.thumbnail.thumbnails.length - 1].height
+        },
         url: url,
         verified: Boolean(badge?.metadataBadgeRenderer?.style?.toLowerCase().includes("verified")),
-        subscribers: data.channelRenderer.subscriberCountText.simpleText
+        subscribers: (data.channelRenderer.subscriberCountText?.simpleText) ? data.channelRenderer.subscriberCountText.simpleText : '0 subscribers'
     });
 
     return res;

node-youtube-dl/index.ts

@@ -1,8 +1,8 @@
-import { search } from "./YouTube/";
+import { playlist_info } from "./YouTube";
 
 let main = async() => {
     let time_start = Date.now()
-    await search('https://www.youtube.com/results?search_query=Hello+Neghibour')
+    let playlist = await playlist_info('https://www.youtube.com/watch?v=bM7SZ5SBzyY&list=PLzkuLC6Yvumv_Rd5apfPRWEcjf9b1JRnq')
     let time_end = Date.now()
     console.log(`Time Taken : ${(time_end - time_start)/1000} seconds`)
 }