Merge pull request #199 from play-dl/developer

1.5.5

README.md

@@ -23,7 +23,7 @@ npm install play-dl@latest
 ### Importing
 
 ```ts
-import * as play from 'play-dl' // ES-6 import or TS import
+import play from 'play-dl' // ES-6 import or TS import
 
 const play = require('play-dl') //JS importing
 ```

play-dl/YouTube/classes/Playlist.ts

@@ -94,7 +94,7 @@ export class YouTubePlayList {
         this.views = data.views || 0;
         this.link = data.link || undefined;
         this.channel = new YouTubeChannel(data.channel) || undefined;
-        this.thumbnail = (data.thumbnail) ? new YouTubeThumbnail(data.thumbnail) : undefined;
+        this.thumbnail = data.thumbnail ? new YouTubeThumbnail(data.thumbnail) : undefined;
         this.videos = data.videos || [];
         this.__count++;
         this.fetched_videos.set(`${this.__count}`, this.videos as YouTubeVideo[]);

play-dl/YouTube/classes/SeekStream.ts

@@ -0,0 +1,223 @@
+import { IncomingMessage } from 'http';
+import { request_stream } from '../../Request';
+import { parseAudioFormats, StreamOptions, StreamType } from '../stream';
+import { video_info } from '../utils';
+import { Timer } from './LiveStream';
+import { WebmSeeker, WebmSeekerState } from './WebmSeeker';
+
+/**
+ * YouTube Stream Class for seeking audio to a timeStamp.
+ */
+export class SeekStream {
+    /**
+     * WebmSeeker Stream through which data passes
+     */
+    stream: WebmSeeker;
+    /**
+     * Type of audio data that we recieved from normal youtube url.
+     */
+    type: StreamType;
+    /**
+     * Audio Endpoint Format Url to get data from.
+     */
+    private url: string;
+    /**
+     * Used to calculate no of bytes data that we have recieved
+     */
+    private bytes_count: number;
+    /**
+     * Calculate per second bytes by using contentLength (Total bytes) / Duration (in seconds)
+     */
+    private per_sec_bytes: number;
+    /**
+     * Total length of audio file in bytes
+     */
+    private content_length: number;
+    /**
+     * YouTube video url. [ Used only for retrying purposes only. ]
+     */
+    private video_url: string;
+    /**
+     * Timer for looping data every 265 seconds.
+     */
+    private timer: Timer;
+    /**
+     * Quality given by user. [ Used only for retrying purposes only. ]
+     */
+    private quality: number;
+    /**
+     * Incoming message that we recieve.
+     *
+     * Storing this is essential.
+     * This helps to destroy the TCP connection completely if you stopped player in between the stream
+     */
+    private request: IncomingMessage | null;
+    /**
+     * YouTube Stream Class constructor
+     * @param url Audio Endpoint url.
+     * @param type Type of Stream
+     * @param duration Duration of audio playback [ in seconds ]
+     * @param contentLength Total length of Audio file in bytes.
+     * @param video_url YouTube video url.
+     * @param options Options provided to stream function.
+     */
+    constructor(url: string, duration: number, contentLength: number, video_url: string, options: StreamOptions) {
+        this.stream = new WebmSeeker({
+            highWaterMark: 5 * 1000 * 1000,
+            readableObjectMode: true,
+            mode: options.seekMode
+        });
+        this.url = url;
+        this.quality = options.quality as number;
+        this.type = StreamType.Opus;
+        this.bytes_count = 0;
+        this.video_url = video_url;
+        this.per_sec_bytes = Math.ceil(contentLength / duration);
+        this.content_length = contentLength;
+        this.request = null;
+        this.timer = new Timer(() => {
+            this.timer.reuse();
+            this.loop();
+        }, 265);
+        this.stream.on('close', () => {
+            this.timer.destroy();
+            this.cleanup();
+        });
+        this.seek(options.seek!);
+    }
+    /**
+     * **INTERNAL Function**
+     *
+     * Uses stream functions to parse Webm Head and gets Offset byte to seek to.
+     * @param sec No of seconds to seek to
+     * @returns Nothing
+     */
+    private async seek(sec: number) {
+        await new Promise(async (res) => {
+            if (!this.stream.headerparsed) {
+                const stream = await request_stream(this.url, {
+                    headers: {
+                        range: `bytes=0-1000`
+                    }
+                }).catch((err: Error) => err);
+
+                if (stream instanceof Error) {
+                    this.stream.emit('error', stream);
+                    this.bytes_count = 0;
+                    this.per_sec_bytes = 0;
+                    this.cleanup();
+                    return;
+                }
+
+                this.request = stream;
+                stream.pipe(this.stream, { end: false });
+
+                stream.once('end', () => {
+                    this.stream.state = WebmSeekerState.READING_DATA;
+                    res('');
+                });
+            } else res('');
+        });
+
+        const bytes = this.stream.seek(sec);
+        if (bytes instanceof Error) {
+            this.stream.emit('error', bytes);
+            this.bytes_count = 0;
+            this.per_sec_bytes = 0;
+            this.cleanup();
+            return;
+        }
+
+        this.stream.seekfound = false;
+        this.bytes_count = bytes;
+        this.timer.reuse();
+        this.loop();
+    }
+    /**
+     * Retry if we get 404 or 403 Errors.
+     */
+    private async retry() {
+        const info = await video_info(this.video_url);
+        const audioFormat = parseAudioFormats(info.format);
+        this.url = audioFormat[this.quality].url;
+    }
+    /**
+     * This cleans every used variable in class.
+     *
+     * This is used to prevent re-use of this class and helping garbage collector to collect it.
+     */
+    private cleanup() {
+        this.request?.destroy();
+        this.request = null;
+        this.url = '';
+    }
+    /**
+     * Getting data from audio endpoint url and passing it to stream.
+     *
+     * If 404 or 403 occurs, it will retry again.
+     */
+    private async loop() {
+        if (this.stream.destroyed) {
+            this.timer.destroy();
+            this.cleanup();
+            return;
+        }
+        const end: number = this.bytes_count + this.per_sec_bytes * 300;
+        const stream = await request_stream(this.url, {
+            headers: {
+                range: `bytes=${this.bytes_count}-${end >= this.content_length ? '' : end}`
+            }
+        }).catch((err: Error) => err);
+        if (stream instanceof Error) {
+            this.stream.emit('error', stream);
+            this.bytes_count = 0;
+            this.per_sec_bytes = 0;
+            this.cleanup();
+            return;
+        }
+        if (Number(stream.statusCode) >= 400) {
+            this.cleanup();
+            await this.retry();
+            this.timer.reuse();
+            this.loop();
+            return;
+        }
+        this.request = stream;
+        stream.pipe(this.stream, { end: false });
+
+        stream.once('error', async () => {
+            this.cleanup();
+            await this.retry();
+            this.timer.reuse();
+            this.loop();
+        });
+
+        stream.on('data', (chunk: any) => {
+            this.bytes_count += chunk.length;
+        });
+
+        stream.on('end', () => {
+            if (end >= this.content_length) {
+                this.timer.destroy();
+                this.stream.write(null);
+                this.cleanup();
+            }
+        });
+    }
+    /**
+     * Pauses timer.
+     * Stops running of loop.
+     *
+     * Useful if you don't want to get excess data to be stored in stream.
+     */
+    pause() {
+        this.timer.pause();
+    }
+    /**
+     * Resumes timer.
+     * Starts running of loop.
+     */
+    resume() {
+        this.timer.resume();
+    }
+}

play-dl/YouTube/classes/Video.ts

@@ -50,14 +50,6 @@ interface VideoOptions {
      * YouTube Video's likes
      */
     likes: number;
-    /**
-     * YouTube Video's dislikes
-     */
-    dislikes: number;
-    /**
-     * YouTube Video's average Rating
-     */
-    averageRating: number;
     /**
      * YouTube Video live status
      */
@@ -123,14 +115,6 @@ export class YouTubeVideo {
      * YouTube Video's likes
      */
     likes: number;
-    /**
-     * YouTube Video's dislikes
-     */
-    dislikes: number;
-    /**
-     * YouTube Video's average Rating
-     */
-    averageRating: number;
     /**
      * YouTube Video live status
      */
@@ -166,8 +150,6 @@ export class YouTubeVideo {
         this.thumbnails = thumbnails || [];
         this.channel = new YouTubeChannel(data.channel) || {};
         this.likes = data.likes || 0;
-        this.averageRating = data.averageRating || 0;
-        this.dislikes = Math.floor((this.likes * (5 - this.averageRating)) / (this.averageRating - 1)) || 0;
         this.live = !!data.live;
         this.private = !!data.private;
         this.tags = data.tags || [];
@@ -197,8 +179,6 @@ export class YouTubeVideo {
             views: this.views,
             tags: this.tags,
             likes: this.likes,
-            dislikes: this.dislikes,
-            averageRating: this.averageRating,
             live: this.live,
             private: this.private
         };

play-dl/YouTube/classes/WebmSeeker.ts

@@ -0,0 +1,229 @@
+import { WebmElements, WebmHeader } from 'play-audio';
+import { Duplex, DuplexOptions } from 'stream';
+
+enum DataType {
+    master,
+    string,
+    uint,
+    binary,
+    float
+}
+
+export enum WebmSeekerState {
+    READING_HEAD = 'READING_HEAD',
+    READING_DATA = 'READING_DATA'
+}
+
+interface WebmSeekerOptions extends DuplexOptions {
+    mode?: 'precise' | 'granular';
+}
+
+export class WebmSeeker extends Duplex {
+    remaining?: Buffer;
+    state: WebmSeekerState;
+    mode: 'precise' | 'granular';
+    chunk?: Buffer;
+    cursor: number;
+    header: WebmHeader;
+    headfound: boolean;
+    headerparsed: boolean;
+    time_left: number;
+    seekfound: boolean;
+    private data_size: number;
+    private data_length: number;
+
+    constructor(options: WebmSeekerOptions) {
+        super(options);
+        this.state = WebmSeekerState.READING_HEAD;
+        this.cursor = 0;
+        this.header = new WebmHeader();
+        this.headfound = false;
+        this.time_left = 0;
+        this.headerparsed = false;
+        this.seekfound = false;
+        this.data_length = 0;
+        this.mode = options.mode || 'granular';
+        this.data_size = 0;
+    }
+
+    private get vint_length(): number {
+        let i = 0;
+        for (; i < 8; i++) {
+            if ((1 << (7 - i)) & this.chunk![this.cursor]) break;
+        }
+        return ++i;
+    }
+
+    private get vint_value(): boolean {
+        if (!this.chunk) return false;
+        const length = this.vint_length;
+        if (this.chunk.length < this.cursor + length) return false;
+        let value = this.chunk[this.cursor] & ((1 << (8 - length)) - 1);
+        for (let i = this.cursor + 1; i < this.cursor + length; i++) value = (value << 8) + this.chunk[i];
+        this.data_size = length;
+        this.data_length = value;
+        return true;
+    }
+
+    cleanup() {
+        this.cursor = 0;
+        this.chunk = undefined;
+        this.remaining = undefined;
+    }
+
+    _read() {}
+
+    seek(sec: number): Error | number {
+        let position = 0;
+        let time = Math.floor(sec / 10) * 10;
+        this.time_left = (sec - time) * 1000 || 0;
+        if (!this.header.segment.cues) return new Error('Failed to Parse Cues');
+
+        for (const data of this.header.segment.cues) {
+            if (Math.floor((data.time as number) / 1000) === time) {
+                position = data.position as number;
+                break;
+            } else continue;
+        }
+        if (position === 0) return Error('Failed to find Cluster Position');
+        else return position;
+    }
+
+    _write(chunk: Buffer, _: BufferEncoding, callback: (error?: Error | null) => void): void {
+        if (this.remaining) {
+            this.chunk = Buffer.concat([this.remaining, chunk]);
+            this.remaining = undefined;
+        } else this.chunk = chunk;
+
+        let err: Error | undefined;
+
+        if (this.state === WebmSeekerState.READING_HEAD) err = this.readHead();
+        else if (!this.seekfound) err = this.getClosetCluster();
+        else err = this.readTag();
+
+        if (err) callback(err);
+        else callback();
+    }
+
+    private readHead(): Error | undefined {
+        if (!this.chunk) return new Error('Chunk is missing');
+
+        while (this.chunk.length > this.cursor) {
+            const oldCursor = this.cursor;
+            const id = this.vint_length;
+            if (this.chunk.length < this.cursor + id) break;
+
+            const ebmlID = this.parseEbmlID(this.chunk.slice(this.cursor, this.cursor + id).toString('hex'));
+            this.cursor += id;
+            const vint = this.vint_value;
+
+            if (!vint) {
+                this.cursor = oldCursor;
+                break;
+            }
+            if (!ebmlID) {
+                this.cursor += this.data_size + this.data_length;
+                continue;
+            }
+
+            if (!this.headfound) {
+                if (ebmlID.name === 'ebml') this.headfound = true;
+                else return new Error('Failed to find EBML ID at start of stream.');
+            }
+            const data = this.chunk.slice(
+                this.cursor + this.data_size,
+                this.cursor + this.data_size + this.data_length
+            );
+            const parse = this.header.parse(ebmlID, data);
+            if (parse instanceof Error) return parse;
+
+            if (ebmlID.type === DataType.master) {
+                this.cursor += this.data_size;
+                continue;
+            }
+
+            if (this.chunk.length < this.cursor + this.data_size + this.data_length) {
+                this.cursor = oldCursor;
+                break;
+            } else this.cursor += this.data_size + this.data_length;
+        }
+        this.remaining = this.chunk.slice(this.cursor);
+        this.cursor = 0;
+    }
+
+    private readTag(): Error | undefined {
+        if (!this.chunk) return new Error('Chunk is missing');
+
+        while (this.chunk.length > this.cursor) {
+            const oldCursor = this.cursor;
+            const id = this.vint_length;
+            if (this.chunk.length < this.cursor + id) break;
+
+            const ebmlID = this.parseEbmlID(this.chunk.slice(this.cursor, this.cursor + id).toString('hex'));
+            this.cursor += id;
+            const vint = this.vint_value;
+
+            if (!vint) {
+                this.cursor = oldCursor;
+                break;
+            }
+            if (!ebmlID) {
+                this.cursor += this.data_size + this.data_length;
+                continue;
+            }
+
+            const data = this.chunk.slice(
+                this.cursor + this.data_size,
+                this.cursor + this.data_size + this.data_length
+            );
+            const parse = this.header.parse(ebmlID, data);
+            if (parse instanceof Error) return parse;
+
+            if (ebmlID.type === DataType.master) {
+                this.cursor += this.data_size;
+                continue;
+            }
+
+            if (this.chunk.length < this.cursor + this.data_size + this.data_length) {
+                this.cursor = oldCursor;
+                break;
+            } else this.cursor += this.data_size + this.data_length;
+
+            if (ebmlID.name === 'simpleBlock') {
+                if (this.time_left !== 0 && this.mode === 'precise') {
+                    if (data.readUInt16BE(1) === this.time_left) this.time_left = 0;
+                    else continue;
+                }
+                const track = this.header.segment.tracks![this.header.audioTrack];
+                if (!track || track.trackType !== 2) return new Error('No audio Track in this webm file.');
+                if ((data[0] & 0xf) === track.trackNumber) this.push(data.slice(4));
+            }
+        }
+        this.remaining = this.chunk.slice(this.cursor);
+        this.cursor = 0;
+    }
+
+    private getClosetCluster(): Error | undefined {
+        if (!this.chunk) return new Error('Chunk is missing');
+        const count = this.chunk.indexOf('1f43b675', 0, 'hex');
+        if (count === -1) throw new Error('Failed to find nearest Cluster.');
+        else this.chunk = this.chunk.slice(count);
+        this.seekfound = true;
+        return this.readTag();
+    }
+
+    private parseEbmlID(ebmlID: string) {
+        if (Object.keys(WebmElements).includes(ebmlID)) return WebmElements[ebmlID];
+        else return false;
+    }
+
+    _destroy(error: Error | null, callback: (error: Error | null) => void): void {
+        this.cleanup();
+        callback(error);
+    }
+
+    _final(callback: (error?: Error | null) => void): void {
+        this.cleanup();
+        callback();
+    }
+}

play-dl/YouTube/stream.ts

@@ -1,4 +1,5 @@
 import { LiveStream, Stream } from './classes/LiveStream';
+import { SeekStream } from './classes/SeekStream';
 import { InfoData, StreamInfoData } from './utils/constants';
 import { video_stream_info } from './utils/extractor';
 
@@ -11,6 +12,8 @@ export enum StreamType {
 }
 
 export interface StreamOptions {
+    seekMode?: 'precise' | 'granular';
+    seek?: number;
     quality?: number;
     htmldata?: boolean;
 }
@@ -35,7 +38,7 @@ export function parseAudioFormats(formats: any[]) {
 /**
  * Type for YouTube Stream
  */
-export type YouTubeStream = Stream | LiveStream;
+export type YouTubeStream = Stream | LiveStream | SeekStream;
 /**
  * Stream command for YouTube
  * @param url YouTube URL
@@ -77,6 +80,19 @@ export async function stream_from_info(
     else final.push(info.format[info.format.length - 1]);
     let type: StreamType =
         final[0].codec === 'opus' && final[0].container === 'webm' ? StreamType.WebmOpus : StreamType.Arbitrary;
+    if (options.seek) {
+        if (type === StreamType.WebmOpus) {
+            if (options.seek >= info.video_details.durationInSec || options.seek <= 0)
+                throw new Error(`Seeking beyond limit. [ 1 - ${info.video_details.durationInSec - 1}]`);
+            return new SeekStream(
+                final[0].url,
+                info.video_details.durationInSec,
+                Number(final[0].contentLength),
+                info.video_details.url,
+                options
+            );
+        } else throw new Error('Seek is only supported in Webm Opus Files.');
+    } else
         return new Stream(
             final[0].url,
             type,

play-dl/YouTube/utils/extractor.ts

@@ -13,12 +13,12 @@ interface PlaylistOptions {
 }
 
 const video_id_pattern = /^[a-zA-Z\d_-]{11,12}$/;
-const playlist_id_pattern = /^(PL|UU|LL|RD|OL)[a-zA-Z\d_-]{16,41}$/;
+const playlist_id_pattern = /^(PL|UU|LL|RD|OL)[a-zA-Z\d_-]{10,}$/;
 const DEFAULT_API_KEY = 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8';
 const video_pattern =
     /^((?:https?:)?\/\/)?(?:(?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w\-]+\?v=|shorts\/|embed\/|v\/)?)([\w\-]+)(\S+)?$/;
 const playlist_pattern =
-    /^((?:https?:)?\/\/)?(?:(?:www|m)\.)?(youtube\.com)\/(?:(playlist|watch))(.*)?((\?|\&)list=)(PL|UU|LL|RD|OL)[a-zA-Z\d_-]{16,41}(.*)?$/;
+    /^((?:https?:)?\/\/)?(?:(?:www|m)\.)?(youtube\.com)\/(?:(playlist|watch))(.*)?((\?|\&)list=)(PL|UU|LL|RD|OL)[a-zA-Z\d_-]{10,}(.*)?$/;
 /**
  * Validate YouTube URL or ID.
  *
@@ -172,7 +172,6 @@ export async function video_basic_info(url: string, options: InfoOptions = {}):
         },
         views: vid.viewCount,
         tags: vid.keywords,
-        averageRating: vid.averageRating,
         likes: parseInt(
             ratingButtons
                 .find((button: any) => button.toggleButtonRenderer.defaultIcon.iconType === 'LIKE')
@@ -328,12 +327,12 @@ export async function decipher_info<T extends InfoData | StreamInfoData>(data: T
  */
 export async function playlist_info(url: string, options: PlaylistOptions = {}): Promise<YouTubePlayList> {
     if (!url || typeof url !== 'string') throw new Error(`Expected playlist url, received ${typeof url}!`);
-    if (!url.startsWith('https')) url = `https://www.youtube.com/playlist?list=${url}`
+    if (!url.startsWith('https')) url = `https://www.youtube.com/playlist?list=${url}`;
     if (url.indexOf('list=') === -1) throw new Error('This is not a Playlist URL');
 
     if (yt_validate(url) === 'playlist') {
-        const id = extractID(url)
-        url = `https://www.youtube.com/playlist?list=${id}`
+        const id = extractID(url);
+        url = `https://www.youtube.com/playlist?list=${id}`;
     }
 
     const body = await request(url, {
@@ -355,9 +354,8 @@ export async function playlist_info(url: string, options: PlaylistOptions = {}):
         else throw new Error('While parsing playlist url\nUnknown Playlist Error');
     }
     if (url.indexOf('watch?v=') !== -1) {
-        return getWatchPlaylist(response, body)
-    }
-    else return getNormalPlaylist(response, body) 
+        return getWatchPlaylist(response, body);
+    } else return getNormalPlaylist(response, body);
 }
 /**
  * Function to parse Playlist from YouTube search
@@ -404,19 +402,18 @@ export function getContinuationToken(data: any): string {
         .continuationEndpoint?.continuationCommand?.token;
 }
 
-
 function getWatchPlaylist(response: any, body: any): YouTubePlayList {
-    const playlist_details = response.contents.twoColumnWatchNextResults.playlist.playlist
+    const playlist_details = response.contents.twoColumnWatchNextResults.playlist.playlist;
 
-    const videos = getWatchPlaylistVideos(playlist_details.contents)
+    const videos = getWatchPlaylistVideos(playlist_details.contents);
     const API_KEY =
         body.split('INNERTUBE_API_KEY":"')[1]?.split('"')[0] ??
         body.split('innertubeApiKey":"')[1]?.split('"')[0] ??
         DEFAULT_API_KEY;
 
-    const videoCount = playlist_details.totalVideos
-    const channel = playlist_details.shortBylineText?.runs?.[0]
-    const badge = playlist_details.badges?.[0]?.metadataBadgeRenderer?.style.toLowerCase()
+    const videoCount = playlist_details.totalVideos;
+    const channel = playlist_details.shortBylineText?.runs?.[0];
+    const badge = playlist_details.badges?.[0]?.metadataBadgeRenderer?.style.toLowerCase();
 
     return new YouTubePlayList({
         continuation: {
@@ -442,12 +439,13 @@ function getWatchPlaylist(response : any, body : any) : YouTubePlayList{
             verified: Boolean(badge?.includes('verified')),
             artist: Boolean(badge?.includes('artist'))
         }
-    })
+    });
 }
 
 function getNormalPlaylist(response: any, body: any): YouTubePlayList {
-
-    const json_data = response.contents.twoColumnBrowseResultsRenderer.tabs[0].tabRenderer.content.sectionListRenderer.contents[0].itemSectionRenderer.contents[0].playlistVideoListRenderer.contents;
+    const json_data =
+        response.contents.twoColumnBrowseResultsRenderer.tabs[0].tabRenderer.content.sectionListRenderer.contents[0]
+            .itemSectionRenderer.contents[0].playlistVideoListRenderer.contents;
     const playlist_details = response.sidebar.playlistSidebarRenderer.items;
 
     const API_KEY =
@@ -509,13 +507,13 @@ function getNormalPlaylist(response : any, body : any): YouTubePlayList{
 }
 
 function getWatchPlaylistVideos(data: any, limit = Infinity): YouTubeVideo[] {
-    const videos: YouTubeVideo[] = []
+    const videos: YouTubeVideo[] = [];
 
     for (let i = 0; i < data.length; i++) {
         if (limit === videos.length) break;
         const info = data[i].playlistPanelVideoRenderer;
         if (!info || !info.shortBylineText) continue;
-        const channel_info = info.shortBylineText.runs[0]
+        const channel_info = info.shortBylineText.runs[0];
 
         videos.push(
             new YouTubeVideo({
@@ -537,21 +535,21 @@ function getWatchPlaylistVideos(data : any, limit = Infinity): YouTubeVideo[] {
         );
     }
 
-    return videos
+    return videos;
 }
 
 function parseDuration(text: string): number {
-    if(!text) return 0
-    const split = text.split(':')
+    if (!text) return 0;
+    const split = text.split(':');
 
     switch (split.length) {
         case 2:
-            return (parseInt(split[0]) * 60) + (parseInt(split[1]))
+            return parseInt(split[0]) * 60 + parseInt(split[1]);
 
         case 3:
-            return (parseInt(split[0]) * 60 * 60) + (parseInt(split[1]) * 60) + (parseInt(split[2]))
+            return parseInt(split[0]) * 60 * 60 + parseInt(split[1]) * 60 + parseInt(split[2]);
 
         default:
-            return 0
+            return 0;
     }
 }