Merge pull request #261 from play-dl/absidue-fixes

Various fixes and feature request implementations

play-dl/Request/index.ts

@@ -136,9 +136,8 @@ export function request_resolve_redirect(url: string): Promise<string> {
             resolve(url);
         } else if (statusCode < 400) {
             const resolved = await request_resolve_redirect(res.headers.location as string).catch((err) => err);
-
-            if (res instanceof Error) {
-                reject(res);
+            if (resolved instanceof Error) {
+                reject(resolved);
                 return;
             }
 
@@ -149,6 +148,38 @@ export function request_resolve_redirect(url: string): Promise<string> {
     });
 }
 
+export function request_content_length(url: string): Promise<number> {
+    return new Promise(async (resolve, reject) => {
+        let res = await https_getter(url, { method: 'HEAD' }).catch((err: Error) => err);
+        if (res instanceof Error) {
+            reject(res);
+            return;
+        }
+        const statusCode = Number(res.statusCode);
+        if (statusCode < 300) {
+            resolve(Number(res.headers['content-length']));
+        } else if (statusCode < 400) {
+            const newURL = await request_resolve_redirect(res.headers.location as string).catch((err) => err);
+            if (newURL instanceof Error) {
+                reject(newURL);
+                return;
+            }
+
+            const res2 = await request_content_length(newURL).catch((err) => err);
+            if (res2 instanceof Error) {
+                reject(res2);
+                return;
+            }
+
+            resolve(res2);
+        } else {
+            reject(
+                new Error(`Failed to get content length with error: ${res.statusCode}, ${res.statusMessage}, ${url}`)
+            );
+        }
+    });
+}
+
 /**
  * Main module that play-dl uses for making a https request
  * @param req_url URL to make https request to

play-dl/SoundCloud/classes.ts

@@ -104,6 +104,10 @@ export class SoundCloudTrack {
      * SoundCloud Track url
      */
     url: string;
+    /**
+     * User friendly SoundCloud track URL
+     */
+    permalink: string;
     /**
      * SoundCloud Track fetched status
      */
@@ -150,6 +154,7 @@ export class SoundCloudTrack {
         this.name = data.title;
         this.id = data.id;
         this.url = data.uri;
+        this.permalink = data.permalink_url;
         this.fetched = true;
         this.type = 'track';
         this.durationInSec = Math.round(Number(data.duration) / 1000);
@@ -187,6 +192,7 @@ export class SoundCloudTrack {
             name: this.name,
             id: this.id,
             url: this.url,
+            permalink: this.permalink,
             fetched: this.fetched,
             durationInMs: this.durationInMs,
             durationInSec: this.durationInSec,

play-dl/SoundCloud/constants.ts

@@ -13,6 +13,10 @@ export interface SoundTrackJSON {
      * SoundCloud Track url
      */
     url: string;
+    /**
+     * User friendly SoundCloud track URL
+     */
+    permalink: string;
     /**
      * SoundCloud Track fetched status
      */

play-dl/YouTube/stream.ts

@@ -1,4 +1,4 @@
-import { request_stream } from '../Request';
+import { request_content_length, request_stream } from '../Request';
 import { LiveStream, Stream } from './classes/LiveStream';
 import { SeekStream } from './classes/SeekStream';
 import { InfoData, StreamInfoData } from './utils/constants';
@@ -104,11 +104,19 @@ export async function stream_from_info(
             );
         } else if (options.seek) throw new Error('Can not seek with discordPlayerCompatibility set to true.');
     }
+
+    let contentLength;
+    if (final[0].contentLength) {
+        contentLength = Number(final[0].contentLength);
+    } else {
+        contentLength = await request_content_length(final[0].url);
+    }
+
     return new Stream(
         final[0].url,
         type,
         info.video_details.durationInSec,
-        Number(final[0].contentLength),
+        contentLength,
         info.video_details.url,
         options
     );

play-dl/YouTube/utils/extractor.ts

@@ -22,7 +22,7 @@ const DEFAULT_API_KEY = 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8';
 const video_pattern =
     /^((?:https?:)?\/\/)?(?:(?:www|m|music)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w\-]+\?v=|shorts\/|embed\/|v\/)?)([\w\-]+)(\S+)?$/;
 const playlist_pattern =
-    /^((?:https?:)?\/\/)?(?:(?:www|m|music)\.)?(youtube\.com)\/(?:(playlist|watch))(.*)?((\?|\&)list=)(PL|UU|LL|RD|OL)[a-zA-Z\d_-]{10,}(.*)?$/;
+    /^((?:https?:)?\/\/)?(?:(?:www|m|music)\.)?((?:youtube\.com|youtu.be))\/(?:(playlist|watch))?(.*)?((\?|\&)list=)(PL|UU|LL|RD|OL)[a-zA-Z\d_-]{10,}(.*)?$/;
 /**
  * Validate YouTube URL or ID.
  *
@@ -60,7 +60,7 @@ export function yt_validate(url: string): 'playlist' | 'video' | 'search' | fals
             else return 'search';
         }
     } else {
-        if (!url.match(playlist_pattern)) return false;
+        if (!url.match(playlist_pattern)) return yt_validate(url.replace(/(\?|\&)list=[a-zA-Z\d_-]+/, ''));
         else return 'playlist';
     }
 }
@@ -274,6 +274,13 @@ export async function video_basic_info(url: string, options: InfoOptions = {}):
     if (!upcoming) {
         format.push(...(player_response.streamingData.formats ?? []));
         format.push(...(player_response.streamingData.adaptiveFormats ?? []));
+
+        // get the formats for the android player for legacy videos
+        // fixes the stream being closed because not enough data
+        // arrived in time for ffmpeg to be able to extract audio data
+        if (parseAudioFormats(format).length === 0 && !options.htmldata) {
+            format = await getAndroidFormats(vid.videoId, cookieJar, body);
+        }
     }
     const LiveStreamData = {
         isLive: video_details.live,
@@ -373,6 +380,13 @@ export async function video_stream_info(url: string, options: InfoOptions = {}):
     if (!upcoming) {
         format.push(...(player_response.streamingData.formats ?? []));
         format.push(...(player_response.streamingData.adaptiveFormats ?? []));
+
+        // get the formats for the android player for legacy videos
+        // fixes the stream being closed because not enough data
+        // arrived in time for ffmpeg to be able to extract audio data
+        if (parseAudioFormats(format).length === 0 && !options.htmldata) {
+            format = await getAndroidFormats(player_response.videoDetails.videoId, cookieJar, body);
+        }
     }
 
     const LiveStreamData = {
@@ -639,6 +653,36 @@ async function acceptViewerDiscretion(
     return { streamingData };
 }
 
+async function getAndroidFormats(videoId: string, cookieJar: { [key: string]: string }, body: string): Promise<any[]> {
+    const apiKey =
+        body.split('INNERTUBE_API_KEY":"')[1]?.split('"')[0] ??
+        body.split('innertubeApiKey":"')[1]?.split('"')[0] ??
+        DEFAULT_API_KEY;
+
+    const response = await request(`https://www.youtube.com/youtubei/v1/player?key=${apiKey}`, {
+        method: 'POST',
+        body: JSON.stringify({
+            context: {
+                client: {
+                    clientName: 'ANDROID',
+                    clientVersion: '16.49',
+                    hl: 'en',
+                    timeZone: 'UTC',
+                    utcOffsetMinutes: 0
+                }
+            },
+            videoId: videoId,
+            playbackContext: { contentPlaybackContext: { html5Preference: 'HTML5_PREF_WANTS' } },
+            contentCheckOk: true,
+            racyCheckOk: true
+        }),
+        cookies: true,
+        cookieJar
+    });
+
+    return JSON.parse(response).streamingData.formats;
+}
+
 function getWatchPlaylist(response: any, body: any, url: string): YouTubePlayList {
     const playlist_details = response.contents.twoColumnWatchNextResults.playlist.playlist;
 

play-dl/YouTube/utils/parser.ts

@@ -45,11 +45,12 @@ export function ParseSearchResult(html: string, options?: ParseSearchInterface):
     const json_data = JSON.parse(data);
     const results = [];
     const details =
-        json_data.contents.twoColumnSearchResultsRenderer.primaryContents.sectionListRenderer.contents[0]
-            .itemSectionRenderer.contents;
+        json_data.contents.twoColumnSearchResultsRenderer.primaryContents.sectionListRenderer.contents.flatMap(
+            (s: any) => s.itemSectionRenderer?.contents
+        );
     for (const detail of details) {
         if (hasLimit && results.length === options.limit) break;
-        if (!detail.videoRenderer && !detail.channelRenderer && !detail.playlistRenderer) continue;
+        if (!detail || (!detail.videoRenderer && !detail.channelRenderer && !detail.playlistRenderer)) continue;
         switch (options.type) {
             case 'video': {
                 const parsed = parseVideo(detail);