From ad1ad2b918d46499b09cc329feb3afc2d761f140 Mon Sep 17 00:00:00 2001 From: killer069 <65385476+killer069@users.noreply.github.com> Date: Fri, 20 Aug 2021 12:06:12 +0530 Subject: [PATCH] LiveStream Support --- package-lock.json | 4 +- package.json | 2 +- play-dl/YouTube/classes/LiveStream.ts | 139 +++++++++++++++++++++++++ play-dl/YouTube/index.ts | 4 +- play-dl/YouTube/stream.ts | 142 +++++++++++++++++--------- play-dl/YouTube/utils/extractor.ts | 33 +++++- play-dl/YouTube/utils/request.ts | 1 - play-dl/index.ts | 4 +- 8 files changed, 269 insertions(+), 60 deletions(-) create mode 100644 play-dl/YouTube/classes/LiveStream.ts diff --git a/package-lock.json b/package-lock.json index 697e707..78e1b4a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "play-dl", - "version": "0.1.6", + "version": "0.1.7", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "play-dl", - "version": "0.1.6", + "version": "0.1.7", "license": "MIT", "dependencies": { "got": "^11.8.2" diff --git a/package.json b/package.json index 341bf67..c213cd7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "play-dl", - "version": "0.1.6", + "version": "0.1.7", "description": "YouTube, SoundCloud, Spotify downloader", "main": "dist/index.js", "typings": "dist/index.d.ts", diff --git a/play-dl/YouTube/classes/LiveStream.ts b/play-dl/YouTube/classes/LiveStream.ts new file mode 100644 index 0000000..706641d --- /dev/null +++ b/play-dl/YouTube/classes/LiveStream.ts @@ -0,0 +1,139 @@ +import { PassThrough } from 'stream' +import got from 'got' + +export interface FormatInterface{ + url : string; + targetDurationSec : number; + maxDvrDurationSec : number +} + +export class LiveStreaming{ + smooth : boolean; + private __stream : PassThrough + private format : FormatInterface + private interval : number + private packet_count : number + private timer : NodeJS.Timer | null + private segments_urls : string[] + constructor(format : FormatInterface, smooth : boolean){ + this.smooth = smooth || false + this.format = format + this.__stream = new PassThrough({ highWaterMark : 10 * 1000 * 1000 }) + this.segments_urls = [] + this.packet_count = 0 + this.interval = 0 + this.timer = null + this.__stream.on('close', () => { + this.cleanup() + }) + if(this.smooth === true) this.__stream.pause() + this.start() + } + + async manifest_getter(){ + let response = await got(this.format.url) + this.segments_urls = response.body.split('\n').filter((x) => x.startsWith('https')) + } + + get stream(){ + return this.__stream + } + + private cleanup(){ + clearInterval(this.timer as NodeJS.Timer) + this.segments_urls = [] + this.packet_count = 0 + } + + async start(){ + if(this.__stream.destroyed) this.cleanup() + await this.manifest_getter() + if(this.packet_count === 0) this.packet_count = Number(this.segments_urls[0].split('index.m3u8/sq/')[1].split('/')[0]) + for await (let url of this.segments_urls){ + await (async () => { + return new Promise(async (resolve, reject) => { + if(Number(url.split('index.m3u8/sq/')[1].split('/')[0]) !== this.packet_count){ + resolve('') + return + } + let stream = this.got_stream(url) + stream.on('data', (chunk) => this.__stream.write(chunk)) + stream.on('end', () => { + this.packet_count++ + resolve('') + }) + }) + })() + } + this.interval = (this.segments_urls.length / 2) * 1000 + this.timer = setTimeout(async () => { + if(this.smooth === true){ + this.__stream.resume() + this.smooth = false + } + await this.start() + }, this.interval) + } + + private got_stream(url: string){ + return got.stream(url) + } +} + +export class LiveEnded{ + private __stream : PassThrough + private format : FormatInterface + private packet_count : number + private segments_urls : string[] + constructor(format : FormatInterface){ + this.format = format + this.__stream = new PassThrough({ highWaterMark : 10 * 1000 * 1000 }) + this.segments_urls = [] + this.packet_count = 0 + this.__stream.on('close', () => { + this.cleanup() + }) + this.start() + } + + async manifest_getter(){ + let response = await got(this.format.url) + this.segments_urls = response.body.split('\n').filter((x) => x.startsWith('https')) + } + + get stream(){ + return this.__stream + } + + private cleanup(){ + this.segments_urls = [] + this.packet_count = 0 + } + + async start(){ + if(this.__stream.destroyed) this.cleanup() + await this.manifest_getter() + if(this.packet_count === 0) this.packet_count = Number(this.segments_urls[0].split('index.m3u8/sq/')[1].split('/')[0]) + for await (let url of this.segments_urls){ + await (async () => { + return new Promise(async (resolve, reject) => { + if(Number(url.split('index.m3u8/sq/')[1].split('/')[0]) !== this.packet_count){ + resolve('') + return + } + let stream = this.got_stream(url) + stream.on('data', (chunk) => this.__stream.write(chunk)) + stream.on('end', () => { + this.packet_count++ + resolve('') + }) + }) + })() + } + } + + private got_stream(url: string){ + return got.stream(url) + } +} + diff --git a/play-dl/YouTube/index.ts b/play-dl/YouTube/index.ts index 9459d6a..8cfd2ce 100644 --- a/play-dl/YouTube/index.ts +++ b/play-dl/YouTube/index.ts @@ -1,5 +1,3 @@ -import { stream } from './stream' - export { search } from './search' -export { stream } +export { stream, stream_from_info, stream_type } from './stream' export * from './utils' \ No newline at end of file diff --git a/play-dl/YouTube/stream.ts b/play-dl/YouTube/stream.ts index a254ca6..4964d8b 100644 --- a/play-dl/YouTube/stream.ts +++ b/play-dl/YouTube/stream.ts @@ -2,10 +2,29 @@ import got from "got" import { video_info } from "." import { PassThrough } from 'stream' import https from 'https' +import { FormatInterface, LiveEnded, LiveStreaming } from "./classes/LiveStream" +enum StreamType{ + Arbitrary = 'arbitrary', + Raw = 'raw', + OggOpus = 'ogg/opus', + WebmOpus = 'webm/opus', + Opus = 'opus', +} interface StreamOptions { - filter : "bestaudio" | "bestvideo" | "live" + smooth : boolean +} + +interface InfoData{ + LiveStreamData : { + isLive : boolean + dashManifestUrl : string + hlsManifestUrl : string + } + html5player : string + format : any[] + video_details : any } function parseAudioFormats(formats : any[]){ @@ -21,55 +40,61 @@ function parseAudioFormats(formats : any[]){ return result } -function parseVideoFormats(formats : any[]){ - let result: any[] = [] - formats.forEach((format) => { - let type = format.mimeType as string - if(type.startsWith('audio')){ - format.codec = type.split('codecs="')[1].split('"')[0] - format.container = type.split('audio/')[1].split(';')[0] - result.push(format) - } - }) - return result -} - -export async function stream(url : string, options? : StreamOptions): Promise{ +export async function stream(url : string, options : StreamOptions = { smooth : false }): Promise{ let info = await video_info(url) let final: any[] = []; - - if(info.video_details.live === true && options) { - options.filter = "live" + + if(info.LiveStreamData.isLive === true && info.LiveStreamData.hlsManifestUrl !== null) { + return await live_stream(info as InfoData, options.smooth) } - if(options?.filter){ - switch(options.filter){ - case "bestaudio": - let audioFormat = parseAudioFormats(info.format) - if(audioFormat.length === 0) await stream(url, { filter : "bestvideo" }) - let opusFormats = filterFormat(audioFormat, "opus") - if(opusFormats.length === 0){ - final.push(audioFormat[audioFormat.length - 1]) - } - else{ - final.push(opusFormats[opusFormats.length - 1]) - } - break - case "bestvideo" : - let videoFormat = parseVideoFormats(info.format) - if(videoFormat.length === 0) throw new Error('Can\'t Find Video Formats ') - let qual_1080 = filterVideo(videoFormat, "1080p") - if(qual_1080.length === 0) { - let qual_720 = filterVideo(videoFormat, "720p") - if(qual_720.length === 0) final.push(videoFormat[0]) - else final.push(qual_720) - break - } - else final.push(qual_1080) - break - - } + let audioFormat = parseAudioFormats(info.format) + let opusFormats = filterFormat(audioFormat, "opus") + + if(opusFormats.length === 0){ + final.push(audioFormat[audioFormat.length - 1]) } + else{ + final.push(opusFormats[opusFormats.length - 1]) + } + + if(final.length === 0) final.push(info.format[info.format.length - 1]) + let piping_stream = got.stream(final[0].url, { + retry : 5, + headers: { + 'Connection': 'keep-alive', + 'Accept-Encoding': '', + 'Accept-Language': 'en-US,en;q=0.8', + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36' + }, + agent : { + https : new https.Agent({ keepAlive : true }) + }, + http2 : true + }) + let playing_stream = new PassThrough({ highWaterMark: 10 * 1000 * 1000 }) + + piping_stream.pipe(playing_stream) + return playing_stream +} + +export async function stream_from_info(info : InfoData, options : StreamOptions){ + let final: any[] = []; + + if(info.LiveStreamData.isLive === true) { + return await live_stream(info as InfoData, options.smooth) + } + + let audioFormat = parseAudioFormats(info.format) + let opusFormats = filterFormat(audioFormat, "opus") + + if(opusFormats.length === 0){ + final.push(audioFormat[audioFormat.length - 1]) + } + else{ + final.push(opusFormats[opusFormats.length - 1]) + } + if(final.length === 0) final.push(info.format[info.format.length - 1]) let piping_stream = got.stream(final[0].url, { retry : 5, @@ -98,10 +123,27 @@ function filterFormat(formats : any[], codec : string){ return result } -function filterVideo(formats : any[], quality : string) { - let result: any[] = [] - formats.forEach((format) => { - if(format.qualityLabel === quality) result.push(format) +export function stream_type(info:InfoData): StreamType{ + if(info.LiveStreamData.isLive === true && info.LiveStreamData.hlsManifestUrl !== null) return StreamType.Arbitrary + else return StreamType.WebmOpus +} + +async function live_stream(info : InfoData, smooth : boolean): Promise{ + let res_144 : FormatInterface = { + url : '', + targetDurationSec : 0, + maxDvrDurationSec : 0 + } + info.format.forEach((format) => { + if(format.qualityLabel === '144p') res_144 = format + else return }) - return result + let stream : LiveStreaming | LiveEnded + if(info.video_details.duration === '0') { + stream = new LiveStreaming((res_144.url.length !== 0) ? res_144 : info.format[info.format.length - 1], smooth) + } + else { + stream = new LiveEnded((res_144.url.length !== 0) ? res_144 : info.format[info.format.length - 1]) + } + return stream.stream } \ No newline at end of file diff --git a/play-dl/YouTube/utils/extractor.ts b/play-dl/YouTube/utils/extractor.ts index c5350cf..e616a28 100644 --- a/play-dl/YouTube/utils/extractor.ts +++ b/play-dl/YouTube/utils/extractor.ts @@ -12,8 +12,9 @@ export async function video_basic_info(url : string){ let video_id = url.split('watch?v=')[1].split('&')[0] let new_url = 'https://www.youtube.com/watch?v=' + video_id let body = await url_get(new_url) - let player_response = JSON.parse(body.split("var ytInitialPlayerResponse = ")[1].split(";")[0]) + let player_response = JSON.parse(body.split("var ytInitialPlayerResponse = ")[1].split("}};")[0] + '}}') if(player_response.playabilityStatus.status === 'ERROR') throw new Error(`While getting info from url \n ${player_response.playabilityStatus.reason}`) + if(player_response.playabilityStatus.status === 'LOGIN_REQUIRED') throw new Error(`While getting info from url \n ${player_response.playabilityStatus.messages[0]}`) let html5player = 'https://www.youtube.com' + body.split('"jsUrl":"')[1].split('"')[0] let format = [] let vid = player_response.videoDetails @@ -43,7 +44,13 @@ export async function video_basic_info(url : string){ } if(!video_details.live) format.push(player_response.streamingData.formats[0]) format.push(...player_response.streamingData.adaptiveFormats) + let LiveStreamData = { + isLive : video_details.live, + dashManifestUrl : (player_response.streamingData?.dashManifestUrl) ? player_response.streamingData?.dashManifestUrl : null, + hlsManifestUrl : (player_response.streamingData?.hlsManifestUrl) ? player_response.streamingData?.hlsManifestUrl : null + } return { + LiveStreamData, html5player, format, video_details @@ -56,11 +63,35 @@ export async function video_info(url : string) { data.format = await format_decipher(data.format, data.html5player) return data } + else if(data.LiveStreamData.isLive === true && data.LiveStreamData.hlsManifestUrl !== null){ + let m3u8 = await url_get(data.LiveStreamData.hlsManifestUrl) + data.format = await parseM3U8(m3u8, data.format) + return data + } else { return data } } +async function parseM3U8(m3u8_data : string, formats : any[]): Promise{ + let lines = m3u8_data.split('\n') + formats.forEach((format) => { + if(!format.qualityLabel) return + let reso = format.width + 'x' + format.height + let index = -1; + let line_count = 0 + lines.forEach((line) => { + index = line.search(reso) + if(index !== -1) { + format.url = lines[line_count+1] + } + line_count++ + index = -1 + }) + }) + return formats +} + export async function playlist_info(url : string) { if (!url || typeof url !== "string") throw new Error(`Expected playlist url, received ${typeof url}!`); if(url.search('(\\?|\\&)list\\=') === -1) throw new Error('This is not a PlayList URL') diff --git a/play-dl/YouTube/utils/request.ts b/play-dl/YouTube/utils/request.ts index 2041b0b..999c8f7 100644 --- a/play-dl/YouTube/utils/request.ts +++ b/play-dl/YouTube/utils/request.ts @@ -3,7 +3,6 @@ import got, { OptionsOfTextResponseBody } from 'got/dist/source' export async function url_get (url : string, options? : OptionsOfTextResponseBody) : Promise{ return new Promise(async(resolve, reject) => { let response = await got(url, options) - if(response.statusCode === 200) { resolve(response.body) } diff --git a/play-dl/index.ts b/play-dl/index.ts index 47a0e0d..7e28b2f 100644 --- a/play-dl/index.ts +++ b/play-dl/index.ts @@ -1,4 +1,4 @@ //This File is in testing stage, everything will change in this -import { playlist_info, video_basic_info, video_info, search, stream } from "./YouTube"; +import { playlist_info, video_basic_info, video_info, search, stream, stream_from_info, stream_type } from "./YouTube"; -export var youtube = { playlist_info, video_basic_info, video_info, search , stream} +export let youtube = { playlist_info, video_basic_info, video_info, search , stream, stream_from_info, stream_type}