From 327bcce3607a5150c6569d98a0e86c23ce7c443d Mon Sep 17 00:00:00 2001 From: killer069 <65385476+killer069@users.noreply.github.com> Date: Tue, 7 Dec 2021 10:43:23 +0530 Subject: [PATCH] Added User agents support --- package-lock.json | 4 +- play-dl/Request/classes.ts | 89 -------------- play-dl/Request/index.ts | 163 +++++++------------------- play-dl/Request/useragent.ts | 27 +++++ play-dl/YouTube/classes/LiveStream.ts | 9 +- play-dl/YouTube/search.ts | 5 +- play-dl/YouTube/stream.ts | 4 +- play-dl/YouTube/utils/extractor.ts | 18 +-- play-dl/index.ts | 29 ----- play-dl/token.ts | 5 + 10 files changed, 86 insertions(+), 267 deletions(-) delete mode 100644 play-dl/Request/classes.ts create mode 100644 play-dl/Request/useragent.ts diff --git a/package-lock.json b/package-lock.json index 54564ff..84d3fff 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "play-dl", - "version": "1.4.2", + "version": "1.4.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "play-dl", - "version": "1.4.2", + "version": "1.4.4", "license": "GPL-3.0", "devDependencies": { "@types/node": "^16.9.4", diff --git a/play-dl/Request/classes.ts b/play-dl/Request/classes.ts deleted file mode 100644 index b61f307..0000000 --- a/play-dl/Request/classes.ts +++ /dev/null @@ -1,89 +0,0 @@ -import tls, { TLSSocket } from 'node:tls'; -import { URL } from 'node:url'; - -interface ProxyOptions extends tls.ConnectionOptions { - method: 'GET'; - headers?: Object; -} - -export class Proxy { - parsed_url: URL; - statusCode: number; - rawHeaders: string; - headers: Object; - body: string; - socket: TLSSocket; - sentHeaders: string; - private options: ProxyOptions; - constructor(parsed_url: URL, options: ProxyOptions) { - this.parsed_url = parsed_url; - this.sentHeaders = ''; - this.statusCode = 0; - this.rawHeaders = ''; - this.body = ''; - this.headers = {}; - this.options = options; - this.socket = tls.connect( - { - host: this.parsed_url.hostname, - port: Number(this.parsed_url.port) || 443, - socket: options.socket, - rejectUnauthorized: false - }, - () => this.onConnect() - ); - if (options.headers) { - for (const [key, value] of Object.entries(options.headers)) { - this.sentHeaders += `${key}: ${value}\r\n`; - } - } - } - - private onConnect() { - this.socket.write( - `${this.options.method} ${this.parsed_url.pathname}${this.parsed_url.search} HTTP/1.1\r\n` + - `Host: ${this.parsed_url.hostname}\r\n` + - this.sentHeaders + - `Connection: close\r\n` + - `\r\n` - ); - } - - private parseHeaders() { - const head_arr = this.rawHeaders.split('\r\n'); - this.statusCode = Number(head_arr.shift()?.split(' ')[1]) ?? -1; - for (const head of head_arr) { - let [key, value] = head.split(': '); - if (!value) break; - key = key.trim().toLowerCase(); - value = value.trim(); - if (Object.keys(this.headers).includes(key)) { - let val = (this.headers as any)[key]; - if (typeof val === 'string') val = [val]; - Object.assign(this.headers, { [key]: [...val, value] }); - } else Object.assign(this.headers, { [key]: value }); - } - } - - fetch(): Promise { - return new Promise((resolve, reject) => { - this.socket.setEncoding('utf-8'); - this.socket.once('error', (err) => reject(err)); - const parts: string[] = []; - this.socket.on('data', (chunk: string) => { - if (this.rawHeaders.length === 0) { - this.rawHeaders = chunk; - this.parseHeaders(); - } else { - const arr = chunk.split('\r\n'); - if (arr.length > 1 && arr[0].length < 5) arr.shift(); - parts.push(...arr); - } - }); - this.socket.on('end', () => { - this.body = parts.join(''); - resolve(this); - }); - }); - } -} diff --git a/play-dl/Request/index.ts b/play-dl/Request/index.ts index 607977e..3ba0f43 100644 --- a/play-dl/Request/index.ts +++ b/play-dl/Request/index.ts @@ -1,15 +1,13 @@ -import http, { ClientRequest, IncomingMessage } from 'node:http'; +import { IncomingMessage } from 'node:http'; import https, { RequestOptions } from 'node:https'; import { URL } from 'node:url'; +import zlib, { BrotliDecompress, Deflate, Gunzip } from 'node:zlib'; import { cookieHeaders, getCookies } from '../YouTube/utils/cookie'; -import { Proxy } from './classes'; - -export type ProxyOptions = ProxyOpts | string; +import { getRandomUserAgent } from './useragent'; interface RequestOpts extends RequestOptions { body?: string; method?: 'GET' | 'POST' | 'HEAD'; - proxies?: ProxyOptions[]; cookies?: boolean; } @@ -48,56 +46,46 @@ export function request_stream(req_url: string, options: RequestOpts = { method: */ export function request(req_url: string, options: RequestOpts = { method: 'GET' }): Promise { return new Promise(async (resolve, reject) => { - if (!options?.proxies || options.proxies.length === 0) { - let cookies_added = false; - if (options.cookies) { - let cook = getCookies(); - if (typeof cook === 'string' && options.headers) { - Object.assign(options.headers, { cookie: cook }); - cookies_added = true; - } + let cookies_added = false; + if (options.cookies) { + let cook = getCookies(); + if (typeof cook === 'string' && options.headers) { + Object.assign(options.headers, { cookie: cook }); + cookies_added = true; } - let res = await https_getter(req_url, options).catch((err: Error) => err); - if (res instanceof Error) { - reject(res); - return; - } - if (res.headers && res.headers['set-cookie'] && cookies_added) { - cookieHeaders(res.headers['set-cookie']); - } - if (Number(res.statusCode) >= 300 && Number(res.statusCode) < 400) { - res = await https_getter(res.headers.location as string, options); - } else if (Number(res.statusCode) > 400) { - reject(new Error(`Got ${res.statusCode} from the request`)); - } - const data: string[] = []; - res.setEncoding('utf-8'); - res.on('data', (c) => data.push(c)); - res.on('end', () => resolve(data.join(''))); - } else { - let cookies_added = false; - if (options.cookies) { - let cook = getCookies(); - if (typeof cook === 'string' && options.headers) { - Object.assign(options.headers, { cookie: cook }); - cookies_added = true; - } - } - let res = await proxy_getter(req_url, options.proxies, options.headers).catch((e: Error) => e); - if (res instanceof Error) { - reject(res); - return; - } - if (res.headers && (res.headers as any)['set-cookie'] && cookies_added) { - cookieHeaders((res.headers as any)['set-cookie']); - } - if (res.statusCode >= 300 && res.statusCode < 400) { - res = await proxy_getter((res.headers as any)['location'], options.proxies, options.headers); - } else if (res.statusCode > 400) { - reject(new Error(`GOT ${res.statusCode} from proxy request`)); - } - resolve(res.body); } + if (options.headers) { + options.headers = { + ...options.headers, + 'accept-encoding': 'gzip, deflate, br', + 'user-agent': getRandomUserAgent() + }; + } + let res = await https_getter(req_url, options).catch((err: Error) => err); + if (res instanceof Error) { + reject(res); + return; + } + if (res.headers && res.headers['set-cookie'] && cookies_added) { + cookieHeaders(res.headers['set-cookie']); + } + if (Number(res.statusCode) >= 300 && Number(res.statusCode) < 400) { + res = await https_getter(res.headers.location as string, options).catch((err) => err); + if (res instanceof Error) throw res; + } else if (Number(res.statusCode) > 400) { + reject(new Error(`Got ${res.statusCode} from the request`)); + } + const data: string[] = []; + let decoder: BrotliDecompress | Gunzip | Deflate; + const encoding = res.headers['content-encoding']; + if (encoding === 'gzip') decoder = zlib.createGunzip(); + else if (encoding === 'br') decoder = zlib.createBrotliDecompress(); + else decoder = zlib.createDeflate(); + + res.pipe(decoder); + decoder.setEncoding('utf-8'); + decoder.on('data', (c) => data.push(c)); + decoder.on('end', () => resolve(data.join(''))); }); } @@ -126,75 +114,6 @@ export function request_resolve_redirect(url: string): Promise { }); } -/** - * Chooses one random number between max and min number. - * @param min Minimum number - * @param max Maximum number - * @returns Random Number - */ -function randomIntFromInterval(min: number, max: number): number { - let x = Math.floor(Math.random() * (max - min + 1) + min); - if (x === 0) return 0; - else return x - 1; -} -/** - * Main module that play-dl uses for proxy. - * @param req_url URL to make https request to - * @param req_proxy Proxies array - * @returns Object with statusCode, head and body - */ -function proxy_getter(req_url: string, req_proxy: ProxyOptions[], headers?: Object): Promise { - return new Promise((resolve, reject) => { - const proxy: string | ProxyOpts = req_proxy[randomIntFromInterval(0, req_proxy.length)]; - const parsed_url = new URL(req_url); - let opts: ProxyOpts; - if (typeof proxy === 'string') { - const parsed = new URL(proxy); - opts = { - host: parsed.hostname, - port: Number(parsed.port), - authentication: { - username: parsed.username, - password: parsed.password - } - }; - } else - opts = { - host: proxy.host, - port: Number(proxy.port) - }; - let req: ClientRequest; - if (!opts.authentication) { - req = http.request({ - host: opts.host, - port: opts.port, - method: 'CONNECT', - path: `${parsed_url.host}:443` - }); - } else { - req = http.request({ - host: opts.host, - port: opts.port, - method: 'CONNECT', - path: `${parsed_url.host}:443`, - headers: { - 'Proxy-Authorization': `Basic ${Buffer.from( - `${opts.authentication?.username}:${opts.authentication?.password}` - ).toString('base64')}` - } - }); - } - req.on('connect', async function (res, socket) { - const conn_proxy = new Proxy(parsed_url, { method: 'GET', socket: socket, headers: headers }); - await conn_proxy.fetch(); - socket.end(); - resolve(conn_proxy); - }); - req.on('error', (e: Error) => reject(e)); - req.end(); - }); -} - /** * Main module that play-dl uses for making a https request * @param req_url URL to make https request to diff --git a/play-dl/Request/useragent.ts b/play-dl/Request/useragent.ts new file mode 100644 index 0000000..05ee668 --- /dev/null +++ b/play-dl/Request/useragent.ts @@ -0,0 +1,27 @@ +const useragents: string[] = [ + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36 Edg/96.0.1054.43', + 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0', + 'Mozilla/5.0 (Windows NT 6.3) AppleWebKit/537.36.0 (KHTML, like Gecko) Chrome/61.0.0.0 Safari/537.36.0', + 'Mozilla/5.0 (Windows; U; Windows NT 6.1) AppleWebKit/531.35.5 (KHTML, like Gecko) Version/4.0.3 Safari/531.35.5', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246', + 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1' +]; + +export function setUserAgent(array: string[]): void { + useragents.push(...array); +} + +function getRandomInt(min: number, max: number): number { + min = Math.ceil(min); + max = Math.floor(max); + return Math.floor(Math.random() * (max - min + 1)) + min; +} + +export function getRandomUserAgent() { + const random = getRandomInt(0, useragents.length - 1); + return useragents[random]; +} diff --git a/play-dl/YouTube/classes/LiveStream.ts b/play-dl/YouTube/classes/LiveStream.ts index c47925a..918ef14 100644 --- a/play-dl/YouTube/classes/LiveStream.ts +++ b/play-dl/YouTube/classes/LiveStream.ts @@ -1,7 +1,7 @@ import { Readable } from 'node:stream'; import { IncomingMessage } from 'node:http'; import { parseAudioFormats, StreamOptions, StreamType } from '../stream'; -import { ProxyOptions as Proxy, request, request_stream } from '../../Request'; +import { request, request_stream } from '../../Request'; import { video_info } from '..'; export interface FormatInterface { @@ -230,10 +230,6 @@ export class Stream { * Quality given by user. [ Used only for retrying purposes only. ] */ private quality: number; - /** - * Proxy config given by user. [ Used only for retrying purposes only. ] - */ - private proxy: Proxy[] | undefined; /** * Incoming message that we recieve. * @@ -261,7 +257,6 @@ export class Stream { this.stream = new Readable({ highWaterMark: 5 * 1000 * 1000, read() {} }); this.url = url; this.quality = options.quality as number; - this.proxy = options.proxy || undefined; this.type = type; this.bytes_count = 0; this.video_url = video_url; @@ -282,7 +277,7 @@ export class Stream { * Retry if we get 404 or 403 Errors. */ private async retry() { - const info = await video_info(this.video_url, { proxy: this.proxy }); + const info = await video_info(this.video_url); const audioFormat = parseAudioFormats(info.format); this.url = audioFormat[this.quality].url; } diff --git a/play-dl/YouTube/search.ts b/play-dl/YouTube/search.ts index 852056b..208ee2a 100644 --- a/play-dl/YouTube/search.ts +++ b/play-dl/YouTube/search.ts @@ -40,9 +40,8 @@ export async function yt_search(search: string, options: ParseSearchInterface = } } const body = await request(url, { - headers: { - 'accept-language': 'en-US,en;q=0.9', - 'user-agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36', + headers: { + 'accept-language': 'en-US,en;q=0.9' } }); if (body.indexOf('Our systems have detected unusual traffic from your computer network.') !== -1) diff --git a/play-dl/YouTube/stream.ts b/play-dl/YouTube/stream.ts index 201012e..62a0040 100644 --- a/play-dl/YouTube/stream.ts +++ b/play-dl/YouTube/stream.ts @@ -1,6 +1,5 @@ import { video_info } from '.'; import { LiveStream, Stream } from './classes/LiveStream'; -import { ProxyOptions as Proxy } from './../Request'; import { InfoData } from './utils/constants'; export enum StreamType { @@ -13,7 +12,6 @@ export enum StreamType { export interface StreamOptions { quality?: number; - proxy?: Proxy[]; htmldata?: boolean; } @@ -45,7 +43,7 @@ export type YouTubeStream = Stream | LiveStream; * @returns Stream class with type and stream for playing. */ export async function stream(url: string, options: StreamOptions = {}): Promise { - const info = await video_info(url, { proxy: options.proxy, htmldata: options.htmldata }); + const info = await video_info(url, { htmldata: options.htmldata }); return await stream_from_info(info, options); } /** diff --git a/play-dl/YouTube/utils/extractor.ts b/play-dl/YouTube/utils/extractor.ts index 20b0421..1ad83ec 100644 --- a/play-dl/YouTube/utils/extractor.ts +++ b/play-dl/YouTube/utils/extractor.ts @@ -1,17 +1,15 @@ -import { ProxyOptions as Proxy, request } from './../../Request/index'; +import { request } from './../../Request/index'; import { format_decipher } from './cipher'; import { YouTubeVideo } from '../classes/Video'; import { YouTubePlayList } from '../classes/Playlist'; import { InfoData } from './constants'; interface InfoOptions { - proxy?: Proxy[]; htmldata?: boolean; } interface PlaylistOptions { incomplete?: boolean; - proxy?: Proxy[]; } const video_id_pattern = /^[a-zA-Z\d_-]{11,12}$/; @@ -123,11 +121,9 @@ export async function video_basic_info(url: string, options: InfoOptions = {}): const video_id: string = extractID(url); const new_url = `https://www.youtube.com/watch?v=${video_id}&has_verified=1`; body = await request(new_url, { - proxies: options.proxy ?? [], - headers: { - 'accept-language': 'en-US,en-IN;q=0.9,en;q=0.8,hi;q=0.7', - 'user-agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36', - }, + headers: { + 'accept-language': 'en-US,en-IN;q=0.9,en;q=0.8,hi;q=0.7' + }, cookies: true }); } @@ -325,10 +321,8 @@ export async function playlist_info(url: string, options: PlaylistOptions = {}): const new_url = `https://www.youtube.com/playlist?list=${Playlist_id}`; const body = await request(new_url, { - proxies: options.proxy ?? undefined, - headers: { - 'accept-language': 'en-US,en-IN;q=0.9,en;q=0.8,hi;q=0.7', - 'user-agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36', + headers: { + 'accept-language': 'en-US,en-IN;q=0.9,en;q=0.8,hi;q=0.7' } }); if (body.indexOf('Our systems have detected unusual traffic from your computer network.') !== -1) diff --git a/play-dl/index.ts b/play-dl/index.ts index 253cb44..6bab669 100644 --- a/play-dl/index.ts +++ b/play-dl/index.ts @@ -123,35 +123,6 @@ export async function stream(url: string, options: StreamOptions = {}): Promise< else return await yt_stream(url, options); } -/** - * Searches through a particular source and gives respective info. - * - * Example - * ```ts - * const searched = await play.search('Rick Roll', { source : { youtube : "video" } }) // YouTube Video Search - * - * const searched = await play.search('Rick Roll', { limit : 1 }) // YouTube Video Search but returns only 1 video. - * - * const searched = await play.search('Rick Roll', { source : { spotify : "track" } }) // Spotify Track Search - * - * const searched = await play.search('Rick Roll', { source : { soundcloud : "tracks" } }) // SoundCloud Track Search - * - * const searched = await play.search('Rick Roll', { source : { deezer : "track" } }) // Deezer Track Search - * ``` - * @param query string to search. - * @param options - * - * - `number` limit : No of searches you want to have. - * - `boolean` fuzzy : Whether the search should be fuzzy or only return exact matches. Defaults to `true`. [ for `Deezer` Only ] - * - `Object` source : Contains type of source and type of result you want to have - * ```ts - * - youtube : 'video' | 'playlist' | 'channel'; - - spotify : 'album' | 'playlist' | 'track'; - - soundcloud : 'tracks' | 'playlists' | 'albums'; - - deezer : 'track' | 'playlist' | 'album'; - ``` - * @returns Array of {@link YouTube} or {@link Spotify} or {@link SoundCloud} or {@link Deezer} type - */ export async function search( query: string, options: { source: { deezer: 'album' } } & SearchOptions diff --git a/play-dl/token.ts b/play-dl/token.ts index 9722765..b302329 100644 --- a/play-dl/token.ts +++ b/play-dl/token.ts @@ -1,3 +1,4 @@ +import { setUserAgent } from './Request/useragent'; import { setSoundCloudToken } from './SoundCloud'; import { setSpotifyToken } from './Spotify'; import { setCookieToken } from './YouTube/utils/cookie'; @@ -15,6 +16,7 @@ interface tokenOptions { youtube?: { cookie: string; }; + useragent?: string[]; } /** * Sets @@ -25,6 +27,8 @@ interface tokenOptions { * * iii> Spotify :- client ID, client secret, refresh token, market. * + * iv> Useragents :- array of string. + * * locally in memory. * @param options {@link tokenOptions} */ @@ -32,4 +36,5 @@ export function setToken(options: tokenOptions) { if (options.spotify) setSpotifyToken(options.spotify); if (options.soundcloud) setSoundCloudToken(options.soundcloud); if (options.youtube) setCookieToken(options.youtube); + if (options.useragent) setUserAgent(options.useragent); }