From 40f60a8490b2a052bef00957421dbe8109e92da4 Mon Sep 17 00:00:00 2001 From: ashley Date: Mon, 18 Aug 2025 01:06:49 +0200 Subject: [PATCH] Update src/libpoketube/libpoketube-core.js --- src/libpoketube/libpoketube-core.js | 230 +++++++--------------------- 1 file changed, 52 insertions(+), 178 deletions(-) diff --git a/src/libpoketube/libpoketube-core.js b/src/libpoketube/libpoketube-core.js index 93b5587e..2b8ebfe8 100644 --- a/src/libpoketube/libpoketube-core.js +++ b/src/libpoketube/libpoketube-core.js @@ -10,7 +10,7 @@ const { toJson } = require("xml2json"); const { curly } = require("node-libcurl"); const getdislikes = require("../libpoketube/libpoketube-dislikes.js"); const getColors = require("get-image-colors"); -const config = require("../../config.json"); +const config = require("../../config.json") class InnerTubePokeVidious { constructor(config) { @@ -27,34 +27,6 @@ class InnerTubePokeVidious { this.INNERTUBE_CONTEXT_CLIENT_VERSION = "1"; this.region = "region=US"; this.sqp = "-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLBy_x4UUHLNDZtJtH0PXeQGoRFTgw"; - - // Lazy-initialized undici pieces for connection reuse + pipelining - this._fetch = null; - this._agent = null; - this._undiciInit = null; - - // Small LRU-ish caches to avoid redundant work within the same process lifespan - this._channelCache = new Map(); // key: authorId -> {data, ts} - this._commentsCache = new Map(); // key: videoId -> {data, ts} - } - - async _ensureUndici() { - if (this._undiciInit) return this._undiciInit; - this._undiciInit = (async () => { - const { fetch, Agent, setGlobalDispatcher } = await import("undici"); - // Aggressive keep-alive + pipelining to cut handshake latency - const agent = new Agent({ - keepAliveTimeout: 60_000, - keepAliveMaxTimeout: 60_000, - connections: 16, - pipelining: 1, // keep conservative; many APIs dislike >1 - maxRedirections: 2 - }); - setGlobalDispatcher(agent); - this._fetch = fetch; - this._agent = agent; - })(); - return this._undiciInit; } getJson(str) { @@ -73,256 +45,158 @@ class InnerTubePokeVidious { return new Promise(r => setTimeout(r, ms)); } - // Faster, low-tail decorrelated jitter backoff - // See AWS "Exponential Backoff And Jitter" — decorrelated variant - backoffDelay(attempt, base = 80, cap = 2000) { - const prev = attempt === 0 ? base : Math.min(cap, base * Math.pow(2, attempt - 1)); - const next = Math.min(cap, base + Math.floor(Math.random() * (prev * 3))); - return next; + backoffDelay(attempt, base = 160, cap = 12000) { + const exp = Math.min(cap, base * Math.pow(2, attempt)); + const jitter = Math.floor(Math.random() * (base + 1)); + return Math.min(cap, exp + jitter); } shouldRetryStatus(status) { if (!status) return true; if (status === 408 || status === 425 || status === 429) return true; if (status >= 500 && status <= 599) return true; - // hard no-retry statuses - if (status === 400 || status === 401 || status === 403 || status === 404 || status === 409 || status === 410) return false; return false; } - _mergeHeaders(a = {}, b = {}) { - const o = { ...a }; - for (const k of Object.keys(b)) { - o[k] = b[k]; - } - return o; - } - - // Ultra-lean fetch with: - // - connection reuse (undici agent) - // - short per-attempt timeouts (attempt-scaled) - // - decorrelated jitter backoff - // - minimal retries by default, but smart on 429/5xx async fetchWithRetry(url, options = {}, cfg = {}) { - await this._ensureUndici(); - const fetch = this._fetch; - - const maxRetries = Number.isInteger(cfg.retries) ? Math.max(0, cfg.retries) : 5; - const baseDelay = cfg.baseDelay ?? 80; - const maxDelay = cfg.maxDelay ?? 2000; - const hardCapMs = cfg.hardCapMs ?? 8000; // total time budget + const { fetch } = await import("undici"); + const maxRetries = Number.isInteger(cfg.retries) ? Math.max(0, cfg.retries) : 8; + const baseDelay = cfg.baseDelay ?? 160; + const maxDelay = cfg.maxDelay ?? 12000; + const perAttemptTimeout = cfg.timeout ?? 12000; const extraRetryOn = cfg.retryOnStatuses || []; - const started = Date.now(); - const uah = { "User-Agent": this.useragent, - "Accept": "application/json, text/plain, */*", - "Accept-Encoding": "gzip, deflate, br" }; - let lastErr = null; - for (let attempt = 0; attempt <= maxRetries; attempt++) { - const elapsed = Date.now() - started; - if (elapsed >= hardCapMs) break; - - const perAttemptBudget = Math.max(500, Math.min(2500, hardCapMs - elapsed)); const ac = new AbortController(); - const tm = setTimeout(() => ac.abort(new Error("timeout")), perAttemptBudget); - + const t = setTimeout(() => ac.abort(new Error("timeout")), perAttemptTimeout); try { const res = await fetch(url, { ...options, signal: ac.signal, - headers: this._mergeHeaders(options.headers, uah) + headers: { ...(options.headers || {}), ...uah } }); - clearTimeout(tm); - + clearTimeout(t); if (res.ok) return res; - const should = this.shouldRetryStatus(res.status) || extraRetryOn.includes(res.status); if (!should || attempt === maxRetries) return res; } catch (e) { - clearTimeout(tm); + clearTimeout(t); lastErr = e; if (attempt === maxRetries) throw e; } - - const backoff = this.backoffDelay(attempt, baseDelay, maxDelay); - const remain = hardCapMs - (Date.now() - started); - if (remain <= 0) break; - await this.wait(Math.min(backoff, Math.max(0, remain - 50))); + await this.wait(this.backoffDelay(attempt, baseDelay, maxDelay)); } - if (lastErr) throw lastErr; throw new Error("fetchWithRetry failed"); } - // Hedged requests with early secondary, winner-takes-all, loser aborted. - // Cuts tail latency when a backend is slow/spotty. async hedgedGetJsonFromBases(bases, path, query) { - await this._ensureUndici(); - const fetch = this._fetch; - const qs = query ? (query.startsWith("?") ? query : "?" + query) : ""; const primary = `${bases[0]}${path}${qs}`; const secondary = bases[1] ? `${bases[1]}${path}${qs}` : null; - - const headers = { - "User-Agent": this.useragent, - "Accept": "application/json, text/plain, */*", - "Accept-Encoding": "gzip, deflate, br" - }; - - const attemptOnce = async (url, signal) => { - const res = await this.fetchWithRetry( - url, - { headers, signal }, - { retries: 3, baseDelay: 80, maxDelay: 1200, hardCapMs: 4000 } - ); - const tx = await res.text(); + const attemptOnce = async (url) => { + const r = await this.fetchWithRetry(url, {}, { retries: 4, baseDelay: 120, maxDelay: 6000, timeout: 10000 }); + const tx = await r.text(); return this.getJson(tx); }; - if (!secondary) return attemptOnce(primary); - - const acPrimary = new AbortController(); - const acSecondary = new AbortController(); - - // Fire secondary quickly (200ms) to hedge - const secondaryKick = (async () => { - await this.wait(200); - if (!acPrimary.signal.aborted) { - return attemptOnce(secondary, acSecondary.signal); - } - throw new Error("secondary canceled"); + let winner; + let errorPrimary, errorSecondary; + const delayedSecondary = (async () => { + await this.wait(300); + return attemptOnce(secondary); })(); - - const primaryP = attemptOnce(primary, acPrimary.signal); - + const primaryP = attemptOnce(primary); try { - const winner = await Promise.race([ - primaryP.then(v => ({ v, who: "p" })), - secondaryKick.then(v => ({ v, who: "s" })) - ]); - - // Abort the loser ASAP - if (winner.who === "p") acSecondary.abort(); - else acPrimary.abort(); - - if (winner.v) return winner.v; - - // Fallback: await the other if the winner returned null-ish - const other = winner.who === "p" ? secondaryKick : primaryP; - const v2 = await other.catch(() => null); - return v2 || null; + winner = await Promise.any([primaryP, delayedSecondary]); } catch { - // Final fallback: try primary once more, fast budget try { - return await attemptOnce(primary, acPrimary.signal); - } catch { - try { - return await attemptOnce(secondary, acSecondary.signal); - } catch { - return null; - } + const a = await primaryP; + if (a) return a; + } catch (e) { + errorPrimary = e; } - } finally { - acPrimary.abort(); - acSecondary.abort(); + try { + const b = await delayedSecondary; + if (b) return b; + } catch (e) { + errorSecondary = e; + } + if (errorPrimary) throw errorPrimary; + if (errorSecondary) throw errorSecondary; + return null; } + return winner; } async getColorsSafe(url) { - for (let i = 0; i < 2; i++) { + for (let i = 0; i < 3; i++) { try { const c = await getColors(url); if (Array.isArray(c) && c[0] && c[1]) return [c[0].hex(), c[1].hex()]; } catch {} - await this.wait(this.backoffDelay(i, 80, 600)); + await this.wait(this.backoffDelay(i, 120, 4000)); } return ["#0ea5e9", "#111827"]; } - // Fast libcurl retry: fewer attempts, tighter timeouts, early exit on non-retryable async curlGetWithRetry(url, httpHeader) { let lastErr = null; - for (let i = 0; i <= 3; i++) { + for (let i = 0; i <= 4; i++) { try { - const res = await curly.get(url, { httpHeader, timeoutMs: 7000, acceptEncoding: "gzip, deflate, br" }); + const res = await curly.get(url, { httpHeader, timeoutMs: 12000 }); if (res && res.statusCode && res.statusCode >= 200 && res.statusCode < 300 && res.data) return res; if (res && res.statusCode && !this.shouldRetryStatus(res.statusCode)) return res; } catch (e) { lastErr = e; } - await this.wait(this.backoffDelay(i, 80, 1500)); + await this.wait(this.backoffDelay(i, 160, 8000)); } if (lastErr) throw lastErr; throw new Error("curlGetWithRetry failed"); } async getYouTubeApiVideo(f, v, contentlang, contentregion) { + const { fetch } = await import("undici"); if (v == null) return "Gib ID"; - - // 1h video-level cache if (this.cache[v] && Date.now() - this.cache[v].timestamp < 3600000) { return this.cache[v].result; } - - const headersArr = Object.entries({ + const headers = { "User-Agent": this.useragent, - "Accept": "application/json, text/plain, */*", - "Accept-Encoding": "gzip, deflate, br" - }).map(([k, v]) => `${k}: ${v}`); - + }; const bases = [this.config.invapi, this.config.invapi_alt]; const b64ts = Buffer.from(String(Date.now())).toString("base64"); const q = `hl=${contentlang}®ion=${contentregion}&h=${b64ts}`; - try { const [comments, vid, videoData] = await Promise.all([ - (async () => { - const hit = this._commentsCache.get(v); - if (hit && Date.now() - hit.ts < 300_000) return hit.data; - const data = await this.hedgedGetJsonFromBases(bases, `/comments/${v}`, q); - this._commentsCache.set(v, { data, ts: Date.now() }); - return data; - })(), + this.hedgedGetJsonFromBases(bases, `/comments/${v}`, q), this.hedgedGetJsonFromBases(bases, `/videos/${v}`, q), (async () => { - const res = await this.curlGetWithRetry(`${this.config.tubeApi}video?v=${v}`, headersArr); + const res = await this.curlGetWithRetry(`${this.config.tubeApi}video?v=${v}`, Object.entries(headers).map(([k, v]) => `${k}: ${v}`)); const str = Buffer.isBuffer(res.data) ? res.data.toString("utf8") : String(res.data || ""); const jsonStr = toJson(str); const video = this.getJson(jsonStr); return { json: jsonStr, video }; })() ]); - let p = {}; if (f === "true" && vid && vid.authorId) { - const cKey = vid.authorId; - const cached = this._channelCache.get(cKey); - if (cached && Date.now() - cached.ts < 300_000) { - p = cached.data; - } else { - p = await this.hedgedGetJsonFromBases(bases, `/channels/${vid.authorId}`, `hl=${contentlang}®ion=${contentregion}`); - this._channelCache.set(cKey, { data: p || {}, ts: Date.now() }); - } + p = await this.hedgedGetJsonFromBases(bases, `/channels/${vid.authorId}`, `hl=${contentlang}®ion=${contentregion}`); } - if (!vid) { this.initError("Video JSON missing", new Error("no vid")); return null; } - if (this.checkUnexistingObject(vid)) { let fe = { engagement: null }; try { fe = await getdislikes(v); } catch {} - const [c1, c2] = await this.getColorsSafe(`https://i.ytimg.com/vi/${v}/hqdefault.jpg?sqp=${this.sqp}`); - this.cache[v] = { result: { json: videoData?.json?.video, @@ -336,7 +210,7 @@ class InnerTubePokeVidious { color: c1, color2: c2 }, - timestamp: Date.now() + timestamp: Date.now(), }; return this.cache[v].result; } @@ -363,7 +237,7 @@ const pokeTubeApiCore = new InnerTubePokeVidious({ invapi_alt: config.proxylocation === "EU" ? "https://invid-api.poketube.fun/api/v1" : "https://iv.ggtyler.dev/api/v1", dislikes: "https://returnyoutubedislikeapi.com/votes?videoId=", t_url: "https://t.poketube.fun/", - useragent: config.useragent + useragent: config.useragent, }); module.exports = pokeTubeApiCore;