diff --git a/src/libpoketube/libpoketube-core.js b/src/libpoketube/libpoketube-core.js index 93b5587e..9c21b927 100644 --- a/src/libpoketube/libpoketube-core.js +++ b/src/libpoketube/libpoketube-core.js @@ -13,9 +13,9 @@ const getColors = require("get-image-colors"); const config = require("../../config.json"); class InnerTubePokeVidious { - constructor(config) { - this.config = config; - this.cache = {}; + constructor(cfg) { + this.config = cfg; + this.cache = {}; // 1h video-level cache this.language = "hl=en-US"; this.param = "2AMB"; this.param_legacy = "CgIIAdgDAQ%3D%3D"; @@ -23,31 +23,36 @@ class InnerTubePokeVidious { this.ANDROID_API_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w"; this.ANDROID_APP_VERSION = "20.20.41"; this.ANDROID_VERSION = "16"; - this.useragent = config.useragent || "PokeTube/2.0.0 (GNU/Linux; Android 14; Trisquel 11; poketube-vidious; like FreeTube)"; + this.useragent = cfg.useragent || "PokeTube/2.0.0 (GNU/Linux; Android 14; Trisquel 11; poketube-vidious; like FreeTube)"; this.INNERTUBE_CONTEXT_CLIENT_VERSION = "1"; this.region = "region=US"; this.sqp = "-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLBy_x4UUHLNDZtJtH0PXeQGoRFTgw"; - // Lazy-initialized undici pieces for connection reuse + pipelining + // undici primitives (lazy) this._fetch = null; this._agent = null; this._undiciInit = null; - // Small LRU-ish caches to avoid redundant work within the same process lifespan - this._channelCache = new Map(); // key: authorId -> {data, ts} - this._commentsCache = new Map(); // key: videoId -> {data, ts} + // Micro-caches (short TTLs) + this._commentsCache = new Map(); // videoId -> {data, ts} + this._channelCache = new Map(); // authorId -> {data, ts} + + // In-flight de-dup (so concurrent identical asks share one network trip) + this._inflight = new Map(); // key -> Promise + + // Per-host circuit breaker state + this._cb = new Map(); // host -> {fail: number, openUntil: number} } async _ensureUndici() { if (this._undiciInit) return this._undiciInit; this._undiciInit = (async () => { const { fetch, Agent, setGlobalDispatcher } = await import("undici"); - // Aggressive keep-alive + pipelining to cut handshake latency const agent = new Agent({ keepAliveTimeout: 60_000, keepAliveMaxTimeout: 60_000, - connections: 16, - pipelining: 1, // keep conservative; many APIs dislike >1 + connections: 24, // push a bit harder + pipelining: 1, maxRedirections: 2 }); setGlobalDispatcher(agent); @@ -58,24 +63,17 @@ class InnerTubePokeVidious { } getJson(str) { - try { - return JSON.parse(str); - } catch { - return null; - } + try { return JSON.parse(str); } catch { return null; } } checkUnexistingObject(obj) { return obj && "authorId" in obj; } - async wait(ms) { - return new Promise(r => setTimeout(r, ms)); - } + async wait(ms) { return new Promise(r => setTimeout(r, ms)); } - // Faster, low-tail decorrelated jitter backoff - // See AWS "Exponential Backoff And Jitter" — decorrelated variant - backoffDelay(attempt, base = 80, cap = 2000) { + // Decorrelated jitter backoff (fast caps) + backoffDelay(attempt, base = 60, cap = 1600) { const prev = attempt === 0 ? base : Math.min(cap, base * Math.pow(2, attempt - 1)); const next = Math.min(cap, base + Math.floor(Math.random() * (prev * 3))); return next; @@ -85,34 +83,52 @@ class InnerTubePokeVidious { if (!status) return true; if (status === 408 || status === 425 || status === 429) return true; if (status >= 500 && status <= 599) return true; - // hard no-retry statuses + // do not retry on common hard errors if (status === 400 || status === 401 || status === 403 || status === 404 || status === 409 || status === 410) return false; return false; } _mergeHeaders(a = {}, b = {}) { const o = { ...a }; - for (const k of Object.keys(b)) { - o[k] = b[k]; - } + for (const k of Object.keys(b)) o[k] = b[k]; return o; } - // Ultra-lean fetch with: - // - connection reuse (undici agent) - // - short per-attempt timeouts (attempt-scaled) - // - decorrelated jitter backoff - // - minimal retries by default, but smart on 429/5xx + _hostFromUrl(u) { + try { return new URL(u).host; } catch { return ""; } + } + + _cbIsOpen(host) { + const s = this._cb.get(host); + return s && s.openUntil && Date.now() < s.openUntil; + } + + _cbReport(host, ok) { + const s = this._cb.get(host) || { fail: 0, openUntil: 0 }; + if (ok) { + s.fail = 0; + s.openUntil = 0; + } else { + s.fail += 1; + if (s.fail >= 3) { + // open for a short period; we hedge to other bases meanwhile + s.openUntil = Date.now() + 15_000; + } + } + this._cb.set(host, s); + } + async fetchWithRetry(url, options = {}, cfg = {}) { await this._ensureUndici(); const fetch = this._fetch; - const maxRetries = Number.isInteger(cfg.retries) ? Math.max(0, cfg.retries) : 5; - const baseDelay = cfg.baseDelay ?? 80; - const maxDelay = cfg.maxDelay ?? 2000; - const hardCapMs = cfg.hardCapMs ?? 8000; // total time budget + const maxRetries = Number.isInteger(cfg.retries) ? Math.max(0, cfg.retries) : 6; + const baseDelay = cfg.baseDelay ?? 60; + const maxDelay = cfg.maxDelay ?? 1600; + const hardCapMs = cfg.hardCapMs ?? 7000; // aggressive total budget const extraRetryOn = cfg.retryOnStatuses || []; const started = Date.now(); + const host = this._hostFromUrl(url); const uah = { "User-Agent": this.useragent, @@ -120,13 +136,21 @@ class InnerTubePokeVidious { "Accept-Encoding": "gzip, deflate, br" }; + // If breaker open, we still *try* once quickly (in case it recovered), + // but with tiny per-attempt budget to fail fast. + const breakerPenalty = this._cbIsOpen(host) ? 300 : 0; + let lastErr = null; for (let attempt = 0; attempt <= maxRetries; attempt++) { const elapsed = Date.now() - started; if (elapsed >= hardCapMs) break; - const perAttemptBudget = Math.max(500, Math.min(2500, hardCapMs - elapsed)); + const perAttemptBudget = Math.max( + 350 - breakerPenalty, + Math.min(2000, hardCapMs - elapsed) + ); + const ac = new AbortController(); const tm = setTimeout(() => ac.abort(new Error("timeout")), perAttemptBudget); @@ -138,232 +162,40 @@ class InnerTubePokeVidious { }); clearTimeout(tm); - if (res.ok) return res; + if (res.ok) { + this._cbReport(host, true); + return res; + } - const should = this.shouldRetryStatus(res.status) || extraRetryOn.includes(res.status); - if (!should || attempt === maxRetries) return res; + const retryable = this.shouldRetryStatus(res.status) || extraRetryOn.includes(res.status); + if (!retryable || attempt === maxRetries) { + this._cbReport(host, false); + return res; + } + + // Honor Retry-After if present (429/503) + const ra = res.headers.get("retry-after"); + if (ra) { + let delay = 0; + if (/^\d+$/.test(ra)) delay = parseInt(ra, 10) * 1000; + else { + const when = Date.parse(ra); + if (!Number.isNaN(when)) delay = Math.max(0, when - Date.now()); + } + if (delay > 0) { + // cap to our budget but still respect server intent + const cap = Math.min(delay, 3000); + await this.wait(cap); + } + } } catch (e) { clearTimeout(tm); lastErr = e; - if (attempt === maxRetries) throw e; + if (attempt === maxRetries) { + this._cbReport(host, false); + throw e; + } } const backoff = this.backoffDelay(attempt, baseDelay, maxDelay); - const remain = hardCapMs - (Date.now() - started); - if (remain <= 0) break; - await this.wait(Math.min(backoff, Math.max(0, remain - 50))); - } - - if (lastErr) throw lastErr; - throw new Error("fetchWithRetry failed"); - } - - // Hedged requests with early secondary, winner-takes-all, loser aborted. - // Cuts tail latency when a backend is slow/spotty. - async hedgedGetJsonFromBases(bases, path, query) { - await this._ensureUndici(); - const fetch = this._fetch; - - const qs = query ? (query.startsWith("?") ? query : "?" + query) : ""; - const primary = `${bases[0]}${path}${qs}`; - const secondary = bases[1] ? `${bases[1]}${path}${qs}` : null; - - const headers = { - "User-Agent": this.useragent, - "Accept": "application/json, text/plain, */*", - "Accept-Encoding": "gzip, deflate, br" - }; - - const attemptOnce = async (url, signal) => { - const res = await this.fetchWithRetry( - url, - { headers, signal }, - { retries: 3, baseDelay: 80, maxDelay: 1200, hardCapMs: 4000 } - ); - const tx = await res.text(); - return this.getJson(tx); - }; - - if (!secondary) return attemptOnce(primary); - - const acPrimary = new AbortController(); - const acSecondary = new AbortController(); - - // Fire secondary quickly (200ms) to hedge - const secondaryKick = (async () => { - await this.wait(200); - if (!acPrimary.signal.aborted) { - return attemptOnce(secondary, acSecondary.signal); - } - throw new Error("secondary canceled"); - })(); - - const primaryP = attemptOnce(primary, acPrimary.signal); - - try { - const winner = await Promise.race([ - primaryP.then(v => ({ v, who: "p" })), - secondaryKick.then(v => ({ v, who: "s" })) - ]); - - // Abort the loser ASAP - if (winner.who === "p") acSecondary.abort(); - else acPrimary.abort(); - - if (winner.v) return winner.v; - - // Fallback: await the other if the winner returned null-ish - const other = winner.who === "p" ? secondaryKick : primaryP; - const v2 = await other.catch(() => null); - return v2 || null; - } catch { - // Final fallback: try primary once more, fast budget - try { - return await attemptOnce(primary, acPrimary.signal); - } catch { - try { - return await attemptOnce(secondary, acSecondary.signal); - } catch { - return null; - } - } - } finally { - acPrimary.abort(); - acSecondary.abort(); - } - } - - async getColorsSafe(url) { - for (let i = 0; i < 2; i++) { - try { - const c = await getColors(url); - if (Array.isArray(c) && c[0] && c[1]) return [c[0].hex(), c[1].hex()]; - } catch {} - await this.wait(this.backoffDelay(i, 80, 600)); - } - return ["#0ea5e9", "#111827"]; - } - - // Fast libcurl retry: fewer attempts, tighter timeouts, early exit on non-retryable - async curlGetWithRetry(url, httpHeader) { - let lastErr = null; - for (let i = 0; i <= 3; i++) { - try { - const res = await curly.get(url, { httpHeader, timeoutMs: 7000, acceptEncoding: "gzip, deflate, br" }); - if (res && res.statusCode && res.statusCode >= 200 && res.statusCode < 300 && res.data) return res; - if (res && res.statusCode && !this.shouldRetryStatus(res.statusCode)) return res; - } catch (e) { - lastErr = e; - } - await this.wait(this.backoffDelay(i, 80, 1500)); - } - if (lastErr) throw lastErr; - throw new Error("curlGetWithRetry failed"); - } - - async getYouTubeApiVideo(f, v, contentlang, contentregion) { - if (v == null) return "Gib ID"; - - // 1h video-level cache - if (this.cache[v] && Date.now() - this.cache[v].timestamp < 3600000) { - return this.cache[v].result; - } - - const headersArr = Object.entries({ - "User-Agent": this.useragent, - "Accept": "application/json, text/plain, */*", - "Accept-Encoding": "gzip, deflate, br" - }).map(([k, v]) => `${k}: ${v}`); - - const bases = [this.config.invapi, this.config.invapi_alt]; - const b64ts = Buffer.from(String(Date.now())).toString("base64"); - const q = `hl=${contentlang}®ion=${contentregion}&h=${b64ts}`; - - try { - const [comments, vid, videoData] = await Promise.all([ - (async () => { - const hit = this._commentsCache.get(v); - if (hit && Date.now() - hit.ts < 300_000) return hit.data; - const data = await this.hedgedGetJsonFromBases(bases, `/comments/${v}`, q); - this._commentsCache.set(v, { data, ts: Date.now() }); - return data; - })(), - this.hedgedGetJsonFromBases(bases, `/videos/${v}`, q), - (async () => { - const res = await this.curlGetWithRetry(`${this.config.tubeApi}video?v=${v}`, headersArr); - const str = Buffer.isBuffer(res.data) ? res.data.toString("utf8") : String(res.data || ""); - const jsonStr = toJson(str); - const video = this.getJson(jsonStr); - return { json: jsonStr, video }; - })() - ]); - - let p = {}; - if (f === "true" && vid && vid.authorId) { - const cKey = vid.authorId; - const cached = this._channelCache.get(cKey); - if (cached && Date.now() - cached.ts < 300_000) { - p = cached.data; - } else { - p = await this.hedgedGetJsonFromBases(bases, `/channels/${vid.authorId}`, `hl=${contentlang}®ion=${contentregion}`); - this._channelCache.set(cKey, { data: p || {}, ts: Date.now() }); - } - } - - if (!vid) { - this.initError("Video JSON missing", new Error("no vid")); - return null; - } - - if (this.checkUnexistingObject(vid)) { - let fe = { engagement: null }; - try { - fe = await getdislikes(v); - } catch {} - - const [c1, c2] = await this.getColorsSafe(`https://i.ytimg.com/vi/${v}/hqdefault.jpg?sqp=${this.sqp}`); - - this.cache[v] = { - result: { - json: videoData?.json?.video, - video: videoData?.video, - vid, - comments, - channel_uploads: p || {}, - engagement: fe.engagement, - wiki: "", - desc: "", - color: c1, - color2: c2 - }, - timestamp: Date.now() - }; - return this.cache[v].result; - } - } catch (error) { - this.initError("Error getting video", error); - } - } - - isvalidvideo(v) { - if (v != "assets" && v != "cdn-cgi" && v != "404") { - return /^([a-zA-Z0-9_-]{11})/.test(v); - } - return false; - } - - initError(args, error) { - console.error("[LIBPT CORE ERROR] " + args, error); - } -} - -const pokeTubeApiCore = new InnerTubePokeVidious({ - tubeApi: "https://inner-api.poketube.fun/api/", - invapi: "https://invid-api.poketube.fun/bHj665PpYhUdPWuKPfZuQGoX/api/v1", - invapi_alt: config.proxylocation === "EU" ? "https://invid-api.poketube.fun/api/v1" : "https://iv.ggtyler.dev/api/v1", - dislikes: "https://returnyoutubedislikeapi.com/votes?videoId=", - t_url: "https://t.poketube.fun/", - useragent: config.useragent -}); - -module.exports = pokeTubeApiCore; + const remain = hardCapMs - (Date.now()