diff --git a/src/libpoketube/libpoketube-core.js b/src/libpoketube/libpoketube-core.js index 9c21b927..93b5587e 100644 --- a/src/libpoketube/libpoketube-core.js +++ b/src/libpoketube/libpoketube-core.js @@ -13,9 +13,9 @@ const getColors = require("get-image-colors"); const config = require("../../config.json"); class InnerTubePokeVidious { - constructor(cfg) { - this.config = cfg; - this.cache = {}; // 1h video-level cache + constructor(config) { + this.config = config; + this.cache = {}; this.language = "hl=en-US"; this.param = "2AMB"; this.param_legacy = "CgIIAdgDAQ%3D%3D"; @@ -23,36 +23,31 @@ class InnerTubePokeVidious { this.ANDROID_API_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w"; this.ANDROID_APP_VERSION = "20.20.41"; this.ANDROID_VERSION = "16"; - this.useragent = cfg.useragent || "PokeTube/2.0.0 (GNU/Linux; Android 14; Trisquel 11; poketube-vidious; like FreeTube)"; + this.useragent = config.useragent || "PokeTube/2.0.0 (GNU/Linux; Android 14; Trisquel 11; poketube-vidious; like FreeTube)"; this.INNERTUBE_CONTEXT_CLIENT_VERSION = "1"; this.region = "region=US"; this.sqp = "-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLBy_x4UUHLNDZtJtH0PXeQGoRFTgw"; - // undici primitives (lazy) + // Lazy-initialized undici pieces for connection reuse + pipelining this._fetch = null; this._agent = null; this._undiciInit = null; - // Micro-caches (short TTLs) - this._commentsCache = new Map(); // videoId -> {data, ts} - this._channelCache = new Map(); // authorId -> {data, ts} - - // In-flight de-dup (so concurrent identical asks share one network trip) - this._inflight = new Map(); // key -> Promise - - // Per-host circuit breaker state - this._cb = new Map(); // host -> {fail: number, openUntil: number} + // Small LRU-ish caches to avoid redundant work within the same process lifespan + this._channelCache = new Map(); // key: authorId -> {data, ts} + this._commentsCache = new Map(); // key: videoId -> {data, ts} } async _ensureUndici() { if (this._undiciInit) return this._undiciInit; this._undiciInit = (async () => { const { fetch, Agent, setGlobalDispatcher } = await import("undici"); + // Aggressive keep-alive + pipelining to cut handshake latency const agent = new Agent({ keepAliveTimeout: 60_000, keepAliveMaxTimeout: 60_000, - connections: 24, // push a bit harder - pipelining: 1, + connections: 16, + pipelining: 1, // keep conservative; many APIs dislike >1 maxRedirections: 2 }); setGlobalDispatcher(agent); @@ -63,17 +58,24 @@ class InnerTubePokeVidious { } getJson(str) { - try { return JSON.parse(str); } catch { return null; } + try { + return JSON.parse(str); + } catch { + return null; + } } checkUnexistingObject(obj) { return obj && "authorId" in obj; } - async wait(ms) { return new Promise(r => setTimeout(r, ms)); } + async wait(ms) { + return new Promise(r => setTimeout(r, ms)); + } - // Decorrelated jitter backoff (fast caps) - backoffDelay(attempt, base = 60, cap = 1600) { + // Faster, low-tail decorrelated jitter backoff + // See AWS "Exponential Backoff And Jitter" — decorrelated variant + backoffDelay(attempt, base = 80, cap = 2000) { const prev = attempt === 0 ? base : Math.min(cap, base * Math.pow(2, attempt - 1)); const next = Math.min(cap, base + Math.floor(Math.random() * (prev * 3))); return next; @@ -83,52 +85,34 @@ class InnerTubePokeVidious { if (!status) return true; if (status === 408 || status === 425 || status === 429) return true; if (status >= 500 && status <= 599) return true; - // do not retry on common hard errors + // hard no-retry statuses if (status === 400 || status === 401 || status === 403 || status === 404 || status === 409 || status === 410) return false; return false; } _mergeHeaders(a = {}, b = {}) { const o = { ...a }; - for (const k of Object.keys(b)) o[k] = b[k]; + for (const k of Object.keys(b)) { + o[k] = b[k]; + } return o; } - _hostFromUrl(u) { - try { return new URL(u).host; } catch { return ""; } - } - - _cbIsOpen(host) { - const s = this._cb.get(host); - return s && s.openUntil && Date.now() < s.openUntil; - } - - _cbReport(host, ok) { - const s = this._cb.get(host) || { fail: 0, openUntil: 0 }; - if (ok) { - s.fail = 0; - s.openUntil = 0; - } else { - s.fail += 1; - if (s.fail >= 3) { - // open for a short period; we hedge to other bases meanwhile - s.openUntil = Date.now() + 15_000; - } - } - this._cb.set(host, s); - } - + // Ultra-lean fetch with: + // - connection reuse (undici agent) + // - short per-attempt timeouts (attempt-scaled) + // - decorrelated jitter backoff + // - minimal retries by default, but smart on 429/5xx async fetchWithRetry(url, options = {}, cfg = {}) { await this._ensureUndici(); const fetch = this._fetch; - const maxRetries = Number.isInteger(cfg.retries) ? Math.max(0, cfg.retries) : 6; - const baseDelay = cfg.baseDelay ?? 60; - const maxDelay = cfg.maxDelay ?? 1600; - const hardCapMs = cfg.hardCapMs ?? 7000; // aggressive total budget + const maxRetries = Number.isInteger(cfg.retries) ? Math.max(0, cfg.retries) : 5; + const baseDelay = cfg.baseDelay ?? 80; + const maxDelay = cfg.maxDelay ?? 2000; + const hardCapMs = cfg.hardCapMs ?? 8000; // total time budget const extraRetryOn = cfg.retryOnStatuses || []; const started = Date.now(); - const host = this._hostFromUrl(url); const uah = { "User-Agent": this.useragent, @@ -136,21 +120,13 @@ class InnerTubePokeVidious { "Accept-Encoding": "gzip, deflate, br" }; - // If breaker open, we still *try* once quickly (in case it recovered), - // but with tiny per-attempt budget to fail fast. - const breakerPenalty = this._cbIsOpen(host) ? 300 : 0; - let lastErr = null; for (let attempt = 0; attempt <= maxRetries; attempt++) { const elapsed = Date.now() - started; if (elapsed >= hardCapMs) break; - const perAttemptBudget = Math.max( - 350 - breakerPenalty, - Math.min(2000, hardCapMs - elapsed) - ); - + const perAttemptBudget = Math.max(500, Math.min(2500, hardCapMs - elapsed)); const ac = new AbortController(); const tm = setTimeout(() => ac.abort(new Error("timeout")), perAttemptBudget); @@ -162,40 +138,232 @@ class InnerTubePokeVidious { }); clearTimeout(tm); - if (res.ok) { - this._cbReport(host, true); - return res; - } + if (res.ok) return res; - const retryable = this.shouldRetryStatus(res.status) || extraRetryOn.includes(res.status); - if (!retryable || attempt === maxRetries) { - this._cbReport(host, false); - return res; - } - - // Honor Retry-After if present (429/503) - const ra = res.headers.get("retry-after"); - if (ra) { - let delay = 0; - if (/^\d+$/.test(ra)) delay = parseInt(ra, 10) * 1000; - else { - const when = Date.parse(ra); - if (!Number.isNaN(when)) delay = Math.max(0, when - Date.now()); - } - if (delay > 0) { - // cap to our budget but still respect server intent - const cap = Math.min(delay, 3000); - await this.wait(cap); - } - } + const should = this.shouldRetryStatus(res.status) || extraRetryOn.includes(res.status); + if (!should || attempt === maxRetries) return res; } catch (e) { clearTimeout(tm); lastErr = e; - if (attempt === maxRetries) { - this._cbReport(host, false); - throw e; - } + if (attempt === maxRetries) throw e; } const backoff = this.backoffDelay(attempt, baseDelay, maxDelay); - const remain = hardCapMs - (Date.now() + const remain = hardCapMs - (Date.now() - started); + if (remain <= 0) break; + await this.wait(Math.min(backoff, Math.max(0, remain - 50))); + } + + if (lastErr) throw lastErr; + throw new Error("fetchWithRetry failed"); + } + + // Hedged requests with early secondary, winner-takes-all, loser aborted. + // Cuts tail latency when a backend is slow/spotty. + async hedgedGetJsonFromBases(bases, path, query) { + await this._ensureUndici(); + const fetch = this._fetch; + + const qs = query ? (query.startsWith("?") ? query : "?" + query) : ""; + const primary = `${bases[0]}${path}${qs}`; + const secondary = bases[1] ? `${bases[1]}${path}${qs}` : null; + + const headers = { + "User-Agent": this.useragent, + "Accept": "application/json, text/plain, */*", + "Accept-Encoding": "gzip, deflate, br" + }; + + const attemptOnce = async (url, signal) => { + const res = await this.fetchWithRetry( + url, + { headers, signal }, + { retries: 3, baseDelay: 80, maxDelay: 1200, hardCapMs: 4000 } + ); + const tx = await res.text(); + return this.getJson(tx); + }; + + if (!secondary) return attemptOnce(primary); + + const acPrimary = new AbortController(); + const acSecondary = new AbortController(); + + // Fire secondary quickly (200ms) to hedge + const secondaryKick = (async () => { + await this.wait(200); + if (!acPrimary.signal.aborted) { + return attemptOnce(secondary, acSecondary.signal); + } + throw new Error("secondary canceled"); + })(); + + const primaryP = attemptOnce(primary, acPrimary.signal); + + try { + const winner = await Promise.race([ + primaryP.then(v => ({ v, who: "p" })), + secondaryKick.then(v => ({ v, who: "s" })) + ]); + + // Abort the loser ASAP + if (winner.who === "p") acSecondary.abort(); + else acPrimary.abort(); + + if (winner.v) return winner.v; + + // Fallback: await the other if the winner returned null-ish + const other = winner.who === "p" ? secondaryKick : primaryP; + const v2 = await other.catch(() => null); + return v2 || null; + } catch { + // Final fallback: try primary once more, fast budget + try { + return await attemptOnce(primary, acPrimary.signal); + } catch { + try { + return await attemptOnce(secondary, acSecondary.signal); + } catch { + return null; + } + } + } finally { + acPrimary.abort(); + acSecondary.abort(); + } + } + + async getColorsSafe(url) { + for (let i = 0; i < 2; i++) { + try { + const c = await getColors(url); + if (Array.isArray(c) && c[0] && c[1]) return [c[0].hex(), c[1].hex()]; + } catch {} + await this.wait(this.backoffDelay(i, 80, 600)); + } + return ["#0ea5e9", "#111827"]; + } + + // Fast libcurl retry: fewer attempts, tighter timeouts, early exit on non-retryable + async curlGetWithRetry(url, httpHeader) { + let lastErr = null; + for (let i = 0; i <= 3; i++) { + try { + const res = await curly.get(url, { httpHeader, timeoutMs: 7000, acceptEncoding: "gzip, deflate, br" }); + if (res && res.statusCode && res.statusCode >= 200 && res.statusCode < 300 && res.data) return res; + if (res && res.statusCode && !this.shouldRetryStatus(res.statusCode)) return res; + } catch (e) { + lastErr = e; + } + await this.wait(this.backoffDelay(i, 80, 1500)); + } + if (lastErr) throw lastErr; + throw new Error("curlGetWithRetry failed"); + } + + async getYouTubeApiVideo(f, v, contentlang, contentregion) { + if (v == null) return "Gib ID"; + + // 1h video-level cache + if (this.cache[v] && Date.now() - this.cache[v].timestamp < 3600000) { + return this.cache[v].result; + } + + const headersArr = Object.entries({ + "User-Agent": this.useragent, + "Accept": "application/json, text/plain, */*", + "Accept-Encoding": "gzip, deflate, br" + }).map(([k, v]) => `${k}: ${v}`); + + const bases = [this.config.invapi, this.config.invapi_alt]; + const b64ts = Buffer.from(String(Date.now())).toString("base64"); + const q = `hl=${contentlang}®ion=${contentregion}&h=${b64ts}`; + + try { + const [comments, vid, videoData] = await Promise.all([ + (async () => { + const hit = this._commentsCache.get(v); + if (hit && Date.now() - hit.ts < 300_000) return hit.data; + const data = await this.hedgedGetJsonFromBases(bases, `/comments/${v}`, q); + this._commentsCache.set(v, { data, ts: Date.now() }); + return data; + })(), + this.hedgedGetJsonFromBases(bases, `/videos/${v}`, q), + (async () => { + const res = await this.curlGetWithRetry(`${this.config.tubeApi}video?v=${v}`, headersArr); + const str = Buffer.isBuffer(res.data) ? res.data.toString("utf8") : String(res.data || ""); + const jsonStr = toJson(str); + const video = this.getJson(jsonStr); + return { json: jsonStr, video }; + })() + ]); + + let p = {}; + if (f === "true" && vid && vid.authorId) { + const cKey = vid.authorId; + const cached = this._channelCache.get(cKey); + if (cached && Date.now() - cached.ts < 300_000) { + p = cached.data; + } else { + p = await this.hedgedGetJsonFromBases(bases, `/channels/${vid.authorId}`, `hl=${contentlang}®ion=${contentregion}`); + this._channelCache.set(cKey, { data: p || {}, ts: Date.now() }); + } + } + + if (!vid) { + this.initError("Video JSON missing", new Error("no vid")); + return null; + } + + if (this.checkUnexistingObject(vid)) { + let fe = { engagement: null }; + try { + fe = await getdislikes(v); + } catch {} + + const [c1, c2] = await this.getColorsSafe(`https://i.ytimg.com/vi/${v}/hqdefault.jpg?sqp=${this.sqp}`); + + this.cache[v] = { + result: { + json: videoData?.json?.video, + video: videoData?.video, + vid, + comments, + channel_uploads: p || {}, + engagement: fe.engagement, + wiki: "", + desc: "", + color: c1, + color2: c2 + }, + timestamp: Date.now() + }; + return this.cache[v].result; + } + } catch (error) { + this.initError("Error getting video", error); + } + } + + isvalidvideo(v) { + if (v != "assets" && v != "cdn-cgi" && v != "404") { + return /^([a-zA-Z0-9_-]{11})/.test(v); + } + return false; + } + + initError(args, error) { + console.error("[LIBPT CORE ERROR] " + args, error); + } +} + +const pokeTubeApiCore = new InnerTubePokeVidious({ + tubeApi: "https://inner-api.poketube.fun/api/", + invapi: "https://invid-api.poketube.fun/bHj665PpYhUdPWuKPfZuQGoX/api/v1", + invapi_alt: config.proxylocation === "EU" ? "https://invid-api.poketube.fun/api/v1" : "https://iv.ggtyler.dev/api/v1", + dislikes: "https://returnyoutubedislikeapi.com/votes?videoId=", + t_url: "https://t.poketube.fun/", + useragent: config.useragent +}); + +module.exports = pokeTubeApiCore;