Update src/libpoketube/libpoketube-core.js
This commit is contained in:
ashley
parent
5af078a9e7
commit
4e0f05b856
@ -13,9 +13,9 @@ const getColors = require("get-image-colors");
|
||||
const config = require("../../config.json");
|
||||
|
||||
class InnerTubePokeVidious {
|
||||
constructor(cfg) {
|
||||
this.config = cfg;
|
||||
this.cache = {}; // 1h video-level cache
|
||||
constructor(config) {
|
||||
this.config = config;
|
||||
this.cache = {};
|
||||
this.language = "hl=en-US";
|
||||
this.param = "2AMB";
|
||||
this.param_legacy = "CgIIAdgDAQ%3D%3D";
|
||||
@ -23,36 +23,31 @@ class InnerTubePokeVidious {
|
||||
this.ANDROID_API_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w";
|
||||
this.ANDROID_APP_VERSION = "20.20.41";
|
||||
this.ANDROID_VERSION = "16";
|
||||
this.useragent = cfg.useragent || "PokeTube/2.0.0 (GNU/Linux; Android 14; Trisquel 11; poketube-vidious; like FreeTube)";
|
||||
this.useragent = config.useragent || "PokeTube/2.0.0 (GNU/Linux; Android 14; Trisquel 11; poketube-vidious; like FreeTube)";
|
||||
this.INNERTUBE_CONTEXT_CLIENT_VERSION = "1";
|
||||
this.region = "region=US";
|
||||
this.sqp = "-oaymwEbCKgBEF5IVfKriqkDDggBFQAAiEIYAXABwAEG&rs=AOn4CLBy_x4UUHLNDZtJtH0PXeQGoRFTgw";
|
||||
|
||||
// undici primitives (lazy)
|
||||
// Lazy-initialized undici pieces for connection reuse + pipelining
|
||||
this._fetch = null;
|
||||
this._agent = null;
|
||||
this._undiciInit = null;
|
||||
|
||||
// Micro-caches (short TTLs)
|
||||
this._commentsCache = new Map(); // videoId -> {data, ts}
|
||||
this._channelCache = new Map(); // authorId -> {data, ts}
|
||||
|
||||
// In-flight de-dup (so concurrent identical asks share one network trip)
|
||||
this._inflight = new Map(); // key -> Promise
|
||||
|
||||
// Per-host circuit breaker state
|
||||
this._cb = new Map(); // host -> {fail: number, openUntil: number}
|
||||
// Small LRU-ish caches to avoid redundant work within the same process lifespan
|
||||
this._channelCache = new Map(); // key: authorId -> {data, ts}
|
||||
this._commentsCache = new Map(); // key: videoId -> {data, ts}
|
||||
}
|
||||
|
||||
async _ensureUndici() {
|
||||
if (this._undiciInit) return this._undiciInit;
|
||||
this._undiciInit = (async () => {
|
||||
const { fetch, Agent, setGlobalDispatcher } = await import("undici");
|
||||
// Aggressive keep-alive + pipelining to cut handshake latency
|
||||
const agent = new Agent({
|
||||
keepAliveTimeout: 60_000,
|
||||
keepAliveMaxTimeout: 60_000,
|
||||
connections: 24, // push a bit harder
|
||||
pipelining: 1,
|
||||
connections: 16,
|
||||
pipelining: 1, // keep conservative; many APIs dislike >1
|
||||
maxRedirections: 2
|
||||
});
|
||||
setGlobalDispatcher(agent);
|
||||
@ -63,17 +58,24 @@ class InnerTubePokeVidious {
|
||||
}
|
||||
|
||||
getJson(str) {
|
||||
try { return JSON.parse(str); } catch { return null; }
|
||||
try {
|
||||
return JSON.parse(str);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
checkUnexistingObject(obj) {
|
||||
return obj && "authorId" in obj;
|
||||
}
|
||||
|
||||
async wait(ms) { return new Promise(r => setTimeout(r, ms)); }
|
||||
async wait(ms) {
|
||||
return new Promise(r => setTimeout(r, ms));
|
||||
}
|
||||
|
||||
// Decorrelated jitter backoff (fast caps)
|
||||
backoffDelay(attempt, base = 60, cap = 1600) {
|
||||
// Faster, low-tail decorrelated jitter backoff
|
||||
// See AWS "Exponential Backoff And Jitter" — decorrelated variant
|
||||
backoffDelay(attempt, base = 80, cap = 2000) {
|
||||
const prev = attempt === 0 ? base : Math.min(cap, base * Math.pow(2, attempt - 1));
|
||||
const next = Math.min(cap, base + Math.floor(Math.random() * (prev * 3)));
|
||||
return next;
|
||||
@ -83,52 +85,34 @@ class InnerTubePokeVidious {
|
||||
if (!status) return true;
|
||||
if (status === 408 || status === 425 || status === 429) return true;
|
||||
if (status >= 500 && status <= 599) return true;
|
||||
// do not retry on common hard errors
|
||||
// hard no-retry statuses
|
||||
if (status === 400 || status === 401 || status === 403 || status === 404 || status === 409 || status === 410) return false;
|
||||
return false;
|
||||
}
|
||||
|
||||
_mergeHeaders(a = {}, b = {}) {
|
||||
const o = { ...a };
|
||||
for (const k of Object.keys(b)) o[k] = b[k];
|
||||
for (const k of Object.keys(b)) {
|
||||
o[k] = b[k];
|
||||
}
|
||||
return o;
|
||||
}
|
||||
|
||||
_hostFromUrl(u) {
|
||||
try { return new URL(u).host; } catch { return ""; }
|
||||
}
|
||||
|
||||
_cbIsOpen(host) {
|
||||
const s = this._cb.get(host);
|
||||
return s && s.openUntil && Date.now() < s.openUntil;
|
||||
}
|
||||
|
||||
_cbReport(host, ok) {
|
||||
const s = this._cb.get(host) || { fail: 0, openUntil: 0 };
|
||||
if (ok) {
|
||||
s.fail = 0;
|
||||
s.openUntil = 0;
|
||||
} else {
|
||||
s.fail += 1;
|
||||
if (s.fail >= 3) {
|
||||
// open for a short period; we hedge to other bases meanwhile
|
||||
s.openUntil = Date.now() + 15_000;
|
||||
}
|
||||
}
|
||||
this._cb.set(host, s);
|
||||
}
|
||||
|
||||
// Ultra-lean fetch with:
|
||||
// - connection reuse (undici agent)
|
||||
// - short per-attempt timeouts (attempt-scaled)
|
||||
// - decorrelated jitter backoff
|
||||
// - minimal retries by default, but smart on 429/5xx
|
||||
async fetchWithRetry(url, options = {}, cfg = {}) {
|
||||
await this._ensureUndici();
|
||||
const fetch = this._fetch;
|
||||
|
||||
const maxRetries = Number.isInteger(cfg.retries) ? Math.max(0, cfg.retries) : 6;
|
||||
const baseDelay = cfg.baseDelay ?? 60;
|
||||
const maxDelay = cfg.maxDelay ?? 1600;
|
||||
const hardCapMs = cfg.hardCapMs ?? 7000; // aggressive total budget
|
||||
const maxRetries = Number.isInteger(cfg.retries) ? Math.max(0, cfg.retries) : 5;
|
||||
const baseDelay = cfg.baseDelay ?? 80;
|
||||
const maxDelay = cfg.maxDelay ?? 2000;
|
||||
const hardCapMs = cfg.hardCapMs ?? 8000; // total time budget
|
||||
const extraRetryOn = cfg.retryOnStatuses || [];
|
||||
const started = Date.now();
|
||||
const host = this._hostFromUrl(url);
|
||||
|
||||
const uah = {
|
||||
"User-Agent": this.useragent,
|
||||
@ -136,21 +120,13 @@ class InnerTubePokeVidious {
|
||||
"Accept-Encoding": "gzip, deflate, br"
|
||||
};
|
||||
|
||||
// If breaker open, we still *try* once quickly (in case it recovered),
|
||||
// but with tiny per-attempt budget to fail fast.
|
||||
const breakerPenalty = this._cbIsOpen(host) ? 300 : 0;
|
||||
|
||||
let lastErr = null;
|
||||
|
||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||
const elapsed = Date.now() - started;
|
||||
if (elapsed >= hardCapMs) break;
|
||||
|
||||
const perAttemptBudget = Math.max(
|
||||
350 - breakerPenalty,
|
||||
Math.min(2000, hardCapMs - elapsed)
|
||||
);
|
||||
|
||||
const perAttemptBudget = Math.max(500, Math.min(2500, hardCapMs - elapsed));
|
||||
const ac = new AbortController();
|
||||
const tm = setTimeout(() => ac.abort(new Error("timeout")), perAttemptBudget);
|
||||
|
||||
@ -162,40 +138,232 @@ class InnerTubePokeVidious {
|
||||
});
|
||||
clearTimeout(tm);
|
||||
|
||||
if (res.ok) {
|
||||
this._cbReport(host, true);
|
||||
return res;
|
||||
}
|
||||
if (res.ok) return res;
|
||||
|
||||
const retryable = this.shouldRetryStatus(res.status) || extraRetryOn.includes(res.status);
|
||||
if (!retryable || attempt === maxRetries) {
|
||||
this._cbReport(host, false);
|
||||
return res;
|
||||
}
|
||||
|
||||
// Honor Retry-After if present (429/503)
|
||||
const ra = res.headers.get("retry-after");
|
||||
if (ra) {
|
||||
let delay = 0;
|
||||
if (/^\d+$/.test(ra)) delay = parseInt(ra, 10) * 1000;
|
||||
else {
|
||||
const when = Date.parse(ra);
|
||||
if (!Number.isNaN(when)) delay = Math.max(0, when - Date.now());
|
||||
}
|
||||
if (delay > 0) {
|
||||
// cap to our budget but still respect server intent
|
||||
const cap = Math.min(delay, 3000);
|
||||
await this.wait(cap);
|
||||
}
|
||||
}
|
||||
const should = this.shouldRetryStatus(res.status) || extraRetryOn.includes(res.status);
|
||||
if (!should || attempt === maxRetries) return res;
|
||||
} catch (e) {
|
||||
clearTimeout(tm);
|
||||
lastErr = e;
|
||||
if (attempt === maxRetries) {
|
||||
this._cbReport(host, false);
|
||||
throw e;
|
||||
}
|
||||
if (attempt === maxRetries) throw e;
|
||||
}
|
||||
|
||||
const backoff = this.backoffDelay(attempt, baseDelay, maxDelay);
|
||||
const remain = hardCapMs - (Date.now()
|
||||
const remain = hardCapMs - (Date.now() - started);
|
||||
if (remain <= 0) break;
|
||||
await this.wait(Math.min(backoff, Math.max(0, remain - 50)));
|
||||
}
|
||||
|
||||
if (lastErr) throw lastErr;
|
||||
throw new Error("fetchWithRetry failed");
|
||||
}
|
||||
|
||||
// Hedged requests with early secondary, winner-takes-all, loser aborted.
|
||||
// Cuts tail latency when a backend is slow/spotty.
|
||||
async hedgedGetJsonFromBases(bases, path, query) {
|
||||
await this._ensureUndici();
|
||||
const fetch = this._fetch;
|
||||
|
||||
const qs = query ? (query.startsWith("?") ? query : "?" + query) : "";
|
||||
const primary = `${bases[0]}${path}${qs}`;
|
||||
const secondary = bases[1] ? `${bases[1]}${path}${qs}` : null;
|
||||
|
||||
const headers = {
|
||||
"User-Agent": this.useragent,
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Accept-Encoding": "gzip, deflate, br"
|
||||
};
|
||||
|
||||
const attemptOnce = async (url, signal) => {
|
||||
const res = await this.fetchWithRetry(
|
||||
url,
|
||||
{ headers, signal },
|
||||
{ retries: 3, baseDelay: 80, maxDelay: 1200, hardCapMs: 4000 }
|
||||
);
|
||||
const tx = await res.text();
|
||||
return this.getJson(tx);
|
||||
};
|
||||
|
||||
if (!secondary) return attemptOnce(primary);
|
||||
|
||||
const acPrimary = new AbortController();
|
||||
const acSecondary = new AbortController();
|
||||
|
||||
// Fire secondary quickly (200ms) to hedge
|
||||
const secondaryKick = (async () => {
|
||||
await this.wait(200);
|
||||
if (!acPrimary.signal.aborted) {
|
||||
return attemptOnce(secondary, acSecondary.signal);
|
||||
}
|
||||
throw new Error("secondary canceled");
|
||||
})();
|
||||
|
||||
const primaryP = attemptOnce(primary, acPrimary.signal);
|
||||
|
||||
try {
|
||||
const winner = await Promise.race([
|
||||
primaryP.then(v => ({ v, who: "p" })),
|
||||
secondaryKick.then(v => ({ v, who: "s" }))
|
||||
]);
|
||||
|
||||
// Abort the loser ASAP
|
||||
if (winner.who === "p") acSecondary.abort();
|
||||
else acPrimary.abort();
|
||||
|
||||
if (winner.v) return winner.v;
|
||||
|
||||
// Fallback: await the other if the winner returned null-ish
|
||||
const other = winner.who === "p" ? secondaryKick : primaryP;
|
||||
const v2 = await other.catch(() => null);
|
||||
return v2 || null;
|
||||
} catch {
|
||||
// Final fallback: try primary once more, fast budget
|
||||
try {
|
||||
return await attemptOnce(primary, acPrimary.signal);
|
||||
} catch {
|
||||
try {
|
||||
return await attemptOnce(secondary, acSecondary.signal);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
acPrimary.abort();
|
||||
acSecondary.abort();
|
||||
}
|
||||
}
|
||||
|
||||
async getColorsSafe(url) {
|
||||
for (let i = 0; i < 2; i++) {
|
||||
try {
|
||||
const c = await getColors(url);
|
||||
if (Array.isArray(c) && c[0] && c[1]) return [c[0].hex(), c[1].hex()];
|
||||
} catch {}
|
||||
await this.wait(this.backoffDelay(i, 80, 600));
|
||||
}
|
||||
return ["#0ea5e9", "#111827"];
|
||||
}
|
||||
|
||||
// Fast libcurl retry: fewer attempts, tighter timeouts, early exit on non-retryable
|
||||
async curlGetWithRetry(url, httpHeader) {
|
||||
let lastErr = null;
|
||||
for (let i = 0; i <= 3; i++) {
|
||||
try {
|
||||
const res = await curly.get(url, { httpHeader, timeoutMs: 7000, acceptEncoding: "gzip, deflate, br" });
|
||||
if (res && res.statusCode && res.statusCode >= 200 && res.statusCode < 300 && res.data) return res;
|
||||
if (res && res.statusCode && !this.shouldRetryStatus(res.statusCode)) return res;
|
||||
} catch (e) {
|
||||
lastErr = e;
|
||||
}
|
||||
await this.wait(this.backoffDelay(i, 80, 1500));
|
||||
}
|
||||
if (lastErr) throw lastErr;
|
||||
throw new Error("curlGetWithRetry failed");
|
||||
}
|
||||
|
||||
async getYouTubeApiVideo(f, v, contentlang, contentregion) {
|
||||
if (v == null) return "Gib ID";
|
||||
|
||||
// 1h video-level cache
|
||||
if (this.cache[v] && Date.now() - this.cache[v].timestamp < 3600000) {
|
||||
return this.cache[v].result;
|
||||
}
|
||||
|
||||
const headersArr = Object.entries({
|
||||
"User-Agent": this.useragent,
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Accept-Encoding": "gzip, deflate, br"
|
||||
}).map(([k, v]) => `${k}: ${v}`);
|
||||
|
||||
const bases = [this.config.invapi, this.config.invapi_alt];
|
||||
const b64ts = Buffer.from(String(Date.now())).toString("base64");
|
||||
const q = `hl=${contentlang}®ion=${contentregion}&h=${b64ts}`;
|
||||
|
||||
try {
|
||||
const [comments, vid, videoData] = await Promise.all([
|
||||
(async () => {
|
||||
const hit = this._commentsCache.get(v);
|
||||
if (hit && Date.now() - hit.ts < 300_000) return hit.data;
|
||||
const data = await this.hedgedGetJsonFromBases(bases, `/comments/${v}`, q);
|
||||
this._commentsCache.set(v, { data, ts: Date.now() });
|
||||
return data;
|
||||
})(),
|
||||
this.hedgedGetJsonFromBases(bases, `/videos/${v}`, q),
|
||||
(async () => {
|
||||
const res = await this.curlGetWithRetry(`${this.config.tubeApi}video?v=${v}`, headersArr);
|
||||
const str = Buffer.isBuffer(res.data) ? res.data.toString("utf8") : String(res.data || "");
|
||||
const jsonStr = toJson(str);
|
||||
const video = this.getJson(jsonStr);
|
||||
return { json: jsonStr, video };
|
||||
})()
|
||||
]);
|
||||
|
||||
let p = {};
|
||||
if (f === "true" && vid && vid.authorId) {
|
||||
const cKey = vid.authorId;
|
||||
const cached = this._channelCache.get(cKey);
|
||||
if (cached && Date.now() - cached.ts < 300_000) {
|
||||
p = cached.data;
|
||||
} else {
|
||||
p = await this.hedgedGetJsonFromBases(bases, `/channels/${vid.authorId}`, `hl=${contentlang}®ion=${contentregion}`);
|
||||
this._channelCache.set(cKey, { data: p || {}, ts: Date.now() });
|
||||
}
|
||||
}
|
||||
|
||||
if (!vid) {
|
||||
this.initError("Video JSON missing", new Error("no vid"));
|
||||
return null;
|
||||
}
|
||||
|
||||
if (this.checkUnexistingObject(vid)) {
|
||||
let fe = { engagement: null };
|
||||
try {
|
||||
fe = await getdislikes(v);
|
||||
} catch {}
|
||||
|
||||
const [c1, c2] = await this.getColorsSafe(`https://i.ytimg.com/vi/${v}/hqdefault.jpg?sqp=${this.sqp}`);
|
||||
|
||||
this.cache[v] = {
|
||||
result: {
|
||||
json: videoData?.json?.video,
|
||||
video: videoData?.video,
|
||||
vid,
|
||||
comments,
|
||||
channel_uploads: p || {},
|
||||
engagement: fe.engagement,
|
||||
wiki: "",
|
||||
desc: "",
|
||||
color: c1,
|
||||
color2: c2
|
||||
},
|
||||
timestamp: Date.now()
|
||||
};
|
||||
return this.cache[v].result;
|
||||
}
|
||||
} catch (error) {
|
||||
this.initError("Error getting video", error);
|
||||
}
|
||||
}
|
||||
|
||||
isvalidvideo(v) {
|
||||
if (v != "assets" && v != "cdn-cgi" && v != "404") {
|
||||
return /^([a-zA-Z0-9_-]{11})/.test(v);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
initError(args, error) {
|
||||
console.error("[LIBPT CORE ERROR] " + args, error);
|
||||
}
|
||||
}
|
||||
|
||||
const pokeTubeApiCore = new InnerTubePokeVidious({
|
||||
tubeApi: "https://inner-api.poketube.fun/api/",
|
||||
invapi: "https://invid-api.poketube.fun/bHj665PpYhUdPWuKPfZuQGoX/api/v1",
|
||||
invapi_alt: config.proxylocation === "EU" ? "https://invid-api.poketube.fun/api/v1" : "https://iv.ggtyler.dev/api/v1",
|
||||
dislikes: "https://returnyoutubedislikeapi.com/votes?videoId=",
|
||||
t_url: "https://t.poketube.fun/",
|
||||
useragent: config.useragent
|
||||
});
|
||||
|
||||
module.exports = pokeTubeApiCore;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user