Modernize the downloadFile
test helper function
The test helper code largely predates the introduction of modern JavaScript features and should be refactored to improve readability. In particular callbacks make the code harder to understand and maintain. This commit: - replaces the callback argument with returning a promise; - uses `const` instead of `var`; - uses arrow functions for shorter code; - uses template strings for shorter string formatting code; - uses `Array.includes` for shorter response code checking code.
This commit is contained in:
parent
91188cf2f8
commit
86bee4409a
@ -34,48 +34,47 @@ function rewriteWebArchiveUrl(url) {
|
|||||||
return url;
|
return url;
|
||||||
}
|
}
|
||||||
|
|
||||||
function downloadFile(file, url, callback, redirects) {
|
function downloadFile(file, url, redirects = 0) {
|
||||||
url = rewriteWebArchiveUrl(url);
|
url = rewriteWebArchiveUrl(url);
|
||||||
|
const protocol = /^https:\/\//.test(url) ? https : http;
|
||||||
|
|
||||||
var protocol = /^https:\/\//.test(url) ? https : http;
|
return new Promise((resolve, reject) => {
|
||||||
protocol
|
protocol
|
||||||
.get(url, function (response) {
|
.get(url, async function (response) {
|
||||||
if (
|
if ([301, 302, 307, 308].includes(response.statusCode)) {
|
||||||
response.statusCode === 301 ||
|
if (redirects > 10) {
|
||||||
response.statusCode === 302 ||
|
reject(new Error("Too many redirects"));
|
||||||
response.statusCode === 307 ||
|
return;
|
||||||
response.statusCode === 308
|
}
|
||||||
) {
|
const redirectTo = urlResolve(url, response.headers.location);
|
||||||
if (redirects > 10) {
|
try {
|
||||||
callback("Too many redirects");
|
await downloadFile(file, redirectTo, ++redirects);
|
||||||
|
resolve();
|
||||||
|
} catch (ex) {
|
||||||
|
reject(ex);
|
||||||
|
}
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
var redirectTo = response.headers.location;
|
|
||||||
redirectTo = urlResolve(url, redirectTo);
|
|
||||||
downloadFile(file, redirectTo, callback, (redirects || 0) + 1);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (response.statusCode !== 200) {
|
if (response.statusCode !== 200) {
|
||||||
callback("HTTP " + response.statusCode);
|
reject(new Error(`HTTP ${response.statusCode}`));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
var stream = fs.createWriteStream(file);
|
|
||||||
stream.on("error", function (err) {
|
const stream = fs.createWriteStream(file);
|
||||||
callback(err);
|
stream.on("error", error => reject(error));
|
||||||
});
|
stream.on("finish", () => {
|
||||||
response.pipe(stream);
|
stream.end();
|
||||||
stream.on("finish", function () {
|
resolve();
|
||||||
stream.end();
|
});
|
||||||
callback();
|
response.pipe(stream);
|
||||||
});
|
})
|
||||||
})
|
.on("error", error => reject(error));
|
||||||
.on("error", function (err) {
|
});
|
||||||
callback(err);
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function downloadManifestFiles(manifest, callback) {
|
function downloadManifestFiles(manifest, callback) {
|
||||||
function downloadNext() {
|
async function downloadNext() {
|
||||||
if (i >= links.length) {
|
if (i >= links.length) {
|
||||||
callback();
|
callback();
|
||||||
return;
|
return;
|
||||||
@ -83,15 +82,15 @@ function downloadManifestFiles(manifest, callback) {
|
|||||||
var file = links[i].file;
|
var file = links[i].file;
|
||||||
var url = links[i].url;
|
var url = links[i].url;
|
||||||
console.log("Downloading " + url + " to " + file + "...");
|
console.log("Downloading " + url + " to " + file + "...");
|
||||||
downloadFile(file, url, function (err) {
|
try {
|
||||||
if (err) {
|
await downloadFile(file, url);
|
||||||
console.error("Error during downloading of " + url + ": " + err);
|
} catch (ex) {
|
||||||
fs.writeFileSync(file, ""); // making it empty file
|
console.error(`Error during downloading of ${url}: ${ex}`);
|
||||||
fs.writeFileSync(file + ".error", err);
|
fs.writeFileSync(file, ""); // making it empty file
|
||||||
}
|
fs.writeFileSync(`${file}.error`, ex);
|
||||||
i++;
|
}
|
||||||
downloadNext();
|
i++;
|
||||||
});
|
downloadNext();
|
||||||
}
|
}
|
||||||
|
|
||||||
var links = manifest
|
var links = manifest
|
||||||
|
Loading…
x
Reference in New Issue
Block a user