Drop obsolete logic from the downloadFile
function in test/downloadutils.js
This code is old and predates the improvements we made to the test manifest to only contain working URLs (either Web Archive or GitHub/Bugzilla links), so the fallback logic to try the Web Archive is no longer necessary. This greatly simplifies the function and also makes sure that we fail directly in case a bad URL is added to the manifest, instead of having it work "accidentally" because of this logic, since we want the manifest to be correct at all times (and otherwise fail loudly).
This commit is contained in:
parent
0df1a56619
commit
99430225b0
@ -38,11 +38,9 @@ function rewriteWebArchiveUrl(url) {
|
||||
function downloadFile(file, url, callback, redirects) {
|
||||
url = rewriteWebArchiveUrl(url);
|
||||
|
||||
var completed = false;
|
||||
var protocol = /^https:\/\//.test(url) ? https : http;
|
||||
protocol
|
||||
.get(url, function (response) {
|
||||
var redirectTo;
|
||||
if (
|
||||
response.statusCode === 301 ||
|
||||
response.statusCode === 302 ||
|
||||
@ -52,56 +50,28 @@ function downloadFile(file, url, callback, redirects) {
|
||||
if (redirects > 10) {
|
||||
callback("Too many redirects");
|
||||
}
|
||||
redirectTo = response.headers.location;
|
||||
var redirectTo = response.headers.location;
|
||||
redirectTo = require("url").resolve(url, redirectTo);
|
||||
downloadFile(file, redirectTo, callback, (redirects || 0) + 1);
|
||||
return;
|
||||
}
|
||||
if (response.statusCode === 404 && !url.includes("web.archive.org")) {
|
||||
// trying waybackmachine
|
||||
redirectTo = "http://web.archive.org/web/" + url;
|
||||
downloadFile(file, redirectTo, callback, (redirects || 0) + 1);
|
||||
return;
|
||||
}
|
||||
|
||||
if (response.statusCode !== 200) {
|
||||
if (!completed) {
|
||||
completed = true;
|
||||
callback("HTTP " + response.statusCode);
|
||||
}
|
||||
callback("HTTP " + response.statusCode);
|
||||
return;
|
||||
}
|
||||
var stream = fs.createWriteStream(file);
|
||||
stream.on("error", function (err) {
|
||||
if (!completed) {
|
||||
completed = true;
|
||||
callback(err);
|
||||
}
|
||||
callback(err);
|
||||
});
|
||||
response.pipe(stream);
|
||||
stream.on("finish", function () {
|
||||
stream.end();
|
||||
if (!completed) {
|
||||
completed = true;
|
||||
callback();
|
||||
}
|
||||
callback();
|
||||
});
|
||||
})
|
||||
.on("error", function (err) {
|
||||
if (!completed) {
|
||||
if (
|
||||
typeof err === "object" &&
|
||||
err.errno === "ENOTFOUND" &&
|
||||
!url.includes("web.archive.org")
|
||||
) {
|
||||
// trying waybackmachine
|
||||
var redirectTo = "http://web.archive.org/web/" + url;
|
||||
downloadFile(file, redirectTo, callback, (redirects || 0) + 1);
|
||||
return;
|
||||
}
|
||||
completed = true;
|
||||
callback(err);
|
||||
}
|
||||
callback(err);
|
||||
});
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user