Merge pull request #17431 from timvandermeij/modernize-test
Modernize the code in the `test/downloadutils.mjs` file
This commit is contained in:
Tim van der Meij
GitHub
commit
34eb6fecac
@ -13,7 +13,6 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
/* eslint-disable no-var */
|
||||
|
||||
import crypto from "crypto";
|
||||
import fs from "fs";
|
||||
@ -25,89 +24,73 @@ function rewriteWebArchiveUrl(url) {
|
||||
// Web Archive URLs need to be transformed to add `if_` after the ID.
|
||||
// Without this, an HTML page containing an iframe with the PDF file
|
||||
// will be served instead (issue 8920).
|
||||
var webArchiveRegex =
|
||||
const webArchiveRegex =
|
||||
/(^https?:\/\/web\.archive\.org\/web\/)(\d+)(\/https?:\/\/.+)/g;
|
||||
var urlParts = webArchiveRegex.exec(url);
|
||||
const urlParts = webArchiveRegex.exec(url);
|
||||
if (urlParts) {
|
||||
return urlParts[1] + (urlParts[2] + "if_") + urlParts[3];
|
||||
return `${urlParts[1]}${urlParts[2]}if_${urlParts[3]}`;
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
function downloadFile(file, url, callback, redirects) {
|
||||
function downloadFile(file, url, redirects = 0) {
|
||||
url = rewriteWebArchiveUrl(url);
|
||||
const protocol = /^https:\/\//.test(url) ? https : http;
|
||||
|
||||
var protocol = /^https:\/\//.test(url) ? https : http;
|
||||
protocol
|
||||
.get(url, function (response) {
|
||||
if (
|
||||
response.statusCode === 301 ||
|
||||
response.statusCode === 302 ||
|
||||
response.statusCode === 307 ||
|
||||
response.statusCode === 308
|
||||
) {
|
||||
if (redirects > 10) {
|
||||
callback("Too many redirects");
|
||||
return new Promise((resolve, reject) => {
|
||||
protocol
|
||||
.get(url, async function (response) {
|
||||
if ([301, 302, 307, 308].includes(response.statusCode)) {
|
||||
if (redirects > 10) {
|
||||
reject(new Error("Too many redirects"));
|
||||
return;
|
||||
}
|
||||
const redirectTo = urlResolve(url, response.headers.location);
|
||||
try {
|
||||
await downloadFile(file, redirectTo, ++redirects);
|
||||
resolve();
|
||||
} catch (ex) {
|
||||
reject(ex);
|
||||
}
|
||||
return;
|
||||
}
|
||||
var redirectTo = response.headers.location;
|
||||
redirectTo = urlResolve(url, redirectTo);
|
||||
downloadFile(file, redirectTo, callback, (redirects || 0) + 1);
|
||||
return;
|
||||
}
|
||||
|
||||
if (response.statusCode !== 200) {
|
||||
callback("HTTP " + response.statusCode);
|
||||
return;
|
||||
}
|
||||
var stream = fs.createWriteStream(file);
|
||||
stream.on("error", function (err) {
|
||||
callback(err);
|
||||
});
|
||||
response.pipe(stream);
|
||||
stream.on("finish", function () {
|
||||
stream.end();
|
||||
callback();
|
||||
});
|
||||
})
|
||||
.on("error", function (err) {
|
||||
callback(err);
|
||||
});
|
||||
if (response.statusCode !== 200) {
|
||||
reject(new Error(`HTTP ${response.statusCode}`));
|
||||
return;
|
||||
}
|
||||
|
||||
const stream = fs.createWriteStream(file);
|
||||
stream.on("error", error => reject(error));
|
||||
stream.on("finish", () => {
|
||||
stream.end();
|
||||
resolve();
|
||||
});
|
||||
response.pipe(stream);
|
||||
})
|
||||
.on("error", error => reject(error));
|
||||
});
|
||||
}
|
||||
|
||||
function downloadManifestFiles(manifest, callback) {
|
||||
function downloadNext() {
|
||||
if (i >= links.length) {
|
||||
callback();
|
||||
return;
|
||||
}
|
||||
var file = links[i].file;
|
||||
var url = links[i].url;
|
||||
console.log("Downloading " + url + " to " + file + "...");
|
||||
downloadFile(file, url, function (err) {
|
||||
if (err) {
|
||||
console.error("Error during downloading of " + url + ": " + err);
|
||||
fs.writeFileSync(file, ""); // making it empty file
|
||||
fs.writeFileSync(file + ".error", err);
|
||||
}
|
||||
i++;
|
||||
downloadNext();
|
||||
});
|
||||
}
|
||||
|
||||
var links = manifest
|
||||
.filter(function (item) {
|
||||
return item.link && !fs.existsSync(item.file);
|
||||
})
|
||||
.map(function (item) {
|
||||
var file = item.file;
|
||||
var linkfile = file + ".link";
|
||||
var url = fs.readFileSync(linkfile).toString();
|
||||
async function downloadManifestFiles(manifest) {
|
||||
const links = manifest
|
||||
.filter(item => item.link && !fs.existsSync(item.file))
|
||||
.map(item => {
|
||||
let url = fs.readFileSync(`${item.file}.link`).toString();
|
||||
url = url.replace(/\s+$/, "");
|
||||
return { file, url };
|
||||
return { file: item.file, url };
|
||||
});
|
||||
|
||||
var i = 0;
|
||||
downloadNext();
|
||||
for (const { file, url } of links) {
|
||||
console.log(`Downloading ${url} to ${file}...`);
|
||||
try {
|
||||
await downloadFile(file, url);
|
||||
} catch (ex) {
|
||||
console.error(`Error during downloading of ${url}: ${ex}`);
|
||||
fs.writeFileSync(file, ""); // making it empty file
|
||||
fs.writeFileSync(`${file}.error`, ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function calculateMD5(file) {
|
||||
|
||||
@ -268,7 +268,7 @@ function examineRefImages() {
|
||||
});
|
||||
}
|
||||
|
||||
function startRefTest(masterMode, showRefImages) {
|
||||
async function startRefTest(masterMode, showRefImages) {
|
||||
function finalize() {
|
||||
stopServer();
|
||||
let numRuns = 0;
|
||||
@ -402,11 +402,10 @@ function startRefTest(masterMode, showRefImages) {
|
||||
if (!manifest) {
|
||||
return;
|
||||
}
|
||||
if (options.noDownload) {
|
||||
checkRefsTmp();
|
||||
} else {
|
||||
ensurePDFsDownloaded(checkRefsTmp);
|
||||
if (!options.noDownload) {
|
||||
await ensurePDFsDownloaded();
|
||||
}
|
||||
checkRefsTmp();
|
||||
}
|
||||
|
||||
function handleSessionTimeout(session) {
|
||||
@ -1071,47 +1070,43 @@ async function closeSession(browser) {
|
||||
}
|
||||
}
|
||||
|
||||
function ensurePDFsDownloaded(callback) {
|
||||
var manifest = getTestManifest();
|
||||
downloadManifestFiles(manifest, async function () {
|
||||
try {
|
||||
await verifyManifestFiles(manifest);
|
||||
} catch {
|
||||
console.log(
|
||||
"Unable to verify the checksum for the files that are " +
|
||||
"used for testing."
|
||||
);
|
||||
console.log(
|
||||
"Please re-download the files, or adjust the MD5 " +
|
||||
"checksum in the manifest for the files listed above.\n"
|
||||
);
|
||||
if (options.strictVerify) {
|
||||
process.exit(1);
|
||||
}
|
||||
async function ensurePDFsDownloaded() {
|
||||
const manifest = getTestManifest();
|
||||
await downloadManifestFiles(manifest);
|
||||
try {
|
||||
await verifyManifestFiles(manifest);
|
||||
} catch {
|
||||
console.log(
|
||||
"Unable to verify the checksum for the files that are " +
|
||||
"used for testing."
|
||||
);
|
||||
console.log(
|
||||
"Please re-download the files, or adjust the MD5 " +
|
||||
"checksum in the manifest for the files listed above.\n"
|
||||
);
|
||||
if (options.strictVerify) {
|
||||
process.exit(1);
|
||||
}
|
||||
callback();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function main() {
|
||||
async function main() {
|
||||
if (options.statsFile) {
|
||||
stats = [];
|
||||
}
|
||||
|
||||
if (options.downloadOnly) {
|
||||
ensurePDFsDownloaded(function () {});
|
||||
await ensurePDFsDownloaded();
|
||||
} else if (options.unitTest) {
|
||||
// Allows linked PDF files in unit-tests as well.
|
||||
ensurePDFsDownloaded(function () {
|
||||
startUnitTest("/test/unit/unit_test.html", "unit");
|
||||
});
|
||||
await ensurePDFsDownloaded();
|
||||
startUnitTest("/test/unit/unit_test.html", "unit");
|
||||
} else if (options.fontTest) {
|
||||
startUnitTest("/test/font/font_test.html", "font");
|
||||
} else if (options.integration) {
|
||||
// Allows linked PDF files in integration-tests as well.
|
||||
ensurePDFsDownloaded(function () {
|
||||
startIntegrationTest();
|
||||
});
|
||||
await ensurePDFsDownloaded();
|
||||
startIntegrationTest();
|
||||
} else {
|
||||
startRefTest(options.masterMode, options.reftest);
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user