Handles HTTP redirects

test/downloadutils.js

@@ -24,10 +24,27 @@ var crypto = require('crypto');
 var http = require('http');
 var https = require('https');
 
-function downloadFile(file, url, callback) {
+function downloadFile(file, url, callback, redirects) {
   var completed = false;
   var protocol = /^https:\/\//.test(url) ? https : http;
   protocol.get(url, function (response) {
+    if (response.statusCode === 301 || response.statusCode === 302 ||
+        response.statusCode === 307 || response.statusCode === 308) {
+      if (redirects > 10) {
+        callback('Too many redirects');
+      }
+      var redirectTo = response.headers.location;
+      redirectTo = require('url').resolve(url, redirectTo);
+      downloadFile(file, redirectTo, callback, (redirects || 0) + 1);
+      return;
+    }
+    if (response.statusCode === 404 && url.indexOf('web.archive.org') < 0) {
+      // trying waybackmachine
+      var redirectTo = 'http://web.archive.org/web/' + url;
+      downloadFile(file, redirectTo, callback, (redirects || 0) + 1);
+      return;
+    }
+
     if (response.statusCode !== 200) {
       if (!completed) {
         completed = true;
@@ -52,6 +69,13 @@ function downloadFile(file, url, callback) {
     });
   }).on('error', function (err) {
     if (!completed) {
+      if (typeof err === 'object' && err.errno === 'ENOTFOUND' &&
+          url.indexOf('web.archive.org') < 0) {
+        // trying waybackmachine
+        var redirectTo = 'http://web.archive.org/web/' + url;
+        downloadFile(file, redirectTo, callback, (redirects || 0) + 1);
+        return;
+      }
       completed = true;
       callback(err);
     }