🚧 Write large files to disk instead of to memory

src/cdn/downloadUrlContents.ts

@@ -0,0 +1,23 @@
+import * as fs from 'fs';
+import * as http from 'http';
+import saveResponse from './funcs/saveResponse';
+
+/** Downloads the given URL and saves it to disk. Throws error if download fails. */
+export default function downloadUrlContents({ host, path }: {host: string, path: string}): Promise<fs.ReadStream> {
+  return new Promise((resolve, reject) => {
+
+    const request = http.request({
+      family: 4,
+      host,
+      path,
+    }, saveResponse.bind(null, resolve, (reason: string) => { request.abort(); reject(reason); }));
+
+    //In case of connection errors, exit early
+    request.on('error', (error) => {
+      request.abort();
+      reject(error);
+    });
+
+    request.end();
+  });
+}

src/cdn/funcs/handleResponse.ts

@@ -4,19 +4,19 @@ import * as http from 'http';
 const MAX_MEMORY_SIZE = 100 * 1024 * 1024;
 
 export default function handleResponse(
-  resolve: (value: ArrayBuffer) => void,
+  resolve: (arrayBuffer: ArrayBuffer) => void,
   reject: (reason: string) => void,
   response: http.IncomingMessage,
 ) {
   //Check that file exists (200 HTTP status code)
   if (response.statusCode !== 200) {
-    return reject(`Expected status code 200 but received ${response.statusCode}`);
+    return reject(`Expected status code 200 but received ${response.statusCode}.`);
   }
 
   //Check file size
   const headerLength = Number(response.headers['content-length']);
   if (headerLength > MAX_MEMORY_SIZE) {
-    return reject('File size too large to be handled in memory.');
+    return reject(`File size (${headerLength} bytes) too large to be handled in memory.`);
   }
 
   //If we receive a part of the response, store it
@@ -27,7 +27,7 @@ export default function handleResponse(
 
     //Exit early if we received too much data
     if (totalLength > headerLength) {
-      return reject(`Expected length ${headerLength} but received at least ${totalLength}`);
+      return reject(`Expected length ${headerLength} but received at least ${totalLength}.`);
     }
 
     //Add chunk to array
@@ -38,7 +38,7 @@ export default function handleResponse(
   response.on('end', () => {
     //Check that length is correct
     if (totalLength !== headerLength) {
-      return reject(`Expected length ${headerLength} but received ${totalLength}`);
+      return reject(`Expected length ${headerLength} but received ${totalLength}.`);
     }
 
     //Return file contents as ArrayBuffer

src/cdn/funcs/saveResponse.ts

@@ -0,0 +1,63 @@
+import * as fs from 'fs';
+import * as http from 'http';
+
+/** Too avoid overloading the server, do not allow large files (> 100 MB) to be handled in memory. */
+const MAX_MEMORY_SIZE = 100 * 1024 * 1024;
+
+export default function saveResponse(
+  resolve: (fileName: fs.ReadStream) => void,
+  reject: (reason: string) => void,
+  response: http.IncomingMessage,
+) {
+  //Check that file exists (200 HTTP status code)
+  if (response.statusCode !== 200) {
+    return reject(`Expected status code 200 but received ${response.statusCode}.`);
+  }
+
+  //Check file size
+  const headerLength = Number(response.headers['content-length']);
+  if (headerLength > MAX_MEMORY_SIZE) {
+    return reject(`File size (${headerLength} bytes) too large to be handled in memory.`);
+  }
+
+  const tempFileName = `'/tmp/swtor-patcher-download-${Math.random()}.tmp`;
+
+  //If we receive a part of the response, write it to disk
+  let previousChunk: Promise<void> = new Promise((innerResolve) => { innerResolve(); });
+  let totalLength = 0;
+  response.on('data', (chunk: Buffer) => {
+    totalLength += chunk.length;
+
+    //Exit early if we received too much data
+    if (totalLength > headerLength) {
+      return reject(`Expected length ${headerLength} but received at least ${totalLength}.`);
+    }
+
+    //If previous chunk was not yet written to disk, wait until it finished to avoid a race condition
+    previousChunk.then(() => {
+      previousChunk = new Promise((innerResolve, innerReject) => {
+        //Write chunk to disk
+        fs.appendFile(tempFileName, chunk, { encoding: 'binary' }, (error) => {
+          if (error) {
+            return reject(`Could not write to disk: [${error.code}] ${error.name}: ${error.message}.`);
+          }
+          innerResolve();
+        });
+      });
+    });
+  });
+
+  //If we finished reading response, check for correctness, then return it
+  response.on('end', () => {
+    //Check that length is correct
+    if (totalLength !== headerLength) {
+      return reject(`Expected length ${headerLength} but received ${totalLength}.`);
+    }
+
+    //Return file reader
+    //TODO: need to automatically delete file once it is no longer used
+    //TODO: need to provide methods to seek through file
+    const stream = fs.createReadStream(tempFileName, { encoding: 'binary' });
+    return resolve(stream);
+  });
+}