🚧 Write large files to disk instead of to memory

src/cdn/downloadUrlContents.ts

@@ -0,0 +1,23 @@
+import * as fs from 'fs';
+import * as http from 'http';
+import saveResponse from './funcs/saveResponse';
+
+/** Downloads the given URL and saves it to disk. Throws error if download fails. */
+export default function downloadUrlContents({ host, path }: {host: string, path: string}): Promise<fs.ReadStream> {
+  return new Promise((resolve, reject) => {
+
+    const request = http.request({
+      family: 4,
+      host,
+      path,
+    }, saveResponse.bind(null, resolve, (reason: string) => { request.abort(); reject(reason); }));
+
+    //In case of connection errors, exit early
+    request.on('error', (error) => {
+      request.abort();
+      reject(error);
+    });
+
+    request.end();
+  });
+}

src/cdn/funcs/handleResponse.ts

@@ -4,19 +4,19 @@ import * as http from 'http';
 const MAX_MEMORY_SIZE = 100 * 1024 * 1024;
 
 export default function handleResponse(
-  resolve: (value: ArrayBuffer) => void,
+  resolve: (arrayBuffer: ArrayBuffer) => void,
   reject: (reason: string) => void,
   response: http.IncomingMessage,
 ) {
   //Check that file exists (200 HTTP status code)
   if (response.statusCode !== 200) {
-    return reject(`Expected status code 200 but received ${response.statusCode}`);
+    return reject(`Expected status code 200 but received ${response.statusCode}.`);
   }
 
   //Check file size
   const headerLength = Number(response.headers['content-length']);
   if (headerLength > MAX_MEMORY_SIZE) {
-    return reject('File size too large to be handled in memory.');
+    return reject(`File size (${headerLength} bytes) too large to be handled in memory.`);
   }
 
   //If we receive a part of the response, store it
@@ -27,7 +27,7 @@ export default function handleResponse(
 
     //Exit early if we received too much data
     if (totalLength > headerLength) {
-      return reject(`Expected length ${headerLength} but received at least ${totalLength}`);
+      return reject(`Expected length ${headerLength} but received at least ${totalLength}.`);
     }
 
     //Add chunk to array
@@ -38,7 +38,7 @@ export default function handleResponse(
   response.on('end', () => {
     //Check that length is correct
     if (totalLength !== headerLength) {
-      return reject(`Expected length ${headerLength} but received ${totalLength}`);
+      return reject(`Expected length ${headerLength} but received ${totalLength}.`);
     }
 
     //Return file contents as ArrayBuffer

src/cdn/funcs/saveResponse.ts

@@ -0,0 +1,63 @@
+import * as fs from 'fs';
+import * as http from 'http';
+
+/** Too avoid overloading the server, do not allow large files (> 100 MB) to be handled in memory. */
+const MAX_MEMORY_SIZE = 100 * 1024 * 1024;
+
+export default function saveResponse(
+  resolve: (fileName: fs.ReadStream) => void,
+  reject: (reason: string) => void,
+  response: http.IncomingMessage,
+) {
+  //Check that file exists (200 HTTP status code)
+  if (response.statusCode !== 200) {
+    return reject(`Expected status code 200 but received ${response.statusCode}.`);
+  }
+
+  //Check file size
+  const headerLength = Number(response.headers['content-length']);
+  if (headerLength > MAX_MEMORY_SIZE) {
+    return reject(`File size (${headerLength} bytes) too large to be handled in memory.`);
+  }
+
+  const tempFileName = `'/tmp/swtor-patcher-download-${Math.random()}.tmp`;
+
+  //If we receive a part of the response, write it to disk
+  let previousChunk: Promise<void> = new Promise((innerResolve) => { innerResolve(); });
+  let totalLength = 0;
+  response.on('data', (chunk: Buffer) => {
+    totalLength += chunk.length;
+
+    //Exit early if we received too much data
+    if (totalLength > headerLength) {
+      return reject(`Expected length ${headerLength} but received at least ${totalLength}.`);
+    }
+
+    //If previous chunk was not yet written to disk, wait until it finished to avoid a race condition
+    previousChunk.then(() => {
+      previousChunk = new Promise((innerResolve, innerReject) => {
+        //Write chunk to disk
+        fs.appendFile(tempFileName, chunk, { encoding: 'binary' }, (error) => {
+          if (error) {
+            return reject(`Could not write to disk: [${error.code}] ${error.name}: ${error.message}.`);
+          }
+          innerResolve();
+        });
+      });
+    });
+  });
+
+  //If we finished reading response, check for correctness, then return it
+  response.on('end', () => {
+    //Check that length is correct
+    if (totalLength !== headerLength) {
+      return reject(`Expected length ${headerLength} but received ${totalLength}.`);
+    }
+
+    //Return file reader
+    //TODO: need to automatically delete file once it is no longer used
+    //TODO: need to provide methods to seek through file
+    const stream = fs.createReadStream(tempFileName, { encoding: 'binary' });
+    return resolve(stream);
+  });
+}

src/ssn/getPatch.ts

@@ -1,3 +1,4 @@
+import downloadUrlContents from '../cdn/downloadUrlContents';
 import getUrlContents from '../cdn/getUrlContents';
 import { Product } from '../interfaces/ISettings';
 import { SsnDiffType } from '../interfaces/ISsnFileEntry';
@@ -9,14 +10,16 @@ export default async function getPatch(product: Product, from: number, to: numbe
   const solidpkg = await getSolidpkg(product, from, to);
   console.debug(solidpkg.files);
 
-  const downloadPromises = solidpkg.files.map((file) => getUrlContents({ host: 'cdn-patch.swtor.com', path: `/patch/${product}/${product}_${from}to${to}/${file.name}` }));
+  function createUrlObject(fileName: string) {
-  const bufferArray = await Promise.all(downloadPromises);
+    return { host: 'cdn-patch.swtor.com', path: `/patch/${product}/${product}_${from}to${to}/${fileName}` };
-  const zipFile = bufferArray[bufferArray.length - 1];
+  }
-  const dvArray = bufferArray.map((buffer) => new DataView(buffer));
 
-  //TODO: Verify that downloaded files match the hash in `solidpkg.pieces`
+  //start download, making sure that .zip file downloads first
+  const zipFile = getUrlContents(createUrlObject(solidpkg.files[solidpkg.files.length - 1].name));
+  const diskFiles = solidpkg.files.slice(0, solidpkg.files.length - 1).map((file) => downloadUrlContents(createUrlObject(file.name)));
 
-  const fileEntries = readSsnFile(zipFile);
+  //we can parse the file entries as soon as the .zip file is loaded
+  const fileEntries = readSsnFile(await zipFile);
   console.debug(fileEntries);
 
   //Verify file entries
@@ -27,16 +30,22 @@ export default async function getPatch(product: Product, from: number, to: numbe
   }
   //TODO: last file must always be `${product}.version` with diff type. Other files depend on diffType.
 
+  //Then we need to wait for other files before we can extract them
+  await Promise.all(diskFiles);
+  //const dvArray = bufferArray.map((buffer) => new DataView(buffer));
+
+  //TODO: Verify that downloaded files match the hash in `solidpkg.pieces`
+
   //Extract newly added files
-  fileEntries.filter((file) => file.diffType === SsnDiffType.NewFile).forEach(async (file) => {
+  /*fileEntries.filter((file) => file.diffType === SsnDiffType.NewFile).forEach(async (file) => {
-    const fileContents = await extractFile(file, dvArray);
+    const fileContents = await extractFile(file, diskFiles);
     console.debug(new Uint8Array(fileContents));
     //TODO
   });
 
   //Extract changed files
   fileEntries.filter((file) => file.diffType === SsnDiffType.Changed).forEach(async (file) => {
-    const fileContents = await extractFile(file, dvArray);
+    const fileContents = await extractFile(file, diskFiles);
     console.debug(new Uint8Array(fileContents));
     //TODO
   });
@@ -44,5 +53,5 @@ export default async function getPatch(product: Product, from: number, to: numbe
   //Need to delete deleted files
   fileEntries.filter((file) => file.diffType === SsnDiffType.Deleted).forEach((file) => {
     //TODO
-  });
+  });*/
 }

src/ssn/verify/verifySolidpkg.ts

@@ -28,6 +28,11 @@ export default function verifySolidpkg(file: ISolid, { product, from, to }: {pro
   if (!Array.isArray(file.info.files)) {
     throw new Error(`Expected files field to be an array but it isn't.`);
   }
+
+  if (file.info.files.length < 2) {
+    throw new Error(`Expected files array to contain at least two files but it had ${file.info.files.length} files.`);
+  }
+
   for (let i = 0, il = file.info.files.length; i < il; i += 1) {
     const fileEntry = file.info.files[i];
     if (typeof fileEntry.length !== 'number' && fileEntry.length >= 0 && fileEntry.length <= 1700000000) {
@@ -36,8 +41,15 @@ export default function verifySolidpkg(file: ISolid, { product, from, to }: {pro
     if (!Array.isArray(fileEntry.path) || fileEntry.path.length !== 1) {
       throw new Error(`Expected valid file name but it was not an array with one element.`);
     }
-    if (typeof fileEntry.path[0] !== 'string' || !fileEntry.path[0].match(new RegExp(`${product}_${from}to${to}\.z(ip|0[1-9]|[1-9][0-9])$`))) {
+    const fileName = fileEntry.path[0];
-      throw new Error(`Expected valid file name but it was ${fileEntry.path[0]}.`);
+    if (typeof fileName !== 'string') {
+      throw new Error(`Expected valid file name to be a string but it was a ${typeof fileName}.`);
+    }
+    //Last file must end with .zip, other files are called .z01, .z02 etc.
+    const validExtension = (i === il - 1) ? 'ip' : String(i + 1).padStart(2, '0');
+    const validName = `${product}_${from}to${to}.z${validExtension}`;
+    if (fileName !== validName) {
+      throw new Error(`Expected file name "${validName}" but it was "${fileName}".`);
     }
   }