🚧 Partial switch to streams, and add utility endpoints

README.md

@@ -7,12 +7,34 @@ For this tool to work, ```tsc``` and ```tslint``` must be globally available, e.
 npm install -g typescript tslint
 ```
 
-Then start it as follows:
+Transpile the TypeScript files to JavaScript by running:
 
 ```bash
-npm start && node dist/installPatch.js
+npm start
 ```
 
+# Usage
+Either directly run whatever script you need:
+
+```bash
+node dist/getManifest.js assets_swtor_main
+node dist/getSolidPkg.js assets_swtor_main 126 127
+node dist/installPatch.js
+```
+
+Or import the functions into your Node.js application:
+
+```js
+import * as ssn from './dist';
+
+(async function() {
+  const manifestContents = await ssn.getManifest('assets_swtor_main');
+  console.log(manifestContents);
+
+  const solidpkgContents = await ssn.getSolidpkg('assets_swtor_main', 126, 127);
+  console.log(solidpkgContents);
+}())
+```
 
 # Introduction
 SWTOR’s patcher is licensed from Solid State Networks, so it is mostly using code from SSN. However, SWTOR does not use the original software by SSN; instead they wrote their own patch deploy pipeline which interfaces with the SSN command line tools.

src/cdn/downloadUrlContents.ts

@@ -5,7 +5,7 @@ import saveResponse from './funcs/saveResponse';
 /** Downloads the given URL and saves it to disk. Throws error if download fails. */
 export default function downloadUrlContents({ host, path }: {host: string, path: string}): Promise<fs.ReadStream> {
   return new Promise((resolve, reject) => {
-
+    //Create HTTP request
     const request = http.request({
       family: 4,
       host,

src/cdn/funcs/resolveDns.ts

@@ -36,69 +36,69 @@ const assert = (condition: boolean) => { if (!condition) { console.warn('Assert
 /** Looks up the given domain and returns a list of IP addresses, along with their time-to-live */
 async function resolveDns(domain: string): Promise<IDnsResult[]> {
   return new Promise((resolve) => {
-      //check given string for correctness to prevent injection attacks
-      if (!domain.match(/^[a-z0-9]+([-.]{1}[a-z0-9]+)*\.[a-z]{2,3}$/)) { return resolve([]); }
+    //check given string for correctness to prevent injection attacks
+    if (!domain.match(/^[a-z0-9]+([-.]{1}[a-z0-9]+)*\.[a-z]{2,3}$/)) { return resolve([]); }
 
-      //Check Level3/North_America separately
-      if (domain !== 'cdn-patch.swtor.com') {
-          dns.resolve4(domain, { ttl: true }, (err, result) => {
-            return resolve(result.map(({ address, ttl }) => ({ address, ttl, type: 'level3-us' as IDnsResult['type'] })));
-          });
-      } else {
-          //Use bash so we get more information.
-          //Also do plenty of asserts to ensure that overall CDN structure has stayed unchanged, and TODO send e-mail if it's different (max. once per hour)
-          exec('dig +noall +answer "cdn-patch.swtor.com"', { timeout: 10000 }, (error, stdout) => {
-              //check for error
-              assert(!error);
-              if (error) {
-                return resolve([]);
-              }
+    //Check Level3/North_America separately
+    if (domain !== 'cdn-patch.swtor.com') {
+      dns.resolve4(domain, { ttl: true }, (err, result) => {
+        return resolve(result.map(({ address, ttl }) => ({ address, ttl, type: 'level3-us' as IDnsResult['type'] })));
+      });
+    } else {
+      //Use bash so we get more information.
+      //Also do plenty of asserts to ensure that overall CDN structure has stayed unchanged, and TODO send e-mail if it's different (max. once per hour)
+      exec('dig +noall +answer "cdn-patch.swtor.com"', { timeout: 10000 }, (error, stdout) => {
+        //check for error
+        assert(!error);
+        if (error) {
+          return resolve([]);
+        }
 
-              const data = stdout.trim().split('\n').map((line) => line.split(/\t| /));
+        const data = stdout.trim().split('\n').map((line) => line.split(/\t| /));
 
-              //Verify output
-              assert(data.length > 3);
-              data.forEach((dataLine) => assert(dataLine.length === 5));
-              assert(data[0][0] === 'cdn-patch.swtor.com.');
-              assert(data[0][1].match(/^[0-9]{1,3}$/) !== null); //at least up to 598
-              assert(data[0][2] === 'IN');
-              assert(data[0][3] === 'CNAME');
-              assert(data[0][4] === 'gslb-patch.swtor.biowareonline.net.');
+        //Verify output
+        assert(data.length > 3);
+        data.forEach((dataLine) => assert(dataLine.length === 5));
+        assert(data[0][0] === 'cdn-patch.swtor.com.');
+        assert(data[0][1].match(/^[0-9]{1,3}$/) !== null); //at least up to 598
+        assert(data[0][2] === 'IN');
+        assert(data[0][3] === 'CNAME');
+        assert(data[0][4] === 'gslb-patch.swtor.biowareonline.net.');
 
-              assert(data[1][0] === 'gslb-patch.swtor.biowareonline.net.');
-              assert(data[1][1].match(/^[0-9]{1,2}$/) !== null); //up to 60 seconds
-              assert(data[1][2] === 'IN');
-              assert(data[1][3] === 'CNAME');
-              assert(data[1][4] === 'cdn-patch.swtor.com.edgesuite.net.' || data[1][4] === 'cdn-patch.swtor.com.c.footprint.net.');
+        assert(data[1][0] === 'gslb-patch.swtor.biowareonline.net.');
+        assert(data[1][1].match(/^[0-9]{1,2}$/) !== null); //up to 60 seconds
+        assert(data[1][2] === 'IN');
+        assert(data[1][3] === 'CNAME');
+        assert(data[1][4] === 'cdn-patch.swtor.com.edgesuite.net.' || data[1][4] === 'cdn-patch.swtor.com.c.footprint.net.');
 
-              assert(data[2][0] === data[1][4]);
-              assert(data[2][1].match(/^[0-9]{1,5}$/) !== null); //at least up to 15092 if Akamai, at least up to 627 if Level3
-              assert(data[2][2] === 'IN');
-              assert(data[2][3] === 'CNAME');
-              assert(
-                  (data[2][4] === 'a56.d.akamai.net.' && data[1][4] === 'cdn-patch.swtor.com.edgesuite.net.') ||
-                  (data[2][4] === 'eu.lvlt.cdn.ea.com.c.footprint.net.' && data[1][4] === 'cdn-patch.swtor.com.c.footprint.net.'),
-              );
+        assert(data[2][0] === data[1][4]);
+        assert(data[2][1].match(/^[0-9]{1,5}$/) !== null); //at least up to 15092 if Akamai, at least up to 627 if Level3
+        assert(data[2][2] === 'IN');
+        assert(data[2][3] === 'CNAME');
+        assert(
+          (data[2][4] === 'a56.d.akamai.net.' && data[1][4] === 'cdn-patch.swtor.com.edgesuite.net.') ||
+          (data[2][4] === 'eu.lvlt.cdn.ea.com.c.footprint.net.' && data[1][4] === 'cdn-patch.swtor.com.c.footprint.net.'),
+        );
 
-              for (let i = 3, il = data.length; i < il; i += 1) {
-                  assert(data[i][0] === data[2][4]);
-                  assert(data[i][1].match(/^[0-9]{1,3}$/) !== null); //up to 60 seconds if Akamai, at least up to 218 if Level3
-                  assert(data[i][2] === 'IN');
-                  assert(data[i][3] === 'A');
-                  assert(data[i][4].match(/^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$/) !== null);
-              }
+        for (let i = 3, il = data.length; i < il; i += 1) {
+          assert(data[i][0] === data[2][4]);
+          assert(data[i][1].match(/^[0-9]{1,3}$/) !== null); //up to 60 seconds if Akamai, at least up to 218 if Level3
+          assert(data[i][2] === 'IN');
+          assert(data[i][3] === 'A');
+          assert(data[i][4].match(/^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$/) !== null);
+        }
 
-              //Prepare return values
-              let type: IDnsResult['type'];
-              switch (data[1][4]) {
-                  case 'cdn-patch.swtor.com.edgesuite.net.': type = 'akamai'; break;
-                  case 'cdn-patch.swtor.com.c.footprint.net.': type = 'level3-eu'; break;
-                  default: type = 'unknown';
-              }
+        //Prepare return values
+        let type: IDnsResult['type'];
+        switch (data[1][4]) {
+          case 'cdn-patch.swtor.com.edgesuite.net.': type = 'akamai'; break;
+          case 'cdn-patch.swtor.com.c.footprint.net.': type = 'level3-eu'; break;
+          default: type = 'unknown';
+        }
 
-              resolve(data.filter((e, index) => index >= 3).map(([, ttl, , , address]) => ({ ttl: Math.min(Number(ttl), Number(data[0][1]), Number(data[1][1]), Number(data[2][1])), address, type })));
-          });
-      }
+        resolve(data.filter((e, index) => index >= 3).map(([, ttl, , , address]) => ({ ttl: Math.min(Number(ttl), Number(data[0][1]), Number(data[1][1]), Number(data[2][1])), address, type })));
+      });
+    }
   }) as Promise<IDnsResult[]>;
 }
 

src/cdn/getUrlContents.ts

@@ -4,6 +4,7 @@ import handleResponse from './funcs/handleResponse';
 /** Downloads the given URL into memory and returns it as an ArrayBuffer. Throws error if download fails or file is too large to be handled in memory. */
 export default function getUrlContents({ host, path }: {host: string, path: string}): Promise<ArrayBuffer> {
   return new Promise((resolve, reject) => {
+    //Create HTTP request
     const request = http.request({
       family: 4,
       host,

src/cdn/heartbeatDns.ts

@@ -12,78 +12,78 @@ servers.push({ ip: '159.153.92.51', type: 'master', lastSeen: Infinity, weight:
 
 /** Updates the list of servers based on current DNS data */
 async function heartbeatDns(domain: string) {
-    //Get list of current patch servers
-    const dnsResults = await resolveDns(domain);
+  //Get list of current patch servers
+  const dnsResults = await resolveDns(domain);
 
-    //Remember time when response came in
-    const now = Date.now() - startTime;
+  //Remember time when response came in
+  const now = Date.now() - startTime;
 
-    //Schedule next check based on time-to-live, but never longer than 1 minute
-    const ttl = Math.min(60, ...(dnsResults.map((obj) => obj.ttl))) + 1;
-    setTimeout(heartbeatDns.bind(null, domain), ttl * 1000);
+  //Schedule next check based on time-to-live, but never longer than 1 minute
+  const ttl = Math.min(60, ...(dnsResults.map((obj) => obj.ttl))) + 1;
+  setTimeout(heartbeatDns.bind(null, domain), ttl * 1000);
 
-    //Update array with new information
-    dnsResults.forEach(
-        ({ address, type }, index) => {
-            //Calculate weight:
-            //on cdn-patch.swtor.com: 3 if first, 2 if second, otherwise 1
-            let weight = (index < 2) ? (3 - index) : 1;
-            //on Level3 US: 1.2 is first, 1 if second
-            if (domain !== 'cdn-patch.swtor.com') {
-              weight = (index === 0) ? 1.2 : 1;
-            }
+  //Update array with new information
+  dnsResults.forEach(
+    ({ address, type }, index) => {
+      //Calculate weight:
+      //on cdn-patch.swtor.com: 3 if first, 2 if second, otherwise 1
+      let weight = (index < 2) ? (3 - index) : 1;
+      //on Level3 US: 1.2 is first, 1 if second
+      if (domain !== 'cdn-patch.swtor.com') {
+        weight = (index === 0) ? 1.2 : 1;
+      }
 
-            //if ip is already contained
-            for (let i = 0, il = servers.length; i < il; i += 1) {
-                const server = servers[i];
-                if (server.ip === address) {
-                    server.lastSeen = now;
-                    server.weight += weight;
-                    if (server.type !== type) { server.type = type; }
-                    return;
-                }
-            }
-
-            //if not yet contained, add to array
-            servers.push({
-                ip: address,
-                lastSeen: now,
-                type,
-                weight: weight + 1, //give a boost to new values compared to existing values
-            });
-        },
-    );
-
-    //Remove old entries - old = not seen for one hour
-    servers = servers.filter((server) => (now - server.lastSeen) < 3600000);
-
-    //Decay weights - reduce them based on update frequency (-50% if full minute, but less if TTL was shorter than a minute)
-    const decayFactor = 0.5 ** ((now - lastUpdate) / 60000);
-    lastUpdate = now;
-    servers.forEach((server) => { server.weight *= decayFactor; });
-
-    //Sort the array by weight
-    servers.sort((a, b) => b.weight - a.weight);
-
-    //Output current list
-    let output = '';
-    servers.forEach((server) => {
-        //set colors based on server type, see https://en.wikipedia.org/wiki/ANSI_escape_code#Colors
-        //bright color if seen within last 5 minutes
-        if (now - server.lastSeen < 300000) { output += '\x1b[1m'; } else { output += '\x1b[0m'; }
-        switch (server.type) {
-            case 'master': output += '\x1b[37m'; break; //white
-            case 'akamai': output += '\x1b[35m'; break; //magenta
-            case 'level3-us': output += '\x1b[32m'; break; //green
-            case 'level3-eu': output += '\x1b[36m'; break; //cyan
-            case 'unknown': default: output += '\x1b[31m'; //red
+      //if ip is already contained
+      for (let i = 0, il = servers.length; i < il; i += 1) {
+        const server = servers[i];
+        if (server.ip === address) {
+          server.lastSeen = now;
+          server.weight += weight;
+          if (server.type !== type) { server.type = type; }
+          return;
         }
-        output += server.ip;
-        output += '\t';
-    });
-    //Reset color to default
-    output += '\x1b[0m';
-    console.log(output);
+      }
+
+      //if not yet contained, add to array
+      servers.push({
+        ip: address,
+        lastSeen: now,
+        type,
+        weight: weight + 1, //give a boost to new values compared to existing values
+      });
+    },
+  );
+
+  //Remove old entries - old = not seen for one hour
+  servers = servers.filter((server) => (now - server.lastSeen) < 3600000);
+
+  //Decay weights - reduce them based on update frequency (-50% if full minute, but less if TTL was shorter than a minute)
+  const decayFactor = 0.5 ** ((now - lastUpdate) / 60000);
+  lastUpdate = now;
+  servers.forEach((server) => { server.weight *= decayFactor; });
+
+  //Sort the array by weight
+  servers.sort((a, b) => b.weight - a.weight);
+
+  //Output current list
+  let output = '';
+  servers.forEach((server) => {
+    //set colors based on server type, see https://en.wikipedia.org/wiki/ANSI_escape_code#Colors
+    //bright color if seen within last 5 minutes
+    if (now - server.lastSeen < 300000) { output += '\x1b[1m'; } else { output += '\x1b[0m'; }
+    switch (server.type) {
+      case 'master': output += '\x1b[37m'; break; //white
+      case 'akamai': output += '\x1b[35m'; break; //magenta
+      case 'level3-us': output += '\x1b[32m'; break; //green
+      case 'level3-eu': output += '\x1b[36m'; break; //cyan
+      case 'unknown': default: output += '\x1b[31m'; //red
+    }
+    output += server.ip;
+    output += '\t';
+  });
+  //Reset color to default
+  output += '\x1b[0m';
+  console.log(output);
 }
 
 //start loading additional addresses, both from CDN, and specifically from Level3/North_America so we have more than just European servers

src/config.ts

@@ -1,60 +0,0 @@
-import { ISettings, Product } from './interfaces/ISettings';
-import verifyProductName from './ssn/verify/verifyProductName';
-
-const settings: ISettings = {};
-
-/** Sets the given setting to the given value. Throws an error if key is invalid, or value doesn't match the key. */
-export const set = (key: string, value: any) => {
-  switch (key) {
-    case 'product':
-      //TODO: need to verify input (one of allowed products)
-      if (typeof value !== 'string') { throw new Error(`product must be a string but it's a "${typeof value}".`); }
-      if (!verifyProductName(value)) { throw new Error(`"${value}" is not an allowed product.`); }
-      settings.product = value as Product;
-      break;
-    case 'release':
-      //verify input (must be a number >=0, and >settings.from)
-      if (typeof value !== 'number') { throw new Error(`release must be a number but it's a "${typeof value}".`); }
-      if ((value | 0) !== value) { throw new Error(`release must be an integer but it's ${value}.`); }
-      if (value < 0) { throw new Error(`release must be a non-negative integer but it's ${value}.`); }
-      if (settings.from !== undefined && value <= settings.from ) { throw new Error(`release must be greater than from but ${value} is not greater than ${settings.from}.`); }
-      settings.release = value;
-      break;
-    case 'from':
-      //TODO: need to verify input (it's a number >=-1 and <setting.release)
-      if (typeof value !== 'number') { throw new Error(`from must be a number but it's a "${typeof value}".`); }
-      if ((value | 0) !== value) { throw new Error(`from must be an integer but it's ${value}.`); }
-      if (value < -1) { throw new Error(`from must be a non-negative integer or -1, but it's ${value}.`); }
-      if (settings.release !== undefined && value >= settings.release ) { throw new Error(`from must be less than release but ${value} is not less than ${settings.release}.`); }
-      settings.from = value;
-      break;
-    case 'outputType':
-      //need to verify input (it's info or file)
-      if (typeof value !== 'string') { throw new Error(`outputType must be a string but it's a "${typeof value}".`); }
-      if (value !== 'info' && value !== 'file') { throw new Error(`outputType must be "info" or "file" but it's "${value}".`); }
-      settings.outputType = value;
-      break;
-    default:
-      throw new Error(`The configuration setting ${key} does not exist.`);
-  }
-};
-
-/** Verify that all required settings were set. Throws an error if not. */
-export const verify = () => {
-  if (settings.product === undefined) {
-    throw new Error('No product set.');
-  }
-  if (settings.release === undefined) {
-    throw new Error('No release set.');
-  }
-  if (settings.from === undefined) {
-    throw new Error('No from set.');
-  }
-  if (settings.outputType === undefined) {
-    throw new Error('No outputType set.');
-  }
-  return true;
-};
-
-/** Gets the value for the given setting. */
-export const get = (key: string) => settings[key];

src/getManifest.ts

@@ -0,0 +1,28 @@
+import IManifest from './interfaces/IManifest';
+import { Product } from './interfaces/ISettings';
+import getManifest from './ssn/getManifest';
+import verifyProductName from './ssn/verify/verifyProductName';
+
+function failWithError(msg?: string) {
+  if (msg !== undefined) {
+    process.stderr.write(msg);
+  }
+  process.stderr.write('Usage: node dist/getManifest.js <product>');
+  process.exit(1);
+}
+
+if (process.argv.length !== 2) {
+  failWithError(`Error: Expected 1 argument but ${process.argv.length - 1} arguments were supplied.`);
+}
+
+//Check that product name is valid
+const product = process.argv[1];
+if (!verifyProductName(product)) {
+  failWithError(`Error: "${product} is not a valid product name.`);
+}
+
+//Get manifest and write output to console
+getManifest(product as Product).then((output: IManifest) => {
+  process.stdout.write(JSON.stringify(output));
+  //process.exit(0);
+});

src/getSolidpkg.ts

@@ -0,0 +1,38 @@
+import { Product } from './interfaces/ISettings';
+import ISolidSimple from './interfaces/ISolidSimple';
+import getSolidpkg from './ssn/getSolidpkg';
+import verifyProductName from './ssn/verify/verifyProductName';
+
+function failWithError(msg?: string) {
+  if (msg !== undefined) {
+    process.stderr.write(msg);
+  }
+  process.stderr.write('Usage: node dist/getSolidpkg.js <product> <from> <to>');
+  process.exit(1);
+}
+
+if (process.argv.length !== 4) {
+  failWithError(`Error: Expected 3 arguments but ${process.argv.length - 1} arguments were supplied.`);
+}
+
+//Check that product name is valid
+const product = process.argv[1];
+if (!verifyProductName(product)) {
+  failWithError(`Error: "${product.substring(0, 300)}" is not a valid product name.`);
+}
+
+//Check that from and to are valid numbers
+const from = process.argv[4];
+const to = process.argv[3];
+if (!from.match(/^(-1|0|[1-9][0-9]{0,2})$/)) {
+  failWithError(`Error: from value "${from.substring(0, 300)}" is not a valid integer; it must be in range [-1, 999].`);
+}
+if (!to.match(/^(0|[1-9][0-9]{0,2})$/)) {
+  failWithError(`Error: to value "${to.substring(0, 300)}" is not a valid integer; it must be in range [0, 999].`);
+}
+
+//Get solidpkg and write output to console
+getSolidpkg(product as Product, Number(from), Number(to)).then((output: ISolidSimple) => {
+  process.stdout.write(JSON.stringify(output));
+  //process.exit(0);
+});

src/index.ts

@@ -1,18 +1,8 @@
-import * as config from './config';
+//Make endpoints available for import by other modules
 
-//TODO: read arguments from command-line instead: process.argv
-config.set('product', 'assets_swtor_en_us');
-config.set('release', 120);
-config.set('from', -1);
-config.set('outputType', 'info');
-//config.set('output', '/home/c3po/swtor-test/');
-config.verify();
+import findReleasePath from './ssn/findReleasePath';
+import getPatch from './ssn/getPatch';
+import getManifest from './ssn/getPatchManifest';
+import getSolidpkg from './ssn/getSolidpkg';
 
-/*TODO:
-- Configuration (which patch to download)
-    Need to install patch X for product Y, given install path Z and optionally previous version A already installed in path B.
-- File download: .solidpkg, .zip, z01
-- File verify
-- Show output: list of .tor files etc. (and also which of the files have changed)
-- File install
-*/
+export { findReleasePath, getManifest, getSolidpkg, getPatch };

src/installPatch.ts

@@ -1,9 +1,9 @@
+import getManifest from './ssn/getManifest';
 import getPatch from './ssn/getPatch';
-import getPatchmanifest from './ssn/getPatchmanifest';
 
 (async () => {
   //----- PATCHMANIFEST -----
-  //const patchmanifestJson = await getPatchmanifest('assets_swtor_de_de');
+  //const patchmanifestJson = await getManifest('assets_swtor_de_de');
   //console.log(patchmanifestJson);
 
   //----- PATCH -----

src/ssn/extractFileStream.ts

@@ -0,0 +1,46 @@
+//Similar to extractFile.ts, but instead of receiving and returning an ArrayBuffer, works with Node.js streams.
+
+import * as stream from 'stream';
+import * as zlib from 'zlib';
+import { ISsnFileEntry } from '../interfaces/ISsnFileEntry';
+import streamSetMaxLength from './streams/streamSetMaxLength';
+
+/** Extracts the file with the given metadata from the stream.
+ * The stream must already start at the .zip's local file header
+ * and must transparently span across multiple disks if necessary.
+ */
+export default function extractFileStream(file: ISsnFileEntry, inputStream: stream.Readable): stream.Readable {
+  const localFileHeader = new DataView(inputStream.read(30));
+
+  //Local file header signature
+  if (localFileHeader.getUint32(0, true) !== 0x04034B50) {
+    throw new Error('Local file header had wrong magic');
+  }
+  //All fields in the local file header are copies of the central file header, so we can skip them.
+  //FIXME: Maybe we should actually read these fields to verify that they are identical?
+  //skip 22 bytes
+  const localFilenameSize = localFileHeader.getUint16(26, true);
+  const localExtraSize = localFileHeader.getUint16(28, true);
+
+  //skip local file name and extra field
+  inputStream.read(localFilenameSize + localExtraSize);
+
+  //TODO: pipe into decryption if file is encrypted
+  const decryptTransform = new stream.Transform({
+    read() {
+      //...
+    },
+  });
+  inputStream.pipe(decryptTransform);
+
+  //TODO: pipe into decompression if file is compressed
+  const decompressTransform = zlib.createInflateRaw();
+  decryptTransform.pipe(decompressTransform);
+
+  //TODO: output file
+  return streamSetMaxLength(decompressTransform, file.size);
+
+  /*const out = new stream.Readable();
+
+  return out;*/
+}

src/ssn/getManifest.ts

@@ -1,17 +1,16 @@
-import { TextDecoder } from 'util';
 import * as xmlJs from 'xml-js';
 import getUrlContents from '../cdn/getUrlContents';
 import IManifest from '../interfaces/IManifest';
 import { Product } from '../interfaces/ISettings';
-import extractFile from './extractFile';
+import extractFileStream from './extractFileStream';
 import parsePatchmanifest from './reader/parsePatchmanifest';
 import readSsnFile from './reader/readSsnFile';
+import arrayBufferToStream from './streams/arrayBufferToStream';
+import streamToString from './streams/streamToString';
 import verifyPatchmanifest from './verify/verifyPatchmanifest';
 import verifyProductName from './verify/verifyProductName';
 
-const Decoder = new TextDecoder('utf-8');
-
-export default async function getPatchmanifest(product: Product): Promise<IManifest> {
+export default async function getManifest(product: Product): Promise<IManifest> {
   //Verify function arguments
   if (!verifyProductName(product)) {
     throw new TypeError(`"${product}" is not a valid product.`);
@@ -33,11 +32,13 @@ export default async function getPatchmanifest(product: Product): Promise<IManif
     throw new Error(`Expected .patchmanifest to contain a file called manifest.xml but it is called "${firstFile.name}".`);
   }
 
+  const stream = arrayBufferToStream(ssnFile, firstFile.offset);
+
   //Extract manifest.xml file
-  const patchmanifestFile = await extractFile(firstFile, [new DataView(ssnFile)]);
+  const patchmanifestStream = extractFileStream(firstFile, stream);
 
   //Convert ArrayBuffer to string
-  const patchmanifestXml = Decoder.decode(patchmanifestFile);
+  const patchmanifestXml = await streamToString(patchmanifestStream);
 
   //convert XML to JSON-converted XML
   const patchManifestJson = xmlJs.xml2js(patchmanifestXml) as xmlJs.Element;

src/ssn/getPatch.ts

@@ -18,14 +18,15 @@ export default async function getPatch(product: Product, from: number, to: numbe
   const zipFile = getUrlContents(createUrlObject(solidpkg.files[indexOfLastFile].name));
   const diskFiles = solidpkg.files.slice(0, indexOfLastFile).map((file) => downloadUrlContents(createUrlObject(file.name)));
 
-  //we can parse the file entries as soon as the .zip file is loaded
+  //we can parse the file entries as soon as the .zip file is downloaded
   const fileEntries = readSsnFile(await zipFile);
   console.debug(fileEntries);
 
   //Verify file entries
   verifyPatch(fileEntries, product, from);
 
-  //Then we need to wait for other files before we can extract them
+  //Then we need to wait for disks to finish download before we can extract individual files
+  //TODO: we can optimize this to already extract some files as soon as their relevant parts are downloaded
   await Promise.all(diskFiles);
   //const dvArray = bufferArray.map((buffer) => new DataView(buffer));
 

src/ssn/streams/arrayBufferToStream.ts

@@ -0,0 +1,32 @@
+import * as stream from 'stream';
+
+const BUFFER_SIZE = 16 * 1024;
+
+export default function arrayBufferToStream(arrayBuffer: ArrayBuffer, offset = 0): stream.Readable {
+  if (offset < 0 || offset + 1 >= arrayBuffer.byteLength) {
+    throw new Error('Could not convert ArrayBuffer to ReadableStream; out of bounds.');
+  }
+
+  let position = offset;
+  const outStream = new stream.Readable({
+    encoding: 'binary',
+    read(size) {
+      const chunkSize = size || BUFFER_SIZE; //TODO: we can probably remove BUFFER_SIZE
+      let needMoreData: boolean;
+      do {
+        //If end is reached
+        if (position + 1 >= arrayBuffer.byteLength) {
+          this.push(null);
+          return;
+        }
+
+        //Write chunk to stream
+        const chunk = Buffer.from(arrayBuffer, position, chunkSize);
+        position += chunk.length;
+        needMoreData = this.push(chunk);
+      } while (needMoreData);
+    },
+  });
+
+  return outStream;
+}

src/ssn/streams/streamSetMaxLength.ts

@@ -0,0 +1,35 @@
+import * as stream from 'stream';
+
+/** Takes the given ReadableStream and returns a ReadableStream with the same contents but that terminates after the given length. */
+export default function streamSetMaxLength(inputStream: stream.Readable, maxLength: number): stream.Readable {
+  if (maxLength <= 0) {
+    throw new Error('maxLength is out of bounds.');
+  }
+
+  let remaining = maxLength;
+
+  const outStream = new stream.Readable({
+    encoding: 'binary',
+    read(size) {
+      //If no size is provided, just pass through all remaining bytes
+      if (size === undefined) {
+        this.push(inputStream.read(remaining));
+        remaining = 0;
+        //End is reached, terminate stream
+        this.push(null);
+      } else {
+        //Otherwise, pass through however many bytes we can
+        const clampedSize = Math.min(size, remaining);
+        this.push(inputStream.read(clampedSize));
+        remaining -= clampedSize;
+
+        //If end is reached, terminate stream
+        if (remaining <= 0) {
+          this.push(null);
+        }
+      }
+    },
+  });
+
+  return outStream;
+}

src/ssn/streams/streamToString.ts

@@ -0,0 +1,26 @@
+import * as stream from 'stream';
+import { TextDecoder } from 'util';
+
+const decoder = new TextDecoder('utf-8');
+
+export default function streamToString(inputStream: stream.Readable): Promise<string> {
+  return new Promise((resolve, reject) => {
+    let outputString = '';
+
+    //Convert chunks to string
+    inputStream.on('data', (chunk: Buffer) => {
+      outputString += decoder.decode(chunk, { stream: true });
+    });
+
+    //Output final string
+    inputStream.on('end', () => {
+      outputString += decoder.decode();
+      resolve(outputString);
+    });
+
+    //Exit on error
+    inputStream.on('error', (error) => {
+      reject(error);
+    });
+  });
+}

src/ssn/verify/verifyProductName.ts

@@ -1,6 +1,28 @@
 import { Product } from '../../interfaces/ISettings';
 
-const allowedProducts: Product[] = ['assets_swtor_de_de', 'assets_swtor_en_us', 'assets_swtor_fr_fr', 'assets_swtor_main', 'assets_swtor_test_de_de', 'assets_swtor_test_en_us', 'assets_swtor_test_fr_fr', 'assets_swtor_test_main', 'eualas', 'movies_de_de', 'movies_en_us', 'movies_fr_fr', 'patcher2014', 'patcher2017', 'retailclient_betatest', 'retailclient_cstraining', 'retailclient_liveeptest', 'retailclient_liveqatest', 'retailclient_publictest', 'retailclient_squadron157', 'retailclient_swtor'];
+const allowedProducts: Product[] = [
+  'assets_swtor_de_de',
+  'assets_swtor_en_us',
+  'assets_swtor_fr_fr',
+  'assets_swtor_main',
+  'assets_swtor_test_de_de',
+  'assets_swtor_test_en_us',
+  'assets_swtor_test_fr_fr',
+  'assets_swtor_test_main',
+  'eualas',
+  'movies_de_de',
+  'movies_en_us',
+  'movies_fr_fr',
+  'patcher2014',
+  'patcher2017',
+  'retailclient_betatest',
+  'retailclient_cstraining',
+  'retailclient_liveeptest',
+  'retailclient_liveqatest',
+  'retailclient_publictest',
+  'retailclient_squadron157',
+  'retailclient_swtor',
+];
 
 export default function verifyProductName(name: string) {
   return allowedProducts.includes(name as Product);