🚧 Partial switch to streams, and add utility endpoints

README.md

@@ -7,12 +7,34 @@ For this tool to work, ```tsc``` and ```tslint``` must be globally available, e.
 npm install -g typescript tslint
 ```
 
-Then start it as follows:
+Transpile the TypeScript files to JavaScript by running:
 
 ```bash
-npm start && node dist/installPatch.js
+npm start
 ```
 
+# Usage
+Either directly run whatever script you need:
+
+```bash
+node dist/getManifest.js assets_swtor_main
+node dist/getSolidPkg.js assets_swtor_main 126 127
+node dist/installPatch.js
+```
+
+Or import the functions into your Node.js application:
+
+```js
+import * as ssn from './dist';
+
+(async function() {
+  const manifestContents = await ssn.getManifest('assets_swtor_main');
+  console.log(manifestContents);
+
+  const solidpkgContents = await ssn.getSolidpkg('assets_swtor_main', 126, 127);
+  console.log(solidpkgContents);
+}())
+```
 
 # Introduction
 SWTOR’s patcher is licensed from Solid State Networks, so it is mostly using code from SSN. However, SWTOR does not use the original software by SSN; instead they wrote their own patch deploy pipeline which interfaces with the SSN command line tools.

src/cdn/downloadUrlContents.ts

@@ -5,7 +5,7 @@ import saveResponse from './funcs/saveResponse';
 /** Downloads the given URL and saves it to disk. Throws error if download fails. */
 export default function downloadUrlContents({ host, path }: {host: string, path: string}): Promise<fs.ReadStream> {
   return new Promise((resolve, reject) => {
-
+    //Create HTTP request
     const request = http.request({
       family: 4,
       host,

src/cdn/getUrlContents.ts

@@ -4,6 +4,7 @@ import handleResponse from './funcs/handleResponse';
 /** Downloads the given URL into memory and returns it as an ArrayBuffer. Throws error if download fails or file is too large to be handled in memory. */
 export default function getUrlContents({ host, path }: {host: string, path: string}): Promise<ArrayBuffer> {
   return new Promise((resolve, reject) => {
+    //Create HTTP request
     const request = http.request({
       family: 4,
       host,

src/config.ts

@@ -1,60 +0,0 @@
-import { ISettings, Product } from './interfaces/ISettings';
-import verifyProductName from './ssn/verify/verifyProductName';
-
-const settings: ISettings = {};
-
-/** Sets the given setting to the given value. Throws an error if key is invalid, or value doesn't match the key. */
-export const set = (key: string, value: any) => {
-  switch (key) {
-    case 'product':
-      //TODO: need to verify input (one of allowed products)
-      if (typeof value !== 'string') { throw new Error(`product must be a string but it's a "${typeof value}".`); }
-      if (!verifyProductName(value)) { throw new Error(`"${value}" is not an allowed product.`); }
-      settings.product = value as Product;
-      break;
-    case 'release':
-      //verify input (must be a number >=0, and >settings.from)
-      if (typeof value !== 'number') { throw new Error(`release must be a number but it's a "${typeof value}".`); }
-      if ((value | 0) !== value) { throw new Error(`release must be an integer but it's ${value}.`); }
-      if (value < 0) { throw new Error(`release must be a non-negative integer but it's ${value}.`); }
-      if (settings.from !== undefined && value <= settings.from ) { throw new Error(`release must be greater than from but ${value} is not greater than ${settings.from}.`); }
-      settings.release = value;
-      break;
-    case 'from':
-      //TODO: need to verify input (it's a number >=-1 and <setting.release)
-      if (typeof value !== 'number') { throw new Error(`from must be a number but it's a "${typeof value}".`); }
-      if ((value | 0) !== value) { throw new Error(`from must be an integer but it's ${value}.`); }
-      if (value < -1) { throw new Error(`from must be a non-negative integer or -1, but it's ${value}.`); }
-      if (settings.release !== undefined && value >= settings.release ) { throw new Error(`from must be less than release but ${value} is not less than ${settings.release}.`); }
-      settings.from = value;
-      break;
-    case 'outputType':
-      //need to verify input (it's info or file)
-      if (typeof value !== 'string') { throw new Error(`outputType must be a string but it's a "${typeof value}".`); }
-      if (value !== 'info' && value !== 'file') { throw new Error(`outputType must be "info" or "file" but it's "${value}".`); }
-      settings.outputType = value;
-      break;
-    default:
-      throw new Error(`The configuration setting ${key} does not exist.`);
-  }
-};
-
-/** Verify that all required settings were set. Throws an error if not. */
-export const verify = () => {
-  if (settings.product === undefined) {
-    throw new Error('No product set.');
-  }
-  if (settings.release === undefined) {
-    throw new Error('No release set.');
-  }
-  if (settings.from === undefined) {
-    throw new Error('No from set.');
-  }
-  if (settings.outputType === undefined) {
-    throw new Error('No outputType set.');
-  }
-  return true;
-};
-
-/** Gets the value for the given setting. */
-export const get = (key: string) => settings[key];

src/getManifest.ts

@@ -0,0 +1,28 @@
+import IManifest from './interfaces/IManifest';
+import { Product } from './interfaces/ISettings';
+import getManifest from './ssn/getManifest';
+import verifyProductName from './ssn/verify/verifyProductName';
+
+function failWithError(msg?: string) {
+  if (msg !== undefined) {
+    process.stderr.write(msg);
+  }
+  process.stderr.write('Usage: node dist/getManifest.js <product>');
+  process.exit(1);
+}
+
+if (process.argv.length !== 2) {
+  failWithError(`Error: Expected 1 argument but ${process.argv.length - 1} arguments were supplied.`);
+}
+
+//Check that product name is valid
+const product = process.argv[1];
+if (!verifyProductName(product)) {
+  failWithError(`Error: "${product} is not a valid product name.`);
+}
+
+//Get manifest and write output to console
+getManifest(product as Product).then((output: IManifest) => {
+  process.stdout.write(JSON.stringify(output));
+  //process.exit(0);
+});

src/getSolidpkg.ts

@@ -0,0 +1,38 @@
+import { Product } from './interfaces/ISettings';
+import ISolidSimple from './interfaces/ISolidSimple';
+import getSolidpkg from './ssn/getSolidpkg';
+import verifyProductName from './ssn/verify/verifyProductName';
+
+function failWithError(msg?: string) {
+  if (msg !== undefined) {
+    process.stderr.write(msg);
+  }
+  process.stderr.write('Usage: node dist/getSolidpkg.js <product> <from> <to>');
+  process.exit(1);
+}
+
+if (process.argv.length !== 4) {
+  failWithError(`Error: Expected 3 arguments but ${process.argv.length - 1} arguments were supplied.`);
+}
+
+//Check that product name is valid
+const product = process.argv[1];
+if (!verifyProductName(product)) {
+  failWithError(`Error: "${product.substring(0, 300)}" is not a valid product name.`);
+}
+
+//Check that from and to are valid numbers
+const from = process.argv[4];
+const to = process.argv[3];
+if (!from.match(/^(-1|0|[1-9][0-9]{0,2})$/)) {
+  failWithError(`Error: from value "${from.substring(0, 300)}" is not a valid integer; it must be in range [-1, 999].`);
+}
+if (!to.match(/^(0|[1-9][0-9]{0,2})$/)) {
+  failWithError(`Error: to value "${to.substring(0, 300)}" is not a valid integer; it must be in range [0, 999].`);
+}
+
+//Get solidpkg and write output to console
+getSolidpkg(product as Product, Number(from), Number(to)).then((output: ISolidSimple) => {
+  process.stdout.write(JSON.stringify(output));
+  //process.exit(0);
+});

src/index.ts

@@ -1,18 +1,8 @@
-import * as config from './config';
+//Make endpoints available for import by other modules
 
-//TODO: read arguments from command-line instead: process.argv
+import findReleasePath from './ssn/findReleasePath';
-config.set('product', 'assets_swtor_en_us');
+import getPatch from './ssn/getPatch';
-config.set('release', 120);
+import getManifest from './ssn/getPatchManifest';
-config.set('from', -1);
+import getSolidpkg from './ssn/getSolidpkg';
-config.set('outputType', 'info');
-//config.set('output', '/home/c3po/swtor-test/');
-config.verify();
 
-/*TODO:
+export { findReleasePath, getManifest, getSolidpkg, getPatch };
-- Configuration (which patch to download)
-    Need to install patch X for product Y, given install path Z and optionally previous version A already installed in path B.
-- File download: .solidpkg, .zip, z01
-- File verify
-- Show output: list of .tor files etc. (and also which of the files have changed)
-- File install
-*/

src/installPatch.ts

@@ -1,9 +1,9 @@
+import getManifest from './ssn/getManifest';
 import getPatch from './ssn/getPatch';
-import getPatchmanifest from './ssn/getPatchmanifest';
 
 (async () => {
   //----- PATCHMANIFEST -----
-  //const patchmanifestJson = await getPatchmanifest('assets_swtor_de_de');
+  //const patchmanifestJson = await getManifest('assets_swtor_de_de');
   //console.log(patchmanifestJson);
 
   //----- PATCH -----

src/ssn/extractFileStream.ts

@@ -0,0 +1,46 @@
+//Similar to extractFile.ts, but instead of receiving and returning an ArrayBuffer, works with Node.js streams.
+
+import * as stream from 'stream';
+import * as zlib from 'zlib';
+import { ISsnFileEntry } from '../interfaces/ISsnFileEntry';
+import streamSetMaxLength from './streams/streamSetMaxLength';
+
+/** Extracts the file with the given metadata from the stream.
+ * The stream must already start at the .zip's local file header
+ * and must transparently span across multiple disks if necessary.
+ */
+export default function extractFileStream(file: ISsnFileEntry, inputStream: stream.Readable): stream.Readable {
+  const localFileHeader = new DataView(inputStream.read(30));
+
+  //Local file header signature
+  if (localFileHeader.getUint32(0, true) !== 0x04034B50) {
+    throw new Error('Local file header had wrong magic');
+  }
+  //All fields in the local file header are copies of the central file header, so we can skip them.
+  //FIXME: Maybe we should actually read these fields to verify that they are identical?
+  //skip 22 bytes
+  const localFilenameSize = localFileHeader.getUint16(26, true);
+  const localExtraSize = localFileHeader.getUint16(28, true);
+
+  //skip local file name and extra field
+  inputStream.read(localFilenameSize + localExtraSize);
+
+  //TODO: pipe into decryption if file is encrypted
+  const decryptTransform = new stream.Transform({
+    read() {
+      //...
+    },
+  });
+  inputStream.pipe(decryptTransform);
+
+  //TODO: pipe into decompression if file is compressed
+  const decompressTransform = zlib.createInflateRaw();
+  decryptTransform.pipe(decompressTransform);
+
+  //TODO: output file
+  return streamSetMaxLength(decompressTransform, file.size);
+
+  /*const out = new stream.Readable();
+
+  return out;*/
+}

src/ssn/getManifest.ts

@@ -1,17 +1,16 @@
-import { TextDecoder } from 'util';
 import * as xmlJs from 'xml-js';
 import getUrlContents from '../cdn/getUrlContents';
 import IManifest from '../interfaces/IManifest';
 import { Product } from '../interfaces/ISettings';
-import extractFile from './extractFile';
+import extractFileStream from './extractFileStream';
 import parsePatchmanifest from './reader/parsePatchmanifest';
 import readSsnFile from './reader/readSsnFile';
+import arrayBufferToStream from './streams/arrayBufferToStream';
+import streamToString from './streams/streamToString';
 import verifyPatchmanifest from './verify/verifyPatchmanifest';
 import verifyProductName from './verify/verifyProductName';
 
-const Decoder = new TextDecoder('utf-8');
+export default async function getManifest(product: Product): Promise<IManifest> {
-
-export default async function getPatchmanifest(product: Product): Promise<IManifest> {
   //Verify function arguments
   if (!verifyProductName(product)) {
     throw new TypeError(`"${product}" is not a valid product.`);
@@ -33,11 +32,13 @@ export default async function getPatchmanifest(product: Product): Promise<IManif
     throw new Error(`Expected .patchmanifest to contain a file called manifest.xml but it is called "${firstFile.name}".`);
   }
 
+  const stream = arrayBufferToStream(ssnFile, firstFile.offset);
+
   //Extract manifest.xml file
-  const patchmanifestFile = await extractFile(firstFile, [new DataView(ssnFile)]);
+  const patchmanifestStream = extractFileStream(firstFile, stream);
 
   //Convert ArrayBuffer to string
-  const patchmanifestXml = Decoder.decode(patchmanifestFile);
+  const patchmanifestXml = await streamToString(patchmanifestStream);
 
   //convert XML to JSON-converted XML
   const patchManifestJson = xmlJs.xml2js(patchmanifestXml) as xmlJs.Element;

src/ssn/getPatch.ts

@@ -18,14 +18,15 @@ export default async function getPatch(product: Product, from: number, to: numbe
   const zipFile = getUrlContents(createUrlObject(solidpkg.files[indexOfLastFile].name));
   const diskFiles = solidpkg.files.slice(0, indexOfLastFile).map((file) => downloadUrlContents(createUrlObject(file.name)));
 
-  //we can parse the file entries as soon as the .zip file is loaded
+  //we can parse the file entries as soon as the .zip file is downloaded
   const fileEntries = readSsnFile(await zipFile);
   console.debug(fileEntries);
 
   //Verify file entries
   verifyPatch(fileEntries, product, from);
 
-  //Then we need to wait for other files before we can extract them
+  //Then we need to wait for disks to finish download before we can extract individual files
+  //TODO: we can optimize this to already extract some files as soon as their relevant parts are downloaded
   await Promise.all(diskFiles);
   //const dvArray = bufferArray.map((buffer) => new DataView(buffer));
 

src/ssn/streams/arrayBufferToStream.ts

@@ -0,0 +1,32 @@
+import * as stream from 'stream';
+
+const BUFFER_SIZE = 16 * 1024;
+
+export default function arrayBufferToStream(arrayBuffer: ArrayBuffer, offset = 0): stream.Readable {
+  if (offset < 0 || offset + 1 >= arrayBuffer.byteLength) {
+    throw new Error('Could not convert ArrayBuffer to ReadableStream; out of bounds.');
+  }
+
+  let position = offset;
+  const outStream = new stream.Readable({
+    encoding: 'binary',
+    read(size) {
+      const chunkSize = size || BUFFER_SIZE; //TODO: we can probably remove BUFFER_SIZE
+      let needMoreData: boolean;
+      do {
+        //If end is reached
+        if (position + 1 >= arrayBuffer.byteLength) {
+          this.push(null);
+          return;
+        }
+
+        //Write chunk to stream
+        const chunk = Buffer.from(arrayBuffer, position, chunkSize);
+        position += chunk.length;
+        needMoreData = this.push(chunk);
+      } while (needMoreData);
+    },
+  });
+
+  return outStream;
+}

src/ssn/streams/streamSetMaxLength.ts

@@ -0,0 +1,35 @@
+import * as stream from 'stream';
+
+/** Takes the given ReadableStream and returns a ReadableStream with the same contents but that terminates after the given length. */
+export default function streamSetMaxLength(inputStream: stream.Readable, maxLength: number): stream.Readable {
+  if (maxLength <= 0) {
+    throw new Error('maxLength is out of bounds.');
+  }
+
+  let remaining = maxLength;
+
+  const outStream = new stream.Readable({
+    encoding: 'binary',
+    read(size) {
+      //If no size is provided, just pass through all remaining bytes
+      if (size === undefined) {
+        this.push(inputStream.read(remaining));
+        remaining = 0;
+        //End is reached, terminate stream
+        this.push(null);
+      } else {
+        //Otherwise, pass through however many bytes we can
+        const clampedSize = Math.min(size, remaining);
+        this.push(inputStream.read(clampedSize));
+        remaining -= clampedSize;
+
+        //If end is reached, terminate stream
+        if (remaining <= 0) {
+          this.push(null);
+        }
+      }
+    },
+  });
+
+  return outStream;
+}

src/ssn/streams/streamToString.ts

@@ -0,0 +1,26 @@
+import * as stream from 'stream';
+import { TextDecoder } from 'util';
+
+const decoder = new TextDecoder('utf-8');
+
+export default function streamToString(inputStream: stream.Readable): Promise<string> {
+  return new Promise((resolve, reject) => {
+    let outputString = '';
+
+    //Convert chunks to string
+    inputStream.on('data', (chunk: Buffer) => {
+      outputString += decoder.decode(chunk, { stream: true });
+    });
+
+    //Output final string
+    inputStream.on('end', () => {
+      outputString += decoder.decode();
+      resolve(outputString);
+    });
+
+    //Exit on error
+    inputStream.on('error', (error) => {
+      reject(error);
+    });
+  });
+}

src/ssn/verify/verifyProductName.ts

@@ -1,6 +1,28 @@
 import { Product } from '../../interfaces/ISettings';
 
-const allowedProducts: Product[] = ['assets_swtor_de_de', 'assets_swtor_en_us', 'assets_swtor_fr_fr', 'assets_swtor_main', 'assets_swtor_test_de_de', 'assets_swtor_test_en_us', 'assets_swtor_test_fr_fr', 'assets_swtor_test_main', 'eualas', 'movies_de_de', 'movies_en_us', 'movies_fr_fr', 'patcher2014', 'patcher2017', 'retailclient_betatest', 'retailclient_cstraining', 'retailclient_liveeptest', 'retailclient_liveqatest', 'retailclient_publictest', 'retailclient_squadron157', 'retailclient_swtor'];
+const allowedProducts: Product[] = [
+  'assets_swtor_de_de',
+  'assets_swtor_en_us',
+  'assets_swtor_fr_fr',
+  'assets_swtor_main',
+  'assets_swtor_test_de_de',
+  'assets_swtor_test_en_us',
+  'assets_swtor_test_fr_fr',
+  'assets_swtor_test_main',
+  'eualas',
+  'movies_de_de',
+  'movies_en_us',
+  'movies_fr_fr',
+  'patcher2014',
+  'patcher2017',
+  'retailclient_betatest',
+  'retailclient_cstraining',
+  'retailclient_liveeptest',
+  'retailclient_liveqatest',
+  'retailclient_publictest',
+  'retailclient_squadron157',
+  'retailclient_swtor',
+];
 
 export default function verifyProductName(name: string) {
   return allowedProducts.includes(name as Product);