From ae2ac6d278917331136f988048cceb183503a17f Mon Sep 17 00:00:00 2001 From: C-3PO Date: Thu, 5 Jul 2018 00:58:40 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=A7=20Partial=20switch=20to=20streams,?= =?UTF-8?q?=20and=20add=20utility=20endpoints?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 26 +++- src/cdn/downloadUrlContents.ts | 2 +- src/cdn/funcs/resolveDns.ts | 110 +++++++------- src/cdn/getUrlContents.ts | 1 + src/cdn/heartbeatDns.ts | 134 +++++++++--------- src/config.ts | 60 -------- src/getManifest.ts | 28 ++++ src/getSolidpkg.ts | 38 +++++ src/index.ts | 22 +-- src/installPatch.ts | 4 +- src/ssn/extractFileStream.ts | 46 ++++++ .../{getPatchmanifest.ts => getManifest.ts} | 15 +- src/ssn/getPatch.ts | 5 +- src/ssn/streams/arrayBufferToStream.ts | 32 +++++ src/ssn/streams/streamSetMaxLength.ts | 35 +++++ src/ssn/streams/streamToString.ts | 26 ++++ src/ssn/verify/verifyProductName.ts | 24 +++- 17 files changed, 395 insertions(+), 213 deletions(-) delete mode 100644 src/config.ts create mode 100644 src/getManifest.ts create mode 100644 src/getSolidpkg.ts create mode 100644 src/ssn/extractFileStream.ts rename src/ssn/{getPatchmanifest.ts => getManifest.ts} (78%) create mode 100644 src/ssn/streams/arrayBufferToStream.ts create mode 100644 src/ssn/streams/streamSetMaxLength.ts create mode 100644 src/ssn/streams/streamToString.ts diff --git a/README.md b/README.md index 1c670ec..392a2f5 100644 --- a/README.md +++ b/README.md @@ -7,12 +7,34 @@ For this tool to work, ```tsc``` and ```tslint``` must be globally available, e. npm install -g typescript tslint ``` -Then start it as follows: +Transpile the TypeScript files to JavaScript by running: ```bash -npm start && node dist/installPatch.js +npm start ``` +# Usage +Either directly run whatever script you need: + +```bash +node dist/getManifest.js assets_swtor_main +node dist/getSolidPkg.js assets_swtor_main 126 127 +node dist/installPatch.js +``` + +Or import the functions into your Node.js application: + +```js +import * as ssn from './dist'; + +(async function() { + const manifestContents = await ssn.getManifest('assets_swtor_main'); + console.log(manifestContents); + + const solidpkgContents = await ssn.getSolidpkg('assets_swtor_main', 126, 127); + console.log(solidpkgContents); +}()) +``` # Introduction SWTOR’s patcher is licensed from Solid State Networks, so it is mostly using code from SSN. However, SWTOR does not use the original software by SSN; instead they wrote their own patch deploy pipeline which interfaces with the SSN command line tools. diff --git a/src/cdn/downloadUrlContents.ts b/src/cdn/downloadUrlContents.ts index baea55e..1cbbd63 100644 --- a/src/cdn/downloadUrlContents.ts +++ b/src/cdn/downloadUrlContents.ts @@ -5,7 +5,7 @@ import saveResponse from './funcs/saveResponse'; /** Downloads the given URL and saves it to disk. Throws error if download fails. */ export default function downloadUrlContents({ host, path }: {host: string, path: string}): Promise { return new Promise((resolve, reject) => { - + //Create HTTP request const request = http.request({ family: 4, host, diff --git a/src/cdn/funcs/resolveDns.ts b/src/cdn/funcs/resolveDns.ts index 38d2e77..b5e89d7 100644 --- a/src/cdn/funcs/resolveDns.ts +++ b/src/cdn/funcs/resolveDns.ts @@ -36,69 +36,69 @@ const assert = (condition: boolean) => { if (!condition) { console.warn('Assert /** Looks up the given domain and returns a list of IP addresses, along with their time-to-live */ async function resolveDns(domain: string): Promise { return new Promise((resolve) => { - //check given string for correctness to prevent injection attacks - if (!domain.match(/^[a-z0-9]+([-.]{1}[a-z0-9]+)*\.[a-z]{2,3}$/)) { return resolve([]); } + //check given string for correctness to prevent injection attacks + if (!domain.match(/^[a-z0-9]+([-.]{1}[a-z0-9]+)*\.[a-z]{2,3}$/)) { return resolve([]); } - //Check Level3/North_America separately - if (domain !== 'cdn-patch.swtor.com') { - dns.resolve4(domain, { ttl: true }, (err, result) => { - return resolve(result.map(({ address, ttl }) => ({ address, ttl, type: 'level3-us' as IDnsResult['type'] }))); - }); - } else { - //Use bash so we get more information. - //Also do plenty of asserts to ensure that overall CDN structure has stayed unchanged, and TODO send e-mail if it's different (max. once per hour) - exec('dig +noall +answer "cdn-patch.swtor.com"', { timeout: 10000 }, (error, stdout) => { - //check for error - assert(!error); - if (error) { - return resolve([]); - } + //Check Level3/North_America separately + if (domain !== 'cdn-patch.swtor.com') { + dns.resolve4(domain, { ttl: true }, (err, result) => { + return resolve(result.map(({ address, ttl }) => ({ address, ttl, type: 'level3-us' as IDnsResult['type'] }))); + }); + } else { + //Use bash so we get more information. + //Also do plenty of asserts to ensure that overall CDN structure has stayed unchanged, and TODO send e-mail if it's different (max. once per hour) + exec('dig +noall +answer "cdn-patch.swtor.com"', { timeout: 10000 }, (error, stdout) => { + //check for error + assert(!error); + if (error) { + return resolve([]); + } - const data = stdout.trim().split('\n').map((line) => line.split(/\t| /)); + const data = stdout.trim().split('\n').map((line) => line.split(/\t| /)); - //Verify output - assert(data.length > 3); - data.forEach((dataLine) => assert(dataLine.length === 5)); - assert(data[0][0] === 'cdn-patch.swtor.com.'); - assert(data[0][1].match(/^[0-9]{1,3}$/) !== null); //at least up to 598 - assert(data[0][2] === 'IN'); - assert(data[0][3] === 'CNAME'); - assert(data[0][4] === 'gslb-patch.swtor.biowareonline.net.'); + //Verify output + assert(data.length > 3); + data.forEach((dataLine) => assert(dataLine.length === 5)); + assert(data[0][0] === 'cdn-patch.swtor.com.'); + assert(data[0][1].match(/^[0-9]{1,3}$/) !== null); //at least up to 598 + assert(data[0][2] === 'IN'); + assert(data[0][3] === 'CNAME'); + assert(data[0][4] === 'gslb-patch.swtor.biowareonline.net.'); - assert(data[1][0] === 'gslb-patch.swtor.biowareonline.net.'); - assert(data[1][1].match(/^[0-9]{1,2}$/) !== null); //up to 60 seconds - assert(data[1][2] === 'IN'); - assert(data[1][3] === 'CNAME'); - assert(data[1][4] === 'cdn-patch.swtor.com.edgesuite.net.' || data[1][4] === 'cdn-patch.swtor.com.c.footprint.net.'); + assert(data[1][0] === 'gslb-patch.swtor.biowareonline.net.'); + assert(data[1][1].match(/^[0-9]{1,2}$/) !== null); //up to 60 seconds + assert(data[1][2] === 'IN'); + assert(data[1][3] === 'CNAME'); + assert(data[1][4] === 'cdn-patch.swtor.com.edgesuite.net.' || data[1][4] === 'cdn-patch.swtor.com.c.footprint.net.'); - assert(data[2][0] === data[1][4]); - assert(data[2][1].match(/^[0-9]{1,5}$/) !== null); //at least up to 15092 if Akamai, at least up to 627 if Level3 - assert(data[2][2] === 'IN'); - assert(data[2][3] === 'CNAME'); - assert( - (data[2][4] === 'a56.d.akamai.net.' && data[1][4] === 'cdn-patch.swtor.com.edgesuite.net.') || - (data[2][4] === 'eu.lvlt.cdn.ea.com.c.footprint.net.' && data[1][4] === 'cdn-patch.swtor.com.c.footprint.net.'), - ); + assert(data[2][0] === data[1][4]); + assert(data[2][1].match(/^[0-9]{1,5}$/) !== null); //at least up to 15092 if Akamai, at least up to 627 if Level3 + assert(data[2][2] === 'IN'); + assert(data[2][3] === 'CNAME'); + assert( + (data[2][4] === 'a56.d.akamai.net.' && data[1][4] === 'cdn-patch.swtor.com.edgesuite.net.') || + (data[2][4] === 'eu.lvlt.cdn.ea.com.c.footprint.net.' && data[1][4] === 'cdn-patch.swtor.com.c.footprint.net.'), + ); - for (let i = 3, il = data.length; i < il; i += 1) { - assert(data[i][0] === data[2][4]); - assert(data[i][1].match(/^[0-9]{1,3}$/) !== null); //up to 60 seconds if Akamai, at least up to 218 if Level3 - assert(data[i][2] === 'IN'); - assert(data[i][3] === 'A'); - assert(data[i][4].match(/^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$/) !== null); - } + for (let i = 3, il = data.length; i < il; i += 1) { + assert(data[i][0] === data[2][4]); + assert(data[i][1].match(/^[0-9]{1,3}$/) !== null); //up to 60 seconds if Akamai, at least up to 218 if Level3 + assert(data[i][2] === 'IN'); + assert(data[i][3] === 'A'); + assert(data[i][4].match(/^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$/) !== null); + } - //Prepare return values - let type: IDnsResult['type']; - switch (data[1][4]) { - case 'cdn-patch.swtor.com.edgesuite.net.': type = 'akamai'; break; - case 'cdn-patch.swtor.com.c.footprint.net.': type = 'level3-eu'; break; - default: type = 'unknown'; - } + //Prepare return values + let type: IDnsResult['type']; + switch (data[1][4]) { + case 'cdn-patch.swtor.com.edgesuite.net.': type = 'akamai'; break; + case 'cdn-patch.swtor.com.c.footprint.net.': type = 'level3-eu'; break; + default: type = 'unknown'; + } - resolve(data.filter((e, index) => index >= 3).map(([, ttl, , , address]) => ({ ttl: Math.min(Number(ttl), Number(data[0][1]), Number(data[1][1]), Number(data[2][1])), address, type }))); - }); - } + resolve(data.filter((e, index) => index >= 3).map(([, ttl, , , address]) => ({ ttl: Math.min(Number(ttl), Number(data[0][1]), Number(data[1][1]), Number(data[2][1])), address, type }))); + }); + } }) as Promise; } diff --git a/src/cdn/getUrlContents.ts b/src/cdn/getUrlContents.ts index ee1228b..efbab4a 100644 --- a/src/cdn/getUrlContents.ts +++ b/src/cdn/getUrlContents.ts @@ -4,6 +4,7 @@ import handleResponse from './funcs/handleResponse'; /** Downloads the given URL into memory and returns it as an ArrayBuffer. Throws error if download fails or file is too large to be handled in memory. */ export default function getUrlContents({ host, path }: {host: string, path: string}): Promise { return new Promise((resolve, reject) => { + //Create HTTP request const request = http.request({ family: 4, host, diff --git a/src/cdn/heartbeatDns.ts b/src/cdn/heartbeatDns.ts index cc23a11..cd47ef5 100644 --- a/src/cdn/heartbeatDns.ts +++ b/src/cdn/heartbeatDns.ts @@ -12,78 +12,78 @@ servers.push({ ip: '159.153.92.51', type: 'master', lastSeen: Infinity, weight: /** Updates the list of servers based on current DNS data */ async function heartbeatDns(domain: string) { - //Get list of current patch servers - const dnsResults = await resolveDns(domain); + //Get list of current patch servers + const dnsResults = await resolveDns(domain); - //Remember time when response came in - const now = Date.now() - startTime; + //Remember time when response came in + const now = Date.now() - startTime; - //Schedule next check based on time-to-live, but never longer than 1 minute - const ttl = Math.min(60, ...(dnsResults.map((obj) => obj.ttl))) + 1; - setTimeout(heartbeatDns.bind(null, domain), ttl * 1000); + //Schedule next check based on time-to-live, but never longer than 1 minute + const ttl = Math.min(60, ...(dnsResults.map((obj) => obj.ttl))) + 1; + setTimeout(heartbeatDns.bind(null, domain), ttl * 1000); - //Update array with new information - dnsResults.forEach( - ({ address, type }, index) => { - //Calculate weight: - //on cdn-patch.swtor.com: 3 if first, 2 if second, otherwise 1 - let weight = (index < 2) ? (3 - index) : 1; - //on Level3 US: 1.2 is first, 1 if second - if (domain !== 'cdn-patch.swtor.com') { - weight = (index === 0) ? 1.2 : 1; - } + //Update array with new information + dnsResults.forEach( + ({ address, type }, index) => { + //Calculate weight: + //on cdn-patch.swtor.com: 3 if first, 2 if second, otherwise 1 + let weight = (index < 2) ? (3 - index) : 1; + //on Level3 US: 1.2 is first, 1 if second + if (domain !== 'cdn-patch.swtor.com') { + weight = (index === 0) ? 1.2 : 1; + } - //if ip is already contained - for (let i = 0, il = servers.length; i < il; i += 1) { - const server = servers[i]; - if (server.ip === address) { - server.lastSeen = now; - server.weight += weight; - if (server.type !== type) { server.type = type; } - return; - } - } - - //if not yet contained, add to array - servers.push({ - ip: address, - lastSeen: now, - type, - weight: weight + 1, //give a boost to new values compared to existing values - }); - }, - ); - - //Remove old entries - old = not seen for one hour - servers = servers.filter((server) => (now - server.lastSeen) < 3600000); - - //Decay weights - reduce them based on update frequency (-50% if full minute, but less if TTL was shorter than a minute) - const decayFactor = 0.5 ** ((now - lastUpdate) / 60000); - lastUpdate = now; - servers.forEach((server) => { server.weight *= decayFactor; }); - - //Sort the array by weight - servers.sort((a, b) => b.weight - a.weight); - - //Output current list - let output = ''; - servers.forEach((server) => { - //set colors based on server type, see https://en.wikipedia.org/wiki/ANSI_escape_code#Colors - //bright color if seen within last 5 minutes - if (now - server.lastSeen < 300000) { output += '\x1b[1m'; } else { output += '\x1b[0m'; } - switch (server.type) { - case 'master': output += '\x1b[37m'; break; //white - case 'akamai': output += '\x1b[35m'; break; //magenta - case 'level3-us': output += '\x1b[32m'; break; //green - case 'level3-eu': output += '\x1b[36m'; break; //cyan - case 'unknown': default: output += '\x1b[31m'; //red + //if ip is already contained + for (let i = 0, il = servers.length; i < il; i += 1) { + const server = servers[i]; + if (server.ip === address) { + server.lastSeen = now; + server.weight += weight; + if (server.type !== type) { server.type = type; } + return; } - output += server.ip; - output += '\t'; - }); - //Reset color to default - output += '\x1b[0m'; - console.log(output); + } + + //if not yet contained, add to array + servers.push({ + ip: address, + lastSeen: now, + type, + weight: weight + 1, //give a boost to new values compared to existing values + }); + }, + ); + + //Remove old entries - old = not seen for one hour + servers = servers.filter((server) => (now - server.lastSeen) < 3600000); + + //Decay weights - reduce them based on update frequency (-50% if full minute, but less if TTL was shorter than a minute) + const decayFactor = 0.5 ** ((now - lastUpdate) / 60000); + lastUpdate = now; + servers.forEach((server) => { server.weight *= decayFactor; }); + + //Sort the array by weight + servers.sort((a, b) => b.weight - a.weight); + + //Output current list + let output = ''; + servers.forEach((server) => { + //set colors based on server type, see https://en.wikipedia.org/wiki/ANSI_escape_code#Colors + //bright color if seen within last 5 minutes + if (now - server.lastSeen < 300000) { output += '\x1b[1m'; } else { output += '\x1b[0m'; } + switch (server.type) { + case 'master': output += '\x1b[37m'; break; //white + case 'akamai': output += '\x1b[35m'; break; //magenta + case 'level3-us': output += '\x1b[32m'; break; //green + case 'level3-eu': output += '\x1b[36m'; break; //cyan + case 'unknown': default: output += '\x1b[31m'; //red + } + output += server.ip; + output += '\t'; + }); + //Reset color to default + output += '\x1b[0m'; + console.log(output); } //start loading additional addresses, both from CDN, and specifically from Level3/North_America so we have more than just European servers diff --git a/src/config.ts b/src/config.ts deleted file mode 100644 index 36d3f4b..0000000 --- a/src/config.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { ISettings, Product } from './interfaces/ISettings'; -import verifyProductName from './ssn/verify/verifyProductName'; - -const settings: ISettings = {}; - -/** Sets the given setting to the given value. Throws an error if key is invalid, or value doesn't match the key. */ -export const set = (key: string, value: any) => { - switch (key) { - case 'product': - //TODO: need to verify input (one of allowed products) - if (typeof value !== 'string') { throw new Error(`product must be a string but it's a "${typeof value}".`); } - if (!verifyProductName(value)) { throw new Error(`"${value}" is not an allowed product.`); } - settings.product = value as Product; - break; - case 'release': - //verify input (must be a number >=0, and >settings.from) - if (typeof value !== 'number') { throw new Error(`release must be a number but it's a "${typeof value}".`); } - if ((value | 0) !== value) { throw new Error(`release must be an integer but it's ${value}.`); } - if (value < 0) { throw new Error(`release must be a non-negative integer but it's ${value}.`); } - if (settings.from !== undefined && value <= settings.from ) { throw new Error(`release must be greater than from but ${value} is not greater than ${settings.from}.`); } - settings.release = value; - break; - case 'from': - //TODO: need to verify input (it's a number >=-1 and = settings.release ) { throw new Error(`from must be less than release but ${value} is not less than ${settings.release}.`); } - settings.from = value; - break; - case 'outputType': - //need to verify input (it's info or file) - if (typeof value !== 'string') { throw new Error(`outputType must be a string but it's a "${typeof value}".`); } - if (value !== 'info' && value !== 'file') { throw new Error(`outputType must be "info" or "file" but it's "${value}".`); } - settings.outputType = value; - break; - default: - throw new Error(`The configuration setting ${key} does not exist.`); - } -}; - -/** Verify that all required settings were set. Throws an error if not. */ -export const verify = () => { - if (settings.product === undefined) { - throw new Error('No product set.'); - } - if (settings.release === undefined) { - throw new Error('No release set.'); - } - if (settings.from === undefined) { - throw new Error('No from set.'); - } - if (settings.outputType === undefined) { - throw new Error('No outputType set.'); - } - return true; -}; - -/** Gets the value for the given setting. */ -export const get = (key: string) => settings[key]; diff --git a/src/getManifest.ts b/src/getManifest.ts new file mode 100644 index 0000000..4153ca7 --- /dev/null +++ b/src/getManifest.ts @@ -0,0 +1,28 @@ +import IManifest from './interfaces/IManifest'; +import { Product } from './interfaces/ISettings'; +import getManifest from './ssn/getManifest'; +import verifyProductName from './ssn/verify/verifyProductName'; + +function failWithError(msg?: string) { + if (msg !== undefined) { + process.stderr.write(msg); + } + process.stderr.write('Usage: node dist/getManifest.js '); + process.exit(1); +} + +if (process.argv.length !== 2) { + failWithError(`Error: Expected 1 argument but ${process.argv.length - 1} arguments were supplied.`); +} + +//Check that product name is valid +const product = process.argv[1]; +if (!verifyProductName(product)) { + failWithError(`Error: "${product} is not a valid product name.`); +} + +//Get manifest and write output to console +getManifest(product as Product).then((output: IManifest) => { + process.stdout.write(JSON.stringify(output)); + //process.exit(0); +}); diff --git a/src/getSolidpkg.ts b/src/getSolidpkg.ts new file mode 100644 index 0000000..442b2d7 --- /dev/null +++ b/src/getSolidpkg.ts @@ -0,0 +1,38 @@ +import { Product } from './interfaces/ISettings'; +import ISolidSimple from './interfaces/ISolidSimple'; +import getSolidpkg from './ssn/getSolidpkg'; +import verifyProductName from './ssn/verify/verifyProductName'; + +function failWithError(msg?: string) { + if (msg !== undefined) { + process.stderr.write(msg); + } + process.stderr.write('Usage: node dist/getSolidpkg.js '); + process.exit(1); +} + +if (process.argv.length !== 4) { + failWithError(`Error: Expected 3 arguments but ${process.argv.length - 1} arguments were supplied.`); +} + +//Check that product name is valid +const product = process.argv[1]; +if (!verifyProductName(product)) { + failWithError(`Error: "${product.substring(0, 300)}" is not a valid product name.`); +} + +//Check that from and to are valid numbers +const from = process.argv[4]; +const to = process.argv[3]; +if (!from.match(/^(-1|0|[1-9][0-9]{0,2})$/)) { + failWithError(`Error: from value "${from.substring(0, 300)}" is not a valid integer; it must be in range [-1, 999].`); +} +if (!to.match(/^(0|[1-9][0-9]{0,2})$/)) { + failWithError(`Error: to value "${to.substring(0, 300)}" is not a valid integer; it must be in range [0, 999].`); +} + +//Get solidpkg and write output to console +getSolidpkg(product as Product, Number(from), Number(to)).then((output: ISolidSimple) => { + process.stdout.write(JSON.stringify(output)); + //process.exit(0); +}); diff --git a/src/index.ts b/src/index.ts index 3455040..df05768 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,18 +1,8 @@ -import * as config from './config'; +//Make endpoints available for import by other modules -//TODO: read arguments from command-line instead: process.argv -config.set('product', 'assets_swtor_en_us'); -config.set('release', 120); -config.set('from', -1); -config.set('outputType', 'info'); -//config.set('output', '/home/c3po/swtor-test/'); -config.verify(); +import findReleasePath from './ssn/findReleasePath'; +import getPatch from './ssn/getPatch'; +import getManifest from './ssn/getPatchManifest'; +import getSolidpkg from './ssn/getSolidpkg'; -/*TODO: -- Configuration (which patch to download) - Need to install patch X for product Y, given install path Z and optionally previous version A already installed in path B. -- File download: .solidpkg, .zip, z01 -- File verify -- Show output: list of .tor files etc. (and also which of the files have changed) -- File install -*/ +export { findReleasePath, getManifest, getSolidpkg, getPatch }; diff --git a/src/installPatch.ts b/src/installPatch.ts index 9fa9bba..dcc4ef3 100644 --- a/src/installPatch.ts +++ b/src/installPatch.ts @@ -1,9 +1,9 @@ +import getManifest from './ssn/getManifest'; import getPatch from './ssn/getPatch'; -import getPatchmanifest from './ssn/getPatchmanifest'; (async () => { //----- PATCHMANIFEST ----- - //const patchmanifestJson = await getPatchmanifest('assets_swtor_de_de'); + //const patchmanifestJson = await getManifest('assets_swtor_de_de'); //console.log(patchmanifestJson); //----- PATCH ----- diff --git a/src/ssn/extractFileStream.ts b/src/ssn/extractFileStream.ts new file mode 100644 index 0000000..0f6a37f --- /dev/null +++ b/src/ssn/extractFileStream.ts @@ -0,0 +1,46 @@ +//Similar to extractFile.ts, but instead of receiving and returning an ArrayBuffer, works with Node.js streams. + +import * as stream from 'stream'; +import * as zlib from 'zlib'; +import { ISsnFileEntry } from '../interfaces/ISsnFileEntry'; +import streamSetMaxLength from './streams/streamSetMaxLength'; + +/** Extracts the file with the given metadata from the stream. + * The stream must already start at the .zip's local file header + * and must transparently span across multiple disks if necessary. + */ +export default function extractFileStream(file: ISsnFileEntry, inputStream: stream.Readable): stream.Readable { + const localFileHeader = new DataView(inputStream.read(30)); + + //Local file header signature + if (localFileHeader.getUint32(0, true) !== 0x04034B50) { + throw new Error('Local file header had wrong magic'); + } + //All fields in the local file header are copies of the central file header, so we can skip them. + //FIXME: Maybe we should actually read these fields to verify that they are identical? + //skip 22 bytes + const localFilenameSize = localFileHeader.getUint16(26, true); + const localExtraSize = localFileHeader.getUint16(28, true); + + //skip local file name and extra field + inputStream.read(localFilenameSize + localExtraSize); + + //TODO: pipe into decryption if file is encrypted + const decryptTransform = new stream.Transform({ + read() { + //... + }, + }); + inputStream.pipe(decryptTransform); + + //TODO: pipe into decompression if file is compressed + const decompressTransform = zlib.createInflateRaw(); + decryptTransform.pipe(decompressTransform); + + //TODO: output file + return streamSetMaxLength(decompressTransform, file.size); + + /*const out = new stream.Readable(); + + return out;*/ +} diff --git a/src/ssn/getPatchmanifest.ts b/src/ssn/getManifest.ts similarity index 78% rename from src/ssn/getPatchmanifest.ts rename to src/ssn/getManifest.ts index b695388..e0e1abb 100644 --- a/src/ssn/getPatchmanifest.ts +++ b/src/ssn/getManifest.ts @@ -1,17 +1,16 @@ -import { TextDecoder } from 'util'; import * as xmlJs from 'xml-js'; import getUrlContents from '../cdn/getUrlContents'; import IManifest from '../interfaces/IManifest'; import { Product } from '../interfaces/ISettings'; -import extractFile from './extractFile'; +import extractFileStream from './extractFileStream'; import parsePatchmanifest from './reader/parsePatchmanifest'; import readSsnFile from './reader/readSsnFile'; +import arrayBufferToStream from './streams/arrayBufferToStream'; +import streamToString from './streams/streamToString'; import verifyPatchmanifest from './verify/verifyPatchmanifest'; import verifyProductName from './verify/verifyProductName'; -const Decoder = new TextDecoder('utf-8'); - -export default async function getPatchmanifest(product: Product): Promise { +export default async function getManifest(product: Product): Promise { //Verify function arguments if (!verifyProductName(product)) { throw new TypeError(`"${product}" is not a valid product.`); @@ -33,11 +32,13 @@ export default async function getPatchmanifest(product: Product): Promise downloadUrlContents(createUrlObject(file.name))); - //we can parse the file entries as soon as the .zip file is loaded + //we can parse the file entries as soon as the .zip file is downloaded const fileEntries = readSsnFile(await zipFile); console.debug(fileEntries); //Verify file entries verifyPatch(fileEntries, product, from); - //Then we need to wait for other files before we can extract them + //Then we need to wait for disks to finish download before we can extract individual files + //TODO: we can optimize this to already extract some files as soon as their relevant parts are downloaded await Promise.all(diskFiles); //const dvArray = bufferArray.map((buffer) => new DataView(buffer)); diff --git a/src/ssn/streams/arrayBufferToStream.ts b/src/ssn/streams/arrayBufferToStream.ts new file mode 100644 index 0000000..bd4066f --- /dev/null +++ b/src/ssn/streams/arrayBufferToStream.ts @@ -0,0 +1,32 @@ +import * as stream from 'stream'; + +const BUFFER_SIZE = 16 * 1024; + +export default function arrayBufferToStream(arrayBuffer: ArrayBuffer, offset = 0): stream.Readable { + if (offset < 0 || offset + 1 >= arrayBuffer.byteLength) { + throw new Error('Could not convert ArrayBuffer to ReadableStream; out of bounds.'); + } + + let position = offset; + const outStream = new stream.Readable({ + encoding: 'binary', + read(size) { + const chunkSize = size || BUFFER_SIZE; //TODO: we can probably remove BUFFER_SIZE + let needMoreData: boolean; + do { + //If end is reached + if (position + 1 >= arrayBuffer.byteLength) { + this.push(null); + return; + } + + //Write chunk to stream + const chunk = Buffer.from(arrayBuffer, position, chunkSize); + position += chunk.length; + needMoreData = this.push(chunk); + } while (needMoreData); + }, + }); + + return outStream; +} diff --git a/src/ssn/streams/streamSetMaxLength.ts b/src/ssn/streams/streamSetMaxLength.ts new file mode 100644 index 0000000..0270349 --- /dev/null +++ b/src/ssn/streams/streamSetMaxLength.ts @@ -0,0 +1,35 @@ +import * as stream from 'stream'; + +/** Takes the given ReadableStream and returns a ReadableStream with the same contents but that terminates after the given length. */ +export default function streamSetMaxLength(inputStream: stream.Readable, maxLength: number): stream.Readable { + if (maxLength <= 0) { + throw new Error('maxLength is out of bounds.'); + } + + let remaining = maxLength; + + const outStream = new stream.Readable({ + encoding: 'binary', + read(size) { + //If no size is provided, just pass through all remaining bytes + if (size === undefined) { + this.push(inputStream.read(remaining)); + remaining = 0; + //End is reached, terminate stream + this.push(null); + } else { + //Otherwise, pass through however many bytes we can + const clampedSize = Math.min(size, remaining); + this.push(inputStream.read(clampedSize)); + remaining -= clampedSize; + + //If end is reached, terminate stream + if (remaining <= 0) { + this.push(null); + } + } + }, + }); + + return outStream; +} diff --git a/src/ssn/streams/streamToString.ts b/src/ssn/streams/streamToString.ts new file mode 100644 index 0000000..5f77126 --- /dev/null +++ b/src/ssn/streams/streamToString.ts @@ -0,0 +1,26 @@ +import * as stream from 'stream'; +import { TextDecoder } from 'util'; + +const decoder = new TextDecoder('utf-8'); + +export default function streamToString(inputStream: stream.Readable): Promise { + return new Promise((resolve, reject) => { + let outputString = ''; + + //Convert chunks to string + inputStream.on('data', (chunk: Buffer) => { + outputString += decoder.decode(chunk, { stream: true }); + }); + + //Output final string + inputStream.on('end', () => { + outputString += decoder.decode(); + resolve(outputString); + }); + + //Exit on error + inputStream.on('error', (error) => { + reject(error); + }); + }); +} diff --git a/src/ssn/verify/verifyProductName.ts b/src/ssn/verify/verifyProductName.ts index 4737a39..c8d70d6 100644 --- a/src/ssn/verify/verifyProductName.ts +++ b/src/ssn/verify/verifyProductName.ts @@ -1,6 +1,28 @@ import { Product } from '../../interfaces/ISettings'; -const allowedProducts: Product[] = ['assets_swtor_de_de', 'assets_swtor_en_us', 'assets_swtor_fr_fr', 'assets_swtor_main', 'assets_swtor_test_de_de', 'assets_swtor_test_en_us', 'assets_swtor_test_fr_fr', 'assets_swtor_test_main', 'eualas', 'movies_de_de', 'movies_en_us', 'movies_fr_fr', 'patcher2014', 'patcher2017', 'retailclient_betatest', 'retailclient_cstraining', 'retailclient_liveeptest', 'retailclient_liveqatest', 'retailclient_publictest', 'retailclient_squadron157', 'retailclient_swtor']; +const allowedProducts: Product[] = [ + 'assets_swtor_de_de', + 'assets_swtor_en_us', + 'assets_swtor_fr_fr', + 'assets_swtor_main', + 'assets_swtor_test_de_de', + 'assets_swtor_test_en_us', + 'assets_swtor_test_fr_fr', + 'assets_swtor_test_main', + 'eualas', + 'movies_de_de', + 'movies_en_us', + 'movies_fr_fr', + 'patcher2014', + 'patcher2017', + 'retailclient_betatest', + 'retailclient_cstraining', + 'retailclient_liveeptest', + 'retailclient_liveqatest', + 'retailclient_publictest', + 'retailclient_squadron157', + 'retailclient_swtor', +]; export default function verifyProductName(name: string) { return allowedProducts.includes(name as Product);