From e4d906e48ca1ba88d39a0379d62b13aba34b9c7e Mon Sep 17 00:00:00 2001 From: C-3PO Date: Wed, 4 Jul 2018 22:07:23 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=A7=20Write=20large=20files=20to=20dis?= =?UTF-8?q?k=20instead=20of=20to=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/cdn/downloadUrlContents.ts | 23 ++++++++++++ src/cdn/funcs/handleResponse.ts | 10 ++--- src/cdn/funcs/saveResponse.ts | 63 ++++++++++++++++++++++++++++++++ src/ssn/getPatch.ts | 29 ++++++++++----- src/ssn/verify/verifySolidpkg.ts | 16 +++++++- 5 files changed, 124 insertions(+), 17 deletions(-) create mode 100644 src/cdn/downloadUrlContents.ts create mode 100644 src/cdn/funcs/saveResponse.ts diff --git a/src/cdn/downloadUrlContents.ts b/src/cdn/downloadUrlContents.ts new file mode 100644 index 0000000..baea55e --- /dev/null +++ b/src/cdn/downloadUrlContents.ts @@ -0,0 +1,23 @@ +import * as fs from 'fs'; +import * as http from 'http'; +import saveResponse from './funcs/saveResponse'; + +/** Downloads the given URL and saves it to disk. Throws error if download fails. */ +export default function downloadUrlContents({ host, path }: {host: string, path: string}): Promise { + return new Promise((resolve, reject) => { + + const request = http.request({ + family: 4, + host, + path, + }, saveResponse.bind(null, resolve, (reason: string) => { request.abort(); reject(reason); })); + + //In case of connection errors, exit early + request.on('error', (error) => { + request.abort(); + reject(error); + }); + + request.end(); + }); +} diff --git a/src/cdn/funcs/handleResponse.ts b/src/cdn/funcs/handleResponse.ts index e58bb94..793507e 100644 --- a/src/cdn/funcs/handleResponse.ts +++ b/src/cdn/funcs/handleResponse.ts @@ -4,19 +4,19 @@ import * as http from 'http'; const MAX_MEMORY_SIZE = 100 * 1024 * 1024; export default function handleResponse( - resolve: (value: ArrayBuffer) => void, + resolve: (arrayBuffer: ArrayBuffer) => void, reject: (reason: string) => void, response: http.IncomingMessage, ) { //Check that file exists (200 HTTP status code) if (response.statusCode !== 200) { - return reject(`Expected status code 200 but received ${response.statusCode}`); + return reject(`Expected status code 200 but received ${response.statusCode}.`); } //Check file size const headerLength = Number(response.headers['content-length']); if (headerLength > MAX_MEMORY_SIZE) { - return reject('File size too large to be handled in memory.'); + return reject(`File size (${headerLength} bytes) too large to be handled in memory.`); } //If we receive a part of the response, store it @@ -27,7 +27,7 @@ export default function handleResponse( //Exit early if we received too much data if (totalLength > headerLength) { - return reject(`Expected length ${headerLength} but received at least ${totalLength}`); + return reject(`Expected length ${headerLength} but received at least ${totalLength}.`); } //Add chunk to array @@ -38,7 +38,7 @@ export default function handleResponse( response.on('end', () => { //Check that length is correct if (totalLength !== headerLength) { - return reject(`Expected length ${headerLength} but received ${totalLength}`); + return reject(`Expected length ${headerLength} but received ${totalLength}.`); } //Return file contents as ArrayBuffer diff --git a/src/cdn/funcs/saveResponse.ts b/src/cdn/funcs/saveResponse.ts new file mode 100644 index 0000000..79fed99 --- /dev/null +++ b/src/cdn/funcs/saveResponse.ts @@ -0,0 +1,63 @@ +import * as fs from 'fs'; +import * as http from 'http'; + +/** Too avoid overloading the server, do not allow large files (> 100 MB) to be handled in memory. */ +const MAX_MEMORY_SIZE = 100 * 1024 * 1024; + +export default function saveResponse( + resolve: (fileName: fs.ReadStream) => void, + reject: (reason: string) => void, + response: http.IncomingMessage, +) { + //Check that file exists (200 HTTP status code) + if (response.statusCode !== 200) { + return reject(`Expected status code 200 but received ${response.statusCode}.`); + } + + //Check file size + const headerLength = Number(response.headers['content-length']); + if (headerLength > MAX_MEMORY_SIZE) { + return reject(`File size (${headerLength} bytes) too large to be handled in memory.`); + } + + const tempFileName = `'/tmp/swtor-patcher-download-${Math.random()}.tmp`; + + //If we receive a part of the response, write it to disk + let previousChunk: Promise = new Promise((innerResolve) => { innerResolve(); }); + let totalLength = 0; + response.on('data', (chunk: Buffer) => { + totalLength += chunk.length; + + //Exit early if we received too much data + if (totalLength > headerLength) { + return reject(`Expected length ${headerLength} but received at least ${totalLength}.`); + } + + //If previous chunk was not yet written to disk, wait until it finished to avoid a race condition + previousChunk.then(() => { + previousChunk = new Promise((innerResolve, innerReject) => { + //Write chunk to disk + fs.appendFile(tempFileName, chunk, { encoding: 'binary' }, (error) => { + if (error) { + return reject(`Could not write to disk: [${error.code}] ${error.name}: ${error.message}.`); + } + innerResolve(); + }); + }); + }); + }); + + //If we finished reading response, check for correctness, then return it + response.on('end', () => { + //Check that length is correct + if (totalLength !== headerLength) { + return reject(`Expected length ${headerLength} but received ${totalLength}.`); + } + + //Return file reader + //TODO: need to automatically delete file once it is no longer used + //TODO: need to provide methods to seek through file + const stream = fs.createReadStream(tempFileName, { encoding: 'binary' }); + return resolve(stream); + }); +} diff --git a/src/ssn/getPatch.ts b/src/ssn/getPatch.ts index 88fefc0..8bccc96 100644 --- a/src/ssn/getPatch.ts +++ b/src/ssn/getPatch.ts @@ -1,3 +1,4 @@ +import downloadUrlContents from '../cdn/downloadUrlContents'; import getUrlContents from '../cdn/getUrlContents'; import { Product } from '../interfaces/ISettings'; import { SsnDiffType } from '../interfaces/ISsnFileEntry'; @@ -9,14 +10,16 @@ export default async function getPatch(product: Product, from: number, to: numbe const solidpkg = await getSolidpkg(product, from, to); console.debug(solidpkg.files); - const downloadPromises = solidpkg.files.map((file) => getUrlContents({ host: 'cdn-patch.swtor.com', path: `/patch/${product}/${product}_${from}to${to}/${file.name}` })); - const bufferArray = await Promise.all(downloadPromises); - const zipFile = bufferArray[bufferArray.length - 1]; - const dvArray = bufferArray.map((buffer) => new DataView(buffer)); + function createUrlObject(fileName: string) { + return { host: 'cdn-patch.swtor.com', path: `/patch/${product}/${product}_${from}to${to}/${fileName}` }; + } - //TODO: Verify that downloaded files match the hash in `solidpkg.pieces` + //start download, making sure that .zip file downloads first + const zipFile = getUrlContents(createUrlObject(solidpkg.files[solidpkg.files.length - 1].name)); + const diskFiles = solidpkg.files.slice(0, solidpkg.files.length - 1).map((file) => downloadUrlContents(createUrlObject(file.name))); - const fileEntries = readSsnFile(zipFile); + //we can parse the file entries as soon as the .zip file is loaded + const fileEntries = readSsnFile(await zipFile); console.debug(fileEntries); //Verify file entries @@ -27,16 +30,22 @@ export default async function getPatch(product: Product, from: number, to: numbe } //TODO: last file must always be `${product}.version` with diff type. Other files depend on diffType. + //Then we need to wait for other files before we can extract them + await Promise.all(diskFiles); + //const dvArray = bufferArray.map((buffer) => new DataView(buffer)); + + //TODO: Verify that downloaded files match the hash in `solidpkg.pieces` + //Extract newly added files - fileEntries.filter((file) => file.diffType === SsnDiffType.NewFile).forEach(async (file) => { - const fileContents = await extractFile(file, dvArray); + /*fileEntries.filter((file) => file.diffType === SsnDiffType.NewFile).forEach(async (file) => { + const fileContents = await extractFile(file, diskFiles); console.debug(new Uint8Array(fileContents)); //TODO }); //Extract changed files fileEntries.filter((file) => file.diffType === SsnDiffType.Changed).forEach(async (file) => { - const fileContents = await extractFile(file, dvArray); + const fileContents = await extractFile(file, diskFiles); console.debug(new Uint8Array(fileContents)); //TODO }); @@ -44,5 +53,5 @@ export default async function getPatch(product: Product, from: number, to: numbe //Need to delete deleted files fileEntries.filter((file) => file.diffType === SsnDiffType.Deleted).forEach((file) => { //TODO - }); + });*/ } diff --git a/src/ssn/verify/verifySolidpkg.ts b/src/ssn/verify/verifySolidpkg.ts index d5be80d..1f4dd54 100644 --- a/src/ssn/verify/verifySolidpkg.ts +++ b/src/ssn/verify/verifySolidpkg.ts @@ -28,6 +28,11 @@ export default function verifySolidpkg(file: ISolid, { product, from, to }: {pro if (!Array.isArray(file.info.files)) { throw new Error(`Expected files field to be an array but it isn't.`); } + + if (file.info.files.length < 2) { + throw new Error(`Expected files array to contain at least two files but it had ${file.info.files.length} files.`); + } + for (let i = 0, il = file.info.files.length; i < il; i += 1) { const fileEntry = file.info.files[i]; if (typeof fileEntry.length !== 'number' && fileEntry.length >= 0 && fileEntry.length <= 1700000000) { @@ -36,8 +41,15 @@ export default function verifySolidpkg(file: ISolid, { product, from, to }: {pro if (!Array.isArray(fileEntry.path) || fileEntry.path.length !== 1) { throw new Error(`Expected valid file name but it was not an array with one element.`); } - if (typeof fileEntry.path[0] !== 'string' || !fileEntry.path[0].match(new RegExp(`${product}_${from}to${to}\.z(ip|0[1-9]|[1-9][0-9])$`))) { - throw new Error(`Expected valid file name but it was ${fileEntry.path[0]}.`); + const fileName = fileEntry.path[0]; + if (typeof fileName !== 'string') { + throw new Error(`Expected valid file name to be a string but it was a ${typeof fileName}.`); + } + //Last file must end with .zip, other files are called .z01, .z02 etc. + const validExtension = (i === il - 1) ? 'ip' : String(i + 1).padStart(2, '0'); + const validName = `${product}_${from}to${to}.z${validExtension}`; + if (fileName !== validName) { + throw new Error(`Expected file name "${validName}" but it was "${fileName}".`); } }