From 96ed471afeb4060f965fb457923167e036a36653 Mon Sep 17 00:00:00 2001 From: C-3PO Date: Sun, 8 Jul 2018 18:10:41 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Use=20local=20cache=20instead=20of?= =?UTF-8?q?=20always=20downloading=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/cdn/downloadUrlContents.ts | 46 +++++++++++++------- src/cdn/funcs/checkLocalCache.ts | 27 ++++++++++++ src/cdn/funcs/createDirRecursively.ts | 49 ++++++++++++++++++++++ src/cdn/funcs/saveResponse.ts | 13 ++---- src/installPatch.ts | 2 +- src/ssn/getPatch.ts | 8 ++-- src/ssn/getPatchTest.ts | 60 --------------------------- 7 files changed, 116 insertions(+), 89 deletions(-) create mode 100644 src/cdn/funcs/checkLocalCache.ts create mode 100644 src/cdn/funcs/createDirRecursively.ts delete mode 100644 src/ssn/getPatchTest.ts diff --git a/src/cdn/downloadUrlContents.ts b/src/cdn/downloadUrlContents.ts index 053587b..0856415 100644 --- a/src/cdn/downloadUrlContents.ts +++ b/src/cdn/downloadUrlContents.ts @@ -1,23 +1,41 @@ -import * as fs from 'fs'; import * as http from 'http'; +import * as os from 'os'; +import * as nodePath from 'path'; +import checkLocalCache from './funcs/checkLocalCache'; +import createDirRecursively from './funcs/createDirRecursively'; import saveResponse from './funcs/saveResponse'; /** Downloads the given URL and saves it to disk. Throws error if download fails. */ -export default function downloadUrlContents({ host, path }: {host: string, path: string}): Promise { +export default function downloadUrlContents({ host, path, size }: {host: string, path: string, size: number}): Promise { return new Promise((resolve, reject) => { - //Create HTTP request - const request = http.request({ - family: 4, - host, - path, - }, saveResponse.bind(null, resolve, (reason: string) => { request.abort(); reject(reason); })); + //Generate file name we want to save it under + //e.g. on Linux: /tmp/patcher/patch/assets_swtor_main/assets_swtor_main_-1to0/assets_swtor_main_-1to0.zip + const tempFileName = nodePath.join(os.tmpdir(), 'patcher', host, path); - //In case of connection errors, exit early - request.on('error', (error) => { - request.abort(); - reject(error); + //Create parent directory recursively + const folderName = nodePath.dirname(tempFileName); + createDirRecursively(folderName).then(() => { + //Check if file already exists locally + checkLocalCache(tempFileName, size).then((cacheStatus) => { + if (cacheStatus) { + return resolve(tempFileName); + } + + //Create HTTP request + const request = http.request({ + family: 4, + host, + path, + }, saveResponse.bind(null, path, resolve, (reason: string) => { request.abort(); reject(reason); })); + + //In case of connection errors, exit early + request.on('error', (error) => { + request.abort(); + reject(error); + }); + + request.end(); + }); }); - - request.end(); }); } diff --git a/src/cdn/funcs/checkLocalCache.ts b/src/cdn/funcs/checkLocalCache.ts new file mode 100644 index 0000000..28e1eb2 --- /dev/null +++ b/src/cdn/funcs/checkLocalCache.ts @@ -0,0 +1,27 @@ +import * as fs from 'fs'; + +/** Checks if this file already exists on the local disk, so we don't need to download it again. */ +export default function checkLocalCache(fileName: string, size: number): Promise { + return new Promise((resolve, reject) => { + //Check if file already exists + fs.exists(fileName, (exists) => { + if (exists) { + const fileStats = fs.statSync(fileName); + //check if file size matches + if (fileStats.size === size) { + resolve(true); + } else { + //delete file so we can overwrite it + fs.unlink(fileName, (error) => { + if (error) { + reject(error); + } + resolve(false); + }); + } + } else { + resolve(false); + } + }); + }); +} diff --git a/src/cdn/funcs/createDirRecursively.ts b/src/cdn/funcs/createDirRecursively.ts new file mode 100644 index 0000000..e7b2fa0 --- /dev/null +++ b/src/cdn/funcs/createDirRecursively.ts @@ -0,0 +1,49 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +/** Recursively creates the given directory. */ +export default function createDirRecursively(folderName: string): Promise { + return new Promise((resolve, reject) => { + //Try to create directory + fs.mkdir(folderName, (error) => { + //If it fails, we first need to create parent directory + if (error) { + switch (error.code) { + case 'ENOENT': //parent does not exist + //Create parent + const parentFolder = path.dirname(folderName); + createDirRecursively(parentFolder).then(() => { + //Then try again + try { + resolve(createDirRecursively(folderName)); + } catch (error) { + reject(error); + } + }).catch((parentError) => { + reject(parentError); + }); + break; + + case 'EEXIST': { //already exists (either as file or directory) + fs.stat(folderName, (statError, stats) => { + if (statError) { + reject(statError); + } + if (stats.isDirectory()) { + resolve(); + } else { + reject('Is not a directory'); + } + }) + break; + } + + default: //other error, just propagate onwards + reject(error); + } + } else { + resolve(); + } + }); + }); +} diff --git a/src/cdn/funcs/saveResponse.ts b/src/cdn/funcs/saveResponse.ts index 7765dcc..94e8ef8 100644 --- a/src/cdn/funcs/saveResponse.ts +++ b/src/cdn/funcs/saveResponse.ts @@ -2,6 +2,7 @@ import * as fs from 'fs'; import * as http from 'http'; export default function saveResponse( + filePath: string, resolve: (fileName: string) => void, reject: (reason: string) => void, response: http.IncomingMessage, @@ -14,14 +15,6 @@ export default function saveResponse( //Remember file size const headerLength = Number(response.headers['content-length']); - //TODO: replace by fs.mkdtemp. Also, file name should be decided in downloadUrlContents(), not here, and we must return a ReadableStream instead of the file name, and automatically delete the file - /*fs.mkdtemp(path.join(os.tmpdir(), 'foo-'), (err, folder) => { - if (err) throw err; - console.log(folder); - // Prints: /tmp/foo-itXde2 or C:\Users\...\AppData\Local\Temp\foo-itXde2 - });*/ - const tempFileName = `/tmp/swtor-patcher-download-${Math.random()}.tmp`; - //If we receive a part of the response, write it to disk let previousChunk: Promise = new Promise((innerResolve) => { innerResolve(); }); let totalLength = 0; @@ -37,7 +30,7 @@ export default function saveResponse( previousChunk.then(() => { previousChunk = new Promise((innerResolve, innerReject) => { //Write chunk to disk - fs.appendFile(tempFileName, chunk, { encoding: 'binary' }, (error) => { + fs.appendFile(filePath, chunk, { encoding: 'binary' }, (error) => { if (error) { return reject(`Could not write to disk: [${error.code}] ${error.name}: ${error.message}.`); } @@ -58,7 +51,7 @@ export default function saveResponse( //TODO: need to automatically delete file once it is no longer used //TODO: need to provide methods to seek through file previousChunk.then(() => { - resolve(tempFileName); + resolve(filePath); }); }); } diff --git a/src/installPatch.ts b/src/installPatch.ts index b4ac023..dcc4ef3 100644 --- a/src/installPatch.ts +++ b/src/installPatch.ts @@ -1,5 +1,5 @@ import getManifest from './ssn/getManifest'; -import getPatch from './ssn/getPatchTest'; +import getPatch from './ssn/getPatch'; (async () => { //----- PATCHMANIFEST ----- diff --git a/src/ssn/getPatch.ts b/src/ssn/getPatch.ts index a99f379..27f5233 100644 --- a/src/ssn/getPatch.ts +++ b/src/ssn/getPatch.ts @@ -13,14 +13,14 @@ export default async function getPatch(product: Product, from: number, to: numbe const solidpkg = await getSolidpkg(product, from, to); console.debug(solidpkg.files); - function createUrlObject(fileName: string) { - return { host: 'cdn-patch.swtor.com', path: `/patch/${product}/${product}_${from}to${to}/${fileName}` }; + function createUrlObject({ name, length }: {name: string, length: number}) { + return { host: 'cdn-patch.swtor.com', path: `/patch/${product}/${product}_${from}to${to}/${name}`, size: length }; } //start download, making sure that .zip file downloads first const indexOfLastFile = solidpkg.files.length - 1; - const zipFile = getUrlContents(createUrlObject(solidpkg.files[indexOfLastFile].name)); - const diskFiles = solidpkg.files.slice(0, indexOfLastFile).map((file) => downloadUrlContents(createUrlObject(file.name))); + const zipFile = getUrlContents(createUrlObject(solidpkg.files[indexOfLastFile])); + const diskFiles = solidpkg.files.slice(0, indexOfLastFile).map((file) => downloadUrlContents(createUrlObject(file))); //we can parse the file entries as soon as the .zip file is downloaded const fileEntries = readSsnFile(await zipFile); diff --git a/src/ssn/getPatchTest.ts b/src/ssn/getPatchTest.ts deleted file mode 100644 index cd3c270..0000000 --- a/src/ssn/getPatchTest.ts +++ /dev/null @@ -1,60 +0,0 @@ -import downloadUrlContents from '../cdn/downloadUrlContents'; -import getUrlContents from '../cdn/getUrlContents'; -import { Product } from '../interfaces/ISettings'; -import { SsnDiffType } from '../interfaces/ISsnFileEntry'; -import extractFileStream from './extractFileStream'; -import getSolidpkg from './getSolidpkg'; -import readSsnFile from './reader/readSsnFile'; -import getFileFromDisks from './streams/getFileFromDisks'; -import streamToArrayBuffer from './streams/streamToArrayBuffer'; -import verifyPatch from './verify/verifyPatch'; - -export default async function getPatch(product: Product, from: number, to: number) { - const solidpkg = await getSolidpkg(product, from, to); - console.debug(solidpkg.files); - - function createUrlObject(fileName: string) { - return { host: 'cdn-patch.swtor.com', path: `/patch/${product}/${product}_${from}to${to}/${fileName}` }; - } - - //start download, making sure that .zip file downloads first - const indexOfLastFile = solidpkg.files.length - 1; - const zipFile = getUrlContents(createUrlObject(solidpkg.files[indexOfLastFile].name)); - //const diskFiles = solidpkg.files.slice(0, indexOfLastFile).map((file) => downloadUrlContents(createUrlObject(file.name))); - - //we can parse the file entries as soon as the .zip file is downloaded - const fileEntries = readSsnFile(await zipFile); - console.debug(fileEntries); - - //Verify file entries - verifyPatch(fileEntries, product, from); - - //Then we need to wait for disks to finish download before we can extract individual files - //TODO: we can optimize this to already extract some files as soon as their relevant parts are downloaded - //const diskFilenames = await Promise.all(diskFiles); - const diskFilenames = ['../assets_swtor_main_-1to0.z01']; - //const dvArray = bufferArray.map((buffer) => new DataView(buffer)); - - //TODO: Verify that downloaded files match the hash in `solidpkg.pieces` - - //Extract newly added files - fileEntries.filter((file) => file.diffType === SsnDiffType.NewFile && file.diskNumberStart === 0).forEach(async (file) => { - const fileStream = await getFileFromDisks(diskFilenames, { diskStart: file.diskNumberStart, offset: file.offset, storedSize: file.compressedSize }); - const fileContents = extractFileStream(file, fileStream); - console.debug(await streamToArrayBuffer(fileContents)); - //TODO: need to write to disk - }); - - //Extract changed files - fileEntries.filter((file) => file.diffType === SsnDiffType.Changed && file.diskNumberStart === 0).forEach(async (file) => { - const fileStream = await getFileFromDisks(diskFilenames, { diskStart: file.diskNumberStart, offset: file.offset, storedSize: file.compressedSize }); - const fileContents = extractFileStream(file, fileStream); - console.debug(await streamToArrayBuffer(fileContents)); - //TODO: need to apply diffing, then write to disk - }); - - //Need to delete deleted files - fileEntries.filter((file) => file.diffType === SsnDiffType.Deleted).forEach((file) => { - //TODO: need to delete file - }); -}