✨ Use local cache instead of always downloading files

src/cdn/downloadUrlContents.ts

@@ -1,23 +1,41 @@
-import * as fs from 'fs';
 import * as http from 'http';
+import * as os from 'os';
+import * as nodePath from 'path';
+import checkLocalCache from './funcs/checkLocalCache';
+import createDirRecursively from './funcs/createDirRecursively';
 import saveResponse from './funcs/saveResponse';
 
 /** Downloads the given URL and saves it to disk. Throws error if download fails. */
-export default function downloadUrlContents({ host, path }: {host: string, path: string}): Promise<string> {
+export default function downloadUrlContents({ host, path, size }: {host: string, path: string, size: number}): Promise<string> {
   return new Promise((resolve, reject) => {
-    //Create HTTP request
-    const request = http.request({
-      family: 4,
-      host,
-      path,
-    }, saveResponse.bind(null, resolve, (reason: string) => { request.abort(); reject(reason); }));
+    //Generate file name we want to save it under
+    //e.g. on Linux: /tmp/patcher/patch/assets_swtor_main/assets_swtor_main_-1to0/assets_swtor_main_-1to0.zip
+    const tempFileName = nodePath.join(os.tmpdir(), 'patcher', host, path);
 
-    //In case of connection errors, exit early
-    request.on('error', (error) => {
-      request.abort();
-      reject(error);
+    //Create parent directory recursively
+    const folderName = nodePath.dirname(tempFileName);
+    createDirRecursively(folderName).then(() => {
+      //Check if file already exists locally
+      checkLocalCache(tempFileName, size).then((cacheStatus) => {
+        if (cacheStatus) {
+          return resolve(tempFileName);
+        }
+
+        //Create HTTP request
+        const request = http.request({
+          family: 4,
+          host,
+          path,
+        }, saveResponse.bind(null, path, resolve, (reason: string) => { request.abort(); reject(reason); }));
+
+        //In case of connection errors, exit early
+        request.on('error', (error) => {
+          request.abort();
+          reject(error);
+        });
+
+        request.end();
+      });
     });
-
-    request.end();
   });
 }

src/cdn/funcs/checkLocalCache.ts

@@ -0,0 +1,27 @@
+import * as fs from 'fs';
+
+/** Checks if this file already exists on the local disk, so we don't need to download it again. */
+export default function checkLocalCache(fileName: string, size: number): Promise<boolean> {
+  return new Promise((resolve, reject) => {
+    //Check if file already exists
+    fs.exists(fileName, (exists) => {
+      if (exists) {
+        const fileStats = fs.statSync(fileName);
+        //check if file size matches
+        if (fileStats.size === size) {
+          resolve(true);
+        } else {
+          //delete file so we can overwrite it
+          fs.unlink(fileName, (error) => {
+            if (error) {
+              reject(error);
+            }
+            resolve(false);
+          });
+        }
+      } else {
+        resolve(false);
+      }
+    });
+  });
+}

src/cdn/funcs/createDirRecursively.ts

@@ -0,0 +1,49 @@
+import * as fs from 'fs';
+import * as path from 'path';
+
+/** Recursively creates the given directory. */
+export default function createDirRecursively(folderName: string): Promise<void> {
+  return new Promise((resolve, reject) => {
+    //Try to create directory
+    fs.mkdir(folderName, (error) => {
+      //If it fails, we first need to create parent directory
+      if (error) {
+        switch (error.code) {
+          case 'ENOENT': //parent does not exist
+            //Create parent
+            const parentFolder = path.dirname(folderName);
+            createDirRecursively(parentFolder).then(() => {
+              //Then try again
+              try {
+                resolve(createDirRecursively(folderName));
+              } catch (error) {
+                reject(error);
+              }
+            }).catch((parentError) => {
+              reject(parentError);
+            });
+            break;
+
+          case 'EEXIST': { //already exists (either as file or directory)
+            fs.stat(folderName, (statError, stats) => {
+              if (statError) {
+                reject(statError);
+              }
+              if (stats.isDirectory()) {
+                resolve();
+              } else {
+                reject('Is not a directory');
+              }
+            })
+            break;
+          }
+
+          default: //other error, just propagate onwards
+            reject(error);
+        }
+      } else {
+        resolve();
+      }
+    });
+  });
+}

src/cdn/funcs/saveResponse.ts

@@ -2,6 +2,7 @@ import * as fs from 'fs';
 import * as http from 'http';
 
 export default function saveResponse(
+  filePath: string,
   resolve: (fileName: string) => void,
   reject: (reason: string) => void,
   response: http.IncomingMessage,
@@ -14,14 +15,6 @@ export default function saveResponse(
   //Remember file size
   const headerLength = Number(response.headers['content-length']);
 
-  //TODO: replace by fs.mkdtemp. Also, file name should be decided in downloadUrlContents(), not here, and we must return a ReadableStream instead of the file name, and automatically delete the file
-  /*fs.mkdtemp(path.join(os.tmpdir(), 'foo-'), (err, folder) => {
-    if (err) throw err;
-    console.log(folder);
-    // Prints: /tmp/foo-itXde2 or C:\Users\...\AppData\Local\Temp\foo-itXde2
-  });*/
-  const tempFileName = `/tmp/swtor-patcher-download-${Math.random()}.tmp`;
-
   //If we receive a part of the response, write it to disk
   let previousChunk: Promise<void> = new Promise((innerResolve) => { innerResolve(); });
   let totalLength = 0;
@@ -37,7 +30,7 @@ export default function saveResponse(
     previousChunk.then(() => {
       previousChunk = new Promise((innerResolve, innerReject) => {
         //Write chunk to disk
-        fs.appendFile(tempFileName, chunk, { encoding: 'binary' }, (error) => {
+        fs.appendFile(filePath, chunk, { encoding: 'binary' }, (error) => {
           if (error) {
             return reject(`Could not write to disk: [${error.code}] ${error.name}: ${error.message}.`);
           }
@@ -58,7 +51,7 @@ export default function saveResponse(
     //TODO: need to automatically delete file once it is no longer used
     //TODO: need to provide methods to seek through file
     previousChunk.then(() => {
-      resolve(tempFileName);
+      resolve(filePath);
     });
   });
 }

src/installPatch.ts

@@ -1,5 +1,5 @@
 import getManifest from './ssn/getManifest';
-import getPatch from './ssn/getPatchTest';
+import getPatch from './ssn/getPatch';
 
 (async () => {
   //----- PATCHMANIFEST -----

src/ssn/getPatch.ts

@@ -13,14 +13,14 @@ export default async function getPatch(product: Product, from: number, to: numbe
   const solidpkg = await getSolidpkg(product, from, to);
   console.debug(solidpkg.files);
 
-  function createUrlObject(fileName: string) {
-    return { host: 'cdn-patch.swtor.com', path: `/patch/${product}/${product}_${from}to${to}/${fileName}` };
+  function createUrlObject({ name, length }: {name: string, length: number}) {
+    return { host: 'cdn-patch.swtor.com', path: `/patch/${product}/${product}_${from}to${to}/${name}`, size: length };
   }
 
   //start download, making sure that .zip file downloads first
   const indexOfLastFile = solidpkg.files.length - 1;
-  const zipFile = getUrlContents(createUrlObject(solidpkg.files[indexOfLastFile].name));
-  const diskFiles = solidpkg.files.slice(0, indexOfLastFile).map((file) => downloadUrlContents(createUrlObject(file.name)));
+  const zipFile = getUrlContents(createUrlObject(solidpkg.files[indexOfLastFile]));
+  const diskFiles = solidpkg.files.slice(0, indexOfLastFile).map((file) => downloadUrlContents(createUrlObject(file)));
 
   //we can parse the file entries as soon as the .zip file is downloaded
   const fileEntries = readSsnFile(await zipFile);

src/ssn/getPatchTest.ts

@@ -1,60 +0,0 @@
-import downloadUrlContents from '../cdn/downloadUrlContents';
-import getUrlContents from '../cdn/getUrlContents';
-import { Product } from '../interfaces/ISettings';
-import { SsnDiffType } from '../interfaces/ISsnFileEntry';
-import extractFileStream from './extractFileStream';
-import getSolidpkg from './getSolidpkg';
-import readSsnFile from './reader/readSsnFile';
-import getFileFromDisks from './streams/getFileFromDisks';
-import streamToArrayBuffer from './streams/streamToArrayBuffer';
-import verifyPatch from './verify/verifyPatch';
-
-export default async function getPatch(product: Product, from: number, to: number) {
-  const solidpkg = await getSolidpkg(product, from, to);
-  console.debug(solidpkg.files);
-
-  function createUrlObject(fileName: string) {
-    return { host: 'cdn-patch.swtor.com', path: `/patch/${product}/${product}_${from}to${to}/${fileName}` };
-  }
-
-  //start download, making sure that .zip file downloads first
-  const indexOfLastFile = solidpkg.files.length - 1;
-  const zipFile = getUrlContents(createUrlObject(solidpkg.files[indexOfLastFile].name));
-  //const diskFiles = solidpkg.files.slice(0, indexOfLastFile).map((file) => downloadUrlContents(createUrlObject(file.name)));
-
-  //we can parse the file entries as soon as the .zip file is downloaded
-  const fileEntries = readSsnFile(await zipFile);
-  console.debug(fileEntries);
-
-  //Verify file entries
-  verifyPatch(fileEntries, product, from);
-
-  //Then we need to wait for disks to finish download before we can extract individual files
-  //TODO: we can optimize this to already extract some files as soon as their relevant parts are downloaded
-  //const diskFilenames = await Promise.all(diskFiles);
-  const diskFilenames = ['../assets_swtor_main_-1to0.z01'];
-  //const dvArray = bufferArray.map((buffer) => new DataView(buffer));
-
-  //TODO: Verify that downloaded files match the hash in `solidpkg.pieces`
-
-  //Extract newly added files
-  fileEntries.filter((file) => file.diffType === SsnDiffType.NewFile && file.diskNumberStart === 0).forEach(async (file) => {
-    const fileStream = await getFileFromDisks(diskFilenames, { diskStart: file.diskNumberStart, offset: file.offset, storedSize: file.compressedSize });
-    const fileContents = extractFileStream(file, fileStream);
-    console.debug(await streamToArrayBuffer(fileContents));
-    //TODO: need to write to disk
-  });
-
-  //Extract changed files
-  fileEntries.filter((file) => file.diffType === SsnDiffType.Changed && file.diskNumberStart === 0).forEach(async (file) => {
-    const fileStream = await getFileFromDisks(diskFilenames, { diskStart: file.diskNumberStart, offset: file.offset, storedSize: file.compressedSize });
-    const fileContents = extractFileStream(file, fileStream);
-    console.debug(await streamToArrayBuffer(fileContents));
-    //TODO: need to apply diffing, then write to disk
-  });
-
-  //Need to delete deleted files
-  fileEntries.filter((file) => file.diffType === SsnDiffType.Deleted).forEach((file) => {
-    //TODO: need to delete file
-  });
-}