WIP - add extractor, generate snippet_data

node_modules/cache-manager-fs-hash/lib/index.js

@@ -0,0 +1,286 @@
+"use strict";
+
+function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
+
+function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
+
+const fs = require('fs');
+
+const crypto = require('crypto');
+
+const path = require('path');
+
+const _require = require('es6-promisify'),
+      promisify = _require.promisify;
+
+const lockFile = require('lockfile');
+
+const jsonFileStore = require('./json-file-store');
+
+const wrapCallback = require('./wrap-callback');
+/**
+ * construction of the disk storage
+ * @param {object} [args] options of disk store
+ * @param {string} [args.path] path for cached files
+ * @param {number} [args.ttl] time to life in seconds
+ * @todo {number} [args.maxsize] max size in bytes on disk
+ * @param {boolean} [args.subdirs] create subdirectories
+ * @returns {DiskStore}
+ */
+
+
+exports.create = function (args) {
+  return new DiskStore(args && args.options ? args.options : args);
+};
+
+function DiskStore(options) {
+  options = options || {};
+  this.options = {
+    path: options.path || './cache',
+
+    /* path for cached files  */
+    ttl: options.ttl >= 0 ? +options.ttl : 60 * 60,
+
+    /* time before expiring in seconds */
+    maxsize: options.maxsize || Infinity,
+
+    /* max size in bytes on disk */
+    subdirs: options.subdirs || false,
+    lockFile: {
+      //check lock at 0ms 50ms 100ms ... 400ms 1400ms 1450ms... up to 10 seconds, after that just asume the lock is staled
+      wait: 400,
+      pollPeriod: 50,
+      stale: 10 * 1000,
+      retries: 10,
+      retryWait: 600
+    }
+  }; // check storage directory for existence (or create it)
+
+  if (!fs.existsSync(this.options.path)) {
+    fs.mkdirSync(this.options.path);
+  }
+}
+/**
+ * save an entry in store
+ * @param {string} key
+ * @param {*} val
+ * @param {object} [options]
+ * @param {number} options.ttl time to life in seconds
+ * @param {function} [cb]
+ * @returns {Promise}
+ */
+
+
+DiskStore.prototype.set = wrapCallback(
+/*#__PURE__*/
+function () {
+  var _ref = _asyncToGenerator(function* (key, val, options) {
+    key = key + '';
+
+    const filePath = this._getFilePathByKey(key);
+
+    const ttl = options && options.ttl >= 0 ? +options.ttl : this.options.ttl;
+    const data = {
+      expireTime: Date.now() + ttl * 1000,
+      key: key,
+      val: val
+    };
+
+    if (this.options.subdirs) {
+      //check if subdir exists or create it
+      const dir = path.dirname(filePath);
+      yield promisify(fs.access)(dir, fs.constants.W_OK).catch(function () {
+        return promisify(fs.mkdir)(dir);
+      });
+    }
+
+    try {
+      yield this._lock(filePath);
+      yield jsonFileStore.write(filePath, data);
+    } catch (err) {
+      throw err;
+    } finally {
+      yield this._unlock(filePath);
+    }
+  });
+
+  return function (_x, _x2, _x3) {
+    return _ref.apply(this, arguments);
+  };
+}());
+/**
+ * get an entry from store
+ * @param {string} key
+ * @param {function} [cb]
+ * @returns {Promise}
+ */
+
+DiskStore.prototype.get = wrapCallback(
+/*#__PURE__*/
+function () {
+  var _ref2 = _asyncToGenerator(function* (key) {
+    var _this = this;
+
+    key = key + '';
+
+    const filePath = this._getFilePathByKey(key);
+
+    try {
+      const data = yield jsonFileStore.read(filePath).catch(
+      /*#__PURE__*/
+      function () {
+        var _ref3 = _asyncToGenerator(function* (err) {
+          if (err.code === 'ENOENT') {
+            throw err;
+          } //maybe the file is currently written to, lets lock it and read again
+
+
+          try {
+            yield _this._lock(filePath);
+            return yield jsonFileStore.read(filePath);
+          } catch (err2) {
+            throw err2;
+          } finally {
+            yield _this._unlock(filePath);
+          }
+        });
+
+        return function (_x5) {
+          return _ref3.apply(this, arguments);
+        };
+      }());
+
+      if (data.expireTime <= Date.now()) {
+        //cache expired
+        this.del(key).catch(() => 0
+        /* ignore */
+        );
+        return undefined;
+      }
+
+      if (data.key !== key) {
+        //hash collision
+        return undefined;
+      }
+
+      return data.val;
+    } catch (err) {
+      //file does not exist lets return a cache miss
+      if (err.code === 'ENOENT') {
+        return undefined;
+      } else {
+        throw err;
+      }
+    }
+  });
+
+  return function (_x4) {
+    return _ref2.apply(this, arguments);
+  };
+}());
+/**
+ * delete entry from cache
+ */
+
+DiskStore.prototype.del = wrapCallback(
+/*#__PURE__*/
+function () {
+  var _ref4 = _asyncToGenerator(function* (key) {
+    const filePath = this._getFilePathByKey(key);
+
+    try {
+      yield this._lock(filePath);
+      yield jsonFileStore.delete(filePath);
+    } catch (err) {
+      //ignore deleting non existing keys
+      if (err.code !== 'ENOENT') {
+        throw err;
+      }
+    } finally {
+      yield this._unlock(filePath);
+    }
+  });
+
+  return function (_x6) {
+    return _ref4.apply(this, arguments);
+  };
+}());
+/**
+ * cleanup cache on disk -> delete all files from the cache
+ */
+
+DiskStore.prototype.reset = wrapCallback(
+/*#__PURE__*/
+_asyncToGenerator(function* () {
+  const readdir = promisify(fs.readdir);
+  const stat = promisify(fs.stat);
+  const unlink = promisify(fs.unlink);
+  return yield deletePath(this.options.path, 2);
+
+  function deletePath(_x7, _x8) {
+    return _deletePath.apply(this, arguments);
+  }
+
+  function _deletePath() {
+    _deletePath = _asyncToGenerator(function* (fileOrDir, maxDeep) {
+      if (maxDeep < 0) {
+        return;
+      }
+
+      const stats = yield stat(fileOrDir);
+
+      if (stats.isDirectory()) {
+        const files = yield readdir(fileOrDir);
+
+        for (let i = 0; i < files.length; i++) {
+          yield deletePath(path.join(fileOrDir, files[i]), maxDeep - 1);
+        }
+      } else if (stats.isFile() && /[/\\]diskstore-[0-9a-fA-F/\\]+(\.json|-\d\.bin)/.test(fileOrDir)) {
+        //delete the file if it is a diskstore file
+        yield unlink(fileOrDir);
+      }
+    });
+    return _deletePath.apply(this, arguments);
+  }
+}));
+/**
+ * locks a file so other forks that want to use the same file have to wait
+ * @param {string} filePath
+ * @returns {Promise}
+ * @private
+ */
+
+DiskStore.prototype._lock = function (filePath) {
+  return promisify(lockFile.lock)(filePath + '.lock', JSON.parse(JSON.stringify(this.options.lockFile)) //the options are modified -> create a copy to prevent that
+  );
+};
+/**
+ * unlocks a file path
+ * @type {Function}
+ * @param {string} filePath
+ * @returns {Promise}
+ * @private
+ */
+
+
+DiskStore.prototype._unlock = function (filePath) {
+  return promisify(lockFile.unlock)(filePath + '.lock');
+};
+/**
+ * returns the location where the value should be stored
+ * @param {string} key
+ * @returns {string}
+ * @private
+ */
+
+
+DiskStore.prototype._getFilePathByKey = function (key) {
+  const hash = crypto.createHash('md5').update(key + '').digest('hex');
+
+  if (this.options.subdirs) {
+    //create subdirs with the first 3 chars of the hash
+    return path.join(this.options.path, 'diskstore-' + hash.substr(0, 3), hash.substr(3));
+  } else {
+    return path.join(this.options.path, 'diskstore-' + hash);
+  }
+};

node_modules/cache-manager-fs-hash/lib/json-file-store.js

@@ -0,0 +1,122 @@
+"use strict";
+
+function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
+
+function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
+
+const _require = require('es6-promisify'),
+      promisify = _require.promisify;
+
+const fs = require('fs');
+
+exports.write =
+/*#__PURE__*/
+function () {
+  var _ref = _asyncToGenerator(function* (path, data) {
+    const externalBuffers = [];
+    const dataString = JSON.stringify(data, function replacerFunction(k, value) {
+      //Buffers searilize to {data: [...], type: "Buffer"}
+      if (value && value.type === 'Buffer' && value.data && value.data.length >= 1024
+      /* only save bigger Buffers external, small ones can be inlined */
+      ) {
+          const buffer = Buffer.from(value.data);
+          externalBuffers.push({
+            index: externalBuffers.length,
+            buffer: buffer
+          });
+          return {
+            type: 'ExternalBuffer',
+            index: externalBuffers.length - 1,
+            size: buffer.length
+          };
+        } else {
+        return value;
+      }
+    }); //save main json file
+
+    yield promisify(fs.writeFile)(path + '.json', dataString, 'utf8'); //save external buffers
+
+    yield Promise.all(externalBuffers.map(
+    /*#__PURE__*/
+    function () {
+      var _ref2 = _asyncToGenerator(function* (externalBuffer) {
+        yield promisify(fs.writeFile)(path + '-' + externalBuffer.index + '.bin', externalBuffer.buffer, 'utf8');
+      });
+
+      return function (_x3) {
+        return _ref2.apply(this, arguments);
+      };
+    }()));
+  });
+
+  return function (_x, _x2) {
+    return _ref.apply(this, arguments);
+  };
+}();
+
+exports.read =
+/*#__PURE__*/
+function () {
+  var _ref3 = _asyncToGenerator(function* (path) {
+    //read main json file
+    const dataString = yield promisify(fs.readFile)(path + '.json', 'utf8');
+    const externalBuffers = [];
+    const data = JSON.parse(dataString, function bufferReceiver(k, value) {
+      if (value && value.type === 'Buffer' && value.data) {
+        return Buffer.from(value.data);
+      } else if (value && value.type === 'ExternalBuffer' && typeof value.index === 'number' && typeof value.size === 'number') {
+        //JSON.parse is sync so we need to return a buffer sync, we will fill the buffer later
+        const buffer = Buffer.alloc(value.size);
+        externalBuffers.push({
+          index: +value.index,
+          buffer: buffer
+        });
+        return buffer;
+      } else {
+        return value;
+      }
+    }); //read external buffers
+
+    yield Promise.all(externalBuffers.map(
+    /*#__PURE__*/
+    function () {
+      var _ref4 = _asyncToGenerator(function* (externalBuffer) {
+        const fd = yield promisify(fs.open)(path + '-' + +externalBuffer.index + '.bin', 'r');
+        yield promisify(fs.read)(fd, externalBuffer.buffer, 0, externalBuffer.buffer.length, 0);
+        yield promisify(fs.close)(fd);
+      });
+
+      return function (_x5) {
+        return _ref4.apply(this, arguments);
+      };
+    }()));
+    return data;
+  });
+
+  return function (_x4) {
+    return _ref3.apply(this, arguments);
+  };
+}();
+
+exports.delete =
+/*#__PURE__*/
+function () {
+  var _ref5 = _asyncToGenerator(function* (path) {
+    yield promisify(fs.unlink)(path + '.json'); //delete binary files
+
+    try {
+      for (let i = 0; i < Infinity; i++) {
+        yield promisify(fs.unlink)(path + '-' + i + '.bin');
+      }
+    } catch (err) {
+      if (err.code === 'ENOENT') {// every binary is deleted, we are done
+      } else {
+        throw err;
+      }
+    }
+  });
+
+  return function (_x6) {
+    return _ref5.apply(this, arguments);
+  };
+}();

node_modules/cache-manager-fs-hash/lib/wrap-callback.js

@@ -0,0 +1,24 @@
+"use strict";
+
+/**
+ * adds an callback param to the original function
+ * @param {function} fn 
+ * @returns {function}
+ */
+module.exports = function wrapCallback(fn) {
+  return function (...args) {
+    let cb;
+
+    if (typeof args[args.length - 1] === 'function') {
+      cb = args.pop();
+    }
+
+    const promise = fn.apply(this, args);
+
+    if (typeof cb === 'function') {
+      promise.then(value => setImmediate(cb, null, value), err => setImmediate(cb, err));
+    }
+
+    return promise;
+  };
+};