WIP - add extractor, generate snippet_data

node_modules/better-queue/.editorconfig

@@ -0,0 +1,3 @@
+[*.{js,json}]
+indent_style = space
+indent_size = 2

node_modules/better-queue/.npmignore

@@ -0,0 +1,32 @@
+# Logs
+logs
+*.log
+
+# Runtime data
+pids
+*.pid
+*.seed
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directory
+# https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
+node_modules
+.DS_Store
+
+# Test file
+test.js
+*.test.js

node_modules/better-queue/.travis.yml

@@ -0,0 +1 @@
+language: node_js

node_modules/better-queue/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Leander
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

node_modules/better-queue/README.md

@@ -0,0 +1,709 @@
+# Better Queue - Powerful flow control
+
+[![npm package](https://nodei.co/npm/better-queue.png?downloads=true&downloadRank=true&stars=true)](https://nodei.co/npm/better-queue/)
+
+[![Build status](https://img.shields.io/travis/diamondio/better-queue.svg?style=flat-square)](https://travis-ci.org/diamondio/better-queue)
+[![Dependency Status](https://img.shields.io/david/diamondio/better-queue.svg?style=flat-square)](https://david-dm.org/diamondio/better-queue)
+[![Known Vulnerabilities](https://snyk.io/test/npm/better-queue/badge.svg?style=flat-square)](https://snyk.io/test/npm/better-queue)
+[![Gitter](https://img.shields.io/badge/gitter-join_chat-blue.svg?style=flat-square)](https://gitter.im/diamondio/better-queue?utm_source=badge)
+
+
+## Super simple to use
+
+Better Queue is designed to be simple to set up but still let you do complex things.
+
+- Persistent (and extendable) storage
+- Batched processing
+- Prioritize tasks
+- Merge/filter tasks
+- Progress events (with ETA!)
+- Fine-tuned timing controls
+- Retry on fail
+- Concurrent batch processing
+- Task statistics (average completion time, failure rate and peak queue size)
+- ... and more!
+
+---
+
+#### Install (via npm)
+
+```bash
+npm install --save better-queue
+```
+
+---
+
+#### Quick Example
+
+```js
+var Queue = require('better-queue');
+
+var q = new Queue(function (input, cb) {
+  
+  // Some processing here ...
+
+  cb(null, result);
+})
+
+q.push(1)
+q.push({ x: 1 })
+```
+
+## Table of contents
+
+- [Queuing](#queuing)
+- [Task Management](#task-management)
+- [Queue Management](#queue-management)
+- [Advanced](#advanced)
+- [Storage](#storage)
+- [Full Documentation](#full-documentation)
+
+---
+
+You will be able to combine any (and all) of these options
+for your queue!
+
+
+## Queuing
+
+It's very easy to push tasks into the queue.
+
+```js
+var q = new Queue(fn);
+q.push(1);
+q.push({ x: 1, y: 2 });
+q.push("hello");
+```
+
+You can also include a callback as a second parameter to the push
+function, which would be called when that task is done. For example:
+
+```js
+var q = new Queue(fn);
+q.push(1, function (err, result) {
+  // Results from the task!
+});
+```
+
+You can also listen to events on the results of the `push` call.
+
+```js
+var q = new Queue(fn);
+q.push(1)
+  .on('finish', function (result) {
+    // Task succeeded with {result}!
+  })
+  .on('failed', function (err) {
+    // Task failed!
+  })
+```
+
+Alternatively, you can subscribe to the queue's events.
+
+```js
+var q = new Queue(fn);
+q.on('task_finish', function (taskId, result, stats) {
+  // taskId = 1, result: 3, stats = { elapsed: <time taken> }
+  // taskId = 2, result: 5, stats = { elapsed: <time taken> }
+})
+q.on('task_failed', function (taskId, err, stats) {
+  // Handle error, stats = { elapsed: <time taken> }
+})
+q.on('empty', function (){})
+q.on('drain', function (){})
+q.push({ id: 1, a: 1, b: 2 });
+q.push({ id: 2, a: 2, b: 3 });
+```
+
+`empty` event fires when all of the tasks have been pulled off of
+the queue (there may still be tasks running!)
+
+`drain` event fires when there are no more tasks on the queue _and_
+when no more tasks are running.
+
+You can control how many tasks process at the same time.
+
+```js
+var q = new Queue(fn, { concurrent: 3 })
+```
+
+Now the queue will allow 3 tasks running at the same time. (By 
+default, we handle tasks one at a time.)
+
+You can also turn the queue into a stack by turning on `filo`.
+
+```js
+var q = new Queue(fn, { filo: true })
+```
+
+Now items you push on will be handled first.
+
+
+
+[back to top](#table-of-contents)
+
+---
+
+## Task Management
+
+#### Task ID
+
+Tasks can be given an ID to help identify and track it as it goes through
+the queue.
+
+By default, we look for `task.id` to see if it's a string property,
+otherwise we generate a random ID for the task.
+
+You can pass in an `id` property to options to change this behaviour.
+Here are some examples of how:
+
+```js
+var q = new Queue(fn, {
+  id: 'id',   // Default: task's `id` property
+  id: 'name', // task's `name` property
+  id: function (task, cb) {
+    // Compute the ID
+    cb(null, 'computed_id');
+  }
+})
+```
+
+One thing you can do with Task ID is merge tasks:
+
+```js
+var counter = new Queue(function (task, cb) {
+  console.log("I have %d %ss.", task.count, task.id);
+  cb();
+}, {
+  merge: function (oldTask, newTask, cb) {
+    oldTask.count += newTask.count;
+    cb(null, oldTask);
+  }
+})
+counter.push({ id: 'apple', count: 2 });
+counter.push({ id: 'apple', count: 1 });
+counter.push({ id: 'orange', count: 1 });
+counter.push({ id: 'orange', count: 1 });
+// Prints out:
+//   I have 3 apples.
+//   I have 2 oranges.
+```
+
+By default, if tasks have the same ID they replace the previous task.
+
+```js
+var counter = new Queue(function (task, cb) {
+  console.log("I have %d %ss.", task.count, task.id);
+  cb();
+})
+counter.push({ id: 'apple', count: 1 });
+counter.push({ id: 'apple', count: 3 });
+counter.push({ id: 'orange', count: 1 });
+counter.push({ id: 'orange', count: 2 });
+// Prints out:
+//   I have 3 apples.
+//   I have 2 oranges.
+```
+
+You can also use the task ID when subscribing to events from Queue.
+
+```js
+var counter = new Queue(fn)
+counter.on('task_finish', function (taskId, result) {
+  // taskId will be 'jim' or 'bob'
+})
+counter.push({ id: 'jim', count: 2 });
+counter.push({ id: 'bob', count: 1 });
+```
+
+
+#### Batch Processing
+
+Your processing function can also be modified to handle multiple
+tasks at the same time. For example:
+
+```js
+var ages = new Queue(function (batch, cb) {
+  // Batch 1:
+  //   [ { id: 'steve', age: 21 },
+  //     { id: 'john', age: 34 },
+  //     { id: 'joe', age: 18 } ]
+  // Batch 2:
+  //   [ { id: 'mary', age: 23 } ]
+  cb();
+}, { batchSize: 3 })
+ages.push({ id: 'steve', age: 21 });
+ages.push({ id: 'john', age: 34 });
+ages.push({ id: 'joe', age: 18 });
+ages.push({ id: 'mary', age: 23 });
+```
+
+Note how the queue will only handle at most 3 items at a time.
+
+Below is another example of a batched call with numbers.
+
+```js
+var ages = new Queue(function (batch, cb) {
+  // batch = [1,2,3]
+  cb();
+}, { batchSize: 3 })
+ages.push(1);
+ages.push(2);
+ages.push(3);
+```
+
+
+#### Filtering, Validation and Priority
+
+You can also format (and filter) the input that arrives from a push
+before it gets processed by the queue by passing in a `filter` 
+function.
+
+```js
+var greeter = new Queue(function (name, cb) {
+  console.log("Hello, %s!", name)
+  cb();
+}, {
+  filter: function (input, cb) {
+    if (input === 'Bob') {
+      return cb('not_allowed');
+    }
+    return cb(null, input.toUpperCase())
+  }
+});
+greeter.push('anna'); // Prints 'Hello, ANNA!'
+```
+
+This can be particularly useful if your queue needs to do some pre-processing,
+input validation, database lookup, etc. before you load it onto the queue.
+
+You can also define a priority function to control which tasks get
+processed first.
+
+```js
+var greeter = new Queue(function (name, cb) {
+  console.log("Greetings, %s.", name);
+  cb();
+}, {
+  priority: function (name, cb) {
+    if (name === "Steve") return cb(null, 10);
+    if (name === "Mary") return cb(null, 5);
+    if (name === "Joe") return cb(null, 5);
+    cb(null, 1);
+  }
+})
+greeter.push("Steve");
+greeter.push("John");
+greeter.push("Joe");
+greeter.push("Mary");
+
+// Prints out:
+//   Greetings, Steve.
+//   Greetings, Joe.
+//   Greetings, Mary.
+//   Greetings, John.
+```
+
+If `filo` is set to `true` in the example above, then Joe and Mary 
+would swap order.
+
+
+[back to top](#table-of-contents)
+
+---
+
+## Queue Management
+
+#### Retry
+
+You can set tasks to retry `maxRetries` times if they fail. By default,
+tasks will fail (and will not retry.) Optionally, you can set a `retryDelay`
+to wait a little while before retrying.
+
+```js
+var q = new Queue(fn, { maxRetries: 10, retryDelay: 1000 })
+```
+
+
+#### Timing
+
+You can configure the queue to have a `maxTimeout`.
+
+```js
+var q = new Queue(function (name, cb) {
+  someLongTask(function () {
+    cb();
+  })
+}, { maxTimeout: 2000 })
+```
+
+After 2 seconds, the process will throw an error instead of waiting for the
+callback to finish.
+
+You can also delay the queue before it starts its processing. This is the 
+behaviour of a timed cargo.
+
+```js
+var q = new Queue(function (batch, cb) {
+  // Batch [1,2] will process after 2s.
+  cb();
+}, { batchSize: 5, batchDelay: 2000 })
+q.push(1);
+setTimeout(function () {
+  q.push(2);
+}, 1000)
+```
+
+You can also set `afterProcessDelay`, which will delay processing between tasks.
+
+```js
+var q = new Queue(function (task, cb) {
+  cb(); // Will wait 1 second before taking the next task
+}, { afterProcessDelay: 1000 })
+q.push(1);
+q.push(2);
+```
+
+Instead of just the `batchDelay`, you can add a `batchDelayTimeout`, which is for firing off a batch if it hasn't had any new tasks pushed to the queue in the `batchDelayTimeout` time (in milliseconds.)
+
+```js
+var q = new Queue(fn, {
+  batchSize: 50,
+  batchDelay: 5000,
+  batchDelayTimeout: 1000
+})
+q.push(1);
+q.push(2);
+```
+
+In the example above, the queue will wait for 50 items to fill up in 5s or process the queue if no new tasks were added in 1s.
+
+#### Precondition
+
+You can define a function called `precondition` that checks that it's ok to process
+the next batch. If the preconditions fail, it will keep calling this function until
+it passes again.
+
+```js
+var q = new Queue(function (batch, cb) {
+
+  // Do something that requires internet
+
+}, {
+  precondition: function (cb) {
+    isOnline(function (err, ok) {
+      if (ok) {
+        cb(null, true);
+      } else {
+        cb(null, false);
+      }
+    })
+  },
+  preconditionRetryTimeout: 10*1000 // If we go offline, retry every 10s
+})
+```
+
+
+#### Pause/Resume
+
+There are options to control processes while they are running.
+
+You can return an object in your processing function with the functions
+`cancel`, `pause` and `resume`. This will allow operations to pause, resume 
+or cancel while it's running.
+
+```js
+var uploader = new Queue(function (file, cb) {
+  
+  var worker = someLongProcess(file);
+
+  return {
+    cancel: function () {
+      // Cancel the file upload
+    },
+    pause: function () {
+      // Pause the file upload
+    },
+    resume: function () {
+      // Resume the file upload
+    }
+  }
+})
+uploader.push('/path/to/file.pdf');
+uploader.pause();
+uploader.resume();
+```
+
+#### Cancel/Abort
+
+You can also set `cancelIfRunning` to `true`. This will cancel a running task if
+a task with the same ID is pushed onto the queue.
+
+```js
+var uploader = new Queue(function (file, cb) {
+  var request = someLongProcess(file);
+  return {
+    cancel: function () {
+      request.cancel();
+    }
+  }
+}, {
+  id: 'path',
+  cancelIfRunning: true
+})
+uploader.push({ path: '/path/to/file.pdf' });
+// ... Some time later
+uploader.push({ path: '/path/to/file.pdf' });
+```
+
+In the example above, the first upload process is cancelled and the task is requeued.
+
+You can also call `.cancel(taskId)` to cancel and unqueue the task.
+
+```js
+uploader.cancel('/path/to/file.pdf');
+```
+
+Note that if you enable this option in batch mode, it will cancel the entire batch!
+
+
+[back to top](#table-of-contents)
+
+---
+
+## Advanced
+
+#### Updating Task Status
+
+The process function will be run in a context with `progress`,
+`finishBatch` and `failedBatch` functions.
+
+The example below illustrates how you can use these:
+
+```js
+var uploader = new Queue(function (file, cb) {
+  this.failedBatch('some_error')
+  this.finishBatch(result)
+  this.progressBatch(bytesUploaded, totalBytes, "uploading")
+});
+uploader.on('task_finish', function (taskId, result) {
+  // Handle finished result
+})
+uploader.on('task_failed', function (taskId, errorMessage) {
+  // Handle error
+})
+uploader.on('task_progress', function (taskId, completed, total) {
+  // Handle task progress
+})
+
+uploader.push('/some/file.jpg')
+  .on('finish', function (result) {
+    // Handle upload result
+  })
+  .on('failed', function (err) {
+    // Handle error
+  })
+  .on('progress', function (progress) {
+    // progress.eta - human readable string estimating time remaining
+    // progress.pct - % complete (out of 100)
+    // progress.complete - # completed so far
+    // progress.total - # for completion
+    // progress.message - status message
+  })
+```
+
+#### Update Status in Batch mode (batchSize > 1)
+
+You can also complete individual tasks in a batch by using `failedTask` and
+`finishTask` functions.
+
+```js
+var uploader = new Queue(function (files, cb) {
+  this.failedTask(0, 'some_error')         // files[0] has failed with 'some_error'
+  this.finishTask(1, result)               // files[1] has finished with {result}
+  this.progressTask(2, 30, 100, "copying") // files[2] is 30% done, currently copying
+}, { batchSize: 3 });
+uploader.push('/some/file1.jpg')
+uploader.push('/some/file2.jpg')
+uploader.push('/some/file3.jpg')
+```
+
+Note that if you use *-Task and *-Batch functions together, the batch functions will only
+apply to the tasks that have not yet finished/failed.
+
+
+#### Queue Statistics
+
+You can inspect the queue at any given time to see information about how many items are
+queued, average queue time, success rate and total item processed.
+
+```js
+var q = new Queue(fn);
+var stats = q.getStats();
+
+// stats.total = Total tasks processed
+// stats.average = Average process time
+// stats.successRate = % success (between 0 and 1)
+// stats.peak = Most tasks queued at any given point in time
+```
+
+
+[back to top](#table-of-contents)
+
+---
+
+
+## Storage
+
+
+#### Using a store
+
+For your convenience, we have added compatibility for a few storage options.
+
+By default, we are using an in-memory store that doesn't persist. You can change
+to one of our other built in stores by passing in the `store` option.
+
+#### Built-in store
+
+Currently, we support the following stores:
+
+ - memory
+ - sql (SQLite, PostgreSQL)
+
+#### SQLite store (`npm install sqlite3`)
+```
+var q = new Queue(fn, {
+  store: {
+    type: 'sql',
+    dialect: 'sqlite',
+    path: '/path/to/sqlite/file'
+  }
+});
+```
+
+#### PostgreSQL store (`npm install pg`)
+```
+var q = new Queue(fn, {
+  store: {
+    type: 'sql',
+    dialect: 'postgres',
+    host: 'localhost',
+    port: 5432,
+    username: 'username',
+    password: 'password',
+    dbname: 'template1',
+    tableName: 'tasks'
+  }
+});
+```
+
+Please help us add support for more stores; contributions are welcome!
+
+#### Custom Store
+
+Writing your own store is very easy; you just need to implement a few functions
+then call `queue.use(store)` on your store.
+
+```js
+var q = new Queue(fn, { store: myStore });
+```
+
+or
+
+```js
+q.use(myStore);
+```
+
+Your store needs the following functions:
+```js
+q.use({
+  connect: function (cb) {
+    // Connect to your db
+  },
+  getTask: function (taskId, cb) {
+    // Retrieves a task
+  },
+  putTask: function (taskId, task, priority, cb) {
+    // Save task with given priority
+  },
+  takeFirstN: function (n, cb) {
+    // Removes the first N items (sorted by priority and age)
+  },
+  takeLastN: function (n, cb) {
+    // Removes the last N items (sorted by priority and recency)
+  }
+})
+```
+
+[back to top](#table-of-contents)
+
+---
+
+## Full Documentation
+
+#### new Queue(process, options)
+
+The first argument can be either the process function or the `options` object.
+
+A process function is required, all other options are optional.
+
+- `process` - function to process tasks. Will be run with either one single task (if `batchSize` is 1) or as an array of at most `batchSize` items. The second argument will be a callback `cb(error, result)` that must be called regardless of success or failure.
+
+---
+
+- `filter` - function to filter input. Will be run with `input` whatever was passed to `q.push()`. If you define this function, then you will be expected to call the callback `cb(error, task)`. If an error is sent in the callback then the input is rejected.
+- `merge` - function to merge tasks with the same task ID. Will be run with `oldTask`, `newTask` and a callback `cb(error, mergedTask)`. If you define this function then the callback is expected to be called.
+- `priority` - function to determine the priority of a task. Takes in a task and returns callback `cb(error, priority)`.
+- `precondition` - function that runs a check before processing to ensure it can process the next batch. Takes a callback `cb(error, passOrFail)`.
+
+---
+
+- `id` - The property to use as the task ID. This can be a string or a function (for more complicated IDs). The function `(task, cb)` and must call the callback with `cb(error, taskId)`.
+- `cancelIfRunning` - If true, when a task with the same ID is running, its worker will be cancelled. Defaults to `false`.
+- `autoResume` - If true, tasks in the store will automatically start processing once it connects to the store. Defaults to `true`.
+- `failTaskOnProcessException` - If true, when the process function throws an error the batch fails. Defaults to `true`.
+- `filo` - If true, tasks will be completed in a first in, last out order. Defaults to `false`.
+- `batchSize` - The number of tasks (at most) that can be processed at once. Defaults to `1`.
+- `batchDelay` - Number of milliseconds to delay before starting to popping items off the queue. Defaults to `0`.
+- `batchDelayTimeout` - Number of milliseconds to wait for a new task to arrive before firing off the batch. Defaults to `Infinity`.
+- `concurrent` - Number of workers that can be running at any given time. Defaults to `1`.
+- `maxTimeout` - Number of milliseconds before a task is considered timed out. Defaults to `Infinity`.
+- `afterProcessDelay` - Number of milliseconds to delay before processing the next batch of items. Defaults to `1`.
+- `maxRetries` - Maximum number of attempts to retry on a failed task. Defaults to `0`.
+- `retryDelay` - Number of milliseconds before retrying. Defaults to `0`.
+- `storeMaxRetries` - Maximum number of attempts before giving up on the store. Defaults to `Infinity`.
+- `storeRetryTimeout` - Number of milliseconds to delay before trying to connect to the store again. Defaults to `1000`.
+- `preconditionRetryTimeout` - Number of milliseconds to delay before checking the precondition function again. Defaults to `1000`.
+- `store` - Represents the options for the initial store. Can be an object containing `{ type: storeType, ... options ... }`, or the store instance itself.
+
+#### Methods on Queue
+
+- `push(task, cb)` - Push a task onto the queue, with an optional callback when it completes. Returns a `Ticket` object.
+- `pause()` - Pauses the queue: tries to pause running tasks and prevents tasks from getting processed until resumed.
+- `resume()` - Resumes the queue and its runnign tasks.
+- `destroy(cb)` - Destroys the queue: closes the store, tries to clean up.
+- `use(store)` - Sets the queue to read from and write to the given store.
+- `getStats()` - Gets the aggregate stats for the queue. Returns an object with properties `successRate`, `peak`, `total` and `average`, representing the success rate on tasks, peak number of items queued, total number of items processed and average processing time, respectively.
+- `resetStats()` - Resets all of the aggregate stats.
+
+#### Events on Queue
+
+- `task_queued` - When a task is queued
+- `task_accepted` - When a task is accepted
+- `task_started` - When a task begins processing
+- `task_finish` - When a task is completed
+- `task_failed` - When a task fails
+- `task_progress` - When a task progress changes
+- `batch_finish` - When a batch of tasks (or worker) completes
+- `batch_failed` - When a batch of tasks (or worker) fails
+- `batch_progress` - When a batch of tasks (or worker) updates its progress
+
+#### Events on Ticket
+
+- `accept` - When the corresponding task is accepted (has passed filter)
+- `queued` - When the corresponding task is queued (and saved into the store)
+- `started` - When the corresponding task is started
+- `progress` - When the corresponding task progress changes
+- `finish` - When the corresponding task completes
+- `failed` - When the corresponding task fails
+

node_modules/better-queue/circle.yml

@@ -0,0 +1,5 @@
+test:
+  override:
+    - mocha test --reporter mocha-junit-reporter:
+        environment:
+          MOCHA_FILE: $CIRCLE_TEST_REPORTS/junit/test-results.xml

node_modules/better-queue/lib/queue.js

@@ -0,0 +1,711 @@
+var uuid = require('uuid');
+var util = require('util');
+var EE   = require('events').EventEmitter;
+var Ticket = require('./ticket');
+var Worker = require('./worker');
+var Tickets = require('./tickets');
+
+function Queue(process, opts) {
+  var self = this;
+  opts = opts || {};
+  if (typeof process === 'object') {
+    opts = process || {};
+  }
+  if (typeof process === 'function') {
+    opts.process = process;
+  }
+  if (!opts.process) {
+    throw new Error("Queue has no process function.");
+  }
+
+  opts = opts || {};
+
+  self.process = opts.process || function (task, cb) { cb(null, {}) };
+  self.filter = opts.filter || function (input, cb) { cb(null, input) };
+  self.merge = opts.merge || function (oldTask, newTask, cb) { cb(null, newTask) };
+  self.precondition = opts.precondition || function (cb) { cb(null, true) };
+  self.setImmediate = opts.setImmediate || setImmediate;
+  self.id = opts.id || 'id';
+  self.priority = opts.priority || null;
+
+  self.cancelIfRunning = (opts.cancelIfRunning === undefined ? false : !!opts.cancelIfRunning);
+  self.autoResume = (opts.autoResume === undefined ? true : !!opts.autoResume);
+  self.failTaskOnProcessException = (opts.failTaskOnProcessException === undefined ? true : !!opts.failTaskOnProcessException);
+  self.filo = opts.filo || false;
+  self.batchSize = opts.batchSize || 1;
+  self.batchDelay = opts.batchDelay || 0;
+  self.batchDelayTimeout = opts.batchDelayTimeout || Infinity;
+  self.afterProcessDelay = opts.afterProcessDelay || 0;
+  self.concurrent = opts.concurrent || 1;
+  self.maxTimeout = opts.maxTimeout || Infinity;
+  self.maxRetries = opts.maxRetries || 0;
+  self.retryDelay = opts.retryDelay || 0;
+  self.storeMaxRetries = opts.storeMaxRetries || Infinity;
+  self.storeRetryTimeout = opts.storeRetryTimeout || 1000;
+  self.preconditionRetryTimeout = opts.preconditionRetryTimeout || 1000;
+
+  // Statuses
+  self._queuedPeak = 0;
+  self._queuedTime = {};
+  self._processedTotalElapsed = 0;
+  self._processedAverage = 0;
+  self._processedTotal = 0;
+  self._failedTotal = 0;
+  self.length = 0;
+  self._stopped = false;
+  self._saturated = false;
+
+  self._preconditionRetryTimeoutId = null;
+  self._batchTimeoutId = null;
+  self._batchDelayTimeoutId = null;
+  self._connected = false;
+  self._storeRetries = 0;
+  
+  // Locks
+  self._hasMore = false;
+  self._isWriting = false;
+  self._writeQueue = [];
+  self._writing = {};
+  self._tasksWaitingForConnect = [];
+
+  self._calledDrain = true;
+  self._calledEmpty = true;
+  self._fetching = 0;
+  self._running = 0;  // Active running tasks
+  self._retries = {}; // Map of taskId => retries
+  self._workers = {}; // Map of taskId => active job
+  self._tickets = {}; // Map of taskId => tickets
+
+  // Initialize Storage
+  self.use(opts.store || 'memory');
+  if (!self._store) {
+    throw new Error('Queue cannot continue without a valid store.')
+  }
+}
+
+util.inherits(Queue, EE);
+
+Queue.prototype.destroy = function (cb) {
+  cb = cb || function () {};
+  var self = this;
+
+  // Statuses
+  self._hasMore = false;
+  self._isWriting = false;
+  self._writeQueue = [];
+  self._writing = {};
+  self._tasksWaitingForConnect = [];
+
+  // Clear internals
+  self._tickets = {};
+  self._workers = {};
+  self._fetching = 0;
+  self._running = {};
+  self._retries = {};
+  self._calledEmpty = true;
+  self._calledDrain = true;
+  self._connected = false;
+  self.pause();
+
+  if (typeof self._store.close === 'function') {
+    self._store.close(cb);
+  } else {
+    cb();
+  }
+}
+
+Queue.prototype.resetStats = function () {
+  this._queuedPeak = 0;
+  this._processedTotalElapsed = 0;
+  this._processedAverage = 0;
+  this._processedTotal = 0;
+  this._failedTotal = 0;
+}
+
+Queue.prototype.getStats = function () {
+  var successRate = this._processedTotal === 0 ? 0 : (1 - (this._failedTotal / this._processedTotal));
+  return {
+    successRate: successRate,
+    peak: this._queuedPeak,
+    average: this._processedAverage,
+    total: this._processedTotal
+  }
+}
+
+Queue.prototype.use = function (store, opts) {
+  var self = this;
+  var loadStore = function (store) {
+    var Store;
+    try {
+      Store = require('better-queue-' + store);
+    } catch (e) {
+      throw new Error('Attempting to require better-queue-' + store + ', but failed.\nPlease ensure you have this store installed via npm install --save better-queue-' + store)
+    }
+    return Store;
+  }
+  if (typeof store === 'string') {
+    var Store = loadStore(store);
+    self._store = new Store(opts);
+  } else if (typeof store === 'object' && typeof store.type === 'string') {
+    var Store = loadStore(store.type);
+    self._store = new Store(store);
+  } else if (typeof store === 'object' && store.putTask && store.getTask && ((self.filo && store.takeLastN) || (!self.filo && store.takeFirstN))) {
+    self._store = store;
+  } else {
+    throw new Error('unknown_store');
+  }
+  self._connected = false;
+  self._tasksWaitingForConnect = [];
+  self._connectToStore();
+}
+
+Queue.prototype._connectToStore = function () {
+  var self = this;
+  if (self._connected) return;
+  if (self._storeRetries >= self.storeMaxRetries) {
+    return self.emit('error', new Error('failed_connect_to_store'));
+  }
+  self._storeRetries++;
+  self._store.connect(function (err, len) {
+    if (err) return setTimeout(function () {
+      self._connectToStore();
+    }, self.storeRetryTimeout);
+    if (len === undefined || len === null) throw new Error("store_not_returning_length");
+    self.length = parseInt(len);
+    if (isNaN(self.length)) throw new Error("length_is_not_a_number");
+    if (self.length) self._calledDrain = false;
+    self._connected = true;
+    self._storeRetries = 0;
+    self._store.getRunningTasks(function (err, running) {
+      if (!self._stopped && self.autoResume) {
+        Object.keys(running).forEach(function (lockId) {
+          self._running++;
+          self._startBatch(running[lockId], {}, lockId);
+        })
+        self.resume();
+      }
+      for (var i = 0; i < self._tasksWaitingForConnect.length; i++) {
+        self.push(self._tasksWaitingForConnect[i].input, self._tasksWaitingForConnect[i].ticket);
+      }
+    })
+  })
+
+}
+
+Queue.prototype.resume = function () {
+  var self = this;
+  self._stopped = false;
+  self._getWorkers().forEach(function (worker) {
+    if (typeof worker.resume === 'function') {
+      worker.resume();
+    }
+  })
+  setTimeout(function () {
+    self._processNextAfterTimeout();
+  }, 0)
+}
+
+Queue.prototype.pause = function () {
+  this._stopped = true;
+  this._getWorkers().forEach(function (worker) {
+    if (typeof worker.pause === 'function') {
+      worker.pause();
+    }
+  })
+}
+
+Queue.prototype.cancel = function (taskId, cb) {
+  cb = cb || function(){};
+  var self = this;
+  var worker = self._workers[taskId];
+  if (worker) {
+    worker.cancel();
+  }
+  self._store.deleteTask(taskId, cb);
+}
+
+Queue.prototype.push = function (input, cb) {
+  var self = this;
+  var ticket = new Ticket();
+  if (cb instanceof Ticket) {
+    ticket = cb;
+  } else if (cb) {
+    ticket
+      .on('finish', function (result) { cb(null, result) })
+      .on('failed', function (err) { cb(err) })
+  }
+  if (!self._connected) {
+    self._tasksWaitingForConnect.push({ input: input, ticket: ticket });
+    return ticket;
+  }
+
+  self.filter(input, function (err, task) {
+    if (err || task === undefined || task === false || task === null) {
+      return ticket.failed('input_rejected');
+    }
+    var acceptTask = function (taskId) {
+      setTimeout(function () {
+        self._queueTask(taskId, task, ticket);
+      }, 0)
+    }
+    if (typeof self.id === 'function') {
+      self.id(task, function (err, id) {
+        if (err) return ticket.failed('id_error');
+        acceptTask(id);
+      })
+    } else if (typeof self.id === 'string' && typeof task === 'object') {
+      acceptTask(task[self.id])
+    } else {
+      acceptTask();
+    }
+  })
+  return ticket;
+}
+
+Queue.prototype._getWorkers = function () {
+  var self = this;
+  var workers = [];
+  Object.keys(self._workers).forEach(function (taskId) {
+    var worker = self._workers[taskId];
+    if (worker && workers.indexOf(worker) === -1) {
+      workers.push(worker);
+    }
+  })
+  return workers;
+}
+
+Queue.prototype._writeNextTask = function () {
+  var self = this;
+  if (self._isWriting) return;
+  if (!self._writeQueue.length) return;
+  self._isWriting = true;
+
+  var taskId = self._writeQueue.shift();
+  var finishedWrite = function () {
+    self._isWriting = false;
+    self.setImmediate(function () {
+      self._writeNextTask();
+    })
+  }
+
+  if (!self._writing[taskId]) {
+    delete self._writing[taskId];
+    return finishedWrite();
+  }
+
+  var task = self._writing[taskId].task;
+  var priority = self._writing[taskId].priority;
+  var isNew = self._writing[taskId].isNew;
+  var writeId = self._writing[taskId].id;
+  var tickets = self._writing[taskId].tickets;
+
+  self._store.putTask(taskId, task, priority, function (err) {
+
+    // Check if task has changed since put
+    if (self._writing[taskId] && self._writing[taskId].id !== writeId) {
+      self._writeQueue.unshift(taskId);
+      return finishedWrite();
+    }
+    delete self._writing[taskId];
+    
+    // If something else has written to taskId, then wait.
+    if (err) {
+      tickets.failed('failed_to_put_task');
+      return finishedWrite();
+    }
+    
+    // Task is in the queue -- update stats
+    if (isNew) {
+      self.length++;
+      if (self._queuedPeak < self.length) {
+        self._queuedPeak = self.length;
+      }
+      self._queuedTime[taskId] = new Date().getTime();
+    }
+
+    // Notify the ticket
+    if (self._tickets[taskId]) {
+      self._tickets[taskId].push(tickets);
+    } else {
+      self._tickets[taskId] = tickets;
+    }
+    self.emit('task_queued', taskId, task);
+    tickets.queued();
+
+    // If it's a new task, make sure to call drain after.
+    if (isNew) {
+      self._calledDrain = false;
+      self._calledEmpty = false;
+    }
+
+    // If already fetching, mark that there are additions to the queue
+    if (self._fetching > 0) {
+      self._hasMore = true;
+    }
+
+    // Clear batchDelayTimeout
+    if (self.batchDelayTimeout < Infinity) {
+      if (self._batchDelayTimeoutId) clearTimeout(self._batchDelayTimeoutId)
+      self._batchDelayTimeoutId = setTimeout(function () {
+        self._batchDelayTimeoutId = null;
+        if (self._batchTimeoutId) clearTimeout(self._batchTimeoutId);
+        self._batchTimeoutId = null;
+        self._processNextIfAllowed();
+      }, self.batchDelayTimeout)
+    }
+
+    // Finish writing
+    finishedWrite();
+    self._processNextAfterTimeout();
+  })
+}
+
+Queue.prototype._queueTask = function (taskId, newTask, ticket) {
+  var self = this;
+  var emptyTicket = new Ticket();
+  ticket = ticket || emptyTicket;
+  var isUUID = false;
+  if (!taskId) {
+    taskId = uuid.v4();
+    isUUID = true;
+  }
+  var priority;
+  var oldTask = null;
+  var isNew = true;
+  var putTask = function () {
+    if (!self._connected) return;
+
+    // Save ticket
+    var tickets = (self._writing[taskId] && self._writing[taskId].tickets) || new Tickets();
+    if (ticket !== emptyTicket) {
+      tickets.push(ticket);
+    }
+
+    // Add to queue
+    var alreadyQueued = !!self._writing[taskId];
+    self._writing[taskId] = {
+      id: uuid.v4(),
+      isNew: isNew,
+      task: newTask,
+      priority: priority,
+      tickets: tickets
+    };
+    if (!alreadyQueued) {
+      self._writeQueue.push(taskId);
+    }
+
+    self._writeNextTask();
+  }
+  var updateTask = function () {
+    ticket.accept();
+    self.emit('task_accepted', taskId, newTask);
+
+    if (!self.priority) return putTask();
+    self.priority(newTask, function (err, p) {
+      if (err) return ticket.failed('failed_to_prioritize');
+      priority = p;
+      putTask();
+    })
+  }
+  var mergeTask = function () {
+    if (!oldTask) return updateTask();
+    self.merge(oldTask, newTask, function (err, mergedTask) {
+      if (err) return ticket.failed('failed_task_merge');
+      if (mergedTask === undefined) return;
+      newTask = mergedTask;
+      updateTask();
+    })
+  }
+
+  if (isUUID) {
+    return updateTask();
+  }
+
+  var worker = self._workers[taskId];
+  if (self.cancelIfRunning && worker) {
+    worker.cancel();
+  }
+
+  // Check if task is writing
+  if (self._writing[taskId]) {
+    oldTask = self._writing[taskId].task;
+    return mergeTask();
+  }
+
+  // Check store for task
+  self._store.getTask(taskId, function (err, savedTask) {
+    if (err) return ticket.failed('failed_to_get');
+
+    // Check if it's already in the store
+    if (savedTask !== undefined) {
+      isNew = false;
+    }
+
+    // Check if task is writing
+    if (self._writing[taskId]) {
+      oldTask = self._writing[taskId].task;
+      return mergeTask();
+    }
+
+    // No task before
+    if (savedTask === undefined) {
+      return updateTask();
+    }
+
+    oldTask = savedTask;
+    mergeTask();
+  })
+}
+
+Queue.prototype._emptied = function () {
+  if (this._calledEmpty) return;
+  this._calledEmpty = true;
+  this.emit('empty');
+}
+
+Queue.prototype._drained = function () {
+  if (this._calledDrain) return;
+  this._calledDrain = true;
+  this.emit('drain');
+}
+
+Queue.prototype._getNextBatch = function (cb) {
+  this._store[this.filo ? 'takeLastN' : 'takeFirstN'](this.batchSize, cb)
+}
+
+Queue.prototype._processNextAfterTimeout = function () {
+  var self = this;
+  if (self.length >= self.batchSize) {
+    if (self._batchTimeoutId) {
+      clearTimeout(self._batchTimeoutId);
+      self._batchTimeoutId = null;
+    }
+    self.setImmediate(function () {
+      self._processNextIfAllowed();
+    })
+  } else if (!self._batchTimeoutId && self.batchDelay < Infinity) {
+    self._batchTimeoutId = setTimeout(function () {
+      self._batchTimeoutId = null;
+      self._processNextIfAllowed();
+    }, self.batchDelay)
+  }
+}
+
+Queue.prototype._processNextIfAllowed = function () {
+  var self = this;
+  if (!self._connected) return;
+  if (self._stopped) return;
+  
+  self._saturated = (self._running + self._fetching >= self.concurrent);
+  if (self._saturated) return;
+  if (!self.length) {
+    if (!self._hasMore) {
+      self._emptied();
+      if (!self._running) {
+        self._drained();
+      }
+    }
+    return;
+  }
+
+  self.precondition(function (err, pass) {
+    if (err || !pass) {
+      if (!self._preconditionRetryTimeoutId && self.preconditionRetryTimeout) {
+        self._preconditionRetryTimeoutId = setTimeout(function () {
+          self._preconditionRetryTimeoutId = null;
+          self._processNextIfAllowed();
+        }, self.preconditionRetryTimeout)
+      }
+    } else {
+      self._processNext();
+    }
+  })
+}
+
+Queue.prototype._processNext = function () {
+  var self = this;
+  // FIXME: There may still be things writing
+  self._hasMore = false;
+  self._fetching++;
+  self._getNextBatch(function (err, lockId) {
+    self._fetching--;
+    if (err || lockId === undefined) return;
+    self._store.getLock(lockId, function (err, batch) {
+      if (err || !batch) return;
+      var batchSize = Object.keys(batch).length;
+      var isEmpty = (batchSize === 0);
+
+      if (self.length < batchSize) {
+        self.length = batchSize;
+      }
+
+      if (!self._hasMore && isEmpty) {
+        self._emptied();
+        if (!self._running) {
+          self._drained();
+        }
+        return;
+      }
+
+      // The write queue wasn't empty on fetch, so we should fetch more.
+      if (self._hasMore && isEmpty) {
+        return self._processNextAfterTimeout()
+      }
+
+      var tickets = {};
+      Object.keys(batch).forEach(function (taskId) {
+        var ticket = self._tickets[taskId];
+        if (ticket) {
+          ticket.started();
+          tickets[taskId] = ticket;
+          delete self._tickets[taskId];
+        }
+      })
+
+      // Acquire lock on process
+      self._running++;
+      self._startBatch(batch, tickets, lockId);
+      
+      if (self.concurrent - self._running > 1) {
+        // Continue processing until saturated
+        self._processNextIfAllowed();
+      }
+    });
+  });
+}
+
+Queue.prototype._startBatch = function (batch, tickets, lockId) {
+  var self = this;
+  var taskIds = Object.keys(batch);
+  var timeout = null;
+  var worker = new Worker({
+    fn: self.process,
+    batch: batch,
+    single: (self.batchSize === 1),
+    failTaskOnProcessException: self.failTaskOnProcessException
+  })
+  var updateStatsForEndedTask = function (taskId) {
+    self._processedTotal++;
+    var stats = {};
+    if (!self._queuedTime[taskId]) return stats;
+
+    var elapsed = (new Date().getTime() - self._queuedTime[taskId]);
+    delete self._queuedTime[taskId];
+    
+    if (elapsed > 0) {
+      stats.elapsed = elapsed;
+      self._processedTotalElapsed += elapsed;
+      self._processedAverage = self._processedTotalElapsed/self._processedTotal;
+    }
+    return stats;
+  }
+
+  if (self.maxTimeout < Infinity) {
+    timeout = setTimeout(function () {
+      worker.failedBatch('task_timeout');
+    }, self.maxTimeout);
+  }
+  worker.on('task_failed', function (id, msg) {
+    var taskId = taskIds[id];
+    self._retries[taskId] = self._retries[taskId] || 0;
+    self._retries[taskId]++;
+    if (worker.cancelled || self._retries[taskId] >= self.maxRetries) {
+      var stats = updateStatsForEndedTask(taskId);
+      if (tickets[taskId]) {
+        // Mark as a failure
+        tickets[taskId].failed(msg);
+        delete tickets[taskId];
+      }
+      self._failedTotal++;
+      self.emit('task_failed', taskId, msg, stats);
+    } else {
+      if (self.retryDelay) {
+        // Pop back onto queue and retry
+        setTimeout(function () {
+          self.emit('task_retry', taskId, self._retries[taskId]);
+          self._queueTask(taskId, batch[taskId], tickets[taskId]);
+        }, self.retryDelay)
+      } else {
+        self.setImmediate(function () {
+          self.emit('task_retry', taskId, self._retries[taskId]);
+          self._queueTask(taskId, batch[taskId], tickets[taskId]);
+        });
+      }
+    }
+  })
+  worker.on('task_finish', function (id, result) {
+    var taskId = taskIds[id];
+    var stats = updateStatsForEndedTask(taskId);
+    if (tickets[taskId]) {
+      tickets[taskId].finish(result);
+      delete tickets[taskId];
+    }
+    self.emit('task_finish', taskId, result, stats);
+  })
+  worker.on('task_progress', function (id, progress) {
+    var taskId = taskIds[id];
+    if (tickets[taskId]) {
+      tickets[taskId].progress(progress);
+      delete tickets[taskId];
+    }
+    self.emit('task_progress', taskId, progress);
+  })
+  worker.on('progress', function (progress) {
+    self.emit('batch_progress', progress);
+  })
+  worker.on('finish', function (result) {
+    self.emit('batch_finish', result);
+  })
+  worker.on('failed', function (err) {
+    self.emit('batch_failed', err);
+  })
+  worker.on('end', function () {
+    self.length -= Object.keys(batch).length;
+    if (timeout) {
+      clearTimeout(timeout);
+    }
+    var finishAndGetNext = function () {
+      if (!self._connected) return;
+      self._store.releaseLock(lockId, function (err) {
+        if (err) {
+          // If we cannot release the lock then retry
+          return setTimeout(function () {
+            finishAndGetNext();
+          }, 1)
+        }
+        self._running--;
+        taskIds.forEach(function (taskId) {
+          if (self._workers[taskId] && !self._workers[taskId].active) {
+            delete self._workers[taskId];
+          }
+        });
+        self._processNextAfterTimeout();
+      })
+    }
+    if (self.afterProcessDelay) {
+      setTimeout(function () {
+        finishAndGetNext()
+      }, self.afterProcessDelay);
+    } else {
+      self.setImmediate(function () {
+        finishAndGetNext()
+      })
+    }
+  })
+
+  taskIds.forEach(function (taskId) {
+    self._workers[taskId] = worker;
+  });
+  
+  try {
+    worker.start();
+    taskIds.forEach(function (taskId) {
+      self.emit('task_started', taskId, batch[taskId])
+    });
+  } catch (e) {
+    self.emit('error', e);
+  }
+
+}
+
+module.exports = Queue;

node_modules/better-queue/lib/ticket.js

@@ -0,0 +1,81 @@
+
+var util = require('util');
+var EE   = require('events').EventEmitter;
+var ETA  = require('node-eta');
+
+function Ticket(opts) {
+  this.isAccepted = false;
+  this.isQueued = false;
+  this.isStarted = false;
+  this.isFailed = false;
+  this.isFinished = false;
+  this.result = null;
+  this.status = 'created';
+  this.eta = new ETA();
+}
+
+util.inherits(Ticket, EE);
+
+Ticket.prototype.accept = function () {
+  this.status = 'accepted';
+  this.isAccepted = true;
+  this.emit('accepted');
+}
+
+Ticket.prototype.queued = function () {
+  this.status = 'queued';
+  this.isQueued = true;
+  this.emit('queued');
+}
+
+Ticket.prototype.unqueued = function () {
+  this.status = 'accepted';
+  this.isQueued = false;
+  this.emit('unqueued');
+}
+
+Ticket.prototype.started = function () {
+  this.eta.count = 1;
+  this.eta.start();
+  this.isStarted = true;
+  this.status = 'in-progress';
+  this.emit('started');
+}
+
+Ticket.prototype.failed = function (msg) {
+  this.isFailed = true;
+  this.isFinished = true;
+  this.status = 'failed';
+  this.emit('failed', msg);
+}
+
+Ticket.prototype.finish = function (result) {
+  this.eta.done = this.eta.count;
+  this.isFinished = true;
+  this.status = 'finished';
+  this.result = result;
+  this.emit('finish', this.result);
+}
+
+Ticket.prototype.stopped = function () {
+  this.eta = new ETA();
+  this.isFinished = false;
+  this.isStarted = false;
+  this.status = 'queued';
+  this.result = null;
+  this.emit('stopped');
+}
+
+Ticket.prototype.progress = function (progress) {
+  this.eta.done = progress.complete;
+  this.eta.count = progress.total;
+  this.emit('progress', {
+    complete: this.eta.done,
+    total: this.eta.count,
+    pct: (this.eta.done/this.eta.count)*100,
+    eta: this.eta.format('{{etah}}'),
+    message: progress.message
+  });
+}
+
+module.exports = Ticket;

node_modules/better-queue/lib/tickets.js

@@ -0,0 +1,34 @@
+
+var Ticket = require('./ticket');
+
+function Tickets() {
+  this.tickets = [];
+}
+
+Tickets.prototype._apply = function (fn, args) {
+  this.tickets.forEach(function (ticket) {
+    ticket[fn].apply(ticket, args);
+  })
+}
+
+Tickets.prototype.push = function (ticket) {
+  var self = this;
+  if (ticket instanceof Tickets) {
+    return ticket.tickets.forEach(function (ticket) {
+      self.push(ticket)
+    })
+  }
+  if (ticket instanceof Ticket) {
+    if (self.tickets.indexOf(ticket) === -1) {
+      self.tickets.push(ticket);
+    }
+  }
+}
+
+Object.keys(Ticket.prototype).forEach(function (method) {
+  Tickets.prototype[method] = function () {
+    this._apply(method, arguments);
+  }
+})
+
+module.exports = Tickets;

node_modules/better-queue/lib/worker.js

@@ -0,0 +1,192 @@
+
+var util = require('util');
+var EE   = require('events').EventEmitter;
+var ETA  = require('node-eta');
+
+function Worker(opts) {
+  this.fn = opts.fn;
+  this.batch = opts.batch;
+  this.single = opts.single;
+  this.active = false;
+  this.cancelled = false;
+  this.failTaskOnProcessException = opts.failTaskOnProcessException;
+}
+
+util.inherits(Worker, EE);
+
+Worker.prototype.setup = function () {
+  var self = this;
+
+  // Internal
+  self._taskIds = Object.keys(self.batch);
+  self._process = {};
+  self._waiting = {};
+  self._eta = new ETA();
+
+  // Task counts
+  self.counts = {
+    finished: 0,
+    failed: 0,
+    completed: 0,
+    total: self._taskIds.length,
+  };
+
+  // Progress
+  self.status = 'ready';
+  self.progress = {
+    tasks: {},
+    complete: 0,
+    total: self._taskIds.length,
+    eta: '',
+  };
+
+  // Setup
+  self._taskIds.forEach(function (taskId, id) {
+    self._waiting[id] = true;
+    self.progress.tasks[id] = {
+      pct: 0,
+      complete: 0,
+      total: 1,
+    }
+  })
+}
+
+Worker.prototype.start = function () {
+  var self = this;
+  if (self.active) return;
+
+  self.setup();
+  self._eta.count = self.progress.total;
+  self._eta.start();
+
+  self.active = true;
+  self.status = 'in-progress';
+  var tasks = self._taskIds.map(function (taskId) { return self.batch[taskId] });
+  if (self.single) {
+    tasks = tasks[0]
+  }
+  try {
+    self._process = self.fn.call(self, tasks, function (err, result) {
+      if (!self.active) return;
+      if (err) {
+        self.failedBatch(err);
+      } else {
+        self.finishBatch(result);
+      }
+    })
+  } catch (err) {
+    if (self.failTaskOnProcessException) {
+      self.failedBatch(err);
+    } else {
+      throw new Error(err);
+    }
+  }
+  self._process = self._process || {};
+}
+
+Worker.prototype.end = function () {
+  if (!this.active) return;
+  this.status = 'finished';
+  this.active = false;
+  this.emit('end');
+}
+
+Worker.prototype.resume = function () {
+  if (typeof this._process.resume === 'function') {
+    this._process.resume();
+  }
+  this.status = 'in-progress';
+}
+
+Worker.prototype.pause = function () {
+  if (typeof this._process.pause === 'function') {
+    this._process.pause();
+  }
+  this.status = 'paused';
+}
+
+Worker.prototype.cancel = function () {
+  this.cancelled = true;
+  if (typeof this._process.cancel === 'function') {
+    this._process.cancel();
+  }
+  if (typeof this._process.abort === 'function') {
+    this._process.abort();
+  }
+  this.failedBatch('cancelled');
+}
+
+Worker.prototype.failedBatch = function (msg) {
+  var self = this;
+  if (!self.active) return;
+  Object.keys(self._waiting).forEach(function (id) {
+    if (!self._waiting[id]) return;
+    self.failedTask(id, msg);
+  })
+  self.emit('failed', msg);
+  self.end();
+}
+
+Worker.prototype.failedTask = function (id, msg) {
+  var self = this;
+  if (!self.active) return;
+  if (self._waiting[id]) {
+    self._waiting[id] = false;
+    self.counts.failed++;
+    self.counts.completed++;
+    self.emit('task_failed', id, msg);
+  }
+}
+
+Worker.prototype.finishBatch = function (result) {
+  var self = this;
+  if (!self.active) return;
+  Object.keys(self._waiting).forEach(function (id) {
+    if (!self._waiting[id]) return;
+    self.finishTask(id, result);
+  })
+  self.emit('finish', result);
+  self.end();
+}
+
+Worker.prototype.finishTask = function (id, result) {
+  var self = this;
+  if (!self.active) return;
+  if (self._waiting[id]) {
+    self._waiting[id] = false;
+    self.counts.finished++;
+    self.counts.completed++;
+    self.emit('task_finish', id, result);
+  }
+}
+
+Worker.prototype.progressBatch = function (complete, total, msg) {
+  var self = this;
+  if (!self.active) return;
+  Object.keys(self._waiting).forEach(function (id) {
+    if (!self._waiting[id]) return;
+    self.progressTask(id, complete, total, msg);
+  })
+  self.progress.complete = 0;
+  self._taskIds.forEach(function (taskId, id) {
+    self.progress.complete += self.progress.tasks[id].pct;
+  })
+  self._eta.done = self.progress.complete;
+  self.progress.eta = self._eta.format('{{etah}}')
+  self.progress.message = msg || '';
+  self.emit('progress', self.progress);
+}
+
+Worker.prototype.progressTask = function (id, complete, total, msg) {
+  var self = this;
+  if (!self.active) return;
+  if (self._waiting[id]) {
+    self.progress.tasks[id].complete = complete;
+    self.progress.tasks[id].total = self.progress.tasks[id].total || total;
+    self.progress.tasks[id].message = self.progress.tasks[id].message || msg;
+    self.progress.tasks[id].pct = Math.max(0, Math.min(1, complete/total));
+    self.emit('task_progress', id, self.progress.tasks[id]);
+  }
+}
+
+module.exports = Worker;

node_modules/better-queue/package.json

@@ -0,0 +1,67 @@
+{
+  "_from": "better-queue@^3.8.6",
+  "_id": "better-queue@3.8.10",
+  "_inBundle": false,
+  "_integrity": "sha512-e3gwNZgDCnNWl0An0Tz6sUjKDV9m6aB+K9Xg//vYeo8+KiH8pWhLFxkawcXhm6FpM//GfD9IQv/kmvWCAVVpKA==",
+  "_location": "/better-queue",
+  "_phantomChildren": {},
+  "_requested": {
+    "type": "range",
+    "registry": true,
+    "raw": "better-queue@^3.8.6",
+    "name": "better-queue",
+    "escapedName": "better-queue",
+    "rawSpec": "^3.8.6",
+    "saveSpec": null,
+    "fetchSpec": "^3.8.6"
+  },
+  "_requiredBy": [
+    "/gatsby",
+    "/gatsby-source-filesystem"
+  ],
+  "_resolved": "https://registry.npmjs.org/better-queue/-/better-queue-3.8.10.tgz",
+  "_shasum": "1c93b9ec4cb3d1b72eb91d0efcb84fc80e8c6835",
+  "_spec": "better-queue@^3.8.6",
+  "_where": "/Users/stefanfejes/Projects/30-seconds-of-python-code/node_modules/gatsby",
+  "author": {
+    "name": "Diamond Inc.",
+    "email": "ops@diamond.io"
+  },
+  "bugs": {
+    "url": "https://github.com/diamondio/better-queue/issues"
+  },
+  "bundleDependencies": false,
+  "dependencies": {
+    "better-queue-memory": "^1.0.1",
+    "node-eta": "^0.9.0",
+    "uuid": "^3.0.0"
+  },
+  "deprecated": false,
+  "description": "Better Queue for NodeJS",
+  "devDependencies": {
+    "mocha": "^2.3.4",
+    "mocha-junit-reporter": "^1.12.1"
+  },
+  "directories": {
+    "test": "test"
+  },
+  "homepage": "https://github.com/diamondio/better-queue",
+  "keywords": [
+    "queue",
+    "cargo",
+    "async",
+    "timeout",
+    "priority"
+  ],
+  "license": "MIT",
+  "main": "lib/queue.js",
+  "name": "better-queue",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/diamondio/better-queue.git"
+  },
+  "scripts": {
+    "test": "mocha"
+  },
+  "version": "3.8.10"
+}

node_modules/better-queue/test/basic.js

@@ -0,0 +1,488 @@
+var assert = require('assert');
+var helper = require('./lib/helper');
+var Queue = require('../lib/queue');
+
+describe('Basic Queue', function() {
+  afterEach(helper.destroyQueues);
+
+  it('should succeed', function (done) {
+    var q = new Queue(function (n, cb) {
+      cb(null, n+1)
+    }, { autoResume: true })
+    q.on('task_finish', function (taskId, r) {
+      assert.equal(r, 2);
+      done();
+    })
+    q.push(1, function (err, r) {
+      assert.equal(r, 2);
+    })
+    this.q = q;
+  });
+
+  it('should fail task if failTaskOnProcessException is true', function (done) {
+    var q = new Queue(function (n, cb) {
+      throw new Error("failed");
+    }, { autoResume: true })
+    q.on('task_failed', function (taskId, err) {
+      assert.equal(err.message, "failed");
+      done();
+    })
+    q.push(1)
+    this.q = q;
+  });
+
+  it('should emit an error if failTaskOnProcessException is false', function (done) {
+    var q = new Queue(function (n, cb) {
+      throw new Error("failed");
+    }, { failTaskOnProcessException: false, autoResume: true })
+    q.on('error', function () {
+      done();
+    })
+    q.push(1)
+    this.q = q;
+  });
+
+  it('should fail', function (done) {
+    var q = new Queue(function (n, cb) {
+      cb('nope')
+    }, { autoResume: true })
+    q.on('task_failed', function (taskId, msg) {
+      assert.equal(msg, 'nope');
+      done();
+    })
+    q.push(1, function (err, r) {
+      assert.equal(err, 'nope');
+    })
+    this.q = q;
+  });
+
+  it('should run fifo', function (done) {
+    var finished = 0;
+    var queued = 0;
+    var q = new Queue(function (num, cb) { cb() })
+    q.on('task_finish', function () {
+      if (finished >= 3) {
+        done();
+      }
+    })
+    q.on('task_queued', function () {
+      queued++;
+      if (queued >= 3) {
+        q.resume();
+      }
+    })
+    q.pause();
+    q.push(1, function (err, r) {
+      assert.equal(finished, 0);
+      finished++;
+    })
+    q.push(2, function (err, r) {
+      assert.equal(finished, 1);
+      finished++;
+    })
+    q.push(3, function (err, r) {
+      assert.equal(finished, 2);
+      finished++;
+    })
+    this.q = q;
+  })
+
+  it('should prioritize', function (done) {
+    var q = new Queue(function (num, cb) { cb() }, {
+      priority: function (n, cb) {
+        if (n === 2) return cb(null, 10);
+        if (n === 1) return cb(null, 5);
+        return cb(null, 1);
+      }
+    })
+    q.pause();
+    var finished = 0;
+    var queued = 0;
+    q.on('task_queued', function () {
+      queued++;
+      if (queued === 3) {
+        q.resume();
+      }
+    })
+    q.push(3, function (err, r) {
+      assert.equal(finished, 2);
+      finished++;
+    });
+    q.push(2, function (err, r) {
+      assert.equal(finished, 0);
+      finished++;
+    });
+    q.push(1, function (err, r) {
+      assert.equal(finished, 1);
+      finished++;
+      done()
+    });
+    this.q = q;
+  })
+
+  it('should run filo', function (done) {
+    var finished = 0;
+    var queued = 0;
+    var q = new Queue(function (num, cb) {
+      cb();
+    }, { filo: true })
+    q.on('task_finish', function () {
+      if (finished >= 3) {
+        done();
+      }
+    })
+    q.on('task_queued', function () {
+      queued++;
+      if (queued >= 3) {
+        q.resume();
+      }
+    })
+    q.pause();
+    q.push(1, function (err, r) {
+      assert.equal(finished, 2);
+      finished++;
+    })
+    q.push(2, function (err, r) {
+      assert.equal(finished, 1);
+      finished++;
+    })
+    q.push(3, function (err, r) {
+      assert.equal(finished, 0);
+      finished++;
+    })
+    this.q = q;
+  })
+
+  it('should filter before process', function (done) {
+    var q = new Queue(function (n, cb) { cb(null, n) }, {
+      filter: function (n, cb) {
+        cb(null, n === 2 ? false : n);
+      }
+    })
+    q.push(2, function (err, r) {
+      assert.equal(err, 'input_rejected');
+    })
+    q.push(3, function (err, r) {
+      assert.equal(r, 3);
+      done();
+    })
+    this.q = q;
+  })
+
+  it('should batch delay', function (done) {
+    var batches = 0;
+    var q = new Queue(function (batch, cb) {
+      batches++;
+      if (batches === 1) {
+        assert.equal(batch.length, 2);
+        return cb();
+      }
+      if (batches === 2) {
+        assert.equal(batch.length, 1);
+        cb();
+        return done();
+      }
+    }, { batchSize: 2, batchDelay: 5, failTaskOnProcessException: false });
+    q.push(1);
+    q.push(2);
+    q.push(3);
+    this.q = q;
+  })
+
+  it('should batch 2', function (done) {
+    var finished = 0;
+    var q = new Queue(function (batch, cb) {
+      finished++;
+      assert.equal(batch.length, 1);
+      if (finished >= 2) {
+        done();
+      }
+      cb();
+    }, { batchSize: 2, batchDelay: 1, autoResume: true });
+    q.push(1)
+      .on('queued', function () {
+        setTimeout(function () {
+          q.push(2);
+        }, 2)
+      })
+    this.q = q;
+  })
+
+  it('should drain and empty', function (done) {
+    var emptied = false;
+    var q = new Queue(function (n, cb) { cb() })
+    q.on('empty', function () {
+      emptied = true;
+    }, { autoResume: true })
+    q.on('drain', function () {
+      assert.ok(emptied);
+      done();
+    });
+    var queued = 0;
+    q.on('task_queued', function () {
+      queued++;
+      if (queued >= 3) {
+        q.resume();
+      }
+    })
+    q.pause();
+    q.push(1)
+    q.push(2)
+    q.push(3)
+    this.q = q;
+  })
+
+  it('should drain only once the task is complete', function (done) {
+    var finished_task = false;
+    var q = new Queue(function (n, cb) {
+        finished_task = true;
+        cb();
+    }, { concurrent: 2 });
+    q.on('drain', function () {
+      assert.ok(finished_task);
+      done();
+    });
+    q.push(1);
+    this.q = q;
+  });
+
+  it('should queue 50 things', function (done) {
+    var q = new Queue(function (n, cb) {
+      cb(null, n+1);
+    })
+    var finished = 0;
+    for (var i = 0; i < 50; i++) {
+      (function (n) {
+        q.push(n, function (err, r) {
+          assert.equal(r, n+1);
+          finished++;
+          if (finished === 50) {
+            done();
+          }
+        })
+      })(i)
+    }
+    this.q = q;
+  });
+
+  it('should concurrently handle tasks', function (done) {
+    var concurrent = 0;
+    var ok = false;
+    var q = new Queue(function (n, cb) {
+      var wait = function () {
+        if (concurrent === 3) {
+          ok = true;
+        }
+        if (ok) return cb();
+        setImmediate(function () {
+          wait();
+        })
+      }
+      concurrent++;
+      wait();
+    }, { concurrent: 3 })
+    var finished = 0;
+    var finish = function () {
+      finished++;
+      if (finished >= 4) {
+        done();
+      }
+    }
+    q.push(0, finish);
+    q.push(1, finish);
+    q.push(2, finish);
+    q.push(3, finish);
+    this.q = q;
+  })
+
+  it('should pause and resume', function (done) {
+    var running = false;
+    var q = new Queue(function (n, cb) {
+      running = true;
+      return {
+        pause: function () {
+          running = false;
+        },
+        resume: function () {
+          running = true;
+          cb();
+          done();
+        }
+      }
+    })
+    q.pause();
+    q.push(1)
+      .on('started', function () {
+        setTimeout(function () {
+          assert.ok(running);
+          q.pause();
+          assert.ok(!running);
+          q.resume();
+        }, 1)
+      })
+    assert.ok(!running);
+    q.resume();
+    this.q = q;
+  })
+
+  it('should timeout and fail', function (done) {
+    var tries = 0;
+    var q = new Queue(function (n, cb) {
+      tries++;
+      setTimeout(function () {
+        cb(null, 'done!')
+      }, 3)
+    }, { maxTimeout: 1, maxRetries: 2 })
+    q.push(1)
+      .on('finish', function (result) {
+        assert.ok(false)
+      })
+      .on('failed', function (err) {
+        assert.equal(tries, 2);
+        setTimeout(function () {
+          done();
+        }, 5)
+      })
+    this.q = q;
+  })
+
+  it('should cancel while running and in queue', function (done) {
+    var q = new Queue(function (task, cb) {
+      assert.ok(task.n, 2)
+      setTimeout(function () {
+        q.cancel(1);
+      }, 1)
+      return {
+        cancel: function () {
+          done();
+        }
+      }
+    }, {
+      id: 'id',
+      merge: function (a,b) {
+        assert.ok(false);
+      }
+    })
+    q.push({ id: 1, n: 1 })
+      .on('queued', function () {
+        q.cancel(1, function () {
+          q.push({ id: 1, n: 2 });
+        })
+      });
+    this.q = q;
+  })
+
+  it('should stop if precondition fails', function (done) {
+    var retries = 0;
+    var q = new Queue(function (n) {
+      assert.equal(retries, 2);
+      done();
+    }, {
+      precondition: function (cb) {
+        retries++;
+        cb(null, retries === 2)
+      },
+      preconditionRetryTimeout: 1
+    })
+    q.push(1);
+    this.q = q;
+  })
+
+  it('should call cb on throw', function (done) {
+    var called = false;
+    var q = new Queue(function (task, cb) {
+      throw new Error('fail');
+    });
+    q.push(1, function (err) {
+      called = true;
+      assert.ok(err);
+    });
+    q.on('drain', function () {
+      assert.ok(called);
+      done();
+    });
+    this.q = q;
+  })
+
+  it('should respect batchDelayTimeout', function (done) {
+    var q = new Queue(function (arr) {
+      assert.equal(arr.length, 2);
+      done();
+    }, {
+      batchSize: 3,
+      batchDelay: Infinity,
+      batchDelayTimeout: 5
+    })
+    q.push(1);
+    setTimeout(function () {
+      q.push(2);
+    }, 1)
+    this.q = q;
+  })
+
+  it('should merge but not batch until the delay has happened', function (done) {
+    var running = false;
+    var q = new Queue(function (arr) {
+      running = true;
+    }, {
+      autoResume: true,
+      batchSize: 2,
+      batchDelay: Infinity,
+      id: 'id'
+    })
+    setTimeout(function () {
+      q.push({ id: 'a', x: 1 });
+      q.push({ id: 'a', x: 2 });
+    }, 1)
+    setTimeout(function () {
+      assert.ok(!running);
+      done();
+    }, 10)
+    this.q = q;
+  })
+
+  it('merge batches should call all push callbacks', function (done) {
+    var count = 0
+    function finish() {
+      count++
+      if (count === 2) done()
+    }
+    var q = new Queue(function (arr, cb) {
+      cb()
+    }, {
+      autoResume: true,
+      batchSize: 2,
+      id: 'id'
+    })
+    q.push({ id: 'a', x: 1 }, finish)
+    q.push({ id: 'a', x: 2 }, finish)
+    this.q = q;
+  })
+
+  it('cancel should not retry', function (done) {
+    var count = 0;
+    var q = new Queue(function (n, cb) {
+      count++;
+      if (count === 2) {
+        q.cancel('a', function () {
+          cb('failed again');
+          setTimeout(function () {
+            if (count === 2) {
+              done();
+            }
+          }, 100)
+        })
+      } else {
+        cb('failed');
+      }
+    }, {
+      autoResume: true,
+      failTaskOnProcessException: true,
+      maxRetries: Infinity,
+      id: 'id'
+    })
+    q.push({ id: 'a', x: 1 });
+    this.q = q;
+  })
+
+})

node_modules/better-queue/test/complex.js

@@ -0,0 +1,361 @@
+var assert = require('assert');
+var helper = require('./lib/helper');
+
+var Queue = require('../lib/queue');
+var MemoryStore = require('better-queue-memory');
+
+describe('Complex Queue', function() {
+  afterEach(helper.destroyQueues);
+
+  it('should run in batch mode', function (done) {
+    var q = new Queue({
+      batchSize: 3,
+      process: function (batch, cb) {
+        assert.equal(batch.length, 3);
+        var total = 0;
+        batch.forEach(function (task) {
+          total += task;
+        })
+        cb(null, total);
+      },
+    })
+    var queued = 0;
+    q.on('task_queued', function () {
+      queued++;
+      if (queued >= 3) {
+        q.resume();
+      }
+    })
+    q.pause();
+    q.push(1, function (err, total) {
+      assert.equal(total, 6);
+    })
+    q.push(2, function (err, total) {
+      assert.equal(total, 6);
+    })
+    q.push(3, function (err, total) {
+      assert.equal(total, 6);
+      done();
+    })
+    this.q = q;
+  })
+
+  it('should store properly', function (done) {
+    var self = this;
+    var s = new MemoryStore();
+    var finished = 0;
+    var queued = 0;
+    var q1 = new Queue(function (n, cb) { throw new Error('failed') }, { store: s })
+    q1.on('task_queued', function () {
+      queued++;
+      if (queued >= 3) {
+        var q2 = new Queue(function (n, cb) {
+          finished++;
+          cb();
+          if (finished === 3) {
+            done();
+          }
+        }, { store: s });
+        self.q2 = q2;
+      }
+    })
+    q1.pause();
+    q1.push(1);
+    q1.push(2);
+    q1.push(3);
+    this.q1 = q1;
+  })
+
+  it('should retry', function (done) {
+    var tries = 0;
+    var q = new Queue(function (n, cb) {
+      tries++;
+      if (tries === 3) {
+        cb();
+        done();
+      } else {
+        cb('fail');
+      }
+    }, { maxRetries: 3 });
+    q.push(1);
+    this.q = q;
+  })
+
+  it('should fail retry', function (done) {
+    var tries = 0;
+    var q = new Queue(function (n, cb) {
+      tries++;
+      if (tries === 3) {
+        cb();
+      } else {
+        cb('fail');
+      }
+    }, { maxRetries: 2, autoResume: true })
+    q.on('task_failed', function () {
+      done();
+    });
+    q.push(1);
+    this.q = q;
+  })
+
+  it('should respect afterProcessDelay', function (done) {
+    var delay = 100;
+    var finished = 0;
+    var startTime;
+    var q = new Queue(function (task, cb) {
+      finished++;
+      cb();
+      if (finished === 1) {
+        startTime = +(new Date());
+      } else if (finished === 2) {
+        var endTime = +(new Date());
+        var elapsedTime = endTime - startTime;
+        assert(elapsedTime >= delay);
+        done();
+      }
+    },  { batchSize: 1, afterProcessDelay: delay });
+    var queued = 0;
+    q.on('task_queued', function () {
+      queued++;
+      if (queued >= 2) {
+        q.resume();
+      }
+    })
+    q.pause();
+    q.push(1);
+    q.push(2);
+    this.q = q;
+  })
+
+  it('should max timeout', function (done) {
+    var q = new Queue(function (tasks, cb) {}, { maxTimeout: 1 })
+    q.on('task_failed', function (taskId, msg) {
+      assert.equal(msg, 'task_timeout');
+      done();
+    });
+    q.push(1, function (err, r) {
+      assert.equal(err, 'task_timeout');
+    });
+    this.q = q;
+  })
+
+  it('should merge tasks', function (done) {
+    var q = new Queue(function (o, cb) {
+      if (o.id === 1) {
+        assert.equal(o.x, 3);
+        cb();
+      } else {
+        cb();
+      }
+    }, {
+      id: 'id',
+      merge: function (a, b, cb) {
+        a.x += b.x;
+        cb(null, a);
+      }
+    })
+    var queued = 0;
+    q.on('task_queued', function () {
+      queued++;
+      if (queued >= 2) {
+        q.resume();
+      }
+    })
+    q.on('task_finish', function (taskId, r) {
+      if (taskId === '1') {
+        done();
+      }
+    })
+    q.pause()
+    q.push({ id: '0', x: 4 });
+    q.push({ id: '1', x: 1 }, function (err, r) {
+      assert.ok(!err)
+    });
+    q.push({ id: '1', x: 2 }, function (err, r) {
+      assert.ok(!err);
+    });
+    this.q = q;
+  })
+  
+  it('should respect id property (string)', function (done) {
+    var q = new Queue(function (o, cb) {
+      if (o.name === 'john') {
+        assert.equal(o.x, 4);
+        cb();
+      }
+      if (o.name === 'mary') {
+        assert.equal(o.x, 5);
+        cb();
+      }
+      if (o.name === 'jim') {
+        assert.equal(o.x, 2);
+        cb();
+      }
+    }, {
+      id: 'name',
+      merge: function (a, b, cb) {
+        a.x += b.x;
+        cb(null, a);
+      }
+    })
+    var finished = 0;
+    var queued = 0;
+    q.on('task_finish', function (taskId, r) {
+      finished++;
+      if (finished >= 3) done();
+    })
+    q.on('task_queued', function (taskId, r) {
+      queued++;
+      if (queued >= 3) {
+        q.resume();
+      }
+    })
+    q.pause();
+    q.push({ name: 'john', x: 4 });
+    q.push({ name: 'mary', x: 3 });
+    q.push({ name: 'jim', x: 1 });
+    q.push({ name: 'jim', x: 1 });
+    q.push({ name: 'mary', x: 2 });
+    this.q = q;
+  })
+  
+  it('should respect id property (function)', function (done) {
+    var finished = 0;
+    var q = new Queue(function (n, cb) {
+      cb(null, n)
+    }, {
+      batchDelay: 3,
+      id: function (n, cb) {
+        cb(null, n % 2 === 0 ? 'even' : 'odd');
+      },
+      merge: function (a, b, cb) {
+        cb(null, a+b);
+      }
+    })
+    var finished = 0;
+    var queued = 0;
+    q.on('task_queued', function (taskId, r) {
+    })
+    q.on('task_finish', function (taskId, r) {
+      finished++;
+      if (taskId === 'odd') {
+        assert.equal(r, 9);
+      }
+      if (taskId === 'even') {
+        assert.equal(r, 6);
+      }
+      if (finished >= 2) {
+        done();
+      }
+    })
+    q.push(1);
+    q.push(2);
+    q.push(3);
+    q.push(4);
+    q.push(5);
+    this.q = q;
+  })
+  
+  it('should cancel if running', function (done) {
+    var ran = 0;
+    var cancelled = false;
+    var q = new Queue(function (n, cb) {
+      ran++;
+      if (ran >= 2) {
+        cb();
+      }
+      if (ran === 3) {
+        assert.ok(cancelled);
+        done();
+      }
+      return {
+        cancel: function () {
+          cancelled = true;
+        }
+      }
+    }, { id: 'id', cancelIfRunning: true })
+    q.push({ id: 1 })
+      .on('started', function () {
+        q.push({ id: 2 });
+        setTimeout(function () {
+          q.push({ id: 1 });
+        }, 1)
+      });
+    this.q = q;
+  })
+  
+  it('failed task should not stack overflow', function (done) {
+    var count = 0;
+    var q = new Queue(function (n, cb) {
+      count++
+      if (count > 100) {
+        cb();
+        done();
+      } else {
+        cb('fail');
+      }
+    }, {
+      maxRetries: Infinity
+    })
+    q.push(1);
+    this.q = q;
+  })
+
+  // it('drain should still work with persistent queues', function (done) {
+  //   var q = new Queue(function (n, cb) {
+  //     setTimeout(cb, 1);
+  //   }, {
+  //     store: {
+  //       type: 'sql',
+  //       dialect: 'sqlite',
+  //       path: 'testqueue.sql'
+  //     }
+  //   })
+  //   var drained = false;
+  //   q.on('drain', function () {
+  //     drained = true;
+  //     done();
+  //   });
+  //   q.push(1);
+  //   this.q = q;
+  // })
+
+  // it('drain should still work when there are persisted items at load time', function (done) {
+  //   var initialQueue = new Queue(function (n, cb) {
+  //     setTimeout(cb, 100);
+  //   }, {
+  //     store: {
+  //       type: 'sql',
+  //       dialect: 'sqlite',
+  //       path: 'testqueue.sql'
+  //     }
+  //   });
+  //   initialQueue.push('' + 1);
+  //   initialQueue.push('' + 2);
+  //   setTimeout(function () {
+  //     // This effectively captures the queue in a state where there were unprocessed items
+  //     fs.copySync('testqueue.sql', 'testqueue2.sql');
+  //     initialQueue.destroy();
+  //     var persistedQueue = new Queue(function (n, cb) {
+  //       setTimeout(cb, 1);
+  //     }, {
+  //       store: {
+  //         type: 'sql',
+  //         dialect: 'sqlite',
+  //         path: 'testqueue2.sql'
+  //       }
+  //     })
+  //     var drained = false;
+  //     persistedQueue.on('drain', function () {
+  //       drained = true;
+  //     });
+  //     persistedQueue.push(2);
+  //     setTimeout(function () {
+  //       persistedQueue.destroy();
+
+  //       assert.ok(drained);
+  //       done();
+  //     }, 140)
+  //   }, 40)
+  // })
+})

node_modules/better-queue/test/fixtures/PostgresAdapter.js

@@ -0,0 +1,12 @@
+var PostgresAdapter = require('../../lib/stores/PostgresAdapter');
+
+function MockPostgresAdapter(opts) {
+  opts.verbose = false;
+  opts.username = 'diamond';
+  opts.dbname = 'diamond';
+  PostgresAdapter.call(this, opts);
+}
+
+MockPostgresAdapter.prototype = Object.create(PostgresAdapter.prototype);
+
+module.exports = MockPostgresAdapter;

node_modules/better-queue/test/fixtures/SqliteAdapter.js

@@ -0,0 +1,23 @@
+var fs            = require('fs-extra');
+var uuid          = require('uuid');
+var SqliteAdapter = require('../../lib/stores/SqliteAdapter');
+
+function MockSqliteAdapter(opts) {
+  opts.verbose = false;
+  opts.path = opts.path || uuid.v4() + '.sqlite';
+  SqliteAdapter.call(this, opts);
+}
+
+MockSqliteAdapter.prototype = Object.create(SqliteAdapter.prototype);
+
+MockSqliteAdapter.prototype.close = function (cb) {
+  var after = function () {
+    SqliteAdapter.prototype.close.call(this, cb)
+  }
+  if (this.path === ':memory:') return after();
+  fs.unlink(this.path, function (err) {
+    after();
+  });
+}
+
+module.exports = MockSqliteAdapter;

node_modules/better-queue/test/lib/helper.js

@@ -0,0 +1,9 @@
+
+exports.destroyQueues = function () {
+  [this.q, this.q1, this.q2].forEach(function (q) {
+    if (!q) return;
+    setTimeout(function () {
+      q.destroy();
+    }, 15);
+  });
+};

node_modules/better-queue/test/stats.js

@@ -0,0 +1,60 @@
+var assert = require('assert');
+var helper = require('./lib/helper');
+var Queue = require('../lib/queue');
+
+describe('Stats', function() {
+  afterEach(helper.destroyQueues);
+
+  it('should get stat', function (done) {
+    var completed = 0;
+    var elapsedTotals = 0;
+    var q = new Queue(function (wait, cb) {
+      setTimeout(function () {
+        cb()
+      }, wait)
+    })
+    q.on('task_finish', function (id, result, stat) {
+      completed++;
+      elapsedTotals += stat.elapsed;
+    })
+    q.on('drain', function () {
+      var stats = q.getStats();
+      assert.ok(stats.peak);
+      assert.equal(3, stats.total);
+      assert.equal(elapsedTotals/3, stats.average);
+      done();
+    })
+    q.push(1);
+    q.push(1);
+    q.push(1);
+    this.q = q;
+  })
+
+  it('should reset stat', function (done) {
+    var queued = 0;
+    var elapsedTotal = 0;
+    var q = new Queue(function (wait, cb) {
+      setTimeout(function () {
+        cb()
+      }, wait)
+    }, { id: function (n, cb) { cb(null, n) } })
+    q.push(1, function () {
+      q.push(1, function () {
+        q.resetStats();
+        q.on('task_finish', function (id, result, stat) {
+          if (id !== '2') return;
+          assert.ok(stat.elapsed > 0);
+          var stats = q.getStats();
+          assert.equal(1, stats.peak);
+          assert.equal(1, stats.total);
+          assert.equal(stats.average, stat.elapsed);
+          done();
+        })
+        q.push(2);
+      });
+    });
+    this.q = q;
+  })
+
+
+})

node_modules/better-queue/test/store.js

@@ -0,0 +1,90 @@
+var assert = require('assert');
+var Queue = require('../lib/queue');
+
+describe('Store Usage', function() {
+
+  it('should retry connect', function (done) {
+    var tries = 0;
+    var s = {
+      connect: function (cb) {
+        tries++;
+        if (tries < 3) {
+          return cb('failed');
+        }
+        done();
+      },
+      getTask: function (taskId, cb) { cb() },
+      putTask: function (taskId, task, priority, cb) { cb() },
+      takeFirstN: function (n, cb) { cb() },
+      takeLastN: function (n, cb) { cb() }
+    }
+    var q = new Queue(function (batch, cb) { cb() }, {
+      storeMaxRetries: 5,
+      storeRetryTimeout: 1,
+      store: s
+    })
+  })
+
+  it('should fail retry', function (done) {
+    var tries = 0;
+    var s = {
+      connect: function (cb) {
+        tries++;
+        cb('failed');
+      },
+      getTask: function (taskId, cb) { cb() },
+      putTask: function (taskId, task, priority, cb) { cb() },
+      takeFirstN: function (n, cb) { cb() },
+      takeLastN: function (n, cb) { cb() }
+    }
+    var q = new Queue(function (batch, cb) { cb() }, {
+      storeMaxRetries: 2,
+      storeRetryTimeout: 1,
+      store: s
+    })
+      .on('error', function (e) {
+        assert.ok(e);
+        done();
+      })
+  })
+
+  it('should queue length', function (done) {
+    var queued = false;
+    var s = {
+      connect: function (cb) { cb(null, 5) },
+      getTask: function (taskId, cb) { cb() },
+      putTask: function (taskId, task, priority, cb) { cb() },
+      getLock: function (lockId, cb) { cb(null, { 'task-id': queued ? 2 : 1 }) },
+      getRunningTasks: function (cb) { cb(null, {}) },
+      takeFirstN: function (n, cb) { cb(null, 'lock-id') },
+      takeLastN: function (n, cb) { cb() },
+      releaseLock: function (lockId, cb) { cb(null) },
+    }
+    var q = new Queue(function (n, cb) {
+      if (n === 2) {
+        assert.equal(q.length, 5);
+        done();
+      }
+      cb();
+    }, { store: s, autoResume: false })
+    q.push(1).on('queued', function (e) {
+      queued = true;
+      assert.equal(q.length, 6);
+    })
+  })
+
+  it('should fail if there is no length on connect', function (done) {
+    var queued = false;
+    var s = {
+      connect: function (cb) { cb() }
+    }
+    try {
+      var q = new Queue(function (n, cb) {}, { store: s })
+    } catch (e) {
+      done();
+    }
+  })
+
+  // TODO: Test progress
+
+})

node_modules/better-queue/test/ticket.js

@@ -0,0 +1,73 @@
+var assert = require('assert');
+var Ticket = require('../lib/ticket');
+
+describe('Ticket', function() {
+  var t;
+
+  before(function () {
+    t = new Ticket();
+  })
+
+  it('should instantiate', function () {
+    assert.ok(t);
+  })
+
+  it('should accept', function () {
+    assert.ok(!t.isAccepted, 'ticket is not accepted');
+    t.accept();
+    assert.ok(t.isAccepted, 'ticket is accepted');
+  })
+
+  it('should queue', function () {
+    assert.ok(!t.isQueued, 'ticket is not queued');
+    t.queued();
+    assert.ok(t.isQueued, 'ticket is queued');
+  })
+
+  it('should start and stop', function () {
+    assert.ok(!t.isStarted, 'ticket is not started');
+    t.started();
+    assert.ok(t.isStarted, 'ticket is started');
+    t.stopped();
+    assert.ok(!t.isStarted, 'ticket is stopped');
+  })
+
+  it('should finish and emit', function (done) {
+    assert.ok(!t.isFinished, 'ticket is not finished');
+    t.once('finish', function (result) {
+      assert.deepEqual(result, { x: 1 });
+      assert.ok(t.isFinished, 'ticket is finished');
+      done();
+    })
+    t.finish({ x: 1 });
+  })
+
+  it('should fail and emit', function (done) {
+    assert.ok(!t.isFailed, 'ticket not failed');
+    t.once('failed', function (err) {
+      assert.equal(err, 'some_error');
+      assert.ok(t.isFailed, 'ticket failed');
+      done();
+    })
+    t.failed('some_error');
+  })
+
+  it('should progress and emit', function (done) {
+    t.started();
+    t.once('progress', function (progress) {
+      assert.equal(progress.pct, 50);
+      assert.equal(progress.complete, 1);
+      assert.equal(progress.total, 2);
+      assert.equal(progress.message, 'test');
+      assert.equal(typeof progress.eta, 'string');
+      done()
+    });
+    t.progress({
+      complete: 1,
+      total: 2,
+      message: 'test'
+    });
+  })
+  
+  
+})

node_modules/better-queue/test/tickets.js

@@ -0,0 +1,90 @@
+var assert = require('assert');
+var Ticket = require('../lib/ticket');
+var Tickets = require('../lib/tickets');
+
+describe('Tickets', function() {
+  var ts, t1, t2;
+
+  before(function () {
+    t1 = new Ticket();
+    t2 = new Ticket();
+    ts = new Tickets();
+    ts.push(t1);
+    ts.push(t2);
+  })
+
+  it('should accept', function () {
+    assert.ok(!t1.isAccepted, 'ticket 1 is not accepted');
+    assert.ok(!t2.isAccepted, 'ticket 2 is not accepted');
+    ts.accept();
+    assert.ok(t1.isAccepted, 'ticket 1 is accepted');
+    assert.ok(t2.isAccepted, 'ticket 2 is accepted');
+  })
+
+  it('should queue', function () {
+    assert.ok(!t1.isQueued, 'ticket 1 is not queued');
+    assert.ok(!t2.isQueued, 'ticket 2 is not queued');
+    ts.queued();
+    assert.ok(t1.isQueued, 'ticket 1 is queued');
+    assert.ok(t2.isQueued, 'ticket 2 is queued');
+  })
+
+  it('should start and stop', function () {
+    assert.ok(!t1.isStarted, 'ticket 1 is not started');
+    assert.ok(!t2.isStarted, 'ticket 2 is not started');
+    ts.started();
+    assert.ok(t1.isStarted, 'ticket 1 is started');
+    assert.ok(t2.isStarted, 'ticket 2 is started');
+    ts.stopped();
+    assert.ok(!t1.isStarted, 'ticket 1 is stopped');
+    assert.ok(!t2.isStarted, 'ticket 2 is stopped');
+  })
+
+  it('should finish and emit', function (done) {
+    assert.ok(!t1.isFinished, 'ticket 1 is not finished');
+    assert.ok(!t2.isFinished, 'ticket 2 is not finished');
+    t2.once('finish', function (result) {
+      assert.deepEqual(result, { x: 1 });
+      assert.ok(t2.isFinished, 'ticket 2 is finished');
+      done();
+    })
+    ts.finish({ x: 1 });
+  })
+
+  it('should fail and emit', function (done) {
+    assert.ok(!t1.isFailed, 'ticket 1 not failed');
+    assert.ok(!t2.isFailed, 'ticket 2 not failed');
+    var called = 0;
+    t1.once('failed', function (err) {
+      assert.equal(err, 'some_error');
+      assert.ok(t1.isFailed, 'ticket 1 failed');
+      called++;
+      if (called == 2) { done() }
+    })
+    t2.once('failed', function (err) {
+      assert.equal(err, 'some_error');
+      assert.ok(t2.isFailed, 'ticket 2 failed');
+      called++;
+      if (called == 2) { done() }
+    })
+    ts.failed('some_error');
+  })
+
+  it('should progress and emit', function (done) {
+    t1.once('progress', function (progress) {
+      assert.equal(progress.pct, 50);
+      assert.equal(progress.complete, 1);
+      assert.equal(progress.total, 2);
+      assert.equal(progress.message, 'test');
+      assert.equal(typeof progress.eta, 'string');
+      done()
+    });
+    ts.progress({
+      complete: 1,
+      total: 2,
+      message: 'test'
+    });
+  })
+  
+  
+})