Update extractor script

scripts/util/environmentCheck.js

@@ -0,0 +1,12 @@
+// Checks if current environment is Travis CI, Cron builds, API builds
+const isTravisCI = () => 'TRAVIS' in process.env && 'CI' in process.env;
+const isTravisCronOrAPI = () =>
+  process.env['TRAVIS_EVENT_TYPE'] === 'cron' ||
+  process.env['TRAVIS_EVENT_TYPE'] === 'api';
+const isNotTravisCronOrAPI = () => !isTravisCronOrAPI();
+
+module.exports = {
+  isTravisCI,
+  isTravisCronOrAPI,
+  isNotTravisCronOrAPI,
+};

scripts/util/helpers.js

@@ -0,0 +1,60 @@
+const config = require('../../config');
+
+const getMarkDownAnchor = paragraphTitle =>
+  paragraphTitle
+    .trim()
+    .toLowerCase()
+    .replace(/[^\w\- ]+/g, '')
+    .replace(/\s/g, '-')
+    .replace(/\-+$/, '');
+// Creates an object from pairs
+const objectFromPairs = arr => arr.reduce((a, v) => ((a[v[0]] = v[1]), a), {});
+// Optimizes nodes in an HTML document
+const optimizeNodes = (data, regexp, replacer) => {
+  let count = 0;
+  let output = data;
+  do {
+    output = output.replace(regexp, replacer);
+    count = 0;
+    while (regexp.exec(output) !== null) ++count;
+  } while (count > 0);
+  return output;
+};
+// Capitalizes the first letter of a string
+const capitalize = (str, lowerRest = false) =>
+  str.slice(0, 1).toUpperCase() +
+  (lowerRest ? str.slice(1).toLowerCase() : str.slice(1));
+const prepTaggedData = tagDbData =>
+  [...new Set(Object.entries(tagDbData).map(t => t[1][0]))]
+    .filter(v => v)
+    .sort((a, b) =>
+      capitalize(a, true) === 'Uncategorized'
+        ? 1
+        : capitalize(b, true) === 'Uncategorized'
+        ? -1
+        : a.localeCompare(b),
+    );
+const makeExamples = data => {
+  data =
+    data.slice(0, data.lastIndexOf(`\`\`\`${config.language}`)).trim() +
+    misc.collapsible(
+      'Examples',
+      data.slice(
+        data.lastIndexOf(`\`\`\`${config.language}`),
+        data.lastIndexOf('```'),
+      ) + data.slice(data.lastIndexOf('```')),
+    );
+  return `${data}\n<br>${misc.link(
+    '⬆ Back to top',
+    misc.anchor('Contents'),
+  )}\n\n`;
+};
+
+module.exports = {
+  getMarkDownAnchor,
+  objectFromPairs,
+  optimizeNodes,
+  capitalize,
+  prepTaggedData,
+  makeExamples,
+};

scripts/util/index.js

@@ -0,0 +1,37 @@
+const {
+  isTravisCI,
+  isTravisCronOrAPI,
+  isNotTravisCronOrAPI,
+} = require('./environmentCheck');
+const {
+  getMarkDownAnchor,
+  objectFromPairs,
+  optimizeNodes,
+  capitalize,
+  prepTaggedData,
+  makeExamples,
+} = require('./helpers');
+const {
+  getFilesInDir,
+  hashData,
+  getCodeBlocks,
+  getTextualContent,
+  readSnippets,
+} = require('./snippetParser');
+
+module.exports = {
+  isTravisCI,
+  isTravisCronOrAPI,
+  isNotTravisCronOrAPI,
+  getMarkDownAnchor,
+  objectFromPairs,
+  optimizeNodes,
+  capitalize,
+  prepTaggedData,
+  makeExamples,
+  getFilesInDir,
+  hashData,
+  getCodeBlocks,
+  getTextualContent,
+  readSnippets,
+};

scripts/util/snippetParser.js

@@ -0,0 +1,121 @@
+const fs = require('fs-extra'),
+  path = require('path'),
+  { red } = require('kleur'),
+  crypto = require('crypto'),
+  frontmatter = require('front-matter'),
+  babel = require('@babel/core');
+const config = require('../../config');
+
+// Reade all files in a directory
+const getFilesInDir = (directoryPath, withPath, exclude = null) => {
+  try {
+    let directoryFilenames = fs.readdirSync(directoryPath);
+    directoryFilenames.sort((a, b) => {
+      a = a.toLowerCase();
+      b = b.toLowerCase();
+      if (a < b) return -1;
+      if (a > b) return 1;
+      return 0;
+    });
+
+    if (withPath) {
+      // a hacky way to do conditional array.map
+      return directoryFilenames.reduce((fileNames, fileName) => {
+        if (
+          exclude == null ||
+          !exclude.some(toExclude => fileName === toExclude)
+        )
+          fileNames.push(`${directoryPath}/${fileName}`);
+        return fileNames;
+      }, []);
+    }
+    return directoryFilenames.filter(v => v !== 'README.md');
+  } catch (err) {
+    console.log(`${red('ERROR!')} During snippet loading: ${err}`);
+    process.exit(1);
+  }
+};
+// Creates a hash for a value using the SHA-256 algorithm.
+const hashData = val =>
+  crypto
+    .createHash('sha256')
+    .update(val)
+    .digest('hex');
+// Gets the code blocks for a snippet file.
+const getCodeBlocks = str => {
+  const regex = /```[.\S\s]*?```/g;
+  let results = [];
+  let m = null;
+  while ((m = regex.exec(str)) !== null) {
+    if (m.index === regex.lastIndex) regex.lastIndex += 1;
+
+    m.forEach((match, groupIndex) => {
+      results.push(match);
+    });
+  }
+  const replacer = new RegExp(
+    `\`\`\`${config.language}([\\s\\S]*?)\`\`\``,
+    'g',
+  );
+  results = results.map(v => v.replace(replacer, '$1').trim());
+  return {
+    es6: results[0],
+    es5: babel.transformSync(results[0], { presets: ['@babel/preset-env'] }).code.replace('"use strict";\n\n', ''),
+    example: results[1],
+  };
+};
+// Gets the textual content for a snippet file.
+const getTextualContent = str => {
+  const regex = /([\s\S]*?)```/g;
+  const results = [];
+  let m = null;
+  while ((m = regex.exec(str)) !== null) {
+    if (m.index === regex.lastIndex) regex.lastIndex += 1;
+
+    m.forEach((match, groupIndex) => {
+      results.push(match);
+    });
+  }
+  if (!results.length) return str.replace(/\r\n/g, '\n');
+  return results[1].replace(/\r\n/g, '\n');
+};
+
+// Synchronously read all snippets and sort them as necessary (case-insensitive)
+const readSnippets = snippetsPath => {
+  const snippetFilenames = getFilesInDir(snippetsPath, false);
+
+  let snippets = {};
+  try {
+    for (let snippet of snippetFilenames) {
+      let data = frontmatter(
+        fs.readFileSync(path.join(snippetsPath, snippet), 'utf8'),
+      );
+      snippets[snippet] = {
+        id: snippet.slice(0, -3),
+        title: data.attributes.title,
+        type: 'snippet',
+        attributes: {
+          fileName: snippet,
+          text: getTextualContent(data.body),
+          codeBlocks: getCodeBlocks(data.body),
+          tags: data.attributes.tags.split(',').map(t => t.trim()),
+        },
+        meta: {
+          hash: hashData(data.body),
+        },
+      };
+    }
+  } catch (err) {
+    console.log(`${red('ERROR!')} During snippet loading: ${err}`);
+    process.exit(1);
+  }
+  return snippets;
+};
+
+module.exports = {
+  getFilesInDir,
+  hashData,
+  getCodeBlocks,
+  getTextualContent,
+  readSnippets,
+};