diff --git a/.travis.yml b/.travis.yml index 11ad50a15..63055201f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ script: - npm run builder - npm run packager - npm run tester +- npm run extractor after_success: - chmod +x .travis/push.sh - .travis/push.sh diff --git a/package.json b/package.json index 9bee117d2..c0bc1c84d 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "tagger": "node ./scripts/tag.js", "webber": "node ./scripts/web.js", "tester": "node ./scripts/tdd.js", + "extractor": "node ./scripts/extract.js", "packager": "node ./scripts/module.js", "localizer": "node ./scripts/localize.js", "test": "tape test/**/*.test.js | tap-spec" diff --git a/scripts/extract.js b/scripts/extract.js new file mode 100644 index 000000000..fe14f29a3 --- /dev/null +++ b/scripts/extract.js @@ -0,0 +1,68 @@ +/* + This is the extractor script that generates the snippets.json and snippetsArchive.json files. + Run using `npm run extractor`. +*/ +// Load modules +const fs = require('fs-extra'); +const path = require('path'); +const chalk = require('chalk'); +const util = require('./util'); +// Paths +const SNIPPETS_PATH = './snippets'; +const SNIPPETS_ARCHIVE_PATH = './snippets_archive'; +const OUTPUT_PATH = './snippet_data'; +// Check if running on Travis - only build for cron jobs and custom builds +if(util.isTravisCI() && process.env['TRAVIS_EVENT_TYPE'] !== 'cron' && process.env['TRAVIS_EVENT_TYPE'] !== 'api') { + console.log(`${chalk.green('NOBUILD')} snippet extraction terminated, not a cron or api build!`); + process.exit(0); +} +// Read data +let snippets = {}, archivedSnippets = {}, tagDbData = {}; +console.time('Extractor'); +snippets = util.readSnippets(SNIPPETS_PATH); +archivedSnippets = util.readSnippets(SNIPPETS_ARCHIVE_PATH); +tagDbData = util.readTags(); +// Extract snippet data +let snippetData = { + data: Object.keys(snippets).map(key => { + return { + id: key.slice(0,-3), + type: 'snippet', + attributes: { + fileName: key, + text: util.getTextualContent(snippets[key]).trim(), + codeBlocks: util.getCodeBlocks(snippets[key]).map(v => v.replace(/```js([\s\S]*?)```/g, '$1').trim()), + tags: tagDbData[key.slice(0,-3)] + }, + meta: { + archived: false, + hash: util.hashData(snippets[key]) + } + } + }) +}; +// Extract archived snippet data +let snippetArchiveData = { + data: Object.keys(archivedSnippets).map(key => { + return { + id: key.slice(0,-3), + type: 'snippet', + attributes: { + fileName: key, + text: util.getTextualContent(archivedSnippets[key]).trim(), + codeBlocks: util.getCodeBlocks(archivedSnippets[key]).map(v => v.replace(/```js([\s\S]*?)```/g, '$1').trim()), + tags: [] + }, + meta: { + archived: true, + hash: util.hashData(archivedSnippets[key]) + } + } + }) +}; +// Write files +fs.writeFileSync(path.join(OUTPUT_PATH, 'snippets.json'), JSON.stringify(snippetData, null, 2)); +fs.writeFileSync(path.join(OUTPUT_PATH, 'snippetsArchive.json'), JSON.stringify(snippetArchiveData, null, 2)); +// Display messages and time +console.log(`${chalk.green('SUCCESS!')} snippets.json and snippetsArchive.json files generated!`); +console.timeEnd('Extractor'); diff --git a/scripts/util.js b/scripts/util.js index c4bc0517e..7ff6f0208 100644 --- a/scripts/util.js +++ b/scripts/util.js @@ -78,4 +78,34 @@ const capitalize = (str, lowerRest = false) => const isTravisCI = () => 'TRAVIS' in process.env && 'CI' in process.env; // Creates a hash for a value using the SHA-256 algorithm. const hashData = val => crypto.createHash('sha256').update(val).digest('hex'); -module.exports = {readSnippets, readTags, optimizeNodes, capitalize, objectFromPairs, isTravisCI, hashData, shuffle}; +// Gets the code blocks for a snippet file. +const getCodeBlocks = str => { + const regex = /```[.\S\s]*?```/g; + const results = []; + let m = null; + while ((m = regex.exec(str)) !== null) { + if (m.index === regex.lastIndex) { + regex.lastIndex += 1; + } + m.forEach((match, groupIndex) => { + results.push(match); + }) + } + return results; +} +// Gets the textual content for a snippet file. +const getTextualContent = str => { + const regex = /###.*\n*([\s\S]*?)```/g; + const results = []; + let m = null; + while ((m = regex.exec(str)) !== null) { + if (m.index === regex.lastIndex) { + regex.lastIndex += 1; + } + m.forEach((match, groupIndex) => { + results.push(match); + }) + } + return results[1]; +} +module.exports = {readSnippets, readTags, optimizeNodes, capitalize, objectFromPairs, isTravisCI, hashData, shuffle, getCodeBlocks, getTextualContent}; diff --git a/snippet_data/snippets.json b/snippet_data/snippets.json new file mode 100644 index 000000000..89cc398ec --- /dev/null +++ b/snippet_data/snippets.json @@ -0,0 +1,6040 @@ +{ + "data": [ + { + "id": "all", + "type": "snippet", + "attributes": { + "fileName": "all.md", + "text": "Returns `true` if the provided predicate function returns `true` for all elements in a collection, `false` otherwise.\r\n\r\nUse `Array.every()` to test if all elements in the collection return `true` based on `fn`.\r\nOmit the second argument, `fn`, to use `Boolean` as a default.", + "codeBlocks": [ + "const all = (arr, fn = Boolean) => arr.every(fn);", + "all([4, 2, 3], x => x > 1); // true\r\nall([1, 2, 3]); // true" + ], + "tags": [ + "array", + "function" + ] + }, + "meta": { + "archived": false, + "hash": "69ea1bb81e034e198af4c61f9440f4e125e3828cbf36fe0ec6efb9da9f98710b" + } + }, + { + "id": "any", + "type": "snippet", + "attributes": { + "fileName": "any.md", + "text": "Returns `true` if the provided predicate function returns `true` for at least one element in a collection, `false` otherwise.\r\n\r\nUse `Array.some()` to test if any elements in the collection return `true` based on `fn`.\r\nOmit the second argument, `fn`, to use `Boolean` as a default.", + "codeBlocks": [ + "const any = (arr, fn = Boolean) => arr.some(fn);", + "any([0, 1, 2, 0], x => x >= 2); // true\r\nany([0, 0, 1, 0]); // true" + ], + "tags": [ + "array", + "function" + ] + }, + "meta": { + "archived": false, + "hash": "3788d0787d355e1e8c3014cb3e61055351eba582cb5866bd25f754c37b0e7ba2" + } + }, + { + "id": "approximatelyEqual", + "type": "snippet", + "attributes": { + "fileName": "approximatelyEqual.md", + "text": "Checks if two numbers are approximately equal to each other.\r\n\r\nUse `Math.abs()` to compare the absolute difference of the two values to `epsilon`.\r\nOmit the third parameter, `epsilon`, to use a default value of `0.001`.", + "codeBlocks": [ + "const approximatelyEqual = (v1, v2, epsilon = 0.001) => Math.abs(v1 - v2) < epsilon;", + "approximatelyEqual(Math.PI / 2.0, 1.5708); // true" + ], + "tags": [ + "math" + ] + }, + "meta": { + "archived": false, + "hash": "2b68e79d3ebb806379c3b84ae70152e033297d6e4fd4bc779b10be81ca2fc0cf" + } + }, + { + "id": "arrayToHtmlList", + "type": "snippet", + "attributes": { + "fileName": "arrayToHtmlList.md", + "text": "Converts the given array elements into `