Initial testing of extractor and builder

Configured appropriately
This commit is contained in:
Angelos Chalaris
2019-08-21 12:50:30 +03:00
parent b5fb49b43e
commit 255df75ad3
16 changed files with 20915 additions and 903 deletions

View File

@ -1,97 +1,141 @@
/*
This is the builder script that generates the README file.
Run using `npm run builder`.
*/
// Load modules
const fs = require('fs-extra');
const path = require('path');
const chalk = require('chalk');
const util = require('./util.js');
const { green, red } = require('kleur');
const util = require('./util');
const markdown = require('markdown-builder');
const snippets = require('../data/snippet_data.json');
const { headers, misc, lists } = markdown;
const TAG_NAMES = [...new Set(snippets.reduce((acc, v) => [...acc, v.tags[0]], []))].sort((a, b) =>
a.localeCompare(b)
);
console.log(TAG_NAMES);
const config = require('../config');
const STATIC_PARTS_PATH = './static-parts';
// Paths (relative to package.json)
const SNIPPETS_PATH = `./${config.snippetPath}`;
const STATIC_PARTS_PATH = `./${config.staticPartsPath}`;
let startPart = '';
let endPart = '';
let output = '';
// Terminate if parent commit is a Travis build
if (
util.isTravisCI() &&
/^Travis build: \d+/g.test(process.env['TRAVIS_COMMIT_MESSAGE'])
) {
console.log(
`${green(
'NOBUILD',
)} README build terminated, parent commit is a Travis build!`,
);
process.exit(0);
}
const detailsTOC = (title, snippetsArray) =>
`\n${misc
.collapsible(
title,
lists
.ul(snippetsArray, snippet =>
misc.link(
snippet.title
.replace('\n', '')
.split('```')[0]
.trim(),
misc.anchor(
snippet.title
.replace('\n', '')
.split('```')[0]
.trim()
)
)
)
.trim()
)
.trim()}\n\n`;
// Setup everything
let snippets = {},
snippetsArray = [],
startPart = '',
endPart = '',
output = '';
const EMOJIS = {};
console.time('Builder');
// Synchronously read all snippets from snippets folder and sort them as necessary (case-insensitive)
snippets = util.readSnippets(SNIPPETS_PATH);
snippetsArray = Object.keys(snippets).reduce((acc, key) => {
acc.push(snippets[key]);
return acc;
}, []);
// Load static parts for the README file
try {
startPart = fs.readFileSync(path.join(STATIC_PARTS_PATH, 'README-start.md'), 'utf8');
endPart = fs.readFileSync(path.join(STATIC_PARTS_PATH, 'README-end.md'), 'utf8');
startPart = fs.readFileSync(
path.join(STATIC_PARTS_PATH, 'README-start.md'),
'utf8',
);
endPart = fs.readFileSync(
path.join(STATIC_PARTS_PATH, 'README-end.md'),
'utf8',
);
} catch (err) {
console.log(`${chalk.red('ERROR!')} During static part loading: ${err}`);
console.log(`${red('ERROR!')} During static part loading: ${err}`);
process.exit(1);
}
// Create the output for the README file
try {
// add static part for start
const tags = util.prepTaggedData(
Object.keys(snippets).reduce((acc, key) => {
acc[key] = snippets[key].attributes.tags;
return acc;
}, {}),
);
output += `${startPart}\n`;
const snippetsInTag = {};
// Loop over tags and snippets to create the table of contents
for (const tag of tags) {
const capitalizedTag = util.capitalize(tag, true);
const taggedSnippets = snippetsArray.filter(
snippet => snippet.attributes.tags[0] === tag,
);
output += headers.h3((EMOJIS[tag] || '') + ' ' + capitalizedTag).trim();
TAG_NAMES.forEach(tag => (snippetsInTag[tag] = snippets.filter(v => v.tags[0] == tag)));
output +=
misc.collapsible(
'View contents',
lists.ul(taggedSnippets, snippet =>
misc.link(
`\`${snippet.title}\``,
`${misc.anchor(snippet.title)}${
snippet.attributes.tags.includes('advanced') ? '-' : ''
}`,
),
),
) + '\n';
}
// write Table of Contents
TAG_NAMES.forEach(tag => {
const taggedSnippets = snippetsInTag[tag];
output += headers.h3(util.capitalize(tag));
output += detailsTOC('View contents', taggedSnippets);
});
for (const tag of tags) {
const capitalizedTag = util.capitalize(tag, true);
const taggedSnippets = snippetsArray.filter(
snippet => snippet.attributes.tags[0] === tag,
);
// delimeter after TOC
output += misc.hr();
output +=
misc.hr() + headers.h2((EMOJIS[tag] || '') + ' ' + capitalizedTag) + '\n';
// write actual snippets
TAG_NAMES.forEach(tag => {
output += headers.h2(util.capitalize(tag));
const taggedSnippets = snippetsInTag[tag];
taggedSnippets.forEach(snippet => {
output += headers.h3(snippet.title).trim();
output += `\n\n${snippet.text}${snippet.codeBlocks.slice(0, -1).join('\n\n')}`;
if (snippet.notes && snippet.notes.length) {
output += headers.h4('Notes');
output += `\n${snippet.notes}`;
}
output += misc.collapsible('Examples', snippet.codeBlocks.slice(-1));
output += `\n<br>${misc.link('⬆ Back to top', misc.anchor('Table of Contents'))}\n\n`;
});
});
for (let snippet of taggedSnippets) {
if (snippet.attributes.tags.includes('advanced'))
output +=
headers.h3(
snippet.title + ' ' + misc.image('advanced', '/advanced.svg'),
) + '\n';
else output += headers.h3(snippet.title) + '\n';
// add static part for end
output += snippet.attributes.text;
if (snippet.attributes.codeBlocks.style !== '')
output += `\`\`\`${config.optionalLanguage}\n${snippet.attributes.codeBlocks.style}\n\`\`\`\n\n`;
output += `\`\`\`${config.language}\n${snippet.attributes.codeBlocks.code}\n\`\`\``;
output += misc.collapsible(
'Examples',
`\`\`\`${config.language}\n${snippet.attributes.codeBlocks.example}\n\`\`\``,
);
output +=
'\n<br>' + misc.link('⬆ Back to top', misc.anchor('Contents')) + '\n';
}
}
// Add the ending static part
output += `\n${endPart}\n`;
// Write to the README file
fs.writeFileSync('README.md', output);
} catch (err) {
console.log(`${chalk.red('ERROR!')} During README generation: ${err}`);
console.log(`${red('ERROR!')} During README generation: ${err}`);
process.exit(1);
}
console.log(`${chalk.green('SUCCESS!')} README file generated!`);
console.log(`${green('SUCCESS!')} README file generated!`);
console.timeEnd('Builder');

View File

@ -1,42 +1,80 @@
/*
This is the extractor script that generates the snippets.json file.
Run using `npm run extractor`.
*/
// Load modules
const fs = require('fs-extra');
const path = require('path');
const chalk = require('chalk');
const {
attempt,
readSnippets,
getCodeBlocks,
getSection,
getTitle,
getTextualContent
} = require('./util');
const { green } = require('kleur');
const util = require('./util');
const config = require('../config');
// Paths (relative to package.json)
const SNIPPETS_PATH = `./${config.snippetPath}`;
const OUTPUT_PATH = `./${config.snippetDataPath}`;
// Check if running on Travis, only build for cron jobs and custom builds
if (
util.isTravisCI() &&
process.env['TRAVIS_EVENT_TYPE'] !== 'cron' &&
process.env['TRAVIS_EVENT_TYPE'] !== 'api'
) {
console.log(
`${green(
'NOBUILD',
)} snippet extraction terminated, not a cron or api build!`,
);
process.exit(0);
}
// Setup everything
let snippets = {},
snippetsArray = [];
console.time('Extractor');
attempt('snippet_data.json generation', () => {
const output = Object.entries(readSnippets()).map(([name, contents]) => {
const title = getTitle(contents);
const text = getTextualContent(contents);
const codeBlocks = getCodeBlocks(contents);
const notes = getSection('#### Notes', contents, false)
.split('\n')
.map(v => v.replace(/[*-] /g, ''))
.filter(v => v.trim() !== '');
// Synchronously read all snippets from snippets folder and sort them as necessary (case-insensitive)
snippets = util.readSnippets(SNIPPETS_PATH);
snippetsArray = Object.keys(snippets).reduce((acc, key) => {
acc.push(snippets[key]);
return acc;
}, []);
return {
name,
title,
text,
codeBlocks,
expertise: parseInt((contents.match(/<!--\s*expertise:\s*\(*(.+)\)*/) || [])[1], 10),
tags: (contents.match(/<!--\s*tags:\s*\(*(.+)\)*\s*-->/) || [])[1]
.split(',')
.map(v => v.trim()),
notes
};
});
fs.writeFileSync('./data/snippet_data.json', JSON.stringify(output, null, 2));
});
console.log(`${chalk.green('SUCCESS!')} snippet_data.json file generated!`);
const completeData = {
data: [...snippetsArray],
meta: {
specification: 'http://jsonapi.org/format/',
type: 'snippetArray',
},
};
let listingData = {
data: completeData.data.map(v => ({
id: v.id,
type: 'snippetListing',
title: v.title,
attributes: {
text: v.attributes.text,
tags: v.attributes.tags,
},
meta: {
hash: v.meta.hash,
},
})),
meta: {
specification: 'http://jsonapi.org/format/',
type: 'snippetListingArray',
},
};
// Write files
fs.writeFileSync(
path.join(OUTPUT_PATH, 'snippets.json'),
JSON.stringify(completeData, null, 2),
);
fs.writeFileSync(
path.join(OUTPUT_PATH, 'snippetList.json'),
JSON.stringify(listingData, null, 2),
);
// Display messages and time
console.log(
`${green('SUCCESS!')} snippets.json and snippetList.json files generated!`,
);
console.timeEnd('Extractor');

View File

@ -1,91 +0,0 @@
const fs = require('fs-extra');
const path = require('path');
const chalk = require('chalk');
const SNIPPETS_PATH = './snippets';
const attempt = (task, cb) => {
try {
return cb();
} catch (e) {
console.log(`${chalk.red('ERROR!')} During ${task}: ${e}`);
process.exit(1);
return null;
}
};
const capitalize = ([first, ...rest], lowerRest = false) =>
first.toUpperCase() + (lowerRest ? rest.join('').toLowerCase() : rest.join(''));
const readSnippets = () =>
attempt('read snippets', () =>
fs
.readdirSync(SNIPPETS_PATH)
.sort((a, b) => (a.toLowerCase() < b.toLowerCase() ? -1 : 1))
.reduce((acc, name) => {
acc[name] = fs.readFileSync(path.join(SNIPPETS_PATH, name), 'utf8').replace(/\r\n/g, '\n');
return acc;
}, {})
);
const getCodeBlocks = str => {
const regex = /```[.\S\s]*?```/g;
const results = [];
let m = null;
while ((m = regex.exec(str)) !== null) {
if (m.index === regex.lastIndex) {
regex.lastIndex += 1;
}
m.forEach(match => results.push(match));
}
return results;
};
const getSection = (searchString, contents, includeSubsections = true) => {
const indexOfSearch = contents.indexOf(searchString);
if (indexOfSearch < 0) return '';
let endSearch = '\\n#';
if (includeSubsections) {
let i;
for (i = 0; searchString[i] === '#' && i < searchString.length; i++);
if (i > 0) {
endSearch += `{${i - 1},${i}}[^#]`;
}
}
const endRegex = new RegExp(endSearch);
const sliceStart = indexOfSearch + searchString.length + 1;
const endIndex = contents.slice(sliceStart).search(endRegex);
const sliceEnd = endIndex === -1 ? undefined : endIndex + sliceStart;
return contents.slice(sliceStart, sliceEnd).trim();
};
const getTextualContent = str => {
const regex = /###.*\n*([\s\S]*?)```/g;
const results = [];
let m = null;
while ((m = regex.exec(str)) !== null) {
if (m.index === regex.lastIndex) regex.lastIndex += 1;
m.forEach((match, groupIndex) => {
results.push(match);
});
}
return results[1];
};
const getTitle = contents => contents.split('\n')[0].replace(/^#+\s+/g, '');
module.exports = {
attempt,
readSnippets,
SNIPPETS_PATH,
capitalize,
getTextualContent,
getCodeBlocks,
getSection,
getTitle
};

View File

@ -0,0 +1,12 @@
// Checks if current environment is Travis CI, Cron builds, API builds
const isTravisCI = () => 'TRAVIS' in process.env && 'CI' in process.env;
const isTravisCronOrAPI = () =>
process.env['TRAVIS_EVENT_TYPE'] === 'cron' ||
process.env['TRAVIS_EVENT_TYPE'] === 'api';
const isNotTravisCronOrAPI = () => !isTravisCronOrAPI();
module.exports = {
isTravisCI,
isTravisCronOrAPI,
isNotTravisCronOrAPI,
};

60
scripts/util/helpers.js Normal file
View File

@ -0,0 +1,60 @@
const config = require('../../config');
const getMarkDownAnchor = paragraphTitle =>
paragraphTitle
.trim()
.toLowerCase()
.replace(/[^\w\- ]+/g, '')
.replace(/\s/g, '-')
.replace(/\-+$/, '');
// Creates an object from pairs
const objectFromPairs = arr => arr.reduce((a, v) => ((a[v[0]] = v[1]), a), {});
// Optimizes nodes in an HTML document
const optimizeNodes = (data, regexp, replacer) => {
let count = 0;
let output = data;
do {
output = output.replace(regexp, replacer);
count = 0;
while (regexp.exec(output) !== null) ++count;
} while (count > 0);
return output;
};
// Capitalizes the first letter of a string
const capitalize = (str, lowerRest = false) =>
str.slice(0, 1).toUpperCase() +
(lowerRest ? str.slice(1).toLowerCase() : str.slice(1));
const prepTaggedData = tagDbData =>
[...new Set(Object.entries(tagDbData).map(t => t[1][0]))]
.filter(v => v)
.sort((a, b) =>
capitalize(a, true) === 'Uncategorized'
? 1
: capitalize(b, true) === 'Uncategorized'
? -1
: a.localeCompare(b),
);
const makeExamples = data => {
data =
data.slice(0, data.lastIndexOf(`\`\`\`${config.language}`)).trim() +
misc.collapsible(
'Examples',
data.slice(
data.lastIndexOf(`\`\`\`${config.language}`),
data.lastIndexOf('```'),
) + data.slice(data.lastIndexOf('```')),
);
return `${data}\n<br>${misc.link(
'⬆ Back to top',
misc.anchor('Contents'),
)}\n\n`;
};
module.exports = {
getMarkDownAnchor,
objectFromPairs,
optimizeNodes,
capitalize,
prepTaggedData,
makeExamples,
};

37
scripts/util/index.js Normal file
View File

@ -0,0 +1,37 @@
const {
isTravisCI,
isTravisCronOrAPI,
isNotTravisCronOrAPI,
} = require('./environmentCheck');
const {
getMarkDownAnchor,
objectFromPairs,
optimizeNodes,
capitalize,
prepTaggedData,
makeExamples,
} = require('./helpers');
const {
getFilesInDir,
hashData,
getCodeBlocks,
getTextualContent,
readSnippets,
} = require('./snippetParser');
module.exports = {
isTravisCI,
isTravisCronOrAPI,
isNotTravisCronOrAPI,
getMarkDownAnchor,
objectFromPairs,
optimizeNodes,
capitalize,
prepTaggedData,
makeExamples,
getFilesInDir,
hashData,
getCodeBlocks,
getTextualContent,
readSnippets,
};

View File

@ -0,0 +1,129 @@
const fs = require('fs-extra'),
path = require('path'),
{ red } = require('kleur'),
crypto = require('crypto'),
frontmatter = require('front-matter');
const config = require('../../config');
// Reade all files in a directory
const getFilesInDir = (directoryPath, withPath, exclude = null) => {
try {
let directoryFilenames = fs.readdirSync(directoryPath);
directoryFilenames.sort((a, b) => {
a = a.toLowerCase();
b = b.toLowerCase();
if (a < b) return -1;
if (a > b) return 1;
return 0;
});
if (withPath) {
// a hacky way to do conditional array.map
return directoryFilenames.reduce((fileNames, fileName) => {
if (
exclude == null ||
!exclude.some(toExclude => fileName === toExclude)
)
fileNames.push(`${directoryPath}/${fileName}`);
return fileNames;
}, []);
}
return directoryFilenames.filter(v => v !== 'README.md');
} catch (err) {
console.log(`${red('ERROR!')} During snippet loading: ${err}`);
process.exit(1);
}
};
// Creates a hash for a value using the SHA-256 algorithm.
const hashData = val =>
crypto
.createHash('sha256')
.update(val)
.digest('hex');
// Gets the code blocks for a snippet file.
const getCodeBlocks = str => {
const regex = /```[.\S\s]*?```/g;
let results = [];
let m = null;
while ((m = regex.exec(str)) !== null) {
if (m.index === regex.lastIndex) regex.lastIndex += 1;
m.forEach((match, groupIndex) => {
results.push(match);
});
}
const replacer = new RegExp(
`\`\`\`${config.language}([\\s\\S]*?)\`\`\``,
'g',
);
const optionalReplacer = new RegExp(
`\`\`\`${config.optionalLanguage}([\\s\\S]*?)\`\`\``,
'g',
);
results = results.map(v => v.replace(replacer, '$1').replace(optionalReplacer, '$1').trim());
if(results.length > 2)
return {
style: results[0],
code: results[1],
example: results[2],
};
return {
style: '',
code: results[0],
example: results[1],
};
};
// Gets the textual content for a snippet file.
const getTextualContent = str => {
const regex = /([\s\S]*?)```/g;
const results = [];
let m = null;
while ((m = regex.exec(str)) !== null) {
if (m.index === regex.lastIndex) regex.lastIndex += 1;
m.forEach((match, groupIndex) => {
results.push(match);
});
}
return results[1].replace(/\r\n/g, '\n');
};
// Synchronously read all snippets and sort them as necessary (case-insensitive)
const readSnippets = snippetsPath => {
const snippetFilenames = getFilesInDir(snippetsPath, false);
let snippets = {};
try {
for (let snippet of snippetFilenames) {
let data = frontmatter(
fs.readFileSync(path.join(snippetsPath, snippet), 'utf8'),
);
snippets[snippet] = {
id: snippet.slice(0, -3),
title: data.attributes.title,
type: 'snippet',
attributes: {
fileName: snippet,
text: getTextualContent(data.body),
codeBlocks: getCodeBlocks(data.body),
tags: data.attributes.tags.split(',').map(t => t.trim()),
},
meta: {
hash: hashData(data.body),
},
};
}
} catch (err) {
console.log(`${red('ERROR!')} During snippet loading: ${err}`);
process.exit(1);
}
return snippets;
};
module.exports = {
getFilesInDir,
hashData,
getCodeBlocks,
getTextualContent,
readSnippets,
};