Travis build: 407

This commit is contained in:
30secondsofcode
2019-08-26 10:03:49 +00:00
parent 2776b342dd
commit 983bc086c1
57 changed files with 376 additions and 873 deletions

View File

@ -3,29 +3,22 @@
Run using `npm run builder`.
*/
// Load modules
const fs = require('fs-extra');
const path = require('path');
const { green, red } = require('kleur');
const util = require('./util');
const markdown = require('markdown-builder');
const { headers, misc, lists } = markdown;
const config = require('../config');
const fs = require('fs-extra')
const path = require('path')
const { green, red } = require('kleur')
const util = require('./util')
const markdown = require('markdown-builder')
const { headers, misc, lists } = markdown
const config = require('../config')
// Paths (relative to package.json)
const SNIPPETS_PATH = `./${config.snippetPath}`;
const STATIC_PARTS_PATH = `./${config.staticPartsPath}`;
const SNIPPETS_PATH = `./${config.snippetPath}`
const STATIC_PARTS_PATH = `./${config.staticPartsPath}`
// Terminate if parent commit is a Travis build
if (
util.isTravisCI() &&
/^Travis build: \d+/g.test(process.env['TRAVIS_COMMIT_MESSAGE'])
) {
console.log(
`${green(
'NOBUILD',
)} README build terminated, parent commit is a Travis build!`,
);
process.exit(0);
if (util.isTravisCI() && /^Travis build: \d+/g.test(process.env['TRAVIS_COMMIT_MESSAGE'])) {
console.log(`${green('NOBUILD')} README build terminated, parent commit is a Travis build!`)
process.exit(0)
}
// Setup everything
@ -33,51 +26,43 @@ let snippets = {},
snippetsArray = [],
startPart = '',
endPart = '',
output = '';
const EMOJIS = {};
output = ''
const EMOJIS = {}
console.time('Builder');
console.time('Builder')
// Synchronously read all snippets from snippets folder and sort them as necessary (case-insensitive)
snippets = util.readSnippets(SNIPPETS_PATH);
snippets = util.readSnippets(SNIPPETS_PATH)
snippetsArray = Object.keys(snippets).reduce((acc, key) => {
acc.push(snippets[key]);
return acc;
}, []);
acc.push(snippets[key])
return acc
}, [])
// Load static parts for the README file
try {
startPart = fs.readFileSync(
path.join(STATIC_PARTS_PATH, 'README-start.md'),
'utf8',
);
endPart = fs.readFileSync(
path.join(STATIC_PARTS_PATH, 'README-end.md'),
'utf8',
);
startPart = fs.readFileSync(path.join(STATIC_PARTS_PATH, 'README-start.md'), 'utf8')
endPart = fs.readFileSync(path.join(STATIC_PARTS_PATH, 'README-end.md'), 'utf8')
} catch (err) {
console.log(`${red('ERROR!')} During static part loading: ${err}`);
process.exit(1);
console.log(`${red('ERROR!')} During static part loading: ${err}`)
process.exit(1)
}
// Create the output for the README file
try {
const tags = util.prepTaggedData(
Object.keys(snippets).reduce((acc, key) => {
acc[key] = snippets[key].attributes.tags;
return acc;
}, {}),
);
acc[key] = snippets[key].attributes.tags
return acc
}, {})
)
output += `${startPart}\n`;
output += `${startPart}\n`
// Loop over tags and snippets to create the table of contents
for (const tag of tags) {
const capitalizedTag = util.capitalize(tag, true);
const taggedSnippets = snippetsArray.filter(
snippet => snippet.attributes.tags[0] === tag,
);
output += headers.h3((EMOJIS[tag] || '') + ' ' + capitalizedTag).trim();
const capitalizedTag = util.capitalize(tag, true)
const taggedSnippets = snippetsArray.filter(snippet => snippet.attributes.tags[0] === tag)
output += headers.h3((EMOJIS[tag] || '') + ' ' + capitalizedTag).trim()
output +=
misc.collapsible(
@ -87,56 +72,49 @@ try {
`\`${snippet.title}\``,
`${misc.anchor(snippet.title)}${
snippet.attributes.tags.includes('advanced') ? '-' : ''
}`,
),
),
) + '\n';
}`
)
)
) + '\n'
}
for (const tag of tags) {
const capitalizedTag = util.capitalize(tag, true);
const taggedSnippets = snippetsArray.filter(
snippet => snippet.attributes.tags[0] === tag,
);
const capitalizedTag = util.capitalize(tag, true)
const taggedSnippets = snippetsArray.filter(snippet => snippet.attributes.tags[0] === tag)
output +=
misc.hr() + headers.h2((EMOJIS[tag] || '') + ' ' + capitalizedTag) + '\n';
output += misc.hr() + headers.h2((EMOJIS[tag] || '') + ' ' + capitalizedTag) + '\n'
for (let snippet of taggedSnippets) {
if (snippet.attributes.tags.includes('advanced'))
output +=
headers.h3(
snippet.title + ' ' + misc.image('advanced', '/advanced.svg'),
) + '\n';
else output += headers.h3(snippet.title) + '\n';
output += headers.h3(snippet.title + ' ' + misc.image('advanced', '/advanced.svg')) + '\n'
else output += headers.h3(snippet.title) + '\n'
output += snippet.attributes.text;
output += snippet.attributes.text
output += `\`\`\`${config.secondLanguage}\n${snippet.attributes.codeBlocks.html}\n\`\`\`\n\n`;
output += `\`\`\`${config.language}\n${snippet.attributes.codeBlocks.css}\n\`\`\`\n\n`;
output += `\`\`\`${config.secondLanguage}\n${snippet.attributes.codeBlocks.html}\n\`\`\`\n\n`
output += `\`\`\`${config.language}\n${snippet.attributes.codeBlocks.css}\n\`\`\`\n\n`
if (snippet.attributes.codeBlocks.js)
output += `\`\`\`${config.optionalLanguage}\n${snippet.attributes.codeBlocks.js}\n\`\`\`\n\n`;
output += `\`\`\`${config.optionalLanguage}\n${snippet.attributes.codeBlocks.js}\n\`\`\`\n\n`
output += headers.h4('Explanation');
output += snippet.attributes.explanation;
output += headers.h4('Explanation')
output += snippet.attributes.explanation
output += headers.h4('Browser support') + '\n';
output += snippet.attributes.browserSupport.supportPercentage.toFixed(1) + '%';
output += snippet.attributes.browserSupport.text;
output += headers.h4('Browser support') + '\n'
output += snippet.attributes.browserSupport.supportPercentage.toFixed(1) + '%'
output += snippet.attributes.browserSupport.text
output +=
'\n<br>' + misc.link('⬆ Back to top', misc.anchor('Contents')) + '\n';
output += '\n<br>' + misc.link('⬆ Back to top', misc.anchor('Contents')) + '\n'
}
}
// Add the ending static part
output += `\n${endPart}\n`;
output += `\n${endPart}\n`
// Write to the README file
fs.writeFileSync('README.md', output);
fs.writeFileSync('README.md', output)
} catch (err) {
console.log(`${red('ERROR!')} During README generation: ${err}`);
process.exit(1);
console.log(`${red('ERROR!')} During README generation: ${err}`)
process.exit(1)
}
console.log(`${green('SUCCESS!')} README file generated!`);
console.timeEnd('Builder');
console.log(`${green('SUCCESS!')} README file generated!`)
console.timeEnd('Builder')

View File

@ -3,15 +3,15 @@
Run using `npm run extractor`.
*/
// Load modules
const fs = require('fs-extra');
const path = require('path');
const { green } = require('kleur');
const util = require('./util');
const config = require('../config');
const fs = require('fs-extra')
const path = require('path')
const { green } = require('kleur')
const util = require('./util')
const config = require('../config')
// Paths (relative to package.json)
const SNIPPETS_PATH = `./${config.snippetPath}`;
const OUTPUT_PATH = `./${config.snippetDataPath}`;
const SNIPPETS_PATH = `./${config.snippetPath}`
const OUTPUT_PATH = `./${config.snippetDataPath}`
// Check if running on Travis, only build for cron jobs and custom builds
if (
@ -19,33 +19,29 @@ if (
process.env['TRAVIS_EVENT_TYPE'] !== 'cron' &&
process.env['TRAVIS_EVENT_TYPE'] !== 'api'
) {
console.log(
`${green(
'NOBUILD',
)} snippet extraction terminated, not a cron or api build!`,
);
process.exit(0);
console.log(`${green('NOBUILD')} snippet extraction terminated, not a cron or api build!`)
process.exit(0)
}
// Setup everything
let snippets = {},
snippetsArray = [];
console.time('Extractor');
snippetsArray = []
console.time('Extractor')
// Synchronously read all snippets from snippets folder and sort them as necessary (case-insensitive)
snippets = util.readSnippets(SNIPPETS_PATH);
snippets = util.readSnippets(SNIPPETS_PATH)
snippetsArray = Object.keys(snippets).reduce((acc, key) => {
acc.push(snippets[key]);
return acc;
}, []);
acc.push(snippets[key])
return acc
}, [])
const completeData = {
data: [...snippetsArray],
meta: {
specification: 'http://jsonapi.org/format/',
type: 'snippetArray',
},
};
type: 'snippetArray'
}
}
let listingData = {
data: completeData.data.map(v => ({
id: v.id,
@ -53,28 +49,20 @@ let listingData = {
title: v.title,
attributes: {
text: v.attributes.text,
tags: v.attributes.tags,
tags: v.attributes.tags
},
meta: {
hash: v.meta.hash,
},
hash: v.meta.hash
}
})),
meta: {
specification: 'http://jsonapi.org/format/',
type: 'snippetListingArray',
},
};
type: 'snippetListingArray'
}
}
// Write files
fs.writeFileSync(
path.join(OUTPUT_PATH, 'snippets.json'),
JSON.stringify(completeData, null, 2),
);
fs.writeFileSync(
path.join(OUTPUT_PATH, 'snippetList.json'),
JSON.stringify(listingData, null, 2),
);
fs.writeFileSync(path.join(OUTPUT_PATH, 'snippets.json'), JSON.stringify(completeData, null, 2))
fs.writeFileSync(path.join(OUTPUT_PATH, 'snippetList.json'), JSON.stringify(listingData, null, 2))
// Display messages and time
console.log(
`${green('SUCCESS!')} snippets.json and snippetList.json files generated!`,
);
console.timeEnd('Extractor');
console.log(`${green('SUCCESS!')} snippets.json and snippetList.json files generated!`)
console.timeEnd('Extractor')

View File

@ -1,12 +1,11 @@
// Checks if current environment is Travis CI, Cron builds, API builds
const isTravisCI = () => 'TRAVIS' in process.env && 'CI' in process.env;
const isTravisCI = () => 'TRAVIS' in process.env && 'CI' in process.env
const isTravisCronOrAPI = () =>
process.env['TRAVIS_EVENT_TYPE'] === 'cron' ||
process.env['TRAVIS_EVENT_TYPE'] === 'api';
const isNotTravisCronOrAPI = () => !isTravisCronOrAPI();
process.env['TRAVIS_EVENT_TYPE'] === 'cron' || process.env['TRAVIS_EVENT_TYPE'] === 'api'
const isNotTravisCronOrAPI = () => !isTravisCronOrAPI()
module.exports = {
isTravisCI,
isTravisCronOrAPI,
isNotTravisCronOrAPI,
};
isNotTravisCronOrAPI
}

View File

@ -1,4 +1,4 @@
const config = require('../../config');
const config = require('../../config')
const getMarkDownAnchor = paragraphTitle =>
paragraphTitle
@ -6,24 +6,23 @@ const getMarkDownAnchor = paragraphTitle =>
.toLowerCase()
.replace(/[^\w\- ]+/g, '')
.replace(/\s/g, '-')
.replace(/\-+$/, '');
.replace(/\-+$/, '')
// Creates an object from pairs
const objectFromPairs = arr => arr.reduce((a, v) => ((a[v[0]] = v[1]), a), {});
const objectFromPairs = arr => arr.reduce((a, v) => ((a[v[0]] = v[1]), a), {})
// Optimizes nodes in an HTML document
const optimizeNodes = (data, regexp, replacer) => {
let count = 0;
let output = data;
let count = 0
let output = data
do {
output = output.replace(regexp, replacer);
count = 0;
while (regexp.exec(output) !== null) ++count;
} while (count > 0);
return output;
};
output = output.replace(regexp, replacer)
count = 0
while (regexp.exec(output) !== null) ++count
} while (count > 0)
return output
}
// Capitalizes the first letter of a string
const capitalize = (str, lowerRest = false) =>
str.slice(0, 1).toUpperCase() +
(lowerRest ? str.slice(1).toLowerCase() : str.slice(1));
str.slice(0, 1).toUpperCase() + (lowerRest ? str.slice(1).toLowerCase() : str.slice(1))
const prepTaggedData = tagDbData =>
[...new Set(Object.entries(tagDbData).map(t => t[1][0]))]
.filter(v => v)
@ -32,23 +31,18 @@ const prepTaggedData = tagDbData =>
? 1
: capitalize(b, true) === 'Uncategorized'
? -1
: a.localeCompare(b),
);
: a.localeCompare(b)
)
const makeExamples = data => {
data =
data.slice(0, data.lastIndexOf(`\`\`\`${config.language}`)).trim() +
misc.collapsible(
'Examples',
data.slice(
data.lastIndexOf(`\`\`\`${config.language}`),
data.lastIndexOf('```'),
) + data.slice(data.lastIndexOf('```')),
);
return `${data}\n<br>${misc.link(
'⬆ Back to top',
misc.anchor('Contents'),
)}\n\n`;
};
data.slice(data.lastIndexOf(`\`\`\`${config.language}`), data.lastIndexOf('```')) +
data.slice(data.lastIndexOf('```'))
)
return `${data}\n<br>${misc.link('⬆ Back to top', misc.anchor('Contents'))}\n\n`
}
module.exports = {
getMarkDownAnchor,
@ -56,5 +50,5 @@ module.exports = {
optimizeNodes,
capitalize,
prepTaggedData,
makeExamples,
};
makeExamples
}

View File

@ -1,23 +1,19 @@
const {
isTravisCI,
isTravisCronOrAPI,
isNotTravisCronOrAPI,
} = require('./environmentCheck');
const { isTravisCI, isTravisCronOrAPI, isNotTravisCronOrAPI } = require('./environmentCheck')
const {
getMarkDownAnchor,
objectFromPairs,
optimizeNodes,
capitalize,
prepTaggedData,
makeExamples,
} = require('./helpers');
makeExamples
} = require('./helpers')
const {
getFilesInDir,
hashData,
getCodeBlocks,
getTextualContent,
readSnippets,
} = require('./snippetParser');
readSnippets
} = require('./snippetParser')
module.exports = {
isTravisCI,
@ -33,5 +29,5 @@ module.exports = {
hashData,
getCodeBlocks,
getTextualContent,
readSnippets,
};
readSnippets
}

View File

@ -2,164 +2,150 @@ const fs = require('fs-extra'),
path = require('path'),
{ red } = require('kleur'),
crypto = require('crypto'),
frontmatter = require('front-matter');
const sass = require('node-sass');
const caniuseDb = require('caniuse-db/data.json');
const config = require('../../config');
frontmatter = require('front-matter')
const sass = require('node-sass')
const caniuseDb = require('caniuse-db/data.json')
const config = require('../../config')
// Reade all files in a directory
const getFilesInDir = (directoryPath, withPath, exclude = null) => {
try {
let directoryFilenames = fs.readdirSync(directoryPath);
let directoryFilenames = fs.readdirSync(directoryPath)
directoryFilenames.sort((a, b) => {
a = a.toLowerCase();
b = b.toLowerCase();
if (a < b) return -1;
if (a > b) return 1;
return 0;
});
a = a.toLowerCase()
b = b.toLowerCase()
if (a < b) return -1
if (a > b) return 1
return 0
})
if (withPath) {
// a hacky way to do conditional array.map
return directoryFilenames.reduce((fileNames, fileName) => {
if (
exclude == null ||
!exclude.some(toExclude => fileName === toExclude)
)
fileNames.push(`${directoryPath}/${fileName}`);
return fileNames;
}, []);
if (exclude == null || !exclude.some(toExclude => fileName === toExclude))
fileNames.push(`${directoryPath}/${fileName}`)
return fileNames
}, [])
}
return directoryFilenames.filter(v => v !== 'README.md');
return directoryFilenames.filter(v => v !== 'README.md')
} catch (err) {
console.log(`${red('ERROR!')} During snippet loading: ${err}`);
process.exit(1);
console.log(`${red('ERROR!')} During snippet loading: ${err}`)
process.exit(1)
}
};
}
// Creates a hash for a value using the SHA-256 algorithm.
const hashData = val =>
crypto
.createHash('sha256')
.update(val)
.digest('hex');
.digest('hex')
// Gets the code blocks for a snippet file.
const getCodeBlocks = str => {
const regex = /```[.\S\s]*?```/g;
let results = [];
let m = null;
const regex = /```[.\S\s]*?```/g
let results = []
let m = null
while ((m = regex.exec(str)) !== null) {
if (m.index === regex.lastIndex) regex.lastIndex += 1;
if (m.index === regex.lastIndex) regex.lastIndex += 1
m.forEach((match, groupIndex) => {
results.push(match);
});
results.push(match)
})
}
const replacer = new RegExp(
`\`\`\`${config.language}([\\s\\S]*?)\`\`\``,
'g',
);
const secondReplacer = new RegExp(
`\`\`\`${config.secondLanguage}([\\s\\S]*?)\`\`\``,
'g',
);
const optionalReplacer = new RegExp(
`\`\`\`${config.optionalLanguage}([\\s\\S]*?)\`\`\``,
'g',
);
const replacer = new RegExp(`\`\`\`${config.language}([\\s\\S]*?)\`\`\``, 'g')
const secondReplacer = new RegExp(`\`\`\`${config.secondLanguage}([\\s\\S]*?)\`\`\``, 'g')
const optionalReplacer = new RegExp(`\`\`\`${config.optionalLanguage}([\\s\\S]*?)\`\`\``, 'g')
results = results.map(v =>
v
.replace(replacer, '$1')
.replace(secondReplacer, '$1')
.replace(optionalReplacer, '$1')
.trim()
);
)
if (results.length > 2)
return {
html: results[0],
css: results[1],
js: results[2],
};
js: results[2]
}
return {
html: results[0],
css: results[1],
js: '',
};
};
js: ''
}
}
// Gets the textual content for a snippet file.
const getTextualContent = str => {
const regex = /([\s\S]*?)```/g;
const results = [];
let m = null;
const regex = /([\s\S]*?)```/g
const results = []
let m = null
while ((m = regex.exec(str)) !== null) {
if (m.index === regex.lastIndex) regex.lastIndex += 1;
if (m.index === regex.lastIndex) regex.lastIndex += 1
m.forEach((match, groupIndex) => {
results.push(match);
});
results.push(match)
})
}
return results[1].replace(/\r\n/g, '\n');
};
return results[1].replace(/\r\n/g, '\n')
}
// Gets the explanation for a snippet file.
const getExplanation = str => {
const regex = /####\s*Explanation([\s\S]*)####/g;
const results = [];
let m = null;
const regex = /####\s*Explanation([\s\S]*)####/g
const results = []
let m = null
while ((m = regex.exec(str)) !== null) {
if (m.index === regex.lastIndex) regex.lastIndex += 1;
if (m.index === regex.lastIndex) regex.lastIndex += 1
m.forEach((match, groupIndex) => {
results.push(match);
});
results.push(match)
})
}
// console.log(results);
return results[1].replace(/\r\n/g, '\n');
};
return results[1].replace(/\r\n/g, '\n')
}
// Gets the browser support for a snippet file.
const getBrowserSupport = str => {
const regex = /####\s*Browser [s|S]upport([\s\S]*)/g;
const results = [];
let m = null;
const regex = /####\s*Browser [s|S]upport([\s\S]*)/g
const results = []
let m = null
while ((m = regex.exec(str)) !== null) {
if (m.index === regex.lastIndex) regex.lastIndex += 1;
if (m.index === regex.lastIndex) regex.lastIndex += 1
m.forEach((match, groupIndex) => {
results.push(match);
});
results.push(match)
})
}
let browserSupportText = results[1].replace(/\r\n/g, '\n');
const supportPercentage = (browserSupportText.match(/https?:\/\/caniuse\.com\/#feat=.*/g) || []).map(
feat => {
const featData = caniuseDb.data[feat.match(/#feat=(.*)/)[1]];
// caniuse doesn't count "untracked" users, which makes the overall share appear much lower
// than it probably is. Most of these untracked browsers probably support these features.
// Currently it's around 5.3% untracked, so we'll use 4% as probably supporting the feature.
// Also the npm package appears to be show higher usage % than the main website, this shows
// about 0.2% lower than the main website when selecting "tracked users" (as of Feb 2019).
const UNTRACKED_PERCENT = 4;
const usage = featData
? Number(featData.usage_perc_y + featData.usage_perc_a) + UNTRACKED_PERCENT
: 100;
return Math.min(100, usage);
}
)
let browserSupportText = results[1].replace(/\r\n/g, '\n')
const supportPercentage = (
browserSupportText.match(/https?:\/\/caniuse\.com\/#feat=.*/g) || []
).map(feat => {
const featData = caniuseDb.data[feat.match(/#feat=(.*)/)[1]]
// caniuse doesn't count "untracked" users, which makes the overall share appear much lower
// than it probably is. Most of these untracked browsers probably support these features.
// Currently it's around 5.3% untracked, so we'll use 4% as probably supporting the feature.
// Also the npm package appears to be show higher usage % than the main website, this shows
// about 0.2% lower than the main website when selecting "tracked users" (as of Feb 2019).
const UNTRACKED_PERCENT = 4
const usage = featData
? Number(featData.usage_perc_y + featData.usage_perc_a) + UNTRACKED_PERCENT
: 100
return Math.min(100, usage)
})
return {
text: browserSupportText,
supportPercentage: Math.min(...supportPercentage,100)
supportPercentage: Math.min(...supportPercentage, 100)
}
};
}
// Synchronously read all snippets and sort them as necessary (case-insensitive)
const readSnippets = snippetsPath => {
const snippetFilenames = getFilesInDir(snippetsPath, false);
const snippetFilenames = getFilesInDir(snippetsPath, false)
let snippets = {};
let snippets = {}
try {
for (let snippet of snippetFilenames) {
let data = frontmatter(
fs.readFileSync(path.join(snippetsPath, snippet), 'utf8'),
);
let data = frontmatter(fs.readFileSync(path.join(snippetsPath, snippet), 'utf8'))
snippets[snippet] = {
id: snippet.slice(0, -3),
title: data.attributes.title,
@ -170,22 +156,24 @@ const readSnippets = snippetsPath => {
explanation: getExplanation(data.body),
browserSupport: getBrowserSupport(data.body),
codeBlocks: getCodeBlocks(data.body),
tags: data.attributes.tags.split(',').map(t => t.trim()),
tags: data.attributes.tags.split(',').map(t => t.trim())
},
meta: {
hash: hashData(data.body),
},
};
snippets[snippet].attributes.codeBlocks.scopedCss = sass.renderSync({
data: `[data-scope="${snippets[snippet].id}"] { ${snippets[snippet].attributes.codeBlocks.css} }`
}).css.toString();
hash: hashData(data.body)
}
}
snippets[snippet].attributes.codeBlocks.scopedCss = sass
.renderSync({
data: `[data-scope="${snippets[snippet].id}"] { ${snippets[snippet].attributes.codeBlocks.css} }`
})
.css.toString()
}
} catch (err) {
console.log(`${red('ERROR!')} During snippet loading: ${err}`);
process.exit(1);
console.log(`${red('ERROR!')} During snippet loading: ${err}`)
process.exit(1)
}
return snippets;
};
return snippets
}
module.exports = {
getFilesInDir,
@ -194,5 +182,5 @@ module.exports = {
getTextualContent,
getExplanation,
getBrowserSupport,
readSnippets,
};
readSnippets
}