andre/30-seconds-of-code
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updated extractor, removed analyzer

Browse Source
This commit is contained in:
Angelos Chalaris
2018-10-26 18:34:57 +03:00
parent c6f2e2d1e7
commit 8d2ddd7e94
9 changed files with 8132 additions and 7706 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
scripts/analyze.js
View File

@ -1,58 +0,0 @@
/*
This is the analyzer script that generates the snippetAnalytics.json and snippetArchiveAnalytics.json files.
Run using `npm run analyzer`.
*/
// Load modules
const fs = require('fs-extra');
const path = require('path');
const chalk = require('chalk');
const prism = require('prismjs');
let snippetsData = require('../snippet_data/snippets.json');
let snippetsArchiveData = require('../snippet_data/snippetsArchive.json');
// Paths
const OUTPUT_PATH = './snippet_data';
console.time('Analyzer');
// Read data
let [snippetTokens, snippetArchiveTokens] = [snippetsData, snippetsArchiveData].map(v => ({
data: v.data.map(snippet => {
let tokens = prism.tokenize(
snippet.attributes.codeBlocks[0],
prism.languages.javascript,
'javascript'
);
return {
id: snippet.id,
type: 'snippetAnalysis',
attributes: {
codeLength: snippet.attributes.codeBlocks[0].trim().length,
tokenCount: tokens.length,
functionCount: tokens.filter(t => t.type === 'function').length,
operatorCount: tokens.filter(t => t.type === 'operator').length,
keywordCount: tokens.filter(t => t.type === 'keyword').length,
distinctFunctionCount: [
...new Set(tokens.filter(t => t.type === 'function').map(t => t.content))
].length
},
meta: {
hash: snippet.meta.hash
}
};
}),
meta: { specification: 'http://jsonapi.org/format/' }
}));
// Write data
fs.writeFileSync(
path.join(OUTPUT_PATH, 'snippetAnalytics.json'),
JSON.stringify(snippetTokens, null, 2)
);
fs.writeFileSync(
path.join(OUTPUT_PATH, 'snippetArchiveAnalytics.json'),
JSON.stringify(snippetArchiveTokens, null, 2)
);
// Display messages and time
console.log(
`${chalk.green(
'SUCCESS!'
)} snippetAnalyticss.json and snippetArchiveAnalytics.json files generated!`
);
console.timeEnd('Analyzer');

87
scripts/extract.js
View File

@ -29,59 +29,66 @@ snippets = util.readSnippets(SNIPPETS_PATH);
archivedSnippets = util.readSnippets(SNIPPETS_ARCHIVE_PATH);
tagDbData = util.readTags();
// Extract snippet data
let snippetData = {
data: Object.keys(snippets).map(key => {
return {
id: key.slice(0, -3),
type: 'snippet',
attributes: {
fileName: key,
text: util.getTextualContent(snippets[key]).trim(),
codeBlocks: util
.getCodeBlocks(snippets[key])
.map(v => v.replace(/```js([\s\S]*?)```/g, '$1').trim()),
tags: tagDbData[key.slice(0, -3)]
},
meta: {
archived: false,
hash: util.hashData(snippets[key])
}
};
}),
let snippetData = Object.keys(snippets).map(key => {
return {
id: key.slice(0, -3),
type: 'snippet',
attributes: {
fileName: key,
text: util.getTextualContent(snippets[key]).trim(),
codeBlocks: util.getCodeBlocks(snippets[key]),
tags: tagDbData[key.slice(0, -3)]
},
meta: {
archived: false,
hash: util.hashData(snippets[key])
}
};
});
// Extract archived snippet data
let snippetArchiveData = Object.keys(archivedSnippets).map(key => {
return {
id: key.slice(0, -3),
type: 'snippet',
attributes: {
fileName: key,
text: util.getTextualContent(archivedSnippets[key]).trim(),
codeBlocks: util.getCodeBlocks(archivedSnippets[key]),
tags: []
},
meta: {
archived: true,
hash: util.hashData(archivedSnippets[key])
}
};
});
const completeData = {
data: [...snippetData, ...snippetArchiveData],
meta: {
specification: 'http://jsonapi.org/format/'
}
};
// Extract archived snippet data
let snippetArchiveData = {
data: Object.keys(archivedSnippets).map(key => {
return {
id: key.slice(0, -3),
type: 'snippet',
let listingData = {
data:
completeData.data.map(v => ({
id: v.id,
type: 'snippetListing',
attributes: {
fileName: key,
text: util.getTextualContent(archivedSnippets[key]).trim(),
codeBlocks: util
.getCodeBlocks(archivedSnippets[key])
.map(v => v.replace(/```js([\s\S]*?)```/g, '$1').trim()),
tags: []
tags: v.attributes.tags,
archived: v.meta.archived
},
meta: {
archived: true,
hash: util.hashData(archivedSnippets[key])
hash: v.meta.hash
}
};
}),
}))
,
meta: {
specification: 'http://jsonapi.org/format/'
}
};
// Write files
fs.writeFileSync(path.join(OUTPUT_PATH, 'snippets.json'), JSON.stringify(snippetData, null, 2));
fs.writeFileSync(
path.join(OUTPUT_PATH, 'snippetsArchive.json'),
JSON.stringify(snippetArchiveData, null, 2)
);
fs.writeFileSync(path.join(OUTPUT_PATH, 'snippets.json'), JSON.stringify(completeData, null, 2));
fs.writeFileSync(path.join(OUTPUT_PATH, 'snippetList.json'), JSON.stringify(listingData, null, 2));
// Display messages and time
console.log(`${chalk.green('SUCCESS!')} snippets.json and snippetsArchive.json files generated!`);
console.timeEnd('Extractor');

10
scripts/util.js
View File

@ -2,6 +2,7 @@ const fs = require('fs-extra'),
path = require('path'),
chalk = require('chalk'),
crypto = require('crypto');
const babel = require('@babel/core');
const getMarkDownAnchor = paragraphTitle =>
paragraphTitle
@ -117,7 +118,7 @@ const hashData = val =>
// Gets the code blocks for a snippet file.
const getCodeBlocks = str => {
const regex = /```[.\S\s]*?```/g;
const results = [];
let results = [];
let m = null;
while ((m = regex.exec(str)) !== null) {
if (m.index === regex.lastIndex)
@ -127,7 +128,12 @@ const getCodeBlocks = str => {
results.push(match);
});
}
return results;
results = results.map(v => v.replace(/```js([\s\S]*?)```/g, '$1').trim());
return {
es6: results[0],
es5: babel.transformSync(results[0], { presets: ['@babel/preset-env'] }).code,
example: results[1]
}
};
// Gets the textual content for a snippet file.
const getTextualContent = str => {