diff --git a/scripts/analyze.js b/scripts/analyze.js index 1997fc925..70c1666f5 100644 --- a/scripts/analyze.js +++ b/scripts/analyze.js @@ -13,8 +13,8 @@ let snippetsArchiveData = require('../snippet_data/snippetsArchive.json'); const OUTPUT_PATH = './snippet_data'; console.time('Analyzer'); // Read data -let snippetTokens = { - data: snippetsData.data.map(snippet => { +let [snippetTokens, snippetArchiveTokens] = [snippetsData, snippetsArchiveData].map(v => ({ + data: v.data.map(snippet => { let tokens = prism.tokenize( snippet.attributes.codeBlocks[0], prism.languages.javascript, @@ -39,34 +39,7 @@ let snippetTokens = { }; }), meta: { specification: 'http://jsonapi.org/format/' } -}; -let snippetArchiveTokens = { - data: snippetsArchiveData.data.map(snippet => { - let tokens = prism.tokenize( - snippet.attributes.codeBlocks[0], - prism.languages.javascript, - 'javascript' - ); - return { - id: snippet.id, - type: 'snippetAnalysis', - attributes: { - codeLength: snippet.attributes.codeBlocks[0].trim().length, - tokenCount: tokens.length, - functionCount: tokens.filter(t => t.type === 'function').length, - operatorCount: tokens.filter(t => t.type === 'operator').length, - keywordCount: tokens.filter(t => t.type === 'keyword').length, - distinctFunctionCount: [ - ...new Set(tokens.filter(t => t.type === 'function').map(t => t.content)) - ].length - }, - meta: { - hash: snippet.meta.hash - } - }; - }), - meta: { specification: 'http://jsonapi.org/format/' } -}; +})); // Write data fs.writeFileSync( path.join(OUTPUT_PATH, 'snippetAnalytics.json'), diff --git a/snippet_data/snippetArchiveAnalytics.json b/snippet_data/snippetArchiveAnalytics.json index cdd182da3..d7f8678ab 100644 --- a/snippet_data/snippetArchiveAnalytics.json +++ b/snippet_data/snippetArchiveAnalytics.json @@ -132,14 +132,14 @@ "distinctFunctionCount": 4 }, "meta": { - "hash": "552bc0487c1bf8e4ff89a0a439840b32b6688e47c8e9a551bf924f91e856aa5d" + "hash": "0b543697856c3c91e3e0a58d3928d743dbe8a369e7263fbc9806aedf487ff723" } }, { "id": "httpPut", "type": "snippetAnalysis", "attributes": { - "codeLength": 336, + "codeLength": 324, "tokenCount": 97, "functionCount": 5, "operatorCount": 8, @@ -147,7 +147,7 @@ "distinctFunctionCount": 5 }, "meta": { - "hash": "8ef860126cb08a63458f113653502ee2eb53d42e39128459cad1ebc92ed55639" + "hash": "59572ee267c7f0bc7c4e411ebde24834a2f5e8e928e2e50c16ac787034b1a53a" } }, { @@ -169,7 +169,7 @@ "id": "isSimilar", "type": "snippetAnalysis", "attributes": { - "codeLength": 214, + "codeLength": 251, "tokenCount": 78, "functionCount": 3, "operatorCount": 10, @@ -177,7 +177,7 @@ "distinctFunctionCount": 2 }, "meta": { - "hash": "669cc6d3a6f96a65b6e4bf678bb0b95cde711baaabe206db66d69599f6e93a51" + "hash": "570b1b21e29f3793d7c1f001958ec68a7c27b913d65e37cf5f078b1e6a839437" } }, { @@ -199,7 +199,7 @@ "id": "levenshteinDistance", "type": "snippetAnalysis", "attributes": { - "codeLength": 709, + "codeLength": 733, "tokenCount": 3, "functionCount": 0, "operatorCount": 0, @@ -207,7 +207,7 @@ "distinctFunctionCount": 0 }, "meta": { - "hash": "170ef4eafeb5ac639c721f1a8e02d309e1ad62941c613ce60462fa72274f4c25" + "hash": "3cc34a842404de0aea2752a6b1398b8a0825cb1a359ff36ab5275f7d15eff107" } }, { @@ -237,14 +237,14 @@ "distinctFunctionCount": 3 }, "meta": { - "hash": "1f1d7def5d8b149626518181acb9f0e7fd0b44b31e5b42ce1e2f53e07eb01bc0" + "hash": "86af9032bb3fd1afc0b6e32aeca6a25c69549594804ff93f9eb0fe6e9e37236b" } }, { "id": "removeVowels", "type": "snippetAnalysis", "attributes": { - "codeLength": 71, + "codeLength": 72, "tokenCount": 23, "functionCount": 1, "operatorCount": 3, @@ -252,7 +252,7 @@ "distinctFunctionCount": 1 }, "meta": { - "hash": "00040c26975307a1be29e9a933ef09c2b3d92bc986d8798b56339dbefce4d6fa" + "hash": "5f3ccc861fd5a18aa561957aadb6494402c1b3bdc1ec38b768e524c600ac756b" } }, {