Analyze script cleanup

scripts/analyze.js

@@ -13,8 +13,8 @@ let snippetsArchiveData = require('../snippet_data/snippetsArchive.json');
 const OUTPUT_PATH = './snippet_data';
 console.time('Analyzer');
 // Read data
-let snippetTokens = {
-  data: snippetsData.data.map(snippet => {
+let [snippetTokens, snippetArchiveTokens] = [snippetsData, snippetsArchiveData].map(v => ({
+  data: v.data.map(snippet => {
     let tokens = prism.tokenize(
       snippet.attributes.codeBlocks[0],
       prism.languages.javascript,
@@ -39,34 +39,7 @@ let snippetTokens = {
     };
   }),
   meta: { specification: 'http://jsonapi.org/format/' }
-};
-let snippetArchiveTokens = {
-  data: snippetsArchiveData.data.map(snippet => {
-    let tokens = prism.tokenize(
-      snippet.attributes.codeBlocks[0],
-      prism.languages.javascript,
-      'javascript'
-    );
-    return {
-      id: snippet.id,
-      type: 'snippetAnalysis',
-      attributes: {
-        codeLength: snippet.attributes.codeBlocks[0].trim().length,
-        tokenCount: tokens.length,
-        functionCount: tokens.filter(t => t.type === 'function').length,
-        operatorCount: tokens.filter(t => t.type === 'operator').length,
-        keywordCount: tokens.filter(t => t.type === 'keyword').length,
-        distinctFunctionCount: [
-          ...new Set(tokens.filter(t => t.type === 'function').map(t => t.content))
-        ].length
-      },
-      meta: {
-        hash: snippet.meta.hash
-      }
-    };
-  }),
-  meta: { specification: 'http://jsonapi.org/format/' }
-};
+}));
 // Write data
 fs.writeFileSync(
   path.join(OUTPUT_PATH, 'snippetAnalytics.json'),

snippet_data/snippetArchiveAnalytics.json

@@ -132,14 +132,14 @@
         "distinctFunctionCount": 4
       },
       "meta": {
-        "hash": "552bc0487c1bf8e4ff89a0a439840b32b6688e47c8e9a551bf924f91e856aa5d"
+        "hash": "0b543697856c3c91e3e0a58d3928d743dbe8a369e7263fbc9806aedf487ff723"
       }
     },
     {
       "id": "httpPut",
       "type": "snippetAnalysis",
       "attributes": {
-        "codeLength": 336,
+        "codeLength": 324,
         "tokenCount": 97,
         "functionCount": 5,
         "operatorCount": 8,
@@ -147,7 +147,7 @@
         "distinctFunctionCount": 5
       },
       "meta": {
-        "hash": "8ef860126cb08a63458f113653502ee2eb53d42e39128459cad1ebc92ed55639"
+        "hash": "59572ee267c7f0bc7c4e411ebde24834a2f5e8e928e2e50c16ac787034b1a53a"
       }
     },
     {
@@ -169,7 +169,7 @@
       "id": "isSimilar",
       "type": "snippetAnalysis",
       "attributes": {
-        "codeLength": 214,
+        "codeLength": 251,
         "tokenCount": 78,
         "functionCount": 3,
         "operatorCount": 10,
@@ -177,7 +177,7 @@
         "distinctFunctionCount": 2
       },
       "meta": {
-        "hash": "669cc6d3a6f96a65b6e4bf678bb0b95cde711baaabe206db66d69599f6e93a51"
+        "hash": "570b1b21e29f3793d7c1f001958ec68a7c27b913d65e37cf5f078b1e6a839437"
       }
     },
     {
@@ -199,7 +199,7 @@
       "id": "levenshteinDistance",
       "type": "snippetAnalysis",
       "attributes": {
-        "codeLength": 709,
+        "codeLength": 733,
         "tokenCount": 3,
         "functionCount": 0,
         "operatorCount": 0,
@@ -207,7 +207,7 @@
         "distinctFunctionCount": 0
       },
       "meta": {
-        "hash": "170ef4eafeb5ac639c721f1a8e02d309e1ad62941c613ce60462fa72274f4c25"
+        "hash": "3cc34a842404de0aea2752a6b1398b8a0825cb1a359ff36ab5275f7d15eff107"
       }
     },
     {
@@ -237,14 +237,14 @@
         "distinctFunctionCount": 3
       },
       "meta": {
-        "hash": "1f1d7def5d8b149626518181acb9f0e7fd0b44b31e5b42ce1e2f53e07eb01bc0"
+        "hash": "86af9032bb3fd1afc0b6e32aeca6a25c69549594804ff93f9eb0fe6e9e37236b"
       }
     },
     {
       "id": "removeVowels",
       "type": "snippetAnalysis",
       "attributes": {
-        "codeLength": 71,
+        "codeLength": 72,
         "tokenCount": 23,
         "functionCount": 1,
         "operatorCount": 3,
@@ -252,7 +252,7 @@
         "distinctFunctionCount": 1
       },
       "meta": {
-        "hash": "00040c26975307a1be29e9a933ef09c2b3d92bc986d8798b56339dbefce4d6fa"
+        "hash": "5f3ccc861fd5a18aa561957aadb6494402c1b3bdc1ec38b768e524c600ac756b"
       }
     },
     {