WIP - add extractor, generate snippet_data

node_modules/@webassemblyjs/wast-parser/lib/tokenizer.js

@@ -0,0 +1,447 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.tokenize = tokenize;
+exports.tokens = exports.keywords = void 0;
+
+var _helperFsm = require("@webassemblyjs/helper-fsm");
+
+var _helperCodeFrame = require("@webassemblyjs/helper-code-frame");
+
+// eslint-disable-next-line
+function getCodeFrame(source, line, column) {
+  var loc = {
+    start: {
+      line: line,
+      column: column
+    }
+  };
+  return "\n" + (0, _helperCodeFrame.codeFrameFromSource)(source, loc) + "\n";
+}
+
+var WHITESPACE = /\s/;
+var PARENS = /\(|\)/;
+var LETTERS = /[a-z0-9_/]/i;
+var idchar = /[a-z0-9!#$%&*+./:<=>?@\\[\]^_`|~-]/i;
+var valtypes = ["i32", "i64", "f32", "f64"];
+var NUMBERS = /[0-9|.|_]/;
+var NUMBER_KEYWORDS = /nan|inf/;
+
+function isNewLine(char) {
+  return char.charCodeAt(0) === 10 || char.charCodeAt(0) === 13;
+}
+
+function Token(type, value, start, end) {
+  var opts = arguments.length > 4 && arguments[4] !== undefined ? arguments[4] : {};
+  var token = {
+    type: type,
+    value: value,
+    loc: {
+      start: start,
+      end: end
+    }
+  };
+
+  if (Object.keys(opts).length > 0) {
+    // $FlowIgnore
+    token["opts"] = opts;
+  }
+
+  return token;
+}
+
+var tokenTypes = {
+  openParen: "openParen",
+  closeParen: "closeParen",
+  number: "number",
+  string: "string",
+  name: "name",
+  identifier: "identifier",
+  valtype: "valtype",
+  dot: "dot",
+  comment: "comment",
+  equal: "equal",
+  keyword: "keyword"
+};
+var keywords = {
+  module: "module",
+  func: "func",
+  param: "param",
+  result: "result",
+  export: "export",
+  loop: "loop",
+  block: "block",
+  if: "if",
+  then: "then",
+  else: "else",
+  call: "call",
+  call_indirect: "call_indirect",
+  import: "import",
+  memory: "memory",
+  table: "table",
+  global: "global",
+  anyfunc: "anyfunc",
+  mut: "mut",
+  data: "data",
+  type: "type",
+  elem: "elem",
+  start: "start",
+  offset: "offset"
+};
+exports.keywords = keywords;
+var NUMERIC_SEPARATOR = "_";
+/**
+ * Build the FSM for number literals
+ */
+
+var numberLiteralFSM = new _helperFsm.FSM({
+  START: [(0, _helperFsm.makeTransition)(/-|\+/, "AFTER_SIGN"), (0, _helperFsm.makeTransition)(/nan:0x/, "NAN_HEX", {
+    n: 6
+  }), (0, _helperFsm.makeTransition)(/nan|inf/, "STOP", {
+    n: 3
+  }), (0, _helperFsm.makeTransition)(/0x/, "HEX", {
+    n: 2
+  }), (0, _helperFsm.makeTransition)(/[0-9]/, "DEC"), (0, _helperFsm.makeTransition)(/\./, "DEC_FRAC")],
+  AFTER_SIGN: [(0, _helperFsm.makeTransition)(/nan:0x/, "NAN_HEX", {
+    n: 6
+  }), (0, _helperFsm.makeTransition)(/nan|inf/, "STOP", {
+    n: 3
+  }), (0, _helperFsm.makeTransition)(/0x/, "HEX", {
+    n: 2
+  }), (0, _helperFsm.makeTransition)(/[0-9]/, "DEC"), (0, _helperFsm.makeTransition)(/\./, "DEC_FRAC")],
+  DEC_FRAC: [(0, _helperFsm.makeTransition)(/[0-9]/, "DEC_FRAC", {
+    allowedSeparator: NUMERIC_SEPARATOR
+  }), (0, _helperFsm.makeTransition)(/e|E/, "DEC_SIGNED_EXP")],
+  DEC: [(0, _helperFsm.makeTransition)(/[0-9]/, "DEC", {
+    allowedSeparator: NUMERIC_SEPARATOR
+  }), (0, _helperFsm.makeTransition)(/\./, "DEC_FRAC"), (0, _helperFsm.makeTransition)(/e|E/, "DEC_SIGNED_EXP")],
+  DEC_SIGNED_EXP: [(0, _helperFsm.makeTransition)(/\+|-/, "DEC_EXP"), (0, _helperFsm.makeTransition)(/[0-9]/, "DEC_EXP")],
+  DEC_EXP: [(0, _helperFsm.makeTransition)(/[0-9]/, "DEC_EXP", {
+    allowedSeparator: NUMERIC_SEPARATOR
+  })],
+  HEX: [(0, _helperFsm.makeTransition)(/[0-9|A-F|a-f]/, "HEX", {
+    allowedSeparator: NUMERIC_SEPARATOR
+  }), (0, _helperFsm.makeTransition)(/\./, "HEX_FRAC"), (0, _helperFsm.makeTransition)(/p|P/, "HEX_SIGNED_EXP")],
+  HEX_FRAC: [(0, _helperFsm.makeTransition)(/[0-9|A-F|a-f]/, "HEX_FRAC", {
+    allowedSeparator: NUMERIC_SEPARATOR
+  }), (0, _helperFsm.makeTransition)(/p|P|/, "HEX_SIGNED_EXP")],
+  HEX_SIGNED_EXP: [(0, _helperFsm.makeTransition)(/[0-9|+|-]/, "HEX_EXP")],
+  HEX_EXP: [(0, _helperFsm.makeTransition)(/[0-9]/, "HEX_EXP", {
+    allowedSeparator: NUMERIC_SEPARATOR
+  })],
+  NAN_HEX: [(0, _helperFsm.makeTransition)(/[0-9|A-F|a-f]/, "NAN_HEX", {
+    allowedSeparator: NUMERIC_SEPARATOR
+  })],
+  STOP: []
+}, "START", "STOP");
+
+function tokenize(input) {
+  var current = 0;
+  var char = input[current]; // Used by SourceLocation
+
+  var column = 1;
+  var line = 1;
+  var tokens = [];
+  /**
+   * Creates a pushToken function for a given type
+   */
+
+  function pushToken(type) {
+    return function (v) {
+      var opts = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
+      var startColumn = opts.startColumn || column - String(v).length;
+      delete opts.startColumn;
+      var endColumn = opts.endColumn || startColumn + String(v).length - 1;
+      delete opts.endColumn;
+      var start = {
+        line: line,
+        column: startColumn
+      };
+      var end = {
+        line: line,
+        column: endColumn
+      };
+      tokens.push(Token(type, v, start, end, opts));
+    };
+  }
+  /**
+   * Functions to save newly encountered tokens
+   */
+
+
+  var pushCloseParenToken = pushToken(tokenTypes.closeParen);
+  var pushOpenParenToken = pushToken(tokenTypes.openParen);
+  var pushNumberToken = pushToken(tokenTypes.number);
+  var pushValtypeToken = pushToken(tokenTypes.valtype);
+  var pushNameToken = pushToken(tokenTypes.name);
+  var pushIdentifierToken = pushToken(tokenTypes.identifier);
+  var pushKeywordToken = pushToken(tokenTypes.keyword);
+  var pushDotToken = pushToken(tokenTypes.dot);
+  var pushStringToken = pushToken(tokenTypes.string);
+  var pushCommentToken = pushToken(tokenTypes.comment);
+  var pushEqualToken = pushToken(tokenTypes.equal);
+  /**
+   * Can be used to look at the next character(s).
+   *
+   * The default behavior `lookahead()` simply returns the next character without consuming it.
+   * Letters are always returned in lowercase.
+   *
+   * @param {number} length How many characters to query. Default = 1
+   * @param {number} offset How many characters to skip forward from current one. Default = 1
+   *
+   */
+
+  function lookahead() {
+    var length = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 1;
+    var offset = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 1;
+    return input.substring(current + offset, current + offset + length).toLowerCase();
+  }
+  /**
+   * Advances the cursor in the input by a certain amount
+   *
+   * @param {number} amount How many characters to consume. Default = 1
+   */
+
+
+  function eatCharacter() {
+    var amount = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 1;
+    column += amount;
+    current += amount;
+    char = input[current];
+  }
+
+  while (current < input.length) {
+    // ;;
+    if (char === ";" && lookahead() === ";") {
+      var startColumn = column;
+      eatCharacter(2);
+      var text = "";
+
+      while (!isNewLine(char)) {
+        text += char;
+        eatCharacter();
+
+        if (char === undefined) {
+          break;
+        }
+      }
+
+      var endColumn = column;
+      pushCommentToken(text, {
+        type: "leading",
+        startColumn: startColumn,
+        endColumn: endColumn
+      });
+      continue;
+    } // (;
+
+
+    if (char === "(" && lookahead() === ";") {
+      var _startColumn = column;
+      eatCharacter(2);
+      var _text = ""; // ;)
+
+      while (true) {
+        char = input[current];
+
+        if (char === ";" && lookahead() === ")") {
+          eatCharacter(2);
+          break;
+        }
+
+        _text += char;
+        eatCharacter();
+
+        if (isNewLine(char)) {
+          line++;
+          column = 0;
+        }
+      }
+
+      var _endColumn = column;
+      pushCommentToken(_text, {
+        type: "block",
+        startColumn: _startColumn,
+        endColumn: _endColumn
+      });
+      continue;
+    }
+
+    if (char === "(") {
+      pushOpenParenToken(char);
+      eatCharacter();
+      continue;
+    }
+
+    if (char === "=") {
+      pushEqualToken(char);
+      eatCharacter();
+      continue;
+    }
+
+    if (char === ")") {
+      pushCloseParenToken(char);
+      eatCharacter();
+      continue;
+    }
+
+    if (isNewLine(char)) {
+      line++;
+      eatCharacter();
+      column = 0;
+      continue;
+    }
+
+    if (WHITESPACE.test(char)) {
+      eatCharacter();
+      continue;
+    }
+
+    if (char === "$") {
+      var _startColumn2 = column;
+      eatCharacter();
+      var value = "";
+
+      while (idchar.test(char)) {
+        value += char;
+        eatCharacter();
+      }
+
+      var _endColumn2 = column;
+      pushIdentifierToken(value, {
+        startColumn: _startColumn2,
+        endColumn: _endColumn2
+      });
+      continue;
+    }
+
+    if (NUMBERS.test(char) || NUMBER_KEYWORDS.test(lookahead(3, 0)) || char === "-" || char === "+") {
+      var _startColumn3 = column;
+
+      var _value = numberLiteralFSM.run(input.slice(current));
+
+      if (_value === "") {
+        throw new Error(getCodeFrame(input, line, column) + "Unexpected character " + JSON.stringify(char));
+      }
+
+      pushNumberToken(_value, {
+        startColumn: _startColumn3
+      });
+      eatCharacter(_value.length);
+
+      if (char && !PARENS.test(char) && !WHITESPACE.test(char)) {
+        throw new Error(getCodeFrame(input, line, column) + "Unexpected character " + JSON.stringify(char));
+      }
+
+      continue;
+    }
+
+    if (char === '"') {
+      var _startColumn4 = column;
+      var _value2 = "";
+      eatCharacter(); // "
+
+      while (char !== '"') {
+        if (isNewLine(char)) {
+          throw new Error(getCodeFrame(input, line, column) + "Unexpected character " + JSON.stringify(char));
+        }
+
+        _value2 += char;
+        eatCharacter(); // char
+      }
+
+      eatCharacter(); // "
+
+      var _endColumn3 = column;
+      pushStringToken(_value2, {
+        startColumn: _startColumn4,
+        endColumn: _endColumn3
+      });
+      continue;
+    }
+
+    if (LETTERS.test(char)) {
+      var _value3 = "";
+      var _startColumn5 = column;
+
+      while (char && LETTERS.test(char)) {
+        _value3 += char;
+        eatCharacter();
+      }
+      /*
+       * Handle MemberAccess
+       */
+
+
+      if (char === ".") {
+        var dotStartColumn = column;
+
+        if (valtypes.indexOf(_value3) !== -1) {
+          pushValtypeToken(_value3, {
+            startColumn: _startColumn5
+          });
+        } else {
+          pushNameToken(_value3);
+        }
+
+        eatCharacter();
+        _value3 = "";
+        var nameStartColumn = column;
+
+        while (LETTERS.test(char)) {
+          _value3 += char;
+          eatCharacter();
+        }
+
+        pushDotToken(".", {
+          startColumn: dotStartColumn
+        });
+        pushNameToken(_value3, {
+          startColumn: nameStartColumn
+        });
+        continue;
+      }
+      /*
+       * Handle keywords
+       */
+      // $FlowIgnore
+
+
+      if (typeof keywords[_value3] === "string") {
+        pushKeywordToken(_value3, {
+          startColumn: _startColumn5
+        });
+        continue;
+      }
+      /*
+       * Handle types
+       */
+
+
+      if (valtypes.indexOf(_value3) !== -1) {
+        pushValtypeToken(_value3, {
+          startColumn: _startColumn5
+        });
+        continue;
+      }
+      /*
+       * Handle literals
+       */
+
+
+      pushNameToken(_value3, {
+        startColumn: _startColumn5
+      });
+      continue;
+    }
+
+    throw new Error(getCodeFrame(input, line, column) + "Unexpected character " + JSON.stringify(char));
+  }
+
+  return tokens;
+}
+
+var tokens = tokenTypes;
+exports.tokens = tokens;