Minor refactorings for numeric parsing in tokenizer (#2472)

Author: MaxGraey

src/tokenizer.ts

@@ -31,6 +31,7 @@ import {
   isIdentifierPart,
   isDecimal,
   isOctal,
+  isHexBase,
   isHighSurrogate,
   isLowSurrogate
 } from "./util";
@@ -1313,30 +1314,24 @@ export class Tokenizer extends DiagnosticEmitter {
     var end = this.end;
     var start = pos;
     var sepEnd = start;
-    var value = i64_new(0);
+    var value = i64_zero;
     var i64_4 = i64_new(4);
     var nextValue = value;
     var overflowOccurred = false;
 
     while (pos < end) {
       let c = text.charCodeAt(pos);
-      if (c >= CharCode._0 && c <= CharCode._9) {
-        // value = (value << 4) + c - CharCode._0;
+      if (isDecimal(c)) {
+        // (value << 4) + c - CharCode._0
         nextValue = i64_add(
           i64_shl(value, i64_4),
           i64_new(c - CharCode._0)
         );
-      } else if (c >= CharCode.A && c <= CharCode.F) {
-        // value = (value << 4) + 10 + c - CharCode.A;
+      } else if (isHexBase(c)) {
+        // (value << 4) + (c | 32) + (10 - CharCode.a)
         nextValue = i64_add(
           i64_shl(value, i64_4),
-          i64_new(10 + c - CharCode.A)
-        );
-      } else if (c >= CharCode.a && c <= CharCode.f) {
-        // value = (value << 4) + 10 + c - CharCode.a;
-        nextValue = i64_add(
-          i64_shl(value, i64_4),
-          i64_new(10 + c - CharCode.a)
+          i64_new((c | 32) + (10 - CharCode.a))
         );
       } else if (c == CharCode._) {
         if (sepEnd == pos) {
@@ -1386,14 +1381,14 @@ export class Tokenizer extends DiagnosticEmitter {
     var end = this.end;
     var start = pos;
     var sepEnd = start;
-    var value = i64_new(0);
+    var value = i64_zero;
     var i64_10 = i64_new(10);
     var nextValue = value;
     var overflowOccurred = false;
 
     while (pos < end) {
       let c = text.charCodeAt(pos);
-      if (c >= CharCode._0 && c <= CharCode._9) {
+      if (isDecimal(c)) {
         // value = value * 10 + c - CharCode._0;
         nextValue = i64_add(
           i64_mul(value, i64_10),
@@ -1451,15 +1446,15 @@ export class Tokenizer extends DiagnosticEmitter {
     var end = this.end;
     var start = pos;
     var sepEnd = start;
-    var value = i64_new(0);
+    var value = i64_zero;
     var i64_3 = i64_new(3);
     var nextValue = value;
     var overflowOccurred = false;
 
     while (pos < end) {
       let c = text.charCodeAt(pos);
-      if (c >= CharCode._0 && c <= CharCode._7) {
-        // value = (value << 3) + c - CharCode._0;
+      if (isOctal(c)) {
+        // (value << 3) + c - CharCode._0
         nextValue = i64_add(
           i64_shl(value, i64_3),
           i64_new(c - CharCode._0)
@@ -1511,21 +1506,20 @@ export class Tokenizer extends DiagnosticEmitter {
     var end = this.end;
     var start = pos;
     var sepEnd = start;
-    var value = i64_new(0);
-    var i64_1 = i64_new(1);
+    var value = i64_zero;
     var nextValue = value;
     var overflowOccurred = false;
 
     while (pos < end) {
       let c = text.charCodeAt(pos);
       if (c == CharCode._0) {
-        // value = (value << 1);
-        nextValue = i64_shl(value, i64_1);
+        // value << 1 | 0
+        nextValue = i64_shl(value, i64_one);
       } else if (c == CharCode._1) {
-        // value = (value << 1) + 1;
-        nextValue = i64_add(
-          i64_shl(value, i64_1),
-          i64_1
+        // value << 1 | 1
+        nextValue = i64_or(
+          i64_shl(value, i64_one),
+          i64_one
         );
       } else if (c == CharCode._) {
         if (sepEnd == pos) {
@@ -1665,12 +1659,10 @@ export class Tokenizer extends DiagnosticEmitter {
     var end = this.end;
     while (pos < end) {
       let c = text.charCodeAt(pos++);
-      if (c >= CharCode._0 && c <= CharCode._9) {
+      if (isDecimal(c)) {
         value = (value << 4) + c - CharCode._0;
-      } else if (c >= CharCode.A && c <= CharCode.F) {
-        value = (value << 4) + c + (10 - CharCode.A);
-      } else if (c >= CharCode.a && c <= CharCode.f) {
-        value = (value << 4) + c + (10 - CharCode.a);
+      } else if (isHexBase(c)) {
+        value = (value << 4) + (c | 32) + (10 - CharCode.a);
       } else if (~startIfTaggedTemplate) {
         this.pos = --pos;
         return text.substring(startIfTaggedTemplate, pos);

src/util/text.ts

@@ -234,10 +234,15 @@ export function isOctal(c: i32): bool {
   return c >= CharCode._0 && c <= CharCode._7;
 }
 
-/** Tests if the specified character code is a valid hexadecimal digit. */
-export function isHex(c: i32): bool {
+/** Tests if the specified character code is a valid hexadecimal symbol [a-f]. */
+export function isHexBase(c: i32): bool {
   let c0 = c | 32; // unify uppercases and lowercases a|A - f|F
-  return isDecimal(c) || (c0 >= CharCode.a && c0 <= CharCode.f);
+  return c0 >= CharCode.a && c0 <= CharCode.f;
+}
+
+/** Tests if the specified character code is a valid hexadecimal digit. */
+export function isHexOrDecimal(c: i32): bool {
+  return isDecimal(c) || isHexBase(c);
 }
 
 /** Tests if the specified character code is trivially alphanumeric. */