feat: Support hexadecimal floating point literals

src/tokenizer.ts

@@ -32,8 +32,10 @@ import {
   isDecimal,
   isOctal,
   isHexBase,
+  isHexOrDecimal,
   isHighSurrogate,
-  isLowSurrogate
+  isLowSurrogate,
+  parseHexFloat
 } from "./util";
 
 /** Named token types. */
@@ -1259,18 +1261,34 @@ export class Tokenizer extends DiagnosticEmitter {
     var text = this.source.text;
     var pos = this.pos;
     var end = this.end;
-    if (pos + 1 < end && text.charCodeAt(pos) == CharCode._0) {
-      switch (text.charCodeAt(pos + 2) | 32) {
-        case CharCode.x:
+    var hex = false;
+    if (pos + 2 < end && text.charCodeAt(pos) == CharCode._0) {
+      switch (text.charCodeAt(pos + 1) | 32) {
+        case CharCode.x: {
+          // Don't early return for CharCode.x
+          // It possible a hexadecimal float.
+          hex = true;
+          pos += 2;
+          break;
+        }
         case CharCode.b:
         case CharCode.o: return true;
+        case CharCode.DOT: return false;
       }
     }
     while (pos < end) {
       let c = text.charCodeAt(pos);
-      if (c == CharCode.DOT || (c | 32) == CharCode.e) return false;
-      if (c != CharCode._ && (c < CharCode._0 || c > CharCode._9)) break;
+      if (c == CharCode.DOT) return false;
       // does not validate separator placement (this is done in readXYInteger)
+      if (c != CharCode._) {
+        if (hex) {
+          if ((c | 32) == CharCode.p) return false;
+          if (!isHexOrDecimal(c)) break;
+        } else {
+          if ((c | 32) == CharCode.e) return false;
+          if (!isDecimal(c)) break;
+        }
+      }
       pos++;
     }
     return true;
@@ -1389,7 +1407,7 @@ export class Tokenizer extends DiagnosticEmitter {
     while (pos < end) {
       let c = text.charCodeAt(pos);
       if (isDecimal(c)) {
-        // value = value * 10 + c - CharCode._0;
+        // value * 10 + c - CharCode._0
         nextValue = i64_add(
           i64_mul(value, i64_10),
           i64_new(c - CharCode._0)
@@ -1563,27 +1581,23 @@ export class Tokenizer extends DiagnosticEmitter {
   }
 
   readFloat(): f64 {
-    // var text = this.source.text;
-    // if (text.charCodeAt(this.pos) == CharCode._0 && this.pos + 2 < this.end) {
-    //   switch (text.charCodeAt(this.pos + 1)) {
-    //     case CharCode.X:
-    //     case CharCode.x: {
-    //       this.pos += 2;
-    //       return this.readHexFloat();
-    //     }
-    //   }
-    // }
+    var text = this.source.text;
+    if (text.charCodeAt(this.pos) == CharCode._0 && this.pos + 2 < this.end) {
+      if ((text.charCodeAt(this.pos + 1) | 32) == CharCode.x) {
+        return this.readHexFloat();
+      }
+    }
     return this.readDecimalFloat();
   }
 
   readDecimalFloat(): f64 {
     var text = this.source.text;
     var end = this.end;
     var start = this.pos;
-    var sepCount = this.readDecimalFloatPartial(false);
+    var sepCount = this.readFloatPartial(false, false);
     if (this.pos < end && text.charCodeAt(this.pos) == CharCode.DOT) {
       ++this.pos;
-      sepCount += this.readDecimalFloatPartial();
+      sepCount += this.readFloatPartial(true, false);
     }
     if (this.pos < end) {
       let c = text.charCodeAt(this.pos);
@@ -1595,7 +1609,7 @@ export class Tokenizer extends DiagnosticEmitter {
         ) {
           ++this.pos;
         }
-        sepCount += this.readDecimalFloatPartial();
+        sepCount += this.readFloatPartial(true, false);
       }
     }
     let result = text.substring(start, this.pos);
@@ -1604,7 +1618,7 @@ export class Tokenizer extends DiagnosticEmitter {
   }
 
   /** Reads past one section of a decimal float literal. Returns the number of separators encountered. */
-  private readDecimalFloatPartial(allowLeadingZeroSep: bool = true): u32 {
+  private readFloatPartial(allowLeadingZeroSep: bool, isHexadecimal: bool): u32 {
     var text = this.source.text;
     var pos = this.pos;
     var start = pos;
@@ -1614,7 +1628,6 @@ export class Tokenizer extends DiagnosticEmitter {
 
     while (pos < end) {
       let c = text.charCodeAt(pos);
-
       if (c == CharCode._) {
         if (sepEnd == pos) {
           this.error(
@@ -1631,8 +1644,12 @@ export class Tokenizer extends DiagnosticEmitter {
         }
         sepEnd = pos + 1;
         ++sepCount;
-      } else if (!isDecimal(c)) {
-        break;
+      } else {
+        if (isHexadecimal) {
+          if (!isHexOrDecimal(c)) break;
+        } else {
+          if (!isDecimal(c)) break;
+        }
       }
       ++pos;
     }
@@ -1649,7 +1666,33 @@ export class Tokenizer extends DiagnosticEmitter {
   }
 
   readHexFloat(): f64 {
-    throw new Error("not implemented"); // TBD
+    var text = this.source.text;
+    var pos = this.pos;
+    var start = pos;
+    var end = this.end;
+
+    this.pos += 2; // skip 0x
+    var sepCount = this.readFloatPartial(false, true);
+    if (this.pos < end && text.charCodeAt(this.pos) == CharCode.DOT) {
+      ++this.pos;
+      sepCount += this.readFloatPartial(true, true);
+    }
+    if (this.pos < end) {
+      let c = text.charCodeAt(this.pos);
+      if ((c | 32) == CharCode.p) {
+        if (
+          ++this.pos < end &&
+          (c = text.charCodeAt(this.pos)) == CharCode.MINUS || c == CharCode.PLUS &&
+          isHexOrDecimal(text.charCodeAt(this.pos + 1))
+        ) {
+          ++this.pos;
+        }
+        sepCount += this.readFloatPartial(true, false);
+      }
+    }
+    let result = text.substring(start, this.pos);
+    if (sepCount) result = result.replaceAll("_", "");
+    return parseHexFloat(result);
   }
 
   readHexadecimalEscape(remain: i32 = 2, startIfTaggedTemplate: i32 = -1): string {

src/util/math.ts

@@ -24,3 +24,38 @@ export function accuratePow64(x: f64, y: f64): f64 {
   }
   return Math.pow(x, y);
 }
+
+// see: https://git.musl-libc.org/cgit/musl/tree/src/math/scalbn.c
+/** Equivalent of `x * (2 ** n)` */
+export function scalbn(x: f64, n: i32): f64 {
+  const
+    Ox1p1023 = 8.98846567431158e+307,    // 0x1p1023
+    Ox1p_969 = 2.00416836000897278e-292; // 0x1p-1022 * 0x1p53
+
+  var y = x;
+  if (n > 1023) {
+    y *= Ox1p1023;
+    n -= 1023;
+    if (n > 1023) {
+      y *= Ox1p1023;
+      n -= 1023;
+      if (n > 1023) n = 1023;
+    }
+  } else if (n < -1022) {
+    // make sure final n < -53 to avoid double
+    // rounding in the subnormal range
+    y *= Ox1p_969;
+    n += 1022 - 53;
+    if (n < -1022) {
+      y *= Ox1p_969;
+      n += 1022 - 53;
+      if (n < -1022) n = -1022;
+    }
+  }
+  if (!ASC_TARGET) { // ASC_TARGET == JS
+    return y * i64_as_f64(i64_new(0, 0x3FF + n << 20));
+  } else {
+    // @ts-ignore
+    return y * reinterpret<f64>(u64(0x3FF + n) << 52);
+  }
+}

src/util/text.ts

@@ -3,6 +3,8 @@
  * @license Apache-2.0
  */
 
+import { scalbn } from "./math";
+
 /** An enum of named character codes. */
 export const enum CharCode {
 
@@ -564,3 +566,33 @@ export function escapeString(str: string, quote: CharCode): string {
   if (i > off) sb.push(str.substring(off, i));
   return sb.join("");
 }
+
+export function parseHexFloat(str: string): f64 {
+  var sign = 1, pPos = -1, dotPos = -1;
+  for (let i = 0, k = str.length; i < k; ++i) {
+    const c = str.charCodeAt(i);
+    if (i == 0 && c == CharCode.MINUS) {
+      sign = -1;
+    } else if ((c | 32) == CharCode.p) {
+      pPos = i;
+    } else if (c == CharCode.DOT) {
+      dotPos = i;
+    }
+  }
+  var mant: f64;
+  var mantissa = ~pPos ? str.substring(0, pPos) : str;
+  if (~dotPos) {
+    const integer  = mantissa.substring(0, dotPos);
+    const fraction = mantissa.substring(dotPos + 1);
+    const intVal   = parseInt(integer, 16);
+    const fracVal  = fraction.length
+      ? scalbn(parseInt(fraction, 16), -(fraction.length << 2))
+      : 0;
+    mant = intVal + sign * fracVal;
+  } else {
+    mant = parseInt(mantissa, 16);
+  }
+  return ~pPos
+    ? scalbn(mant, parseInt(str.substring(pPos + 1)))
+    : mant;
+}