JS Code from Building a Basic Interpreter, '80s Style Part 2

tokenizer.js

const tokens = ["CLS", "PRINT", ":" ];

function tokenize(inStr) {
    let out = [];                        // return value is "crunched" array
    let idx = 0;                         // index into inStr
    let ch = "";                         // current character
    while (idx < inStr.length) {
        ch = inStr.charCodeAt(idx);      // get the current character code
        if (ch >= 48 && ch <= 57) {      // NUMERIC LITERAL
            out.push (0xFD);
            const numberLiteralStr = inStr.substr(idx).match(/^[\d|A-F|a-f|x|X]+/)[0];
            idx += numberLiteralStr.length;
            const numberLiteral = Number(numberLiteralStr);
            const bytes = new Uint8Array(new Uint16Array([numberLiteral]).buffer);
            out.push(...bytes)
            continue;
        }
        if (ch === 34) {                 // STRING LITERAL
            out.push (0xFB);             // string-in-code token
            const stringLiteral = inStr.substr(idx+1).match(/^[^"]*/)[0];
            idx += stringLiteral.length+1;
            out.push(...Array.from(stringLiteral, ch => ch.charCodeAt(0)));
            idx++;                       // go past the quote
            out.push(0);                 // end of string
            continue;
        }
        if (ch === 32) {                 // collapse SPACEs
            const allTheSpaces = inStr.substr(idx).match(/^\ +/)[0];
            idx += allTheSpaces.length;
            out.push(ch);
            continue;
        }
        const tokenMatch = tokens.findIndex(token => inStr.substr(idx).startsWith(token));
        if (tokenMatch > -1) {
            out.push(tokenMatch + 128);
            idx += tokens[tokenMatch].length;
            continue;
        }
        const variableMatch = inStr.substr(idx).match(/^[A-Z]+[A-Z0-9]*[\$]*/); console.log(variableMatch);
        if (variableMatch) {
            out.push(0xFA);
            const variableName = variableMatch[0];
            const variableIndexName = variableName.substr(0,2);
            const variableType = variableName.endsWith("$") ? "string" : "number";
            let variableIndex; 
            if (variableIndexName.length === 1) {
                variableIndex = variableIndexName.charCodeAt(0) - 65;
            } else {
                variableIndex = 27 + (variableIndexName.charCodeAt(0) - 65) * 36 + variableIndexName.charCodeAt(1);
            }
            variableIndex += {"number": 0b0000_0000_0000_0000, "string": 0b1000_0000_0000_0000}[variableType];
            const hiByte = (variableIndex & 0xFF00) >> 8;
            const loByte = (variableIndex & 0x00FF);
            out.push(hiByte, loByte);
            out.push(variableName.length);
            out.push(...Array.from(variableName, ch => ch.charCodeAt(0)));
            idx += variableName.length;
            continue;
        }
        
        out.push(ch);
        idx++;    
    }
    out.push(0);                         // NUL
    return out;
}

const toHex = arr => arr.map(val => val.toString(16).toUpperCase().padStart(2,"0")).join(" ");

console.log(toHex(tokenize(`PRINT ZZ    96 0xFF "Hi"`)))
console.log(toHex(tokenize(`CLS: PRINT "Hello, World"`)))