var charCodeDef = require('./char-code-definitions'); var isDigit = charCodeDef.isDigit; var isHexDigit = charCodeDef.isHexDigit; var isUppercaseLetter = charCodeDef.isUppercaseLetter; var isLowercaseLetter = charCodeDef.isLowercaseLetter; var isLetter = charCodeDef.isLetter; var isNonAscii = charCodeDef.isNonAscii; var isNameStart = charCodeDef.isNameStart; var isName = charCodeDef.isName; var isNonPrintable = charCodeDef.isNonPrintable; var isNewline = charCodeDef.isNewline; var isWhiteSpace = charCodeDef.isWhiteSpace; var isValidEscape = charCodeDef.isValidEscape; var isIdentifierStart = charCodeDef.isIdentifierStart; var isNumberStart = charCodeDef.isNumberStart; // detect BOM (https://en.wikipedia.org/wiki/Byte_order_mark) function firstCharOffset(source) { if (source.length > 0) { if (source.charCodeAt(0) === 0xFEFF || // UTF-16BE source.charCodeAt(0) === 0xFFFE) { // UTF-16LE return 1; } } return 0; } function getCharCode(source, offset) { return offset < source.length ? source.charCodeAt(offset) : 0; } function getNewlineLength(source, offset, code) { if (code === 13 /* \r */ && getCharCode(source, offset + 1) === 10 /* \n */) { return 2; } return 1; } function cmpChar(testStr, offset, referenceCode) { var code = testStr.charCodeAt(offset); // code.toLowerCase() for A..Z if (isUppercaseLetter(code)) { code = code | 32; } return code === referenceCode; } function cmpStr(testStr, start, end, referenceStr) { if (end - start !== referenceStr.length) { return false; } if (start < 0 || end > testStr.length) { return false; } for (var i = start; i < end; i++) { var testCode = testStr.charCodeAt(i); var referenceCode = referenceStr.charCodeAt(i - start); // testCode.toLowerCase() for A..Z if (isUppercaseLetter(testCode)) { testCode = testCode | 32; } if (testCode !== referenceCode) { return false; } } return true; } function findWhiteSpaceStart(source, offset) { for (; offset >= 0; offset--) { if (!isWhiteSpace(source.charCodeAt(offset))) { break; } } return offset + 1; } function findWhiteSpaceEnd(source, offset) { for (; offset < source.length; offset++) { if (!isWhiteSpace(source.charCodeAt(offset))) { break; } } return offset; } function findDecimalNumberEnd(source, offset) { for (; offset < source.length; offset++) { if (!isDigit(source.charCodeAt(offset))) { break; } } return offset; } // § 4.3.7. Consume an escaped code point function consumeEscaped(source, offset) { // It assumes that the U+005C REVERSE SOLIDUS (\) has already been consumed and // that the next input code point has already been verified to be part of a valid escape. offset += 2; // hex digit if (isHexDigit(getCharCode(source, offset - 1))) { // Consume as many hex digits as possible, but no more than 5. // Note that this means 1-6 hex digits have been consumed in total. for (var maxOffset = Math.min(source.length, offset + 5); offset < maxOffset; offset++) { if (!isHexDigit(getCharCode(source, offset))) { break; } } // If the next input code point is whitespace, consume it as well. var code = getCharCode(source, offset); if (isWhiteSpace(code)) { offset += getNewlineLength(source, offset, code); } } return offset; } // §4.3.11. Consume a name // Note: This algorithm does not do the verification of the first few code points that are necessary // to ensure the returned code points would constitute an . If that is the intended use, // ensure that the stream starts with an identifier before calling this algorithm. function consumeName(source, offset) { // Let result initially be an empty string. // Repeatedly consume the next input code point from the stream: for (; offset < source.length; offset++) { var code = source.charCodeAt(offset); // name code point if (isName(code)) { // Append the code point to result. continue; } // the stream starts with a valid escape if (isValidEscape(code, getCharCode(source, offset + 1))) { // Consume an escaped code point. Append the returned code point to result. offset = consumeEscaped(source, offset) - 1; continue; } // anything else // Reconsume the current input code point. Return result. break; } return offset; } // §4.3.12. Consume a number function consumeNumber(source, offset) { var code = source.charCodeAt(offset); // 2. If the next input code point is U+002B PLUS SIGN (+) or U+002D HYPHEN-MINUS (-), // consume it and append it to repr. if (code === 0x002B || code === 0x002D) { code = source.charCodeAt(offset += 1); } // 3. While the next input code point is a digit, consume it and append it to repr. if (isDigit(code)) { offset = findDecimalNumberEnd(source, offset + 1); code = source.charCodeAt(offset); } // 4. If the next 2 input code points are U+002E FULL STOP (.) followed by a digit, then: if (code === 0x002E && isDigit(source.charCodeAt(offset + 1))) { // 4.1 Consume them. // 4.2 Append them to repr. code = source.charCodeAt(offset += 2); // 4.3 Set type to "number". // TODO // 4.4 While the next input code point is a digit, consume it and append it to repr. offset = findDecimalNumberEnd(source, offset); } // 5. If the next 2 or 3 input code points are U+0045 LATIN CAPITAL LETTER E (E) // or U+0065 LATIN SMALL LETTER E (e), ... , followed by a digit, then: if (cmpChar(source, offset, 101 /* e */)) { var sign = 0; code = source.charCodeAt(offset + 1); // ... optionally followed by U+002D HYPHEN-MINUS (-) or U+002B PLUS SIGN (+) ... if (code === 0x002D || code === 0x002B) { sign = 1; code = source.charCodeAt(offset + 2); } // ... followed by a digit if (isDigit(code)) { // 5.1 Consume them. // 5.2 Append them to repr. // 5.3 Set type to "number". // TODO // 5.4 While the next input code point is a digit, consume it and append it to repr. offset = findDecimalNumberEnd(source, offset + 1 + sign + 1); } } return offset; } // § 4.3.14. Consume the remnants of a bad url // ... its sole use is to consume enough of the input stream to reach a recovery point // where normal tokenizing can resume. function consumeBadUrlRemnants(source, offset) { // Repeatedly consume the next input code point from the stream: for (; offset < source.length; offset++) { var code = source.charCodeAt(offset); // U+0029 RIGHT PARENTHESIS ()) // EOF if (code === 0x0029) { // Return. offset++; break; } if (isValidEscape(code, getCharCode(source, offset + 1))) { // Consume an escaped code point. // Note: This allows an escaped right parenthesis ("\)") to be encountered // without ending the . This is otherwise identical to // the "anything else" clause. offset = consumeEscaped(source, offset); } } return offset; } module.exports = { firstCharOffset: firstCharOffset, isDigit: isDigit, isHexDigit: isHexDigit, isUppercaseLetter: isUppercaseLetter, isLowercaseLetter: isLowercaseLetter, isLetter: isLetter, isNonAscii: isNonAscii, isNameStart: isNameStart, isName: isName, isNonPrintable: isNonPrintable, isNewline: isNewline, isWhiteSpace: isWhiteSpace, isValidEscape: isValidEscape, isIdentifierStart: isIdentifierStart, isNumberStart: isNumberStart, consumeEscaped: consumeEscaped, consumeName: consumeName, consumeNumber: consumeNumber, consumeBadUrlRemnants: consumeBadUrlRemnants, cmpChar: cmpChar, cmpStr: cmpStr, getNewlineLength: getNewlineLength, findWhiteSpaceStart: findWhiteSpaceStart, findWhiteSpaceEnd: findWhiteSpaceEnd };