public static final int BracketKinds = 3;
public static final boolean DEBUG = false;
+
public Scanner() {
this(false, false);
}
* permissible as the first character in a PHP identifier
*/
public static boolean isPHPIdentifierStart(char ch) {
- return Character.isLetter(ch) || (ch == '_');
+ return Character.isLetter(ch)
+ || (ch == '_')
+ || (0x7F <= ch && ch <= 0xFF);
}
/**
* other than the first character
*/
public static boolean isPHPIdentifierPart(char ch) {
- return Character.isLetterOrDigit(ch) || (ch == '_');
+ return Character.isLetterOrDigit(ch)
+ || (ch == '_')
+ || (0x7F <= ch && ch <= 0xFF);
}
public final boolean atEnd() {
//return the token REAL source (aka unicodes are precomputed)
char[] result;
- if (withoutUnicodePtr != 0)
- //0 is used as a fast test flag so the real first char is in position 1
- System.arraycopy(
- withoutUnicodeBuffer,
- 1,
- result = new char[withoutUnicodePtr],
- 0,
- withoutUnicodePtr);
- else {
+// if (withoutUnicodePtr != 0)
+// //0 is used as a fast test flag so the real first char is in position 1
+// System.arraycopy(
+// withoutUnicodeBuffer,
+// 1,
+// result = new char[withoutUnicodePtr],
+// 0,
+// withoutUnicodePtr);
+// else {
int length = currentPosition - startPosition;
switch (length) { // see OptimizedLength
case 1 :
result = new char[length],
0,
length);
- }
+ // }
return result;
}
public int getCurrentTokenEndPosition() {
// Return the token REAL source (aka unicodes are precomputed)
char[] result;
- if (withoutUnicodePtr != 0)
- // 0 is used as a fast test flag so the real first char is in position 1
- System.arraycopy(
- withoutUnicodeBuffer,
- 1,
- result = new char[withoutUnicodePtr],
- 0,
- withoutUnicodePtr);
- else {
+// if (withoutUnicodePtr != 0)
+// // 0 is used as a fast test flag so the real first char is in position 1
+// System.arraycopy(
+// withoutUnicodeBuffer,
+// 1,
+// result = new char[withoutUnicodePtr],
+// 0,
+// withoutUnicodePtr);
+// else {
int length;
System.arraycopy(
source,
result = new char[length = currentPosition - startPosition],
0,
length);
- }
+// }
return result;
}
// Return the token REAL source (aka unicodes are precomputed)
char[] result;
- if (withoutUnicodePtr != 0)
- // 0 is used as a fast test flag so the real first char is in position 1
- System.arraycopy(
- withoutUnicodeBuffer,
- 1,
- result = new char[withoutUnicodePtr],
- 0,
- withoutUnicodePtr);
- else {
+// if (withoutUnicodePtr != 0)
+// // 0 is used as a fast test flag so the real first char is in position 1
+// System.arraycopy(
+// withoutUnicodeBuffer,
+// 1,
+// result = new char[withoutUnicodePtr],
+// 0,
+// withoutUnicodePtr);
+// else {
int length;
System.arraycopy(
source,
result = new char[length = currentPosition - startPos],
0,
length);
- }
+ // }
return result;
}
int temp = currentPosition;
try {
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- //-------------unicode traitement ------------
- int c1, c2, c3, c4;
- int unicodeSize = 6;
- currentPosition++;
- while (source[currentPosition] == 'u') {
- currentPosition++;
- unicodeSize++;
- }
-
- if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
- || c1 < 0)
- || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15
- || c2 < 0)
- || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15
- || c3 < 0)
- || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15
- || c4 < 0)) {
- currentPosition = temp;
- return false;
- }
-
- currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
- if (currentCharacter != testedChar) {
- currentPosition = temp;
- return false;
- }
- unicodeAsBackSlash = currentCharacter == '\\';
-
- //need the unicode buffer
- if (withoutUnicodePtr == 0) {
- //buffer all the entries that have been left aside....
- withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
- System.arraycopy(
- source,
- startPosition,
- withoutUnicodeBuffer,
- 1,
- withoutUnicodePtr);
- }
- //fill the buffer with the char
- withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
- return true;
-
- } //-------------end unicode traitement--------------
- else {
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// //-------------unicode traitement ------------
+// int c1, c2, c3, c4;
+// int unicodeSize = 6;
+// currentPosition++;
+// while (source[currentPosition] == 'u') {
+// currentPosition++;
+// unicodeSize++;
+// }
+//
+// if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c1 < 0)
+// || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c2 < 0)
+// || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c3 < 0)
+// || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c4 < 0)) {
+// currentPosition = temp;
+// return false;
+// }
+//
+// currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
+// if (currentCharacter != testedChar) {
+// currentPosition = temp;
+// return false;
+// }
+// unicodeAsBackSlash = currentCharacter == '\\';
+//
+// //need the unicode buffer
+// if (withoutUnicodePtr == 0) {
+// //buffer all the entries that have been left aside....
+// withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
+// System.arraycopy(
+// source,
+// startPosition,
+// withoutUnicodeBuffer,
+// 1,
+// withoutUnicodePtr);
+// }
+// //fill the buffer with the char
+// withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+// return true;
+//
+// } //-------------end unicode traitement--------------
+// else {
if (currentCharacter != testedChar) {
currentPosition = temp;
return false;
}
unicodeAsBackSlash = false;
- if (withoutUnicodePtr != 0)
- withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+// if (withoutUnicodePtr != 0)
+// withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
return true;
- }
+// }
} catch (IndexOutOfBoundsException e) {
unicodeAsBackSlash = false;
currentPosition = temp;
int temp = currentPosition;
try {
int result;
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- //-------------unicode traitement ------------
- int c1, c2, c3, c4;
- int unicodeSize = 6;
- currentPosition++;
- while (source[currentPosition] == 'u') {
- currentPosition++;
- unicodeSize++;
- }
-
- if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
- || c1 < 0)
- || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15
- || c2 < 0)
- || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15
- || c3 < 0)
- || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15
- || c4 < 0)) {
- currentPosition = temp;
- return 2;
- }
-
- currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
- if (currentCharacter == testedChar1)
- result = 0;
- else if (currentCharacter == testedChar2)
- result = 1;
- else {
- currentPosition = temp;
- return -1;
- }
-
- //need the unicode buffer
- if (withoutUnicodePtr == 0) {
- //buffer all the entries that have been left aside....
- withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
- System.arraycopy(
- source,
- startPosition,
- withoutUnicodeBuffer,
- 1,
- withoutUnicodePtr);
- }
- //fill the buffer with the char
- withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
- return result;
- } //-------------end unicode traitement--------------
- else {
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// //-------------unicode traitement ------------
+// int c1, c2, c3, c4;
+// int unicodeSize = 6;
+// currentPosition++;
+// while (source[currentPosition] == 'u') {
+// currentPosition++;
+// unicodeSize++;
+// }
+//
+// if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c1 < 0)
+// || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c2 < 0)
+// || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c3 < 0)
+// || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c4 < 0)) {
+// currentPosition = temp;
+// return 2;
+// }
+//
+// currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
+// if (currentCharacter == testedChar1)
+// result = 0;
+// else if (currentCharacter == testedChar2)
+// result = 1;
+// else {
+// currentPosition = temp;
+// return -1;
+// }
+//
+// //need the unicode buffer
+// if (withoutUnicodePtr == 0) {
+// //buffer all the entries that have been left aside....
+// withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
+// System.arraycopy(
+// source,
+// startPosition,
+// withoutUnicodeBuffer,
+// 1,
+// withoutUnicodePtr);
+// }
+// //fill the buffer with the char
+// withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+// return result;
+// } //-------------end unicode traitement--------------
+// else {
if (currentCharacter == testedChar1)
result = 0;
else if (currentCharacter == testedChar2)
return -1;
}
- if (withoutUnicodePtr != 0)
- withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+// if (withoutUnicodePtr != 0)
+// withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
return result;
- }
+ // }
} catch (IndexOutOfBoundsException e) {
currentPosition = temp;
return -1;
int temp = currentPosition;
try {
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- //-------------unicode traitement ------------
- int c1, c2, c3, c4;
- int unicodeSize = 6;
- currentPosition++;
- while (source[currentPosition] == 'u') {
- currentPosition++;
- unicodeSize++;
- }
-
- if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
- || c1 < 0)
- || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15
- || c2 < 0)
- || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15
- || c3 < 0)
- || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15
- || c4 < 0)) {
- currentPosition = temp;
- return false;
- }
-
- currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
- if (!Character.isDigit(currentCharacter)) {
- currentPosition = temp;
- return false;
- }
-
- //need the unicode buffer
- if (withoutUnicodePtr == 0) {
- //buffer all the entries that have been left aside....
- withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
- System.arraycopy(
- source,
- startPosition,
- withoutUnicodeBuffer,
- 1,
- withoutUnicodePtr);
- }
- //fill the buffer with the char
- withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
- return true;
- } //-------------end unicode traitement--------------
- else {
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// //-------------unicode traitement ------------
+// int c1, c2, c3, c4;
+// int unicodeSize = 6;
+// currentPosition++;
+// while (source[currentPosition] == 'u') {
+// currentPosition++;
+// unicodeSize++;
+// }
+//
+// if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c1 < 0)
+// || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c2 < 0)
+// || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c3 < 0)
+// || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c4 < 0)) {
+// currentPosition = temp;
+// return false;
+// }
+//
+// currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
+// if (!Character.isDigit(currentCharacter)) {
+// currentPosition = temp;
+// return false;
+// }
+//
+// //need the unicode buffer
+// if (withoutUnicodePtr == 0) {
+// //buffer all the entries that have been left aside....
+// withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
+// System.arraycopy(
+// source,
+// startPosition,
+// withoutUnicodeBuffer,
+// 1,
+// withoutUnicodePtr);
+// }
+// //fill the buffer with the char
+// withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+// return true;
+// } //-------------end unicode traitement--------------
+// else {
if (!Character.isDigit(currentCharacter)) {
currentPosition = temp;
return false;
}
- if (withoutUnicodePtr != 0)
- withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+// if (withoutUnicodePtr != 0)
+// withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
return true;
- }
+// }
} catch (IndexOutOfBoundsException e) {
currentPosition = temp;
return false;
int temp = currentPosition;
try {
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- //-------------unicode traitement ------------
- int c1, c2, c3, c4;
- int unicodeSize = 6;
- currentPosition++;
- while (source[currentPosition] == 'u') {
- currentPosition++;
- unicodeSize++;
- }
-
- if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
- || c1 < 0)
- || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15
- || c2 < 0)
- || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15
- || c3 < 0)
- || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15
- || c4 < 0)) {
- currentPosition = temp;
- return false;
- }
-
- currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
- if (Character.digit(currentCharacter, radix) == -1) {
- currentPosition = temp;
- return false;
- }
-
- //need the unicode buffer
- if (withoutUnicodePtr == 0) {
- //buffer all the entries that have been left aside....
- withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
- System.arraycopy(
- source,
- startPosition,
- withoutUnicodeBuffer,
- 1,
- withoutUnicodePtr);
- }
- //fill the buffer with the char
- withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
- return true;
- } //-------------end unicode traitement--------------
- else {
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// //-------------unicode traitement ------------
+// int c1, c2, c3, c4;
+// int unicodeSize = 6;
+// currentPosition++;
+// while (source[currentPosition] == 'u') {
+// currentPosition++;
+// unicodeSize++;
+// }
+//
+// if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c1 < 0)
+// || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c2 < 0)
+// || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c3 < 0)
+// || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c4 < 0)) {
+// currentPosition = temp;
+// return false;
+// }
+//
+// currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
+// if (Character.digit(currentCharacter, radix) == -1) {
+// currentPosition = temp;
+// return false;
+// }
+//
+// //need the unicode buffer
+// if (withoutUnicodePtr == 0) {
+// //buffer all the entries that have been left aside....
+// withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
+// System.arraycopy(
+// source,
+// startPosition,
+// withoutUnicodeBuffer,
+// 1,
+// withoutUnicodePtr);
+// }
+// //fill the buffer with the char
+// withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+// return true;
+// } //-------------end unicode traitement--------------
+// else {
if (Character.digit(currentCharacter, radix) == -1) {
currentPosition = temp;
return false;
}
- if (withoutUnicodePtr != 0)
- withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+// if (withoutUnicodePtr != 0)
+// withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
return true;
- }
+// }
} catch (IndexOutOfBoundsException e) {
currentPosition = temp;
return false;
int temp = currentPosition;
try {
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- //-------------unicode traitement ------------
- int c1, c2, c3, c4;
- int unicodeSize = 6;
- currentPosition++;
- while (source[currentPosition] == 'u') {
- currentPosition++;
- unicodeSize++;
- }
-
- if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
- || c1 < 0)
- || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15
- || c2 < 0)
- || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15
- || c3 < 0)
- || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15
- || c4 < 0)) {
- currentPosition = temp;
- return false;
- }
-
- currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
- if (!isPHPIdentifierPart(currentCharacter)) {
- currentPosition = temp;
- return false;
- }
-
- //need the unicode buffer
- if (withoutUnicodePtr == 0) {
- //buffer all the entries that have been left aside....
- withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
- System.arraycopy(
- source,
- startPosition,
- withoutUnicodeBuffer,
- 1,
- withoutUnicodePtr);
- }
- //fill the buffer with the char
- withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
- return true;
- } //-------------end unicode traitement--------------
- else {
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// //-------------unicode traitement ------------
+// int c1, c2, c3, c4;
+// int unicodeSize = 6;
+// currentPosition++;
+// while (source[currentPosition] == 'u') {
+// currentPosition++;
+// unicodeSize++;
+// }
+//
+// if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c1 < 0)
+// || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c2 < 0)
+// || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c3 < 0)
+// || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c4 < 0)) {
+// currentPosition = temp;
+// return false;
+// }
+//
+// currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
+// if (!isPHPIdentifierPart(currentCharacter)) {
+// currentPosition = temp;
+// return false;
+// }
+//
+// //need the unicode buffer
+// if (withoutUnicodePtr == 0) {
+// //buffer all the entries that have been left aside....
+// withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
+// System.arraycopy(
+// source,
+// startPosition,
+// withoutUnicodeBuffer,
+// 1,
+// withoutUnicodePtr);
+// }
+// //fill the buffer with the char
+// withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+// return true;
+// } //-------------end unicode traitement--------------
+// else {
if (!isPHPIdentifierPart(currentCharacter)) {
currentPosition = temp;
return false;
}
- if (withoutUnicodePtr != 0)
- withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+// if (withoutUnicodePtr != 0)
+// withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
return true;
- }
+// }
} catch (IndexOutOfBoundsException e) {
currentPosition = temp;
return false;
}
public int getNextToken() throws InvalidInputException {
+ int htmlPosition = currentPosition;
try {
while (!phpMode) {
- startPosition = currentPosition;
currentCharacter = source[currentPosition++];
if (currentCharacter == '<') {
if (getNextChar('?')) {
// <?
startPosition = currentPosition;
phpMode = true;
+ if (tokenizeWhiteSpace) {
+ // && (whiteStart != currentPosition - 1)) {
+ // reposition scanner in case we are interested by spaces as tokens
+ startPosition = htmlPosition;
+ return TokenNameHTML;
+ }
} else {
boolean phpStart =
(currentCharacter == 'P') || (currentCharacter == 'p');
// <?PHP <?php
startPosition = currentPosition;
phpMode = true;
+
+ if (tokenizeWhiteSpace) {
+ // && (whiteStart != currentPosition - 1)) {
+ // reposition scanner in case we are interested by spaces as tokens
+ startPosition = htmlPosition;
+ return TokenNameHTML;
+ }
}
}
}
}
} //-----------------end switch while try--------------------
catch (IndexOutOfBoundsException e) {
+ if (tokenizeWhiteSpace) {
+ // && (whiteStart != currentPosition - 1)) {
+ // reposition scanner in case we are interested by spaces as tokens
+ startPosition = htmlPosition;
+ }
return TokenNameEOF;
}
boolean isWhiteSpace;
do {
startPosition = currentPosition;
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- isWhiteSpace = jumpOverUnicodeWhiteSpace();
- } else {
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// isWhiteSpace = jumpOverUnicodeWhiteSpace();
+// } else {
if ((currentCharacter == '\r') || (currentCharacter == '\n')) {
checkNonExternalizeString();
if (recordLineSeparator) {
isWhiteSpace =
(currentCharacter == ' ')
|| Character.isWhitespace(currentCharacter);
- }
+// }
} while (isWhiteSpace);
if (tokenizeWhiteSpace && (whiteStart != currentPosition - 1)) {
// reposition scanner in case we are interested by spaces as tokens
try {
// consume next character
unicodeAsBackSlash = false;
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- getNextUnicodeChar();
- } else {
- if (withoutUnicodePtr != 0) {
- withoutUnicodeBuffer[++withoutUnicodePtr] =
- currentCharacter;
- }
- }
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// getNextUnicodeChar();
+// } else {
+// if (withoutUnicodePtr != 0) {
+// withoutUnicodeBuffer[++withoutUnicodePtr] =
+// currentCharacter;
+// }
+// }
while (currentCharacter != '\'') {
}
// consume next character
unicodeAsBackSlash = false;
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- getNextUnicodeChar();
- } else {
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// getNextUnicodeChar();
+// } else {
if (withoutUnicodePtr != 0) {
withoutUnicodeBuffer[++withoutUnicodePtr] =
currentCharacter;
}
- }
+// }
}
} catch (IndexOutOfBoundsException e) {
try {
// consume next character
unicodeAsBackSlash = false;
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- getNextUnicodeChar();
- } else {
- if (withoutUnicodePtr != 0) {
- withoutUnicodeBuffer[++withoutUnicodePtr] =
- currentCharacter;
- }
- }
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// getNextUnicodeChar();
+// } else {
+// if (withoutUnicodePtr != 0) {
+// withoutUnicodeBuffer[++withoutUnicodePtr] =
+// currentCharacter;
+// }
+// }
while (currentCharacter != '"') {
}
// consume next character
unicodeAsBackSlash = false;
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- getNextUnicodeChar();
- } else {
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// getNextUnicodeChar();
+// } else {
if (withoutUnicodePtr != 0) {
withoutUnicodeBuffer[++withoutUnicodePtr] =
currentCharacter;
}
- }
+// }
}
} catch (IndexOutOfBoundsException e) {
try {
// consume next character
unicodeAsBackSlash = false;
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- getNextUnicodeChar();
- } else {
- if (withoutUnicodePtr != 0) {
- withoutUnicodeBuffer[++withoutUnicodePtr] =
- currentCharacter;
- }
- }
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// getNextUnicodeChar();
+// } else {
+// if (withoutUnicodePtr != 0) {
+// withoutUnicodeBuffer[++withoutUnicodePtr] =
+// currentCharacter;
+// }
+// }
while (currentCharacter != '`') {
}
// consume next character
unicodeAsBackSlash = false;
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- getNextUnicodeChar();
- } else {
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// getNextUnicodeChar();
+// } else {
if (withoutUnicodePtr != 0) {
withoutUnicodeBuffer[++withoutUnicodePtr] =
currentCharacter;
}
- }
+// }
}
} catch (IndexOutOfBoundsException e) {
//line comment
int endPositionForLineComment = 0;
try { //get the next char
- if (((currentCharacter = source[currentPosition++])
- == '\\')
- && (source[currentPosition] == 'u')) {
- //-------------unicode traitement ------------
- int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
- currentPosition++;
- while (source[currentPosition] == 'u') {
- currentPosition++;
- }
- if ((c1 =
- Character.getNumericValue(source[currentPosition++]))
- > 15
- || c1 < 0
- || (c2 =
- Character.getNumericValue(source[currentPosition++]))
- > 15
- || c2 < 0
- || (c3 =
- Character.getNumericValue(source[currentPosition++]))
- > 15
- || c3 < 0
- || (c4 =
- Character.getNumericValue(source[currentPosition++]))
- > 15
- || c4 < 0) {
- throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
- } else {
- currentCharacter =
- (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
- }
- }
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++])
+// == '\\')
+// && (source[currentPosition] == 'u')) {
+// //-------------unicode traitement ------------
+// int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
+// currentPosition++;
+// while (source[currentPosition] == 'u') {
+// currentPosition++;
+// }
+// if ((c1 =
+// Character.getNumericValue(source[currentPosition++]))
+// > 15
+// || c1 < 0
+// || (c2 =
+// Character.getNumericValue(source[currentPosition++]))
+// > 15
+// || c2 < 0
+// || (c3 =
+// Character.getNumericValue(source[currentPosition++]))
+// > 15
+// || c3 < 0
+// || (c4 =
+// Character.getNumericValue(source[currentPosition++]))
+// > 15
+// || c4 < 0) {
+// throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
+// } else {
+// currentCharacter =
+// (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
+// }
+// }
//handle the \\u case manually into comment
- if (currentCharacter == '\\') {
- if (source[currentPosition] == '\\')
- currentPosition++;
- } //jump over the \\
+// if (currentCharacter == '\\') {
+// if (source[currentPosition] == '\\')
+// currentPosition++;
+// } //jump over the \\
boolean isUnicode = false;
while (currentCharacter != '\r'
&& currentCharacter != '\n') {
//get the next char
isUnicode = false;
- if (((currentCharacter = source[currentPosition++])
- == '\\')
- && (source[currentPosition] == 'u')) {
- isUnicode = true;
- //-------------unicode traitement ------------
- int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
- currentPosition++;
- while (source[currentPosition] == 'u') {
- currentPosition++;
- }
- if ((c1 =
- Character.getNumericValue(source[currentPosition++]))
- > 15
- || c1 < 0
- || (c2 =
- Character.getNumericValue(
- source[currentPosition++]))
- > 15
- || c2 < 0
- || (c3 =
- Character.getNumericValue(
- source[currentPosition++]))
- > 15
- || c3 < 0
- || (c4 =
- Character.getNumericValue(
- source[currentPosition++]))
- > 15
- || c4 < 0) {
- throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
- } else {
- currentCharacter =
- (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
- }
- }
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++])
+// == '\\')
+// && (source[currentPosition] == 'u')) {
+// isUnicode = true;
+// //-------------unicode traitement ------------
+// int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
+// currentPosition++;
+// while (source[currentPosition] == 'u') {
+// currentPosition++;
+// }
+// if ((c1 =
+// Character.getNumericValue(source[currentPosition++]))
+// > 15
+// || c1 < 0
+// || (c2 =
+// Character.getNumericValue(
+// source[currentPosition++]))
+// > 15
+// || c2 < 0
+// || (c3 =
+// Character.getNumericValue(
+// source[currentPosition++]))
+// > 15
+// || c3 < 0
+// || (c4 =
+// Character.getNumericValue(
+// source[currentPosition++]))
+// > 15
+// || c4 < 0) {
+// throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
+// } else {
+// currentCharacter =
+// (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
+// }
+// }
//handle the \\u case manually into comment
- if (currentCharacter == '\\') {
- if (source[currentPosition] == '\\')
- currentPosition++;
- } //jump over the \\
+// if (currentCharacter == '\\') {
+// if (source[currentPosition] == '\\')
+// currentPosition++;
+// } //jump over the \\
}
if (isUnicode) {
endPositionForLineComment = currentPosition - 6;
boolean isJavadoc = false, star = false;
// consume next character
unicodeAsBackSlash = false;
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- getNextUnicodeChar();
- } else {
- if (withoutUnicodePtr != 0) {
- withoutUnicodeBuffer[++withoutUnicodePtr] =
- currentCharacter;
- }
- }
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// getNextUnicodeChar();
+// } else {
+// if (withoutUnicodePtr != 0) {
+// withoutUnicodeBuffer[++withoutUnicodePtr] =
+// currentCharacter;
+// }
+// }
if (currentCharacter == '*') {
isJavadoc = true;
}
}
try { //get the next char
- if (((currentCharacter = source[currentPosition++])
- == '\\')
- && (source[currentPosition] == 'u')) {
- //-------------unicode traitement ------------
- getNextUnicodeChar();
- }
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++])
+// == '\\')
+// && (source[currentPosition] == 'u')) {
+// //-------------unicode traitement ------------
+// getNextUnicodeChar();
+// }
//handle the \\u case manually into comment
- if (currentCharacter == '\\') {
- if (source[currentPosition] == '\\')
- currentPosition++;
- //jump over the \\
- }
+// if (currentCharacter == '\\') {
+// if (source[currentPosition] == '\\')
+// currentPosition++;
+// //jump over the \\
+// }
// empty comment is not a javadoc /**/
if (currentCharacter == '/') {
isJavadoc = false;
}
star = currentCharacter == '*';
//get next char
- if (((currentCharacter = source[currentPosition++])
- == '\\')
- && (source[currentPosition] == 'u')) {
- //-------------unicode traitement ------------
- getNextUnicodeChar();
- }
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++])
+// == '\\')
+// && (source[currentPosition] == 'u')) {
+// //-------------unicode traitement ------------
+// getNextUnicodeChar();
+// }
//handle the \\u case manually into comment
- if (currentCharacter == '\\') {
- if (source[currentPosition] == '\\')
- currentPosition++;
- } //jump over the \\
+// if (currentCharacter == '\\') {
+// if (source[currentPosition] == '\\')
+// currentPosition++;
+// } //jump over the \\
}
recordComment(isJavadoc);
if (tokenizeComments) {
default :
if (currentCharacter == '$') {
- while ( (currentCharacter = source[currentPosition++])=='$') {
+ while ((currentCharacter = source[currentPosition++]) == '$') {
}
if (currentCharacter == '{')
return TokenNameDOLLAR_LBRACE;
return TokenNameEOF;
}
- public final void getNextUnicodeChar()
- throws IndexOutOfBoundsException, InvalidInputException {
- //VOID
- //handle the case of unicode.
- //when a unicode appears then we must use a buffer that holds char internal values
- //At the end of this method currentCharacter holds the new visited char
- //and currentPosition points right next after it
-
- //ALL getNextChar.... ARE OPTIMIZED COPIES
-
- int c1 = 0, c2 = 0, c3 = 0, c4 = 0, unicodeSize = 6;
- currentPosition++;
- while (source[currentPosition] == 'u') {
- currentPosition++;
- unicodeSize++;
- }
-
- if ((c1 = Character.getNumericValue(source[currentPosition++])) > 15
- || c1 < 0
- || (c2 = Character.getNumericValue(source[currentPosition++])) > 15
- || c2 < 0
- || (c3 = Character.getNumericValue(source[currentPosition++])) > 15
- || c3 < 0
- || (c4 = Character.getNumericValue(source[currentPosition++])) > 15
- || c4 < 0) {
- throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
- } else {
- currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
- //need the unicode buffer
- if (withoutUnicodePtr == 0) {
- //buffer all the entries that have been left aside....
- withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
- System.arraycopy(
- source,
- startPosition,
- withoutUnicodeBuffer,
- 1,
- withoutUnicodePtr);
- }
- //fill the buffer with the char
- withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
- }
- unicodeAsBackSlash = currentCharacter == '\\';
- }
+// public final void getNextUnicodeChar()
+// throws IndexOutOfBoundsException, InvalidInputException {
+// //VOID
+// //handle the case of unicode.
+// //when a unicode appears then we must use a buffer that holds char internal values
+// //At the end of this method currentCharacter holds the new visited char
+// //and currentPosition points right next after it
+//
+// //ALL getNextChar.... ARE OPTIMIZED COPIES
+//
+// int c1 = 0, c2 = 0, c3 = 0, c4 = 0, unicodeSize = 6;
+// currentPosition++;
+// while (source[currentPosition] == 'u') {
+// currentPosition++;
+// unicodeSize++;
+// }
+//
+// if ((c1 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c1 < 0
+// || (c2 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c2 < 0
+// || (c3 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c3 < 0
+// || (c4 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c4 < 0) {
+// throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
+// } else {
+// currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
+// //need the unicode buffer
+// if (withoutUnicodePtr == 0) {
+// //buffer all the entries that have been left aside....
+// withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
+// System.arraycopy(
+// source,
+// startPosition,
+// withoutUnicodeBuffer,
+// 1,
+// withoutUnicodePtr);
+// }
+// //fill the buffer with the char
+// withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+// }
+// unicodeAsBackSlash = currentCharacter == '\\';
+// }
/* Tokenize a method body, assuming that curly brackets are properly balanced.
*/
public final void jumpOverMethodBody() {
boolean isWhiteSpace;
do {
startPosition = currentPosition;
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- isWhiteSpace = jumpOverUnicodeWhiteSpace();
- } else {
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// isWhiteSpace = jumpOverUnicodeWhiteSpace();
+// } else {
if (recordLineSeparator
&& ((currentCharacter == '\r') || (currentCharacter == '\n')))
pushLineSeparator();
isWhiteSpace = Character.isWhitespace(currentCharacter);
- }
+// }
} while (isWhiteSpace);
// -------consume token until } is found---------
} catch (InvalidInputException ex) {
};
} else {
- try { // consume next character
+// try { // consume next character
unicodeAsBackSlash = false;
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- getNextUnicodeChar();
- } else {
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// getNextUnicodeChar();
+// } else {
if (withoutUnicodePtr != 0) {
withoutUnicodeBuffer[++withoutUnicodePtr] =
currentCharacter;
}
- }
- } catch (InvalidInputException ex) {
- };
+// }
+// } catch (InvalidInputException ex) {
+// };
}
getNextChar('\'');
break;
}
case '"' :
try {
- try { // consume next character
+// try { // consume next character
unicodeAsBackSlash = false;
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- getNextUnicodeChar();
- } else {
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// getNextUnicodeChar();
+// } else {
if (withoutUnicodePtr != 0) {
withoutUnicodeBuffer[++withoutUnicodePtr] =
currentCharacter;
}
- }
- } catch (InvalidInputException ex) {
- };
+// }
+// } catch (InvalidInputException ex) {
+// };
while (currentCharacter != '"') {
if (currentCharacter == '\r') {
if (source[currentPosition] == '\n')
} catch (InvalidInputException ex) {
};
}
- try { // consume next character
+// try { // consume next character
unicodeAsBackSlash = false;
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- getNextUnicodeChar();
- } else {
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// getNextUnicodeChar();
+// } else {
if (withoutUnicodePtr != 0) {
withoutUnicodeBuffer[++withoutUnicodePtr] =
currentCharacter;
}
- }
- } catch (InvalidInputException ex) {
- };
+// }
+// } catch (InvalidInputException ex) {
+// };
}
} catch (IndexOutOfBoundsException e) {
return;
//line comment
try {
//get the next char
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- //-------------unicode traitement ------------
- int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
- currentPosition++;
- while (source[currentPosition] == 'u') {
- currentPosition++;
- }
- if ((c1 =
- Character.getNumericValue(source[currentPosition++]))
- > 15
- || c1 < 0
- || (c2 =
- Character.getNumericValue(source[currentPosition++]))
- > 15
- || c2 < 0
- || (c3 =
- Character.getNumericValue(source[currentPosition++]))
- > 15
- || c3 < 0
- || (c4 =
- Character.getNumericValue(source[currentPosition++]))
- > 15
- || c4 < 0) {
- //error don't care of the value
- currentCharacter = 'A';
- } //something different from \n and \r
- else {
- currentCharacter =
- (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
- }
- }
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// //-------------unicode traitement ------------
+// int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
+// currentPosition++;
+// while (source[currentPosition] == 'u') {
+// currentPosition++;
+// }
+// if ((c1 =
+// Character.getNumericValue(source[currentPosition++]))
+// > 15
+// || c1 < 0
+// || (c2 =
+// Character.getNumericValue(source[currentPosition++]))
+// > 15
+// || c2 < 0
+// || (c3 =
+// Character.getNumericValue(source[currentPosition++]))
+// > 15
+// || c3 < 0
+// || (c4 =
+// Character.getNumericValue(source[currentPosition++]))
+// > 15
+// || c4 < 0) {
+// //error don't care of the value
+// currentCharacter = 'A';
+// } //something different from \n and \r
+// else {
+// currentCharacter =
+// (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
+// }
+// }
while (currentCharacter != '\r'
&& currentCharacter != '\n') {
//get the next char
- if (((currentCharacter = source[currentPosition++])
- == '\\')
- && (source[currentPosition] == 'u')) {
- //-------------unicode traitement ------------
- int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
- currentPosition++;
- while (source[currentPosition] == 'u') {
- currentPosition++;
- }
- if ((c1 =
- Character.getNumericValue(source[currentPosition++]))
- > 15
- || c1 < 0
- || (c2 =
- Character.getNumericValue(source[currentPosition++]))
- > 15
- || c2 < 0
- || (c3 =
- Character.getNumericValue(source[currentPosition++]))
- > 15
- || c3 < 0
- || (c4 =
- Character.getNumericValue(source[currentPosition++]))
- > 15
- || c4 < 0) {
- //error don't care of the value
- currentCharacter = 'A';
- } //something different from \n and \r
- else {
- currentCharacter =
- (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
- }
- }
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++])
+// == '\\')
+// && (source[currentPosition] == 'u')) {
+// //-------------unicode traitement ------------
+// int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
+// currentPosition++;
+// while (source[currentPosition] == 'u') {
+// currentPosition++;
+// }
+// if ((c1 =
+// Character.getNumericValue(source[currentPosition++]))
+// > 15
+// || c1 < 0
+// || (c2 =
+// Character.getNumericValue(source[currentPosition++]))
+// > 15
+// || c2 < 0
+// || (c3 =
+// Character.getNumericValue(source[currentPosition++]))
+// > 15
+// || c3 < 0
+// || (c4 =
+// Character.getNumericValue(source[currentPosition++]))
+// > 15
+// || c4 < 0) {
+// //error don't care of the value
+// currentCharacter = 'A';
+// } //something different from \n and \r
+// else {
+// currentCharacter =
+// (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
+// }
+// }
}
if (recordLineSeparator
&& ((currentCharacter == '\r') || (currentCharacter == '\n')))
if (test > 0) {
//traditional and annotation comment
boolean star = false;
- try { // consume next character
+// try { // consume next character
unicodeAsBackSlash = false;
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- getNextUnicodeChar();
- } else {
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// getNextUnicodeChar();
+// } else {
if (withoutUnicodePtr != 0) {
withoutUnicodeBuffer[++withoutUnicodePtr] =
currentCharacter;
}
- };
- } catch (InvalidInputException ex) {
- };
+// };
+// } catch (InvalidInputException ex) {
+// };
if (currentCharacter == '*') {
star = true;
}
&& ((currentCharacter == '\r') || (currentCharacter == '\n')))
pushLineSeparator();
try { //get the next char
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- //-------------unicode traitement ------------
- int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
- currentPosition++;
- while (source[currentPosition] == 'u') {
- currentPosition++;
- }
- if ((c1 =
- Character.getNumericValue(source[currentPosition++]))
- > 15
- || c1 < 0
- || (c2 =
- Character.getNumericValue(source[currentPosition++]))
- > 15
- || c2 < 0
- || (c3 =
- Character.getNumericValue(source[currentPosition++]))
- > 15
- || c3 < 0
- || (c4 =
- Character.getNumericValue(source[currentPosition++]))
- > 15
- || c4 < 0) {
- //error don't care of the value
- currentCharacter = 'A';
- } //something different from * and /
- else {
- currentCharacter =
- (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
- }
- }
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// //-------------unicode traitement ------------
+// int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
+// currentPosition++;
+// while (source[currentPosition] == 'u') {
+// currentPosition++;
+// }
+// if ((c1 =
+// Character.getNumericValue(source[currentPosition++]))
+// > 15
+// || c1 < 0
+// || (c2 =
+// Character.getNumericValue(source[currentPosition++]))
+// > 15
+// || c2 < 0
+// || (c3 =
+// Character.getNumericValue(source[currentPosition++]))
+// > 15
+// || c3 < 0
+// || (c4 =
+// Character.getNumericValue(source[currentPosition++]))
+// > 15
+// || c4 < 0) {
+// //error don't care of the value
+// currentCharacter = 'A';
+// } //something different from * and /
+// else {
+// currentCharacter =
+// (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
+// }
+// }
//loop until end of comment */
while ((currentCharacter != '/') || (!star)) {
if (recordLineSeparator
pushLineSeparator();
star = currentCharacter == '*';
//get next char
- if (((currentCharacter = source[currentPosition++])
- == '\\')
- && (source[currentPosition] == 'u')) {
- //-------------unicode traitement ------------
- int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
- currentPosition++;
- while (source[currentPosition] == 'u') {
- currentPosition++;
- }
- if ((c1 =
- Character.getNumericValue(source[currentPosition++]))
- > 15
- || c1 < 0
- || (c2 =
- Character.getNumericValue(source[currentPosition++]))
- > 15
- || c2 < 0
- || (c3 =
- Character.getNumericValue(source[currentPosition++]))
- > 15
- || c3 < 0
- || (c4 =
- Character.getNumericValue(source[currentPosition++]))
- > 15
- || c4 < 0) {
- //error don't care of the value
- currentCharacter = 'A';
- } //something different from * and /
- else {
- currentCharacter =
- (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
- }
- }
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++])
+// == '\\')
+// && (source[currentPosition] == 'u')) {
+// //-------------unicode traitement ------------
+// int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
+// currentPosition++;
+// while (source[currentPosition] == 'u') {
+// currentPosition++;
+// }
+// if ((c1 =
+// Character.getNumericValue(source[currentPosition++]))
+// > 15
+// || c1 < 0
+// || (c2 =
+// Character.getNumericValue(source[currentPosition++]))
+// > 15
+// || c2 < 0
+// || (c3 =
+// Character.getNumericValue(source[currentPosition++]))
+// > 15
+// || c3 < 0
+// || (c4 =
+// Character.getNumericValue(source[currentPosition++]))
+// > 15
+// || c4 < 0) {
+// //error don't care of the value
+// currentCharacter = 'A';
+// } //something different from * and /
+// else {
+// currentCharacter =
+// (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
+// }
+// }
}
} catch (IndexOutOfBoundsException e) {
return;
}
return;
}
- public final boolean jumpOverUnicodeWhiteSpace()
- throws InvalidInputException {
- //BOOLEAN
- //handle the case of unicode. Jump over the next whiteSpace
- //making startPosition pointing on the next available char
- //On false, the currentCharacter is filled up with a potential
- //correct char
-
- try {
- this.wasAcr = false;
- int c1, c2, c3, c4;
- int unicodeSize = 6;
- currentPosition++;
- while (source[currentPosition] == 'u') {
- currentPosition++;
- unicodeSize++;
- }
-
- if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
- || c1 < 0)
- || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15
- || c2 < 0)
- || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15
- || c3 < 0)
- || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15
- || c4 < 0)) {
- throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
- }
-
- currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
- if (recordLineSeparator
- && ((currentCharacter == '\r') || (currentCharacter == '\n')))
- pushLineSeparator();
- if (Character.isWhitespace(currentCharacter))
- return true;
-
- //buffer the new char which is not a white space
- withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
- //withoutUnicodePtr == 1 is true here
- return false;
- } catch (IndexOutOfBoundsException e) {
- throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
- }
- }
+// public final boolean jumpOverUnicodeWhiteSpace()
+// throws InvalidInputException {
+// //BOOLEAN
+// //handle the case of unicode. Jump over the next whiteSpace
+// //making startPosition pointing on the next available char
+// //On false, the currentCharacter is filled up with a potential
+// //correct char
+//
+// try {
+// this.wasAcr = false;
+// int c1, c2, c3, c4;
+// int unicodeSize = 6;
+// currentPosition++;
+// while (source[currentPosition] == 'u') {
+// currentPosition++;
+// unicodeSize++;
+// }
+//
+// if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c1 < 0)
+// || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c2 < 0)
+// || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c3 < 0)
+// || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15
+// || c4 < 0)) {
+// throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
+// }
+//
+// currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
+// if (recordLineSeparator
+// && ((currentCharacter == '\r') || (currentCharacter == '\n')))
+// pushLineSeparator();
+// if (Character.isWhitespace(currentCharacter))
+// return true;
+//
+// //buffer the new char which is not a white space
+// withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+// //withoutUnicodePtr == 1 is true here
+// return false;
+// } catch (IndexOutOfBoundsException e) {
+// throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
+// }
+// }
public final int[] getLineEnds() {
//return a bounded copy of this.lineEnds
return new char[] { charOne };
}
}
+
final char[] optimizedCurrentTokenSource2() {
//try to return the same char[] build only once
newEntry2 = max;
return r;
}
+
final char[] optimizedCurrentTokenSource3() {
//try to return the same char[] build only once
newEntry3 = max;
return r;
}
+
final char[] optimizedCurrentTokenSource4() {
//try to return the same char[] build only once
return r;
}
+
final char[] optimizedCurrentTokenSource5() {
//try to return the same char[] build only once
return r;
}
+
final char[] optimizedCurrentTokenSource6() {
//try to return the same char[] build only once
newEntry6 = max;
return r;
}
+
public final void pushLineSeparator() throws InvalidInputException {
//see comment on isLineDelimiter(char) for the use of '\n' and '\r'
final int INCREMENT = 250;
// the string with "\\u" is a legal string of two chars \ and u
//thus we use a direct access to the source (for regular cases).
- if (unicodeAsBackSlash) {
- // consume next character
- unicodeAsBackSlash = false;
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- getNextUnicodeChar();
- } else {
- if (withoutUnicodePtr != 0) {
- withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
- }
- }
- } else
+// if (unicodeAsBackSlash) {
+// // consume next character
+// unicodeAsBackSlash = false;
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// getNextUnicodeChar();
+// } else {
+// if (withoutUnicodePtr != 0) {
+// withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+// }
+// }
+// } else
currentCharacter = source[currentPosition++];
switch (currentCharacter) {
case '\'' :
// the string with "\\u" is a legal string of two chars \ and u
//thus we use a direct access to the source (for regular cases).
- if (unicodeAsBackSlash) {
- // consume next character
- unicodeAsBackSlash = false;
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- getNextUnicodeChar();
- } else {
- if (withoutUnicodePtr != 0) {
- withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
- }
- }
- } else
+// if (unicodeAsBackSlash) {
+// // consume next character
+// unicodeAsBackSlash = false;
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// getNextUnicodeChar();
+// } else {
+// if (withoutUnicodePtr != 0) {
+// withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+// }
+// }
+// } else
currentCharacter = source[currentPosition++];
switch (currentCharacter) {
// case 'b' :
// return scanIdentifierOrKeyword( false );
// }
- public int scanIdentifierOrKeyword(boolean isVariable)
+ public int scanIdentifierOrKeyword(boolean isVariable)
throws InvalidInputException {
//test keywords
//disptach on the second char :-)...cool....but fast !
useAssertAsAnIndentifier = false;
-
+
while (getNextCharAsJavaIdentifierPart()) {
};
int index, length;
char[] data;
char firstLetter;
- if (withoutUnicodePtr == 0)
+// if (withoutUnicodePtr == 0)
//quick test on length == 1 but not on length > 12 while most identifier
//have a length which is <= 12...but there are lots of identifier with
//only one char....
- {
+// {
if ((length = currentPosition - startPosition) == 1)
return TokenNameIdentifier;
// data = source;
data[i] = Character.toLowerCase(source[index + i]);
}
index = 0;
- } else {
- if ((length = withoutUnicodePtr) == 1)
- return TokenNameIdentifier;
- // data = withoutUnicodeBuffer;
- data = new char[withoutUnicodeBuffer.length];
- for (int i = 0; i < withoutUnicodeBuffer.length; i++) {
- data[i] = Character.toLowerCase(withoutUnicodeBuffer[i]);
- }
- index = 1;
- }
+// } else {
+// if ((length = withoutUnicodePtr) == 1)
+// return TokenNameIdentifier;
+// // data = withoutUnicodeBuffer;
+// data = new char[withoutUnicodeBuffer.length];
+// for (int i = 0; i < withoutUnicodeBuffer.length; i++) {
+// data[i] = Character.toLowerCase(withoutUnicodeBuffer[i]);
+// }
+// index = 1;
+// }
firstLetter = data[index];
switch (firstLetter) {
//force the first char of the hexa number do exist...
// consume next character
unicodeAsBackSlash = false;
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- getNextUnicodeChar();
- } else {
- if (withoutUnicodePtr != 0) {
- withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
- }
- }
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// getNextUnicodeChar();
+// } else {
+// if (withoutUnicodePtr != 0) {
+// withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+// }
+// }
if (Character.digit(currentCharacter, 16) == -1)
throw new InvalidInputException(INVALID_HEXA);
//---end forcing--
if (getNextChar('e', 'E') >= 0) {
// consume next character
unicodeAsBackSlash = false;
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- getNextUnicodeChar();
- } else {
- if (withoutUnicodePtr != 0) {
- withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
- }
- }
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// getNextUnicodeChar();
+// } else {
+// if (withoutUnicodePtr != 0) {
+// withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+// }
+// }
if ((currentCharacter == '-') || (currentCharacter == '+')) {
// consume next character
unicodeAsBackSlash = false;
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- getNextUnicodeChar();
- } else {
- if (withoutUnicodePtr != 0) {
- withoutUnicodeBuffer[++withoutUnicodePtr] =
- currentCharacter;
- }
- }
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// getNextUnicodeChar();
+// } else {
+// if (withoutUnicodePtr != 0) {
+// withoutUnicodeBuffer[++withoutUnicodePtr] =
+// currentCharacter;
+// }
+// }
}
if (!Character.isDigit(currentCharacter))
throw new InvalidInputException(INVALID_FLOAT);
floating = true;
// consume next character
unicodeAsBackSlash = false;
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- getNextUnicodeChar();
- } else {
- if (withoutUnicodePtr != 0) {
- withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
- }
- }
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// getNextUnicodeChar();
+// } else {
+// if (withoutUnicodePtr != 0) {
+// withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+// }
+// }
if ((currentCharacter == '-')
|| (currentCharacter == '+')) { // consume next character
unicodeAsBackSlash = false;
- if (((currentCharacter = source[currentPosition++]) == '\\')
- && (source[currentPosition] == 'u')) {
- getNextUnicodeChar();
- } else {
- if (withoutUnicodePtr != 0) {
- withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
- }
- }
+ currentCharacter = source[currentPosition++];
+// if (((currentCharacter = source[currentPosition++]) == '\\')
+// && (source[currentPosition] == 'u')) {
+// getNextUnicodeChar();
+// } else {
+// if (withoutUnicodePtr != 0) {
+// withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+// }
+// }
}
if (!Character.isDigit(currentCharacter))
throw new InvalidInputException(INVALID_FLOAT);
git://secure.phpeclipse.com / phpeclipse.git / blobdiff