import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import java.util.Stack;
import net.sourceforge.phpdt.core.compiler.CharOperation;
import net.sourceforge.phpdt.core.compiler.IScanner;
import net.sourceforge.phpdt.core.compiler.ITerminalSymbols;
public boolean containsAssertKeyword = false;
public boolean recordLineSeparator;
public boolean phpMode = false;
+ public Stack encapsedStringStack = null;
public char currentCharacter;
public int startPosition;
public int currentPosition;
// source
public boolean tokenizeComments;
public boolean tokenizeWhiteSpace;
+ public boolean tokenizeStrings;
//source should be viewed as a window (aka a part)
//of a entire very large stream
public char source[];
public char[][] taskTags = null;
public char[][] taskPriorities = null;
public static final boolean DEBUG = false;
- public Scanner() {
- this(false, false);
- }
- public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace) {
- this(tokenizeComments, tokenizeWhiteSpace, false);
- }
+ public static final boolean TRACE = false;
+
/**
* Determines if the specified character is permissible as the first
* character in a PHP identifier
return false;
}
}
- public int getNextToken() throws InvalidInputException {
- int htmlPosition = currentPosition;
+ public int getCastOrParen() {
+ int tempPosition = currentPosition;
+ char tempCharacter = currentCharacter;
+ int tempToken = TokenNameLPAREN;
+ boolean found = false;
+ StringBuffer buf = new StringBuffer();
try {
- while (!phpMode) {
+ do {
currentCharacter = source[currentPosition++];
- if (currentCharacter == '<') {
- if (getNextChar('?')) {
- currentCharacter = source[currentPosition++];
- if ((currentCharacter == ' ')
- || Character.isWhitespace(currentCharacter)) {
- // <?
- startPosition = currentPosition;
- phpMode = true;
- if (tokenizeWhiteSpace) {
- // && (whiteStart != currentPosition - 1)) {
- // reposition scanner in case we are interested by spaces as
- // tokens
- startPosition = htmlPosition;
- return TokenNameHTML;
+ } while (currentCharacter == ' ' || currentCharacter == '\t');
+ while ((currentCharacter >= 'a' && currentCharacter <= 'z')
+ || (currentCharacter >= 'A' && currentCharacter <= 'Z')) {
+ buf.append(currentCharacter);
+ currentCharacter = source[currentPosition++];
+ }
+ if (buf.length() >= 3 && buf.length() <= 7) {
+ char[] data = buf.toString().toCharArray();
+ int index = 0;
+ switch (data.length) {
+ case 3 :
+ // int
+ if ((data[index] == 'i') && (data[++index] == 'n')
+ && (data[++index] == 't')) {
+ found = true;
+ tempToken = TokenNameintCAST;
+ }
+ break;
+ case 4 :
+ // bool real
+ if ((data[index] == 'b') && (data[++index] == 'o')
+ && (data[++index] == 'o') && (data[++index] == 'l')) {
+ found = true;
+ tempToken = TokenNameboolCAST;
+ } else {
+ index = 0;
+ if ((data[index] == 'r') && (data[++index] == 'e')
+ && (data[++index] == 'a') && (data[++index] == 'l')) {
+ found = true;
+ tempToken = TokenNamedoubleCAST;
}
+ }
+ break;
+ case 5 :
+ // array unset float
+ if ((data[index] == 'a') && (data[++index] == 'r')
+ && (data[++index] == 'r') && (data[++index] == 'a')
+ && (data[++index] == 'y')) {
+ found = true;
+ tempToken = TokenNamearrayCAST;
} else {
- boolean phpStart = (currentCharacter == 'P')
- || (currentCharacter == 'p');
- if (phpStart) {
- int test = getNextChar('H', 'h');
- if (test >= 0) {
- test = getNextChar('P', 'p');
- if (test >= 0) {
- // <?PHP <?php
- startPosition = currentPosition;
- phpMode = true;
- if (tokenizeWhiteSpace) {
- // && (whiteStart != currentPosition - 1)) {
- // reposition scanner in case we are interested by spaces
- // as tokens
- startPosition = htmlPosition;
- return TokenNameHTML;
- }
- }
+ index = 0;
+ if ((data[index] == 'u') && (data[++index] == 'n')
+ && (data[++index] == 's') && (data[++index] == 'e')
+ && (data[++index] == 't')) {
+ found = true;
+ tempToken = TokenNameunsetCAST;
+ } else {
+ index = 0;
+ if ((data[index] == 'f') && (data[++index] == 'l')
+ && (data[++index] == 'o') && (data[++index] == 'a')
+ && (data[++index] == 't')) {
+ found = true;
+ tempToken = TokenNamedoubleCAST;
}
}
}
+ break;
+ case 6 :
+ // object string double
+ if ((data[index] == 'o') && (data[++index] == 'b')
+ && (data[++index] == 'j') && (data[++index] == 'e')
+ && (data[++index] == 'c') && (data[++index] == 't')) {
+ found = true;
+ tempToken = TokenNameobjectCAST;
+ } else {
+ index = 0;
+ if ((data[index] == 's') && (data[++index] == 't')
+ && (data[++index] == 'r') && (data[++index] == 'i')
+ && (data[++index] == 'n') && (data[++index] == 'g')) {
+ found = true;
+ tempToken = TokenNamestringCAST;
+ } else {
+ index = 0;
+ if ((data[index] == 'd') && (data[++index] == 'o')
+ && (data[++index] == 'u') && (data[++index] == 'b')
+ && (data[++index] == 'l') && (data[++index] == 'e')) {
+ found = true;
+ tempToken = TokenNamedoubleCAST;
+ }
+ }
+ }
+ break;
+ case 7 :
+ // boolean integer
+ if ((data[index] == 'b') && (data[++index] == 'o')
+ && (data[++index] == 'o') && (data[++index] == 'l')
+ && (data[++index] == 'e') && (data[++index] == 'a')
+ && (data[++index] == 'n')) {
+ found = true;
+ tempToken = TokenNameboolCAST;
+ } else {
+ index = 0;
+ if ((data[index] == 'i') && (data[++index] == 'n')
+ && (data[++index] == 't') && (data[++index] == 'e')
+ && (data[++index] == 'g') && (data[++index] == 'e')
+ && (data[++index] == 'r')) {
+ found = true;
+ tempToken = TokenNameintCAST;
+ }
+ }
+ break;
+ }
+ if (found) {
+ while (currentCharacter == ' ' || currentCharacter == '\t') {
+ currentCharacter = source[currentPosition++];
+ }
+ if (currentCharacter == ')') {
+ return tempToken;
}
}
- if ((currentCharacter == '\r') || (currentCharacter == '\n')) {
- if (recordLineSeparator) {
- pushLineSeparator();
- } else {
- currentLine = null;
+ }
+ } catch (IndexOutOfBoundsException e) {
+ }
+ currentCharacter = tempCharacter;
+ currentPosition = tempPosition;
+ return TokenNameLPAREN;
+ }
+ public void consumeStringInterpolated() throws InvalidInputException {
+ try {
+ // consume next character
+ unicodeAsBackSlash = false;
+ currentCharacter = source[currentPosition++];
+ // if (((currentCharacter = source[currentPosition++]) == '\\')
+ // && (source[currentPosition] == 'u')) {
+ // getNextUnicodeChar();
+ // } else {
+ // if (withoutUnicodePtr != 0) {
+ // withoutUnicodeBuffer[++withoutUnicodePtr] =
+ // currentCharacter;
+ // }
+ // }
+ while (currentCharacter != '`') {
+ /** ** in PHP \r and \n are valid in string literals *** */
+ // if ((currentCharacter == '\n')
+ // || (currentCharacter == '\r')) {
+ // // relocate if finding another quote fairly close: thus unicode
+ // '/u000D' will be fully consumed
+ // for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
+ // if (currentPosition + lookAhead == source.length)
+ // break;
+ // if (source[currentPosition + lookAhead] == '\n')
+ // break;
+ // if (source[currentPosition + lookAhead] == '\"') {
+ // currentPosition += lookAhead + 1;
+ // break;
+ // }
+ // }
+ // throw new InvalidInputException(INVALID_CHAR_IN_STRING);
+ // }
+ if (currentCharacter == '\\') {
+ int escapeSize = currentPosition;
+ boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
+ //scanEscapeCharacter make a side effect on this value and we need
+ // the previous value few lines down this one
+ scanDoubleQuotedEscapeCharacter();
+ escapeSize = currentPosition - escapeSize;
+ if (withoutUnicodePtr == 0) {
+ //buffer all the entries that have been left aside....
+ withoutUnicodePtr = currentPosition - escapeSize - 1
+ - startPosition;
+ System.arraycopy(source, startPosition, withoutUnicodeBuffer, 1,
+ withoutUnicodePtr);
+ withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+ } else { //overwrite the / in the buffer
+ withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter;
+ if (backSlashAsUnicodeInString) { //there are TWO \ in the stream
+ // where only one is correct
+ withoutUnicodePtr--;
+ }
}
}
+ // consume next character
+ unicodeAsBackSlash = false;
+ currentCharacter = source[currentPosition++];
+ // if (((currentCharacter = source[currentPosition++]) == '\\')
+ // && (source[currentPosition] == 'u')) {
+ // getNextUnicodeChar();
+ // } else {
+ if (withoutUnicodePtr != 0) {
+ withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+ }
+ // }
}
- } //-----------------end switch while try--------------------
- catch (IndexOutOfBoundsException e) {
- if (tokenizeWhiteSpace) {
- // && (whiteStart != currentPosition - 1)) {
- // reposition scanner in case we are interested by spaces as tokens
- startPosition = htmlPosition;
+ } catch (IndexOutOfBoundsException e) {
+ throw new InvalidInputException(UNTERMINATED_STRING);
+ } catch (InvalidInputException e) {
+ if (e.getMessage().equals(INVALID_ESCAPE)) {
+ // relocate if finding another quote fairly close: thus unicode
+ // '/u000D' will be fully consumed
+ for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
+ if (currentPosition + lookAhead == source.length)
+ break;
+ if (source[currentPosition + lookAhead] == '\n')
+ break;
+ if (source[currentPosition + lookAhead] == '`') {
+ currentPosition += lookAhead + 1;
+ break;
+ }
+ }
}
- return TokenNameEOF;
+ throw e; // rethrow
+ }
+ if (checkNonExternalizedStringLiterals) { // check for presence of NLS tags
+ // //$NON-NLS-?$ where ? is an
+ // int.
+ if (currentLine == null) {
+ currentLine = new NLSLine();
+ lines.add(currentLine);
+ }
+ currentLine.add(new StringLiteral(getCurrentTokenSourceString(),
+ startPosition, currentPosition - 1));
+ }
+ }
+ public void consumeStringConstant() throws InvalidInputException {
+ try {
+ // consume next character
+ unicodeAsBackSlash = false;
+ currentCharacter = source[currentPosition++];
+ // if (((currentCharacter = source[currentPosition++]) == '\\')
+ // && (source[currentPosition] == 'u')) {
+ // getNextUnicodeChar();
+ // } else {
+ // if (withoutUnicodePtr != 0) {
+ // withoutUnicodeBuffer[++withoutUnicodePtr] =
+ // currentCharacter;
+ // }
+ // }
+ while (currentCharacter != '\'') {
+ /** ** in PHP \r and \n are valid in string literals *** */
+ // if ((currentCharacter == '\n')
+ // || (currentCharacter == '\r')) {
+ // // relocate if finding another quote fairly close: thus unicode
+ // '/u000D' will be fully consumed
+ // for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
+ // if (currentPosition + lookAhead == source.length)
+ // break;
+ // if (source[currentPosition + lookAhead] == '\n')
+ // break;
+ // if (source[currentPosition + lookAhead] == '\"') {
+ // currentPosition += lookAhead + 1;
+ // break;
+ // }
+ // }
+ // throw new InvalidInputException(INVALID_CHAR_IN_STRING);
+ // }
+ if (currentCharacter == '\\') {
+ int escapeSize = currentPosition;
+ boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
+ //scanEscapeCharacter make a side effect on this value and we need
+ // the previous value few lines down this one
+ scanSingleQuotedEscapeCharacter();
+ escapeSize = currentPosition - escapeSize;
+ if (withoutUnicodePtr == 0) {
+ //buffer all the entries that have been left aside....
+ withoutUnicodePtr = currentPosition - escapeSize - 1
+ - startPosition;
+ System.arraycopy(source, startPosition, withoutUnicodeBuffer, 1,
+ withoutUnicodePtr);
+ withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+ } else { //overwrite the / in the buffer
+ withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter;
+ if (backSlashAsUnicodeInString) { //there are TWO \ in the stream
+ // where only one is correct
+ withoutUnicodePtr--;
+ }
+ }
+ }
+ // consume next character
+ unicodeAsBackSlash = false;
+ currentCharacter = source[currentPosition++];
+ // if (((currentCharacter = source[currentPosition++]) == '\\')
+ // && (source[currentPosition] == 'u')) {
+ // getNextUnicodeChar();
+ // } else {
+ if (withoutUnicodePtr != 0) {
+ withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+ }
+ // }
+ }
+ } catch (IndexOutOfBoundsException e) {
+ throw new InvalidInputException(UNTERMINATED_STRING);
+ } catch (InvalidInputException e) {
+ if (e.getMessage().equals(INVALID_ESCAPE)) {
+ // relocate if finding another quote fairly close: thus unicode
+ // '/u000D' will be fully consumed
+ for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
+ if (currentPosition + lookAhead == source.length)
+ break;
+ if (source[currentPosition + lookAhead] == '\n')
+ break;
+ if (source[currentPosition + lookAhead] == '\'') {
+ currentPosition += lookAhead + 1;
+ break;
+ }
+ }
+ }
+ throw e; // rethrow
+ }
+ if (checkNonExternalizedStringLiterals) { // check for presence of NLS tags
+ // //$NON-NLS-?$ where ? is an
+ // int.
+ if (currentLine == null) {
+ currentLine = new NLSLine();
+ lines.add(currentLine);
+ }
+ currentLine.add(new StringLiteral(getCurrentTokenSourceString(),
+ startPosition, currentPosition - 1));
+ }
+ }
+ public void consumeStringLiteral() throws InvalidInputException {
+ try {
+ // consume next character
+ unicodeAsBackSlash = false;
+ currentCharacter = source[currentPosition++];
+ // if (((currentCharacter = source[currentPosition++]) == '\\')
+ // && (source[currentPosition] == 'u')) {
+ // getNextUnicodeChar();
+ // } else {
+ // if (withoutUnicodePtr != 0) {
+ // withoutUnicodeBuffer[++withoutUnicodePtr] =
+ // currentCharacter;
+ // }
+ // }
+ while (currentCharacter != '"') {
+ /** ** in PHP \r and \n are valid in string literals *** */
+ // if ((currentCharacter == '\n')
+ // || (currentCharacter == '\r')) {
+ // // relocate if finding another quote fairly close: thus unicode
+ // '/u000D' will be fully consumed
+ // for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
+ // if (currentPosition + lookAhead == source.length)
+ // break;
+ // if (source[currentPosition + lookAhead] == '\n')
+ // break;
+ // if (source[currentPosition + lookAhead] == '\"') {
+ // currentPosition += lookAhead + 1;
+ // break;
+ // }
+ // }
+ // throw new InvalidInputException(INVALID_CHAR_IN_STRING);
+ // }
+ if (currentCharacter == '\\') {
+ int escapeSize = currentPosition;
+ boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
+ //scanEscapeCharacter make a side effect on this value and we need
+ // the previous value few lines down this one
+ scanDoubleQuotedEscapeCharacter();
+ escapeSize = currentPosition - escapeSize;
+ if (withoutUnicodePtr == 0) {
+ //buffer all the entries that have been left aside....
+ withoutUnicodePtr = currentPosition - escapeSize - 1
+ - startPosition;
+ System.arraycopy(source, startPosition, withoutUnicodeBuffer, 1,
+ withoutUnicodePtr);
+ withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+ } else { //overwrite the / in the buffer
+ withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter;
+ if (backSlashAsUnicodeInString) { //there are TWO \ in the stream
+ // where only one is correct
+ withoutUnicodePtr--;
+ }
+ }
+ }
+ // consume next character
+ unicodeAsBackSlash = false;
+ currentCharacter = source[currentPosition++];
+ // if (((currentCharacter = source[currentPosition++]) == '\\')
+ // && (source[currentPosition] == 'u')) {
+ // getNextUnicodeChar();
+ // } else {
+ if (withoutUnicodePtr != 0) {
+ withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+ }
+ // }
+ }
+ } catch (IndexOutOfBoundsException e) {
+ throw new InvalidInputException(UNTERMINATED_STRING);
+ } catch (InvalidInputException e) {
+ if (e.getMessage().equals(INVALID_ESCAPE)) {
+ // relocate if finding another quote fairly close: thus unicode
+ // '/u000D' will be fully consumed
+ for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
+ if (currentPosition + lookAhead == source.length)
+ break;
+ if (source[currentPosition + lookAhead] == '\n')
+ break;
+ if (source[currentPosition + lookAhead] == '\"') {
+ currentPosition += lookAhead + 1;
+ break;
+ }
+ }
+ }
+ throw e; // rethrow
+ }
+ if (checkNonExternalizedStringLiterals) { // check for presence of NLS tags
+ // //$NON-NLS-?$ where ? is an
+ // int.
+ if (currentLine == null) {
+ currentLine = new NLSLine();
+ lines.add(currentLine);
+ }
+ currentLine.add(new StringLiteral(getCurrentTokenSourceString(),
+ startPosition, currentPosition - 1));
+ }
+ }
+ public int getNextToken() throws InvalidInputException {
+ if (!phpMode) {
+ return getInlinedHTML(currentPosition);
}
if (phpMode) {
this.wasAcr = false;
return currentPosition > source.length ? TokenNameEOF : TokenNameRBRACE;
}
try {
- while (true) { //loop for jumping over comments
+ while (true) {
withoutUnicodePtr = 0;
- //start with a new token (even comment written with unicode )
+ //start with a new token
+ char encapsedChar = ' ';
+ if (!encapsedStringStack.isEmpty()) {
+ encapsedChar = ((Character) encapsedStringStack.peek()).charValue();
+ }
+ if (encapsedChar != '$' && encapsedChar != ' ') {
+ currentCharacter = source[currentPosition++];
+ if (currentCharacter == encapsedChar) {
+ switch (currentCharacter) {
+ case '`' :
+ return TokenNameEncapsedString0;
+ case '\'' :
+ return TokenNameEncapsedString1;
+ case '"' :
+ return TokenNameEncapsedString2;
+ }
+ }
+ while (currentCharacter != encapsedChar) {
+ /** ** in PHP \r and \n are valid in string literals *** */
+ switch (currentCharacter) {
+ case '\\' :
+ int escapeSize = currentPosition;
+ boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
+ //scanEscapeCharacter make a side effect on this value and
+ // we need the previous value few lines down this one
+ scanDoubleQuotedEscapeCharacter();
+ escapeSize = currentPosition - escapeSize;
+ if (withoutUnicodePtr == 0) {
+ //buffer all the entries that have been left aside....
+ withoutUnicodePtr = currentPosition - escapeSize - 1
+ - startPosition;
+ System.arraycopy(source, startPosition,
+ withoutUnicodeBuffer, 1, withoutUnicodePtr);
+ withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+ } else { //overwrite the / in the buffer
+ withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter;
+ if (backSlashAsUnicodeInString) { //there are TWO \ in
+ withoutUnicodePtr--;
+ }
+ }
+ break;
+ case '$' :
+ if (isPHPIdentifierStart(source[currentPosition])
+ || source[currentPosition] == '{') {
+ currentPosition--;
+ encapsedStringStack.push(new Character('$'));
+ return TokenNameSTRING;
+ }
+ break;
+ case '{' :
+ if (source[currentPosition] == '$') { // CURLY_OPEN
+ currentPosition--;
+ encapsedStringStack.push(new Character('$'));
+ return TokenNameSTRING;
+ }
+ }
+ // consume next character
+ unicodeAsBackSlash = false;
+ currentCharacter = source[currentPosition++];
+ if (withoutUnicodePtr != 0) {
+ withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+ }
+ // }
+ } // end while
+ currentPosition--;
+ return TokenNameSTRING;
+ }
// ---------Consume white space and handles startPosition---------
int whiteStart = currentPosition;
- boolean isWhiteSpace;
- do {
+ startPosition = currentPosition;
+ currentCharacter = source[currentPosition++];
+ if (encapsedChar == '$') {
+ switch (currentCharacter) {
+ case '\\' :
+ currentCharacter = source[currentPosition++];
+ return TokenNameSTRING;
+ case '{' :
+ if (encapsedChar == '$') {
+ if (getNextChar('$'))
+ return TokenNameCURLY_OPEN;
+ }
+ return TokenNameLBRACE;
+ case '}' :
+ return TokenNameRBRACE;
+ case '[' :
+ return TokenNameLBRACKET;
+ case ']' :
+ return TokenNameRBRACKET;
+ case '\'' :
+ if (tokenizeStrings) {
+ consumeStringConstant();
+ return TokenNameStringConstant;
+ }
+ return TokenNameEncapsedString1;
+ case '"' :
+ return TokenNameEncapsedString2;
+ case '`' :
+ if (tokenizeStrings) {
+ consumeStringInterpolated();
+ return TokenNameStringInterpolated;
+ }
+ return TokenNameEncapsedString0;
+ case '-' :
+ if (getNextChar('>'))
+ return TokenNameMINUS_GREATER;
+ return TokenNameSTRING;
+ default :
+ if (currentCharacter == '$') {
+ int oldPosition = currentPosition;
+ try {
+ currentCharacter = source[currentPosition++];
+ if (currentCharacter == '{') {
+ return TokenNameDOLLAR_LBRACE;
+ }
+ if (isPHPIdentifierStart(currentCharacter)) {
+ return scanIdentifierOrKeyword(true);
+ } else {
+ currentPosition = oldPosition;
+ return TokenNameSTRING;
+ }
+ } catch (IndexOutOfBoundsException e) {
+ currentPosition = oldPosition;
+ return TokenNameSTRING;
+ }
+ }
+ if (isPHPIdentifierStart(currentCharacter))
+ return scanIdentifierOrKeyword(false);
+ if (Character.isDigit(currentCharacter))
+ return scanNumber(false);
+ return TokenNameERROR;
+ }
+ }
+ // boolean isWhiteSpace;
+
+ while ((currentCharacter == ' ')
+ || Character.isWhitespace(currentCharacter)) {
startPosition = currentPosition;
currentCharacter = source[currentPosition++];
// if (((currentCharacter = source[currentPosition++]) == '\\')
currentLine = null;
}
}
- isWhiteSpace = (currentCharacter == ' ')
- || Character.isWhitespace(currentCharacter);
+ // isWhiteSpace = (currentCharacter == ' ')
+ // || Character.isWhitespace(currentCharacter);
// }
- } while (isWhiteSpace);
+ }
if (tokenizeWhiteSpace && (whiteStart != currentPosition - 1)) {
// reposition scanner in case we are interested by spaces as tokens
currentPosition--;
// ---------Identify the next token-------------
switch (currentCharacter) {
case '(' :
- return TokenNameLPAREN;
+ return getCastOrParen();
case ')' :
return TokenNameRPAREN;
case '{' :
case ',' :
return TokenNameCOMMA;
case '.' :
+ if (getNextChar('='))
+ return TokenNameDOT_EQUAL;
if (getNextCharAsDigit())
return scanNumber(true);
return TokenNameDOT;
return TokenNameREMAINDER;
case '<' :
{
- int test;
- if ((test = getNextChar('=', '<')) == 0)
- return TokenNameLESS_EQUAL;
- if (test > 0) {
- if (getNextChar('='))
- return TokenNameLEFT_SHIFT_EQUAL;
- if (getNextChar('<')) {
- int heredocStart = currentPosition;
- int heredocLength = 0;
- currentCharacter = source[currentPosition++];
- if (isPHPIdentifierStart(currentCharacter)) {
- currentCharacter = source[currentPosition++];
- } else {
- return TokenNameERROR;
- }
- while (isPHPIdentifierPart(currentCharacter)) {
+ int oldPosition = currentPosition;
+ try {
+ currentCharacter = source[currentPosition++];
+ } catch (IndexOutOfBoundsException e) {
+ currentPosition = oldPosition;
+ return TokenNameLESS;
+ }
+ switch (currentCharacter) {
+ case '=' :
+ return TokenNameLESS_EQUAL;
+ case '>' :
+ return TokenNameNOT_EQUAL;
+ case '<' :
+ if (getNextChar('='))
+ return TokenNameLEFT_SHIFT_EQUAL;
+ if (getNextChar('<')) {
currentCharacter = source[currentPosition++];
- }
- heredocLength = currentPosition - heredocStart - 1;
- // heredoc end-tag determination
- boolean endTag = true;
- char ch;
- do {
- ch = source[currentPosition++];
- if (ch == '\r' || ch == '\n') {
- if (recordLineSeparator) {
- pushLineSeparator();
- } else {
- currentLine = null;
- }
- for (int i = 0; i < heredocLength; i++) {
- if (source[currentPosition + i] != source[heredocStart
- + i]) {
- endTag = false;
- break;
+ while (Character.isWhitespace(currentCharacter)) {
+ currentCharacter = source[currentPosition++];
+ }
+ int heredocStart = currentPosition - 1;
+ int heredocLength = 0;
+ if (isPHPIdentifierStart(currentCharacter)) {
+ currentCharacter = source[currentPosition++];
+ } else {
+ return TokenNameERROR;
+ }
+ while (isPHPIdentifierPart(currentCharacter)) {
+ currentCharacter = source[currentPosition++];
+ }
+ heredocLength = currentPosition - heredocStart - 1;
+ // heredoc end-tag determination
+ boolean endTag = true;
+ char ch;
+ do {
+ ch = source[currentPosition++];
+ if (ch == '\r' || ch == '\n') {
+ if (recordLineSeparator) {
+ pushLineSeparator();
+ } else {
+ currentLine = null;
+ }
+ for (int i = 0; i < heredocLength; i++) {
+ if (source[currentPosition + i] != source[heredocStart
+ + i]) {
+ endTag = false;
+ break;
+ }
+ }
+ if (endTag) {
+ currentPosition += heredocLength - 1;
+ currentCharacter = source[currentPosition++];
+ break; // do...while loop
+ } else {
+ endTag = true;
}
}
- if (endTag) {
- currentPosition += heredocLength - 1;
- currentCharacter = source[currentPosition++];
- break; // do...while loop
- } else {
- endTag = true;
- }
- }
- } while (true);
- return TokenNameHEREDOC;
- }
- return TokenNameLEFT_SHIFT;
+ } while (true);
+ return TokenNameHEREDOC;
+ }
+ return TokenNameLEFT_SHIFT;
}
+ currentPosition = oldPosition;
return TokenNameLESS;
}
case '>' :
case '?' :
if (getNextChar('>')) {
phpMode = false;
- return TokenNameStopPHP;
+ if (currentPosition == source.length) {
+ phpMode = true;
+ return TokenNameINLINE_HTML;
+ }
+ return getInlinedHTML(currentPosition - 2);
}
return TokenNameQUESTION;
case ':' :
return TokenNameCOLON;
case '@' :
return TokenNameAT;
- // case '\'' :
- // {
- // int test;
- // if ((test = getNextChar('\n', '\r')) == 0) {
- // throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
- // }
- // if (test > 0) {
- // // relocate if finding another quote fairly close: thus unicode
- // '/u000D' will be fully consumed
- // for (int lookAhead = 0;
- // lookAhead < 3;
- // lookAhead++) {
- // if (currentPosition + lookAhead
- // == source.length)
- // break;
- // if (source[currentPosition + lookAhead]
- // == '\n')
- // break;
- // if (source[currentPosition + lookAhead]
- // == '\'') {
- // currentPosition += lookAhead + 1;
- // break;
- // }
- // }
- // throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
- // }
- // }
- // if (getNextChar('\'')) {
- // // relocate if finding another quote fairly close: thus unicode
- // '/u000D' will be fully consumed
- // for (int lookAhead = 0;
- // lookAhead < 3;
- // lookAhead++) {
- // if (currentPosition + lookAhead
- // == source.length)
- // break;
- // if (source[currentPosition + lookAhead]
- // == '\n')
- // break;
- // if (source[currentPosition + lookAhead]
- // == '\'') {
- // currentPosition += lookAhead + 1;
- // break;
- // }
- // }
- // throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
- // }
- // if (getNextChar('\\'))
- // scanEscapeCharacter();
- // else { // consume next character
- // unicodeAsBackSlash = false;
- // if (((currentCharacter = source[currentPosition++])
- // == '\\')
- // && (source[currentPosition] == 'u')) {
- // getNextUnicodeChar();
- // } else {
- // if (withoutUnicodePtr != 0) {
- // withoutUnicodeBuffer[++withoutUnicodePtr] =
- // currentCharacter;
- // }
- // }
- // }
- // // if (getNextChar('\''))
- // // return TokenNameCharacterLiteral;
- // // relocate if finding another quote fairly close: thus unicode
- // '/u000D' will be fully consumed
- // for (int lookAhead = 0; lookAhead < 20; lookAhead++) {
- // if (currentPosition + lookAhead == source.length)
- // break;
- // if (source[currentPosition + lookAhead] == '\n')
- // break;
- // if (source[currentPosition + lookAhead] == '\'') {
- // currentPosition += lookAhead + 1;
- // break;
- // }
- // }
- // throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
case '\'' :
- try {
- // consume next character
- unicodeAsBackSlash = false;
- currentCharacter = source[currentPosition++];
- // if (((currentCharacter = source[currentPosition++]) == '\\')
- // && (source[currentPosition] == 'u')) {
- // getNextUnicodeChar();
- // } else {
- // if (withoutUnicodePtr != 0) {
- // withoutUnicodeBuffer[++withoutUnicodePtr] =
- // currentCharacter;
- // }
- // }
- while (currentCharacter != '\'') {
- /** ** in PHP \r and \n are valid in string literals *** */
- // if ((currentCharacter == '\n')
- // || (currentCharacter == '\r')) {
- // // relocate if finding another quote fairly close: thus
- // unicode '/u000D' will be fully consumed
- // for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
- // if (currentPosition + lookAhead == source.length)
- // break;
- // if (source[currentPosition + lookAhead] == '\n')
- // break;
- // if (source[currentPosition + lookAhead] == '\"') {
- // currentPosition += lookAhead + 1;
- // break;
- // }
- // }
- // throw new InvalidInputException(INVALID_CHAR_IN_STRING);
- // }
- if (currentCharacter == '\\') {
- int escapeSize = currentPosition;
- boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
- //scanEscapeCharacter make a side effect on this value and
- // we need the previous value few lines down this one
- scanSingleQuotedEscapeCharacter();
- escapeSize = currentPosition - escapeSize;
- if (withoutUnicodePtr == 0) {
- //buffer all the entries that have been left aside....
- withoutUnicodePtr = currentPosition - escapeSize - 1
- - startPosition;
- System.arraycopy(source, startPosition,
- withoutUnicodeBuffer, 1, withoutUnicodePtr);
- withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
- } else { //overwrite the / in the buffer
- withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter;
- if (backSlashAsUnicodeInString) { //there are TWO \ in
- // the stream where
- // only one is correct
- withoutUnicodePtr--;
- }
- }
- }
- // consume next character
- unicodeAsBackSlash = false;
- currentCharacter = source[currentPosition++];
- // if (((currentCharacter = source[currentPosition++]) ==
- // '\\')
- // && (source[currentPosition] == 'u')) {
- // getNextUnicodeChar();
- // } else {
- if (withoutUnicodePtr != 0) {
- withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
- }
- // }
- }
- } catch (IndexOutOfBoundsException e) {
- throw new InvalidInputException(UNTERMINATED_STRING);
- } catch (InvalidInputException e) {
- if (e.getMessage().equals(INVALID_ESCAPE)) {
- // relocate if finding another quote fairly close: thus
- // unicode '/u000D' will be fully consumed
- for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
- if (currentPosition + lookAhead == source.length)
- break;
- if (source[currentPosition + lookAhead] == '\n')
- break;
- if (source[currentPosition + lookAhead] == '\'') {
- currentPosition += lookAhead + 1;
- break;
- }
- }
- }
- throw e; // rethrow
- }
- if (checkNonExternalizedStringLiterals) { // check for presence
- // of NLS tags
- // //$NON-NLS-?$ where
- // ? is an int.
- if (currentLine == null) {
- currentLine = new NLSLine();
- lines.add(currentLine);
- }
- currentLine.add(new StringLiteral(
- getCurrentTokenSourceString(), startPosition,
- currentPosition - 1));
- }
+ consumeStringConstant();
return TokenNameStringConstant;
case '"' :
- try {
- // consume next character
- unicodeAsBackSlash = false;
- currentCharacter = source[currentPosition++];
- // if (((currentCharacter = source[currentPosition++]) == '\\')
- // && (source[currentPosition] == 'u')) {
- // getNextUnicodeChar();
- // } else {
- // if (withoutUnicodePtr != 0) {
- // withoutUnicodeBuffer[++withoutUnicodePtr] =
- // currentCharacter;
- // }
- // }
- while (currentCharacter != '"') {
- /** ** in PHP \r and \n are valid in string literals *** */
- // if ((currentCharacter == '\n')
- // || (currentCharacter == '\r')) {
- // // relocate if finding another quote fairly close: thus
- // unicode '/u000D' will be fully consumed
- // for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
- // if (currentPosition + lookAhead == source.length)
- // break;
- // if (source[currentPosition + lookAhead] == '\n')
- // break;
- // if (source[currentPosition + lookAhead] == '\"') {
- // currentPosition += lookAhead + 1;
- // break;
- // }
- // }
- // throw new InvalidInputException(INVALID_CHAR_IN_STRING);
- // }
- if (currentCharacter == '\\') {
- int escapeSize = currentPosition;
- boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
- //scanEscapeCharacter make a side effect on this value and
- // we need the previous value few lines down this one
- scanDoubleQuotedEscapeCharacter();
- escapeSize = currentPosition - escapeSize;
- if (withoutUnicodePtr == 0) {
- //buffer all the entries that have been left aside....
- withoutUnicodePtr = currentPosition - escapeSize - 1
- - startPosition;
- System.arraycopy(source, startPosition,
- withoutUnicodeBuffer, 1, withoutUnicodePtr);
- withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
- } else { //overwrite the / in the buffer
- withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter;
- if (backSlashAsUnicodeInString) { //there are TWO \ in
- // the stream where
- // only one is correct
- withoutUnicodePtr--;
- }
- }
- }
- // consume next character
- unicodeAsBackSlash = false;
- currentCharacter = source[currentPosition++];
- // if (((currentCharacter = source[currentPosition++]) ==
- // '\\')
- // && (source[currentPosition] == 'u')) {
- // getNextUnicodeChar();
- // } else {
- if (withoutUnicodePtr != 0) {
- withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
- }
- // }
- }
- } catch (IndexOutOfBoundsException e) {
- throw new InvalidInputException(UNTERMINATED_STRING);
- } catch (InvalidInputException e) {
- if (e.getMessage().equals(INVALID_ESCAPE)) {
- // relocate if finding another quote fairly close: thus
- // unicode '/u000D' will be fully consumed
- for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
- if (currentPosition + lookAhead == source.length)
- break;
- if (source[currentPosition + lookAhead] == '\n')
- break;
- if (source[currentPosition + lookAhead] == '\"') {
- currentPosition += lookAhead + 1;
- break;
- }
- }
- }
- throw e; // rethrow
+ if (tokenizeStrings) {
+ consumeStringLiteral();
+ return TokenNameStringLiteral;
}
- if (checkNonExternalizedStringLiterals) { // check for presence
- // of NLS tags
- // //$NON-NLS-?$ where
- // ? is an int.
- if (currentLine == null) {
- currentLine = new NLSLine();
- lines.add(currentLine);
- }
- currentLine.add(new StringLiteral(
- getCurrentTokenSourceString(), startPosition,
- currentPosition - 1));
- }
- return TokenNameStringLiteral;
+ return TokenNameEncapsedString2;
case '`' :
- try {
- // consume next character
- unicodeAsBackSlash = false;
- currentCharacter = source[currentPosition++];
- // if (((currentCharacter = source[currentPosition++]) == '\\')
- // && (source[currentPosition] == 'u')) {
- // getNextUnicodeChar();
- // } else {
- // if (withoutUnicodePtr != 0) {
- // withoutUnicodeBuffer[++withoutUnicodePtr] =
- // currentCharacter;
- // }
- // }
- while (currentCharacter != '`') {
- /** ** in PHP \r and \n are valid in string literals *** */
- // if ((currentCharacter == '\n')
- // || (currentCharacter == '\r')) {
- // // relocate if finding another quote fairly close: thus
- // unicode '/u000D' will be fully consumed
- // for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
- // if (currentPosition + lookAhead == source.length)
- // break;
- // if (source[currentPosition + lookAhead] == '\n')
- // break;
- // if (source[currentPosition + lookAhead] == '\"') {
- // currentPosition += lookAhead + 1;
- // break;
- // }
- // }
- // throw new InvalidInputException(INVALID_CHAR_IN_STRING);
- // }
- if (currentCharacter == '\\') {
- int escapeSize = currentPosition;
- boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
- //scanEscapeCharacter make a side effect on this value and
- // we need the previous value few lines down this one
- scanDoubleQuotedEscapeCharacter();
- escapeSize = currentPosition - escapeSize;
- if (withoutUnicodePtr == 0) {
- //buffer all the entries that have been left aside....
- withoutUnicodePtr = currentPosition - escapeSize - 1
- - startPosition;
- System.arraycopy(source, startPosition,
- withoutUnicodeBuffer, 1, withoutUnicodePtr);
- withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
- } else { //overwrite the / in the buffer
- withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter;
- if (backSlashAsUnicodeInString) { //there are TWO \ in
- // the stream where
- // only one is correct
- withoutUnicodePtr--;
- }
- }
- }
- // consume next character
- unicodeAsBackSlash = false;
- currentCharacter = source[currentPosition++];
- // if (((currentCharacter = source[currentPosition++]) ==
- // '\\')
- // && (source[currentPosition] == 'u')) {
- // getNextUnicodeChar();
- // } else {
- if (withoutUnicodePtr != 0) {
- withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
- }
- // }
- }
- } catch (IndexOutOfBoundsException e) {
- throw new InvalidInputException(UNTERMINATED_STRING);
- } catch (InvalidInputException e) {
- if (e.getMessage().equals(INVALID_ESCAPE)) {
- // relocate if finding another quote fairly close: thus
- // unicode '/u000D' will be fully consumed
- for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
- if (currentPosition + lookAhead == source.length)
- break;
- if (source[currentPosition + lookAhead] == '\n')
- break;
- if (source[currentPosition + lookAhead] == '`') {
- currentPosition += lookAhead + 1;
- break;
- }
- }
- }
- throw e; // rethrow
+ if (tokenizeStrings) {
+ consumeStringInterpolated();
+ return TokenNameStringInterpolated;
}
- if (checkNonExternalizedStringLiterals) { // check for presence
- // of NLS tags
- // //$NON-NLS-?$ where
- // ? is an int.
- if (currentLine == null) {
- currentLine = new NLSLine();
- lines.add(currentLine);
- }
- currentLine.add(new StringLiteral(
- getCurrentTokenSourceString(), startPosition,
- currentPosition - 1));
- }
- return TokenNameStringInterpolated;
+ return TokenNameEncapsedString0;
case '#' :
case '/' :
{
+ char startChar = currentCharacter;
+ if (getNextChar('=')) {
+ return TokenNameDIVIDE_EQUAL;
+ }
int test;
- if ((currentCharacter == '#')
- || (test = getNextChar('/', '*')) == 0) {
+ if ((startChar == '#') || (test = getNextChar('/', '*')) == 0) {
//line comment
int endPositionForLineComment = 0;
try { //get the next char
// Character.getNumericValue(source[currentPosition++]))
// > 15
// || c4 < 0) {
- // throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
+ // throw new
+ // InvalidInputException(INVALID_UNICODE_ESCAPE);
// } else {
// currentCharacter =
// (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
if (getNextChar('>')) {
startPosition = currentPosition - 2;
phpMode = false;
- return TokenNameStopPHP;
+ return TokenNameINLINE_HTML;
}
}
//get the next char
endPositionForLineComment = currentPosition - 1;
}
recordComment(false);
- if ((currentCharacter == '\r')
+ if (this.taskTags != null) checkTaskTag(this.startPosition, this.currentPosition);
+ if ((currentCharacter == '\r')
|| (currentCharacter == '\n')) {
checkNonExternalizeString();
if (recordLineSeparator) {
}
break;
}
- if (getNextChar('='))
- return TokenNameDIVIDE_EQUAL;
return TokenNameDIVIDE;
}
case '\u001a' :
throw new InvalidInputException("Ctrl-Z"); //$NON-NLS-1$
default :
if (currentCharacter == '$') {
- while ((currentCharacter = source[currentPosition++]) == '$') {
+ int oldPosition = currentPosition;
+ try {
+ currentCharacter = source[currentPosition++];
+ if (isPHPIdentifierStart(currentCharacter)) {
+ return scanIdentifierOrKeyword(true);
+ } else {
+ currentPosition = oldPosition;
+ return TokenNameDOLLAR;
+ }
+ } catch (IndexOutOfBoundsException e) {
+ currentPosition = oldPosition;
+ return TokenNameDOLLAR;
}
- if (currentCharacter == '{')
- return TokenNameDOLLAR_LBRACE;
- if (isPHPIdentifierStart(currentCharacter))
- return scanIdentifierOrKeyword(true);
- return TokenNameERROR;
}
if (isPHPIdentifierStart(currentCharacter))
return scanIdentifierOrKeyword(false);
}
return TokenNameEOF;
}
+ /**
+ * @return @throws
+ * InvalidInputException
+ */
+ private int getInlinedHTML(int start) throws InvalidInputException {
+ // int htmlPosition = start;
+ if (currentPosition > source.length) {
+ currentPosition = source.length;
+ return TokenNameEOF;
+ }
+ startPosition = start;
+ try {
+ while (!phpMode) {
+ currentCharacter = source[currentPosition++];
+ if (currentCharacter == '<') {
+ if (getNextChar('?')) {
+ currentCharacter = source[currentPosition++];
+ if ((currentCharacter == ' ')
+ || Character.isWhitespace(currentCharacter)) {
+ // <?
+ phpMode = true;
+ return TokenNameINLINE_HTML;
+ } else {
+ boolean phpStart = (currentCharacter == 'P')
+ || (currentCharacter == 'p');
+ if (phpStart) {
+ int test = getNextChar('H', 'h');
+ if (test >= 0) {
+ test = getNextChar('P', 'p');
+ if (test >= 0) {
+ // <?PHP <?php
+ phpMode = true;
+ return TokenNameINLINE_HTML;
+ }
+ }
+ }
+ }
+ }
+ }
+ if ((currentCharacter == '\r') || (currentCharacter == '\n')) {
+ if (recordLineSeparator) {
+ pushLineSeparator();
+ } else {
+ currentLine = null;
+ }
+ }
+ } //-----------------while--------------------
+ phpMode = true;
+ return TokenNameINLINE_HTML;
+ } //-----------------try--------------------
+ catch (IndexOutOfBoundsException e) {
+ startPosition = start;
+ currentPosition--;
+ }
+ phpMode = true;
+ return TokenNameINLINE_HTML;
+ }
// public final void getNextUnicodeChar()
// throws IndexOutOfBoundsException, InvalidInputException {
// //VOID
while (getNextCharAsJavaIdentifierPart()) {
};
if (isVariable) {
- if (new String(getCurrentTokenSource()).equals("$this")) {
- return TokenNamethis;
- }
+ // if (new String(getCurrentTokenSource()).equals("$this")) {
+ // return TokenNamethis;
+ // }
return TokenNameVariable;
}
int index, length;
case 3 :
//and
if ((data[++index] == 'n') && (data[++index] == 'd')) {
- return TokenNameAND;
+ return TokenNameand;
} else {
return TokenNameIdentifier;
}
return TokenNameIdentifier;
}
case 'c' :
- //case catch class const continue
+ //case catch class clone const continue
switch (length) {
case 4 :
if ((data[++index] == 'a') && (data[++index] == 's')
if ((data[++index] == 'a') && (data[++index] == 't')
&& (data[++index] == 'c') && (data[++index] == 'h'))
return TokenNamecatch;
- if ((data[index] == 'l') && (data[++index] == 'a')
+ index = 0;
+ if ((data[++index] == 'l') && (data[++index] == 'a')
&& (data[++index] == 's') && (data[++index] == 's'))
return TokenNameclass;
- if ((data[index] == 'o') && (data[++index] == 'n')
+ index = 0;
+ if ((data[++index] == 'l') && (data[++index] == 'o')
+ && (data[++index] == 'n') && (data[++index] == 'e'))
+ return TokenNameclone;
+ index = 0;
+ if ((data[++index] == 'o') && (data[++index] == 'n')
&& (data[++index] == 's') && (data[++index] == 't'))
return TokenNameconst;
else
else
return TokenNameIdentifier;
case 5 :
-// if ((data[++index] == 'a') && (data[++index] == 'l')
-// && (data[++index] == 's') && (data[++index] == 'e'))
-// return TokenNamefalse;
+ // if ((data[++index] == 'a') && (data[++index] == 'l')
+ // && (data[++index] == 's') && (data[++index] == 'e'))
+ // return TokenNamefalse;
if ((data[++index] == 'i') && (data[++index] == 'n')
&& (data[++index] == 'a') && (data[++index] == 'l'))
return TokenNamefinal;
return TokenNamenew;
else
return TokenNameIdentifier;
-// case 4 :
-// if ((data[++index] == 'u') && (data[++index] == 'l')
-// && (data[++index] == 'l'))
-// return TokenNamenull;
-// else
-// return TokenNameIdentifier;
+ // case 4 :
+ // if ((data[++index] == 'u') && (data[++index] == 'l')
+ // && (data[++index] == 'l'))
+ // return TokenNamenull;
+ // else
+ // return TokenNameIdentifier;
default :
return TokenNameIdentifier;
}
// or old_function
if (length == 2) {
if (data[++index] == 'r') {
- return TokenNameOR;
+ return TokenNameor;
}
}
// if (length == 12) {
return TokenNametry;
else
return TokenNameIdentifier;
-// case 4 :
-// if ((data[++index] == 'r') && (data[++index] == 'u')
-// && (data[++index] == 'e'))
-// return TokenNametrue;
-// else
-// return TokenNameIdentifier;
+ // case 4 :
+ // if ((data[++index] == 'r') && (data[++index] == 'u')
+ // && (data[++index] == 'e'))
+ // return TokenNametrue;
+ // else
+ // return TokenNameIdentifier;
case 5 :
if ((data[++index] == 'h') && (data[++index] == 'r')
&& (data[++index] == 'o') && (data[++index] == 'w'))
switch (length) {
case 3 :
if ((data[++index] == 'o') && (data[++index] == 'r'))
- return TokenNameXOR;
+ return TokenNamexor;
else
return TokenNameIdentifier;
default :
initialPosition = currentPosition = 0;
containsAssertKeyword = false;
withoutUnicodeBuffer = new char[this.source.length];
+ encapsedStringStack = new Stack();
}
public String toString() {
if (startPosition == source.length)
case TokenNameERROR :
return "ScannerError"; // + new String(getCurrentTokenSource()) + ")";
// //$NON-NLS-1$
- case TokenNameStopPHP :
- return "StopPHP(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
+ case TokenNameINLINE_HTML :
+ return "Inline-HTML(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
case TokenNameIdentifier :
return "Identifier(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
case TokenNameVariable :
return "Variable(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
case TokenNameabstract :
return "abstract"; //$NON-NLS-1$
+ case TokenNameand :
+ return "AND"; //$NON-NLS-1$
case TokenNamearray :
return "array"; //$NON-NLS-1$
case TokenNameas :
return "case"; //$NON-NLS-1$
case TokenNameclass :
return "class"; //$NON-NLS-1$
+ case TokenNamecatch :
+ return "catch"; //$NON-NLS-1$
+ case TokenNameclone :
+ //$NON-NLS-1$
+ return "clone";
+ case TokenNameconst :
+ //$NON-NLS-1$
+ return "const";
case TokenNamecontinue :
return "continue"; //$NON-NLS-1$
case TokenNamedefault :
return "endswitch"; //$NON-NLS-1$
case TokenNameendwhile :
return "endwhile"; //$NON-NLS-1$
+ case TokenNameexit:
+ return "exit";
case TokenNameextends :
return "extends"; //$NON-NLS-1$
-// case TokenNamefalse :
-// return "false"; //$NON-NLS-1$
+ // case TokenNamefalse :
+ // return "false"; //$NON-NLS-1$
case TokenNamefinal :
return "final"; //$NON-NLS-1$
case TokenNamefor :
return "include"; //$NON-NLS-1$
case TokenNameinclude_once :
return "include_once"; //$NON-NLS-1$
+ case TokenNameinstanceof :
+ return "instanceof"; //$NON-NLS-1$
case TokenNameinterface :
return "interface"; //$NON-NLS-1$
+ case TokenNameisset :
+ return "isset"; //$NON-NLS-1$
case TokenNamelist :
return "list"; //$NON-NLS-1$
case TokenNamenew :
return "new"; //$NON-NLS-1$
-// case TokenNamenull :
-// return "null"; //$NON-NLS-1$
+ // case TokenNamenull :
+ // return "null"; //$NON-NLS-1$
+ case TokenNameor :
+ return "OR"; //$NON-NLS-1$
case TokenNameprint :
return "print"; //$NON-NLS-1$
case TokenNameprivate :
return "static"; //$NON-NLS-1$
case TokenNameswitch :
return "switch"; //$NON-NLS-1$
-// case TokenNametrue :
-// return "true"; //$NON-NLS-1$
+ // case TokenNametrue :
+ // return "true"; //$NON-NLS-1$
case TokenNameunset :
return "unset"; //$NON-NLS-1$
case TokenNamevar :
return "var"; //$NON-NLS-1$
case TokenNamewhile :
return "while"; //$NON-NLS-1$
- case TokenNamethis :
- return "$this"; //$NON-NLS-1$
+ case TokenNamexor :
+ return "XOR"; //$NON-NLS-1$
+ // case TokenNamethis :
+ // return "$this"; //$NON-NLS-1$
case TokenNameIntegerLiteral :
return "Integer(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
case TokenNameDoubleLiteral :
return "Double(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
case TokenNameStringLiteral :
- return "String(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
+ return "StringLiteral(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
case TokenNameStringConstant :
return "StringConstant(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
case TokenNameStringInterpolated :
return "StringInterpolated(" + new String(getCurrentTokenSource())
+ ")"; //$NON-NLS-1$ //$NON-NLS-2$
+ case TokenNameEncapsedString0 :
+ return "`"; //$NON-NLS-1$
+ case TokenNameEncapsedString1 :
+ return "\'"; //$NON-NLS-1$
+ case TokenNameEncapsedString2 :
+ return "\""; //$NON-NLS-1$
+ case TokenNameSTRING :
+ return "STRING(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
case TokenNameHEREDOC :
return "HEREDOC(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$
case TokenNamePLUS_PLUS :
return "^="; //$NON-NLS-1$
case TokenNameREMAINDER_EQUAL :
return "%="; //$NON-NLS-1$
+ case TokenNameDOT_EQUAL :
+ return ".="; //$NON-NLS-1$
case TokenNameLEFT_SHIFT_EQUAL :
return "<<="; //$NON-NLS-1$
case TokenNameRIGHT_SHIFT_EQUAL :
return "="; //$NON-NLS-1$
case TokenNameAT :
return "@";
+ case TokenNameDOLLAR :
+ return "$";
case TokenNameDOLLAR_LBRACE :
return "${";
case TokenNameEOF :
return "COMMENT_BLOCK(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$
case TokenNameCOMMENT_PHPDOC :
return "COMMENT_PHPDOC(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$
- case TokenNameHTML :
- return "HTML(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$
+ // case TokenNameHTML :
+ // return "HTML(" + new String(getCurrentTokenSource()) + ")";
+ // //$NON-NLS-1$
case TokenNameFILE :
return "__FILE__"; //$NON-NLS-1$
case TokenNameLINE :
case TokenNameMETHOD_C :
return "__METHOD__"; //$NON-NLS-1$
case TokenNameFUNC_C :
- return "__FUNCTION__"; //$NON-NLS-1$
+ return "__FUNCTION__"; //$NON-NLS-1
+ case TokenNameboolCAST :
+ return "( bool )"; //$NON-NLS-1$
+ case TokenNameintCAST :
+ return "( int )"; //$NON-NLS-1$
+ case TokenNamedoubleCAST :
+ return "( double )"; //$NON-NLS-1$
+ case TokenNameobjectCAST :
+ return "( object )"; //$NON-NLS-1$
+ case TokenNamestringCAST :
+ return "( string )"; //$NON-NLS-1$
default :
return "not-a-token(" + (new Integer(act)) + ") "
+ new String(getCurrentTokenSource()); //$NON-NLS-1$
}
}
+
+ public Scanner() {
+ this(false, false);
+ }
+ public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace) {
+ this(tokenizeComments, tokenizeWhiteSpace, false);
+ }
public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace,
boolean checkNonExternalizedStringLiterals) {
this(tokenizeComments, tokenizeWhiteSpace,
}
public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace,
boolean checkNonExternalizedStringLiterals, boolean assertMode) {
+ this(tokenizeComments, tokenizeWhiteSpace,
+ checkNonExternalizedStringLiterals, assertMode, false, null, null);
+ }
+ public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace,
+ boolean checkNonExternalizedStringLiterals, boolean assertMode,
+ boolean tokenizeStrings,
+ char[][] taskTags,
+ char[][] taskPriorities) {
this.eofPosition = Integer.MAX_VALUE;
this.tokenizeComments = tokenizeComments;
this.tokenizeWhiteSpace = tokenizeWhiteSpace;
+ this.tokenizeStrings = tokenizeStrings;
this.checkNonExternalizedStringLiterals = checkNonExternalizedStringLiterals;
this.assertMode = assertMode;
+ this.encapsedStringStack = null;
+ this.taskTags = taskTags;
+ this.taskPriorities = taskPriorities;
}
private void checkNonExternalizeString() throws InvalidInputException {
if (currentLine == null)
git://secure.phpeclipse.com / phpeclipse.git / blobdiff