bug 1434118, scanner had a faulty behavior scanning variable names like 'retur*'
[phpeclipse.git] / net.sourceforge.phpeclipse / src / net / sourceforge / phpdt / internal / compiler / parser / Scanner.java
index e4ed1ce..b2ae4c6 100644 (file)
@@ -27,8 +27,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
         * stream - currentPosition-1 gives the sourceEnd position into the stream
         */
        // 1.4 feature
-//     private boolean assertMode;
-
+       // private boolean assertMode;
        public boolean useAssertAsAnIndentifier = false;
 
        // flag indicating if processed source contains occurrences of keyword assert
@@ -40,9 +39,13 @@ public class Scanner implements IScanner, ITerminalSymbols {
 
        public boolean phpMode = false;
 
-       public boolean phpExpressionTag = false;
-
-       // public Stack encapsedStringStack = null;
+       /**
+        * This token is set to TokenNameecho if a short tag block begins (i.e. >?= ... )
+        * Directly after the "=" character the getNextToken() method returns TokenNameINLINE_HTML
+        * In the next call to the getNextToken() method the value of fFillerToken (==TokenNameecho) is returned
+        *
+        */
+       int fFillerToken = TokenNameEOF;
 
        public char currentCharacter;
 
@@ -199,6 +202,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
                ObviousIdentCharNatures['"'] = C_SEPARATOR;
                ObviousIdentCharNatures['\''] = C_SEPARATOR;
        }
+
        static final char[] initCharArray = new char[] { '\u0000', '\u0000', '\u0000', '\u0000', '\u0000', '\u0000' };
 
        static final int TableSize = 30, InternalTableSize = 6;
@@ -285,10 +289,11 @@ public class Scanner implements IScanner, ITerminalSymbols {
         */
        public static boolean isPHPIdentOrVarStart(char ch) {
                if (ch < MAX_OBVIOUS) {
-                       return ObviousIdentCharNatures[ch]==C_LETTER || ObviousIdentCharNatures[ch]==C_DOLLAR;
+                       return ObviousIdentCharNatures[ch] == C_LETTER || ObviousIdentCharNatures[ch] == C_DOLLAR;
                }
                return false;
-               //return Character.isLetter(ch) || (ch == '$') || (ch == '_') || (0x7F <= ch && ch <= 0xFF);
+               // return Character.isLetter(ch) || (ch == '$') || (ch == '_') || (0x7F <=
+               // ch && ch <= 0xFF);
        }
 
        /**
@@ -299,10 +304,11 @@ public class Scanner implements IScanner, ITerminalSymbols {
         */
        public static boolean isPHPIdentifierStart(char ch) {
                if (ch < MAX_OBVIOUS) {
-                       return ObviousIdentCharNatures[ch]==C_LETTER;
+                       return ObviousIdentCharNatures[ch] == C_LETTER;
                }
                return false;
-//             return Character.isLetter(ch) || (ch == '_') || (0x7F <= ch && ch <= 0xFF);
+               // return Character.isLetter(ch) || (ch == '_') || (0x7F <= ch && ch <=
+               // 0xFF);
        }
 
        /**
@@ -311,15 +317,16 @@ public class Scanner implements IScanner, ITerminalSymbols {
         */
        public static boolean isPHPIdentifierPart(char ch) {
                if (ch < MAX_OBVIOUS) {
-                       return ObviousIdentCharNatures[ch]==C_LETTER || ObviousIdentCharNatures[ch]==C_DIGIT;
+                       return ObviousIdentCharNatures[ch] == C_LETTER || ObviousIdentCharNatures[ch] == C_DIGIT;
                }
                return false;
-//             return Character.isLetterOrDigit(ch) || (ch == '_') || (0x7F <= ch && ch <= 0xFF);
+               // return Character.isLetterOrDigit(ch) || (ch == '_') || (0x7F <= ch && ch
+               // <= 0xFF);
        }
 
        public static boolean isSQLIdentifierPart(char ch) {
                if (ch < MAX_OBVIOUS) {
-                       return ObviousIdentCharNatures[ch]==C_LETTER || ObviousIdentCharNatures[ch]==C_DIGIT;
+                       return ObviousIdentCharNatures[ch] == C_LETTER || ObviousIdentCharNatures[ch] == C_DIGIT;
                }
                return false;
        }
@@ -419,6 +426,18 @@ public class Scanner implements IScanner, ITerminalSymbols {
                return result;
        }
 
+       public final boolean equalsCurrentTokenSource(char[] word) {
+               if (word.length != currentPosition - startPosition) {
+                       return false;
+               }
+               for (int i = 0; i < word.length; i++) {
+                       if (word[i] != source[startPosition + i]) {
+                               return false;
+                       }
+               }
+               return true;
+       }
+
        public final char[] getRawTokenSourceEnd() {
                int length = this.eofPosition - this.currentPosition - 1;
                char[] sourceEnd = new char[length];
@@ -889,8 +908,9 @@ public class Scanner implements IScanner, ITerminalSymbols {
                        do {
                                currentCharacter = source[currentPosition++];
                        } while (currentCharacter == ' ' || currentCharacter == '\t');
-                       while (ObviousIdentCharNatures[currentCharacter]==C_LETTER) {
-                               //      while((currentCharacter >= 'a' && currentCharacter <= 'z') || (currentCharacter >= 'A' && currentCharacter <= 'Z')) {
+                       while (ObviousIdentCharNatures[currentCharacter] == C_LETTER) {
+                               // while((currentCharacter >= 'a' && currentCharacter <= 'z') ||
+                               // (currentCharacter >= 'A' && currentCharacter <= 'Z')) {
                                buf.append(currentCharacter);
                                currentCharacter = source[currentPosition++];
                        }
@@ -1196,11 +1216,11 @@ public class Scanner implements IScanner, ITerminalSymbols {
 
        public void consumeStringLiteral() throws InvalidInputException {
                try {
-                       boolean openDollarBrace = false;
+                       int openDollarBrace = 0;
                        // consume next character
                        unicodeAsBackSlash = false;
                        currentCharacter = source[currentPosition++];
-                       while (currentCharacter != '"' || openDollarBrace) {
+                       while (currentCharacter != '"' || openDollarBrace>0) {
                                /** ** in PHP \r and \n are valid in string literals *** */
                                if (currentCharacter == '\\') {
                                        int escapeSize = currentPosition;
@@ -1222,11 +1242,11 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                                }
                                        }
                                } else if (currentCharacter == '$' && source[currentPosition] == '{') {
-                                       openDollarBrace = true;
+                                       openDollarBrace++;
                                } else if (currentCharacter == '{' && source[currentPosition] == '$') {
-                                       openDollarBrace = true;
+                                       openDollarBrace++;
                                } else if (currentCharacter == '}') {
-                                       openDollarBrace = false;
+                                       openDollarBrace--;
                                } else if ((currentCharacter == '\r') || (currentCharacter == '\n')) {
                                        if (recordLineSeparator) {
                                                pushLineSeparator();
@@ -1272,11 +1292,16 @@ public class Scanner implements IScanner, ITerminalSymbols {
        }
 
        public int getNextToken() throws InvalidInputException {
-               phpExpressionTag = false;
                if (!phpMode) {
                        return getInlinedHTMLToken(currentPosition);
-               }
-               if (phpMode) {
+               } else {
+                       if (fFillerToken != TokenNameEOF) {
+                               int tempToken;
+                               startPosition = currentPosition;
+                               tempToken = fFillerToken;
+                               fFillerToken = TokenNameEOF;
+                               return tempToken;
+                       }
                        this.wasAcr = false;
                        if (diet) {
                                jumpOverMethodBody();
@@ -1286,154 +1311,14 @@ public class Scanner implements IScanner, ITerminalSymbols {
                        try {
                                while (true) {
                                        withoutUnicodePtr = 0;
-                                       // start with a new token
-                                       char encapsedChar = ' ';
-                                       // if (!encapsedStringStack.isEmpty()) {
-                                       // encapsedChar = ((Character)
-                                       // encapsedStringStack.peek()).charValue();
-                                       // }
-                                       // if (encapsedChar != '$' && encapsedChar != ' ') {
-                                       // currentCharacter = source[currentPosition++];
-                                       // if (currentCharacter == encapsedChar) {
-                                       // switch (currentCharacter) {
-                                       // case '`':
-                                       // return TokenNameEncapsedString0;
-                                       // case '\'':
-                                       // return TokenNameEncapsedString1;
-                                       // case '"':
-                                       // return TokenNameEncapsedString2;
-                                       // }
-                                       // }
-                                       // while (currentCharacter != encapsedChar) {
-                                       // /** ** in PHP \r and \n are valid in string literals *** */
-                                       // switch (currentCharacter) {
-                                       // case '\\':
-                                       // int escapeSize = currentPosition;
-                                       // boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
-                                       // //scanEscapeCharacter make a side effect on this value and
-                                       // // we need the previous value few lines down this one
-                                       // scanDoubleQuotedEscapeCharacter();
-                                       // escapeSize = currentPosition - escapeSize;
-                                       // if (withoutUnicodePtr == 0) {
-                                       // //buffer all the entries that have been left aside....
-                                       // withoutUnicodePtr = currentPosition - escapeSize - 1 -
-                                       // startPosition;
-                                       // System.arraycopy(source, startPosition, withoutUnicodeBuffer, 1,
-                                       // withoutUnicodePtr);
-                                       // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
-                                       // } else { //overwrite the / in the buffer
-                                       // withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter;
-                                       // if (backSlashAsUnicodeInString) { //there are TWO \ in
-                                       // withoutUnicodePtr--;
-                                       // }
-                                       // }
-                                       // break;
-                                       // case '\r':
-                                       // case '\n':
-                                       // if (recordLineSeparator) {
-                                       // pushLineSeparator();
-                                       // }
-                                       // break;
-                                       // case '$':
-                                       // if (isPHPIdentifierStart(source[currentPosition]) ||
-                                       // source[currentPosition] == '{') {
-                                       // currentPosition--;
-                                       // encapsedStringStack.push(new Character('$'));
-                                       // return TokenNameSTRING;
-                                       // }
-                                       // break;
-                                       // case '{':
-                                       // if (source[currentPosition] == '$') { // CURLY_OPEN
-                                       // currentPosition--;
-                                       // encapsedStringStack.push(new Character('$'));
-                                       // return TokenNameSTRING;
-                                       // }
-                                       // }
-                                       // // consume next character
-                                       // unicodeAsBackSlash = false;
-                                       // currentCharacter = source[currentPosition++];
-                                       // if (withoutUnicodePtr != 0) {
-                                       // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
-                                       // }
-                                       // // }
-                                       // } // end while
-                                       // currentPosition--;
-                                       // return TokenNameSTRING;
-                                       // }
                                        // ---------Consume white space and handles startPosition---------
                                        int whiteStart = currentPosition;
                                        startPosition = currentPosition;
                                        currentCharacter = source[currentPosition++];
-                                       // if (encapsedChar == '$') {
-                                       // switch (currentCharacter) {
-                                       // case '\\':
-                                       // currentCharacter = source[currentPosition++];
-                                       // return TokenNameSTRING;
-                                       // case '{':
-                                       // if (encapsedChar == '$') {
-                                       // if (getNextChar('$'))
-                                       // return TokenNameLBRACE_DOLLAR;
-                                       // }
-                                       // return TokenNameLBRACE;
-                                       // case '}':
-                                       // return TokenNameRBRACE;
-                                       // case '[':
-                                       // return TokenNameLBRACKET;
-                                       // case ']':
-                                       // return TokenNameRBRACKET;
-                                       // case '\'':
-                                       // if (tokenizeStrings) {
-                                       // consumeStringConstant();
-                                       // return TokenNameStringSingleQuote;
-                                       // }
-                                       // return TokenNameEncapsedString1;
-                                       // case '"':
-                                       // return TokenNameEncapsedString2;
-                                       // case '`':
-                                       // if (tokenizeStrings) {
-                                       // consumeStringInterpolated();
-                                       // return TokenNameStringInterpolated;
-                                       // }
-                                       // return TokenNameEncapsedString0;
-                                       // case '-':
-                                       // if (getNextChar('>'))
-                                       // return TokenNameMINUS_GREATER;
-                                       // return TokenNameSTRING;
-                                       // default:
-                                       // if (currentCharacter == '$') {
-                                       // int oldPosition = currentPosition;
-                                       // try {
-                                       // currentCharacter = source[currentPosition++];
-                                       // if (currentCharacter == '{') {
-                                       // return TokenNameDOLLAR_LBRACE;
-                                       // }
-                                       // if (isPHPIdentifierStart(currentCharacter)) {
-                                       // return scanIdentifierOrKeyword(true);
-                                       // } else {
-                                       // currentPosition = oldPosition;
-                                       // return TokenNameSTRING;
-                                       // }
-                                       // } catch (IndexOutOfBoundsException e) {
-                                       // currentPosition = oldPosition;
-                                       // return TokenNameSTRING;
-                                       // }
-                                       // }
-                                       // if (isPHPIdentifierStart(currentCharacter))
-                                       // return scanIdentifierOrKeyword(false);
-                                       // if (Character.isDigit(currentCharacter))
-                                       // return scanNumber(false);
-                                       // return TokenNameERROR;
-                                       // }
-                                       // }
-                                       // boolean isWhiteSpace;
 
                                        while ((currentCharacter == ' ') || Character.isWhitespace(currentCharacter)) {
                                                startPosition = currentPosition;
                                                currentCharacter = source[currentPosition++];
-                                               // if (((currentCharacter = source[currentPosition++]) == '\\')
-                                               // && (source[currentPosition] == 'u')) {
-                                               // isWhiteSpace = jumpOverUnicodeWhiteSpace();
-                                               // } else {
                                                if ((currentCharacter == '\r') || (currentCharacter == '\n')) {
                                                        checkNonExternalizeString();
                                                        if (recordLineSeparator) {
@@ -1442,9 +1327,6 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                                                currentLine = null;
                                                        }
                                                }
-                                               // isWhiteSpace = (currentCharacter == ' ')
-                                               // || Character.isWhitespace(currentCharacter);
-                                               // }
                                        }
                                        if (tokenizeWhiteSpace && (whiteStart != currentPosition - 1)) {
                                                // reposition scanner in case we are interested by spaces as tokens
@@ -1935,6 +1817,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
         * @throws InvalidInputException
         */
        private int getInlinedHTMLToken(int start) throws InvalidInputException {
+               boolean phpShortTag = false; // true, if <?= detected
                if (currentPosition > source.length) {
                        currentPosition = source.length;
                        return TokenNameEOF;
@@ -1949,27 +1832,39 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                                if ((currentCharacter != 'P') && (currentCharacter != 'p')) {
                                                        if (currentCharacter != '=') { // <?=
                                                                currentPosition--;
+                                                               phpShortTag = false;
                                                        } else {
-                                                               phpExpressionTag = true;
+                                                               phpShortTag = true;
                                                        }
                                                        // <?
                                                        if (ignorePHPOneLiner) { // for CodeFormatter
                                                                if (lookAheadLinePHPTag() == TokenNameINLINE_HTML) {
                                                                        phpMode = true;
+                                                                       if (phpShortTag) {
+                                                                               fFillerToken = TokenNameECHO_INVISIBLE;
+                                                                       }
                                                                        return TokenNameINLINE_HTML;
                                                                }
                                                        } else {
-                                                               phpMode = true;
+                                                               boolean foundXML=false;
+                                                               if (getNextChar('X','x')>=0) {
+                                                                       if (getNextChar('M','m')>=0) {
+                                                                               if (getNextChar('L','l')>=0) {
+                                                                                       foundXML=true;
+                                                                               }
+                                                                       }
+                                                               }
+                                                               if (!foundXML) {
+                                                                       phpMode = true;
+                                                               }
+                                                               if (phpShortTag) {
+                                                                       fFillerToken = TokenNameECHO_INVISIBLE;
+                                                               }
                                                                return TokenNameINLINE_HTML;
                                                        }
                                                } else {
-                                                       // boolean phpStart = (currentCharacter == 'P') ||
-                                                       // (currentCharacter == 'p');
-                                                       // if (phpStart) {
-                                                       int test = getNextChar('H', 'h');
-                                                       if (test >= 0) {
-                                                               test = getNextChar('P', 'p');
-                                                               if (test >= 0) {
+                                                       if (getNextChar('H', 'h') >= 0) {
+                                                               if (getNextChar('P', 'p') >= 0) {
                                                                        // <?PHP <?php
                                                                        if (ignorePHPOneLiner) {
                                                                                if (lookAheadLinePHPTag() == TokenNameINLINE_HTML) {
@@ -2006,10 +1901,11 @@ public class Scanner implements IScanner, ITerminalSymbols {
        }
 
        /**
+        * check if the PHP is only in this line (for CodeFormatter)
+        *
         * @return
         */
        private int lookAheadLinePHPTag() {
-               // check if the PHP is only in this line (for CodeFormatter)
                int currentPositionInLine = currentPosition;
                char previousCharInLine = ' ';
                char currentCharInLine = ' ';
@@ -2429,7 +2325,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                        break;
                                }
                                default:
-                                       if (isPHPIdentOrVarStart(currentCharacter) ) {
+                                       if (isPHPIdentOrVarStart(currentCharacter)) {
                                                try {
                                                        scanIdentifierOrKeyword((currentCharacter == '$'));
                                                } catch (InvalidInputException ex) {
@@ -2437,8 +2333,8 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                                ;
                                                break;
                                        }
-                               if ( ObviousIdentCharNatures[currentCharacter]==C_DIGIT) {
-//                                     if (Character.isDigit(currentCharacter)) {
+                                       if (ObviousIdentCharNatures[currentCharacter] == C_DIGIT) {
+                                               // if (Character.isDigit(currentCharacter)) {
                                                try {
                                                        scanNumber(false);
                                                } catch (InvalidInputException ex) {
@@ -3158,50 +3054,42 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                // as
                                if ((data[++index] == 's')) {
                                        return TokenNameas;
-                               } else {
-                                       return TokenNameIdentifier;
                                }
+                               return TokenNameIdentifier;
                        case 3:
                                // and
                                if ((data[++index] == 'n') && (data[++index] == 'd')) {
                                        return TokenNameand;
-                               } else {
-                                       return TokenNameIdentifier;
                                }
+                               return TokenNameIdentifier;
                        case 5:
                                // array
                                if ((data[++index] == 'r') && (data[++index] == 'r') && (data[++index] == 'a') && (data[++index] == 'y'))
                                        return TokenNamearray;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 8:
                                if ((data[++index] == 'b') && (data[++index] == 's') && (data[++index] == 't') && (data[++index] == 'r')
                                                && (data[++index] == 'a') && (data[++index] == 'c') && (data[++index] == 't'))
                                        return TokenNameabstract;
-                               else
-                                       return TokenNameIdentifier;
-                       default:
                                return TokenNameIdentifier;
                        }
+                       return TokenNameIdentifier;
                case 'b':
                        // break
                        switch (length) {
                        case 5:
                                if ((data[++index] == 'r') && (data[++index] == 'e') && (data[++index] == 'a') && (data[++index] == 'k'))
                                        return TokenNamebreak;
-                               else
-                                       return TokenNameIdentifier;
-                       default:
                                return TokenNameIdentifier;
                        }
+                       return TokenNameIdentifier;
                case 'c':
                        // case catch class clone const continue
                        switch (length) {
                        case 4:
                                if ((data[++index] == 'a') && (data[++index] == 's') && (data[++index] == 'e'))
                                        return TokenNamecase;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 5:
                                if ((data[++index] == 'a') && (data[++index] == 't') && (data[++index] == 'c') && (data[++index] == 'h'))
                                        return TokenNamecatch;
@@ -3214,17 +3102,14 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                index = 0;
                                if ((data[++index] == 'o') && (data[++index] == 'n') && (data[++index] == 's') && (data[++index] == 't'))
                                        return TokenNameconst;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 8:
                                if ((data[++index] == 'o') && (data[++index] == 'n') && (data[++index] == 't') && (data[++index] == 'i')
                                                && (data[++index] == 'n') && (data[++index] == 'u') && (data[++index] == 'e'))
                                        return TokenNamecontinue;
-                               else
-                                       return TokenNameIdentifier;
-                       default:
                                return TokenNameIdentifier;
                        }
+                       return TokenNameIdentifier;
                case 'd':
                        // declare default do die
                        // TODO delete define ==> no keyword !
@@ -3232,8 +3117,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
                        case 2:
                                if ((data[++index] == 'o'))
                                        return TokenNamedo;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        // case 6 :
                        // if ((data[++index] == 'e')
                        // && (data[++index] == 'f')
@@ -3251,11 +3135,9 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                if ((data[++index] == 'e') && (data[++index] == 'f') && (data[++index] == 'a') && (data[++index] == 'u')
                                                && (data[++index] == 'l') && (data[++index] == 't'))
                                        return TokenNamedefault;
-                               else
-                                       return TokenNameIdentifier;
-                       default:
                                return TokenNameIdentifier;
                        }
+                       return TokenNameIdentifier;
                case 'e':
                        // echo else exit elseif extends eval
                        switch (length) {
@@ -3268,16 +3150,14 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                        return TokenNameexit;
                                else if ((data[index] == 'v') && (data[++index] == 'a') && (data[++index] == 'l'))
                                        return TokenNameeval;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 5:
                                // endif empty
                                if ((data[++index] == 'n') && (data[++index] == 'd') && (data[++index] == 'i') && (data[++index] == 'f'))
                                        return TokenNameendif;
                                if ((data[index] == 'm') && (data[++index] == 'p') && (data[++index] == 't') && (data[++index] == 'y'))
                                        return TokenNameempty;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 6:
                                // endfor
                                if ((data[++index] == 'n') && (data[++index] == 'd') && (data[++index] == 'f') && (data[++index] == 'o')
@@ -3286,28 +3166,24 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                else if ((data[index] == 'l') && (data[++index] == 's') && (data[++index] == 'e') && (data[++index] == 'i')
                                                && (data[++index] == 'f'))
                                        return TokenNameelseif;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 7:
                                if ((data[++index] == 'x') && (data[++index] == 't') && (data[++index] == 'e') && (data[++index] == 'n')
                                                && (data[++index] == 'd') && (data[++index] == 's'))
                                        return TokenNameextends;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 8:
                                // endwhile
                                if ((data[++index] == 'n') && (data[++index] == 'd') && (data[++index] == 'w') && (data[++index] == 'h')
                                                && (data[++index] == 'i') && (data[++index] == 'l') && (data[++index] == 'e'))
                                        return TokenNameendwhile;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 9:
                                // endswitch
                                if ((data[++index] == 'n') && (data[++index] == 'd') && (data[++index] == 's') && (data[++index] == 'w')
                                                && (data[++index] == 'i') && (data[++index] == 't') && (data[++index] == 'c') && (data[++index] == 'h'))
                                        return TokenNameendswitch;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 10:
                                // enddeclare
                                if ((data[++index] == 'n') && (data[++index] == 'd') && (data[++index] == 'd') && (data[++index] == 'e')
@@ -3319,44 +3195,37 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                                && (data[++index] == 'd') && (data[++index] == 'f') && (data[++index] == 'o') && (data[++index] == 'r')
                                                && (data[++index] == 'e') && (data[++index] == 'a') && (data[++index] == 'c') && (data[++index] == 'h'))
                                        return TokenNameendforeach;
-                               else
-                                       return TokenNameIdentifier;
-                       default:
                                return TokenNameIdentifier;
                        }
+                       return TokenNameIdentifier;
                case 'f':
                        // for false final function
                        switch (length) {
                        case 3:
                                if ((data[++index] == 'o') && (data[++index] == 'r'))
                                        return TokenNamefor;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 5:
                                // if ((data[++index] == 'a') && (data[++index] == 'l')
                                // && (data[++index] == 's') && (data[++index] == 'e'))
                                // return TokenNamefalse;
                                if ((data[++index] == 'i') && (data[++index] == 'n') && (data[++index] == 'a') && (data[++index] == 'l'))
                                        return TokenNamefinal;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 7:
                                // foreach
                                if ((data[++index] == 'o') && (data[++index] == 'r') && (data[++index] == 'e') && (data[++index] == 'a')
                                                && (data[++index] == 'c') && (data[++index] == 'h'))
                                        return TokenNameforeach;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 8:
                                // function
                                if ((data[++index] == 'u') && (data[++index] == 'n') && (data[++index] == 'c') && (data[++index] == 't')
                                                && (data[++index] == 'i') && (data[++index] == 'o') && (data[++index] == 'n'))
                                        return TokenNamefunction;
-                               else
-                                       return TokenNameIdentifier;
-                       default:
                                return TokenNameIdentifier;
                        }
+                       return TokenNameIdentifier;
                case 'g':
                        // global
                        if (length == 6) {
@@ -3372,8 +3241,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
                        case 2:
                                if (data[++index] == 'f')
                                        return TokenNameif;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        // case 3 :
                        // if ((data[++index] == 'n') && (data[++index] == 't'))
                        // return TokenNameint;
@@ -3382,21 +3250,18 @@ public class Scanner implements IScanner, ITerminalSymbols {
                        case 5:
                                if ((data[++index] == 's') && (data[++index] == 's') && (data[++index] == 'e') && (data[++index] == 't'))
                                        return TokenNameisset;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 7:
                                if ((data[++index] == 'n') && (data[++index] == 'c') && (data[++index] == 'l') && (data[++index] == 'u')
                                                && (data[++index] == 'd') && (data[++index] == 'e'))
                                        return TokenNameinclude;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 9:
                                // interface
                                if ((data[++index] == 'n') && (data[++index] == 't') && (data[++index] == 'e') && (data[++index] == 'r')
                                                && (data[++index] == 'f') && (data[++index] == 'a') && (data[++index] == 'c') && (data[++index] == 'e'))
                                        return TokenNameinterface;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 10:
                                // instanceof
                                if ((data[++index] == 'n') && (data[++index] == 's') && (data[++index] == 't') && (data[++index] == 'a')
@@ -3407,18 +3272,15 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                                && (data[++index] == 'm') && (data[++index] == 'e') && (data[++index] == 'n') && (data[++index] == 't')
                                                && (data[++index] == 's'))
                                        return TokenNameimplements;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 12:
                                if ((data[++index] == 'n') && (data[++index] == 'c') && (data[++index] == 'l') && (data[++index] == 'u')
                                                && (data[++index] == 'd') && (data[++index] == 'e') && (data[++index] == '_') && (data[++index] == 'o')
                                                && (data[++index] == 'n') && (data[++index] == 'c') && (data[++index] == 'e'))
                                        return TokenNameinclude_once;
-                               else
-                                       return TokenNameIdentifier;
-                       default:
                                return TokenNameIdentifier;
                        }
+                       return TokenNameIdentifier;
                case 'l':
                        // list
                        if (length == 4) {
@@ -3433,17 +3295,15 @@ public class Scanner implements IScanner, ITerminalSymbols {
                        case 3:
                                if ((data[++index] == 'e') && (data[++index] == 'w'))
                                        return TokenNamenew;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        // case 4 :
                        // if ((data[++index] == 'u') && (data[++index] == 'l')
                        // && (data[++index] == 'l'))
                        // return TokenNamenull;
                        // else
                        // return TokenNameIdentifier;
-                       default:
-                               return TokenNameIdentifier;
                        }
+                       return TokenNameIdentifier;
                case 'o':
                        // or old_function
                        if (length == 2) {
@@ -3473,26 +3333,26 @@ public class Scanner implements IScanner, ITerminalSymbols {
                        case 5:
                                if ((data[++index] == 'r') && (data[++index] == 'i') && (data[++index] == 'n') && (data[++index] == 't')) {
                                        return TokenNameprint;
-                               } else
-                                       return TokenNameIdentifier;
+                               }
+                               return TokenNameIdentifier;
                        case 6:
                                if ((data[++index] == 'u') && (data[++index] == 'b') && (data[++index] == 'l') && (data[++index] == 'i')
                                                && (data[++index] == 'c')) {
                                        return TokenNamepublic;
-                               } else
-                                       return TokenNameIdentifier;
+                               }
+                               return TokenNameIdentifier;
                        case 7:
                                if ((data[++index] == 'r') && (data[++index] == 'i') && (data[++index] == 'v') && (data[++index] == 'a')
                                                && (data[++index] == 't') && (data[++index] == 'e')) {
                                        return TokenNameprivate;
-                               } else
-                                       return TokenNameIdentifier;
+                               }
+                               return TokenNameIdentifier;
                        case 9:
                                if ((data[++index] == 'r') && (data[++index] == 'o') && (data[++index] == 't') && (data[++index] == 'e')
                                                && (data[++index] == 'c') && (data[++index] == 't') && (data[++index] == 'e') && (data[++index] == 'd')) {
                                        return TokenNameprotected;
-                               } else
-                                       return TokenNameIdentifier;
+                               }
+                               return TokenNameIdentifier;
                        }
                        return TokenNameIdentifier;
                case 'r':
@@ -3513,11 +3373,17 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                                && (data[++index] == 'n') && (data[++index] == 'c') && (data[++index] == 'e')) {
                                        return TokenNamerequire_once;
                                }
-                       } else
-                               return TokenNameIdentifier;
+                       }
+                       return TokenNameIdentifier;
                case 's':
-                       // static switch
+                       // self static switch
                        switch (length) {
+                       // case 4:
+                       // if ((data[++index] == 'e') && (data[++index] == 'l') && (data[++index]
+                       // == 'f')) {
+                       // return TokenNameself;
+                       // }
+                       // return TokenNameIdentifier;
                        case 6:
                                if (data[++index] == 't')
                                        if ((data[++index] == 'a') && (data[++index] == 't') && (data[++index] == 'i') && (data[++index] == 'c')) {
@@ -3527,19 +3393,14 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                else if ((data[index] == 'w') && (data[++index] == 'i') && (data[++index] == 't') && (data[++index] == 'c')
                                                && (data[++index] == 'h'))
                                        return TokenNameswitch;
-                               else
-                                       return TokenNameIdentifier;
-                       default:
-                               return TokenNameIdentifier;
                        }
+                       return TokenNameIdentifier;
                case 't':
                        // try true throw
                        switch (length) {
                        case 3:
                                if ((data[++index] == 'r') && (data[++index] == 'y'))
                                        return TokenNametry;
-                               else
-                                       return TokenNameIdentifier;
                        // case 4 :
                        // if ((data[++index] == 'r') && (data[++index] == 'u')
                        // && (data[++index] == 'e'))
@@ -3549,55 +3410,41 @@ public class Scanner implements IScanner, ITerminalSymbols {
                        case 5:
                                if ((data[++index] == 'h') && (data[++index] == 'r') && (data[++index] == 'o') && (data[++index] == 'w'))
                                        return TokenNamethrow;
-                               else
-                                       return TokenNameIdentifier;
-                       default:
-                               return TokenNameIdentifier;
                        }
+                       return TokenNameIdentifier;
                case 'u':
                        // use unset
                        switch (length) {
                        case 3:
                                if ((data[++index] == 's') && (data[++index] == 'e'))
                                        return TokenNameuse;
-                               else
-                                       return TokenNameIdentifier;
                        case 5:
                                if ((data[++index] == 'n') && (data[++index] == 's') && (data[++index] == 'e') && (data[++index] == 't'))
                                        return TokenNameunset;
-                               else
-                                       return TokenNameIdentifier;
-                       default:
-                               return TokenNameIdentifier;
                        }
+                       return TokenNameIdentifier;
                case 'v':
                        // var
                        switch (length) {
                        case 3:
                                if ((data[++index] == 'a') && (data[++index] == 'r'))
                                        return TokenNamevar;
-                               else
-                                       return TokenNameIdentifier;
-                       default:
-                               return TokenNameIdentifier;
                        }
+                       return TokenNameIdentifier;
                case 'w':
                        // while
                        switch (length) {
                        case 5:
                                if ((data[++index] == 'h') && (data[++index] == 'i') && (data[++index] == 'l') && (data[++index] == 'e'))
                                        return TokenNamewhile;
-                               else
-                                       return TokenNameIdentifier;
                        // case 6:if ( (data[++index] =='i') && (data[++index]=='d') &&
                        // (data[++index]=='e') && (data[++index]=='f')&&
                        // (data[++index]=='p'))
                        // return TokenNamewidefp ;
                        // else
                        // return TokenNameIdentifier;
-                       default:
-                               return TokenNameIdentifier;
                        }
+                       return TokenNameIdentifier;
                case 'x':
                        // xor
                        switch (length) {
@@ -3606,12 +3453,10 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                        return TokenNamexor;
                                else
                                        return TokenNameIdentifier;
-                       default:
-                               return TokenNameIdentifier;
                        }
-               default:
                        return TokenNameIdentifier;
                }
+               return TokenNameIdentifier;
        }
 
        public int scanNumber(boolean dotPrefix) throws InvalidInputException {
@@ -3811,6 +3656,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
                initialPosition = currentPosition = 0;
                containsAssertKeyword = false;
                withoutUnicodeBuffer = new char[this.source.length];
+               fFillerToken = TokenNameEOF;
                // encapsedStringStack = new Stack();
        }
 
@@ -3843,6 +3689,9 @@ public class Scanner implements IScanner, ITerminalSymbols {
                // //$NON-NLS-1$
                case TokenNameINLINE_HTML:
                        return "Inline-HTML(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
+               case    TokenNameECHO_INVISIBLE:
+                       //0-length token
+                       return "";
                case TokenNameIdentifier:
                        return "Identifier(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
                case TokenNameVariable:
@@ -3945,6 +3794,8 @@ public class Scanner implements IScanner, ITerminalSymbols {
                        return "require_once"; //$NON-NLS-1$
                case TokenNamereturn:
                        return "return"; //$NON-NLS-1$
+                       // case TokenNameself:
+                       // return "self"; //$NON-NLS-1$
                case TokenNamestatic:
                        return "static"; //$NON-NLS-1$
                case TokenNameswitch:
@@ -3973,10 +3824,10 @@ public class Scanner implements IScanner, ITerminalSymbols {
                        return "StringInterpolated(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
                case TokenNameEncapsedString0:
                        return "`"; //$NON-NLS-1$
-                       // case TokenNameEncapsedString1:
-                       // return "\'"; //$NON-NLS-1$
-                       // case TokenNameEncapsedString2:
-                       // return "\""; //$NON-NLS-1$
+               // case TokenNameEncapsedString1:
+               // return "\'"; //$NON-NLS-1$
+               // case TokenNameEncapsedString2:
+               // return "\""; //$NON-NLS-1$
                case TokenNameSTRING:
                        return "STRING_DQ(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
                case TokenNameHEREDOC:
@@ -4151,7 +4002,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
                this.tokenizeWhiteSpace = tokenizeWhiteSpace;
                this.tokenizeStrings = tokenizeStrings;
                this.checkNonExternalizedStringLiterals = checkNonExternalizedStringLiterals;
-//             this.assertMode = assertMode;
+               // this.assertMode = assertMode;
                // this.encapsedStringStack = null;
                this.taskTags = taskTags;
                this.taskPriorities = taskPriorities;
@@ -4325,8 +4176,8 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                                        continue nextTag;
                                                if ((sc = src[i + t]) != (tc = tag[t])) { // case sensitive check
                                                        if (this.isTaskCaseSensitive || (Character.toLowerCase(sc) != Character.toLowerCase(tc))) { // case
-                                                                                                                                                                                                                                                                                                                                                                                                                                       // insensitive
-                                                                                                                                                                                                                                                                                                                                                                                                                                       // check
+                                                               // insensitive
+                                                               // check
                                                                continue nextTag;
                                                        }
                                                }