bug 1434118, scanner had a faulty behavior scanning variable names like 'retur*'
[phpeclipse.git] / net.sourceforge.phpeclipse / src / net / sourceforge / phpdt / internal / compiler / parser / Scanner.java
index a19a44d..b2ae4c6 100644 (file)
@@ -39,9 +39,13 @@ public class Scanner implements IScanner, ITerminalSymbols {
 
        public boolean phpMode = false;
 
-       public boolean phpExpressionTag = false;
-
-       // public Stack encapsedStringStack = null;
+       /**
+        * This token is set to TokenNameecho if a short tag block begins (i.e. >?= ... )
+        * Directly after the "=" character the getNextToken() method returns TokenNameINLINE_HTML
+        * In the next call to the getNextToken() method the value of fFillerToken (==TokenNameecho) is returned
+        *
+        */
+       int fFillerToken = TokenNameEOF;
 
        public char currentCharacter;
 
@@ -1212,11 +1216,11 @@ public class Scanner implements IScanner, ITerminalSymbols {
 
        public void consumeStringLiteral() throws InvalidInputException {
                try {
-                       boolean openDollarBrace = false;
+                       int openDollarBrace = 0;
                        // consume next character
                        unicodeAsBackSlash = false;
                        currentCharacter = source[currentPosition++];
-                       while (currentCharacter != '"' || openDollarBrace) {
+                       while (currentCharacter != '"' || openDollarBrace>0) {
                                /** ** in PHP \r and \n are valid in string literals *** */
                                if (currentCharacter == '\\') {
                                        int escapeSize = currentPosition;
@@ -1238,11 +1242,11 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                                }
                                        }
                                } else if (currentCharacter == '$' && source[currentPosition] == '{') {
-                                       openDollarBrace = true;
+                                       openDollarBrace++;
                                } else if (currentCharacter == '{' && source[currentPosition] == '$') {
-                                       openDollarBrace = true;
+                                       openDollarBrace++;
                                } else if (currentCharacter == '}') {
-                                       openDollarBrace = false;
+                                       openDollarBrace--;
                                } else if ((currentCharacter == '\r') || (currentCharacter == '\n')) {
                                        if (recordLineSeparator) {
                                                pushLineSeparator();
@@ -1288,11 +1292,16 @@ public class Scanner implements IScanner, ITerminalSymbols {
        }
 
        public int getNextToken() throws InvalidInputException {
-               phpExpressionTag = false;
                if (!phpMode) {
                        return getInlinedHTMLToken(currentPosition);
-               }
-               if (phpMode) {
+               } else {
+                       if (fFillerToken != TokenNameEOF) {
+                               int tempToken;
+                               startPosition = currentPosition;
+                               tempToken = fFillerToken;
+                               fFillerToken = TokenNameEOF;
+                               return tempToken;
+                       }
                        this.wasAcr = false;
                        if (diet) {
                                jumpOverMethodBody();
@@ -1302,154 +1311,14 @@ public class Scanner implements IScanner, ITerminalSymbols {
                        try {
                                while (true) {
                                        withoutUnicodePtr = 0;
-                                       // start with a new token
-                                       char encapsedChar = ' ';
-                                       // if (!encapsedStringStack.isEmpty()) {
-                                       // encapsedChar = ((Character)
-                                       // encapsedStringStack.peek()).charValue();
-                                       // }
-                                       // if (encapsedChar != '$' && encapsedChar != ' ') {
-                                       // currentCharacter = source[currentPosition++];
-                                       // if (currentCharacter == encapsedChar) {
-                                       // switch (currentCharacter) {
-                                       // case '`':
-                                       // return TokenNameEncapsedString0;
-                                       // case '\'':
-                                       // return TokenNameEncapsedString1;
-                                       // case '"':
-                                       // return TokenNameEncapsedString2;
-                                       // }
-                                       // }
-                                       // while (currentCharacter != encapsedChar) {
-                                       // /** ** in PHP \r and \n are valid in string literals *** */
-                                       // switch (currentCharacter) {
-                                       // case '\\':
-                                       // int escapeSize = currentPosition;
-                                       // boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
-                                       // //scanEscapeCharacter make a side effect on this value and
-                                       // // we need the previous value few lines down this one
-                                       // scanDoubleQuotedEscapeCharacter();
-                                       // escapeSize = currentPosition - escapeSize;
-                                       // if (withoutUnicodePtr == 0) {
-                                       // //buffer all the entries that have been left aside....
-                                       // withoutUnicodePtr = currentPosition - escapeSize - 1 -
-                                       // startPosition;
-                                       // System.arraycopy(source, startPosition, withoutUnicodeBuffer, 1,
-                                       // withoutUnicodePtr);
-                                       // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
-                                       // } else { //overwrite the / in the buffer
-                                       // withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter;
-                                       // if (backSlashAsUnicodeInString) { //there are TWO \ in
-                                       // withoutUnicodePtr--;
-                                       // }
-                                       // }
-                                       // break;
-                                       // case '\r':
-                                       // case '\n':
-                                       // if (recordLineSeparator) {
-                                       // pushLineSeparator();
-                                       // }
-                                       // break;
-                                       // case '$':
-                                       // if (isPHPIdentifierStart(source[currentPosition]) ||
-                                       // source[currentPosition] == '{') {
-                                       // currentPosition--;
-                                       // encapsedStringStack.push(new Character('$'));
-                                       // return TokenNameSTRING;
-                                       // }
-                                       // break;
-                                       // case '{':
-                                       // if (source[currentPosition] == '$') { // CURLY_OPEN
-                                       // currentPosition--;
-                                       // encapsedStringStack.push(new Character('$'));
-                                       // return TokenNameSTRING;
-                                       // }
-                                       // }
-                                       // // consume next character
-                                       // unicodeAsBackSlash = false;
-                                       // currentCharacter = source[currentPosition++];
-                                       // if (withoutUnicodePtr != 0) {
-                                       // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
-                                       // }
-                                       // // }
-                                       // } // end while
-                                       // currentPosition--;
-                                       // return TokenNameSTRING;
-                                       // }
                                        // ---------Consume white space and handles startPosition---------
                                        int whiteStart = currentPosition;
                                        startPosition = currentPosition;
                                        currentCharacter = source[currentPosition++];
-                                       // if (encapsedChar == '$') {
-                                       // switch (currentCharacter) {
-                                       // case '\\':
-                                       // currentCharacter = source[currentPosition++];
-                                       // return TokenNameSTRING;
-                                       // case '{':
-                                       // if (encapsedChar == '$') {
-                                       // if (getNextChar('$'))
-                                       // return TokenNameLBRACE_DOLLAR;
-                                       // }
-                                       // return TokenNameLBRACE;
-                                       // case '}':
-                                       // return TokenNameRBRACE;
-                                       // case '[':
-                                       // return TokenNameLBRACKET;
-                                       // case ']':
-                                       // return TokenNameRBRACKET;
-                                       // case '\'':
-                                       // if (tokenizeStrings) {
-                                       // consumeStringConstant();
-                                       // return TokenNameStringSingleQuote;
-                                       // }
-                                       // return TokenNameEncapsedString1;
-                                       // case '"':
-                                       // return TokenNameEncapsedString2;
-                                       // case '`':
-                                       // if (tokenizeStrings) {
-                                       // consumeStringInterpolated();
-                                       // return TokenNameStringInterpolated;
-                                       // }
-                                       // return TokenNameEncapsedString0;
-                                       // case '-':
-                                       // if (getNextChar('>'))
-                                       // return TokenNameMINUS_GREATER;
-                                       // return TokenNameSTRING;
-                                       // default:
-                                       // if (currentCharacter == '$') {
-                                       // int oldPosition = currentPosition;
-                                       // try {
-                                       // currentCharacter = source[currentPosition++];
-                                       // if (currentCharacter == '{') {
-                                       // return TokenNameDOLLAR_LBRACE;
-                                       // }
-                                       // if (isPHPIdentifierStart(currentCharacter)) {
-                                       // return scanIdentifierOrKeyword(true);
-                                       // } else {
-                                       // currentPosition = oldPosition;
-                                       // return TokenNameSTRING;
-                                       // }
-                                       // } catch (IndexOutOfBoundsException e) {
-                                       // currentPosition = oldPosition;
-                                       // return TokenNameSTRING;
-                                       // }
-                                       // }
-                                       // if (isPHPIdentifierStart(currentCharacter))
-                                       // return scanIdentifierOrKeyword(false);
-                                       // if (Character.isDigit(currentCharacter))
-                                       // return scanNumber(false);
-                                       // return TokenNameERROR;
-                                       // }
-                                       // }
-                                       // boolean isWhiteSpace;
 
                                        while ((currentCharacter == ' ') || Character.isWhitespace(currentCharacter)) {
                                                startPosition = currentPosition;
                                                currentCharacter = source[currentPosition++];
-                                               // if (((currentCharacter = source[currentPosition++]) == '\\')
-                                               // && (source[currentPosition] == 'u')) {
-                                               // isWhiteSpace = jumpOverUnicodeWhiteSpace();
-                                               // } else {
                                                if ((currentCharacter == '\r') || (currentCharacter == '\n')) {
                                                        checkNonExternalizeString();
                                                        if (recordLineSeparator) {
@@ -1458,9 +1327,6 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                                                currentLine = null;
                                                        }
                                                }
-                                               // isWhiteSpace = (currentCharacter == ' ')
-                                               // || Character.isWhitespace(currentCharacter);
-                                               // }
                                        }
                                        if (tokenizeWhiteSpace && (whiteStart != currentPosition - 1)) {
                                                // reposition scanner in case we are interested by spaces as tokens
@@ -1951,6 +1817,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
         * @throws InvalidInputException
         */
        private int getInlinedHTMLToken(int start) throws InvalidInputException {
+               boolean phpShortTag = false; // true, if <?= detected
                if (currentPosition > source.length) {
                        currentPosition = source.length;
                        return TokenNameEOF;
@@ -1965,27 +1832,39 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                                if ((currentCharacter != 'P') && (currentCharacter != 'p')) {
                                                        if (currentCharacter != '=') { // <?=
                                                                currentPosition--;
+                                                               phpShortTag = false;
                                                        } else {
-                                                               phpExpressionTag = true;
+                                                               phpShortTag = true;
                                                        }
                                                        // <?
                                                        if (ignorePHPOneLiner) { // for CodeFormatter
                                                                if (lookAheadLinePHPTag() == TokenNameINLINE_HTML) {
                                                                        phpMode = true;
+                                                                       if (phpShortTag) {
+                                                                               fFillerToken = TokenNameECHO_INVISIBLE;
+                                                                       }
                                                                        return TokenNameINLINE_HTML;
                                                                }
                                                        } else {
-                                                               phpMode = true;
+                                                               boolean foundXML=false;
+                                                               if (getNextChar('X','x')>=0) {
+                                                                       if (getNextChar('M','m')>=0) {
+                                                                               if (getNextChar('L','l')>=0) {
+                                                                                       foundXML=true;
+                                                                               }
+                                                                       }
+                                                               }
+                                                               if (!foundXML) {
+                                                                       phpMode = true;
+                                                               }
+                                                               if (phpShortTag) {
+                                                                       fFillerToken = TokenNameECHO_INVISIBLE;
+                                                               }
                                                                return TokenNameINLINE_HTML;
                                                        }
                                                } else {
-                                                       // boolean phpStart = (currentCharacter == 'P') ||
-                                                       // (currentCharacter == 'p');
-                                                       // if (phpStart) {
-                                                       int test = getNextChar('H', 'h');
-                                                       if (test >= 0) {
-                                                               test = getNextChar('P', 'p');
-                                                               if (test >= 0) {
+                                                       if (getNextChar('H', 'h') >= 0) {
+                                                               if (getNextChar('P', 'p') >= 0) {
                                                                        // <?PHP <?php
                                                                        if (ignorePHPOneLiner) {
                                                                                if (lookAheadLinePHPTag() == TokenNameINLINE_HTML) {
@@ -2022,10 +1901,11 @@ public class Scanner implements IScanner, ITerminalSymbols {
        }
 
        /**
+        * check if the PHP is only in this line (for CodeFormatter)
+        *
         * @return
         */
        private int lookAheadLinePHPTag() {
-               // check if the PHP is only in this line (for CodeFormatter)
                int currentPositionInLine = currentPosition;
                char previousCharInLine = ' ';
                char currentCharInLine = ' ';
@@ -3174,50 +3054,42 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                // as
                                if ((data[++index] == 's')) {
                                        return TokenNameas;
-                               } else {
-                                       return TokenNameIdentifier;
                                }
+                               return TokenNameIdentifier;
                        case 3:
                                // and
                                if ((data[++index] == 'n') && (data[++index] == 'd')) {
                                        return TokenNameand;
-                               } else {
-                                       return TokenNameIdentifier;
                                }
+                               return TokenNameIdentifier;
                        case 5:
                                // array
                                if ((data[++index] == 'r') && (data[++index] == 'r') && (data[++index] == 'a') && (data[++index] == 'y'))
                                        return TokenNamearray;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 8:
                                if ((data[++index] == 'b') && (data[++index] == 's') && (data[++index] == 't') && (data[++index] == 'r')
                                                && (data[++index] == 'a') && (data[++index] == 'c') && (data[++index] == 't'))
                                        return TokenNameabstract;
-                               else
-                                       return TokenNameIdentifier;
-                       default:
                                return TokenNameIdentifier;
                        }
+                       return TokenNameIdentifier;
                case 'b':
                        // break
                        switch (length) {
                        case 5:
                                if ((data[++index] == 'r') && (data[++index] == 'e') && (data[++index] == 'a') && (data[++index] == 'k'))
                                        return TokenNamebreak;
-                               else
-                                       return TokenNameIdentifier;
-                       default:
                                return TokenNameIdentifier;
                        }
+                       return TokenNameIdentifier;
                case 'c':
                        // case catch class clone const continue
                        switch (length) {
                        case 4:
                                if ((data[++index] == 'a') && (data[++index] == 's') && (data[++index] == 'e'))
                                        return TokenNamecase;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 5:
                                if ((data[++index] == 'a') && (data[++index] == 't') && (data[++index] == 'c') && (data[++index] == 'h'))
                                        return TokenNamecatch;
@@ -3230,17 +3102,14 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                index = 0;
                                if ((data[++index] == 'o') && (data[++index] == 'n') && (data[++index] == 's') && (data[++index] == 't'))
                                        return TokenNameconst;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 8:
                                if ((data[++index] == 'o') && (data[++index] == 'n') && (data[++index] == 't') && (data[++index] == 'i')
                                                && (data[++index] == 'n') && (data[++index] == 'u') && (data[++index] == 'e'))
                                        return TokenNamecontinue;
-                               else
-                                       return TokenNameIdentifier;
-                       default:
                                return TokenNameIdentifier;
                        }
+                       return TokenNameIdentifier;
                case 'd':
                        // declare default do die
                        // TODO delete define ==> no keyword !
@@ -3248,8 +3117,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
                        case 2:
                                if ((data[++index] == 'o'))
                                        return TokenNamedo;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        // case 6 :
                        // if ((data[++index] == 'e')
                        // && (data[++index] == 'f')
@@ -3267,11 +3135,9 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                if ((data[++index] == 'e') && (data[++index] == 'f') && (data[++index] == 'a') && (data[++index] == 'u')
                                                && (data[++index] == 'l') && (data[++index] == 't'))
                                        return TokenNamedefault;
-                               else
-                                       return TokenNameIdentifier;
-                       default:
                                return TokenNameIdentifier;
                        }
+                       return TokenNameIdentifier;
                case 'e':
                        // echo else exit elseif extends eval
                        switch (length) {
@@ -3284,16 +3150,14 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                        return TokenNameexit;
                                else if ((data[index] == 'v') && (data[++index] == 'a') && (data[++index] == 'l'))
                                        return TokenNameeval;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 5:
                                // endif empty
                                if ((data[++index] == 'n') && (data[++index] == 'd') && (data[++index] == 'i') && (data[++index] == 'f'))
                                        return TokenNameendif;
                                if ((data[index] == 'm') && (data[++index] == 'p') && (data[++index] == 't') && (data[++index] == 'y'))
                                        return TokenNameempty;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 6:
                                // endfor
                                if ((data[++index] == 'n') && (data[++index] == 'd') && (data[++index] == 'f') && (data[++index] == 'o')
@@ -3302,28 +3166,24 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                else if ((data[index] == 'l') && (data[++index] == 's') && (data[++index] == 'e') && (data[++index] == 'i')
                                                && (data[++index] == 'f'))
                                        return TokenNameelseif;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 7:
                                if ((data[++index] == 'x') && (data[++index] == 't') && (data[++index] == 'e') && (data[++index] == 'n')
                                                && (data[++index] == 'd') && (data[++index] == 's'))
                                        return TokenNameextends;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 8:
                                // endwhile
                                if ((data[++index] == 'n') && (data[++index] == 'd') && (data[++index] == 'w') && (data[++index] == 'h')
                                                && (data[++index] == 'i') && (data[++index] == 'l') && (data[++index] == 'e'))
                                        return TokenNameendwhile;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 9:
                                // endswitch
                                if ((data[++index] == 'n') && (data[++index] == 'd') && (data[++index] == 's') && (data[++index] == 'w')
                                                && (data[++index] == 'i') && (data[++index] == 't') && (data[++index] == 'c') && (data[++index] == 'h'))
                                        return TokenNameendswitch;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 10:
                                // enddeclare
                                if ((data[++index] == 'n') && (data[++index] == 'd') && (data[++index] == 'd') && (data[++index] == 'e')
@@ -3335,44 +3195,37 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                                && (data[++index] == 'd') && (data[++index] == 'f') && (data[++index] == 'o') && (data[++index] == 'r')
                                                && (data[++index] == 'e') && (data[++index] == 'a') && (data[++index] == 'c') && (data[++index] == 'h'))
                                        return TokenNameendforeach;
-                               else
-                                       return TokenNameIdentifier;
-                       default:
                                return TokenNameIdentifier;
                        }
+                       return TokenNameIdentifier;
                case 'f':
                        // for false final function
                        switch (length) {
                        case 3:
                                if ((data[++index] == 'o') && (data[++index] == 'r'))
                                        return TokenNamefor;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 5:
                                // if ((data[++index] == 'a') && (data[++index] == 'l')
                                // && (data[++index] == 's') && (data[++index] == 'e'))
                                // return TokenNamefalse;
                                if ((data[++index] == 'i') && (data[++index] == 'n') && (data[++index] == 'a') && (data[++index] == 'l'))
                                        return TokenNamefinal;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 7:
                                // foreach
                                if ((data[++index] == 'o') && (data[++index] == 'r') && (data[++index] == 'e') && (data[++index] == 'a')
                                                && (data[++index] == 'c') && (data[++index] == 'h'))
                                        return TokenNameforeach;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 8:
                                // function
                                if ((data[++index] == 'u') && (data[++index] == 'n') && (data[++index] == 'c') && (data[++index] == 't')
                                                && (data[++index] == 'i') && (data[++index] == 'o') && (data[++index] == 'n'))
                                        return TokenNamefunction;
-                               else
-                                       return TokenNameIdentifier;
-                       default:
                                return TokenNameIdentifier;
                        }
+                       return TokenNameIdentifier;
                case 'g':
                        // global
                        if (length == 6) {
@@ -3388,8 +3241,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
                        case 2:
                                if (data[++index] == 'f')
                                        return TokenNameif;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        // case 3 :
                        // if ((data[++index] == 'n') && (data[++index] == 't'))
                        // return TokenNameint;
@@ -3398,21 +3250,18 @@ public class Scanner implements IScanner, ITerminalSymbols {
                        case 5:
                                if ((data[++index] == 's') && (data[++index] == 's') && (data[++index] == 'e') && (data[++index] == 't'))
                                        return TokenNameisset;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 7:
                                if ((data[++index] == 'n') && (data[++index] == 'c') && (data[++index] == 'l') && (data[++index] == 'u')
                                                && (data[++index] == 'd') && (data[++index] == 'e'))
                                        return TokenNameinclude;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 9:
                                // interface
                                if ((data[++index] == 'n') && (data[++index] == 't') && (data[++index] == 'e') && (data[++index] == 'r')
                                                && (data[++index] == 'f') && (data[++index] == 'a') && (data[++index] == 'c') && (data[++index] == 'e'))
                                        return TokenNameinterface;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 10:
                                // instanceof
                                if ((data[++index] == 'n') && (data[++index] == 's') && (data[++index] == 't') && (data[++index] == 'a')
@@ -3423,18 +3272,15 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                                && (data[++index] == 'm') && (data[++index] == 'e') && (data[++index] == 'n') && (data[++index] == 't')
                                                && (data[++index] == 's'))
                                        return TokenNameimplements;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        case 12:
                                if ((data[++index] == 'n') && (data[++index] == 'c') && (data[++index] == 'l') && (data[++index] == 'u')
                                                && (data[++index] == 'd') && (data[++index] == 'e') && (data[++index] == '_') && (data[++index] == 'o')
                                                && (data[++index] == 'n') && (data[++index] == 'c') && (data[++index] == 'e'))
                                        return TokenNameinclude_once;
-                               else
-                                       return TokenNameIdentifier;
-                       default:
                                return TokenNameIdentifier;
                        }
+                       return TokenNameIdentifier;
                case 'l':
                        // list
                        if (length == 4) {
@@ -3449,17 +3295,15 @@ public class Scanner implements IScanner, ITerminalSymbols {
                        case 3:
                                if ((data[++index] == 'e') && (data[++index] == 'w'))
                                        return TokenNamenew;
-                               else
-                                       return TokenNameIdentifier;
+                               return TokenNameIdentifier;
                        // case 4 :
                        // if ((data[++index] == 'u') && (data[++index] == 'l')
                        // && (data[++index] == 'l'))
                        // return TokenNamenull;
                        // else
                        // return TokenNameIdentifier;
-                       default:
-                               return TokenNameIdentifier;
                        }
+                       return TokenNameIdentifier;
                case 'o':
                        // or old_function
                        if (length == 2) {
@@ -3489,26 +3333,26 @@ public class Scanner implements IScanner, ITerminalSymbols {
                        case 5:
                                if ((data[++index] == 'r') && (data[++index] == 'i') && (data[++index] == 'n') && (data[++index] == 't')) {
                                        return TokenNameprint;
-                               } else
-                                       return TokenNameIdentifier;
+                               }
+                               return TokenNameIdentifier;
                        case 6:
                                if ((data[++index] == 'u') && (data[++index] == 'b') && (data[++index] == 'l') && (data[++index] == 'i')
                                                && (data[++index] == 'c')) {
                                        return TokenNamepublic;
-                               } else
-                                       return TokenNameIdentifier;
+                               }
+                               return TokenNameIdentifier;
                        case 7:
                                if ((data[++index] == 'r') && (data[++index] == 'i') && (data[++index] == 'v') && (data[++index] == 'a')
                                                && (data[++index] == 't') && (data[++index] == 'e')) {
                                        return TokenNameprivate;
-                               } else
-                                       return TokenNameIdentifier;
+                               }
+                               return TokenNameIdentifier;
                        case 9:
                                if ((data[++index] == 'r') && (data[++index] == 'o') && (data[++index] == 't') && (data[++index] == 'e')
                                                && (data[++index] == 'c') && (data[++index] == 't') && (data[++index] == 'e') && (data[++index] == 'd')) {
                                        return TokenNameprotected;
-                               } else
-                                       return TokenNameIdentifier;
+                               }
+                               return TokenNameIdentifier;
                        }
                        return TokenNameIdentifier;
                case 'r':
@@ -3529,16 +3373,17 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                                && (data[++index] == 'n') && (data[++index] == 'c') && (data[++index] == 'e')) {
                                        return TokenNamerequire_once;
                                }
-                       } else
-                               return TokenNameIdentifier;
+                       }
+                       return TokenNameIdentifier;
                case 's':
                        // self static switch
                        switch (length) {
-//                     case 4:
-//                             if ((data[++index] == 'e') && (data[++index] == 'l') && (data[++index] == 'f')) {
-//                                     return TokenNameself;
-//                             }
-//                             return TokenNameIdentifier;
+                       // case 4:
+                       // if ((data[++index] == 'e') && (data[++index] == 'l') && (data[++index]
+                       // == 'f')) {
+                       // return TokenNameself;
+                       // }
+                       // return TokenNameIdentifier;
                        case 6:
                                if (data[++index] == 't')
                                        if ((data[++index] == 'a') && (data[++index] == 't') && (data[++index] == 'i') && (data[++index] == 'c')) {
@@ -3548,19 +3393,14 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                else if ((data[index] == 'w') && (data[++index] == 'i') && (data[++index] == 't') && (data[++index] == 'c')
                                                && (data[++index] == 'h'))
                                        return TokenNameswitch;
-                               else
-                                       return TokenNameIdentifier;
-                       default:
-                               return TokenNameIdentifier;
                        }
+                       return TokenNameIdentifier;
                case 't':
                        // try true throw
                        switch (length) {
                        case 3:
                                if ((data[++index] == 'r') && (data[++index] == 'y'))
                                        return TokenNametry;
-                               else
-                                       return TokenNameIdentifier;
                        // case 4 :
                        // if ((data[++index] == 'r') && (data[++index] == 'u')
                        // && (data[++index] == 'e'))
@@ -3570,55 +3410,41 @@ public class Scanner implements IScanner, ITerminalSymbols {
                        case 5:
                                if ((data[++index] == 'h') && (data[++index] == 'r') && (data[++index] == 'o') && (data[++index] == 'w'))
                                        return TokenNamethrow;
-                               else
-                                       return TokenNameIdentifier;
-                       default:
-                               return TokenNameIdentifier;
                        }
+                       return TokenNameIdentifier;
                case 'u':
                        // use unset
                        switch (length) {
                        case 3:
                                if ((data[++index] == 's') && (data[++index] == 'e'))
                                        return TokenNameuse;
-                               else
-                                       return TokenNameIdentifier;
                        case 5:
                                if ((data[++index] == 'n') && (data[++index] == 's') && (data[++index] == 'e') && (data[++index] == 't'))
                                        return TokenNameunset;
-                               else
-                                       return TokenNameIdentifier;
-                       default:
-                               return TokenNameIdentifier;
                        }
+                       return TokenNameIdentifier;
                case 'v':
                        // var
                        switch (length) {
                        case 3:
                                if ((data[++index] == 'a') && (data[++index] == 'r'))
                                        return TokenNamevar;
-                               else
-                                       return TokenNameIdentifier;
-                       default:
-                               return TokenNameIdentifier;
                        }
+                       return TokenNameIdentifier;
                case 'w':
                        // while
                        switch (length) {
                        case 5:
                                if ((data[++index] == 'h') && (data[++index] == 'i') && (data[++index] == 'l') && (data[++index] == 'e'))
                                        return TokenNamewhile;
-                               else
-                                       return TokenNameIdentifier;
                        // case 6:if ( (data[++index] =='i') && (data[++index]=='d') &&
                        // (data[++index]=='e') && (data[++index]=='f')&&
                        // (data[++index]=='p'))
                        // return TokenNamewidefp ;
                        // else
                        // return TokenNameIdentifier;
-                       default:
-                               return TokenNameIdentifier;
                        }
+                       return TokenNameIdentifier;
                case 'x':
                        // xor
                        switch (length) {
@@ -3627,12 +3453,10 @@ public class Scanner implements IScanner, ITerminalSymbols {
                                        return TokenNamexor;
                                else
                                        return TokenNameIdentifier;
-                       default:
-                               return TokenNameIdentifier;
                        }
-               default:
                        return TokenNameIdentifier;
                }
+               return TokenNameIdentifier;
        }
 
        public int scanNumber(boolean dotPrefix) throws InvalidInputException {
@@ -3832,6 +3656,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
                initialPosition = currentPosition = 0;
                containsAssertKeyword = false;
                withoutUnicodeBuffer = new char[this.source.length];
+               fFillerToken = TokenNameEOF;
                // encapsedStringStack = new Stack();
        }
 
@@ -3864,6 +3689,9 @@ public class Scanner implements IScanner, ITerminalSymbols {
                // //$NON-NLS-1$
                case TokenNameINLINE_HTML:
                        return "Inline-HTML(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
+               case    TokenNameECHO_INVISIBLE:
+                       //0-length token
+                       return "";
                case TokenNameIdentifier:
                        return "Identifier(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
                case TokenNameVariable:
@@ -3966,8 +3794,8 @@ public class Scanner implements IScanner, ITerminalSymbols {
                        return "require_once"; //$NON-NLS-1$
                case TokenNamereturn:
                        return "return"; //$NON-NLS-1$
-//             case TokenNameself:
-//                     return "self"; //$NON-NLS-1$
+                       // case TokenNameself:
+                       // return "self"; //$NON-NLS-1$
                case TokenNamestatic:
                        return "static"; //$NON-NLS-1$
                case TokenNameswitch: