X-Git-Url: http://secure.phpeclipse.com

diff --git a/net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java b/net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java
index 36adb95..c897fdb 100644
--- a/net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java
+++ b/net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java
@@ -156,12 +156,35 @@ public class Scanner implements IScanner, ITerminalSymbols {
   public static final int SquareBracket = 1;
   public static final int CurlyBracket = 2;
   public static final int BracketKinds = 3;
+
+  public static final boolean DEBUG = false;
   public Scanner() {
     this(false, false);
   }
   public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace) {
     this(tokenizeComments, tokenizeWhiteSpace, false);
   }
+
+  /**
+   * Determines if the specified character is
+   * permissible as the first character in a PHP identifier
+   */
+  public static boolean isPHPIdentifierStart(char ch) {
+    return Character.isLetter(ch)
+      || (ch == '_')
+      || (0x7F <= ch && ch <= 0xFF);
+  }
+
+  /**
+   * Determines if the specified character may be part of a PHP identifier as
+   * other than the first character
+   */
+  public static boolean isPHPIdentifierPart(char ch) {
+    return Character.isLetterOrDigit(ch)
+      || (ch == '_')
+      || (0x7F <= ch && ch <= 0xFF);
+  }
+
   public final boolean atEnd() {
     // This code is not relevant if source is 
     // Only a part of the real stream input
@@ -232,7 +255,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
     }
     return result;
   }
-  
+
   public final char[] getCurrentTokenSource(int startPos) {
     // Return the token REAL source (aka unicodes are precomputed)
 
@@ -256,7 +279,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
     }
     return result;
   }
-  
+
   public final char[] getCurrentTokenSourceString() {
     //return the token REAL source (aka unicodes are precomputed).
     //REMOVE the two " that are at the beginning and the end.
@@ -663,7 +686,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
         }
 
         currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
-        if (!Character.isJavaIdentifierPart(currentCharacter)) {
+        if (!isPHPIdentifierPart(currentCharacter)) {
           currentPosition = temp;
           return false;
         }
@@ -684,7 +707,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
         return true;
       } //-------------end unicode traitement--------------
       else {
-        if (!Character.isJavaIdentifierPart(currentCharacter)) {
+        if (!isPHPIdentifierPart(currentCharacter)) {
           currentPosition = temp;
           return false;
         }
@@ -700,9 +723,9 @@ public class Scanner implements IScanner, ITerminalSymbols {
   }
 
   public int getNextToken() throws InvalidInputException {
+    int htmlPosition = currentPosition;
     try {
       while (!phpMode) {
-        startPosition = currentPosition;
         currentCharacter = source[currentPosition++];
         if (currentCharacter == '<') {
           if (getNextChar('?')) {
@@ -712,6 +735,12 @@ public class Scanner implements IScanner, ITerminalSymbols {
               // <?
               startPosition = currentPosition;
               phpMode = true;
+              if (tokenizeWhiteSpace) {
+                // && (whiteStart != currentPosition - 1)) {
+                // reposition scanner in case we are interested by spaces as tokens
+                startPosition = htmlPosition;
+                return TokenNameHTML;
+              }
             } else {
               boolean phpStart =
                 (currentCharacter == 'P') || (currentCharacter == 'p');
@@ -723,6 +752,13 @@ public class Scanner implements IScanner, ITerminalSymbols {
                     // <?PHP  <?php
                     startPosition = currentPosition;
                     phpMode = true;
+
+                    if (tokenizeWhiteSpace) {
+                      // && (whiteStart != currentPosition - 1)) {
+                      // reposition scanner in case we are interested by spaces as tokens
+                      startPosition = htmlPosition;
+                      return TokenNameHTML;
+                    }
                   }
                 }
               }
@@ -740,6 +776,11 @@ public class Scanner implements IScanner, ITerminalSymbols {
       }
     } //-----------------end switch while try--------------------
     catch (IndexOutOfBoundsException e) {
+      if (tokenizeWhiteSpace) {
+        // && (whiteStart != currentPosition - 1)) {
+        // reposition scanner in case we are interested by spaces as tokens
+        startPosition = htmlPosition;
+      }
       return TokenNameEOF;
     }
 
@@ -860,12 +901,12 @@ public class Scanner implements IScanner, ITerminalSymbols {
                     int heredocStart = currentPosition;
                     int heredocLength = 0;
                     currentCharacter = source[currentPosition++];
-                    if (Character.isJavaIdentifierStart(currentCharacter)) {
+                    if (isPHPIdentifierStart(currentCharacter)) {
                       currentCharacter = source[currentPosition++];
                     } else {
                       return TokenNameERROR;
                     }
-                    while (Character.isJavaIdentifierPart(currentCharacter)) {
+                    while (isPHPIdentifierPart(currentCharacter)) {
                       currentCharacter = source[currentPosition++];
                     }
 
@@ -1047,28 +1088,28 @@ public class Scanner implements IScanner, ITerminalSymbols {
                 }
 
                 while (currentCharacter != '\'') {
-                  
+
                   /**** in PHP \r and \n are valid in string literals ****/
-//                  if ((currentCharacter == '\n')
-//                    || (currentCharacter == '\r')) {
-//                    // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
-//                    for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
-//                      if (currentPosition + lookAhead == source.length)
-//                        break;
-//                      if (source[currentPosition + lookAhead] == '\n')
-//                        break;
-//                      if (source[currentPosition + lookAhead] == '\"') {
-//                        currentPosition += lookAhead + 1;
-//                        break;
-//                      }
-//                    }
-//                    throw new InvalidInputException(INVALID_CHAR_IN_STRING);
-//                  }
+                  //                  if ((currentCharacter == '\n')
+                  //                    || (currentCharacter == '\r')) {
+                  //                    // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
+                  //                    for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
+                  //                      if (currentPosition + lookAhead == source.length)
+                  //                        break;
+                  //                      if (source[currentPosition + lookAhead] == '\n')
+                  //                        break;
+                  //                      if (source[currentPosition + lookAhead] == '\"') {
+                  //                        currentPosition += lookAhead + 1;
+                  //                        break;
+                  //                      }
+                  //                    }
+                  //                    throw new InvalidInputException(INVALID_CHAR_IN_STRING);
+                  //                  }
                   if (currentCharacter == '\\') {
                     int escapeSize = currentPosition;
                     boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
                     //scanEscapeCharacter make a side effect on this value and we need the previous value few lines down this one
-                    scanEscapeCharacter();
+                    scanSingleQuotedEscapeCharacter();
                     escapeSize = currentPosition - escapeSize;
                     if (withoutUnicodePtr == 0) {
                       //buffer all the entries that have been left aside....
@@ -1149,29 +1190,28 @@ public class Scanner implements IScanner, ITerminalSymbols {
                 }
 
                 while (currentCharacter != '"') {
-                  
-                  
+
                   /**** in PHP \r and \n are valid in string literals ****/
-//                  if ((currentCharacter == '\n')
-//                    || (currentCharacter == '\r')) {
-//                    // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
-//                    for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
-//                      if (currentPosition + lookAhead == source.length)
-//                        break;
-//                      if (source[currentPosition + lookAhead] == '\n')
-//                        break;
-//                      if (source[currentPosition + lookAhead] == '\"') {
-//                        currentPosition += lookAhead + 1;
-//                        break;
-//                      }
-//                    }
-//                    throw new InvalidInputException(INVALID_CHAR_IN_STRING);
-//                  }
+                  //                  if ((currentCharacter == '\n')
+                  //                    || (currentCharacter == '\r')) {
+                  //                    // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
+                  //                    for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
+                  //                      if (currentPosition + lookAhead == source.length)
+                  //                        break;
+                  //                      if (source[currentPosition + lookAhead] == '\n')
+                  //                        break;
+                  //                      if (source[currentPosition + lookAhead] == '\"') {
+                  //                        currentPosition += lookAhead + 1;
+                  //                        break;
+                  //                      }
+                  //                    }
+                  //                    throw new InvalidInputException(INVALID_CHAR_IN_STRING);
+                  //                  }
                   if (currentCharacter == '\\') {
                     int escapeSize = currentPosition;
                     boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
                     //scanEscapeCharacter make a side effect on this value and we need the previous value few lines down this one
-                    scanEscapeCharacter();
+                    scanDoubleQuotedEscapeCharacter();
                     escapeSize = currentPosition - escapeSize;
                     if (withoutUnicodePtr == 0) {
                       //buffer all the entries that have been left aside....
@@ -1238,64 +1278,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
               }
               return TokenNameStringLiteral;
             case '`' :
-            try {
-              // consume next character
-              unicodeAsBackSlash = false;
-              if (((currentCharacter = source[currentPosition++]) == '\\')
-                && (source[currentPosition] == 'u')) {
-                getNextUnicodeChar();
-              } else {
-                if (withoutUnicodePtr != 0) {
-                  withoutUnicodeBuffer[++withoutUnicodePtr] =
-                    currentCharacter;
-                }
-              }
-
-              while (currentCharacter != '`') {
-                  
-                  
-                /**** in PHP \r and \n are valid in string literals ****/
-//                if ((currentCharacter == '\n')
-//                  || (currentCharacter == '\r')) {
-//                  // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
-//                  for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
-//                    if (currentPosition + lookAhead == source.length)
-//                      break;
-//                    if (source[currentPosition + lookAhead] == '\n')
-//                      break;
-//                    if (source[currentPosition + lookAhead] == '\"') {
-//                      currentPosition += lookAhead + 1;
-//                      break;
-//                    }
-//                  }
-//                  throw new InvalidInputException(INVALID_CHAR_IN_STRING);
-//                }
-                if (currentCharacter == '\\') {
-                  int escapeSize = currentPosition;
-                  boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
-                  //scanEscapeCharacter make a side effect on this value and we need the previous value few lines down this one
-                  scanEscapeCharacter();
-                  escapeSize = currentPosition - escapeSize;
-                  if (withoutUnicodePtr == 0) {
-                    //buffer all the entries that have been left aside....
-                    withoutUnicodePtr =
-                      currentPosition - escapeSize - 1 - startPosition;
-                    System.arraycopy(
-                      source,
-                      startPosition,
-                      withoutUnicodeBuffer,
-                      1,
-                      withoutUnicodePtr);
-                    withoutUnicodeBuffer[++withoutUnicodePtr] =
-                      currentCharacter;
-                  } else { //overwrite the / in the buffer
-                    withoutUnicodeBuffer[withoutUnicodePtr] =
-                      currentCharacter;
-                    if (backSlashAsUnicodeInString) { //there are TWO \ in the stream where only one is correct
-                      withoutUnicodePtr--;
-                    }
-                  }
-                }
+              try {
                 // consume next character
                 unicodeAsBackSlash = false;
                 if (((currentCharacter = source[currentPosition++]) == '\\')
@@ -1308,38 +1291,94 @@ public class Scanner implements IScanner, ITerminalSymbols {
                   }
                 }
 
-              }
-            } catch (IndexOutOfBoundsException e) {
-              throw new InvalidInputException(UNTERMINATED_STRING);
-            } catch (InvalidInputException e) {
-              if (e.getMessage().equals(INVALID_ESCAPE)) {
-                // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
-                for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
-                  if (currentPosition + lookAhead == source.length)
-                    break;
-                  if (source[currentPosition + lookAhead] == '\n')
-                    break;
-                  if (source[currentPosition + lookAhead] == '`') {
-                    currentPosition += lookAhead + 1;
-                    break;
+                while (currentCharacter != '`') {
+
+                  /**** in PHP \r and \n are valid in string literals ****/
+                  //                if ((currentCharacter == '\n')
+                  //                  || (currentCharacter == '\r')) {
+                  //                  // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
+                  //                  for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
+                  //                    if (currentPosition + lookAhead == source.length)
+                  //                      break;
+                  //                    if (source[currentPosition + lookAhead] == '\n')
+                  //                      break;
+                  //                    if (source[currentPosition + lookAhead] == '\"') {
+                  //                      currentPosition += lookAhead + 1;
+                  //                      break;
+                  //                    }
+                  //                  }
+                  //                  throw new InvalidInputException(INVALID_CHAR_IN_STRING);
+                  //                }
+                  if (currentCharacter == '\\') {
+                    int escapeSize = currentPosition;
+                    boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
+                    //scanEscapeCharacter make a side effect on this value and we need the previous value few lines down this one
+                    scanDoubleQuotedEscapeCharacter();
+                    escapeSize = currentPosition - escapeSize;
+                    if (withoutUnicodePtr == 0) {
+                      //buffer all the entries that have been left aside....
+                      withoutUnicodePtr =
+                        currentPosition - escapeSize - 1 - startPosition;
+                      System.arraycopy(
+                        source,
+                        startPosition,
+                        withoutUnicodeBuffer,
+                        1,
+                        withoutUnicodePtr);
+                      withoutUnicodeBuffer[++withoutUnicodePtr] =
+                        currentCharacter;
+                    } else { //overwrite the / in the buffer
+                      withoutUnicodeBuffer[withoutUnicodePtr] =
+                        currentCharacter;
+                      if (backSlashAsUnicodeInString) { //there are TWO \ in the stream where only one is correct
+                        withoutUnicodePtr--;
+                      }
+                    }
+                  }
+                  // consume next character
+                  unicodeAsBackSlash = false;
+                  if (((currentCharacter = source[currentPosition++]) == '\\')
+                    && (source[currentPosition] == 'u')) {
+                    getNextUnicodeChar();
+                  } else {
+                    if (withoutUnicodePtr != 0) {
+                      withoutUnicodeBuffer[++withoutUnicodePtr] =
+                        currentCharacter;
+                    }
                   }
+
                 }
+              } catch (IndexOutOfBoundsException e) {
+                throw new InvalidInputException(UNTERMINATED_STRING);
+              } catch (InvalidInputException e) {
+                if (e.getMessage().equals(INVALID_ESCAPE)) {
+                  // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
+                  for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
+                    if (currentPosition + lookAhead == source.length)
+                      break;
+                    if (source[currentPosition + lookAhead] == '\n')
+                      break;
+                    if (source[currentPosition + lookAhead] == '`') {
+                      currentPosition += lookAhead + 1;
+                      break;
+                    }
+                  }
 
+                }
+                throw e; // rethrow
               }
-              throw e; // rethrow
-            }
-            if (checkNonExternalizedStringLiterals) { // check for presence of	NLS tags //$NON-NLS-?$ where ? is an int.
-              if (currentLine == null) {
-                currentLine = new NLSLine();
-                lines.add(currentLine);
+              if (checkNonExternalizedStringLiterals) { // check for presence of	NLS tags //$NON-NLS-?$ where ? is an int.
+                if (currentLine == null) {
+                  currentLine = new NLSLine();
+                  lines.add(currentLine);
+                }
+                currentLine.add(
+                  new StringLiteral(
+                    getCurrentTokenSourceString(),
+                    startPosition,
+                    currentPosition - 1));
               }
-              currentLine.add(
-                new StringLiteral(
-                  getCurrentTokenSourceString(),
-                  startPosition,
-                  currentPosition - 1));
-            }
-            return TokenNameStringInterpolated;
+              return TokenNameStringInterpolated;
             case '#' :
             case '/' :
               {
@@ -1548,7 +1587,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
                     recordComment(isJavadoc);
                     if (tokenizeComments) {
                       if (isJavadoc)
-                        return TokenNameCOMMENT_JAVADOC;
+                        return TokenNameCOMMENT_PHPDOC;
                       return TokenNameCOMMENT_BLOCK;
                     }
                   } catch (IndexOutOfBoundsException e) {
@@ -1568,11 +1607,15 @@ public class Scanner implements IScanner, ITerminalSymbols {
 
             default :
               if (currentCharacter == '$') {
-                if (getNextChar('{'))
+                while ((currentCharacter = source[currentPosition++]) == '$') {
+                }
+                if (currentCharacter == '{')
                   return TokenNameDOLLAR_LBRACE;
-                return scanIdentifierOrKeyword(true);
+                if (isPHPIdentifierStart(currentCharacter))
+                  return scanIdentifierOrKeyword(true);
+                return TokenNameERROR;
               }
-              if (Character.isJavaIdentifierStart(currentCharacter))
+              if (isPHPIdentifierStart(currentCharacter))
                 return scanIdentifierOrKeyword(false);
               if (Character.isDigit(currentCharacter))
                 return scanNumber(false);
@@ -1669,7 +1712,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
               test = getNextChar('\\');
               if (test) {
                 try {
-                  scanEscapeCharacter();
+                  scanDoubleQuotedEscapeCharacter();
                 } catch (InvalidInputException ex) {
                 };
               } else {
@@ -1718,7 +1761,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
                 }
                 if (currentCharacter == '\\') {
                   try {
-                    scanEscapeCharacter();
+                    scanDoubleQuotedEscapeCharacter();
                   } catch (InvalidInputException ex) {
                   };
                 }
@@ -1930,7 +1973,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
             }
 
           default :
-            if (Character.isJavaIdentifierStart(currentCharacter)
+            if (isPHPIdentifierStart(currentCharacter)
               || currentCharacter == '$') {
               try {
                 scanIdentifierOrKeyword((currentCharacter == '$'));
@@ -2070,6 +2113,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
         return new char[] { charOne };
     }
   }
+
   final char[] optimizedCurrentTokenSource2() {
     //try to return the same char[] build only once
 
@@ -2100,6 +2144,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
     newEntry2 = max;
     return r;
   }
+
   final char[] optimizedCurrentTokenSource3() {
     //try to return the same char[] build only once
 
@@ -2132,6 +2177,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
     newEntry3 = max;
     return r;
   }
+
   final char[] optimizedCurrentTokenSource4() {
     //try to return the same char[] build only once
 
@@ -2172,6 +2218,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
     return r;
 
   }
+
   final char[] optimizedCurrentTokenSource5() {
     //try to return the same char[] build only once
 
@@ -2215,6 +2262,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
     return r;
 
   }
+
   final char[] optimizedCurrentTokenSource6() {
     //try to return the same char[] build only once
 
@@ -2260,6 +2308,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
     newEntry6 = max;
     return r;
   }
+
   public final void pushLineSeparator() throws InvalidInputException {
     //see comment on isLineDelimiter(char) for the use of '\n' and '\r'
     final int INCREMENT = 250;
@@ -2421,7 +2470,8 @@ public class Scanner implements IScanner, ITerminalSymbols {
     commentPtr = -1; // reset comment stack
   }
 
-  public final void scanEscapeCharacter() throws InvalidInputException {
+  public final void scanSingleQuotedEscapeCharacter()
+    throws InvalidInputException {
     // the string with "\\u" is a legal string of two chars \ and u
     //thus we use a direct access to the source (for regular cases).
 
@@ -2439,18 +2489,49 @@ public class Scanner implements IScanner, ITerminalSymbols {
     } else
       currentCharacter = source[currentPosition++];
     switch (currentCharacter) {
-      case 'b' :
-        currentCharacter = '\b';
+      case '\'' :
+        currentCharacter = '\'';
+        break;
+      case '\\' :
+        currentCharacter = '\\';
         break;
+      default :
+        currentCharacter = '\\';
+        currentPosition--;
+    }
+  }
+
+  public final void scanDoubleQuotedEscapeCharacter()
+    throws InvalidInputException {
+    // the string with "\\u" is a legal string of two chars \ and u
+    //thus we use a direct access to the source (for regular cases).
+
+    if (unicodeAsBackSlash) {
+      // consume next character
+      unicodeAsBackSlash = false;
+      if (((currentCharacter = source[currentPosition++]) == '\\')
+        && (source[currentPosition] == 'u')) {
+        getNextUnicodeChar();
+      } else {
+        if (withoutUnicodePtr != 0) {
+          withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
+        }
+      }
+    } else
+      currentCharacter = source[currentPosition++];
+    switch (currentCharacter) {
+      //      case 'b' :
+      //        currentCharacter = '\b';
+      //        break;
       case 't' :
         currentCharacter = '\t';
         break;
       case 'n' :
         currentCharacter = '\n';
         break;
-      case 'f' :
-        currentCharacter = '\f';
-        break;
+        //      case 'f' :
+        //        currentCharacter = '\f';
+        //        break;
       case 'r' :
         currentCharacter = '\r';
         break;
@@ -2463,6 +2544,9 @@ public class Scanner implements IScanner, ITerminalSymbols {
       case '\\' :
         currentCharacter = '\\';
         break;
+      case '$' :
+        currentCharacter = '$';
+        break;
       default :
         // -----------octal escape--------------
         // OctalDigit
@@ -2522,6 +2606,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
     //disptach on the second char :-)...cool....but fast !
 
     useAssertAsAnIndentifier = false;
+
     while (getNextCharAsJavaIdentifierPart()) {
     };