X-Git-Url: http://secure.phpeclipse.com

diff --git a/net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java b/net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java
index 655dcef..a483ee4 100644
--- a/net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java
+++ b/net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java
@@ -126,6 +126,17 @@ public class Scanner implements IScanner, ITerminalSymbols {
       charArray_v = new char[] { 'v' }, charArray_w = new char[] { 'w' }, charArray_x = new char[] { 'x' },
       charArray_y = new char[] { 'y' }, charArray_z = new char[] { 'z' };
 
+  static final char[] charArray_va = new char[] { '$', 'a' }, charArray_vb = new char[] { '$', 'b' }, charArray_vc = new char[] {
+      '$',
+      'c' }, charArray_vd = new char[] { '$', 'd' }, charArray_ve = new char[] { '$', 'e' },
+      charArray_vf = new char[] { '$', 'f' }, charArray_vg = new char[] { '$', 'g' }, charArray_vh = new char[] { '$', 'h' },
+      charArray_vi = new char[] { '$', 'i' }, charArray_vj = new char[] { '$', 'j' }, charArray_vk = new char[] { '$', 'k' },
+      charArray_vl = new char[] { '$', 'l' }, charArray_vm = new char[] { '$', 'm' }, charArray_vn = new char[] { '$', 'n' },
+      charArray_vo = new char[] { '$', 'o' }, charArray_vp = new char[] { '$', 'p' }, charArray_vq = new char[] { '$', 'q' },
+      charArray_vr = new char[] { '$', 'r' }, charArray_vs = new char[] { '$', 's' }, charArray_vt = new char[] { '$', 't' },
+      charArray_vu = new char[] { '$', 'u' }, charArray_vv = new char[] { '$', 'v' }, charArray_vw = new char[] { '$', 'w' },
+      charArray_vx = new char[] { '$', 'x' }, charArray_vy = new char[] { '$', 'y' }, charArray_vz = new char[] { '$', 'z' };
+
   static final char[] initCharArray = new char[] { '\u0000', '\u0000', '\u0000', '\u0000', '\u0000', '\u0000' };
 
   static final int TableSize = 30, InternalTableSize = 6;
@@ -193,13 +204,28 @@ public class Scanner implements IScanner, ITerminalSymbols {
 
   public char[][] taskPriorities = null;
 
+  public boolean isTaskCaseSensitive = true;
+
   public static final boolean DEBUG = false;
 
   public static final boolean TRACE = false;
 
   public ICompilationUnit compilationUnit = null;
+
+  /**
+   * Determines if the specified character is permissible as the first character in a PHP identifier or variable
+   * 
+   * The '$' character for PHP variables is regarded as a correct first character !
+   *  
+   */
+  public static boolean isPHPIdentOrVarStart(char ch) {
+    return Character.isLetter(ch) || (ch == '$') || (ch == '_') || (0x7F <= ch && ch <= 0xFF);
+  }
+
   /**
-   * Determines if the specified character is permissible as the first character in a PHP identifier
+   * Determines if the specified character is permissible as the first character in a PHP identifier.
+   * 
+   * The '$' character for PHP variables isn't regarded as the first character !
    */
   public static boolean isPHPIdentifierStart(char ch) {
     return Character.isLetter(ch) || (ch == '_') || (0x7F <= ch && ch <= 0xFF);
@@ -307,6 +333,13 @@ public class Scanner implements IScanner, ITerminalSymbols {
     return result;
   }
 
+  public final char[] getRawTokenSourceEnd() {
+    int length = this.eofPosition - this.currentPosition - 1;
+    char[] sourceEnd = new char[length];
+    System.arraycopy(this.source, this.currentPosition, sourceEnd, 0, length);
+    return sourceEnd;
+  }
+
   public int getCurrentTokenStartPosition() {
     return this.startPosition;
   }
@@ -331,6 +364,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
     //    }
     return result;
   }
+
   /*
    * Search the source position corresponding to the end of a given line number
    * 
@@ -916,6 +950,10 @@ public class Scanner implements IScanner, ITerminalSymbols {
               withoutUnicodePtr--;
             }
           }
+        } else if ((currentCharacter == '\r') || (currentCharacter == '\n')) {
+          if (recordLineSeparator) {
+            pushLineSeparator();
+          }
         }
         // consume next character
         unicodeAsBackSlash = false;
@@ -1012,6 +1050,10 @@ public class Scanner implements IScanner, ITerminalSymbols {
               withoutUnicodePtr--;
             }
           }
+        } else if ((currentCharacter == '\r') || (currentCharacter == '\n')) {
+          if (recordLineSeparator) {
+            pushLineSeparator();
+          }
         }
         // consume next character
         unicodeAsBackSlash = false;
@@ -1108,6 +1150,10 @@ public class Scanner implements IScanner, ITerminalSymbols {
               withoutUnicodePtr--;
             }
           }
+        } else if ((currentCharacter == '\r') || (currentCharacter == '\n')) {
+          if (recordLineSeparator) {
+            pushLineSeparator();
+          }
         }
         // consume next character
         unicodeAsBackSlash = false;
@@ -1206,6 +1252,12 @@ public class Scanner implements IScanner, ITerminalSymbols {
                   }
                 }
                 break;
+              case '\r':
+              case '\n':
+                if (recordLineSeparator) {
+                  pushLineSeparator();
+                }
+                break;
               case '$':
                 if (isPHPIdentifierStart(source[currentPosition]) || source[currentPosition] == '{') {
                   currentPosition--;
@@ -1528,7 +1580,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
           case '#':
           case '/': {
             char startChar = currentCharacter;
-            if (getNextChar('=') && startChar=='/') {
+            if (getNextChar('=') && startChar == '/') {
               return TokenNameDIVIDE_EQUAL;
             }
             int test;
@@ -1751,6 +1803,10 @@ public class Scanner implements IScanner, ITerminalSymbols {
                     return TokenNameCOMMENT_PHPDOC;
                   return TokenNameCOMMENT_BLOCK;
                 }
+
+                if (this.taskTags != null) {
+                  checkTaskTag(this.startPosition, this.currentPosition);
+                }
               } catch (IndexOutOfBoundsException e) {
                 //                  reset end position for error reporting
                 currentPosition -= 2;
@@ -1834,7 +1890,6 @@ public class Scanner implements IScanner, ITerminalSymbols {
    * @throws InvalidInputException
    */
   private int getInlinedHTMLToken(int start) throws InvalidInputException {
-    //    int htmlPosition = start;
     if (currentPosition > source.length) {
       currentPosition = source.length;
       return TokenNameEOF;
@@ -1846,7 +1901,10 @@ public class Scanner implements IScanner, ITerminalSymbols {
         if (currentCharacter == '<') {
           if (getNextChar('?')) {
             currentCharacter = source[currentPosition++];
-            if ((currentCharacter == ' ') || Character.isWhitespace(currentCharacter)) {
+            if ((currentCharacter != 'P') && (currentCharacter != 'p')) {
+              if (currentCharacter != '=') { // <?=
+                currentPosition--;
+              }
               // <?
               if (ignorePHPOneLiner) {
                 if (lookAheadLinePHPTag() == TokenNameINLINE_HTML) {
@@ -1858,25 +1916,25 @@ public class Scanner implements IScanner, ITerminalSymbols {
                 return TokenNameINLINE_HTML;
               }
             } else {
-              boolean phpStart = (currentCharacter == 'P') || (currentCharacter == 'p');
-              if (phpStart) {
-                int test = getNextChar('H', 'h');
+              //              boolean phpStart = (currentCharacter == 'P') || (currentCharacter == 'p');
+              //              if (phpStart) {
+              int test = getNextChar('H', 'h');
+              if (test >= 0) {
+                test = getNextChar('P', 'p');
                 if (test >= 0) {
-                  test = getNextChar('P', 'p');
-                  if (test >= 0) {
-                    // <?PHP <?php
-                    if (ignorePHPOneLiner) {
-                      if (lookAheadLinePHPTag() == TokenNameINLINE_HTML) {
-                        phpMode = true;
-                        return TokenNameINLINE_HTML;
-                      }
-                    } else {
+                  // <?PHP <?php
+                  if (ignorePHPOneLiner) {
+                    if (lookAheadLinePHPTag() == TokenNameINLINE_HTML) {
                       phpMode = true;
                       return TokenNameINLINE_HTML;
                     }
+                  } else {
+                    phpMode = true;
+                    return TokenNameINLINE_HTML;
                   }
                 }
               }
+              //              }
             }
           }
         }
@@ -2466,9 +2524,69 @@ public class Scanner implements IScanner, ITerminalSymbols {
   }
 
   final char[] optimizedCurrentTokenSource2() {
-    //try to return the same char[] build only once
     char c0, c1;
-    int hash = (((c0 = source[startPosition]) << 6) + (c1 = source[startPosition + 1])) % TableSize;
+    c0 = source[startPosition];
+    c1 = source[startPosition + 1];
+    if (c0 == '$') {
+      //return always the same char[] build only once
+      //optimization at no speed cost of 99.5 % of the singleCharIdentifier
+      switch (c1) {
+      case 'a':
+        return charArray_va;
+      case 'b':
+        return charArray_vb;
+      case 'c':
+        return charArray_vc;
+      case 'd':
+        return charArray_vd;
+      case 'e':
+        return charArray_ve;
+      case 'f':
+        return charArray_vf;
+      case 'g':
+        return charArray_vg;
+      case 'h':
+        return charArray_vh;
+      case 'i':
+        return charArray_vi;
+      case 'j':
+        return charArray_vj;
+      case 'k':
+        return charArray_vk;
+      case 'l':
+        return charArray_vl;
+      case 'm':
+        return charArray_vm;
+      case 'n':
+        return charArray_vn;
+      case 'o':
+        return charArray_vo;
+      case 'p':
+        return charArray_vp;
+      case 'q':
+        return charArray_vq;
+      case 'r':
+        return charArray_vr;
+      case 's':
+        return charArray_vs;
+      case 't':
+        return charArray_vt;
+      case 'u':
+        return charArray_vu;
+      case 'v':
+        return charArray_vv;
+      case 'w':
+        return charArray_vw;
+      case 'x':
+        return charArray_vx;
+      case 'y':
+        return charArray_vy;
+      case 'z':
+        return charArray_vz;
+      }
+    }
+    //try to return the same char[] build only once
+    int hash = ((c0 << 6) + c1) % TableSize;
     char[][] table = charArray_length[0][hash];
     int i = newEntry2;
     while (++i < InternalTableSize) {
@@ -3641,7 +3759,7 @@ public class Scanner implements IScanner, ITerminalSymbols {
   public final void setSource(char[] source) {
     setSource(null, source);
   }
-  
+
   public final void setSource(ICompilationUnit compilationUnit, char[] source) {
     //the source-buffer is set to sourceString
     this.compilationUnit = compilationUnit;
@@ -3984,11 +4102,11 @@ public class Scanner implements IScanner, ITerminalSymbols {
 
   public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace, boolean checkNonExternalizedStringLiterals,
       boolean assertMode) {
-    this(tokenizeComments, tokenizeWhiteSpace, checkNonExternalizedStringLiterals, assertMode, false, null, null);
+    this(tokenizeComments, tokenizeWhiteSpace, checkNonExternalizedStringLiterals, assertMode, false, null, null, true);
   }
 
   public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace, boolean checkNonExternalizedStringLiterals,
-      boolean assertMode, boolean tokenizeStrings, char[][] taskTags, char[][] taskPriorities) {
+      boolean assertMode, boolean tokenizeStrings, char[][] taskTags, char[][] taskPriorities, boolean isTaskCaseSensitive) {
     this.eofPosition = Integer.MAX_VALUE;
     this.tokenizeComments = tokenizeComments;
     this.tokenizeWhiteSpace = tokenizeWhiteSpace;
@@ -4132,65 +4250,99 @@ public class Scanner implements IScanner, ITerminalSymbols {
     }
   }
 
-  // chech presence of task: tags
+  //chech presence of task: tags
+  //TODO (frederic) see if we need to take unicode characters into account...
   public void checkTaskTag(int commentStart, int commentEnd) {
+    char[] src = this.source;
+
     // only look for newer task: tags
     if (this.foundTaskCount > 0 && this.foundTaskPositions[this.foundTaskCount - 1][0] >= commentStart) {
       return;
     }
     int foundTaskIndex = this.foundTaskCount;
-    nextChar: for (int i = commentStart; i < commentEnd && i < this.eofPosition; i++) {
+    char previous = src[commentStart + 1]; // should be '*' or '/'
+    nextChar: for (int i = commentStart + 2; i < commentEnd && i < this.eofPosition; i++) {
       char[] tag = null;
       char[] priority = null;
-      // check for tag occurrence
-      nextTag: for (int itag = 0; itag < this.taskTags.length; itag++) {
-        tag = this.taskTags[itag];
-        priority = this.taskPriorities != null && itag < this.taskPriorities.length ? this.taskPriorities[itag] : null;
-        int tagLength = tag.length;
-        for (int t = 0; t < tagLength; t++) {
-          if (this.source[i + t] != tag[t])
+      // check for tag occurrence only if not ambiguous with javadoc tag
+      if (previous != '@') {
+        nextTag: for (int itag = 0; itag < this.taskTags.length; itag++) {
+          tag = this.taskTags[itag];
+          int tagLength = tag.length;
+          if (tagLength == 0)
             continue nextTag;
+
+          // ensure tag is not leaded with letter if tag starts with a letter
+          if (Scanner.isPHPIdentifierStart(tag[0])) {
+            if (Scanner.isPHPIdentifierPart(previous)) {
+              continue nextTag;
+            }
+          }
+
+          for (int t = 0; t < tagLength; t++) {
+            char sc, tc;
+            int x = i + t;
+            if (x >= this.eofPosition || x >= commentEnd)
+              continue nextTag;
+            if ((sc = src[i + t]) != (tc = tag[t])) { // case sensitive check
+              if (this.isTaskCaseSensitive || (Character.toLowerCase(sc) != Character.toLowerCase(tc))) { // case insensitive check
+                continue nextTag;
+              }
+            }
+          }
+          // ensure tag is not followed with letter if tag finishes with a letter
+          if (i + tagLength < commentEnd && Scanner.isPHPIdentifierPart(src[i + tagLength - 1])) {
+            if (Scanner.isPHPIdentifierPart(src[i + tagLength]))
+              continue nextTag;
+          }
+          if (this.foundTaskTags == null) {
+            this.foundTaskTags = new char[5][];
+            this.foundTaskMessages = new char[5][];
+            this.foundTaskPriorities = new char[5][];
+            this.foundTaskPositions = new int[5][];
+          } else if (this.foundTaskCount == this.foundTaskTags.length) {
+            System.arraycopy(this.foundTaskTags, 0, this.foundTaskTags = new char[this.foundTaskCount * 2][], 0,
+                this.foundTaskCount);
+            System.arraycopy(this.foundTaskMessages, 0, this.foundTaskMessages = new char[this.foundTaskCount * 2][], 0,
+                this.foundTaskCount);
+            System.arraycopy(this.foundTaskPriorities, 0, this.foundTaskPriorities = new char[this.foundTaskCount * 2][], 0,
+                this.foundTaskCount);
+            System.arraycopy(this.foundTaskPositions, 0, this.foundTaskPositions = new int[this.foundTaskCount * 2][], 0,
+                this.foundTaskCount);
+          }
+
+          priority = this.taskPriorities != null && itag < this.taskPriorities.length ? this.taskPriorities[itag] : null;
+
+          this.foundTaskTags[this.foundTaskCount] = tag;
+          this.foundTaskPriorities[this.foundTaskCount] = priority;
+          this.foundTaskPositions[this.foundTaskCount] = new int[] { i, i + tagLength - 1 };
+          this.foundTaskMessages[this.foundTaskCount] = CharOperation.NO_CHAR;
+          this.foundTaskCount++;
+          i += tagLength - 1; // will be incremented when looping
+          break nextTag;
         }
-        if (this.foundTaskTags == null) {
-          this.foundTaskTags = new char[5][];
-          this.foundTaskMessages = new char[5][];
-          this.foundTaskPriorities = new char[5][];
-          this.foundTaskPositions = new int[5][];
-        } else if (this.foundTaskCount == this.foundTaskTags.length) {
-          System.arraycopy(this.foundTaskTags, 0, this.foundTaskTags = new char[this.foundTaskCount * 2][], 0, this.foundTaskCount);
-          System.arraycopy(this.foundTaskMessages, 0, this.foundTaskMessages = new char[this.foundTaskCount * 2][], 0,
-              this.foundTaskCount);
-          System.arraycopy(this.foundTaskPriorities, 0, this.foundTaskPriorities = new char[this.foundTaskCount * 2][], 0,
-              this.foundTaskCount);
-          System.arraycopy(this.foundTaskPositions, 0, this.foundTaskPositions = new int[this.foundTaskCount * 2][], 0,
-              this.foundTaskCount);
-        }
-        this.foundTaskTags[this.foundTaskCount] = tag;
-        this.foundTaskPriorities[this.foundTaskCount] = priority;
-        this.foundTaskPositions[this.foundTaskCount] = new int[] { i, i + tagLength - 1 };
-        this.foundTaskMessages[this.foundTaskCount] = CharOperation.NO_CHAR;
-        this.foundTaskCount++;
-        i += tagLength - 1; // will be incremented when looping
       }
+      previous = src[i];
     }
     for (int i = foundTaskIndex; i < this.foundTaskCount; i++) {
       // retrieve message start and end positions
       int msgStart = this.foundTaskPositions[i][0] + this.foundTaskTags[i].length;
       int max_value = i + 1 < this.foundTaskCount ? this.foundTaskPositions[i + 1][0] - 1 : commentEnd - 1;
       // at most beginning of next task
-      if (max_value < msgStart)
+      if (max_value < msgStart) {
         max_value = msgStart; // would only occur if tag is before EOF.
+      }
       int end = -1;
       char c;
       for (int j = msgStart; j < max_value; j++) {
-        if ((c = this.source[j]) == '\n' || c == '\r') {
+        if ((c = src[j]) == '\n' || c == '\r') {
           end = j - 1;
           break;
         }
       }
       if (end == -1) {
         for (int j = max_value; j > msgStart; j--) {
-          if ((c = this.source[j]) == '*') {
+          if ((c = src[j]) == '*') {
             end = j - 1;
             break;
           }
@@ -4201,17 +4353,100 @@ public class Scanner implements IScanner, ITerminalSymbols {
       if (msgStart == end)
         continue; // empty
       // trim the message
-      while (CharOperation.isWhitespace(source[end]) && msgStart <= end)
+      while (CharOperation.isWhitespace(src[end]) && msgStart <= end)
         end--;
-      while (CharOperation.isWhitespace(source[msgStart]) && msgStart <= end)
+      while (CharOperation.isWhitespace(src[msgStart]) && msgStart <= end)
         msgStart++;
       // update the end position of the task
       this.foundTaskPositions[i][1] = end;
       // get the message source
       final int messageLength = end - msgStart + 1;
       char[] message = new char[messageLength];
-      System.arraycopy(source, msgStart, message, 0, messageLength);
+      System.arraycopy(src, msgStart, message, 0, messageLength);
       this.foundTaskMessages[i] = message;
     }
   }
+
+  // chech presence of task: tags
+  //  public void checkTaskTag(int commentStart, int commentEnd) {
+  //    // only look for newer task: tags
+  //    if (this.foundTaskCount > 0 && this.foundTaskPositions[this.foundTaskCount - 1][0] >= commentStart) {
+  //      return;
+  //    }
+  //    int foundTaskIndex = this.foundTaskCount;
+  //    nextChar: for (int i = commentStart; i < commentEnd && i < this.eofPosition; i++) {
+  //      char[] tag = null;
+  //      char[] priority = null;
+  //      // check for tag occurrence
+  //      nextTag: for (int itag = 0; itag < this.taskTags.length; itag++) {
+  //        tag = this.taskTags[itag];
+  //        priority = this.taskPriorities != null && itag < this.taskPriorities.length ? this.taskPriorities[itag] : null;
+  //        int tagLength = tag.length;
+  //        for (int t = 0; t < tagLength; t++) {
+  //          if (this.source[i + t] != tag[t])
+  //            continue nextTag;
+  //        }
+  //        if (this.foundTaskTags == null) {
+  //          this.foundTaskTags = new char[5][];
+  //          this.foundTaskMessages = new char[5][];
+  //          this.foundTaskPriorities = new char[5][];
+  //          this.foundTaskPositions = new int[5][];
+  //        } else if (this.foundTaskCount == this.foundTaskTags.length) {
+  //          System.arraycopy(this.foundTaskTags, 0, this.foundTaskTags = new char[this.foundTaskCount * 2][], 0, this.foundTaskCount);
+  //          System.arraycopy(this.foundTaskMessages, 0, this.foundTaskMessages = new char[this.foundTaskCount * 2][], 0,
+  //              this.foundTaskCount);
+  //          System.arraycopy(this.foundTaskPriorities, 0, this.foundTaskPriorities = new char[this.foundTaskCount * 2][], 0,
+  //              this.foundTaskCount);
+  //          System.arraycopy(this.foundTaskPositions, 0, this.foundTaskPositions = new int[this.foundTaskCount * 2][], 0,
+  //              this.foundTaskCount);
+  //        }
+  //        this.foundTaskTags[this.foundTaskCount] = tag;
+  //        this.foundTaskPriorities[this.foundTaskCount] = priority;
+  //        this.foundTaskPositions[this.foundTaskCount] = new int[] { i, i + tagLength - 1 };
+  //        this.foundTaskMessages[this.foundTaskCount] = CharOperation.NO_CHAR;
+  //        this.foundTaskCount++;
+  //        i += tagLength - 1; // will be incremented when looping
+  //      }
+  //    }
+  //    for (int i = foundTaskIndex; i < this.foundTaskCount; i++) {
+  //      // retrieve message start and end positions
+  //      int msgStart = this.foundTaskPositions[i][0] + this.foundTaskTags[i].length;
+  //      int max_value = i + 1 < this.foundTaskCount ? this.foundTaskPositions[i + 1][0] - 1 : commentEnd - 1;
+  //      // at most beginning of next task
+  //      if (max_value < msgStart)
+  //        max_value = msgStart; // would only occur if tag is before EOF.
+  //      int end = -1;
+  //      char c;
+  //      for (int j = msgStart; j < max_value; j++) {
+  //        if ((c = this.source[j]) == '\n' || c == '\r') {
+  //          end = j - 1;
+  //          break;
+  //        }
+  //      }
+  //      if (end == -1) {
+  //        for (int j = max_value; j > msgStart; j--) {
+  //          if ((c = this.source[j]) == '*') {
+  //            end = j - 1;
+  //            break;
+  //          }
+  //        }
+  //        if (end == -1)
+  //          end = max_value;
+  //      }
+  //      if (msgStart == end)
+  //        continue; // empty
+  //      // trim the message
+  //      while (CharOperation.isWhitespace(source[end]) && msgStart <= end)
+  //        end--;
+  //      while (CharOperation.isWhitespace(source[msgStart]) && msgStart <= end)
+  //        msgStart++;
+  //      // update the end position of the task
+  //      this.foundTaskPositions[i][1] = end;
+  //      // get the message source
+  //      final int messageLength = end - msgStart + 1;
+  //      char[] message = new char[messageLength];
+  //      System.arraycopy(source, msgStart, message, 0, messageLength);
+  //      this.foundTaskMessages[i] = message;
+  //    }
+  //  }
 }
\ No newline at end of file