net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java

   1 /***********************************************************************************************************************************
   2  * Copyright (c) 2000, 2001, 2002 International Business Machines Corp. and others. All rights reserved. This program and the
   3  * accompanying materials are made available under the terms of the Common Public License v0.5 which accompanies this distribution,
   4  * and is available at http://www.eclipse.org/legal/cpl-v05.html
   5  *
   6  * Contributors: IBM Corporation - initial API and implementation
   7  **********************************************************************************************************************************/
   8 package net.sourceforge.phpdt.internal.compiler.parser;
   9
  10 import java.util.ArrayList;
  11 import java.util.Iterator;
  12 import java.util.List;
  13
  14 import net.sourceforge.phpdt.core.compiler.CharOperation;
  15 import net.sourceforge.phpdt.core.compiler.IScanner;
  16 import net.sourceforge.phpdt.core.compiler.ITerminalSymbols;
  17 import net.sourceforge.phpdt.core.compiler.InvalidInputException;
  18 import net.sourceforge.phpdt.internal.compiler.ast.StringLiteral;
  19 import net.sourceforge.phpdt.internal.compiler.env.ICompilationUnit;
  20
  21 public class Scanner implements IScanner, ITerminalSymbols {
  22         /*
  23          * APIs ares - getNextToken() which return the current type of the token
  24          * (this value is not memorized by the scanner) - getCurrentTokenSource()
  25          * which provides with the token "REAL" source (aka all unicode have been
  26          * transformed into a correct char) - sourceStart gives the position into
  27          * the stream - currentPosition-1 gives the sourceEnd position into the
  28          * stream
  29          */
  30         // 1.4 feature
  31         // private boolean assertMode;
  32         public boolean useAssertAsAnIndentifier = false;
  33
  34         // flag indicating if processed source contains occurrences of keyword
  35         // assert
  36         public boolean containsAssertKeyword = false;
  37
  38         public boolean recordLineSeparator;
  39
  40         public boolean ignorePHPOneLiner = false;
  41
  42         public boolean phpMode = false;
  43
  44         /**
  45          * This token is set to TokenName.echo if a short tag block begins (i.e.
  46          * &gt;?= ... ) Directly after the &quot;=&quot; character the
  47          * getNextToken() method returns TokenName.INLINE_HTML In the next call to
  48          * the getNextToken() method the value of fFillerToken (==TokenName.echo) is
  49          * returned
  50          *
  51          */
  52         TokenName fFillerToken = TokenName.EOF;
  53
  54         public char currentCharacter;
  55
  56         public int startPosition;
  57
  58         public int currentPosition;
  59
  60         public int initialPosition, eofPosition;
  61
  62         // after this position eof are generated instead of real token from the
  63         // source
  64         public boolean tokenizeComments;
  65
  66         public boolean tokenizeWhiteSpace;
  67
  68         public boolean tokenizeStrings;
  69
  70         // source should be viewed as a window (aka a part)
  71         // of a entire very large stream
  72         public char source[];
  73
  74         // unicode support
  75         public char[] withoutUnicodeBuffer;
  76
  77         public int withoutUnicodePtr;
  78
  79         // when == 0 ==> no unicode in the current token
  80         public boolean unicodeAsBackSlash = false;
  81
  82         public boolean scanningFloatLiteral = false;
  83
  84         // support for /** comments
  85         public int[] commentStops = new int[10];
  86
  87         public int[] commentStarts = new int[10];
  88
  89         public int commentPtr = -1; // no comment test with commentPtr value -1
  90
  91         protected int lastCommentLinePosition = -1;
  92
  93         // diet parsing support - jump over some method body when requested
  94         public boolean diet = false;
  95
  96         // support for the poor-line-debuggers ....
  97         // remember the position of the cr/lf
  98         public int[] lineEnds = new int[250];
  99
 100         public int linePtr = -1;
 101
 102         public boolean wasAcr = false;
 103
 104         public static final String END_OF_SOURCE = "End_Of_Source"; //$NON-NLS-1$
 105
 106         public static final String INVALID_HEXA = "Invalid_Hexa_Literal"; //$NON-NLS-1$
 107
 108         public static final String INVALID_OCTAL = "Invalid_Octal_Literal"; //$NON-NLS-1$
 109
 110         public static final String INVALID_CHARACTER_CONSTANT = "Invalid_Character_Constant"; //$NON-NLS-1$
 111
 112         public static final String INVALID_ESCAPE = "Invalid_Escape"; //$NON-NLS-1$
 113
 114         public static final String INVALID_INPUT = "Invalid_Input"; //$NON-NLS-1$
 115
 116         public static final String INVALID_UNICODE_ESCAPE = "Invalid_Unicode_Escape"; //$NON-NLS-1$
 117
 118         public static final String INVALID_FLOAT = "Invalid_Float_Literal"; //$NON-NLS-1$
 119
 120         public static final String NULL_SOURCE_STRING = "Null_Source_String"; //$NON-NLS-1$
 121
 122         public static final String UNTERMINATED_STRING = "Unterminated_String"; //$NON-NLS-1$
 123
 124         public static final String UNTERMINATED_COMMENT = "Unterminated_Comment"; //$NON-NLS-1$
 125
 126         public static final String INVALID_CHAR_IN_STRING = "Invalid_Char_In_String"; //$NON-NLS-1$
 127
 128         // ----------------optimized identifier managment------------------
 129         static final char[] charArray_a = new char[] { 'a' },
 130                         charArray_b = new char[] { 'b' }, charArray_c = new char[] { 'c' },
 131                         charArray_d = new char[] { 'd' }, charArray_e = new char[] { 'e' },
 132                         charArray_f = new char[] { 'f' }, charArray_g = new char[] { 'g' },
 133                         charArray_h = new char[] { 'h' }, charArray_i = new char[] { 'i' },
 134                         charArray_j = new char[] { 'j' }, charArray_k = new char[] { 'k' },
 135                         charArray_l = new char[] { 'l' }, charArray_m = new char[] { 'm' },
 136                         charArray_n = new char[] { 'n' }, charArray_o = new char[] { 'o' },
 137                         charArray_p = new char[] { 'p' }, charArray_q = new char[] { 'q' },
 138                         charArray_r = new char[] { 'r' }, charArray_s = new char[] { 's' },
 139                         charArray_t = new char[] { 't' }, charArray_u = new char[] { 'u' },
 140                         charArray_v = new char[] { 'v' }, charArray_w = new char[] { 'w' },
 141                         charArray_x = new char[] { 'x' }, charArray_y = new char[] { 'y' },
 142                         charArray_z = new char[] { 'z' };
 143
 144         static final char[] charArray_va = new char[] { '$', 'a' },
 145                         charArray_vb = new char[] { '$', 'b' }, charArray_vc = new char[] {
 146                                         '$', 'c' }, charArray_vd = new char[] { '$', 'd' },
 147                         charArray_ve = new char[] { '$', 'e' }, charArray_vf = new char[] {
 148                                         '$', 'f' }, charArray_vg = new char[] { '$', 'g' },
 149                         charArray_vh = new char[] { '$', 'h' }, charArray_vi = new char[] {
 150                                         '$', 'i' }, charArray_vj = new char[] { '$', 'j' },
 151                         charArray_vk = new char[] { '$', 'k' }, charArray_vl = new char[] {
 152                                         '$', 'l' }, charArray_vm = new char[] { '$', 'm' },
 153                         charArray_vn = new char[] { '$', 'n' }, charArray_vo = new char[] {
 154                                         '$', 'o' }, charArray_vp = new char[] { '$', 'p' },
 155                         charArray_vq = new char[] { '$', 'q' }, charArray_vr = new char[] {
 156                                         '$', 'r' }, charArray_vs = new char[] { '$', 's' },
 157                         charArray_vt = new char[] { '$', 't' }, charArray_vu = new char[] {
 158                                         '$', 'u' }, charArray_vv = new char[] { '$', 'v' },
 159                         charArray_vw = new char[] { '$', 'w' }, charArray_vx = new char[] {
 160                                         '$', 'x' }, charArray_vy = new char[] { '$', 'y' },
 161                         charArray_vz = new char[] { '$', 'z' };
 162
 163         public final static int MAX_OBVIOUS = 256;
 164
 165         static final int[] ObviousIdentCharNatures = new int[MAX_OBVIOUS];
 166
 167         public final static int C_DOLLAR = 8;
 168
 169         public final static int C_LETTER = 4;
 170
 171         public final static int C_DIGIT = 3;
 172
 173         public final static int C_SEPARATOR = 2;
 174
 175         public final static int C_SPACE = 1;
 176         static {
 177                 for (int i = '0'; i <= '9'; i++)
 178                         ObviousIdentCharNatures[i] = C_DIGIT;
 179
 180                 for (int i = 'a'; i <= 'z'; i++)
 181                         ObviousIdentCharNatures[i] = C_LETTER;
 182                 for (int i = 'A'; i <= 'Z'; i++)
 183                         ObviousIdentCharNatures[i] = C_LETTER;
 184                 ObviousIdentCharNatures['_'] = C_LETTER;
 185                 for (int i = 127; i <= 255; i++)
 186                         ObviousIdentCharNatures[i] = C_LETTER;
 187
 188                 ObviousIdentCharNatures['$'] = C_DOLLAR;
 189
 190                 ObviousIdentCharNatures[10] = C_SPACE; // \ u000a: LINE FEED
 191                 ObviousIdentCharNatures[12] = C_SPACE; // \ u000c: FORM FEED
 192                 ObviousIdentCharNatures[13] = C_SPACE; // \ u000d: CARRIAGE RETURN
 193                 ObviousIdentCharNatures[32] = C_SPACE; // \ u0020: SPACE
 194                 ObviousIdentCharNatures[9] = C_SPACE; // \ u0009: HORIZONTAL
 195                                                                                                 // TABULATION
 196
 197                 ObviousIdentCharNatures['.'] = C_SEPARATOR;
 198                 ObviousIdentCharNatures[':'] = C_SEPARATOR;
 199                 ObviousIdentCharNatures[';'] = C_SEPARATOR;
 200                 ObviousIdentCharNatures[','] = C_SEPARATOR;
 201                 ObviousIdentCharNatures['['] = C_SEPARATOR;
 202                 ObviousIdentCharNatures[']'] = C_SEPARATOR;
 203                 ObviousIdentCharNatures['('] = C_SEPARATOR;
 204                 ObviousIdentCharNatures[')'] = C_SEPARATOR;
 205                 ObviousIdentCharNatures['{'] = C_SEPARATOR;
 206                 ObviousIdentCharNatures['}'] = C_SEPARATOR;
 207                 ObviousIdentCharNatures['+'] = C_SEPARATOR;
 208                 ObviousIdentCharNatures['-'] = C_SEPARATOR;
 209                 ObviousIdentCharNatures['*'] = C_SEPARATOR;
 210                 ObviousIdentCharNatures['/'] = C_SEPARATOR;
 211                 ObviousIdentCharNatures['='] = C_SEPARATOR;
 212                 ObviousIdentCharNatures['&'] = C_SEPARATOR;
 213                 ObviousIdentCharNatures['|'] = C_SEPARATOR;
 214                 ObviousIdentCharNatures['?'] = C_SEPARATOR;
 215                 ObviousIdentCharNatures['<'] = C_SEPARATOR;
 216                 ObviousIdentCharNatures['>'] = C_SEPARATOR;
 217                 ObviousIdentCharNatures['!'] = C_SEPARATOR;
 218                 ObviousIdentCharNatures['%'] = C_SEPARATOR;
 219                 ObviousIdentCharNatures['^'] = C_SEPARATOR;
 220                 ObviousIdentCharNatures['~'] = C_SEPARATOR;
 221                 ObviousIdentCharNatures['"'] = C_SEPARATOR;
 222                 ObviousIdentCharNatures['\''] = C_SEPARATOR;
 223         }
 224
 225         static final char[] initCharArray = new char[] { '\u0000', '\u0000',
 226                         '\u0000', '\u0000', '\u0000', '\u0000' };
 227
 228         static final int TableSize = 30, InternalTableSize = 6;
 229
 230         // 30*6 = 180 entries
 231         public static final int OptimizedLength = 6;
 232
 233         public/* static */
 234         final char[][][][] charArray_length = new char[OptimizedLength][TableSize][InternalTableSize][];
 235
 236         // support for detecting non-externalized string literals
 237         int currentLineNr = -1;
 238
 239         int previousLineNr = -1;
 240
 241         NLSLine currentLine = null;
 242
 243         List lines = new ArrayList();
 244
 245         public static final String TAG_PREFIX = "//$NON-NLS-"; //$NON-NLS-1$
 246
 247         public static final int TAG_PREFIX_LENGTH = TAG_PREFIX.length();
 248
 249         public static final String TAG_POSTFIX = "$"; //$NON-NLS-1$
 250
 251         public static final int TAG_POSTFIX_LENGTH = TAG_POSTFIX.length();
 252
 253         public StringLiteral[] nonNLSStrings = null;
 254
 255         public boolean checkNonExternalizedStringLiterals = true;
 256
 257         public boolean wasNonExternalizedStringLiteral = false;
 258
 259         /* static */{
 260                 for (int i = 0; i < 6; i++) {
 261                         for (int j = 0; j < TableSize; j++) {
 262                                 for (int k = 0; k < InternalTableSize; k++) {
 263                                         charArray_length[i][j][k] = initCharArray;
 264                                 }
 265                         }
 266                 }
 267         }
 268
 269         static int newEntry2 = 0, newEntry3 = 0, newEntry4 = 0, newEntry5 = 0,
 270                         newEntry6 = 0;
 271
 272         public static final int RoundBracket = 0;
 273
 274         public static final int SquareBracket = 1;
 275
 276         public static final int CurlyBracket = 2;
 277
 278         public static final int BracketKinds = 3;
 279
 280         // task tag support
 281         public char[][] foundTaskTags = null;
 282
 283         public char[][] foundTaskMessages;
 284
 285         public char[][] foundTaskPriorities = null;
 286
 287         public int[][] foundTaskPositions;
 288
 289         public int foundTaskCount = 0;
 290
 291         public char[][] taskTags = null;
 292
 293         public char[][] taskPriorities = null;
 294
 295         public boolean isTaskCaseSensitive = true;
 296
 297         public static final boolean DEBUG = false;
 298
 299         public static final boolean TRACE = false;
 300
 301         public ICompilationUnit compilationUnit = null;
 302
 303         /**
 304          * Determines if the specified character is permissible as the first
 305          * character in a PHP identifier or variable
 306          *
 307          * The '$' character for PHP variables is regarded as a correct first
 308          * character !
 309          *
 310          */
 311         public static boolean isPHPIdentOrVarStart(char ch) {
 312                 if (ch < MAX_OBVIOUS) {
 313                         return ObviousIdentCharNatures[ch] == C_LETTER
 314                                         || ObviousIdentCharNatures[ch] == C_DOLLAR;
 315                 }
 316                 return false;
 317                 // return Character.isLetter(ch) || (ch == '$') || (ch == '_') || (0x7F
 318                 // <=
 319                 // ch && ch <= 0xFF);
 320         }
 321
 322         /**
 323          * Determines if the specified character is permissible as the first
 324          * character in a PHP identifier.
 325          *
 326          * The '$' character for PHP variables isn't regarded as the first character !
 327          */
 328         public static boolean isPHPIdentifierStart(char ch) {
 329                 if (ch < MAX_OBVIOUS) {
 330                         return ObviousIdentCharNatures[ch] == C_LETTER;
 331                 }
 332                 return false;
 333                 // return Character.isLetter(ch) || (ch == '_') || (0x7F <= ch && ch <=
 334                 // 0xFF);
 335         }
 336
 337         /**
 338          * Determines if the specified character may be part of a PHP identifier as
 339          * other than the first character
 340          */
 341         public static boolean isPHPIdentifierPart(char ch) {
 342                 if (ch < MAX_OBVIOUS) {
 343                         return ObviousIdentCharNatures[ch] == C_LETTER
 344                                         || ObviousIdentCharNatures[ch] == C_DIGIT;
 345                 }
 346                 return false;
 347                 // return Character.isLetterOrDigit(ch) || (ch == '_') || (0x7F <= ch &&
 348                 // ch
 349                 // <= 0xFF);
 350         }
 351
 352 //      public static boolean isSQLIdentifierPart(char ch) {
 353 //              if (ch < MAX_OBVIOUS) {
 354 //                      return ObviousIdentCharNatures[ch] == C_LETTER
 355 //                                      || ObviousIdentCharNatures[ch] == C_DIGIT;
 356 //              }
 357 //              return false;
 358 //      }
 359
 360         public final boolean atEnd() {
 361                 // This code is not relevant if source is
 362                 // Only a part of the real stream input
 363                 return source.length == currentPosition;
 364         }
 365
 366         public char[] getCurrentIdentifierSource() {
 367                 // return the token REAL source (aka unicodes are precomputed)
 368                 char[] result;
 369                 // if (withoutUnicodePtr != 0)
 370                 // //0 is used as a fast test flag so the real first char is in position
 371                 // 1
 372                 // System.arraycopy(
 373                 // withoutUnicodeBuffer,
 374                 // 1,
 375                 // result = new char[withoutUnicodePtr],
 376                 // 0,
 377                 // withoutUnicodePtr);
 378                 // else {
 379                 int length = currentPosition - startPosition;
 380                 switch (length) { // see OptimizedLength
 381                 case 1:
 382                         return optimizedCurrentTokenSource1();
 383                 case 2:
 384                         return optimizedCurrentTokenSource2();
 385                 case 3:
 386                         return optimizedCurrentTokenSource3();
 387                 case 4:
 388                         return optimizedCurrentTokenSource4();
 389                 case 5:
 390                         return optimizedCurrentTokenSource5();
 391                 case 6:
 392                         return optimizedCurrentTokenSource6();
 393                 }
 394                 // no optimization
 395                 System.arraycopy(source, startPosition, result = new char[length], 0,
 396                                 length);
 397                 // }
 398                 return result;
 399         }
 400
 401         public int getCurrentTokenEndPosition() {
 402                 return this.currentPosition - 1;
 403         }
 404
 405         public final char[] getCurrentTokenSource() {
 406                 // Return the token REAL source (aka unicodes are precomputed)
 407                 char[] result;
 408                 // if (withoutUnicodePtr != 0)
 409                 // // 0 is used as a fast test flag so the real first char is in
 410                 // position 1
 411                 // System.arraycopy(
 412                 // withoutUnicodeBuffer,
 413                 // 1,
 414                 // result = new char[withoutUnicodePtr],
 415                 // 0,
 416                 // withoutUnicodePtr);
 417                 // else {
 418                 int length;
 419                 System.arraycopy(source, startPosition,
 420                                 result = new char[length = currentPosition - startPosition], 0,
 421                                 length);
 422                 // }
 423                 return result;
 424         }
 425
 426         public final char[] getCurrentTokenSource(int startPos) {
 427                 // Return the token REAL source (aka unicodes are precomputed)
 428                 char[] result;
 429                 // if (withoutUnicodePtr != 0)
 430                 // // 0 is used as a fast test flag so the real first char is in
 431                 // position 1
 432                 // System.arraycopy(
 433                 // withoutUnicodeBuffer,
 434                 // 1,
 435                 // result = new char[withoutUnicodePtr],
 436                 // 0,
 437                 // withoutUnicodePtr);
 438                 // else {
 439                 int length;
 440                 System.arraycopy(source, startPos,
 441                                 result = new char[length = currentPosition - startPos], 0,
 442                                 length);
 443                 // }
 444                 return result;
 445         }
 446
 447         public final char[] getCurrentTokenSourceString() {
 448                 // return the token REAL source (aka unicodes are precomputed).
 449                 // REMOVE the two " that are at the beginning and the end.
 450                 char[] result;
 451                 if (withoutUnicodePtr != 0)
 452                         // 0 is used as a fast test flag so the real first char is in
 453                         // position 1
 454                         System.arraycopy(withoutUnicodeBuffer, 2,
 455                                         // 2 is 1 (real start) + 1 (to jump over the ")
 456                                         result = new char[withoutUnicodePtr - 2], 0,
 457                                         withoutUnicodePtr - 2);
 458                 else {
 459                         int length;
 460                         System.arraycopy(source, startPosition + 1,
 461                                         result = new char[length = currentPosition - startPosition
 462                                                         - 2], 0, length);
 463                 }
 464                 return result;
 465         }
 466
 467         public final boolean equalsCurrentTokenSource(char[] word) {
 468                 if (word.length != currentPosition - startPosition) {
 469                         return false;
 470                 }
 471                 for (int i = 0; i < word.length; i++) {
 472                         if (word[i] != source[startPosition + i]) {
 473                                 return false;
 474                         }
 475                 }
 476                 return true;
 477         }
 478
 479 //      public final char[] getRawTokenSourceEnd() {
 480 //              int length = this.eofPosition - this.currentPosition - 1;
 481 //              char[] sourceEnd = new char[length];
 482 //              System.arraycopy(this.source, this.currentPosition, sourceEnd, 0,
 483 //                              length);
 484 //              return sourceEnd;
 485 //      }
 486
 487         public int getCurrentTokenStartPosition() {
 488                 return this.startPosition;
 489         }
 490
 491 //      public final String getCurrentStringLiteral() {
 492 //              char[] result = getCurrentStringLiteralSource();
 493 //              return new String(result);
 494 //      }
 495
 496         public final char[] getCurrentStringLiteralSource() {
 497                 // Return the token REAL source (aka unicodes are precomputed)
 498                 if (startPosition + 1 >= currentPosition) {
 499                         return new char[0];
 500                 }
 501                 char[] result;
 502                 int length;
 503                 System
 504                                 .arraycopy(source, startPosition + 1,
 505                                                 result = new char[length = currentPosition
 506                                                                 - startPosition - 2], 0, length);
 507                 // }
 508                 return result;
 509         }
 510
 511 //      public final char[] getCurrentStringLiteralSource(int startPos) {
 512 //              // Return the token REAL source (aka unicodes are precomputed)
 513 //              char[] result;
 514 //              int length;
 515 //              System.arraycopy(source, startPos + 1,
 516 //                              result = new char[length = currentPosition - startPos - 2], 0,
 517 //                              length);
 518 //              // }
 519 //              return result;
 520 //      }
 521
 522         /*
 523          * Search the source position corresponding to the end of a given line
 524          * number
 525          *
 526          * Line numbers are 1-based, and relative to the scanner initialPosition.
 527          * Character positions are 0-based.
 528          *
 529          * In case the given line number is inconsistent, answers -1.
 530          */
 531         public final int getLineEnd(int lineNumber) {
 532                 if (lineEnds == null)
 533                         return -1;
 534                 if (lineNumber >= lineEnds.length)
 535                         return -1;
 536                 if (lineNumber <= 0)
 537                         return -1;
 538                 if (lineNumber == lineEnds.length - 1)
 539                         return eofPosition;
 540                 return lineEnds[lineNumber - 1];
 541                 // next line start one character behind the lineEnd of the previous line
 542         }
 543
 544         /**
 545          * Search the source position corresponding to the beginning of a given line
 546          * number
 547          *
 548          * Line numbers are 1-based, and relative to the scanner initialPosition.
 549          * Character positions are 0-based.
 550          *
 551          * e.g. getLineStart(1) --> 0 i.e. first line starts at character 0.
 552          *
 553          * In case the given line number is inconsistent, answers -1.
 554          */
 555         public final int getLineStart(int lineNumber) {
 556                 if (lineEnds == null)
 557                         return -1;
 558                 if (lineNumber >= lineEnds.length)
 559                         return -1;
 560                 if (lineNumber <= 0)
 561                         return -1;
 562                 if (lineNumber == 1)
 563                         return initialPosition;
 564                 return lineEnds[lineNumber - 2] + 1;
 565                 // next line start one character behind the lineEnd of the previous line
 566         }
 567
 568         public final boolean getNextChar(char testedChar) {
 569                 // BOOLEAN
 570                 // handle the case of unicode.
 571                 // when a unicode appears then we must use a buffer that holds char
 572                 // internal values
 573                 // At the end of this method currentCharacter holds the new visited char
 574                 // and currentPosition points right next after it
 575                 // Both previous lines are true if the currentCharacter is == to the
 576                 // testedChar
 577                 // On false, no side effect has occured.
 578                 // ALL getNextChar.... ARE OPTIMIZED COPIES
 579                 int temp = currentPosition;
 580                 try {
 581                         currentCharacter = source[currentPosition++];
 582                         // if (((currentCharacter = source[currentPosition++]) == '\\')
 583                         // && (source[currentPosition] == 'u')) {
 584                         // //-------------unicode traitement ------------
 585                         // int c1, c2, c3, c4;
 586                         // int unicodeSize = 6;
 587                         // currentPosition++;
 588                         // while (source[currentPosition] == 'u') {
 589                         // currentPosition++;
 590                         // unicodeSize++;
 591                         // }
 592                         //
 593                         // if (((c1 = Character.getNumericValue(source[currentPosition++]))
 594                         // > 15
 595                         // || c1 < 0)
 596                         // || ((c2 = Character.getNumericValue(source[currentPosition++])) >
 597                         // 15
 598                         // || c2 < 0)
 599                         // || ((c3 = Character.getNumericValue(source[currentPosition++])) >
 600                         // 15
 601                         // || c3 < 0)
 602                         // || ((c4 = Character.getNumericValue(source[currentPosition++])) >
 603                         // 15
 604                         // || c4 < 0)) {
 605                         // currentPosition = temp;
 606                         // return false;
 607                         // }
 608                         //
 609                         // currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 610                         // if (currentCharacter != testedChar) {
 611                         // currentPosition = temp;
 612                         // return false;
 613                         // }
 614                         // unicodeAsBackSlash = currentCharacter == '\\';
 615                         //
 616                         // //need the unicode buffer
 617                         // if (withoutUnicodePtr == 0) {
 618                         // //buffer all the entries that have been left aside....
 619                         // withoutUnicodePtr = currentPosition - unicodeSize -
 620                         // startPosition;
 621                         // System.arraycopy(
 622                         // source,
 623                         // startPosition,
 624                         // withoutUnicodeBuffer,
 625                         // 1,
 626                         // withoutUnicodePtr);
 627                         // }
 628                         // //fill the buffer with the char
 629                         // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 630                         // return true;
 631                         //
 632                         // } //-------------end unicode traitement--------------
 633                         // else {
 634                         if (currentCharacter != testedChar) {
 635                                 currentPosition = temp;
 636                                 return false;
 637                         }
 638                         unicodeAsBackSlash = false;
 639                         // if (withoutUnicodePtr != 0)
 640                         // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 641                         return true;
 642                         // }
 643                 } catch (IndexOutOfBoundsException e) {
 644                         unicodeAsBackSlash = false;
 645                         currentPosition = temp;
 646                         return false;
 647                 }
 648         }
 649
 650         public final int getNextChar(char testedChar1, char testedChar2) {
 651                 // INT 0 : testChar1 \\\\///\\\\ 1 : testedChar2 \\\\///\\\\ -1 : others
 652                 // test can be done with (x==0) for the first and (x>0) for the second
 653                 // handle the case of unicode.
 654                 // when a unicode appears then we must use a buffer that holds char
 655                 // internal values
 656                 // At the end of this method currentCharacter holds the new visited char
 657                 // and currentPosition points right next after it
 658                 // Both previous lines are true if the currentCharacter is == to the
 659                 // testedChar1/2
 660                 // On false, no side effect has occured.
 661                 // ALL getNextChar.... ARE OPTIMIZED COPIES
 662                 int temp = currentPosition;
 663                 try {
 664                         int result;
 665                         currentCharacter = source[currentPosition++];
 666                         // if (((currentCharacter = source[currentPosition++]) == '\\')
 667                         // && (source[currentPosition] == 'u')) {
 668                         // //-------------unicode traitement ------------
 669                         // int c1, c2, c3, c4;
 670                         // int unicodeSize = 6;
 671                         // currentPosition++;
 672                         // while (source[currentPosition] == 'u') {
 673                         // currentPosition++;
 674                         // unicodeSize++;
 675                         // }
 676                         //
 677                         // if (((c1 = Character.getNumericValue(source[currentPosition++]))
 678                         // > 15
 679                         // || c1 < 0)
 680                         // || ((c2 = Character.getNumericValue(source[currentPosition++])) >
 681                         // 15
 682                         // || c2 < 0)
 683                         // || ((c3 = Character.getNumericValue(source[currentPosition++])) >
 684                         // 15
 685                         // || c3 < 0)
 686                         // || ((c4 = Character.getNumericValue(source[currentPosition++])) >
 687                         // 15
 688                         // || c4 < 0)) {
 689                         // currentPosition = temp;
 690                         // return 2;
 691                         // }
 692                         //
 693                         // currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 694                         // if (currentCharacter == testedChar1)
 695                         // result = 0;
 696                         // else if (currentCharacter == testedChar2)
 697                         // result = 1;
 698                         // else {
 699                         // currentPosition = temp;
 700                         // return -1;
 701                         // }
 702                         //
 703                         // //need the unicode buffer
 704                         // if (withoutUnicodePtr == 0) {
 705                         // //buffer all the entries that have been left aside....
 706                         // withoutUnicodePtr = currentPosition - unicodeSize -
 707                         // startPosition;
 708                         // System.arraycopy(
 709                         // source,
 710                         // startPosition,
 711                         // withoutUnicodeBuffer,
 712                         // 1,
 713                         // withoutUnicodePtr);
 714                         // }
 715                         // //fill the buffer with the char
 716                         // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 717                         // return result;
 718                         // } //-------------end unicode traitement--------------
 719                         // else {
 720                         if (currentCharacter == testedChar1)
 721                                 result = 0;
 722                         else if (currentCharacter == testedChar2)
 723                                 result = 1;
 724                         else {
 725                                 currentPosition = temp;
 726                                 return -1;
 727                         }
 728                         // if (withoutUnicodePtr != 0)
 729                         // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 730                         return result;
 731                         // }
 732                 } catch (IndexOutOfBoundsException e) {
 733                         currentPosition = temp;
 734                         return -1;
 735                 }
 736         }
 737
 738         public final boolean getNextCharAsDigit() {
 739                 // BOOLEAN
 740                 // handle the case of unicode.
 741                 // when a unicode appears then we must use a buffer that holds char
 742                 // internal values
 743                 // At the end of this method currentCharacter holds the new visited char
 744                 // and currentPosition points right next after it
 745                 // Both previous lines are true if the currentCharacter is a digit
 746                 // On false, no side effect has occured.
 747                 // ALL getNextChar.... ARE OPTIMIZED COPIES
 748                 int temp = currentPosition;
 749                 try {
 750                         currentCharacter = source[currentPosition++];
 751                         // if (((currentCharacter = source[currentPosition++]) == '\\')
 752                         // && (source[currentPosition] == 'u')) {
 753                         // //-------------unicode traitement ------------
 754                         // int c1, c2, c3, c4;
 755                         // int unicodeSize = 6;
 756                         // currentPosition++;
 757                         // while (source[currentPosition] == 'u') {
 758                         // currentPosition++;
 759                         // unicodeSize++;
 760                         // }
 761                         //
 762                         // if (((c1 = Character.getNumericValue(source[currentPosition++]))
 763                         // > 15
 764                         // || c1 < 0)
 765                         // || ((c2 = Character.getNumericValue(source[currentPosition++])) >
 766                         // 15
 767                         // || c2 < 0)
 768                         // || ((c3 = Character.getNumericValue(source[currentPosition++])) >
 769                         // 15
 770                         // || c3 < 0)
 771                         // || ((c4 = Character.getNumericValue(source[currentPosition++])) >
 772                         // 15
 773                         // || c4 < 0)) {
 774                         // currentPosition = temp;
 775                         // return false;
 776                         // }
 777                         //
 778                         // currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 779                         // if (!Character.isDigit(currentCharacter)) {
 780                         // currentPosition = temp;
 781                         // return false;
 782                         // }
 783                         //
 784                         // //need the unicode buffer
 785                         // if (withoutUnicodePtr == 0) {
 786                         // //buffer all the entries that have been left aside....
 787                         // withoutUnicodePtr = currentPosition - unicodeSize -
 788                         // startPosition;
 789                         // System.arraycopy(
 790                         // source,
 791                         // startPosition,
 792                         // withoutUnicodeBuffer,
 793                         // 1,
 794                         // withoutUnicodePtr);
 795                         // }
 796                         // //fill the buffer with the char
 797                         // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 798                         // return true;
 799                         // } //-------------end unicode traitement--------------
 800                         // else {
 801                         if (!Character.isDigit(currentCharacter)) {
 802                                 currentPosition = temp;
 803                                 return false;
 804                         }
 805                         // if (withoutUnicodePtr != 0)
 806                         // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 807                         return true;
 808                         // }
 809                 } catch (IndexOutOfBoundsException e) {
 810                         currentPosition = temp;
 811                         return false;
 812                 }
 813         }
 814
 815         public final boolean getNextCharAsDigit(int radix) {
 816                 // BOOLEAN
 817                 // handle the case of unicode.
 818                 // when a unicode appears then we must use a buffer that holds char
 819                 // internal values
 820                 // At the end of this method currentCharacter holds the new visited char
 821                 // and currentPosition points right next after it
 822                 // Both previous lines are true if the currentCharacter is a digit base
 823                 // on
 824                 // radix
 825                 // On false, no side effect has occured.
 826                 // ALL getNextChar.... ARE OPTIMIZED COPIES
 827                 int temp = currentPosition;
 828                 try {
 829                         currentCharacter = source[currentPosition++];
 830                         // if (((currentCharacter = source[currentPosition++]) == '\\')
 831                         // && (source[currentPosition] == 'u')) {
 832                         // //-------------unicode traitement ------------
 833                         // int c1, c2, c3, c4;
 834                         // int unicodeSize = 6;
 835                         // currentPosition++;
 836                         // while (source[currentPosition] == 'u') {
 837                         // currentPosition++;
 838                         // unicodeSize++;
 839                         // }
 840                         //
 841                         // if (((c1 = Character.getNumericValue(source[currentPosition++]))
 842                         // > 15
 843                         // || c1 < 0)
 844                         // || ((c2 = Character.getNumericValue(source[currentPosition++])) >
 845                         // 15
 846                         // || c2 < 0)
 847                         // || ((c3 = Character.getNumericValue(source[currentPosition++])) >
 848                         // 15
 849                         // || c3 < 0)
 850                         // || ((c4 = Character.getNumericValue(source[currentPosition++])) >
 851                         // 15
 852                         // || c4 < 0)) {
 853                         // currentPosition = temp;
 854                         // return false;
 855                         // }
 856                         //
 857                         // currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 858                         // if (Character.digit(currentCharacter, radix) == -1) {
 859                         // currentPosition = temp;
 860                         // return false;
 861                         // }
 862                         //
 863                         // //need the unicode buffer
 864                         // if (withoutUnicodePtr == 0) {
 865                         // //buffer all the entries that have been left aside....
 866                         // withoutUnicodePtr = currentPosition - unicodeSize -
 867                         // startPosition;
 868                         // System.arraycopy(
 869                         // source,
 870                         // startPosition,
 871                         // withoutUnicodeBuffer,
 872                         // 1,
 873                         // withoutUnicodePtr);
 874                         // }
 875                         // //fill the buffer with the char
 876                         // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 877                         // return true;
 878                         // } //-------------end unicode traitement--------------
 879                         // else {
 880                         if (Character.digit(currentCharacter, radix) == -1) {
 881                                 currentPosition = temp;
 882                                 return false;
 883                         }
 884                         // if (withoutUnicodePtr != 0)
 885                         // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 886                         return true;
 887                         // }
 888                 } catch (IndexOutOfBoundsException e) {
 889                         currentPosition = temp;
 890                         return false;
 891                 }
 892         }
 893
 894         public boolean getNextCharAsJavaIdentifierPart() {
 895                 // BOOLEAN
 896                 // handle the case of unicode.
 897                 // when a unicode appears then we must use a buffer that holds char
 898                 // internal values
 899                 // At the end of this method currentCharacter holds the new visited char
 900                 // and currentPosition points right next after it
 901                 // Both previous lines are true if the currentCharacter is a
 902                 // JavaIdentifierPart
 903                 // On false, no side effect has occured.
 904                 // ALL getNextChar.... ARE OPTIMIZED COPIES
 905                 int temp = currentPosition;
 906                 try {
 907                         currentCharacter = source[currentPosition++];
 908                         // if (((currentCharacter = source[currentPosition++]) == '\\')
 909                         // && (source[currentPosition] == 'u')) {
 910                         // //-------------unicode traitement ------------
 911                         // int c1, c2, c3, c4;
 912                         // int unicodeSize = 6;
 913                         // currentPosition++;
 914                         // while (source[currentPosition] == 'u') {
 915                         // currentPosition++;
 916                         // unicodeSize++;
 917                         // }
 918                         //
 919                         // if (((c1 = Character.getNumericValue(source[currentPosition++]))
 920                         // > 15
 921                         // || c1 < 0)
 922                         // || ((c2 = Character.getNumericValue(source[currentPosition++])) >
 923                         // 15
 924                         // || c2 < 0)
 925                         // || ((c3 = Character.getNumericValue(source[currentPosition++])) >
 926                         // 15
 927                         // || c3 < 0)
 928                         // || ((c4 = Character.getNumericValue(source[currentPosition++])) >
 929                         // 15
 930                         // || c4 < 0)) {
 931                         // currentPosition = temp;
 932                         // return false;
 933                         // }
 934                         //
 935                         // currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 936                         // if (!isPHPIdentifierPart(currentCharacter)) {
 937                         // currentPosition = temp;
 938                         // return false;
 939                         // }
 940                         //
 941                         // //need the unicode buffer
 942                         // if (withoutUnicodePtr == 0) {
 943                         // //buffer all the entries that have been left aside....
 944                         // withoutUnicodePtr = currentPosition - unicodeSize -
 945                         // startPosition;
 946                         // System.arraycopy(
 947                         // source,
 948                         // startPosition,
 949                         // withoutUnicodeBuffer,
 950                         // 1,
 951                         // withoutUnicodePtr);
 952                         // }
 953                         // //fill the buffer with the char
 954                         // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 955                         // return true;
 956                         // } //-------------end unicode traitement--------------
 957                         // else {
 958                         if (!isPHPIdentifierPart(currentCharacter)) {
 959                                 currentPosition = temp;
 960                                 return false;
 961                         }
 962                         // if (withoutUnicodePtr != 0)
 963                         // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 964                         return true;
 965                         // }
 966                 } catch (IndexOutOfBoundsException e) {
 967                         currentPosition = temp;
 968                         return false;
 969                 }
 970         }
 971
 972         public TokenName getCastOrParen() {
 973                 int tempPosition = currentPosition;
 974                 char tempCharacter = currentCharacter;
 975                 TokenName tempToken = TokenName.LPAREN;
 976                 boolean found = false;
 977                 StringBuffer buf = new StringBuffer();
 978                 try {
 979                         do {
 980                                 currentCharacter = source[currentPosition++];
 981                         } while (currentCharacter == ' ' || currentCharacter == '\t');
 982                         while (ObviousIdentCharNatures[currentCharacter] == C_LETTER) {
 983                                 // while((currentCharacter >= 'a' && currentCharacter <= 'z') ||
 984                                 // (currentCharacter >= 'A' && currentCharacter <= 'Z')) {
 985                                 buf.append(currentCharacter);
 986                                 currentCharacter = source[currentPosition++];
 987                         }
 988                         if (buf.length() >= 3 && buf.length() <= 7) {
 989                                 char[] data = buf.toString().toCharArray();
 990                                 int index = 0;
 991                                 switch (data.length) {
 992                                 case 3:
 993                                         // int
 994                                         if ((data[index] == 'i') && (data[++index] == 'n')
 995                                                         && (data[++index] == 't')) {
 996                                                 found = true;
 997                                                 tempToken = TokenName.INTCAST;
 998                                         }
 999                                         break;
1000                                 case 4:
1001                                         // bool real
1002                                         if ((data[index] == 'b') && (data[++index] == 'o')
1003                                                         && (data[++index] == 'o') && (data[++index] == 'l')) {
1004                                                 found = true;
1005                                                 tempToken = TokenName.BOOLCAST;
1006                                         } else {
1007                                                 index = 0;
1008                                                 if ((data[index] == 'r') && (data[++index] == 'e')
1009                                                                 && (data[++index] == 'a')
1010                                                                 && (data[++index] == 'l')) {
1011                                                         found = true;
1012                                                         tempToken = TokenName.DOUBLECAST;
1013                                                 }
1014                                         }
1015                                         break;
1016                                 case 5:
1017                                         // array unset float
1018                                         if ((data[index] == 'a') && (data[++index] == 'r')
1019                                                         && (data[++index] == 'r') && (data[++index] == 'a')
1020                                                         && (data[++index] == 'y')) {
1021                                                 found = true;
1022                                                 tempToken = TokenName.ARRAYCAST;
1023                                         } else {
1024                                                 index = 0;
1025                                                 if ((data[index] == 'u') && (data[++index] == 'n')
1026                                                                 && (data[++index] == 's')
1027                                                                 && (data[++index] == 'e')
1028                                                                 && (data[++index] == 't')) {
1029                                                         found = true;
1030                                                         tempToken = TokenName.UNSETCAST;
1031                                                 } else {
1032                                                         index = 0;
1033                                                         if ((data[index] == 'f') && (data[++index] == 'l')
1034                                                                         && (data[++index] == 'o')
1035                                                                         && (data[++index] == 'a')
1036                                                                         && (data[++index] == 't')) {
1037                                                                 found = true;
1038                                                                 tempToken = TokenName.DOUBLECAST;
1039                                                         }
1040                                                 }
1041                                         }
1042                                         break;
1043                                 case 6:
1044                                         // object string double
1045                                         if ((data[index] == 'o') && (data[++index] == 'b')
1046                                                         && (data[++index] == 'j') && (data[++index] == 'e')
1047                                                         && (data[++index] == 'c') && (data[++index] == 't')) {
1048                                                 found = true;
1049                                                 tempToken = TokenName.OBJECTCAST;
1050                                         } else {
1051                                                 index = 0;
1052                                                 if ((data[index] == 's') && (data[++index] == 't')
1053                                                                 && (data[++index] == 'r')
1054                                                                 && (data[++index] == 'i')
1055                                                                 && (data[++index] == 'n')
1056                                                                 && (data[++index] == 'g')) {
1057                                                         found = true;
1058                                                         tempToken = TokenName.STRINGCAST;
1059                                                 } else {
1060                                                         index = 0;
1061                                                         if ((data[index] == 'd') && (data[++index] == 'o')
1062                                                                         && (data[++index] == 'u')
1063                                                                         && (data[++index] == 'b')
1064                                                                         && (data[++index] == 'l')
1065                                                                         && (data[++index] == 'e')) {
1066                                                                 found = true;
1067                                                                 tempToken = TokenName.DOUBLECAST;
1068                                                         }
1069                                                 }
1070                                         }
1071                                         break;
1072                                 case 7:
1073                                         // boolean integer
1074                                         if ((data[index] == 'b') && (data[++index] == 'o')
1075                                                         && (data[++index] == 'o') && (data[++index] == 'l')
1076                                                         && (data[++index] == 'e') && (data[++index] == 'a')
1077                                                         && (data[++index] == 'n')) {
1078                                                 found = true;
1079                                                 tempToken = TokenName.BOOLCAST;
1080                                         } else {
1081                                                 index = 0;
1082                                                 if ((data[index] == 'i') && (data[++index] == 'n')
1083                                                                 && (data[++index] == 't')
1084                                                                 && (data[++index] == 'e')
1085                                                                 && (data[++index] == 'g')
1086                                                                 && (data[++index] == 'e')
1087                                                                 && (data[++index] == 'r')) {
1088                                                         found = true;
1089                                                         tempToken = TokenName.INTCAST;
1090                                                 }
1091                                         }
1092                                         break;
1093                                 }
1094                                 if (found) {
1095                                         while (currentCharacter == ' ' || currentCharacter == '\t') {
1096                                                 currentCharacter = source[currentPosition++];
1097                                         }
1098                                         if (currentCharacter == ')') {
1099                                                 return tempToken;
1100                                         }
1101                                 }
1102                         }
1103                 } catch (IndexOutOfBoundsException e) {
1104                 }
1105                 currentCharacter = tempCharacter;
1106                 currentPosition = tempPosition;
1107                 return TokenName.LPAREN;
1108         }
1109
1110         /**
1111          *
1112          *
1113          */
1114         public void consumeStringInterpolated() throws InvalidInputException {
1115                 try {
1116                         // consume next character
1117                         unicodeAsBackSlash = false;
1118                         currentCharacter = source[currentPosition++];
1119                         // if (((currentCharacter = source[currentPosition++]) == '\\')
1120                         // && (source[currentPosition] == 'u')) {
1121                         // getNextUnicodeChar();
1122                         // } else {
1123                         // if (withoutUnicodePtr != 0) {
1124                         // withoutUnicodeBuffer[++withoutUnicodePtr] =
1125                         // currentCharacter;
1126                         // }
1127                         // }
1128                         while (currentCharacter != '`') {
1129                                 /** ** in PHP \r and \n are valid in string literals *** */
1130                                 // if ((currentCharacter == '\n')
1131                                 // || (currentCharacter == '\r')) {
1132                                 // // relocate if finding another quote fairly close: thus
1133                                 // unicode
1134                                 // '/u000D' will be fully consumed
1135                                 // for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
1136                                 // if (currentPosition + lookAhead == source.length)
1137                                 // break;
1138                                 // if (source[currentPosition + lookAhead] == '\n')
1139                                 // break;
1140                                 // if (source[currentPosition + lookAhead] == '\"') {
1141                                 // currentPosition += lookAhead + 1;
1142                                 // break;
1143                                 // }
1144                                 // }
1145                                 // throw new InvalidInputException(INVALID_CHAR_IN_STRING);
1146                                 // }
1147                                 if (currentCharacter == '\\') {
1148                                         int escapeSize = currentPosition;
1149                                         boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
1150                                         // scanEscapeCharacter make a side effect on this value and
1151                                         // we need
1152                                         // the previous value few lines down this one
1153                                         scanDoubleQuotedEscapeCharacter();
1154                                         escapeSize = currentPosition - escapeSize;
1155                                         if (withoutUnicodePtr == 0) {
1156                                                 // buffer all the entries that have been left aside....
1157                                                 withoutUnicodePtr = currentPosition - escapeSize - 1
1158                                                                 - startPosition;
1159                                                 System.arraycopy(source, startPosition,
1160                                                                 withoutUnicodeBuffer, 1, withoutUnicodePtr);
1161                                                 withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1162                                         } else { // overwrite the / in the buffer
1163                                                 withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter;
1164                                                 if (backSlashAsUnicodeInString) { // there are TWO \
1165                                                                                                                         // in the stream
1166                                                         // where only one is correct
1167                                                         withoutUnicodePtr--;
1168                                                 }
1169                                         }
1170                                 } else if ((currentCharacter == '\r')
1171                                                 || (currentCharacter == '\n')) {
1172                                         if (recordLineSeparator) {
1173                                                 pushLineSeparator();
1174                                         }
1175                                 }
1176                                 // consume next character
1177                                 unicodeAsBackSlash = false;
1178                                 currentCharacter = source[currentPosition++];
1179                                 // if (((currentCharacter = source[currentPosition++]) == '\\')
1180                                 // && (source[currentPosition] == 'u')) {
1181                                 // getNextUnicodeChar();
1182                                 // } else {
1183                                 if (withoutUnicodePtr != 0) {
1184                                         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1185                                 }
1186                                 // }
1187                         }
1188                 } catch (IndexOutOfBoundsException e) {
1189                         // reset end position for error reporting
1190                         currentPosition -= 2;
1191                         throw new InvalidInputException(UNTERMINATED_STRING);
1192                 } catch (InvalidInputException e) {
1193                         if (e.getMessage().equals(INVALID_ESCAPE)) {
1194                                 // relocate if finding another quote fairly close: thus unicode
1195                                 // '/u000D' will be fully consumed
1196                                 for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
1197                                         if (currentPosition + lookAhead == source.length)
1198                                                 break;
1199                                         if (source[currentPosition + lookAhead] == '\n')
1200                                                 break;
1201                                         if (source[currentPosition + lookAhead] == '`') {
1202                                                 currentPosition += lookAhead + 1;
1203                                                 break;
1204                                         }
1205                                 }
1206                         }
1207                         throw e; // rethrow
1208                 }
1209                 if (checkNonExternalizedStringLiterals) { // check for presence of NLS
1210                                                                                                         // tags
1211                         // //$NON-NLS-?$ where ? is an
1212                         // int.
1213                         if (currentLine == null) {
1214                                 currentLine = new NLSLine();
1215                                 lines.add(currentLine);
1216                         }
1217                         currentLine.add(new StringLiteral(getCurrentTokenSourceString(),
1218                                         startPosition, currentPosition - 1));
1219                 }
1220         }
1221
1222         public void consumeStringConstant() throws InvalidInputException {
1223                 try {
1224                         // consume next character
1225                         unicodeAsBackSlash = false;
1226                         currentCharacter = source[currentPosition++];
1227                         // if (((currentCharacter = source[currentPosition++]) == '\\')
1228                         // && (source[currentPosition] == 'u')) {
1229                         // getNextUnicodeChar();
1230                         // } else {
1231                         // if (withoutUnicodePtr != 0) {
1232                         // withoutUnicodeBuffer[++withoutUnicodePtr] =
1233                         // currentCharacter;
1234                         // }
1235                         // }
1236                         while (currentCharacter != '\'') {
1237                                 /** ** in PHP \r and \n are valid in string literals *** */
1238                                 // if ((currentCharacter == '\n')
1239                                 // || (currentCharacter == '\r')) {
1240                                 // // relocate if finding another quote fairly close: thus
1241                                 // unicode
1242                                 // '/u000D' will be fully consumed
1243                                 // for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
1244                                 // if (currentPosition + lookAhead == source.length)
1245                                 // break;
1246                                 // if (source[currentPosition + lookAhead] == '\n')
1247                                 // break;
1248                                 // if (source[currentPosition + lookAhead] == '\"') {
1249                                 // currentPosition += lookAhead + 1;
1250                                 // break;
1251                                 // }
1252                                 // }
1253                                 // throw new InvalidInputException(INVALID_CHAR_IN_STRING);
1254                                 // }
1255                                 if (currentCharacter == '\\') {
1256                                         int escapeSize = currentPosition;
1257                                         boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
1258                                         // scanEscapeCharacter make a side effect on this value and
1259                                         // we need
1260                                         // the previous value few lines down this one
1261                                         scanSingleQuotedEscapeCharacter();
1262                                         escapeSize = currentPosition - escapeSize;
1263                                         if (withoutUnicodePtr == 0) {
1264                                                 // buffer all the entries that have been left aside....
1265                                                 withoutUnicodePtr = currentPosition - escapeSize - 1
1266                                                                 - startPosition;
1267                                                 System.arraycopy(source, startPosition,
1268                                                                 withoutUnicodeBuffer, 1, withoutUnicodePtr);
1269                                                 withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1270                                         } else { // overwrite the / in the buffer
1271                                                 withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter;
1272                                                 if (backSlashAsUnicodeInString) { // there are TWO \
1273                                                                                                                         // in the stream
1274                                                         // where only one is correct
1275                                                         withoutUnicodePtr--;
1276                                                 }
1277                                         }
1278                                 } else if ((currentCharacter == '\r')
1279                                                 || (currentCharacter == '\n')) {
1280                                         if (recordLineSeparator) {
1281                                                 pushLineSeparator();
1282                                         }
1283                                 }
1284                                 // consume next character
1285                                 unicodeAsBackSlash = false;
1286                                 currentCharacter = source[currentPosition++];
1287                                 // if (((currentCharacter = source[currentPosition++]) == '\\')
1288                                 // && (source[currentPosition] == 'u')) {
1289                                 // getNextUnicodeChar();
1290                                 // } else {
1291                                 if (withoutUnicodePtr != 0) {
1292                                         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1293                                 }
1294                                 // }
1295                         }
1296                 } catch (IndexOutOfBoundsException e) {
1297                         // reset end position for error reporting
1298                         currentPosition -= 2;
1299                         throw new InvalidInputException(UNTERMINATED_STRING);
1300                 } catch (InvalidInputException e) {
1301                         if (e.getMessage().equals(INVALID_ESCAPE)) {
1302                                 // relocate if finding another quote fairly close: thus unicode
1303                                 // '/u000D' will be fully consumed
1304                                 for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
1305                                         if (currentPosition + lookAhead == source.length)
1306                                                 break;
1307                                         if (source[currentPosition + lookAhead] == '\n')
1308                                                 break;
1309                                         if (source[currentPosition + lookAhead] == '\'') {
1310                                                 currentPosition += lookAhead + 1;
1311                                                 break;
1312                                         }
1313                                 }
1314                         }
1315                         throw e; // rethrow
1316                 }
1317                 if (checkNonExternalizedStringLiterals) { // check for presence of NLS
1318                                                                                                         // tags
1319                         // //$NON-NLS-?$ where ? is an
1320                         // int.
1321                         if (currentLine == null) {
1322                                 currentLine = new NLSLine();
1323                                 lines.add(currentLine);
1324                         }
1325                         currentLine.add(new StringLiteral(getCurrentTokenSourceString(),
1326                                         startPosition, currentPosition - 1));
1327                 }
1328         }
1329
1330         /**
1331          *
1332          *
1333          */
1334         public void consumeStringLiteral() throws InvalidInputException {
1335                 try {
1336                         int openDollarBrace = 0;
1337
1338                         unicodeAsBackSlash  = false;
1339                         currentCharacter    = source[currentPosition++];                                   // consume next character
1340
1341                         while (currentCharacter != '"' ||                                          // As long as the ending '"' isn't found, or
1342                                openDollarBrace > 0) {                                              // the last '}' isn't found
1343                                 if (currentCharacter == '\\') {
1344                                         int     escapeSize                 = currentPosition;
1345                                         boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
1346
1347                                         // scanEscapeCharacter make a side effect on this value and we need
1348                                         // the previous value few lines down this one
1349                                         scanDoubleQuotedEscapeCharacter ();
1350                                         escapeSize = currentPosition - escapeSize;
1351
1352                                         if (withoutUnicodePtr == 0) {                                      // buffer all the entries that have been left aside....
1353                                                 withoutUnicodePtr = currentPosition - escapeSize - 1 - startPosition;
1354                                                 System.arraycopy (source, startPosition, withoutUnicodeBuffer, 1, withoutUnicodePtr);
1355                                                 withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1356                                         }
1357                                         else {                                                             // overwrite the / in the buffer
1358                                                 withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter;
1359
1360                                                 if (backSlashAsUnicodeInString) {                              // there are TWO \ in the stream where only one is correct
1361                                                         withoutUnicodePtr--;
1362                                                 }
1363                                         }
1364                                 }
1365                                 else if (currentCharacter == '$' && source[currentPosition] == '{') {  // If found '${'
1366                                         openDollarBrace++;
1367                                         currentCharacter = source[currentPosition++];                      // consume next character, or we count one open brace to much!
1368                                 }
1369                                 else if (currentCharacter == '{' && source[currentPosition] == '$') {  // If found '{$'
1370                                         openDollarBrace++;
1371                                 }
1372                                 else if (currentCharacter == '}') {                                    // If found '}'
1373                                         openDollarBrace--;
1374                                 }
1375                                 else if ((currentCharacter == '\r') || (currentCharacter == '\n')) {   // In PHP \r and \n are valid in string literals
1376                                         if (recordLineSeparator) {
1377                                                 pushLineSeparator ();
1378                                         }
1379                                 }
1380
1381                                 unicodeAsBackSlash = false;
1382                                 currentCharacter = source[currentPosition++];                          // consume next character
1383
1384                                 if (withoutUnicodePtr != 0) {
1385                                         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1386                                 }
1387                         }
1388                 } catch (IndexOutOfBoundsException e) {
1389                         // reset end position for error reporting
1390                         currentPosition -= 2;
1391                         throw new InvalidInputException(UNTERMINATED_STRING);
1392                 } catch (InvalidInputException e) {
1393                         if (e.getMessage().equals(INVALID_ESCAPE)) {
1394                                 // relocate if finding another quote fairly close: thus unicode
1395                                 // '/u000D' will be fully consumed
1396                                 for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
1397                                         if (currentPosition + lookAhead == source.length)
1398                                                 break;
1399                                         if (source[currentPosition + lookAhead] == '\n')
1400                                                 break;
1401                                         if (source[currentPosition + lookAhead] == '\"') {
1402                                                 currentPosition += lookAhead + 1;
1403                                                 break;
1404                                         }
1405                                 }
1406                         }
1407                         throw e; // rethrow
1408                 }
1409
1410                 if (checkNonExternalizedStringLiterals) { // check for presence of NLS tags
1411                         // $NON-NLS-?$ where ? is an int.
1412                         if (currentLine == null) {
1413                                 currentLine = new NLSLine ();
1414                                 lines.add (currentLine);
1415                         }
1416                         currentLine.add (new StringLiteral (getCurrentTokenSourceString (), startPosition, currentPosition - 1));
1417                 }
1418         }
1419
1420         /**
1421          *
1422          */
1423         public TokenName getNextToken() throws InvalidInputException {
1424                 if (!phpMode) {
1425                         return getInlinedHTMLToken(currentPosition);
1426                 } else {
1427                         if (fFillerToken != TokenName.EOF) {
1428                                 TokenName tempToken;
1429                                 startPosition = currentPosition;
1430                                 tempToken = fFillerToken;
1431                                 fFillerToken = TokenName.EOF;
1432                                 return tempToken;
1433                         }
1434                         this.wasAcr = false;
1435                         if (diet) {
1436                                 jumpOverMethodBody();
1437                                 diet = false;
1438                                 return currentPosition > source.length ? TokenName.EOF
1439                                                 : TokenName.RBRACE;
1440                         }
1441                         try {
1442                                 while (true) {
1443                                         withoutUnicodePtr = 0;
1444                                         // ---------Consume white space and handles
1445                                         // startPosition---------
1446                                         int whiteStart = currentPosition;
1447                                         startPosition = currentPosition;
1448                                         currentCharacter = source[currentPosition++];
1449
1450                                         while ((currentCharacter == ' ') || Character.isWhitespace(currentCharacter)) {
1451                                                 if ((currentCharacter == '\r') || (currentCharacter == '\n')) {
1452                                                         checkNonExternalizeString();
1453
1454                                                         if (recordLineSeparator) {
1455                                                                 pushLineSeparator();
1456                                                         } else {
1457                                                                 currentLine = null;
1458                                                         }
1459                                                 }
1460                                                 startPosition = currentPosition;
1461                                                 currentCharacter = source[currentPosition++];
1462                                         }
1463
1464                                         if (tokenizeWhiteSpace && (whiteStart != currentPosition - 1)) {
1465                                                 // reposition scanner in case we are interested by
1466                                                 // spaces as tokens
1467                                                 currentPosition--;
1468                                                 startPosition = whiteStart;
1469                                                 return TokenName.WHITESPACE;
1470                                         }
1471                                         // little trick to get out in the middle of a source
1472                                         // compuation
1473                                         if (currentPosition > eofPosition)
1474                                                 return TokenName.EOF;
1475                                         // ---------Identify the next token-------------
1476                                         switch (currentCharacter) {
1477                                         case '(':
1478                                                 return getCastOrParen();
1479                                         case ')':
1480                                                 return TokenName.RPAREN;
1481                                         case '{':
1482                                                 return TokenName.LBRACE;
1483                                         case '}':
1484                                                 return TokenName.RBRACE;
1485                                         case '[':
1486                                                 return TokenName.LBRACKET;
1487                                         case ']':
1488                                                 return TokenName.RBRACKET;
1489                                         case ';':
1490                                                 return TokenName.SEMICOLON;
1491                                         case ',':
1492                                                 return TokenName.COMMA;
1493                                         case '.':
1494                                                 if (getNextChar('='))
1495                                                         return TokenName.DOT_EQUAL;
1496                                                 if (getNextCharAsDigit())
1497                                                         return scanNumber(true);
1498                                                 return TokenName.DOT;
1499                                         case '\\':
1500                                             return TokenName.BACKSLASH;
1501                                         case '+': {
1502                                                 int test;
1503                                                 if ((test = getNextChar('+', '=')) == 0)
1504                                                         return TokenName.PLUS_PLUS;
1505                                                 if (test > 0)
1506                                                         return TokenName.PLUS_EQUAL;
1507                                                 return TokenName.PLUS;
1508                                         }
1509                                         case '-': {
1510                                                 int test;
1511                                                 if ((test = getNextChar('-', '=')) == 0)
1512                                                         return TokenName.MINUS_MINUS;
1513                                                 if (test > 0)
1514                                                         return TokenName.MINUS_EQUAL;
1515                                                 if (getNextChar('>'))
1516                                                         return TokenName.MINUS_GREATER;
1517                                                 return TokenName.MINUS;
1518                                         }
1519                                         case '~':
1520                                                 if (getNextChar('='))
1521                                                         return TokenName.TWIDDLE_EQUAL;
1522                                                 return TokenName.TWIDDLE;
1523                                         case '!':
1524                                                 if (getNextChar('=')) {
1525                                                         if (getNextChar('=')) {
1526                                                                 return TokenName.NOT_EQUAL_EQUAL;
1527                                                         }
1528                                                         return TokenName.NOT_EQUAL;
1529                                                 }
1530                                                 return TokenName.NOT;
1531                                         case '*':
1532                                                 if (getNextChar('='))
1533                                                         return TokenName.MULTIPLY_EQUAL;
1534                                                 return TokenName.MULTIPLY;
1535                                         case '%':
1536                                                 if (getNextChar('='))
1537                                                         return TokenName.REMAINDER_EQUAL;
1538                                                 return TokenName.REMAINDER;
1539
1540                                         case '<': {
1541                                                 int oldPosition = currentPosition;
1542
1543                                                 try {
1544                                                         currentCharacter = source[currentPosition++];
1545                                                 } catch (IndexOutOfBoundsException e) {
1546                                                         currentPosition = oldPosition;
1547                                                         return TokenName.LESS;
1548                                                 }
1549
1550                                                 switch (currentCharacter) {
1551                                                         case '=':
1552                                                                 return TokenName.LESS_EQUAL;
1553
1554                                                         case '>':
1555                                                                 return TokenName.NOT_EQUAL;
1556
1557                                                         case '<':
1558                                                                 if (getNextChar ('=')) {
1559                                                                         return TokenName.LEFT_SHIFT_EQUAL;
1560                                                                 }
1561
1562                                                                 if (getNextChar('<')) {
1563                                                                         currentCharacter = source[currentPosition++];
1564
1565                                                                         while (Character.isWhitespace(currentCharacter)) {
1566                                                                                 currentCharacter = source[currentPosition++];
1567                                                                         }
1568
1569                                                                         int heredocStart = currentPosition - 1;
1570                                                                         int heredocLength = 0;
1571
1572                                                                         if (isPHPIdentifierStart (currentCharacter)) {
1573                                                                                 currentCharacter = source[currentPosition++];
1574                                                                         }
1575                                                                         else {
1576                                                                                 return TokenName.ERROR;
1577                                                                         }
1578
1579                                                                         while (isPHPIdentifierPart(currentCharacter)) {
1580                                                                                 currentCharacter = source[currentPosition++];
1581                                                                         }
1582
1583                                                                         heredocLength = currentPosition - heredocStart - 1;
1584
1585                                                                         // heredoc end-tag determination
1586                                                                         boolean endTag = true;
1587                                                                         char ch;
1588                                                                         do {
1589                                                                                 ch = source[currentPosition++];
1590
1591                                                                                 if (ch == '\r' || ch == '\n') {
1592                                                                                         if (recordLineSeparator) {
1593                                                                                                 pushLineSeparator();
1594                                                                                         }
1595                                                                                         else {
1596                                                                                                 currentLine = null;
1597                                                                                         }
1598
1599                                                                                         for (int i = 0; i < heredocLength; i++) {
1600                                                                                                 if (source[currentPosition + i] != source[heredocStart + i]) {
1601                                                                                                         endTag = false;
1602                                                                                                         break;
1603                                                                                                 }
1604                                                                                         }
1605
1606                                                                                         if (endTag) {
1607                                                                                                 currentPosition += heredocLength - 1;
1608                                                                                                 currentCharacter = source[currentPosition++];
1609                                                                                                 break; // do...while loop
1610                                                                                         }
1611                                                                                         else {
1612                                                                                                 endTag = true;
1613                                                                                         }
1614                                                                                 }
1615                                                                         } while (true);
1616
1617                                                                         return TokenName.HEREDOC;
1618                                                                 }
1619                                                                 return TokenName.LEFT_SHIFT;
1620                                                         }
1621                                                         currentPosition = oldPosition;
1622                                                         return TokenName.LESS;
1623                                                 }
1624
1625                                         case '>': {
1626                                                 int test;
1627                                                 if ((test = getNextChar('=', '>')) == 0)
1628                                                         return TokenName.GREATER_EQUAL;
1629                                                 if (test > 0) {
1630                                                         if ((test = getNextChar('=', '>')) == 0)
1631                                                                 return TokenName.RIGHT_SHIFT_EQUAL;
1632                                                         return TokenName.RIGHT_SHIFT;
1633                                                 }
1634                                                 return TokenName.GREATER;
1635                                         }
1636                                         case '=':
1637                                                 if (getNextChar('=')) {
1638                                                         if (getNextChar('=')) {
1639                                                                 return TokenName.EQUAL_EQUAL_EQUAL;
1640                                                         }
1641                                                         return TokenName.EQUAL_EQUAL;
1642                                                 }
1643                                                 if (getNextChar('>'))
1644                                                         return TokenName.EQUAL_GREATER;
1645                                                 return TokenName.EQUAL;
1646                                         case '&': {
1647                                                 int test;
1648                                                 if ((test = getNextChar('&', '=')) == 0)
1649                                                         return TokenName.AND_AND;
1650                                                 if (test > 0)
1651                                                         return TokenName.AND_EQUAL;
1652                                                 return TokenName.OP_AND;
1653                                         }
1654                                         case '|': {
1655                                                 int test;
1656                                                 if ((test = getNextChar('|', '=')) == 0)
1657                                                         return TokenName.OR_OR;
1658                                                 if (test > 0)
1659                                                         return TokenName.OR_EQUAL;
1660                                                 return TokenName.OP_OR;
1661                                         }
1662                                         case '^':
1663                                                 if (getNextChar('='))
1664                                                         return TokenName.XOR_EQUAL;
1665                                                 return TokenName.OP_XOR;
1666                                         case '?':
1667                                                 if (getNextChar('>')) {
1668                                                         phpMode = false;
1669                                                         if (currentPosition == source.length) {
1670                                                                 phpMode = true;
1671                                                                 return TokenName.INLINE_HTML;
1672                                                         }
1673                                                         return getInlinedHTMLToken(currentPosition - 2);
1674                                                 }
1675                                                 else if (getNextChar(':')) {
1676                                                         return TokenName.TERNARY_SHORT;
1677                                                 }
1678
1679                                                 return TokenName.QUESTION;
1680
1681                                         case ':':
1682                                                 if (getNextChar(':'))
1683                                                         return TokenName.PAAMAYIM_NEKUDOTAYIM;
1684                                                 return TokenName.COLON;
1685                                         case '@':
1686                                                 return TokenName.OP_AT;
1687                                         case '\'':
1688                                                 consumeStringConstant();
1689                                                 return TokenName.STRINGSINGLEQUOTE;
1690                                         case '"':
1691                                                 // if (tokenizeStrings) {
1692                                                 consumeStringLiteral();
1693                                                 return TokenName.STRINGDOUBLEQUOTE;
1694                                                 // }
1695                                                 // return TokenName.EncapsedString2;
1696                                         case '`':
1697                                                 // if (tokenizeStrings) {
1698                                                 consumeStringInterpolated();
1699                                                 return TokenName.STRINGINTERPOLATED;
1700                                                 // }
1701                                                 // return TokenName.EncapsedString0;
1702                                         case '#':
1703                                         case '/': {
1704                                                 char startChar = currentCharacter;
1705                                                 if (getNextChar('=') && startChar == '/') {
1706                                                         return TokenName.DIVIDE_EQUAL;
1707                                                 }
1708                                                 int test;
1709                                                 if ((startChar == '#')
1710                                                                 || (test = getNextChar('/', '*')) == 0) {
1711                                                         // line comment
1712                                                         this.lastCommentLinePosition = this.currentPosition;
1713                                                         int endPositionForLineComment = 0;
1714                                                         try { // get the next char
1715                                                                 currentCharacter = source[currentPosition++];
1716                                                                 // if (((currentCharacter =
1717                                                                 // source[currentPosition++])
1718                                                                 // == '\\')
1719                                                                 // && (source[currentPosition] == 'u')) {
1720                                                                 // //-------------unicode traitement
1721                                                                 // ------------
1722                                                                 // int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
1723                                                                 // currentPosition++;
1724                                                                 // while (source[currentPosition] == 'u') {
1725                                                                 // currentPosition++;
1726                                                                 // }
1727                                                                 // if ((c1 =
1728                                                                 // Character.getNumericValue(source[currentPosition++]))
1729                                                                 // > 15
1730                                                                 // || c1 < 0
1731                                                                 // || (c2 =
1732                                                                 // Character.getNumericValue(source[currentPosition++]))
1733                                                                 // > 15
1734                                                                 // || c2 < 0
1735                                                                 // || (c3 =
1736                                                                 // Character.getNumericValue(source[currentPosition++]))
1737                                                                 // > 15
1738                                                                 // || c3 < 0
1739                                                                 // || (c4 =
1740                                                                 // Character.getNumericValue(source[currentPosition++]))
1741                                                                 // > 15
1742                                                                 // || c4 < 0) {
1743                                                                 // throw new
1744                                                                 // InvalidInputException(INVALID_UNICODE_ESCAPE);
1745                                                                 // } else {
1746                                                                 // currentCharacter =
1747                                                                 // (char) (((c1 * 16 + c2) * 16 + c3) * 16 +
1748                                                                 // c4);
1749                                                                 // }
1750                                                                 // }
1751                                                                 // handle the \\u case manually into comment
1752                                                                 // if (currentCharacter == '\\') {
1753                                                                 // if (source[currentPosition] == '\\')
1754                                                                 // currentPosition++;
1755                                                                 // } //jump over the \\
1756                                                                 boolean isUnicode = false;
1757                                                                 while (currentCharacter != '\r'
1758                                                                                 && currentCharacter != '\n') {
1759                                                                         this.lastCommentLinePosition = this.currentPosition;
1760                                                                         if (currentCharacter == '?') {
1761                                                                                 if (getNextChar('>')) {
1762                                                                                         // ?> breaks line comments
1763                                                                                         startPosition = currentPosition - 2;
1764                                                                                         phpMode = false;
1765                                                                                         return TokenName.INLINE_HTML;
1766                                                                                 }
1767                                                                         }
1768                                                                         // get the next char
1769                                                                         isUnicode = false;
1770                                                                         currentCharacter = source[currentPosition++];
1771                                                                         // if (((currentCharacter =
1772                                                                         // source[currentPosition++])
1773                                                                         // == '\\')
1774                                                                         // && (source[currentPosition] == 'u')) {
1775                                                                         // isUnicode = true;
1776                                                                         // //-------------unicode traitement
1777                                                                         // ------------
1778                                                                         // int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
1779                                                                         // currentPosition++;
1780                                                                         // while (source[currentPosition] == 'u') {
1781                                                                         // currentPosition++;
1782                                                                         // }
1783                                                                         // if ((c1 =
1784                                                                         // Character.getNumericValue(source[currentPosition++]))
1785                                                                         // > 15
1786                                                                         // || c1 < 0
1787                                                                         // || (c2 =
1788                                                                         // Character.getNumericValue(
1789                                                                         // source[currentPosition++]))
1790                                                                         // > 15
1791                                                                         // || c2 < 0
1792                                                                         // || (c3 =
1793                                                                         // Character.getNumericValue(
1794                                                                         // source[currentPosition++]))
1795                                                                         // > 15
1796                                                                         // || c3 < 0
1797                                                                         // || (c4 =
1798                                                                         // Character.getNumericValue(
1799                                                                         // source[currentPosition++]))
1800                                                                         // > 15
1801                                                                         // || c4 < 0) {
1802                                                                         // throw new
1803                                                                         // InvalidInputException(INVALID_UNICODE_ESCAPE);
1804                                                                         // } else {
1805                                                                         // currentCharacter =
1806                                                                         // (char) (((c1 * 16 + c2) * 16 + c3) * 16 +
1807                                                                         // c4);
1808                                                                         // }
1809                                                                         // }
1810                                                                         // handle the \\u case manually into comment
1811                                                                         // if (currentCharacter == '\\') {
1812                                                                         // if (source[currentPosition] == '\\')
1813                                                                         // currentPosition++;
1814                                                                         // } //jump over the \\
1815                                                                 }
1816                                                                 if (isUnicode) {
1817                                                                         endPositionForLineComment = currentPosition - 6;
1818                                                                 } else {
1819                                                                         endPositionForLineComment = currentPosition - 1;
1820                                                                 }
1821                                                                 // recordComment(false);
1822                                                                 recordComment(TokenName.COMMENT_LINE);
1823                                                                 if (this.taskTags != null)
1824                                                                         checkTaskTag(this.startPosition,
1825                                                                                         this.currentPosition);
1826                                                                 if ((currentCharacter == '\r')
1827                                                                                 || (currentCharacter == '\n')) {
1828                                                                         checkNonExternalizeString();
1829                                                                         if (recordLineSeparator) {
1830                                                                                 if (isUnicode) {
1831                                                                                         pushUnicodeLineSeparator();
1832                                                                                 } else {
1833                                                                                         pushLineSeparator();
1834                                                                                 }
1835                                                                         } else {
1836                                                                                 currentLine = null;
1837                                                                         }
1838                                                                 }
1839                                                                 if (tokenizeComments) {
1840                                                                         if (!isUnicode) {
1841                                                                                 currentPosition = endPositionForLineComment;
1842                                                                                 // reset one character behind
1843                                                                         }
1844                                                                         return TokenName.COMMENT_LINE;
1845                                                                 }
1846                                                         } catch (IndexOutOfBoundsException e) { // an eof
1847                                                                                                                                         // will them
1848                                                                 // be generated
1849                                                                 if (tokenizeComments) {
1850                                                                         currentPosition--;
1851                                                                         // reset one character behind
1852                                                                         return TokenName.COMMENT_LINE;
1853                                                                 }
1854                                                         }
1855                                                         break;
1856                                                 }
1857                                                 if (test > 0) {
1858                                                         // traditional and annotation comment
1859                                                         boolean isJavadoc = false, star = false;
1860                                                         // consume next character
1861                                                         unicodeAsBackSlash = false;
1862                                                         currentCharacter = source[currentPosition++];
1863                                                         // if (((currentCharacter =
1864                                                         // source[currentPosition++]) ==
1865                                                         // '\\')
1866                                                         // && (source[currentPosition] == 'u')) {
1867                                                         // getNextUnicodeChar();
1868                                                         // } else {
1869                                                         // if (withoutUnicodePtr != 0) {
1870                                                         // withoutUnicodeBuffer[++withoutUnicodePtr] =
1871                                                         // currentCharacter;
1872                                                         // }
1873                                                         // }
1874                                                         if (currentCharacter == '*') {
1875                                                                 isJavadoc = true;
1876                                                                 star = true;
1877                                                         }
1878                                                         if ((currentCharacter == '\r')
1879                                                                         || (currentCharacter == '\n')) {
1880                                                                 checkNonExternalizeString();
1881                                                                 if (recordLineSeparator) {
1882                                                                         pushLineSeparator();
1883                                                                 } else {
1884                                                                         currentLine = null;
1885                                                                 }
1886                                                         }
1887                                                         try { // get the next char
1888                                                                 currentCharacter = source[currentPosition++];
1889                                                                 // if (((currentCharacter =
1890                                                                 // source[currentPosition++])
1891                                                                 // == '\\')
1892                                                                 // && (source[currentPosition] == 'u')) {
1893                                                                 // //-------------unicode traitement
1894                                                                 // ------------
1895                                                                 // getNextUnicodeChar();
1896                                                                 // }
1897                                                                 // handle the \\u case manually into comment
1898                                                                 // if (currentCharacter == '\\') {
1899                                                                 // if (source[currentPosition] == '\\')
1900                                                                 // currentPosition++;
1901                                                                 // //jump over the \\
1902                                                                 // }
1903                                                                 // empty comment is not a javadoc /**/
1904                                                                 if (currentCharacter == '/') {
1905                                                                         isJavadoc = false;
1906                                                                 }
1907                                                                 // loop until end of comment */
1908                                                                 while ((currentCharacter != '/') || (!star)) {
1909                                                                         if ((currentCharacter == '\r')
1910                                                                                         || (currentCharacter == '\n')) {
1911                                                                                 checkNonExternalizeString();
1912                                                                                 if (recordLineSeparator) {
1913                                                                                         pushLineSeparator();
1914                                                                                 } else {
1915                                                                                         currentLine = null;
1916                                                                                 }
1917                                                                         }
1918                                                                         star = currentCharacter == '*';
1919                                                                         // get next char
1920                                                                         currentCharacter = source[currentPosition++];
1921                                                                         // if (((currentCharacter =
1922                                                                         // source[currentPosition++])
1923                                                                         // == '\\')
1924                                                                         // && (source[currentPosition] == 'u')) {
1925                                                                         // //-------------unicode traitement
1926                                                                         // ------------
1927                                                                         // getNextUnicodeChar();
1928                                                                         // }
1929                                                                         // handle the \\u case manually into comment
1930                                                                         // if (currentCharacter == '\\') {
1931                                                                         // if (source[currentPosition] == '\\')
1932                                                                         // currentPosition++;
1933                                                                         // } //jump over the \\
1934                                                                 }
1935                                                                 // recordComment(isJavadoc);
1936                                                                 if (isJavadoc) {
1937                                                                         recordComment(TokenName.COMMENT_PHPDOC);
1938                                                                 } else {
1939                                                                         recordComment(TokenName.COMMENT_BLOCK);
1940                                                                 }
1941
1942                                                                 if (tokenizeComments) {
1943                                                                         if (isJavadoc)
1944                                                                                 return TokenName.COMMENT_PHPDOC;
1945                                                                         return TokenName.COMMENT_BLOCK;
1946                                                                 }
1947
1948                                                                 if (this.taskTags != null) {
1949                                                                         checkTaskTag(this.startPosition,
1950                                                                                         this.currentPosition);
1951                                                                 }
1952                                                         } catch (IndexOutOfBoundsException e) {
1953                                                                 // reset end position for error reporting
1954                                                                 currentPosition -= 2;
1955                                                                 throw new InvalidInputException(
1956                                                                                 UNTERMINATED_COMMENT);
1957                                                         }
1958                                                         break;
1959                                                 }
1960                                                 return TokenName.DIVIDE;
1961                                         }
1962                                         case '\u001a':
1963                                                 if (atEnd())
1964                                                         return TokenName.EOF;
1965                                                 // the atEnd may not be <currentPosition ==
1966                                                 // source.length> if
1967                                                 // source is only some part of a real (external) stream
1968                                                 throw new InvalidInputException("Ctrl-Z"); //$NON-NLS-1$
1969                                         default:
1970                                                 if (currentCharacter == '$') {
1971                                                         int oldPosition = currentPosition;
1972                                                         try {
1973                                                                 currentCharacter = source[currentPosition++];
1974                                                                 if (isPHPIdentifierStart(currentCharacter)) {
1975                                                                         return scanIdentifierOrKeyword(true);
1976                                                                 } else {
1977                                                                         currentPosition = oldPosition;
1978                                                                         return TokenName.DOLLAR;
1979                                                                 }
1980                                                         } catch (IndexOutOfBoundsException e) {
1981                                                                 currentPosition = oldPosition;
1982                                                                 return TokenName.DOLLAR;
1983                                                         }
1984                                                 }
1985
1986                                                 if (isPHPIdentifierStart(currentCharacter)) {
1987                                                         return scanIdentifierOrKeyword(false);
1988                                                 }
1989
1990                                                 if (Character.isDigit(currentCharacter)) {
1991                                                         return scanNumber(false);
1992                                                 }
1993
1994                                                 return TokenName.ERROR;
1995                                         }
1996                                 }
1997                         } // -----------------end switch while try--------------------
1998                         catch (IndexOutOfBoundsException e) {
1999                         }
2000                 }
2001                 return TokenName.EOF;
2002         }
2003
2004         /**
2005          * @return
2006          * @throws InvalidInputException
2007          */
2008         private TokenName getInlinedHTMLToken(int start) throws InvalidInputException {
2009                 boolean phpShortTag = false; // true, if <?= detected
2010                 if (currentPosition > source.length) {
2011                         currentPosition = source.length;
2012                         return TokenName.EOF;
2013                 }
2014                 startPosition = start;
2015                 try {
2016                         while (!phpMode) {
2017                                 currentCharacter = source[currentPosition++];
2018                                 if (currentCharacter == '<') {
2019                                         if (getNextChar('?')) {
2020                                                 currentCharacter = source[currentPosition++];
2021                                                 if ((currentCharacter != 'P')
2022                                                                 && (currentCharacter != 'p')) {
2023                                                         if (currentCharacter != '=') { // <?=
2024                                                                 currentPosition--;
2025                                                                 phpShortTag = false;
2026                                                         } else {
2027                                                                 phpShortTag = true;
2028                                                         }
2029                                                         // <?
2030                                                         if (ignorePHPOneLiner) { // for CodeFormatter
2031                                                                 if (lookAheadLinePHPTag() == TokenName.INLINE_HTML) {
2032                                                                         phpMode = true;
2033                                                                         if (phpShortTag) {
2034                                                                                 fFillerToken = TokenName.ECHO_INVISIBLE;
2035                                                                         }
2036                                                                         return TokenName.INLINE_HTML;
2037                                                                 }
2038                                                         } else {
2039                                                                 boolean foundXML = false;
2040                                                                 if (getNextChar('X', 'x') >= 0) {
2041                                                                         if (getNextChar('M', 'm') >= 0) {
2042                                                                                 if (getNextChar('L', 'l') >= 0) {
2043                                                                                         foundXML = true;
2044                                                                                 }
2045                                                                         }
2046                                                                 }
2047                                                                 if (!foundXML) {
2048                                                                         phpMode = true;
2049                                                                 }
2050                                                                 if (phpShortTag) {
2051                                                                         fFillerToken = TokenName.ECHO_INVISIBLE;
2052                                                                 }
2053                                                                 return TokenName.INLINE_HTML;
2054                                                         }
2055                                                 } else {
2056                                                         if (getNextChar('H', 'h') >= 0) {
2057                                                                 if (getNextChar('P', 'p') >= 0) {
2058                                                                         // <?PHP <?php
2059                                                                         if (ignorePHPOneLiner) {
2060                                                                                 if (lookAheadLinePHPTag() == TokenName.INLINE_HTML) {
2061                                                                                         phpMode = true;
2062                                                                                         return TokenName.INLINE_HTML;
2063                                                                                 }
2064                                                                         } else {
2065                                                                                 phpMode = true;
2066                                                                                 return TokenName.INLINE_HTML;
2067                                                                         }
2068                                                                 }
2069                                                         }
2070                                                         // }
2071                                                 }
2072                                         }
2073                                 }
2074                                 if ((currentCharacter == '\r') || (currentCharacter == '\n')) {
2075                                         if (recordLineSeparator) {
2076                                                 pushLineSeparator();
2077                                         } else {
2078                                                 currentLine = null;
2079                                         }
2080                                 }
2081                         } // -----------------while--------------------
2082                         phpMode = true;
2083                         return TokenName.INLINE_HTML;
2084                 } // -----------------try--------------------
2085                 catch (IndexOutOfBoundsException e) {
2086                         startPosition = start;
2087                         currentPosition--;
2088                 }
2089                 phpMode = true;
2090                 return TokenName.INLINE_HTML;
2091         }
2092
2093         /**
2094          * check if the PHP is only in this line (for CodeFormatter)
2095          *
2096          * @return
2097          */
2098         private TokenName lookAheadLinePHPTag() {
2099                 int currentPositionInLine = currentPosition;
2100                 char previousCharInLine = ' ';
2101                 char currentCharInLine = ' ';
2102                 boolean singleQuotedStringActive = false;
2103                 boolean doubleQuotedStringActive = false;
2104
2105                 try {
2106                         // look ahead in this line
2107                         while (true) {
2108                                 previousCharInLine = currentCharInLine;
2109                                 currentCharInLine = source[currentPositionInLine++];
2110                                 switch (currentCharInLine) {
2111                                 case '>':
2112                                         if (previousCharInLine == '?') {
2113                                                 // update the scanner's current Position in the source
2114                                                 currentPosition = currentPositionInLine;
2115                                                 // use as "dummy" token
2116                                                 return TokenName.EOF;
2117                                         }
2118                                         break;
2119                                 case '\\':
2120                                         if (doubleQuotedStringActive) {
2121                                                 // ignore escaped characters in double quoted strings
2122                                                 previousCharInLine = currentCharInLine;
2123                                                 currentCharInLine = source[currentPositionInLine++];
2124                                         }
2125                                 case '\"':
2126                                         if (doubleQuotedStringActive) {
2127                                                 doubleQuotedStringActive = false;
2128                                         } else {
2129                                                 if (!singleQuotedStringActive) {
2130                                                         doubleQuotedStringActive = true;
2131                                                 }
2132                                         }
2133                                         break;
2134                                 case '\'':
2135                                         if (singleQuotedStringActive) {
2136                                                 if (previousCharInLine != '\\') {
2137                                                         singleQuotedStringActive = false;
2138                                                 }
2139                                         } else {
2140                                                 if (!doubleQuotedStringActive) {
2141                                                         singleQuotedStringActive = true;
2142                                                 }
2143                                         }
2144                                         break;
2145                                 case '\n':
2146                                         phpMode = true;
2147                                         return TokenName.INLINE_HTML;
2148                                 case '#':
2149                                         if (!singleQuotedStringActive && !doubleQuotedStringActive) {
2150                                                 phpMode = true;
2151                                                 return TokenName.INLINE_HTML;
2152                                         }
2153                                         break;
2154                                 case '/':
2155                                         if (previousCharInLine == '/' && !singleQuotedStringActive
2156                                                         && !doubleQuotedStringActive) {
2157                                                 phpMode = true;
2158                                                 return TokenName.INLINE_HTML;
2159                                         }
2160                                         break;
2161                                 case '*':
2162                                         if (previousCharInLine == '/' && !singleQuotedStringActive
2163                                                         && !doubleQuotedStringActive) {
2164                                                 phpMode = true;
2165                                                 return TokenName.INLINE_HTML;
2166                                         }
2167                                         break;
2168                                 }
2169                         }
2170                 } catch (IndexOutOfBoundsException e) {
2171                         phpMode = true;
2172                         currentPosition = currentPositionInLine - 1;
2173                         return TokenName.INLINE_HTML;
2174                 }
2175         }
2176
2177         // public final void getNextUnicodeChar()
2178         // throws IndexOutOfBoundsException, InvalidInputException {
2179         // //VOID
2180         // //handle the case of unicode.
2181         // //when a unicode appears then we must use a buffer that holds char
2182         // internal values
2183         // //At the end of this method currentCharacter holds the new visited char
2184         // //and currentPosition points right next after it
2185         //
2186         // //ALL getNextChar.... ARE OPTIMIZED COPIES
2187         //
2188         // int c1 = 0, c2 = 0, c3 = 0, c4 = 0, unicodeSize = 6;
2189         // currentPosition++;
2190         // while (source[currentPosition] == 'u') {
2191         // currentPosition++;
2192         // unicodeSize++;
2193         // }
2194         //
2195         // if ((c1 = Character.getNumericValue(source[currentPosition++])) > 15
2196         // || c1 < 0
2197         // || (c2 = Character.getNumericValue(source[currentPosition++])) > 15
2198         // || c2 < 0
2199         // || (c3 = Character.getNumericValue(source[currentPosition++])) > 15
2200         // || c3 < 0
2201         // || (c4 = Character.getNumericValue(source[currentPosition++])) > 15
2202         // || c4 < 0) {
2203         // throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
2204         // } else {
2205         // currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
2206         // //need the unicode buffer
2207         // if (withoutUnicodePtr == 0) {
2208         // //buffer all the entries that have been left aside....
2209         // withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
2210         // System.arraycopy(
2211         // source,
2212         // startPosition,
2213         // withoutUnicodeBuffer,
2214         // 1,
2215         // withoutUnicodePtr);
2216         // }
2217         // //fill the buffer with the char
2218         // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
2219         // }
2220         // unicodeAsBackSlash = currentCharacter == '\\';
2221         // }
2222         /*
2223          * Tokenize a method body, assuming that curly brackets are properly
2224          * balanced.
2225          */
2226         public final void jumpOverMethodBody() {
2227                 this.wasAcr = false;
2228                 int found = 1;
2229                 try {
2230                         while (true) { // loop for jumping over comments
2231                                 // ---------Consume white space and handles
2232                                 // startPosition---------
2233                                 boolean isWhiteSpace;
2234                                 do {
2235                                         startPosition = currentPosition;
2236                                         currentCharacter = source[currentPosition++];
2237                                         // if (((currentCharacter = source[currentPosition++]) ==
2238                                         // '\\')
2239                                         // && (source[currentPosition] == 'u')) {
2240                                         // isWhiteSpace = jumpOverUnicodeWhiteSpace();
2241                                         // } else {
2242                                         if (recordLineSeparator
2243                                                         && ((currentCharacter == '\r') || (currentCharacter == '\n')))
2244                                                 pushLineSeparator();
2245                                         isWhiteSpace = Character.isWhitespace(currentCharacter);
2246                                         // }
2247                                 } while (isWhiteSpace);
2248                                 // -------consume token until } is found---------
2249                                 switch (currentCharacter) {
2250                                 case '{':
2251                                         found++;
2252                                         break;
2253                                 case '}':
2254                                         found--;
2255                                         if (found == 0)
2256                                                 return;
2257                                         break;
2258                                 case '\'': {
2259                                         boolean test;
2260                                         test = getNextChar('\\');
2261                                         if (test) {
2262                                                 try {
2263                                                         scanDoubleQuotedEscapeCharacter();
2264                                                 } catch (InvalidInputException ex) {
2265                                                 }
2266                                                 ;
2267                                         } else {
2268                                                 // try { // consume next character
2269                                                 unicodeAsBackSlash = false;
2270                                                 currentCharacter = source[currentPosition++];
2271                                                 // if (((currentCharacter = source[currentPosition++])
2272                                                 // == '\\')
2273                                                 // && (source[currentPosition] == 'u')) {
2274                                                 // getNextUnicodeChar();
2275                                                 // } else {
2276                                                 if (withoutUnicodePtr != 0) {
2277                                                         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
2278                                                 }
2279                                                 // }
2280                                                 // } catch (InvalidInputException ex) {
2281                                                 // };
2282                                         }
2283                                         getNextChar('\'');
2284                                         break;
2285                                 }
2286                                 case '"':
2287                                         try {
2288                                                 // try { // consume next character
2289                                                 unicodeAsBackSlash = false;
2290                                                 currentCharacter = source[currentPosition++];
2291                                                 // if (((currentCharacter = source[currentPosition++])
2292                                                 // == '\\')
2293                                                 // && (source[currentPosition] == 'u')) {
2294                                                 // getNextUnicodeChar();
2295                                                 // } else {
2296                                                 if (withoutUnicodePtr != 0) {
2297                                                         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
2298                                                 }
2299                                                 // }
2300                                                 // } catch (InvalidInputException ex) {
2301                                                 // };
2302                                                 while (currentCharacter != '"') {
2303                                                         if (currentCharacter == '\r') {
2304                                                                 if (source[currentPosition] == '\n')
2305                                                                         currentPosition++;
2306                                                                 break;
2307                                                                 // the string cannot go further that the line
2308                                                         }
2309                                                         if (currentCharacter == '\n') {
2310                                                                 break;
2311                                                                 // the string cannot go further that the line
2312                                                         }
2313                                                         if (currentCharacter == '\\') {
2314                                                                 try {
2315                                                                         scanDoubleQuotedEscapeCharacter();
2316                                                                 } catch (InvalidInputException ex) {
2317                                                                 }
2318                                                                 ;
2319                                                         }
2320                                                         // try { // consume next character
2321                                                         unicodeAsBackSlash = false;
2322                                                         currentCharacter = source[currentPosition++];
2323                                                         // if (((currentCharacter =
2324                                                         // source[currentPosition++]) == '\\')
2325                                                         // && (source[currentPosition] == 'u')) {
2326                                                         // getNextUnicodeChar();
2327                                                         // } else {
2328                                                         if (withoutUnicodePtr != 0) {
2329                                                                 withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
2330                                                         }
2331                                                         // }
2332                                                         // } catch (InvalidInputException ex) {
2333                                                         // };
2334                                                 }
2335                                         } catch (IndexOutOfBoundsException e) {
2336                                                 return;
2337                                         }
2338                                         break;
2339                                 case '/': {
2340                                         int test;
2341                                         if ((test = getNextChar('/', '*')) == 0) {
2342                                                 // line comment
2343                                                 try {
2344                                                         // get the next char
2345                                                         currentCharacter = source[currentPosition++];
2346                                                         // if (((currentCharacter =
2347                                                         // source[currentPosition++]) ==
2348                                                         // '\\')
2349                                                         // && (source[currentPosition] == 'u')) {
2350                                                         // //-------------unicode traitement ------------
2351                                                         // int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
2352                                                         // currentPosition++;
2353                                                         // while (source[currentPosition] == 'u') {
2354                                                         // currentPosition++;
2355                                                         // }
2356                                                         // if ((c1 =
2357                                                         // Character.getNumericValue(source[currentPosition++]))
2358                                                         // > 15
2359                                                         // || c1 < 0
2360                                                         // || (c2 =
2361                                                         // Character.getNumericValue(source[currentPosition++]))
2362                                                         // > 15
2363                                                         // || c2 < 0
2364                                                         // || (c3 =
2365                                                         // Character.getNumericValue(source[currentPosition++]))
2366                                                         // > 15
2367                                                         // || c3 < 0
2368                                                         // || (c4 =
2369                                                         // Character.getNumericValue(source[currentPosition++]))
2370                                                         // > 15
2371                                                         // || c4 < 0) {
2372                                                         // //error don't care of the value
2373                                                         // currentCharacter = 'A';
2374                                                         // } //something different from \n and \r
2375                                                         // else {
2376                                                         // currentCharacter =
2377                                                         // (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
2378                                                         // }
2379                                                         // }
2380                                                         while (currentCharacter != '\r'
2381                                                                         && currentCharacter != '\n') {
2382                                                                 // get the next char
2383                                                                 currentCharacter = source[currentPosition++];
2384                                                                 // if (((currentCharacter =
2385                                                                 // source[currentPosition++])
2386                                                                 // == '\\')
2387                                                                 // && (source[currentPosition] == 'u')) {
2388                                                                 // //-------------unicode traitement
2389                                                                 // ------------
2390                                                                 // int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
2391                                                                 // currentPosition++;
2392                                                                 // while (source[currentPosition] == 'u') {
2393                                                                 // currentPosition++;
2394                                                                 // }
2395                                                                 // if ((c1 =
2396                                                                 // Character.getNumericValue(source[currentPosition++]))
2397                                                                 // > 15
2398                                                                 // || c1 < 0
2399                                                                 // || (c2 =
2400                                                                 // Character.getNumericValue(source[currentPosition++]))
2401                                                                 // > 15
2402                                                                 // || c2 < 0
2403                                                                 // || (c3 =
2404                                                                 // Character.getNumericValue(source[currentPosition++]))
2405                                                                 // > 15
2406                                                                 // || c3 < 0
2407                                                                 // || (c4 =
2408                                                                 // Character.getNumericValue(source[currentPosition++]))
2409                                                                 // > 15
2410                                                                 // || c4 < 0) {
2411                                                                 // //error don't care of the value
2412                                                                 // currentCharacter = 'A';
2413                                                                 // } //something different from \n and \r
2414                                                                 // else {
2415                                                                 // currentCharacter =
2416                                                                 // (char) (((c1 * 16 + c2) * 16 + c3) * 16 +
2417                                                                 // c4);
2418                                                                 // }
2419                                                                 // }
2420                                                         }
2421                                                         if (recordLineSeparator
2422                                                                         && ((currentCharacter == '\r') || (currentCharacter == '\n')))
2423                                                                 pushLineSeparator();
2424                                                 } catch (IndexOutOfBoundsException e) {
2425                                                 } // an eof will them be generated
2426                                                 break;
2427                                         }
2428                                         if (test > 0) {
2429                                                 // traditional and annotation comment
2430                                                 boolean star = false;
2431                                                 // try { // consume next character
2432                                                 unicodeAsBackSlash = false;
2433                                                 currentCharacter = source[currentPosition++];
2434                                                 // if (((currentCharacter = source[currentPosition++])
2435                                                 // == '\\')
2436                                                 // && (source[currentPosition] == 'u')) {
2437                                                 // getNextUnicodeChar();
2438                                                 // } else {
2439                                                 if (withoutUnicodePtr != 0) {
2440                                                         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
2441                                                 }
2442                                                 // };
2443                                                 // } catch (InvalidInputException ex) {
2444                                                 // };
2445                                                 if (currentCharacter == '*') {
2446                                                         star = true;
2447                                                 }
2448                                                 if (recordLineSeparator
2449                                                                 && ((currentCharacter == '\r') || (currentCharacter == '\n')))
2450                                                         pushLineSeparator();
2451                                                 try { // get the next char
2452                                                         currentCharacter = source[currentPosition++];
2453                                                         // if (((currentCharacter =
2454                                                         // source[currentPosition++]) ==
2455                                                         // '\\')
2456                                                         // && (source[currentPosition] == 'u')) {
2457                                                         // //-------------unicode traitement ------------
2458                                                         // int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
2459                                                         // currentPosition++;
2460                                                         // while (source[currentPosition] == 'u') {
2461                                                         // currentPosition++;
2462                                                         // }
2463                                                         // if ((c1 =
2464                                                         // Character.getNumericValue(source[currentPosition++]))
2465                                                         // > 15
2466                                                         // || c1 < 0
2467                                                         // || (c2 =
2468                                                         // Character.getNumericValue(source[currentPosition++]))
2469                                                         // > 15
2470                                                         // || c2 < 0
2471                                                         // || (c3 =
2472                                                         // Character.getNumericValue(source[currentPosition++]))
2473                                                         // > 15
2474                                                         // || c3 < 0
2475                                                         // || (c4 =
2476                                                         // Character.getNumericValue(source[currentPosition++]))
2477                                                         // > 15
2478                                                         // || c4 < 0) {
2479                                                         // //error don't care of the value
2480                                                         // currentCharacter = 'A';
2481                                                         // } //something different from * and /
2482                                                         // else {
2483                                                         // currentCharacter =
2484                                                         // (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
2485                                                         // }
2486                                                         // }
2487                                                         // loop until end of comment */
2488                                                         while ((currentCharacter != '/') || (!star)) {
2489                                                                 if (recordLineSeparator
2490                                                                                 && ((currentCharacter == '\r') || (currentCharacter == '\n')))
2491                                                                         pushLineSeparator();
2492                                                                 star = currentCharacter == '*';
2493                                                                 // get next char
2494                                                                 currentCharacter = source[currentPosition++];
2495                                                                 // if (((currentCharacter =
2496                                                                 // source[currentPosition++])
2497                                                                 // == '\\')
2498                                                                 // && (source[currentPosition] == 'u')) {
2499                                                                 // //-------------unicode traitement
2500                                                                 // ------------
2501                                                                 // int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
2502                                                                 // currentPosition++;
2503                                                                 // while (source[currentPosition] == 'u') {
2504                                                                 // currentPosition++;
2505                                                                 // }
2506                                                                 // if ((c1 =
2507                                                                 // Character.getNumericValue(source[currentPosition++]))
2508                                                                 // > 15
2509                                                                 // || c1 < 0
2510                                                                 // || (c2 =
2511                                                                 // Character.getNumericValue(source[currentPosition++]))
2512                                                                 // > 15
2513                                                                 // || c2 < 0
2514                                                                 // || (c3 =
2515                                                                 // Character.getNumericValue(source[currentPosition++]))
2516                                                                 // > 15
2517                                                                 // || c3 < 0
2518                                                                 // || (c4 =
2519                                                                 // Character.getNumericValue(source[currentPosition++]))
2520                                                                 // > 15
2521                                                                 // || c4 < 0) {
2522                                                                 // //error don't care of the value
2523                                                                 // currentCharacter = 'A';
2524                                                                 // } //something different from * and /
2525                                                                 // else {
2526                                                                 // currentCharacter =
2527                                                                 // (char) (((c1 * 16 + c2) * 16 + c3) * 16 +
2528                                                                 // c4);
2529                                                                 // }
2530                                                                 // }
2531                                                         }
2532                                                 } catch (IndexOutOfBoundsException e) {
2533                                                         return;
2534                                                 }
2535                                                 break;
2536                                         }
2537                                         break;
2538                                 }
2539                                 default:
2540                                         if (isPHPIdentOrVarStart(currentCharacter)) {
2541                                                 try {
2542                                                         scanIdentifierOrKeyword((currentCharacter == '$'));
2543                                                 } catch (InvalidInputException ex) {
2544                                                 }
2545                                                 ;
2546                                                 break;
2547                                         }
2548                                         if (ObviousIdentCharNatures[currentCharacter] == C_DIGIT) {
2549                                                 // if (Character.isDigit(currentCharacter)) {
2550                                                 try {
2551                                                         scanNumber(false);
2552                                                 } catch (InvalidInputException ex) {
2553                                                 }
2554                                                 ;
2555                                                 break;
2556                                         }
2557                                 }
2558                         }
2559                         // -----------------end switch while try--------------------
2560                 } catch (IndexOutOfBoundsException e) {
2561                 } catch (InvalidInputException e) {
2562                 }
2563                 return;
2564         }
2565
2566         // public final boolean jumpOverUnicodeWhiteSpace()
2567         // throws InvalidInputException {
2568         // //BOOLEAN
2569         // //handle the case of unicode. Jump over the next whiteSpace
2570         // //making startPosition pointing on the next available char
2571         // //On false, the currentCharacter is filled up with a potential
2572         // //correct char
2573         //
2574         // try {
2575         // this.wasAcr = false;
2576         // int c1, c2, c3, c4;
2577         // int unicodeSize = 6;
2578         // currentPosition++;
2579         // while (source[currentPosition] == 'u') {
2580         // currentPosition++;
2581         // unicodeSize++;
2582         // }
2583         //
2584         // if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
2585         // || c1 < 0)
2586         // || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15
2587         // || c2 < 0)
2588         // || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15
2589         // || c3 < 0)
2590         // || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15
2591         // || c4 < 0)) {
2592         // throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
2593         // }
2594         //
2595         // currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
2596         // if (recordLineSeparator
2597         // && ((currentCharacter == '\r') || (currentCharacter == '\n')))
2598         // pushLineSeparator();
2599         // if (Character.isWhitespace(currentCharacter))
2600         // return true;
2601         //
2602         // //buffer the new char which is not a white space
2603         // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
2604         // //withoutUnicodePtr == 1 is true here
2605         // return false;
2606         // } catch (IndexOutOfBoundsException e) {
2607         // throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
2608         // }
2609         // }
2610         public final int[] getLineEnds() {
2611                 // return a bounded copy of this.lineEnds
2612                 int[] copy;
2613                 System.arraycopy(lineEnds, 0, copy = new int[linePtr + 1], 0,
2614                                 linePtr + 1);
2615                 return copy;
2616         }
2617
2618         public char[] getSource() {
2619                 return this.source;
2620         }
2621
2622         public static boolean isIdentifierOrKeyword (TokenName token) {
2623                 return (token == TokenName.IDENTIFIER) || (token.compareTo (TokenName.KEYWORD) > 0);
2624         }
2625
2626         final char[] optimizedCurrentTokenSource1() {
2627                 // return always the same char[] build only once
2628                 // optimization at no speed cost of 99.5 % of the singleCharIdentifier
2629                 char charOne = source[startPosition];
2630                 switch (charOne) {
2631                 case 'a':
2632                         return charArray_a;
2633                 case 'b':
2634                         return charArray_b;
2635                 case 'c':
2636                         return charArray_c;
2637                 case 'd':
2638                         return charArray_d;
2639                 case 'e':
2640                         return charArray_e;
2641                 case 'f':
2642                         return charArray_f;
2643                 case 'g':
2644                         return charArray_g;
2645                 case 'h':
2646                         return charArray_h;
2647                 case 'i':
2648                         return charArray_i;
2649                 case 'j':
2650                         return charArray_j;
2651                 case 'k':
2652                         return charArray_k;
2653                 case 'l':
2654                         return charArray_l;
2655                 case 'm':
2656                         return charArray_m;
2657                 case 'n':
2658                         return charArray_n;
2659                 case 'o':
2660                         return charArray_o;
2661                 case 'p':
2662                         return charArray_p;
2663                 case 'q':
2664                         return charArray_q;
2665                 case 'r':
2666                         return charArray_r;
2667                 case 's':
2668                         return charArray_s;
2669                 case 't':
2670                         return charArray_t;
2671                 case 'u':
2672                         return charArray_u;
2673                 case 'v':
2674                         return charArray_v;
2675                 case 'w':
2676                         return charArray_w;
2677                 case 'x':
2678                         return charArray_x;
2679                 case 'y':
2680                         return charArray_y;
2681                 case 'z':
2682                         return charArray_z;
2683                 default:
2684                         return new char[] { charOne };
2685                 }
2686         }
2687
2688         final char[] optimizedCurrentTokenSource2() {
2689                 char c0, c1;
2690                 c0 = source[startPosition];
2691                 c1 = source[startPosition + 1];
2692                 if (c0 == '$') {
2693                         // return always the same char[] build only once
2694                         // optimization at no speed cost of 99.5 % of the
2695                         // singleCharIdentifier
2696                         switch (c1) {
2697                         case 'a':
2698                                 return charArray_va;
2699                         case 'b':
2700                                 return charArray_vb;
2701                         case 'c':
2702                                 return charArray_vc;
2703                         case 'd':
2704                                 return charArray_vd;
2705                         case 'e':
2706                                 return charArray_ve;
2707                         case 'f':
2708                                 return charArray_vf;
2709                         case 'g':
2710                                 return charArray_vg;
2711                         case 'h':
2712                                 return charArray_vh;
2713                         case 'i':
2714                                 return charArray_vi;
2715                         case 'j':
2716                                 return charArray_vj;
2717                         case 'k':
2718                                 return charArray_vk;
2719                         case 'l':
2720                                 return charArray_vl;
2721                         case 'm':
2722                                 return charArray_vm;
2723                         case 'n':
2724                                 return charArray_vn;
2725                         case 'o':
2726                                 return charArray_vo;
2727                         case 'p':
2728                                 return charArray_vp;
2729                         case 'q':
2730                                 return charArray_vq;
2731                         case 'r':
2732                                 return charArray_vr;
2733                         case 's':
2734                                 return charArray_vs;
2735                         case 't':
2736                                 return charArray_vt;
2737                         case 'u':
2738                                 return charArray_vu;
2739                         case 'v':
2740                                 return charArray_vv;
2741                         case 'w':
2742                                 return charArray_vw;
2743                         case 'x':
2744                                 return charArray_vx;
2745                         case 'y':
2746                                 return charArray_vy;
2747                         case 'z':
2748                                 return charArray_vz;
2749                         }
2750                 }
2751                 // try to return the same char[] build only once
2752                 int hash = ((c0 << 6) + c1) % TableSize;
2753                 char[][] table = charArray_length[0][hash];
2754                 int i = newEntry2;
2755                 while (++i < InternalTableSize) {
2756                         char[] charArray = table[i];
2757                         if ((c0 == charArray[0]) && (c1 == charArray[1]))
2758                                 return charArray;
2759                 }
2760                 // ---------other side---------
2761                 i = -1;
2762                 int max = newEntry2;
2763                 while (++i <= max) {
2764                         char[] charArray = table[i];
2765                         if ((c0 == charArray[0]) && (c1 == charArray[1]))
2766                                 return charArray;
2767                 }
2768                 // --------add the entry-------
2769                 if (++max >= InternalTableSize)
2770                         max = 0;
2771                 char[] r;
2772                 table[max] = (r = new char[] { c0, c1 });
2773                 newEntry2 = max;
2774                 return r;
2775         }
2776
2777         final char[] optimizedCurrentTokenSource3() {
2778                 // try to return the same char[] build only once
2779                 char c0, c1, c2;
2780                 int hash = (((c0 = source[startPosition]) << 12)
2781                                 + ((c1 = source[startPosition + 1]) << 6) + (c2 = source[startPosition + 2]))
2782                                 % TableSize;
2783                 char[][] table = charArray_length[1][hash];
2784                 int i = newEntry3;
2785                 while (++i < InternalTableSize) {
2786                         char[] charArray = table[i];
2787                         if ((c0 == charArray[0]) && (c1 == charArray[1])
2788                                         && (c2 == charArray[2]))
2789                                 return charArray;
2790                 }
2791                 // ---------other side---------
2792                 i = -1;
2793                 int max = newEntry3;
2794                 while (++i <= max) {
2795                         char[] charArray = table[i];
2796                         if ((c0 == charArray[0]) && (c1 == charArray[1])
2797                                         && (c2 == charArray[2]))
2798                                 return charArray;
2799                 }
2800                 // --------add the entry-------
2801                 if (++max >= InternalTableSize)
2802                         max = 0;
2803                 char[] r;
2804                 table[max] = (r = new char[] { c0, c1, c2 });
2805                 newEntry3 = max;
2806                 return r;
2807         }
2808
2809         final char[] optimizedCurrentTokenSource4() {
2810                 // try to return the same char[] build only once
2811                 char c0, c1, c2, c3;
2812                 long hash = ((((long) (c0 = source[startPosition])) << 18)
2813                                 + ((c1 = source[startPosition + 1]) << 12)
2814                                 + ((c2 = source[startPosition + 2]) << 6) + (c3 = source[startPosition + 3]))
2815                                 % TableSize;
2816                 char[][] table = charArray_length[2][(int) hash];
2817                 int i = newEntry4;
2818                 while (++i < InternalTableSize) {
2819                         char[] charArray = table[i];
2820                         if ((c0 == charArray[0]) && (c1 == charArray[1])
2821                                         && (c2 == charArray[2]) && (c3 == charArray[3]))
2822                                 return charArray;
2823                 }
2824                 // ---------other side---------
2825                 i = -1;
2826
2827                 int max = newEntry4;
2828                 while (++i <= max) {
2829                         char[] charArray = table[i];
2830                         if ((c0 == charArray[0]) && (c1 == charArray[1])
2831                                         && (c2 == charArray[2]) && (c3 == charArray[3]))
2832                                 return charArray;
2833                 }
2834                 // --------add the entry-------
2835                 if (++max >= InternalTableSize)
2836                         max = 0;
2837                 char[] r;
2838                 table[max] = (r = new char[] { c0, c1, c2, c3 });
2839                 newEntry4 = max;
2840                 return r;
2841         }
2842
2843         final char[] optimizedCurrentTokenSource5() {
2844                 // try to return the same char[] build only once
2845                 char c0, c1, c2, c3, c4;
2846                 long hash = ((((long) (c0 = source[startPosition])) << 24)
2847                                 + (((long) (c1 = source[startPosition + 1])) << 18)
2848                                 + ((c2 = source[startPosition + 2]) << 12)
2849                                 + ((c3 = source[startPosition + 3]) << 6) + (c4 = source[startPosition + 4]))
2850                                 % TableSize;
2851                 char[][] table = charArray_length[3][(int) hash];
2852                 int i = newEntry5;
2853                 while (++i < InternalTableSize) {
2854                         char[] charArray = table[i];
2855                         if ((c0 == charArray[0]) && (c1 == charArray[1])
2856                                         && (c2 == charArray[2]) && (c3 == charArray[3])
2857                                         && (c4 == charArray[4]))
2858                                 return charArray;
2859                 }
2860                 // ---------other side---------
2861                 i = -1;
2862                 int max = newEntry5;
2863                 while (++i <= max) {
2864                         char[] charArray = table[i];
2865                         if ((c0 == charArray[0]) && (c1 == charArray[1])
2866                                         && (c2 == charArray[2]) && (c3 == charArray[3])
2867                                         && (c4 == charArray[4]))
2868                                 return charArray;
2869                 }
2870                 // --------add the entry-------
2871                 if (++max >= InternalTableSize)
2872                         max = 0;
2873                 char[] r;
2874                 table[max] = (r = new char[] { c0, c1, c2, c3, c4 });
2875                 newEntry5 = max;
2876                 return r;
2877         }
2878
2879         final char[] optimizedCurrentTokenSource6() {
2880                 // try to return the same char[] build only once
2881                 char c0, c1, c2, c3, c4, c5;
2882                 long hash = ((((long) (c0 = source[startPosition])) << 32)
2883                                 + (((long) (c1 = source[startPosition + 1])) << 24)
2884                                 + (((long) (c2 = source[startPosition + 2])) << 18)
2885                                 + ((c3 = source[startPosition + 3]) << 12)
2886                                 + ((c4 = source[startPosition + 4]) << 6) + (c5 = source[startPosition + 5]))
2887                                 % TableSize;
2888                 char[][] table = charArray_length[4][(int) hash];
2889                 int i = newEntry6;
2890                 while (++i < InternalTableSize) {
2891                         char[] charArray = table[i];
2892                         if ((c0 == charArray[0]) && (c1 == charArray[1])
2893                                         && (c2 == charArray[2]) && (c3 == charArray[3])
2894                                         && (c4 == charArray[4]) && (c5 == charArray[5]))
2895                                 return charArray;
2896                 }
2897                 // ---------other side---------
2898                 i = -1;
2899                 int max = newEntry6;
2900                 while (++i <= max) {
2901                         char[] charArray = table[i];
2902                         if ((c0 == charArray[0]) && (c1 == charArray[1])
2903                                         && (c2 == charArray[2]) && (c3 == charArray[3])
2904                                         && (c4 == charArray[4]) && (c5 == charArray[5]))
2905                                 return charArray;
2906                 }
2907                 // --------add the entry-------
2908                 if (++max >= InternalTableSize)
2909                         max = 0;
2910                 char[] r;
2911                 table[max] = (r = new char[] { c0, c1, c2, c3, c4, c5 });
2912                 newEntry6 = max;
2913                 return r;
2914         }
2915
2916         public final void pushLineSeparator() throws InvalidInputException {
2917                 // see comment on isLineDelimiter(char) for the use of '\n' and '\r'
2918                 final int INCREMENT = 250;
2919                 if (this.checkNonExternalizedStringLiterals) {
2920                         // reinitialize the current line for non externalize strings purpose
2921                         currentLine = null;
2922                 }
2923                 // currentCharacter is at position currentPosition-1
2924                 // cr 000D
2925                 if (currentCharacter == '\r') {
2926                         int separatorPos = currentPosition - 1;
2927                         if ((linePtr > 0) && (lineEnds[linePtr] >= separatorPos))
2928                                 return;
2929                         // System.out.println("CR-" + separatorPos);
2930                         try {
2931                                 lineEnds[++linePtr] = separatorPos;
2932                         } catch (IndexOutOfBoundsException e) {
2933                                 // linePtr value is correct
2934                                 int oldLength = lineEnds.length;
2935                                 int[] old = lineEnds;
2936                                 lineEnds = new int[oldLength + INCREMENT];
2937                                 System.arraycopy(old, 0, lineEnds, 0, oldLength);
2938                                 lineEnds[linePtr] = separatorPos;
2939                         }
2940                         // look-ahead for merged cr+lf
2941                         try {
2942                                 if (source[currentPosition] == '\n') {
2943                                         // System.out.println("look-ahead LF-" + currentPosition);
2944                                         lineEnds[linePtr] = currentPosition;
2945                                         currentPosition++;
2946                                         wasAcr = false;
2947                                 } else {
2948                                         wasAcr = true;
2949                                 }
2950                         } catch (IndexOutOfBoundsException e) {
2951                                 wasAcr = true;
2952                         }
2953                 } else {
2954                         // lf 000A
2955                         if (currentCharacter == '\n') {
2956                                 // must merge eventual cr followed by lf
2957                                 if (wasAcr && (lineEnds[linePtr] == (currentPosition - 2))) {
2958                                         // System.out.println("merge LF-" + (currentPosition - 1));
2959                                         lineEnds[linePtr] = currentPosition - 1;
2960                                 } else {
2961                                         int separatorPos = currentPosition - 1;
2962                                         if ((linePtr > 0) && (lineEnds[linePtr] >= separatorPos))
2963                                                 return;
2964                                         // System.out.println("LF-" + separatorPos);
2965                                         try {
2966                                                 lineEnds[++linePtr] = separatorPos;
2967                                         } catch (IndexOutOfBoundsException e) {
2968                                                 // linePtr value is correct
2969                                                 int oldLength = lineEnds.length;
2970                                                 int[] old = lineEnds;
2971                                                 lineEnds = new int[oldLength + INCREMENT];
2972                                                 System.arraycopy(old, 0, lineEnds, 0, oldLength);
2973                                                 lineEnds[linePtr] = separatorPos;
2974                                         }
2975                                 }
2976                                 wasAcr = false;
2977                         }
2978                 }
2979         }
2980
2981         public final void pushUnicodeLineSeparator() {
2982                 // isUnicode means that the \r or \n has been read as a unicode
2983                 // character
2984                 // see comment on isLineDelimiter(char) for the use of '\n' and '\r'
2985                 final int INCREMENT = 250;
2986                 // currentCharacter is at position currentPosition-1
2987                 if (this.checkNonExternalizedStringLiterals) {
2988                         // reinitialize the current line for non externalize strings purpose
2989                         currentLine = null;
2990                 }
2991                 // cr 000D
2992                 if (currentCharacter == '\r') {
2993                         int separatorPos = currentPosition - 6;
2994                         if ((linePtr > 0) && (lineEnds[linePtr] >= separatorPos))
2995                                 return;
2996                         // System.out.println("CR-" + separatorPos);
2997                         try {
2998                                 lineEnds[++linePtr] = separatorPos;
2999                         } catch (IndexOutOfBoundsException e) {
3000                                 // linePtr value is correct
3001                                 int oldLength = lineEnds.length;
3002                                 int[] old = lineEnds;
3003                                 lineEnds = new int[oldLength + INCREMENT];
3004                                 System.arraycopy(old, 0, lineEnds, 0, oldLength);
3005                                 lineEnds[linePtr] = separatorPos;
3006                         }
3007                         // look-ahead for merged cr+lf
3008                         if (source[currentPosition] == '\n') {
3009                                 // System.out.println("look-ahead LF-" + currentPosition);
3010                                 lineEnds[linePtr] = currentPosition;
3011                                 currentPosition++;
3012                                 wasAcr = false;
3013                         } else {
3014                                 wasAcr = true;
3015                         }
3016                 } else {
3017                         // lf 000A
3018                         if (currentCharacter == '\n') {
3019                                 // must merge eventual cr followed by lf
3020                                 if (wasAcr && (lineEnds[linePtr] == (currentPosition - 7))) {
3021                                         // System.out.println("merge LF-" + (currentPosition - 1));
3022                                         lineEnds[linePtr] = currentPosition - 6;
3023                                 } else {
3024                                         int separatorPos = currentPosition - 6;
3025                                         if ((linePtr > 0) && (lineEnds[linePtr] >= separatorPos))
3026                                                 return;
3027                                         // System.out.println("LF-" + separatorPos);
3028                                         try {
3029                                                 lineEnds[++linePtr] = separatorPos;
3030                                         } catch (IndexOutOfBoundsException e) {
3031                                                 // linePtr value is correct
3032                                                 int oldLength = lineEnds.length;
3033                                                 int[] old = lineEnds;
3034                                                 lineEnds = new int[oldLength + INCREMENT];
3035                                                 System.arraycopy(old, 0, lineEnds, 0, oldLength);
3036                                                 lineEnds[linePtr] = separatorPos;
3037                                         }
3038                                 }
3039                                 wasAcr = false;
3040                         }
3041                 }
3042         }
3043
3044         public void recordComment(TokenName token) {
3045                 // compute position
3046                 int stopPosition = this.currentPosition;
3047                 switch (token) {
3048                 case COMMENT_LINE:
3049                         stopPosition = -this.lastCommentLinePosition;
3050                         break;
3051                 case COMMENT_BLOCK:
3052                         stopPosition = -this.currentPosition;
3053                         break;
3054                 }
3055
3056                 // a new comment is recorded
3057                 int length = this.commentStops.length;
3058                 if (++this.commentPtr >= length) {
3059                         System.arraycopy(this.commentStops, 0,
3060                                         this.commentStops = new int[length + 30], 0, length);
3061                         // grows the positions buffers too
3062                         System.arraycopy(this.commentStarts, 0,
3063                                         this.commentStarts = new int[length + 30], 0, length);
3064                 }
3065                 this.commentStops[this.commentPtr] = stopPosition;
3066                 this.commentStarts[this.commentPtr] = this.startPosition;
3067         }
3068
3069         // public final void recordComment(boolean isJavadoc) {
3070         // // a new annotation comment is recorded
3071         // try {
3072         // commentStops[++commentPtr] = isJavadoc
3073         // ? currentPosition
3074         // : -currentPosition;
3075         // } catch (IndexOutOfBoundsException e) {
3076         // int oldStackLength = commentStops.length;
3077         // int[] oldStack = commentStops;
3078         // commentStops = new int[oldStackLength + 30];
3079         // System.arraycopy(oldStack, 0, commentStops, 0, oldStackLength);
3080         // commentStops[commentPtr] = isJavadoc ? currentPosition :
3081         // -currentPosition;
3082         // //grows the positions buffers too
3083         // int[] old = commentStarts;
3084         // commentStarts = new int[oldStackLength + 30];
3085         // System.arraycopy(old, 0, commentStarts, 0, oldStackLength);
3086         // }
3087         // //the buffer is of a correct size here
3088         // commentStarts[commentPtr] = startPosition;
3089         // }
3090         public void resetTo(int begin, int end) {
3091                 // reset the scanner to a given position where it may rescan again
3092                 diet = false;
3093                 initialPosition = startPosition = currentPosition = begin;
3094                 eofPosition = end < Integer.MAX_VALUE ? end + 1 : end;
3095                 commentPtr = -1; // reset comment stack
3096         }
3097
3098         public final void scanSingleQuotedEscapeCharacter()
3099                         throws InvalidInputException {
3100                 // the string with "\\u" is a legal string of two chars \ and u
3101                 // thus we use a direct access to the source (for regular cases).
3102                 // if (unicodeAsBackSlash) {
3103                 // // consume next character
3104                 // unicodeAsBackSlash = false;
3105                 // if (((currentCharacter = source[currentPosition++]) == '\\')
3106                 // && (source[currentPosition] == 'u')) {
3107                 // getNextUnicodeChar();
3108                 // } else {
3109                 // if (withoutUnicodePtr != 0) {
3110                 // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
3111                 // }
3112                 // }
3113                 // } else
3114                 currentCharacter = source[currentPosition++];
3115                 switch (currentCharacter) {
3116                 case '\'':
3117                         currentCharacter = '\'';
3118                         break;
3119                 case '\\':
3120                         currentCharacter = '\\';
3121                         break;
3122                 default:
3123                         currentCharacter = '\\';
3124                         currentPosition--;
3125                 }
3126         }
3127
3128         public final void scanDoubleQuotedEscapeCharacter()
3129                         throws InvalidInputException {
3130                 currentCharacter = source[currentPosition++];
3131                 switch (currentCharacter) {
3132                 // case 'b' :
3133                 // currentCharacter = '\b';
3134                 // break;
3135                 case 't':
3136                         currentCharacter = '\t';
3137                         break;
3138                 case 'n':
3139                         currentCharacter = '\n';
3140                         break;
3141                 // case 'f' :
3142                 // currentCharacter = '\f';
3143                 // break;
3144                 case 'r':
3145                         currentCharacter = '\r';
3146                         break;
3147                 case '\"':
3148                         currentCharacter = '\"';
3149                         break;
3150                 case '\'':
3151                         currentCharacter = '\'';
3152                         break;
3153                 case '\\':
3154                         currentCharacter = '\\';
3155                         break;
3156                 case '$':
3157                         currentCharacter = '$';
3158                         break;
3159                 default:
3160                         // -----------octal escape--------------
3161                         // OctalDigit
3162                         // OctalDigit OctalDigit
3163                         // ZeroToThree OctalDigit OctalDigit
3164                         int number = Character.getNumericValue(currentCharacter);
3165                         if (number >= 0 && number <= 7) {
3166                                 boolean zeroToThreeNot = number > 3;
3167                                 if (Character
3168                                                 .isDigit(currentCharacter = source[currentPosition++])) {
3169                                         int digit = Character.getNumericValue(currentCharacter);
3170                                         if (digit >= 0 && digit <= 7) {
3171                                                 number = (number * 8) + digit;
3172                                                 if (Character
3173                                                                 .isDigit(currentCharacter = source[currentPosition++])) {
3174                                                         if (zeroToThreeNot) { // has read \NotZeroToThree
3175                                                                                                         // OctalDigit
3176                                                                 // Digit --> ignore last character
3177                                                                 currentPosition--;
3178                                                         } else {
3179                                                                 digit = Character
3180                                                                                 .getNumericValue(currentCharacter);
3181                                                                 if (digit >= 0 && digit <= 7) {
3182                                                                         // has read \ZeroToThree OctalDigit
3183                                                                         // OctalDigit
3184                                                                         number = (number * 8) + digit;
3185                                                                 } else { // has read \ZeroToThree OctalDigit
3186                                                                                         // NonOctalDigit
3187                                                                         // --> ignore last character
3188                                                                         currentPosition--;
3189                                                                 }
3190                                                         }
3191                                                 } else { // has read \OctalDigit NonDigit--> ignore
3192                                                                         // last
3193                                                         // character
3194                                                         currentPosition--;
3195                                                 }
3196                                         } else { // has read \OctalDigit NonOctalDigit--> ignore
3197                                                                 // last
3198                                                 // character
3199                                                 currentPosition--;
3200                                         }
3201                                 } else { // has read \OctalDigit --> ignore last character
3202                                         currentPosition--;
3203                                 }
3204                                 if (number > 255)
3205                                         throw new InvalidInputException(INVALID_ESCAPE);
3206                                 currentCharacter = (char) number;
3207                         }
3208                         // else
3209                         // throw new InvalidInputException(INVALID_ESCAPE);
3210                 }
3211         }
3212
3213         // public int scanIdentifierOrKeyword() throws InvalidInputException {
3214         // return scanIdentifierOrKeyword( false );
3215         // }
3216         public TokenName scanIdentifierOrKeyword(boolean isVariable)
3217                         throws InvalidInputException {
3218                 // test keywords
3219                 // first dispatch on the first char.
3220                 // then the length. If there are several
3221                 // keywords with the same length AND the same first char, then do another
3222                 // disptach on the second char :-)...cool....but fast !
3223                 useAssertAsAnIndentifier = false;
3224                 while (getNextCharAsJavaIdentifierPart()) {
3225                 }
3226                 ;
3227                 if (isVariable) {
3228                         // if (new String(getCurrentTokenSource()).equals("$this")) {
3229                         // return TokenName.this;
3230                         // }
3231                         return TokenName.VARIABLE;
3232                 }
3233                 int index, length;
3234                 char[] data;
3235                 char firstLetter;
3236                 // if (withoutUnicodePtr == 0)
3237                 // quick test on length == 1 but not on length > 12 while most
3238                 // identifier
3239                 // have a length which is <= 12...but there are lots of identifier with
3240                 // only one char....
3241                 // {
3242                 if ((length = currentPosition - startPosition) == 1)
3243                         return TokenName.IDENTIFIER;
3244                 // data = source;
3245                 data = new char[length];
3246                 index = startPosition;
3247                 for (int i = 0; i < length; i++) {
3248                         data[i] = Character.toLowerCase(source[index + i]);
3249                 }
3250                 index = 0;
3251                 // } else {
3252                 // if ((length = withoutUnicodePtr) == 1)
3253                 // return TokenName.Identifier;
3254                 // // data = withoutUnicodeBuffer;
3255                 // data = new char[withoutUnicodeBuffer.length];
3256                 // for (int i = 0; i < withoutUnicodeBuffer.length; i++) {
3257                 // data[i] = Character.toLowerCase(withoutUnicodeBuffer[i]);
3258                 // }
3259                 // index = 1;
3260                 // }
3261                 firstLetter = data[index];
3262                 switch (firstLetter) {
3263                 case '_':
3264                         switch (length) {
3265                         case 8:
3266                                 // __FILE__
3267                                 if ((data[++index] == '_') && (data[++index] == 'f')
3268                                                 && (data[++index] == 'i') && (data[++index] == 'l')
3269                                                 && (data[++index] == 'e') && (data[++index] == '_')
3270                                                 && (data[++index] == '_'))
3271                                         return TokenName.FILE;
3272                                 index = 0; // __LINE__
3273                                 if ((data[++index] == '_') && (data[++index] == 'l')
3274                                                 && (data[++index] == 'i') && (data[++index] == 'n')
3275                                                 && (data[++index] == 'e') && (data[++index] == '_')
3276                                                 && (data[++index] == '_'))
3277                                         return TokenName.LINE;
3278                                 break;
3279                         case 9:
3280                                 // __CLASS__
3281                                 if ((data[++index] == '_') && (data[++index] == 'c')
3282                                                 && (data[++index] == 'l') && (data[++index] == 'a')
3283                                                 && (data[++index] == 's') && (data[++index] == 's')
3284                                                 && (data[++index] == '_') && (data[++index] == '_'))
3285                                         return TokenName.CLASS_C;
3286                                 break;
3287                         case 11:
3288                                 // __METHOD__
3289                                 if ((data[++index] == '_') && (data[++index] == 'm')
3290                                                 && (data[++index] == 'e') && (data[++index] == 't')
3291                                                 && (data[++index] == 'h') && (data[++index] == 'o')
3292                                                 && (data[++index] == 'd') && (data[++index] == '_')
3293                                                 && (data[++index] == '_'))
3294                                         return TokenName.METHOD_C;
3295                                 break;
3296                         case 12:
3297                                 // __FUNCTION__
3298                                 if ((data[++index] == '_') && (data[++index] == 'f')
3299                                                 && (data[++index] == 'u') && (data[++index] == 'n')
3300                                                 && (data[++index] == 'c') && (data[++index] == 't')
3301                                                 && (data[++index] == 'i') && (data[++index] == 'o')
3302                                                 && (data[++index] == 'n') && (data[++index] == '_')
3303                                                 && (data[++index] == '_'))
3304                                         return TokenName.FUNC_C;
3305                                 break;
3306                         }
3307                         return TokenName.IDENTIFIER;
3308                 case 'a':
3309                         // as and array abstract
3310                         switch (length) {
3311                         case 2:
3312                                 // as
3313                                 if ((data[++index] == 's')) {
3314                                         return TokenName.AS;
3315                                 }
3316                                 return TokenName.IDENTIFIER;
3317                         case 3:
3318                                 // and
3319                                 if ((data[++index] == 'n') && (data[++index] == 'd')) {
3320                                         return TokenName.OP_AND_OLD;
3321                                 }
3322                                 return TokenName.IDENTIFIER;
3323                         case 5:
3324                                 // array
3325                                 if ((data[++index] == 'r') && (data[++index] == 'r')
3326                                                 && (data[++index] == 'a') && (data[++index] == 'y'))
3327                                         return TokenName.ARRAY;
3328                                 return TokenName.IDENTIFIER;
3329                         case 8:
3330                                 if ((data[++index] == 'b') && (data[++index] == 's')
3331                                                 && (data[++index] == 't') && (data[++index] == 'r')
3332                                                 && (data[++index] == 'a') && (data[++index] == 'c')
3333                                                 && (data[++index] == 't'))
3334                                         return TokenName.ABSTRACT;
3335                                 return TokenName.IDENTIFIER;
3336                         }
3337                         return TokenName.IDENTIFIER;
3338                 case 'b':
3339                         // break
3340                         switch (length) {
3341                         case 5:
3342                                 if ((data[++index] == 'r') && (data[++index] == 'e')
3343                                                 && (data[++index] == 'a') && (data[++index] == 'k'))
3344                                         return TokenName.BREAK;
3345                                 return TokenName.IDENTIFIER;
3346                         }
3347                         return TokenName.IDENTIFIER;
3348                 case 'c':
3349                         // case catch class clone const continue
3350                         switch (length) {
3351                         case 4:
3352                                 if ((data[++index] == 'a') && (data[++index] == 's')
3353                                                 && (data[++index] == 'e'))
3354                                         return TokenName.CASE;
3355                                 return TokenName.IDENTIFIER;
3356                         case 5:
3357                                 if ((data[++index] == 'a') && (data[++index] == 't')
3358                                                 && (data[++index] == 'c') && (data[++index] == 'h'))
3359                                         return TokenName.CATCH;
3360                                 index = 0;
3361                                 if ((data[++index] == 'l') && (data[++index] == 'a')
3362                                                 && (data[++index] == 's') && (data[++index] == 's'))
3363                                         return TokenName.CLASS;
3364                                 index = 0;
3365                                 if ((data[++index] == 'l') && (data[++index] == 'o')
3366                                                 && (data[++index] == 'n') && (data[++index] == 'e'))
3367                                         return TokenName.CLONE;
3368                                 index = 0;
3369                                 if ((data[++index] == 'o') && (data[++index] == 'n')
3370                                                 && (data[++index] == 's') && (data[++index] == 't'))
3371                                         return TokenName.CONST;
3372                                 return TokenName.IDENTIFIER;
3373                         case 8:
3374                                 if ((data[++index] == 'o') && (data[++index] == 'n')
3375                                                 && (data[++index] == 't') && (data[++index] == 'i')
3376                                                 && (data[++index] == 'n') && (data[++index] == 'u')
3377                                                 && (data[++index] == 'e'))
3378                                         return TokenName.CONTINUE;
3379                                 return TokenName.IDENTIFIER;
3380                         }
3381                         return TokenName.IDENTIFIER;
3382                 case 'd':
3383                         // declare default do die
3384                         // TODO delete define ==> no keyword !
3385                         switch (length) {
3386                         case 2:
3387                                 if ((data[++index] == 'o'))
3388                                         return TokenName.DO;
3389                                 return TokenName.IDENTIFIER;
3390                                 // case 6 :
3391                                 // if ((data[++index] == 'e')
3392                                 // && (data[++index] == 'f')
3393                                 // && (data[++index] == 'i')
3394                                 // && (data[++index] == 'n')
3395                                 // && (data[++index] == 'e'))
3396                                 // return TokenName.define;
3397                                 // else
3398                                 // return TokenName.Identifier;
3399                         case 7:
3400                                 if ((data[++index] == 'e') && (data[++index] == 'c')
3401                                                 && (data[++index] == 'l') && (data[++index] == 'a')
3402                                                 && (data[++index] == 'r') && (data[++index] == 'e'))
3403                                         return TokenName.DECLARE;
3404                                 index = 0;
3405                                 if ((data[++index] == 'e') && (data[++index] == 'f')
3406                                                 && (data[++index] == 'a') && (data[++index] == 'u')
3407                                                 && (data[++index] == 'l') && (data[++index] == 't'))
3408                                         return TokenName.DEFAULT;
3409                                 return TokenName.IDENTIFIER;
3410                         }
3411                         return TokenName.IDENTIFIER;
3412                 case 'e':
3413                         // echo else exit elseif extends eval
3414                         switch (length) {
3415                         case 4:
3416                                 if ((data[++index] == 'c') && (data[++index] == 'h')
3417                                                 && (data[++index] == 'o'))
3418                                         return TokenName.ECHO;
3419                                 else if ((data[index] == 'l') && (data[++index] == 's')
3420                                                 && (data[++index] == 'e'))
3421                                         return TokenName.ELSE;
3422                                 else if ((data[index] == 'x') && (data[++index] == 'i')
3423                                                 && (data[++index] == 't'))
3424                                         return TokenName.EXIT;
3425                                 else if ((data[index] == 'v') && (data[++index] == 'a')
3426                                                 && (data[++index] == 'l'))
3427                                         return TokenName.EVAL;
3428                                 return TokenName.IDENTIFIER;
3429                         case 5:
3430                                 // endif empty
3431                                 if ((data[++index] == 'n') && (data[++index] == 'd')
3432                                                 && (data[++index] == 'i') && (data[++index] == 'f'))
3433                                         return TokenName.ENDIF;
3434                                 if ((data[index] == 'm') && (data[++index] == 'p')
3435                                                 && (data[++index] == 't') && (data[++index] == 'y'))
3436                                         return TokenName.EMPTY;
3437                                 return TokenName.IDENTIFIER;
3438                         case 6:
3439                                 // endfor
3440                                 if ((data[++index] == 'n') && (data[++index] == 'd')
3441                                                 && (data[++index] == 'f') && (data[++index] == 'o')
3442                                                 && (data[++index] == 'r'))
3443                                         return TokenName.ENDFOR;
3444                                 else if ((data[index] == 'l') && (data[++index] == 's')
3445                                                 && (data[++index] == 'e') && (data[++index] == 'i')
3446                                                 && (data[++index] == 'f'))
3447                                         return TokenName.ELSEIF;
3448                                 return TokenName.IDENTIFIER;
3449                         case 7:
3450                                 if ((data[++index] == 'x') && (data[++index] == 't')
3451                                                 && (data[++index] == 'e') && (data[++index] == 'n')
3452                                                 && (data[++index] == 'd') && (data[++index] == 's'))
3453                                         return TokenName.EXTENDS;
3454                                 return TokenName.IDENTIFIER;
3455                         case 8:
3456                                 // endwhile
3457                                 if ((data[++index] == 'n') && (data[++index] == 'd')
3458                                                 && (data[++index] == 'w') && (data[++index] == 'h')
3459                                                 && (data[++index] == 'i') && (data[++index] == 'l')
3460                                                 && (data[++index] == 'e'))
3461                                         return TokenName.ENDWHILE;
3462                                 return TokenName.IDENTIFIER;
3463                         case 9:
3464                                 // endswitch
3465                                 if ((data[++index] == 'n') && (data[++index] == 'd')
3466                                                 && (data[++index] == 's') && (data[++index] == 'w')
3467                                                 && (data[++index] == 'i') && (data[++index] == 't')
3468                                                 && (data[++index] == 'c') && (data[++index] == 'h'))
3469                                         return TokenName.ENDSWITCH;
3470                                 return TokenName.IDENTIFIER;
3471                         case 10:
3472                                 // enddeclare
3473                                 if ((data[++index] == 'n') && (data[++index] == 'd')
3474                                                 && (data[++index] == 'd') && (data[++index] == 'e')
3475                                                 && (data[++index] == 'c') && (data[++index] == 'l')
3476                                                 && (data[++index] == 'a') && (data[++index] == 'r')
3477                                                 && (data[++index] == 'e'))
3478                                         return TokenName.ENDDECLARE;
3479                                 index = 0;
3480                                 if ((data[++index] == 'n') // endforeach
3481                                                 && (data[++index] == 'd')
3482                                                 && (data[++index] == 'f')
3483                                                 && (data[++index] == 'o')
3484                                                 && (data[++index] == 'r')
3485                                                 && (data[++index] == 'e')
3486                                                 && (data[++index] == 'a')
3487                                                 && (data[++index] == 'c') && (data[++index] == 'h'))
3488                                         return TokenName.ENDFOREACH;
3489                                 return TokenName.IDENTIFIER;
3490                         }
3491                         return TokenName.IDENTIFIER;
3492                 case 'f':
3493                         // for false final function
3494                         switch (length) {
3495                         case 3:
3496                                 if ((data[++index] == 'o') && (data[++index] == 'r'))
3497                                         return TokenName.FOR;
3498                                 return TokenName.IDENTIFIER;
3499                         case 5:
3500                                 // if ((data[++index] == 'a') && (data[++index] == 'l')
3501                                 // && (data[++index] == 's') && (data[++index] == 'e'))
3502                                 // return TokenName.false;
3503                                 if ((data[++index] == 'i') && (data[++index] == 'n')
3504                                                 && (data[++index] == 'a') && (data[++index] == 'l'))
3505                                         return TokenName.FINAL;
3506                                 return TokenName.IDENTIFIER;
3507                         case 7:
3508                                 // foreach
3509                                 if ((data[++index] == 'o') && (data[++index] == 'r')
3510                                                 && (data[++index] == 'e') && (data[++index] == 'a')
3511                                                 && (data[++index] == 'c') && (data[++index] == 'h'))
3512                                         return TokenName.FOREACH;
3513                                 return TokenName.IDENTIFIER;
3514                         case 8:
3515                                 // function
3516                                 if ((data[++index] == 'u') && (data[++index] == 'n')
3517                                                 && (data[++index] == 'c') && (data[++index] == 't')
3518                                                 && (data[++index] == 'i') && (data[++index] == 'o')
3519                                                 && (data[++index] == 'n'))
3520                                         return TokenName.FUNCTION;
3521                                 return TokenName.IDENTIFIER;
3522                         }
3523                         return TokenName.IDENTIFIER;
3524                 case 'g':
3525                         // global
3526                         if (length == 6) {
3527                                 if ((data[++index] == 'l') && (data[++index] == 'o')
3528                                                 && (data[++index] == 'b') && (data[++index] == 'a')
3529                                                 && (data[++index] == 'l')) {
3530                                         return TokenName.GLOBAL;
3531                                 }
3532                         }
3533                         else if (length == 4) {  // goto
3534                 if ((data[++index] == 'o') &&
3535                     (data[++index] == 't') &&
3536                     (data[++index] == 'o')) {
3537                     return TokenName.GOTO;
3538                 }
3539                         }
3540                         return TokenName.IDENTIFIER;
3541                 case 'i':
3542                         // if int isset include include_once instanceof interface implements
3543                         switch (length) {
3544                         case 2:
3545                                 if (data[++index] == 'f')
3546                                         return TokenName.IF;
3547                                 return TokenName.IDENTIFIER;
3548                                 // case 3 :
3549                                 // if ((data[++index] == 'n') && (data[++index] == 't'))
3550                                 // return TokenName.int;
3551                                 // else
3552                                 // return TokenName.IDENTIFIER;
3553                         case 5:
3554                                 if ((data[++index] == 's') && (data[++index] == 's')
3555                                                 && (data[++index] == 'e') && (data[++index] == 't'))
3556                                         return TokenName.ISSET;
3557                                 return TokenName.IDENTIFIER;
3558                         case 7:
3559                                 if ((data[++index] == 'n') && (data[++index] == 'c')
3560                                                 && (data[++index] == 'l') && (data[++index] == 'u')
3561                                                 && (data[++index] == 'd') && (data[++index] == 'e'))
3562                                         return TokenName.INCLUDE;
3563                                 return TokenName.IDENTIFIER;
3564                         case 9:
3565                                 // interface
3566                                 if ((data[++index] == 'n') && (data[++index] == 't')
3567                                                 && (data[++index] == 'e') && (data[++index] == 'r')
3568                                                 && (data[++index] == 'f') && (data[++index] == 'a')
3569                                                 && (data[++index] == 'c') && (data[++index] == 'e'))
3570                                         return TokenName.INTERFACE;
3571                                 return TokenName.IDENTIFIER;
3572                         case 10:
3573                                 // instanceof implements
3574                                 if ((data[++index] == 'n') && (data[++index] == 's')
3575                                                 && (data[++index] == 't') && (data[++index] == 'a')
3576                                                 && (data[++index] == 'n') && (data[++index] == 'c')
3577                                                 && (data[++index] == 'e') && (data[++index] == 'o')
3578                                                 && (data[++index] == 'f'))
3579                                         return TokenName.INSTANCEOF;
3580                                 if ((data[index] == 'm') && (data[++index] == 'p')
3581                                                 && (data[++index] == 'l') && (data[++index] == 'e')
3582                                                 && (data[++index] == 'm') && (data[++index] == 'e')
3583                                                 && (data[++index] == 'n') && (data[++index] == 't')
3584                                                 && (data[++index] == 's'))
3585                                         return TokenName.IMPLEMENTS;
3586                                 return TokenName.IDENTIFIER;
3587                         case 12: // include_once
3588                                 if ((data[++index] == 'n') && (data[++index] == 'c')
3589                                                 && (data[++index] == 'l') && (data[++index] == 'u')
3590                                                 && (data[++index] == 'd') && (data[++index] == 'e')
3591                                                 && (data[++index] == '_') && (data[++index] == 'o')
3592                                                 && (data[++index] == 'n') && (data[++index] == 'c')
3593                                                 && (data[++index] == 'e'))
3594                                         return TokenName.INCLUDE_ONCE;
3595                                 return TokenName.IDENTIFIER;
3596                         }
3597                         return TokenName.IDENTIFIER;
3598                 case 'l':
3599                         // list
3600                         if (length == 4) {
3601                                 if ((data[++index] == 'i') && (data[++index] == 's')
3602                                                 && (data[++index] == 't')) {
3603                                         return TokenName.LIST;
3604                                 }
3605                         }
3606                         return TokenName.IDENTIFIER;
3607                 case 'n':
3608                         // new null namespace
3609                         switch (length) {
3610                         case 3:
3611                                 if ((data[++index] == 'e') && (data[++index] == 'w'))
3612                                         return TokenName.NEW;
3613                                 return TokenName.IDENTIFIER;
3614                                 // case 4 :
3615                                 // if ((data[++index] == 'u') && (data[++index] == 'l')
3616                                 // && (data[++index] == 'l'))
3617                                 // return TokenName.null;
3618                                 // else
3619                                 // return TokenName.IDENTIFIER;
3620                         case 9:
3621                 if ((data[++index] == 'a') && (data[++index] == 'm')
3622                         && (data[++index] == 'e') && (data[++index] == 's')
3623                         && (data[++index] == 'p') && (data[++index] == 'a')
3624                         && (data[++index] == 'c') && (data[++index] == 'e')) {
3625                     return TokenName.NAMESPACE;
3626                 }
3627                             return TokenName.IDENTIFIER;
3628                         }
3629                         return TokenName.IDENTIFIER;
3630                 case 'o':
3631                         // or old_function
3632                         if (length == 2) {
3633                                 if (data[++index] == 'r') {
3634                                         return TokenName.OP_OR_OLD;
3635                                 }
3636                         }
3637                         // if (length == 12) {
3638                         // if ((data[++index] == 'l')
3639                         // && (data[++index] == 'd')
3640                         // && (data[++index] == '_')
3641                         // && (data[++index] == 'f')
3642                         // && (data[++index] == 'u')
3643                         // && (data[++index] == 'n')
3644                         // && (data[++index] == 'c')
3645                         // && (data[++index] == 't')
3646                         // && (data[++index] == 'i')
3647                         // && (data[++index] == 'o')
3648                         // && (data[++index] == 'n')) {
3649                         // return TokenName.old_function;
3650                         // }
3651                         // }
3652                         return TokenName.IDENTIFIER;
3653                 case 'p':
3654                         // print public private protected
3655                         switch (length) {
3656                         case 5:
3657                                 if ((data[++index] == 'r') && (data[++index] == 'i')
3658                                                 && (data[++index] == 'n') && (data[++index] == 't')) {
3659                                         return TokenName.PRINT;
3660                                 }
3661                                 return TokenName.IDENTIFIER;
3662                         case 6:
3663                                 if ((data[++index] == 'u') && (data[++index] == 'b')
3664                                                 && (data[++index] == 'l') && (data[++index] == 'i')
3665                                                 && (data[++index] == 'c')) {
3666                                         return TokenName.PUBLIC;
3667                                 }
3668                                 return TokenName.IDENTIFIER;
3669                         case 7:
3670                                 if ((data[++index] == 'r') && (data[++index] == 'i')
3671                                                 && (data[++index] == 'v') && (data[++index] == 'a')
3672                                                 && (data[++index] == 't') && (data[++index] == 'e')) {
3673                                         return TokenName.PRIVATE;
3674                                 }
3675                                 return TokenName.IDENTIFIER;
3676                         case 9:
3677                                 if ((data[++index] == 'r') && (data[++index] == 'o')
3678                                                 && (data[++index] == 't') && (data[++index] == 'e')
3679                                                 && (data[++index] == 'c') && (data[++index] == 't')
3680                                                 && (data[++index] == 'e') && (data[++index] == 'd')) {
3681                                         return TokenName.PROTECTED;
3682                                 }
3683                                 return TokenName.IDENTIFIER;
3684                         }
3685                         return TokenName.IDENTIFIER;
3686                 case 'r':
3687                         // return require require_once
3688                         if (length == 6) {
3689                                 if ((data[++index] == 'e') && (data[++index] == 't')
3690                                                 && (data[++index] == 'u') && (data[++index] == 'r')
3691                                                 && (data[++index] == 'n')) {
3692                                         return TokenName.RETURN;
3693                                 }
3694                         } else if (length == 7) {
3695                                 if ((data[++index] == 'e') && (data[++index] == 'q')
3696                                                 && (data[++index] == 'u') && (data[++index] == 'i')
3697                                                 && (data[++index] == 'r') && (data[++index] == 'e')) {
3698                                         return TokenName.REQUIRE;
3699                                 }
3700                         } else if (length == 12) {
3701                                 if ((data[++index] == 'e') && (data[++index] == 'q')
3702                                                 && (data[++index] == 'u') && (data[++index] == 'i')
3703                                                 && (data[++index] == 'r') && (data[++index] == 'e')
3704                                                 && (data[++index] == '_') && (data[++index] == 'o')
3705                                                 && (data[++index] == 'n') && (data[++index] == 'c')
3706                                                 && (data[++index] == 'e')) {
3707                                         return TokenName.REQUIRE_ONCE;
3708                                 }
3709                         }
3710                         return TokenName.IDENTIFIER;
3711                 case 's':
3712                         // self static switch
3713                         switch (length) {
3714                         // case 4:
3715                         // if ((data[++index] == 'e') && (data[++index] == 'l') &&
3716                         // (data[++index]
3717                         // == 'f')) {
3718                         // return TokenName.self;
3719                         // }
3720                         // return TokenName.IDENTIFIER;
3721                         case 6:
3722                                 if (data[++index] == 't')
3723                                         if ((data[++index] == 'a') && (data[++index] == 't')
3724                                                         && (data[++index] == 'i') && (data[++index] == 'c')) {
3725                                                 return TokenName.STATIC;
3726                                         } else
3727                                                 return TokenName.IDENTIFIER;
3728                                 else if ((data[index] == 'w') && (data[++index] == 'i')
3729                                                 && (data[++index] == 't') && (data[++index] == 'c')
3730                                                 && (data[++index] == 'h'))
3731                                         return TokenName.SWITCH;
3732                         }
3733                         return TokenName.IDENTIFIER;
3734                 case 't':
3735                         // try true throw
3736                         switch (length) {
3737                         case 3:
3738                                 if ((data[++index] == 'r') && (data[++index] == 'y'))
3739                                         return TokenName.TRY;
3740                                 // case 4 :
3741                                 // if ((data[++index] == 'r') && (data[++index] == 'u')
3742                                 // && (data[++index] == 'e'))
3743                                 // return TokenName.true;
3744                                  else
3745                                          return TokenName.IDENTIFIER;
3746                         case 5:
3747                                 if ((data[++index] == 'h') && (data[++index] == 'r')
3748                                                 && (data[++index] == 'o') && (data[++index] == 'w'))
3749                                         return TokenName.THROW;
3750                         }
3751                         return TokenName.IDENTIFIER;
3752                 case 'u':
3753                         // use unset
3754                         switch (length) {
3755                         case 3:
3756                                 if ((data[++index] == 's') && (data[++index] == 'e'))
3757                                         return TokenName.USE;
3758                                 else
3759                                         return TokenName.IDENTIFIER;
3760                         case 5:
3761                                 if ((data[++index] == 'n') && (data[++index] == 's')
3762                                                 && (data[++index] == 'e') && (data[++index] == 't'))
3763                                         return TokenName.UNSET;
3764                         }
3765                         return TokenName.IDENTIFIER;
3766                 case 'v':
3767                         // var
3768                         switch (length) {
3769                         case 3:
3770                                 if ((data[++index] == 'a') && (data[++index] == 'r'))
3771                                         return TokenName.VAR;
3772                         }
3773                         return TokenName.IDENTIFIER;
3774                 case 'w':
3775                         // while
3776                         switch (length) {
3777                         case 5:
3778                                 if ((data[++index] == 'h') && (data[++index] == 'i')
3779                                                 && (data[++index] == 'l') && (data[++index] == 'e'))
3780                                         return TokenName.WHILE;
3781                                 // case 6:if ( (data[++index] =='i') && (data[++index]=='d') &&
3782                                 // (data[++index]=='e') && (data[++index]=='f')&&
3783                                 // (data[++index]=='p'))
3784                                 // return TokenName.widefp ;
3785                                 // else
3786                                 // return TokenName.IDENTIFIER;
3787                         }
3788                         return TokenName.IDENTIFIER;
3789                 case 'x':
3790                         // xor
3791                         switch (length) {
3792                         case 3:
3793                                 if ((data[++index] == 'o') && (data[++index] == 'r'))
3794                                         return TokenName.OP_XOR_OLD;
3795                                 else
3796                                         return TokenName.IDENTIFIER;
3797                         }
3798                         return TokenName.IDENTIFIER;
3799                 }
3800                 return TokenName.IDENTIFIER;
3801         }
3802
3803         public TokenName scanNumber(boolean dotPrefix) throws InvalidInputException {
3804                 // when entering this method the currentCharacter is the firt
3805                 // digit of the number , i.e. it may be preceeded by a . when
3806                 // dotPrefix is true
3807                 boolean floating = dotPrefix;
3808                 if ((!dotPrefix) && (currentCharacter == '0')) {
3809                         if (getNextChar('x', 'X') >= 0) { // ----------hexa-----------------
3810                                 // force the first char of the hexa number do exist...
3811                                 // consume next character
3812                                 unicodeAsBackSlash = false;
3813                                 currentCharacter = source[currentPosition++];
3814                                 // if (((currentCharacter = source[currentPosition++]) == '\\')
3815                                 // && (source[currentPosition] == 'u')) {
3816                                 // getNextUnicodeChar();
3817                                 // } else {
3818                                 // if (withoutUnicodePtr != 0) {
3819                                 // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
3820                                 // }
3821                                 // }
3822                                 if (Character.digit(currentCharacter, 16) == -1)
3823                                         throw new InvalidInputException(INVALID_HEXA);
3824                                 // ---end forcing--
3825                                 while (getNextCharAsDigit(16)) {
3826                                 }
3827                                 ;
3828                                 // if (getNextChar('l', 'L') >= 0)
3829                                 // return TokenName.LongLiteral;
3830                                 // else
3831                                 return TokenName.INTEGERLITERAL;
3832                         }
3833                         // there is x or X in the number
3834                         // potential octal ! ... some one may write 000099.0 ! thus 00100 <
3835                         // 00078.0 is true !!!!! crazy language
3836                         if (getNextCharAsDigit()) {
3837                                 // -------------potential octal-----------------
3838                                 while (getNextCharAsDigit()) {
3839                                 }
3840                                 ;
3841                                 // if (getNextChar('l', 'L') >= 0) {
3842                                 // return TokenName.LongLiteral;
3843                                 // }
3844                                 //
3845                                 // if (getNextChar('f', 'F') >= 0) {
3846                                 // return TokenName.FloatingPointLiteral;
3847                                 // }
3848                                 if (getNextChar('d', 'D') >= 0) {
3849                                         return TokenName.DOUBLELITERAL;
3850                                 } else { // make the distinction between octal and float ....
3851                                         if (getNextChar('.')) { // bingo ! ....
3852                                                 while (getNextCharAsDigit()) {
3853                                                 }
3854                                                 ;
3855                                                 if (getNextChar('e', 'E') >= 0) {
3856                                                         // consume next character
3857                                                         unicodeAsBackSlash = false;
3858                                                         currentCharacter = source[currentPosition++];
3859                                                         // if (((currentCharacter =
3860                                                         // source[currentPosition++]) == '\\')
3861                                                         // && (source[currentPosition] == 'u')) {
3862                                                         // getNextUnicodeChar();
3863                                                         // } else {
3864                                                         // if (withoutUnicodePtr != 0) {
3865                                                         // withoutUnicodeBuffer[++withoutUnicodePtr] =
3866                                                         // currentCharacter;
3867                                                         // }
3868                                                         // }
3869                                                         if ((currentCharacter == '-')
3870                                                                         || (currentCharacter == '+')) {
3871                                                                 // consume next character
3872                                                                 unicodeAsBackSlash = false;
3873                                                                 currentCharacter = source[currentPosition++];
3874                                                                 // if (((currentCharacter =
3875                                                                 // source[currentPosition++]) == '\\')
3876                                                                 // && (source[currentPosition] == 'u')) {
3877                                                                 // getNextUnicodeChar();
3878                                                                 // } else {
3879                                                                 // if (withoutUnicodePtr != 0) {
3880                                                                 // withoutUnicodeBuffer[++withoutUnicodePtr] =
3881                                                                 // currentCharacter;
3882                                                                 // }
3883                                                                 // }
3884                                                         }
3885                                                         if (!Character.isDigit(currentCharacter))
3886                                                                 throw new InvalidInputException(INVALID_FLOAT);
3887                                                         while (getNextCharAsDigit()) {
3888                                                         }
3889                                                         ;
3890                                                 }
3891                                                 // if (getNextChar('f', 'F') >= 0)
3892                                                 // return TokenName.FloatingPointLiteral;
3893                                                 getNextChar('d', 'D'); // jump over potential d or D
3894                                                 return TokenName.DOUBLELITERAL;
3895                                         } else {
3896                                                 return TokenName.INTEGERLITERAL;
3897                                         }
3898                                 }
3899                         } else {
3900                                 /* carry on */
3901                         }
3902                 }
3903                 while (getNextCharAsDigit()) {
3904                 }
3905                 ;
3906                 // if ((!dotPrefix) && (getNextChar('l', 'L') >= 0))
3907                 // return TokenName.LongLiteral;
3908                 if ((!dotPrefix) && (getNextChar('.'))) { // decimal part that can be
3909                                                                                                         // empty
3910                         while (getNextCharAsDigit()) {
3911                         }
3912                         ;
3913                         floating = true;
3914                 }
3915                 // if floating is true both exponant and suffix may be optional
3916                 if (getNextChar('e', 'E') >= 0) {
3917                         floating = true;
3918                         // consume next character
3919                         unicodeAsBackSlash = false;
3920                         currentCharacter = source[currentPosition++];
3921                         // if (((currentCharacter = source[currentPosition++]) == '\\')
3922                         // && (source[currentPosition] == 'u')) {
3923                         // getNextUnicodeChar();
3924                         // } else {
3925                         // if (withoutUnicodePtr != 0) {
3926                         // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
3927                         // }
3928                         // }
3929                         if ((currentCharacter == '-') || (currentCharacter == '+')) { // consume
3930                                 // next
3931                                 // character
3932                                 unicodeAsBackSlash = false;
3933                                 currentCharacter = source[currentPosition++];
3934                                 // if (((currentCharacter = source[currentPosition++]) == '\\')
3935                                 // && (source[currentPosition] == 'u')) {
3936                                 // getNextUnicodeChar();
3937                                 // } else {
3938                                 // if (withoutUnicodePtr != 0) {
3939                                 // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
3940                                 // }
3941                                 // }
3942                         }
3943                         if (!Character.isDigit(currentCharacter))
3944                                 throw new InvalidInputException(INVALID_FLOAT);
3945                         while (getNextCharAsDigit()) {
3946                         }
3947                         ;
3948                 }
3949                 if (getNextChar('d', 'D') >= 0)
3950                         return TokenName.DOUBLELITERAL;
3951                 // if (getNextChar('f', 'F') >= 0)
3952                 // return TokenName.FloatingPointLiteral;
3953                 // the long flag has been tested before
3954                 return floating ? TokenName.DOUBLELITERAL : TokenName.INTEGERLITERAL;
3955         }
3956
3957         /**
3958          * Search the line number corresponding to a specific position
3959          *
3960          */
3961         public final int getLineNumber(int position) {
3962                 if (lineEnds == null)
3963                         return 1;
3964                 int length = linePtr + 1;
3965                 if (length == 0)
3966                         return 1;
3967                 int g = 0, d = length - 1;
3968                 int m = 0;
3969                 while (g <= d) {
3970                         m = (g + d) / 2;
3971                         if (position < lineEnds[m]) {
3972                                 d = m - 1;
3973                         } else if (position > lineEnds[m]) {
3974                                 g = m + 1;
3975                         } else {
3976                                 return m + 1;
3977                         }
3978                 }
3979                 if (position < lineEnds[m]) {
3980                         return m + 1;
3981                 }
3982                 return m + 2;
3983         }
3984
3985         public void setPHPMode(boolean mode) {
3986                 phpMode = mode;
3987         }
3988
3989         public final void setSource(char[] source) {
3990                 setSource(null, source);
3991         }
3992
3993         public final void setSource(ICompilationUnit compilationUnit, char[] source) {
3994                 // the source-buffer is set to sourceString
3995                 this.compilationUnit = compilationUnit;
3996                 if (source == null) {
3997                         this.source = new char[0];
3998                 } else {
3999                         this.source = source;
4000                 }
4001                 startPosition = -1;
4002                 initialPosition = currentPosition = 0;
4003                 containsAssertKeyword = false;
4004                 withoutUnicodeBuffer = new char[this.source.length];
4005                 fFillerToken = TokenName.EOF;
4006                 // encapsedStringStack = new Stack();
4007         }
4008
4009         public String toString() {
4010                 if (startPosition == source.length)
4011                         return "EOF\n\n" + new String(source); //$NON-NLS-1$
4012                 if (currentPosition > source.length)
4013                         return "behind the EOF :-( ....\n\n" + new String(source); //$NON-NLS-1$
4014                 char front[] = new char[startPosition];
4015                 System.arraycopy(source, 0, front, 0, startPosition);
4016                 int middleLength = (currentPosition - 1) - startPosition + 1;
4017                 char middle[];
4018                 if (middleLength > -1) {
4019                         middle = new char[middleLength];
4020                         System.arraycopy(source, startPosition, middle, 0, middleLength);
4021                 } else {
4022                         middle = new char[0];
4023                 }
4024                 char end[] = new char[source.length - (currentPosition - 1)];
4025                 System.arraycopy(source, (currentPosition - 1) + 1, end, 0,
4026                                 source.length - (currentPosition - 1) - 1);
4027                 return new String(front)
4028                                 + "\n===============================\nStarts here -->" //$NON-NLS-1$
4029                                 + new String(middle)
4030                                 + "<-- Ends here\n===============================\n" //$NON-NLS-1$
4031                                 + new String(end);
4032         }
4033
4034         public final String toStringAction(TokenName act) {
4035                 switch (act) {
4036                 case ERROR:
4037                         return "ScannerError"; // + new String(getCurrentTokenSource()) +
4038                                                                         // ")";
4039                         // //$NON-NLS-1$
4040                 case INLINE_HTML:
4041                         return "Inline-HTML(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4042                 case ECHO_INVISIBLE:
4043                         // 0-length token
4044                         return "";
4045                 case IDENTIFIER:
4046                         return "Identifier(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4047                 case VARIABLE:
4048                         return "Variable(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4049                 case ABSTRACT:
4050                         return "abstract"; //$NON-NLS-1$
4051                 case OP_AND_OLD:
4052                         return "AND"; //$NON-NLS-1$
4053                 case ARRAY:
4054                         return "array"; //$NON-NLS-1$
4055                 case AS:
4056                         return "as"; //$NON-NLS-1$
4057                 case BREAK:
4058                         return "break"; //$NON-NLS-1$
4059                 case CASE:
4060                         return "case"; //$NON-NLS-1$
4061                 case CLASS:
4062                         return "class"; //$NON-NLS-1$
4063                 case CATCH:
4064                         return "catch"; //$NON-NLS-1$
4065                 case CLONE:
4066                         //$NON-NLS-1$
4067                         return "clone";
4068                 case CONST:
4069                         //$NON-NLS-1$
4070                         return "const";
4071                 case CONTINUE:
4072                         return "continue"; //$NON-NLS-1$
4073                 case DEFAULT:
4074                         return "default"; //$NON-NLS-1$
4075                         // case define :
4076                         // return "define"; //$NON-NLS-1$
4077                 case DO:
4078                         return "do"; //$NON-NLS-1$
4079                 case ECHO:
4080                         return "echo"; //$NON-NLS-1$
4081                 case ELSE:
4082                         return "else"; //$NON-NLS-1$
4083                 case ELSEIF:
4084                         return "elseif"; //$NON-NLS-1$
4085                 case ENDFOR:
4086                         return "endfor"; //$NON-NLS-1$
4087                 case ENDFOREACH:
4088                         return "endforeach"; //$NON-NLS-1$
4089                 case ENDIF:
4090                         return "endif"; //$NON-NLS-1$
4091                 case ENDSWITCH:
4092                         return "endswitch"; //$NON-NLS-1$
4093                 case ENDWHILE:
4094                         return "endwhile"; //$NON-NLS-1$
4095                 case EXIT:
4096                         return "exit";
4097                 case EXTENDS:
4098                         return "extends"; //$NON-NLS-1$
4099                         // case false :
4100                         // return "false"; //$NON-NLS-1$
4101                 case FINAL:
4102                         return "final"; //$NON-NLS-1$
4103                 case FOR:
4104                         return "for"; //$NON-NLS-1$
4105                 case FOREACH:
4106                         return "foreach"; //$NON-NLS-1$
4107                 case FUNCTION:
4108                         return "function"; //$NON-NLS-1$
4109                 case GLOBAL:
4110                         return "global"; //$NON-NLS-1$
4111                 case IF:
4112                         return "if"; //$NON-NLS-1$
4113                 case IMPLEMENTS:
4114                         return "implements"; //$NON-NLS-1$
4115                 case INCLUDE:
4116                         return "include"; //$NON-NLS-1$
4117                 case INCLUDE_ONCE:
4118                         return "include_once"; //$NON-NLS-1$
4119                 case INSTANCEOF:
4120                         return "instanceof"; //$NON-NLS-1$
4121                 case INTERFACE:
4122                         return "interface"; //$NON-NLS-1$
4123                 case ISSET:
4124                         return "isset"; //$NON-NLS-1$
4125                 case LIST:
4126                         return "list"; //$NON-NLS-1$
4127                 case NEW:
4128                         return "new"; //$NON-NLS-1$
4129                         // case null :
4130                         // return "null"; //$NON-NLS-1$
4131                 case OP_OR_OLD:
4132                         return "OR"; //$NON-NLS-1$
4133                 case PRINT:
4134                         return "print"; //$NON-NLS-1$
4135                 case PRIVATE:
4136                         return "private"; //$NON-NLS-1$
4137                 case PROTECTED:
4138                         return "protected"; //$NON-NLS-1$
4139                 case PUBLIC:
4140                         return "public"; //$NON-NLS-1$
4141                 case REQUIRE:
4142                         return "require"; //$NON-NLS-1$
4143                 case REQUIRE_ONCE:
4144                         return "require_once"; //$NON-NLS-1$
4145                 case RETURN:
4146                         return "return"; //$NON-NLS-1$
4147                         // case self:
4148                         // return "self"; //$NON-NLS-1$
4149                 case STATIC:
4150                         return "static"; //$NON-NLS-1$
4151                 case SWITCH:
4152                         return "switch"; //$NON-NLS-1$
4153                         // case true :
4154                         // return "true"; //$NON-NLS-1$
4155                 case UNSET:
4156                         return "unset"; //$NON-NLS-1$
4157                 case VAR:
4158                         return "var"; //$NON-NLS-1$
4159                 case WHILE:
4160                         return "while"; //$NON-NLS-1$
4161                 case OP_XOR_OLD:
4162                         return "XOR"; //$NON-NLS-1$
4163                         // case this :
4164                         // return "$this"; //$NON-NLS-1$
4165                 case INTEGERLITERAL:
4166                         return "Integer(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4167                 case DOUBLELITERAL:
4168                         return "Double(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4169                 case STRINGDOUBLEQUOTE:
4170                         return "StringLiteral(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4171                 case STRINGSINGLEQUOTE:
4172                         return "StringConstant(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4173                 case STRINGINTERPOLATED:
4174                         return "StringInterpolated(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4175                 case ENCAPSEDSTRING0:
4176                         return "`"; //$NON-NLS-1$
4177                         // case EncapsedString1:
4178                         // return "\'"; //$NON-NLS-1$
4179                         // case EncapsedString2:
4180                         // return "\""; //$NON-NLS-1$
4181                 case STRING:
4182                         return "STRING_DQ(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4183                 case HEREDOC:
4184                         return "HEREDOC(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$
4185                 case PLUS_PLUS:
4186                         return "++"; //$NON-NLS-1$
4187                 case MINUS_MINUS:
4188                         return "--"; //$NON-NLS-1$
4189                 case EQUAL_EQUAL:
4190                         return "=="; //$NON-NLS-1$
4191                 case EQUAL_EQUAL_EQUAL:
4192                         return "==="; //$NON-NLS-1$
4193                 case EQUAL_GREATER:
4194                         return "=>"; //$NON-NLS-1$
4195                 case LESS_EQUAL:
4196                         return "<="; //$NON-NLS-1$
4197                 case GREATER_EQUAL:
4198                         return ">="; //$NON-NLS-1$
4199                 case NOT_EQUAL:
4200                         return "!="; //$NON-NLS-1$
4201                 case NOT_EQUAL_EQUAL:
4202                         return "!=="; //$NON-NLS-1$
4203                 case LEFT_SHIFT:
4204                         return "<<"; //$NON-NLS-1$
4205                 case RIGHT_SHIFT:
4206                         return ">>"; //$NON-NLS-1$
4207                 case PLUS_EQUAL:
4208                         return "+="; //$NON-NLS-1$
4209                 case MINUS_EQUAL:
4210                         return "-="; //$NON-NLS-1$
4211                 case MULTIPLY_EQUAL:
4212                         return "*="; //$NON-NLS-1$
4213                 case DIVIDE_EQUAL:
4214                         return "/="; //$NON-NLS-1$
4215                 case AND_EQUAL:
4216                         return "&="; //$NON-NLS-1$
4217                 case OR_EQUAL:
4218                         return "|="; //$NON-NLS-1$
4219                 case XOR_EQUAL:
4220                         return "^="; //$NON-NLS-1$
4221                 case REMAINDER_EQUAL:
4222                         return "%="; //$NON-NLS-1$
4223                 case DOT_EQUAL:
4224                         return ".="; //$NON-NLS-1$
4225                 case LEFT_SHIFT_EQUAL:
4226                         return "<<="; //$NON-NLS-1$
4227                 case RIGHT_SHIFT_EQUAL:
4228                         return ">>="; //$NON-NLS-1$
4229                 case OR_OR:
4230                         return "||"; //$NON-NLS-1$
4231                 case AND_AND:
4232                         return "&&"; //$NON-NLS-1$
4233                 case PLUS:
4234                         return "+"; //$NON-NLS-1$
4235                 case MINUS:
4236                         return "-"; //$NON-NLS-1$
4237                 case MINUS_GREATER:
4238                         return "->";
4239                 case NOT:
4240                         return "!"; //$NON-NLS-1$
4241                 case REMAINDER:
4242                         return "%"; //$NON-NLS-1$
4243                 case OP_XOR:
4244                         return "^"; //$NON-NLS-1$
4245                 case OP_AND:
4246                         return "&"; //$NON-NLS-1$
4247                 case MULTIPLY:
4248                         return "*"; //$NON-NLS-1$
4249                 case OP_OR:
4250                         return "|"; //$NON-NLS-1$
4251                 case TWIDDLE:
4252                         return "~"; //$NON-NLS-1$
4253                 case TWIDDLE_EQUAL:
4254                         return "~="; //$NON-NLS-1$
4255                 case DIVIDE:
4256                         return "/"; //$NON-NLS-1$
4257                 case GREATER:
4258                         return ">"; //$NON-NLS-1$
4259                 case LESS:
4260                         return "<"; //$NON-NLS-1$
4261                 case LPAREN:
4262                         return "("; //$NON-NLS-1$
4263                 case RPAREN:
4264                         return ")"; //$NON-NLS-1$
4265                 case LBRACE:
4266                         return "{"; //$NON-NLS-1$
4267                 case RBRACE:
4268                         return "}"; //$NON-NLS-1$
4269                 case LBRACKET:
4270                         return "["; //$NON-NLS-1$
4271                 case RBRACKET:
4272                         return "]"; //$NON-NLS-1$
4273                 case SEMICOLON:
4274                         return ";"; //$NON-NLS-1$
4275                 case QUESTION:
4276                         return "?"; //$NON-NLS-1$
4277                 case COLON:
4278                         return ":"; //$NON-NLS-1$
4279                 case COMMA:
4280                         return ","; //$NON-NLS-1$
4281                 case DOT:
4282                         return "."; //$NON-NLS-1$
4283                 case EQUAL:
4284                         return "="; //$NON-NLS-1$
4285                 case OP_AT:
4286                         return "@";
4287                 case DOLLAR:
4288                         return "$";
4289                 case DOLLAR_LBRACE:
4290                         return "${";
4291                 case LBRACE_DOLLAR:
4292                         return "{$";
4293                 case EOF:
4294                         return "EOF"; //$NON-NLS-1$
4295                 case WHITESPACE:
4296                         return "WHITESPACE(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$
4297                 case COMMENT_LINE:
4298                         return "COMMENT_LINE(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$
4299                 case COMMENT_BLOCK:
4300                         return "COMMENT_BLOCK(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$
4301                 case COMMENT_PHPDOC:
4302                         return "COMMENT_PHPDOC(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$
4303                         // case HTML :
4304                         // return "HTML(" + new String(getCurrentTokenSource()) + ")";
4305                         // //$NON-NLS-1$
4306                 case FILE:
4307                         return "__FILE__"; //$NON-NLS-1$
4308                 case LINE:
4309                         return "__LINE__"; //$NON-NLS-1$
4310                 case CLASS_C:
4311                         return "__CLASS__"; //$NON-NLS-1$
4312                 case METHOD_C:
4313                         return "__METHOD__"; //$NON-NLS-1$
4314                 case FUNC_C:
4315                         return "__FUNCTION__"; //$NON-NLS-1
4316                 case BOOLCAST:
4317                         return "( bool )"; //$NON-NLS-1$
4318                 case INTCAST:
4319                         return "( int )"; //$NON-NLS-1$
4320                 case DOUBLECAST:
4321                         return "( double )"; //$NON-NLS-1$
4322                 case OBJECTCAST:
4323                         return "( object )"; //$NON-NLS-1$
4324                 case STRINGCAST:
4325                         return "( string )"; //$NON-NLS-1$
4326         case NAMESPACE:
4327             return "( namespace )"; //$NON-NLS-1$
4328                 default:
4329                         return "token not handled (" + (act.toString ()) + ") " + new String(getCurrentTokenSource()); //$NON-NLS-1$
4330                 }
4331         }
4332
4333         public Scanner() {
4334                 this(false, false);
4335         }
4336
4337         public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace) {
4338                 this(tokenizeComments, tokenizeWhiteSpace, false);
4339         }
4340
4341         public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace,
4342                         boolean checkNonExternalizedStringLiterals) {
4343                 this(tokenizeComments, tokenizeWhiteSpace,
4344                                 checkNonExternalizedStringLiterals, false);
4345         }
4346
4347         public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace,
4348                         boolean checkNonExternalizedStringLiterals, boolean assertMode) {
4349                 this(tokenizeComments, tokenizeWhiteSpace,
4350                                 checkNonExternalizedStringLiterals, assertMode, false, null,
4351                                 null, true);
4352         }
4353
4354         public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace,
4355                         boolean checkNonExternalizedStringLiterals, boolean assertMode,
4356                         boolean tokenizeStrings, char[][] taskTags,
4357                         char[][] taskPriorities, boolean isTaskCaseSensitive) {
4358                 this.eofPosition = Integer.MAX_VALUE;
4359                 this.tokenizeComments = tokenizeComments;
4360                 this.tokenizeWhiteSpace = tokenizeWhiteSpace;
4361                 this.tokenizeStrings = tokenizeStrings;
4362                 this.checkNonExternalizedStringLiterals = checkNonExternalizedStringLiterals;
4363                 // this.assertMode = assertMode;
4364                 // this.encapsedStringStack = null;
4365                 this.taskTags = taskTags;
4366                 this.taskPriorities = taskPriorities;
4367         }
4368
4369         private void checkNonExternalizeString() throws InvalidInputException {
4370                 if (currentLine == null)
4371                         return;
4372                 parseTags(currentLine);
4373         }
4374
4375         private void parseTags(NLSLine line) throws InvalidInputException {
4376                 String s = new String(getCurrentTokenSource());
4377                 int pos = s.indexOf(TAG_PREFIX);
4378                 int lineLength = line.size();
4379                 while (pos != -1) {
4380                         int start = pos + TAG_PREFIX_LENGTH;
4381                         int end = s.indexOf(TAG_POSTFIX, start);
4382                         String index = s.substring(start, end);
4383                         int i = 0;
4384                         try {
4385                                 i = Integer.parseInt(index) - 1;
4386                                 // Tags are one based not zero based.
4387                         } catch (NumberFormatException e) {
4388                                 i = -1; // we don't want to consider this as a valid NLS tag
4389                         }
4390                         if (line.exists(i)) {
4391                                 line.set(i, null);
4392                         }
4393                         pos = s.indexOf(TAG_PREFIX, start);
4394                 }
4395                 this.nonNLSStrings = new StringLiteral[lineLength];
4396                 int nonNLSCounter = 0;
4397                 for (Iterator iterator = line.iterator(); iterator.hasNext();) {
4398                         StringLiteral literal = (StringLiteral) iterator.next();
4399                         if (literal != null) {
4400                                 this.nonNLSStrings[nonNLSCounter++] = literal;
4401                         }
4402                 }
4403                 if (nonNLSCounter == 0) {
4404                         this.nonNLSStrings = null;
4405                         currentLine = null;
4406                         return;
4407                 }
4408                 this.wasNonExternalizedStringLiteral = true;
4409                 if (nonNLSCounter != lineLength) {
4410                         System.arraycopy(this.nonNLSStrings, 0,
4411                                         (this.nonNLSStrings = new StringLiteral[nonNLSCounter]), 0,
4412                                         nonNLSCounter);
4413                 }
4414                 currentLine = null;
4415         }
4416
4417         public final void scanEscapeCharacter() throws InvalidInputException {
4418                 // the string with "\\u" is a legal string of two chars \ and u
4419                 // thus we use a direct access to the source (for regular cases).
4420                 if (unicodeAsBackSlash) {
4421                         // consume next character
4422                         unicodeAsBackSlash = false;
4423                         // if (((currentCharacter = source[currentPosition++]) == '\\') &&
4424                         // (source[currentPosition] == 'u')) {
4425                         // getNextUnicodeChar();
4426                         // } else {
4427                         if (withoutUnicodePtr != 0) {
4428                                 withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
4429                                 // }
4430                         }
4431                 } else
4432                         currentCharacter = source[currentPosition++];
4433                 switch (currentCharacter) {
4434                 case 'b':
4435                         currentCharacter = '\b';
4436                         break;
4437                 case 't':
4438                         currentCharacter = '\t';
4439                         break;
4440                 case 'n':
4441                         currentCharacter = '\n';
4442                         break;
4443                 case 'f':
4444                         currentCharacter = '\f';
4445                         break;
4446                 case 'r':
4447                         currentCharacter = '\r';
4448                         break;
4449                 case '\"':
4450                         currentCharacter = '\"';
4451                         break;
4452                 case '\'':
4453                         currentCharacter = '\'';
4454                         break;
4455                 case '\\':
4456                         currentCharacter = '\\';
4457                         break;
4458                 default:
4459                         // -----------octal escape--------------
4460                         // OctalDigit
4461                         // OctalDigit OctalDigit
4462                         // ZeroToThree OctalDigit OctalDigit
4463                         int number = Character.getNumericValue(currentCharacter);
4464                         if (number >= 0 && number <= 7) {
4465                                 boolean zeroToThreeNot = number > 3;
4466                                 if (Character
4467                                                 .isDigit(currentCharacter = source[currentPosition++])) {
4468                                         int digit = Character.getNumericValue(currentCharacter);
4469                                         if (digit >= 0 && digit <= 7) {
4470                                                 number = (number * 8) + digit;
4471                                                 if (Character
4472                                                                 .isDigit(currentCharacter = source[currentPosition++])) {
4473                                                         if (zeroToThreeNot) { // has read \NotZeroToThree
4474                                                                                                         // OctalDigit
4475                                                                 // Digit --> ignore last character
4476                                                                 currentPosition--;
4477                                                         } else {
4478                                                                 digit = Character
4479                                                                                 .getNumericValue(currentCharacter);
4480                                                                 if (digit >= 0 && digit <= 7) { // has read
4481                                                                                                                                 // \ZeroToThree
4482                                                                         // OctalDigit OctalDigit
4483                                                                         number = (number * 8) + digit;
4484                                                                 } else { // has read \ZeroToThree OctalDigit
4485                                                                                         // NonOctalDigit
4486                                                                         // --> ignore last character
4487                                                                         currentPosition--;
4488                                                                 }
4489                                                         }
4490                                                 } else { // has read \OctalDigit NonDigit--> ignore
4491                                                                         // last
4492                                                         // character
4493                                                         currentPosition--;
4494                                                 }
4495                                         } else { // has read \OctalDigit NonOctalDigit--> ignore
4496                                                                 // last
4497                                                 // character
4498                                                 currentPosition--;
4499                                         }
4500                                 } else { // has read \OctalDigit --> ignore last character
4501                                         currentPosition--;
4502                                 }
4503                                 if (number > 255)
4504                                         throw new InvalidInputException(INVALID_ESCAPE);
4505                                 currentCharacter = (char) number;
4506                         } else
4507                                 throw new InvalidInputException(INVALID_ESCAPE);
4508                 }
4509         }
4510
4511         // chech presence of task: tags
4512         // TODO (frederic) see if we need to take unicode characters into account...
4513         public void checkTaskTag(int commentStart, int commentEnd) {
4514                 char[] src = this.source;
4515
4516                 // only look for newer task: tags
4517                 if (this.foundTaskCount > 0
4518                                 && this.foundTaskPositions[this.foundTaskCount - 1][0] >= commentStart) {
4519                         return;
4520                 }
4521                 int foundTaskIndex = this.foundTaskCount;
4522                 char previous = src[commentStart + 1]; // should be '*' or '/'
4523                 nextChar: for (int i = commentStart + 2; i < commentEnd
4524                                 && i < this.eofPosition; i++) {
4525                         char[] tag = null;
4526                         char[] priority = null;
4527                         // check for tag occurrence only if not ambiguous with javadoc tag
4528                         if (previous != '@') {
4529                                 nextTag: for (int itag = 0; itag < this.taskTags.length; itag++) {
4530                                         tag = this.taskTags[itag];
4531                                         int tagLength = tag.length;
4532                                         if (tagLength == 0)
4533                                                 continue nextTag;
4534
4535                                         // ensure tag is not leaded with letter if tag starts with a
4536                                         // letter
4537                                         if (Scanner.isPHPIdentifierStart(tag[0])) {
4538                                                 if (Scanner.isPHPIdentifierPart(previous)) {
4539                                                         continue nextTag;
4540                                                 }
4541                                         }
4542
4543                                         for (int t = 0; t < tagLength; t++) {
4544                                                 char sc, tc;
4545                                                 int x = i + t;
4546                                                 if (x >= this.eofPosition || x >= commentEnd)
4547                                                         continue nextTag;
4548                                                 if ((sc = src[i + t]) != (tc = tag[t])) { // case
4549                                                                                                                                         // sensitive
4550                                                                                                                                         // check
4551                                                         if (this.isTaskCaseSensitive
4552                                                                         || (Character.toLowerCase(sc) != Character
4553                                                                                         .toLowerCase(tc))) { // case
4554                                                                 // insensitive
4555                                                                 // check
4556                                                                 continue nextTag;
4557                                                         }
4558                                                 }
4559                                         }
4560                                         // ensure tag is not followed with letter if tag finishes
4561                                         // with a
4562                                         // letter
4563                                         if (i + tagLength < commentEnd
4564                                                         && Scanner.isPHPIdentifierPart(src[i + tagLength
4565                                                                         - 1])) {
4566                                                 if (Scanner.isPHPIdentifierPart(src[i + tagLength]))
4567                                                         continue nextTag;
4568                                         }
4569                                         if (this.foundTaskTags == null) {
4570                                                 this.foundTaskTags = new char[5][];
4571                                                 this.foundTaskMessages = new char[5][];
4572                                                 this.foundTaskPriorities = new char[5][];
4573                                                 this.foundTaskPositions = new int[5][];
4574                                         } else if (this.foundTaskCount == this.foundTaskTags.length) {
4575                                                 System
4576                                                                 .arraycopy(
4577                                                                                 this.foundTaskTags,
4578                                                                                 0,
4579                                                                                 this.foundTaskTags = new char[this.foundTaskCount * 2][],
4580                                                                                 0, this.foundTaskCount);
4581                                                 System
4582                                                                 .arraycopy(
4583                                                                                 this.foundTaskMessages,
4584                                                                                 0,
4585                                                                                 this.foundTaskMessages = new char[this.foundTaskCount * 2][],
4586                                                                                 0, this.foundTaskCount);
4587                                                 System
4588                                                                 .arraycopy(
4589                                                                                 this.foundTaskPriorities,
4590                                                                                 0,
4591                                                                                 this.foundTaskPriorities = new char[this.foundTaskCount * 2][],
4592                                                                                 0, this.foundTaskCount);
4593                                                 System
4594                                                                 .arraycopy(
4595                                                                                 this.foundTaskPositions,
4596                                                                                 0,
4597                                                                                 this.foundTaskPositions = new int[this.foundTaskCount * 2][],
4598                                                                                 0, this.foundTaskCount);
4599                                         }
4600
4601                                         priority = this.taskPriorities != null
4602                                                         && itag < this.taskPriorities.length ? this.taskPriorities[itag]
4603                                                         : null;
4604
4605                                         this.foundTaskTags[this.foundTaskCount] = tag;
4606                                         this.foundTaskPriorities[this.foundTaskCount] = priority;
4607                                         this.foundTaskPositions[this.foundTaskCount] = new int[] {
4608                                                         i, i + tagLength - 1 };
4609                                         this.foundTaskMessages[this.foundTaskCount] = CharOperation.NO_CHAR;
4610                                         this.foundTaskCount++;
4611                                         i += tagLength - 1; // will be incremented when looping
4612                                         break nextTag;
4613                                 }
4614                         }
4615                         previous = src[i];
4616                 }
4617                 for (int i = foundTaskIndex; i < this.foundTaskCount; i++) {
4618                         // retrieve message start and end positions
4619                         int msgStart = this.foundTaskPositions[i][0]
4620                                         + this.foundTaskTags[i].length;
4621                         int max_value = i + 1 < this.foundTaskCount ? this.foundTaskPositions[i + 1][0] - 1
4622                                         : commentEnd - 1;
4623                         // at most beginning of next task
4624                         if (max_value < msgStart) {
4625                                 max_value = msgStart; // would only occur if tag is before
4626                                                                                 // EOF.
4627                         }
4628                         int end = -1;
4629                         char c;
4630                         for (int j = msgStart; j < max_value; j++) {
4631                                 if ((c = src[j]) == '\n' || c == '\r') {
4632                                         end = j - 1;
4633                                         break;
4634                                 }
4635                         }
4636                         if (end == -1) {
4637                                 for (int j = max_value; j > msgStart; j--) {
4638                                         if ((c = src[j]) == '*') {
4639                                                 end = j - 1;
4640                                                 break;
4641                                         }
4642                                 }
4643                                 if (end == -1)
4644                                         end = max_value;
4645                         }
4646                         if (msgStart == end)
4647                                 continue; // empty
4648                         // trim the message
4649                         while (CharOperation.isWhitespace(src[end]) && msgStart <= end)
4650                                 end--;
4651                         while (CharOperation.isWhitespace(src[msgStart]) && msgStart <= end)
4652                                 msgStart++;
4653                         // update the end position of the task
4654                         this.foundTaskPositions[i][1] = end;
4655                         // get the message source
4656                         final int messageLength = end - msgStart + 1;
4657                         char[] message = new char[messageLength];
4658                         System.arraycopy(src, msgStart, message, 0, messageLength);
4659                         this.foundTaskMessages[i] = message;
4660                 }
4661         }
4662
4663         // chech presence of task: tags
4664         // public void checkTaskTag(int commentStart, int commentEnd) {
4665         // // only look for newer task: tags
4666         // if (this.foundTaskCount > 0 &&
4667         // this.foundTaskPositions[this.foundTaskCount
4668         // - 1][0] >= commentStart) {
4669         // return;
4670         // }
4671         // int foundTaskIndex = this.foundTaskCount;
4672         // nextChar: for (int i = commentStart; i < commentEnd && i <
4673         // this.eofPosition; i++) {
4674         // char[] tag = null;
4675         // char[] priority = null;
4676         // // check for tag occurrence
4677         // nextTag: for (int itag = 0; itag < this.taskTags.length; itag++) {
4678         // tag = this.taskTags[itag];
4679         // priority = this.taskPriorities != null && itag <
4680         // this.taskPriorities.length
4681         // ? this.taskPriorities[itag] : null;
4682         // int tagLength = tag.length;
4683         // for (int t = 0; t < tagLength; t++) {
4684         // if (this.source[i + t] != tag[t])
4685         // continue nextTag;
4686         // }
4687         // if (this.foundTaskTags == null) {
4688         // this.foundTaskTags = new char[5][];
4689         // this.foundTaskMessages = new char[5][];
4690         // this.foundTaskPriorities = new char[5][];
4691         // this.foundTaskPositions = new int[5][];
4692         // } else if (this.foundTaskCount == this.foundTaskTags.length) {
4693         // System.arraycopy(this.foundTaskTags, 0, this.foundTaskTags = new
4694         // char[this.foundTaskCount * 2][], 0, this.foundTaskCount);
4695         // System.arraycopy(this.foundTaskMessages, 0, this.foundTaskMessages = new
4696         // char[this.foundTaskCount * 2][], 0,
4697         // this.foundTaskCount);
4698         // System.arraycopy(this.foundTaskPriorities, 0, this.foundTaskPriorities =
4699         // new char[this.foundTaskCount * 2][], 0,
4700         // this.foundTaskCount);
4701         // System.arraycopy(this.foundTaskPositions, 0, this.foundTaskPositions =
4702         // new
4703         // int[this.foundTaskCount * 2][], 0,
4704         // this.foundTaskCount);
4705         // }
4706         // this.foundTaskTags[this.foundTaskCount] = tag;
4707         // this.foundTaskPriorities[this.foundTaskCount] = priority;
4708         // this.foundTaskPositions[this.foundTaskCount] = new int[] { i, i +
4709         // tagLength
4710         // - 1 };
4711         // this.foundTaskMessages[this.foundTaskCount] = CharOperation.NO_CHAR;
4712         // this.foundTaskCount++;
4713         // i += tagLength - 1; // will be incremented when looping
4714         // }
4715         // }
4716         // for (int i = foundTaskIndex; i < this.foundTaskCount; i++) {
4717         // // retrieve message start and end positions
4718         // int msgStart = this.foundTaskPositions[i][0] +
4719         // this.foundTaskTags[i].length;
4720         // int max_value = i + 1 < this.foundTaskCount ? this.foundTaskPositions[i +
4721         // 1][0] - 1 : commentEnd - 1;
4722         // // at most beginning of next task
4723         // if (max_value < msgStart)
4724         // max_value = msgStart; // would only occur if tag is before EOF.
4725         // int end = -1;
4726         // char c;
4727         // for (int j = msgStart; j < max_value; j++) {
4728         // if ((c = this.source[j]) == '\n' || c == '\r') {
4729         // end = j - 1;
4730         // break;
4731         // }
4732         // }
4733         // if (end == -1) {
4734         // for (int j = max_value; j > msgStart; j--) {
4735         // if ((c = this.source[j]) == '*') {
4736         // end = j - 1;
4737         // break;
4738         // }
4739         // }
4740         // if (end == -1)
4741         // end = max_value;
4742         // }
4743         // if (msgStart == end)
4744         // continue; // empty
4745         // // trim the message
4746         // while (CharOperation.isWhitespace(source[end]) && msgStart <= end)
4747         // end--;
4748         // while (CharOperation.isWhitespace(source[msgStart]) && msgStart <= end)
4749         // msgStart++;
4750         // // update the end position of the task
4751         // this.foundTaskPositions[i][1] = end;
4752         // // get the message source
4753         // final int messageLength = end - msgStart + 1;
4754         // char[] message = new char[messageLength];
4755         // System.arraycopy(source, msgStart, message, 0, messageLength);
4756         // this.foundTaskMessages[i] = message;
4757         // }
4758         // }
4759 }
4760