secure.phpeclipse.com Git - phpeclipse.git/blob

   1 /*******************************************************************************
   2  * Copyright (c) 2000, 2001, 2002 International Business Machines Corp. and others.
   3  * All rights reserved. This program and the accompanying materials
   4  * are made available under the terms of the Common Public License v0.5
   5  * which accompanies this distribution, and is available at
   6  * http://www.eclipse.org/legal/cpl-v05.html
   7  *
   8  * Contributors:
   9  *     IBM Corporation - initial API and implementation
  10  ******************************************************************************/
  11 package net.sourceforge.phpdt.internal.codeassist.complete;
  12
  13 /*
  14  * Scanner aware of a cursor location so as to discard trailing portions of identifiers
  15  * containing the cursor location.
  16  *
  17  * Cursor location denotes the position of the last character behind which completion
  18  * got requested:
  19  *  -1 means completion at the very beginning of the source
  20  *      0  means completion behind the first character
  21  *  n  means completion behind the n-th character
  22  */
  23 import net.sourceforge.phpdt.core.compiler.InvalidInputException;
  24 import net.sourceforge.phpdt.internal.compiler.parser.Scanner;
  25
  26 public class CompletionScanner extends Scanner {
  27
  28         public char[] completionIdentifier;
  29         public int cursorLocation;
  30
  31         /* Source positions of the completedIdentifier
  32          * if inside actual identifier, end goes to the actual identifier
  33          * end, i.e. beyond cursor location
  34          */
  35         public int completedIdentifierStart = 0;
  36         public int completedIdentifierEnd = -1;
  37
  38         public static final char[] EmptyCompletionIdentifier = {};
  39 public CompletionScanner(boolean assertMode) {
  40         super(false, false, false, assertMode);
  41 }
  42 /*
  43  * Truncate the current identifier if it is containing the cursor location. Since completion is performed
  44  * on an identifier prefix.
  45  *
  46  */
  47 public char[] getCurrentIdentifierSource() {
  48
  49         if (completionIdentifier == null){
  50                 if (cursorLocation < startPosition && currentPosition == startPosition){ // fake empty identifier got issued
  51                         // remember actual identifier positions
  52                         completedIdentifierStart = startPosition;
  53                         completedIdentifierEnd = completedIdentifierStart - 1;
  54                         return completionIdentifier = EmptyCompletionIdentifier;
  55                 }
  56                 if (cursorLocation+1 >= startPosition && cursorLocation < currentPosition){
  57                         // remember actual identifier positions
  58                         completedIdentifierStart = startPosition;
  59                         completedIdentifierEnd = currentPosition - 1;
  60                         if (withoutUnicodePtr != 0){                    // check unicode scenario
  61                                 System.arraycopy(withoutUnicodeBuffer, 1, completionIdentifier = new char[withoutUnicodePtr], 0, withoutUnicodePtr);
  62                         } else {
  63                                 int length = cursorLocation + 1 - startPosition;
  64                                 // no char[] sharing around completionIdentifier, we want it to be unique so as to use identity checks
  65                                 System.arraycopy(source, startPosition, (completionIdentifier = new char[length]), 0, length);
  66                         }
  67                         return completionIdentifier;
  68                 }
  69         }
  70         return super.getCurrentIdentifierSource();
  71 }
  72 /*
  73  * Identifier splitting for unicodes.
  74  * Only store the current unicode if we did not pass the cursorLocation.
  75  * Note: this does not handle cases where the cursor is in the middle of a unicode
  76  */
  77 public boolean getNextCharAsJavaIdentifierPart() {
  78
  79         int temp = currentPosition;
  80         try {
  81                 if (((currentCharacter = source[currentPosition++]) == '\\')
  82                         && (source[currentPosition] == 'u')) {
  83                         //-------------unicode traitement ------------
  84                         int c1, c2, c3, c4;
  85                         int unicodeSize = 6;
  86                         currentPosition++;
  87                         while (source[currentPosition] == 'u') {
  88                                 currentPosition++;
  89                                 unicodeSize++;
  90                         }
  91
  92                         if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
  93                                 || c1 < 0)
  94                                 || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15 || c2 < 0)
  95                                 || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15 || c3 < 0)
  96                                 || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15 || c4 < 0)) {
  97                                 currentPosition = temp;
  98                                 return false;
  99                         }
 100
 101                         currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 102                         if (!Character.isJavaIdentifierPart(currentCharacter)) {
 103                                 currentPosition = temp;
 104                                 return false;
 105                         }
 106
 107                         //need the unicode buffer
 108                         if (withoutUnicodePtr == 0) {
 109                                 //buffer all the entries that have been left aside....
 110                                 withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
 111                                 System.arraycopy(
 112                                         source,
 113                                         startPosition,
 114                                         withoutUnicodeBuffer,
 115                                         1,
 116                                         withoutUnicodePtr);
 117                         }
 118                         if (temp < cursorLocation && cursorLocation < currentPosition-1){
 119                                 throw new InvalidCursorLocation(InvalidCursorLocation.NO_COMPLETION_INSIDE_UNICODE);
 120                         }
 121                         // store the current unicode, only if we did not pass the cursorLocation
 122                         // Note: this does not handle cases where the cursor is in the middle of a unicode
 123                         if ((completionIdentifier != null)
 124                                 || (startPosition <= cursorLocation+1 && cursorLocation >= currentPosition-1)){
 125                                 withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 126                         }
 127                         return true;
 128                 } //-------------end unicode traitement--------------
 129                 else {
 130                         if (!Character.isJavaIdentifierPart(currentCharacter)) {
 131                                 currentPosition = temp;
 132                                 return false;
 133                         }
 134
 135                         if (withoutUnicodePtr != 0){
 136                                 // store the current unicode, only if we did not pass the cursorLocation
 137                                 // Note: this does not handle cases where the cursor is in the middle of a unicode
 138                                 if ((completionIdentifier != null)
 139                                         || (startPosition <= cursorLocation+1 && cursorLocation >= currentPosition-1)){
 140                                         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 141                                 }
 142                         }
 143                         return true;
 144                 }
 145         } catch (IndexOutOfBoundsException e) {
 146                 currentPosition = temp;
 147                 return false;
 148         }
 149 }
 150 public int getNextToken() throws InvalidInputException {
 151
 152         this.wasAcr = false;
 153         if (diet) {
 154                 jumpOverMethodBody();
 155                 diet = false;
 156                 return currentPosition > source.length ? TokenNameEOF : TokenNameRBRACE;
 157         }
 158         try {
 159                 while (true) { //loop for jumping over comments
 160                         withoutUnicodePtr = 0;
 161                         //start with a new token (even comment written with unicode )
 162
 163                         // ---------Consume white space and handles startPosition---------
 164                         int whiteStart = currentPosition;
 165                         boolean isWhiteSpace;
 166                         do {
 167                                 startPosition = currentPosition;
 168                                 if (((currentCharacter = source[currentPosition++]) == '\\')
 169                                         && (source[currentPosition] == 'u')) {
 170                                         isWhiteSpace = jumpOverUnicodeWhiteSpace();
 171                                 } else {
 172                                         if (recordLineSeparator
 173                                                 && ((currentCharacter == '\r') || (currentCharacter == '\n')))
 174                                                 pushLineSeparator();
 175                                         isWhiteSpace =
 176                                                 (currentCharacter == ' ') || Character.isWhitespace(currentCharacter);
 177                                 }
 178                                 /* completion requesting strictly inside blanks */
 179                                 if ((whiteStart != currentPosition)
 180                                         //&& (previousToken == TokenNameDOT)
 181                                         && (completionIdentifier == null)
 182                                         && (whiteStart <= cursorLocation+1)
 183                                         && (cursorLocation < startPosition)
 184                                         && !Character.isJavaIdentifierStart(currentCharacter)){
 185                                         currentPosition = startPosition; // for next token read
 186                                         return TokenNameIdentifier;
 187                                 }
 188                         } while (isWhiteSpace);
 189                         if (tokenizeWhiteSpace && (whiteStart != currentPosition - 1)) {
 190                                 // reposition scanner in case we are interested by spaces as tokens
 191                                 currentPosition--;
 192                                 startPosition = whiteStart;
 193                                 return TokenNameWHITESPACE;
 194                         }
 195                         //little trick to get out in the middle of a source computation
 196                         if (currentPosition > eofPosition){
 197                                 /* might be completing at eof (e.g. behind a dot) */
 198                                 if (completionIdentifier == null &&
 199                                         startPosition == cursorLocation + 1){
 200                                         currentPosition = startPosition; // for being detected as empty free identifier
 201                                         return TokenNameIdentifier;
 202                                 }
 203                                 return TokenNameEOF;
 204                         }
 205
 206                         // ---------Identify the next token-------------
 207
 208                         switch (currentCharacter) {
 209                                 case '(' :
 210                                         return TokenNameLPAREN;
 211                                 case ')' :
 212                                         return TokenNameRPAREN;
 213                                 case '{' :
 214                                         return TokenNameLBRACE;
 215                                 case '}' :
 216                                         return TokenNameRBRACE;
 217                                 case '[' :
 218                                         return TokenNameLBRACKET;
 219                                 case ']' :
 220                                         return TokenNameRBRACKET;
 221                                 case ';' :
 222                                         return TokenNameSEMICOLON;
 223                                 case ',' :
 224                                         return TokenNameCOMMA;
 225                                 case '.' :
 226                                         if (startPosition <= cursorLocation
 227                                             && cursorLocation < currentPosition){
 228                                                 return TokenNameDOT; // completion inside .<|>12
 229                                     }
 230                                         if (getNextCharAsDigit())
 231                                                 return scanNumber(true);
 232                                         return TokenNameDOT;
 233                                 case '+' :
 234                                         {
 235                                                 int test;
 236                                                 if ((test = getNextChar('+', '=')) == 0)
 237                                                         return TokenNamePLUS_PLUS;
 238                                                 if (test > 0)
 239                                                         return TokenNamePLUS_EQUAL;
 240                                                 return TokenNamePLUS;
 241                                         }
 242                                 case '-' :
 243                                         {
 244                                                 int test;
 245                                                 if ((test = getNextChar('-', '=')) == 0)
 246                                                         return TokenNameMINUS_MINUS;
 247                                                 if (test > 0)
 248                                                         return TokenNameMINUS_EQUAL;
 249                                                 return TokenNameMINUS;
 250                                         }
 251                                 case '~' :
 252                                         return TokenNameTWIDDLE;
 253                                 case '!' :
 254                                         if (getNextChar('='))
 255                                                 return TokenNameNOT_EQUAL;
 256                                         return TokenNameNOT;
 257                                 case '*' :
 258                                         if (getNextChar('='))
 259                                                 return TokenNameMULTIPLY_EQUAL;
 260                                         return TokenNameMULTIPLY;
 261                                 case '%' :
 262                                         if (getNextChar('='))
 263                                                 return TokenNameREMAINDER_EQUAL;
 264                                         return TokenNameREMAINDER;
 265                                 case '<' :
 266                                         {
 267                                                 int test;
 268                                                 if ((test = getNextChar('=', '<')) == 0)
 269                                                         return TokenNameLESS_EQUAL;
 270                                                 if (test > 0) {
 271                                                         if (getNextChar('='))
 272                                                                 return TokenNameLEFT_SHIFT_EQUAL;
 273                                                         return TokenNameLEFT_SHIFT;
 274                                                 }
 275                                                 return TokenNameLESS;
 276                                         }
 277                                 case '>' :
 278                                         {
 279                                                 int test;
 280                                                 if ((test = getNextChar('=', '>')) == 0)
 281                                                         return TokenNameGREATER_EQUAL;
 282                                                 if (test > 0) {
 283                                                         if ((test = getNextChar('=', '>')) == 0)
 284                                                                 return TokenNameRIGHT_SHIFT_EQUAL;
 285                                                         if (test > 0) {
 286                                                                 if (getNextChar('='))
 287                                                                         return TokenNameUNSIGNED_RIGHT_SHIFT_EQUAL;
 288                                                                 return TokenNameUNSIGNED_RIGHT_SHIFT;
 289                                                         }
 290                                                         return TokenNameRIGHT_SHIFT;
 291                                                 }
 292                                                 return TokenNameGREATER;
 293                                         }
 294                                 case '=' :
 295                                         if (getNextChar('='))
 296                                                 return TokenNameEQUAL_EQUAL;
 297                                         return TokenNameEQUAL;
 298                                 case '&' :
 299                                         {
 300                                                 int test;
 301                                                 if ((test = getNextChar('&', '=')) == 0)
 302                                                         return TokenNameAND_AND;
 303                                                 if (test > 0)
 304                                                         return TokenNameAND_EQUAL;
 305                                                 return TokenNameAND;
 306                                         }
 307                                 case '|' :
 308                                         {
 309                                                 int test;
 310                                                 if ((test = getNextChar('|', '=')) == 0)
 311                                                         return TokenNameOR_OR;
 312                                                 if (test > 0)
 313                                                         return TokenNameOR_EQUAL;
 314                                                 return TokenNameOR;
 315                                         }
 316                                 case '^' :
 317                                         if (getNextChar('='))
 318                                                 return TokenNameXOR_EQUAL;
 319                                         return TokenNameXOR;
 320                                 case '?' :
 321                                         return TokenNameQUESTION;
 322                                 case ':' :
 323                                         return TokenNameCOLON;
 324                                 case '\'' :
 325                                         {
 326                                                 int test;
 327                                                 if ((test = getNextChar('\n', '\r')) == 0) {
 328                                                         throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
 329                                                 }
 330                                                 if (test > 0) {
 331                                                         // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
 332                                                         for (int lookAhead = 0; lookAhead < 3; lookAhead++) {
 333                                                                 if (currentPosition + lookAhead == source.length)
 334                                                                         break;
 335                                                                 if (source[currentPosition + lookAhead] == '\n')
 336                                                                         break;
 337                                                                 if (source[currentPosition + lookAhead] == '\'') {
 338                                                                         currentPosition += lookAhead + 1;
 339                                                                         break;
 340                                                                 }
 341                                                         }
 342                                                         throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
 343                                                 }
 344                                         }
 345                                         if (getNextChar('\'')) {
 346                                                 // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
 347                                                 for (int lookAhead = 0; lookAhead < 3; lookAhead++) {
 348                                                         if (currentPosition + lookAhead == source.length)
 349                                                                 break;
 350                                                         if (source[currentPosition + lookAhead] == '\n')
 351                                                                 break;
 352                                                         if (source[currentPosition + lookAhead] == '\'') {
 353                                                                 currentPosition += lookAhead + 1;
 354                                                                 break;
 355                                                         }
 356                                                 }
 357                                                 throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
 358                                         }
 359                                         if (getNextChar('\\'))
 360                                                 scanEscapeCharacter();
 361                                         else { // consume next character
 362                                                 unicodeAsBackSlash = false;
 363                                                 if (((currentCharacter = source[currentPosition++]) == '\\')
 364                                                         && (source[currentPosition] == 'u')) {
 365                                                         getNextUnicodeChar();
 366                                                 } else {
 367                                                         if (withoutUnicodePtr != 0) {
 368                                                                 withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 369                                                         }
 370                                                 }
 371                                         }
 372                                         if (getNextChar('\''))
 373                                                 return TokenNameCharacterLiteral;
 374                                         // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
 375                                         for (int lookAhead = 0; lookAhead < 20; lookAhead++) {
 376                                                 if (currentPosition + lookAhead == source.length)
 377                                                         break;
 378                                                 if (source[currentPosition + lookAhead] == '\n')
 379                                                         break;
 380                                                 if (source[currentPosition + lookAhead] == '\'') {
 381                                                         currentPosition += lookAhead + 1;
 382                                                         break;
 383                                                 }
 384                                         }
 385                                         throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
 386                                 case '"' :
 387                                         try {
 388                                                 // consume next character
 389                                                 unicodeAsBackSlash = false;
 390                                                 if (((currentCharacter = source[currentPosition++]) == '\\')
 391                                                         && (source[currentPosition] == 'u')) {
 392                                                         getNextUnicodeChar();
 393                                                 } else {
 394                                                         if (withoutUnicodePtr != 0) {
 395                                                                 withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 396                                                         }
 397                                                 }
 398
 399                                                 while (currentCharacter != '"') {
 400                                                         /**** \r and \n are not valid in string literals ****/
 401                                                         if ((currentCharacter == '\n') || (currentCharacter == '\r')) {
 402                                                                 // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
 403                                                                 for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
 404                                                                         if (currentPosition + lookAhead == source.length)
 405                                                                                 break;
 406                                                                         if (source[currentPosition + lookAhead] == '\n')
 407                                                                                 break;
 408                                                                         if (source[currentPosition + lookAhead] == '\"') {
 409                                                                                 currentPosition += lookAhead + 1;
 410                                                                                 break;
 411                                                                         }
 412                                                                 }
 413                                                                 throw new InvalidInputException(INVALID_CHAR_IN_STRING);
 414                                                         }
 415                                                         if (currentCharacter == '\\') {
 416                                                                 int escapeSize = currentPosition;
 417                                                                 boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
 418                                                                 //scanEscapeCharacter make a side effect on this value and we need the previous value few lines down this one
 419                                                                 scanEscapeCharacter();
 420                                                                 escapeSize = currentPosition - escapeSize;
 421                                                                 if (withoutUnicodePtr == 0) {
 422                                                                         //buffer all the entries that have been left aside....
 423                                                                         withoutUnicodePtr = currentPosition - escapeSize - 1 - startPosition;
 424                                                                         System.arraycopy(
 425                                                                                 source,
 426                                                                                 startPosition,
 427                                                                                 withoutUnicodeBuffer,
 428                                                                                 1,
 429                                                                                 withoutUnicodePtr);
 430                                                                         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 431                                                                 } else { //overwrite the / in the buffer
 432                                                                         withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter;
 433                                                                         if (backSlashAsUnicodeInString) { //there are TWO \ in the stream where only one is correct
 434                                                                                 withoutUnicodePtr--;
 435                                                                         }
 436                                                                 }
 437                                                         }
 438                                                         // consume next character
 439                                                         unicodeAsBackSlash = false;
 440                                                         if (((currentCharacter = source[currentPosition++]) == '\\')
 441                                                                 && (source[currentPosition] == 'u')) {
 442                                                                 getNextUnicodeChar();
 443                                                         } else {
 444                                                                 if (withoutUnicodePtr != 0) {
 445                                                                         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 446                                                                 }
 447                                                         }
 448
 449                                                 }
 450                                         } catch (IndexOutOfBoundsException e) {
 451                                                 throw new InvalidInputException(UNTERMINATED_STRING);
 452                                         } catch (InvalidInputException e) {
 453                                                 if (e.getMessage().equals(INVALID_ESCAPE)) {
 454                                                         // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
 455                                                         for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
 456                                                                 if (currentPosition + lookAhead == source.length)
 457                                                                         break;
 458                                                                 if (source[currentPosition + lookAhead] == '\n')
 459                                                                         break;
 460                                                                 if (source[currentPosition + lookAhead] == '\"') {
 461                                                                         currentPosition += lookAhead + 1;
 462                                                                         break;
 463                                                                 }
 464                                                         }
 465
 466                                                 }
 467                                                 throw e; // rethrow
 468                                         }
 469                                         if (startPosition <= cursorLocation && cursorLocation <= currentPosition-1){
 470                                                 throw new InvalidCursorLocation(InvalidCursorLocation.NO_COMPLETION_INSIDE_STRING);
 471                                         }
 472                                         return TokenNameStringLiteral;
 473                                 case '/' :
 474                                         {
 475                                                 int test;
 476                                                 if ((test = getNextChar('/', '*')) == 0) { //line comment
 477                                                         try { //get the next char
 478                                                                 if (((currentCharacter = source[currentPosition++]) == '\\')
 479                                                                         && (source[currentPosition] == 'u')) {
 480                                                                         //-------------unicode traitement ------------
 481                                                                         int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
 482                                                                         currentPosition++;
 483                                                                         while (source[currentPosition] == 'u') {
 484                                                                                 currentPosition++;
 485                                                                         }
 486                                                                         if ((c1 = Character.getNumericValue(source[currentPosition++])) > 15
 487                                                                                 || c1 < 0
 488                                                                                 || (c2 = Character.getNumericValue(source[currentPosition++])) > 15
 489                                                                                 || c2 < 0
 490                                                                                 || (c3 = Character.getNumericValue(source[currentPosition++])) > 15
 491                                                                                 || c3 < 0
 492                                                                                 || (c4 = Character.getNumericValue(source[currentPosition++])) > 15
 493                                                                                 || c4 < 0) {
 494                                                                                 throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
 495                                                                         } else {
 496                                                                                 currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 497                                                                         }
 498                                                                 }
 499
 500                                                                 //handle the \\u case manually into comment
 501                                                                 if (currentCharacter == '\\') {
 502                                                                         if (source[currentPosition] == '\\')
 503                                                                                 currentPosition++;
 504                                                                 } //jump over the \\
 505                                                                 while (currentCharacter != '\r' && currentCharacter != '\n') {
 506                                                                         //get the next char
 507                                                                         if (((currentCharacter = source[currentPosition++]) == '\\')
 508                                                                                 && (source[currentPosition] == 'u')) {
 509                                                                                 //-------------unicode traitement ------------
 510                                                                                 int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
 511                                                                                 currentPosition++;
 512                                                                                 while (source[currentPosition] == 'u') {
 513                                                                                         currentPosition++;
 514                                                                                 }
 515                                                                                 if ((c1 = Character.getNumericValue(source[currentPosition++])) > 15
 516                                                                                         || c1 < 0
 517                                                                                         || (c2 = Character.getNumericValue(source[currentPosition++])) > 15
 518                                                                                         || c2 < 0
 519                                                                                         || (c3 = Character.getNumericValue(source[currentPosition++])) > 15
 520                                                                                         || c3 < 0
 521                                                                                         || (c4 = Character.getNumericValue(source[currentPosition++])) > 15
 522                                                                                         || c4 < 0) {
 523                                                                                         throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
 524                                                                                 } else {
 525                                                                                         currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 526                                                                                 }
 527                                                                         }
 528                                                                         //handle the \\u case manually into comment
 529                                                                         if (currentCharacter == '\\') {
 530                                                                                 if (source[currentPosition] == '\\')
 531                                                                                         currentPosition++;
 532                                                                         } //jump over the \\
 533                                                                 }
 534                                                                 recordComment(false);
 535                                                                 if (startPosition <= cursorLocation && cursorLocation < currentPosition-1){
 536                                                                         throw new InvalidCursorLocation(InvalidCursorLocation.NO_COMPLETION_INSIDE_COMMENT);
 537                                                                 }
 538                                                                 if (recordLineSeparator
 539                                                                         && ((currentCharacter == '\r') || (currentCharacter == '\n')))
 540                                                                         pushLineSeparator();
 541                                                                 if (tokenizeComments) {
 542                                                                         currentPosition--; // reset one character behind
 543                                                                         return TokenNameCOMMENT_LINE;
 544                                                                 }
 545                                                         } catch (IndexOutOfBoundsException e) { //an eof will them be generated
 546                                                                 if (tokenizeComments) {
 547                                                                         currentPosition--; // reset one character behind
 548                                                                         return TokenNameCOMMENT_LINE;
 549                                                                 }
 550                                                         }
 551                                                         break;
 552                                                 }
 553                                                 if (test > 0) { //traditional and annotation comment
 554                                                         boolean isJavadoc = false, star = false;
 555                                                         // consume next character
 556                                                         unicodeAsBackSlash = false;
 557                                                         if (((currentCharacter = source[currentPosition++]) == '\\')
 558                                                                 && (source[currentPosition] == 'u')) {
 559                                                                 getNextUnicodeChar();
 560                                                         } else {
 561                                                                 if (withoutUnicodePtr != 0) {
 562                                                                         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 563                                                                 }
 564                                                         }
 565
 566                                                         if (currentCharacter == '*') {
 567                                                                 isJavadoc = true;
 568                                                                 star = true;
 569                                                         }
 570                                                         if (recordLineSeparator
 571                                                                 && ((currentCharacter == '\r') || (currentCharacter == '\n')))
 572                                                                 pushLineSeparator();
 573                                                         try { //get the next char
 574                                                                 if (((currentCharacter = source[currentPosition++]) == '\\')
 575                                                                         && (source[currentPosition] == 'u')) {
 576                                                                         //-------------unicode traitement ------------
 577                                                                         int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
 578                                                                         currentPosition++;
 579                                                                         while (source[currentPosition] == 'u') {
 580                                                                                 currentPosition++;
 581                                                                         }
 582                                                                         if ((c1 = Character.getNumericValue(source[currentPosition++])) > 15
 583                                                                                 || c1 < 0
 584                                                                                 || (c2 = Character.getNumericValue(source[currentPosition++])) > 15
 585                                                                                 || c2 < 0
 586                                                                                 || (c3 = Character.getNumericValue(source[currentPosition++])) > 15
 587                                                                                 || c3 < 0
 588                                                                                 || (c4 = Character.getNumericValue(source[currentPosition++])) > 15
 589                                                                                 || c4 < 0) {
 590                                                                                 throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
 591                                                                         } else {
 592                                                                                 currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 593                                                                         }
 594                                                                 }
 595                                                                 //handle the \\u case manually into comment
 596                                                                 if (currentCharacter == '\\') {
 597                                                                         if (source[currentPosition] == '\\')
 598                                                                                 currentPosition++;
 599                                                                 } //jump over the \\
 600                                                                 // empty comment is not a javadoc /**/
 601                                                                 if (currentCharacter == '/') {
 602                                                                         isJavadoc = false;
 603                                                                 }
 604                                                                 //loop until end of comment */
 605                                                                 while ((currentCharacter != '/') || (!star)) {
 606                                                                         if (recordLineSeparator
 607                                                                                 && ((currentCharacter == '\r') || (currentCharacter == '\n')))
 608                                                                                 pushLineSeparator();
 609                                                                         star = currentCharacter == '*';
 610                                                                         //get next char
 611                                                                         if (((currentCharacter = source[currentPosition++]) == '\\')
 612                                                                                 && (source[currentPosition] == 'u')) {
 613                                                                                 //-------------unicode traitement ------------
 614                                                                                 int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
 615                                                                                 currentPosition++;
 616                                                                                 while (source[currentPosition] == 'u') {
 617                                                                                         currentPosition++;
 618                                                                                 }
 619                                                                                 if ((c1 = Character.getNumericValue(source[currentPosition++])) > 15
 620                                                                                         || c1 < 0
 621                                                                                         || (c2 = Character.getNumericValue(source[currentPosition++])) > 15
 622                                                                                         || c2 < 0
 623                                                                                         || (c3 = Character.getNumericValue(source[currentPosition++])) > 15
 624                                                                                         || c3 < 0
 625                                                                                         || (c4 = Character.getNumericValue(source[currentPosition++])) > 15
 626                                                                                         || c4 < 0) {
 627                                                                                         throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
 628                                                                                 } else {
 629                                                                                         currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 630                                                                                 }
 631                                                                         }
 632                                                                         //handle the \\u case manually into comment
 633                                                                         if (currentCharacter == '\\') {
 634                                                                                 if (source[currentPosition] == '\\')
 635                                                                                         currentPosition++;
 636                                                                         } //jump over the \\
 637                                                                 }
 638                                                                 recordComment(isJavadoc);
 639                                                                 if (startPosition <= cursorLocation && cursorLocation < currentPosition-1){
 640                                                                         throw new InvalidCursorLocation(InvalidCursorLocation.NO_COMPLETION_INSIDE_COMMENT);
 641                                                                 }
 642                                                                 if (tokenizeComments) {
 643                                                                         if (isJavadoc)
 644                                                                                 return TokenNameCOMMENT_JAVADOC;
 645                                                                         return TokenNameCOMMENT_BLOCK;
 646                                                                 }
 647                                                         } catch (IndexOutOfBoundsException e) {
 648                                                                 throw new InvalidInputException(UNTERMINATED_COMMENT);
 649                                                         }
 650                                                         break;
 651                                                 }
 652                                                 if (getNextChar('='))
 653                                                         return TokenNameDIVIDE_EQUAL;
 654                                                 return TokenNameDIVIDE;
 655                                         }
 656                                 case '\u001a' :
 657                                         if (atEnd())
 658                                                 return TokenNameEOF;
 659                                         //the atEnd may not be <currentPosition == source.length> if source is only some part of a real (external) stream
 660                                         throw new InvalidInputException("Ctrl-Z"); //$NON-NLS-1$
 661
 662                                 default :
 663                                         if (Character.isJavaIdentifierStart(currentCharacter))
 664                                                 return scanIdentifierOrKeyword();
 665                                         if (Character.isDigit(currentCharacter))
 666                                                 return scanNumber(false);
 667                                         return TokenNameERROR;
 668                         }
 669                 }
 670         } //-----------------end switch while try--------------------
 671         catch (IndexOutOfBoundsException e) {
 672         }
 673         /* might be completing at very end of file (e.g. behind a dot) */
 674         if (completionIdentifier == null &&
 675                 startPosition == cursorLocation + 1){
 676                 currentPosition = startPosition; // for being detected as empty free identifier
 677                 return TokenNameIdentifier;
 678         }
 679         return TokenNameEOF;
 680 }
 681 /*
 682  * In case we actually read a keyword, but the cursor is located inside,
 683  * we pretend we read an identifier.
 684  */
 685 public int scanIdentifierOrKeyword() throws InvalidInputException {
 686
 687         int id = super.scanIdentifierOrKeyword();
 688
 689         // convert completed keyword into an identifier
 690         if (id != TokenNameIdentifier
 691                 && startPosition <= cursorLocation+1
 692                 && cursorLocation < currentPosition){
 693                 return TokenNameIdentifier;
 694         }
 695         return id;
 696 }
 697 public int scanNumber(boolean dotPrefix) throws InvalidInputException {
 698
 699         int token = super.scanNumber(dotPrefix);
 700
 701         // consider completion just before a number to be ok, will insert before it
 702         if (startPosition <= cursorLocation && cursorLocation < currentPosition){
 703                 throw new InvalidCursorLocation(InvalidCursorLocation.NO_COMPLETION_INSIDE_NUMBER);
 704         }
 705         return token;
 706 }
 707 }