misc changes
[phpeclipse.git] / net.sourceforge.phpeclipse / src / net / sourceforge / phpdt / internal / codeassist / complete / CompletionScanner.java
1 /*******************************************************************************
2  * Copyright (c) 2000, 2001, 2002 International Business Machines Corp. and others.
3  * All rights reserved. This program and the accompanying materials 
4  * are made available under the terms of the Common Public License v0.5 
5  * which accompanies this distribution, and is available at
6  * http://www.eclipse.org/legal/cpl-v05.html
7  * 
8  * Contributors:
9  *     IBM Corporation - initial API and implementation
10  ******************************************************************************/
11 package net.sourceforge.phpdt.internal.codeassist.complete;
12
13 /*
14  * Scanner aware of a cursor location so as to discard trailing portions of identifiers
15  * containing the cursor location.
16  *
17  * Cursor location denotes the position of the last character behind which completion
18  * got requested:
19  *  -1 means completion at the very beginning of the source
20  *      0  means completion behind the first character
21  *  n  means completion behind the n-th character
22  */
23 import net.sourceforge.phpdt.core.compiler.InvalidInputException;
24 import net.sourceforge.phpdt.internal.compiler.parser.Scanner;
25
26 public class CompletionScanner extends Scanner {
27
28         public char[] completionIdentifier;
29         public int cursorLocation;
30                 
31         /* Source positions of the completedIdentifier
32          * if inside actual identifier, end goes to the actual identifier 
33          * end, i.e. beyond cursor location
34          */
35         public int completedIdentifierStart = 0;
36         public int completedIdentifierEnd = -1;
37
38         public static final char[] EmptyCompletionIdentifier = {};
39 public CompletionScanner(boolean assertMode) {
40         super(false, false, false, assertMode);
41 }
42 /* 
43  * Truncate the current identifier if it is containing the cursor location. Since completion is performed
44  * on an identifier prefix.
45  *
46  */
47 public char[] getCurrentIdentifierSource() {
48
49         if (completionIdentifier == null){
50                 if (cursorLocation < startPosition && currentPosition == startPosition){ // fake empty identifier got issued
51                         // remember actual identifier positions
52                         completedIdentifierStart = startPosition;
53                         completedIdentifierEnd = completedIdentifierStart - 1;                  
54                         return completionIdentifier = EmptyCompletionIdentifier;                                        
55                 }
56                 if (cursorLocation+1 >= startPosition && cursorLocation < currentPosition){
57                         // remember actual identifier positions
58                         completedIdentifierStart = startPosition;
59                         completedIdentifierEnd = currentPosition - 1;
60                         if (withoutUnicodePtr != 0){                    // check unicode scenario
61                                 System.arraycopy(withoutUnicodeBuffer, 1, completionIdentifier = new char[withoutUnicodePtr], 0, withoutUnicodePtr);
62                         } else {
63                                 int length = cursorLocation + 1 - startPosition;
64                                 // no char[] sharing around completionIdentifier, we want it to be unique so as to use identity checks  
65                                 System.arraycopy(source, startPosition, (completionIdentifier = new char[length]), 0, length);
66                         }
67                         return completionIdentifier;
68                 }
69         }
70         return super.getCurrentIdentifierSource();
71 }
72 /* 
73  * Identifier splitting for unicodes.
74  * Only store the current unicode if we did not pass the cursorLocation.
75  * Note: this does not handle cases where the cursor is in the middle of a unicode
76  */
77 public boolean getNextCharAsJavaIdentifierPart() {
78
79         int temp = currentPosition;
80         try {
81                 if (((currentCharacter = source[currentPosition++]) == '\\')
82                         && (source[currentPosition] == 'u')) {
83                         //-------------unicode traitement ------------
84                         int c1, c2, c3, c4;
85                         int unicodeSize = 6;
86                         currentPosition++;
87                         while (source[currentPosition] == 'u') {
88                                 currentPosition++;
89                                 unicodeSize++;
90                         }
91
92                         if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
93                                 || c1 < 0)
94                                 || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15 || c2 < 0)
95                                 || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15 || c3 < 0)
96                                 || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15 || c4 < 0)) {
97                                 currentPosition = temp;
98                                 return false;
99                         }
100
101                         currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
102                         if (!Character.isJavaIdentifierPart(currentCharacter)) {
103                                 currentPosition = temp;
104                                 return false;
105                         }
106
107                         //need the unicode buffer
108                         if (withoutUnicodePtr == 0) {
109                                 //buffer all the entries that have been left aside....
110                                 withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
111                                 System.arraycopy(
112                                         source, 
113                                         startPosition, 
114                                         withoutUnicodeBuffer, 
115                                         1, 
116                                         withoutUnicodePtr); 
117                         }
118                         if (temp < cursorLocation && cursorLocation < currentPosition-1){
119                                 throw new InvalidCursorLocation(InvalidCursorLocation.NO_COMPLETION_INSIDE_UNICODE);
120                         }
121                         // store the current unicode, only if we did not pass the cursorLocation
122                         // Note: this does not handle cases where the cursor is in the middle of a unicode
123                         if ((completionIdentifier != null)
124                                 || (startPosition <= cursorLocation+1 && cursorLocation >= currentPosition-1)){
125                                 withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
126                         }
127                         return true;
128                 } //-------------end unicode traitement--------------
129                 else {
130                         if (!Character.isJavaIdentifierPart(currentCharacter)) {
131                                 currentPosition = temp;
132                                 return false;
133                         }
134
135                         if (withoutUnicodePtr != 0){
136                                 // store the current unicode, only if we did not pass the cursorLocation
137                                 // Note: this does not handle cases where the cursor is in the middle of a unicode
138                                 if ((completionIdentifier != null)
139                                         || (startPosition <= cursorLocation+1 && cursorLocation >= currentPosition-1)){
140                                         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
141                                 }
142                         }
143                         return true;
144                 }
145         } catch (IndexOutOfBoundsException e) {
146                 currentPosition = temp;
147                 return false;
148         }
149 }
150 public int getNextToken() throws InvalidInputException {
151
152         this.wasAcr = false;
153         if (diet) {
154                 jumpOverMethodBody();
155                 diet = false;
156                 return currentPosition > source.length ? TokenNameEOF : TokenNameRBRACE;
157         }
158         try {
159                 while (true) { //loop for jumping over comments
160                         withoutUnicodePtr = 0;
161                         //start with a new token (even comment written with unicode )
162
163                         // ---------Consume white space and handles startPosition---------
164                         int whiteStart = currentPosition;
165                         boolean isWhiteSpace;
166                         do {
167                                 startPosition = currentPosition;
168                                 if (((currentCharacter = source[currentPosition++]) == '\\')
169                                         && (source[currentPosition] == 'u')) {
170                                         isWhiteSpace = jumpOverUnicodeWhiteSpace();
171                                 } else {
172                                         if (recordLineSeparator
173                                                 && ((currentCharacter == '\r') || (currentCharacter == '\n')))
174                                                 pushLineSeparator();
175                                         isWhiteSpace = 
176                                                 (currentCharacter == ' ') || Character.isWhitespace(currentCharacter); 
177                                 }
178                                 /* completion requesting strictly inside blanks */
179                                 if ((whiteStart != currentPosition)
180                                         //&& (previousToken == TokenNameDOT)
181                                         && (completionIdentifier == null)
182                                         && (whiteStart <= cursorLocation+1)
183                                         && (cursorLocation < startPosition)
184                                         && !Character.isJavaIdentifierStart(currentCharacter)){
185                                         currentPosition = startPosition; // for next token read
186                                         return TokenNameIdentifier;
187                                 }
188                         } while (isWhiteSpace);
189                         if (tokenizeWhiteSpace && (whiteStart != currentPosition - 1)) {
190                                 // reposition scanner in case we are interested by spaces as tokens
191                                 currentPosition--;
192                                 startPosition = whiteStart;
193                                 return TokenNameWHITESPACE;
194                         }
195                         //little trick to get out in the middle of a source computation
196                         if (currentPosition > eofPosition){
197                                 /* might be completing at eof (e.g. behind a dot) */
198                                 if (completionIdentifier == null && 
199                                         startPosition == cursorLocation + 1){
200                                         currentPosition = startPosition; // for being detected as empty free identifier
201                                         return TokenNameIdentifier;
202                                 }                               
203                                 return TokenNameEOF;
204                         }
205
206                         // ---------Identify the next token-------------
207
208                         switch (currentCharacter) {
209                                 case '(' :
210                                         return TokenNameLPAREN;
211                                 case ')' :
212                                         return TokenNameRPAREN;
213                                 case '{' :
214                                         return TokenNameLBRACE;
215                                 case '}' :
216                                         return TokenNameRBRACE;
217                                 case '[' :
218                                         return TokenNameLBRACKET;
219                                 case ']' :
220                                         return TokenNameRBRACKET;
221                                 case ';' :
222                                         return TokenNameSEMICOLON;
223                                 case ',' :
224                                         return TokenNameCOMMA;
225                                 case '.' :
226                                         if (startPosition <= cursorLocation 
227                                             && cursorLocation < currentPosition){
228                                                 return TokenNameDOT; // completion inside .<|>12
229                                     }
230                                         if (getNextCharAsDigit())
231                                                 return scanNumber(true);
232                                         return TokenNameDOT;
233                                 case '+' :
234                                         {
235                                                 int test;
236                                                 if ((test = getNextChar('+', '=')) == 0)
237                                                         return TokenNamePLUS_PLUS;
238                                                 if (test > 0)
239                                                         return TokenNamePLUS_EQUAL;
240                                                 return TokenNamePLUS;
241                                         }
242                                 case '-' :
243                                         {
244                                                 int test;
245                                                 if ((test = getNextChar('-', '=')) == 0)
246                                                         return TokenNameMINUS_MINUS;
247                                                 if (test > 0)
248                                                         return TokenNameMINUS_EQUAL;
249                                                 return TokenNameMINUS;
250                                         }
251                                 case '~' :
252                                         return TokenNameTWIDDLE;
253                                 case '!' :
254                                         if (getNextChar('='))
255                                                 return TokenNameNOT_EQUAL;
256                                         return TokenNameNOT;
257                                 case '*' :
258                                         if (getNextChar('='))
259                                                 return TokenNameMULTIPLY_EQUAL;
260                                         return TokenNameMULTIPLY;
261                                 case '%' :
262                                         if (getNextChar('='))
263                                                 return TokenNameREMAINDER_EQUAL;
264                                         return TokenNameREMAINDER;
265                                 case '<' :
266                                         {
267                                                 int test;
268                                                 if ((test = getNextChar('=', '<')) == 0)
269                                                         return TokenNameLESS_EQUAL;
270                                                 if (test > 0) {
271                                                         if (getNextChar('='))
272                                                                 return TokenNameLEFT_SHIFT_EQUAL;
273                                                         return TokenNameLEFT_SHIFT;
274                                                 }
275                                                 return TokenNameLESS;
276                                         }
277                                 case '>' :
278                                         {
279                                                 int test;
280                                                 if ((test = getNextChar('=', '>')) == 0)
281                                                         return TokenNameGREATER_EQUAL;
282                                                 if (test > 0) {
283                                                         if ((test = getNextChar('=', '>')) == 0)
284                                                                 return TokenNameRIGHT_SHIFT_EQUAL;
285                                                         if (test > 0) {
286                                                                 if (getNextChar('='))
287                                                                         return TokenNameUNSIGNED_RIGHT_SHIFT_EQUAL;
288                                                                 return TokenNameUNSIGNED_RIGHT_SHIFT;
289                                                         }
290                                                         return TokenNameRIGHT_SHIFT;
291                                                 }
292                                                 return TokenNameGREATER;
293                                         }
294                                 case '=' :
295                                         if (getNextChar('='))
296                                                 return TokenNameEQUAL_EQUAL;
297                                         return TokenNameEQUAL;
298                                 case '&' :
299                                         {
300                                                 int test;
301                                                 if ((test = getNextChar('&', '=')) == 0)
302                                                         return TokenNameAND_AND;
303                                                 if (test > 0)
304                                                         return TokenNameAND_EQUAL;
305                                                 return TokenNameAND;
306                                         }
307                                 case '|' :
308                                         {
309                                                 int test;
310                                                 if ((test = getNextChar('|', '=')) == 0)
311                                                         return TokenNameOR_OR;
312                                                 if (test > 0)
313                                                         return TokenNameOR_EQUAL;
314                                                 return TokenNameOR;
315                                         }
316                                 case '^' :
317                                         if (getNextChar('='))
318                                                 return TokenNameXOR_EQUAL;
319                                         return TokenNameXOR;
320                                 case '?' :
321                                         return TokenNameQUESTION;
322                                 case ':' :
323                                         return TokenNameCOLON;
324                                 case '\'' :
325                                         {
326                                                 int test;
327                                                 if ((test = getNextChar('\n', '\r')) == 0) {
328                                                         throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
329                                                 }
330                                                 if (test > 0) {
331                                                         // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
332                                                         for (int lookAhead = 0; lookAhead < 3; lookAhead++) {
333                                                                 if (currentPosition + lookAhead == source.length)
334                                                                         break;
335                                                                 if (source[currentPosition + lookAhead] == '\n')
336                                                                         break;
337                                                                 if (source[currentPosition + lookAhead] == '\'') {
338                                                                         currentPosition += lookAhead + 1;
339                                                                         break;
340                                                                 }
341                                                         }
342                                                         throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
343                                                 }
344                                         }
345                                         if (getNextChar('\'')) {
346                                                 // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
347                                                 for (int lookAhead = 0; lookAhead < 3; lookAhead++) {
348                                                         if (currentPosition + lookAhead == source.length)
349                                                                 break;
350                                                         if (source[currentPosition + lookAhead] == '\n')
351                                                                 break;
352                                                         if (source[currentPosition + lookAhead] == '\'') {
353                                                                 currentPosition += lookAhead + 1;
354                                                                 break;
355                                                         }
356                                                 }
357                                                 throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
358                                         }
359                                         if (getNextChar('\\'))
360                                                 scanEscapeCharacter();
361                                         else { // consume next character
362                                                 unicodeAsBackSlash = false;
363                                                 if (((currentCharacter = source[currentPosition++]) == '\\')
364                                                         && (source[currentPosition] == 'u')) {
365                                                         getNextUnicodeChar();
366                                                 } else {
367                                                         if (withoutUnicodePtr != 0) {
368                                                                 withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
369                                                         }
370                                                 }
371                                         }
372                                         if (getNextChar('\''))
373                                                 return TokenNameCharacterLiteral;
374                                         // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
375                                         for (int lookAhead = 0; lookAhead < 20; lookAhead++) {
376                                                 if (currentPosition + lookAhead == source.length)
377                                                         break;
378                                                 if (source[currentPosition + lookAhead] == '\n')
379                                                         break;
380                                                 if (source[currentPosition + lookAhead] == '\'') {
381                                                         currentPosition += lookAhead + 1;
382                                                         break;
383                                                 }
384                                         }
385                                         throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
386                                 case '"' :
387                                         try {
388                                                 // consume next character
389                                                 unicodeAsBackSlash = false;
390                                                 if (((currentCharacter = source[currentPosition++]) == '\\')
391                                                         && (source[currentPosition] == 'u')) {
392                                                         getNextUnicodeChar();
393                                                 } else {
394                                                         if (withoutUnicodePtr != 0) {
395                                                                 withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
396                                                         }
397                                                 }
398
399                                                 while (currentCharacter != '"') {
400                                                         /**** \r and \n are not valid in string literals ****/
401                                                         if ((currentCharacter == '\n') || (currentCharacter == '\r')) {
402                                                                 // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
403                                                                 for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
404                                                                         if (currentPosition + lookAhead == source.length)
405                                                                                 break;
406                                                                         if (source[currentPosition + lookAhead] == '\n')
407                                                                                 break;
408                                                                         if (source[currentPosition + lookAhead] == '\"') {
409                                                                                 currentPosition += lookAhead + 1;
410                                                                                 break;
411                                                                         }
412                                                                 }
413                                                                 throw new InvalidInputException(INVALID_CHAR_IN_STRING);
414                                                         }
415                                                         if (currentCharacter == '\\') {
416                                                                 int escapeSize = currentPosition;
417                                                                 boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
418                                                                 //scanEscapeCharacter make a side effect on this value and we need the previous value few lines down this one
419                                                                 scanEscapeCharacter();
420                                                                 escapeSize = currentPosition - escapeSize;
421                                                                 if (withoutUnicodePtr == 0) {
422                                                                         //buffer all the entries that have been left aside....
423                                                                         withoutUnicodePtr = currentPosition - escapeSize - 1 - startPosition;
424                                                                         System.arraycopy(
425                                                                                 source, 
426                                                                                 startPosition, 
427                                                                                 withoutUnicodeBuffer, 
428                                                                                 1, 
429                                                                                 withoutUnicodePtr); 
430                                                                         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
431                                                                 } else { //overwrite the / in the buffer
432                                                                         withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter;
433                                                                         if (backSlashAsUnicodeInString) { //there are TWO \ in the stream where only one is correct
434                                                                                 withoutUnicodePtr--;
435                                                                         }
436                                                                 }
437                                                         }
438                                                         // consume next character
439                                                         unicodeAsBackSlash = false;
440                                                         if (((currentCharacter = source[currentPosition++]) == '\\')
441                                                                 && (source[currentPosition] == 'u')) {
442                                                                 getNextUnicodeChar();
443                                                         } else {
444                                                                 if (withoutUnicodePtr != 0) {
445                                                                         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
446                                                                 }
447                                                         }
448
449                                                 }
450                                         } catch (IndexOutOfBoundsException e) {
451                                                 throw new InvalidInputException(UNTERMINATED_STRING);
452                                         } catch (InvalidInputException e) {
453                                                 if (e.getMessage().equals(INVALID_ESCAPE)) {
454                                                         // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
455                                                         for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
456                                                                 if (currentPosition + lookAhead == source.length)
457                                                                         break;
458                                                                 if (source[currentPosition + lookAhead] == '\n')
459                                                                         break;
460                                                                 if (source[currentPosition + lookAhead] == '\"') {
461                                                                         currentPosition += lookAhead + 1;
462                                                                         break;
463                                                                 }
464                                                         }
465
466                                                 }
467                                                 throw e; // rethrow
468                                         }
469                                         if (startPosition <= cursorLocation && cursorLocation <= currentPosition-1){
470                                                 throw new InvalidCursorLocation(InvalidCursorLocation.NO_COMPLETION_INSIDE_STRING);
471                                         }
472                                         return TokenNameStringLiteral;
473                                 case '/' :
474                                         {
475                                                 int test;
476                                                 if ((test = getNextChar('/', '*')) == 0) { //line comment 
477                                                         try { //get the next char 
478                                                                 if (((currentCharacter = source[currentPosition++]) == '\\')
479                                                                         && (source[currentPosition] == 'u')) {
480                                                                         //-------------unicode traitement ------------
481                                                                         int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
482                                                                         currentPosition++;
483                                                                         while (source[currentPosition] == 'u') {
484                                                                                 currentPosition++;
485                                                                         }
486                                                                         if ((c1 = Character.getNumericValue(source[currentPosition++])) > 15
487                                                                                 || c1 < 0
488                                                                                 || (c2 = Character.getNumericValue(source[currentPosition++])) > 15
489                                                                                 || c2 < 0
490                                                                                 || (c3 = Character.getNumericValue(source[currentPosition++])) > 15
491                                                                                 || c3 < 0
492                                                                                 || (c4 = Character.getNumericValue(source[currentPosition++])) > 15
493                                                                                 || c4 < 0) {
494                                                                                 throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
495                                                                         } else {
496                                                                                 currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
497                                                                         }
498                                                                 }
499
500                                                                 //handle the \\u case manually into comment
501                                                                 if (currentCharacter == '\\') {
502                                                                         if (source[currentPosition] == '\\')
503                                                                                 currentPosition++;
504                                                                 } //jump over the \\
505                                                                 while (currentCharacter != '\r' && currentCharacter != '\n') {
506                                                                         //get the next char 
507                                                                         if (((currentCharacter = source[currentPosition++]) == '\\')
508                                                                                 && (source[currentPosition] == 'u')) {
509                                                                                 //-------------unicode traitement ------------
510                                                                                 int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
511                                                                                 currentPosition++;
512                                                                                 while (source[currentPosition] == 'u') {
513                                                                                         currentPosition++;
514                                                                                 }
515                                                                                 if ((c1 = Character.getNumericValue(source[currentPosition++])) > 15
516                                                                                         || c1 < 0
517                                                                                         || (c2 = Character.getNumericValue(source[currentPosition++])) > 15
518                                                                                         || c2 < 0
519                                                                                         || (c3 = Character.getNumericValue(source[currentPosition++])) > 15
520                                                                                         || c3 < 0
521                                                                                         || (c4 = Character.getNumericValue(source[currentPosition++])) > 15
522                                                                                         || c4 < 0) {
523                                                                                         throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
524                                                                                 } else {
525                                                                                         currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
526                                                                                 }
527                                                                         }
528                                                                         //handle the \\u case manually into comment
529                                                                         if (currentCharacter == '\\') {
530                                                                                 if (source[currentPosition] == '\\')
531                                                                                         currentPosition++;
532                                                                         } //jump over the \\
533                                                                 }
534                                                                 recordComment(false);
535                                                                 if (startPosition <= cursorLocation && cursorLocation < currentPosition-1){
536                                                                         throw new InvalidCursorLocation(InvalidCursorLocation.NO_COMPLETION_INSIDE_COMMENT);
537                                                                 }
538                                                                 if (recordLineSeparator
539                                                                         && ((currentCharacter == '\r') || (currentCharacter == '\n')))
540                                                                         pushLineSeparator();
541                                                                 if (tokenizeComments) {
542                                                                         currentPosition--; // reset one character behind
543                                                                         return TokenNameCOMMENT_LINE;
544                                                                 }
545                                                         } catch (IndexOutOfBoundsException e) { //an eof will them be generated
546                                                                 if (tokenizeComments) {
547                                                                         currentPosition--; // reset one character behind
548                                                                         return TokenNameCOMMENT_LINE;
549                                                                 }
550                                                         }
551                                                         break;
552                                                 }
553                                                 if (test > 0) { //traditional and annotation comment
554                                                         boolean isJavadoc = false, star = false;
555                                                         // consume next character
556                                                         unicodeAsBackSlash = false;
557                                                         if (((currentCharacter = source[currentPosition++]) == '\\')
558                                                                 && (source[currentPosition] == 'u')) {
559                                                                 getNextUnicodeChar();
560                                                         } else {
561                                                                 if (withoutUnicodePtr != 0) {
562                                                                         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
563                                                                 }
564                                                         }
565
566                                                         if (currentCharacter == '*') {
567                                                                 isJavadoc = true;
568                                                                 star = true;
569                                                         }
570                                                         if (recordLineSeparator
571                                                                 && ((currentCharacter == '\r') || (currentCharacter == '\n')))
572                                                                 pushLineSeparator();
573                                                         try { //get the next char 
574                                                                 if (((currentCharacter = source[currentPosition++]) == '\\')
575                                                                         && (source[currentPosition] == 'u')) {
576                                                                         //-------------unicode traitement ------------
577                                                                         int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
578                                                                         currentPosition++;
579                                                                         while (source[currentPosition] == 'u') {
580                                                                                 currentPosition++;
581                                                                         }
582                                                                         if ((c1 = Character.getNumericValue(source[currentPosition++])) > 15
583                                                                                 || c1 < 0
584                                                                                 || (c2 = Character.getNumericValue(source[currentPosition++])) > 15
585                                                                                 || c2 < 0
586                                                                                 || (c3 = Character.getNumericValue(source[currentPosition++])) > 15
587                                                                                 || c3 < 0
588                                                                                 || (c4 = Character.getNumericValue(source[currentPosition++])) > 15
589                                                                                 || c4 < 0) {
590                                                                                 throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
591                                                                         } else {
592                                                                                 currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
593                                                                         }
594                                                                 }
595                                                                 //handle the \\u case manually into comment
596                                                                 if (currentCharacter == '\\') {
597                                                                         if (source[currentPosition] == '\\')
598                                                                                 currentPosition++;
599                                                                 } //jump over the \\
600                                                                 // empty comment is not a javadoc /**/
601                                                                 if (currentCharacter == '/') { 
602                                                                         isJavadoc = false;
603                                                                 }
604                                                                 //loop until end of comment */ 
605                                                                 while ((currentCharacter != '/') || (!star)) {
606                                                                         if (recordLineSeparator
607                                                                                 && ((currentCharacter == '\r') || (currentCharacter == '\n')))
608                                                                                 pushLineSeparator();
609                                                                         star = currentCharacter == '*';
610                                                                         //get next char
611                                                                         if (((currentCharacter = source[currentPosition++]) == '\\')
612                                                                                 && (source[currentPosition] == 'u')) {
613                                                                                 //-------------unicode traitement ------------
614                                                                                 int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
615                                                                                 currentPosition++;
616                                                                                 while (source[currentPosition] == 'u') {
617                                                                                         currentPosition++;
618                                                                                 }
619                                                                                 if ((c1 = Character.getNumericValue(source[currentPosition++])) > 15
620                                                                                         || c1 < 0
621                                                                                         || (c2 = Character.getNumericValue(source[currentPosition++])) > 15
622                                                                                         || c2 < 0
623                                                                                         || (c3 = Character.getNumericValue(source[currentPosition++])) > 15
624                                                                                         || c3 < 0
625                                                                                         || (c4 = Character.getNumericValue(source[currentPosition++])) > 15
626                                                                                         || c4 < 0) {
627                                                                                         throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
628                                                                                 } else {
629                                                                                         currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
630                                                                                 }
631                                                                         }
632                                                                         //handle the \\u case manually into comment
633                                                                         if (currentCharacter == '\\') {
634                                                                                 if (source[currentPosition] == '\\')
635                                                                                         currentPosition++;
636                                                                         } //jump over the \\
637                                                                 }
638                                                                 recordComment(isJavadoc);
639                                                                 if (startPosition <= cursorLocation && cursorLocation < currentPosition-1){
640                                                                         throw new InvalidCursorLocation(InvalidCursorLocation.NO_COMPLETION_INSIDE_COMMENT);
641                                                                 }
642                                                                 if (tokenizeComments) {
643                                                                         if (isJavadoc)
644                                                                                 return TokenNameCOMMENT_JAVADOC;
645                                                                         return TokenNameCOMMENT_BLOCK;
646                                                                 }
647                                                         } catch (IndexOutOfBoundsException e) {
648                                                                 throw new InvalidInputException(UNTERMINATED_COMMENT);
649                                                         }
650                                                         break;
651                                                 }
652                                                 if (getNextChar('='))
653                                                         return TokenNameDIVIDE_EQUAL;
654                                                 return TokenNameDIVIDE;
655                                         }
656                                 case '\u001a' :
657                                         if (atEnd())
658                                                 return TokenNameEOF;
659                                         //the atEnd may not be <currentPosition == source.length> if source is only some part of a real (external) stream
660                                         throw new InvalidInputException("Ctrl-Z"); //$NON-NLS-1$
661
662                                 default :
663                                         if (Character.isJavaIdentifierStart(currentCharacter))
664                                                 return scanIdentifierOrKeyword();
665                                         if (Character.isDigit(currentCharacter))
666                                                 return scanNumber(false);
667                                         return TokenNameERROR;
668                         }
669                 }
670         } //-----------------end switch while try--------------------
671         catch (IndexOutOfBoundsException e) {
672         }
673         /* might be completing at very end of file (e.g. behind a dot) */
674         if (completionIdentifier == null && 
675                 startPosition == cursorLocation + 1){
676                 currentPosition = startPosition; // for being detected as empty free identifier
677                 return TokenNameIdentifier;
678         }
679         return TokenNameEOF;
680 }
681 /*
682  * In case we actually read a keyword, but the cursor is located inside,
683  * we pretend we read an identifier.
684  */
685 public int scanIdentifierOrKeyword() throws InvalidInputException {
686
687         int id = super.scanIdentifierOrKeyword();
688
689         // convert completed keyword into an identifier
690         if (id != TokenNameIdentifier
691                 && startPosition <= cursorLocation+1 
692                 && cursorLocation < currentPosition){
693                 return TokenNameIdentifier;
694         }
695         return id;
696 }
697 public int scanNumber(boolean dotPrefix) throws InvalidInputException {
698         
699         int token = super.scanNumber(dotPrefix);
700
701         // consider completion just before a number to be ok, will insert before it
702         if (startPosition <= cursorLocation && cursorLocation < currentPosition){  
703                 throw new InvalidCursorLocation(InvalidCursorLocation.NO_COMPLETION_INSIDE_NUMBER);
704         }
705         return token;
706 }
707 }