41246b694685e3da4c86d200269412e78c026ee4
[phpeclipse.git] / net.sourceforge.phpeclipse / src / net / sourceforge / phpdt / internal / ui / text / FastJavaPartitionScanner.java
1 package net.sourceforge.phpdt.internal.ui.text;
2
3 /*
4  * (c) Copyright IBM Corp. 2000, 2001.
5  * All Rights Reserved.
6  */
7
8 import org.eclipse.jface.text.IDocument;
9 import org.eclipse.jface.text.rules.ICharacterScanner;
10 import org.eclipse.jface.text.rules.IPartitionTokenScanner;
11 import org.eclipse.jface.text.rules.IToken;
12 import org.eclipse.jface.text.rules.Token;
13
14
15 /**
16  * This scanner recognizes the JavaDoc comments, Java multi line comments, Java single line comments,
17  * Java strings and Java characters.
18  */
19 public class FastJavaPartitionScanner implements IPartitionTokenScanner {
20
21         private final static String SKIP= "__skip"; //$NON-NLS-1$       
22         public final static String JAVA_STRING= "__php_string"; //$NON-NLS-1$
23         public final static String JAVA_SINGLE_LINE_COMMENT= "__php_singleline_comment"; //$NON-NLS-1$
24         public final static String JAVA_MULTI_LINE_COMMENT= "__php_multiline_comment"; //$NON-NLS-1$
25         public final static String JAVA_DOC= "__php_phpdoc"; //$NON-NLS-1$
26
27         // states
28         private static final int JAVA= 0;       
29         private static final int SINGLE_LINE_COMMENT= 1;
30         private static final int MULTI_LINE_COMMENT= 2;
31         private static final int JAVADOC= 3;
32         private static final int CHARACTER= 4;
33         private static final int STRING= 5;
34         
35         // beginning of prefixes and postfixes
36         private static final int NONE= 0;
37         private static final int BACKSLASH= 1; // postfix for STRING and CHARACTER
38         private static final int SLASH= 2; // prefix for SINGLE_LINE or MULTI_LINE or JAVADOC
39         private static final int SLASH_STAR= 3; // prefix for MULTI_LINE_COMMENT or JAVADOC
40         private static final int SLASH_STAR_STAR= 4; // prefix for MULTI_LINE_COMMENT or JAVADOC
41         private static final int STAR= 5; // postfix for MULTI_LINE_COMMENT or JAVADOC
42         private static final int CARRIAGE_RETURN=6; // postfix for STRING, CHARACTER and SINGLE_LINE_COMMENT
43         
44         /** The scanner. */
45 //      private final BufferedRuleBasedScanner fScanner= new BufferedRuleBasedScanner(1000);
46         private final BufferedDocumentScanner fScanner= new BufferedDocumentScanner(1000);      // faster implementation
47         
48         /** The offset of the last returned token. */
49         private int fTokenOffset;
50         /** The length of the last returned token. */
51         private int fTokenLength;
52         
53         /** The state of the scanner. */        
54         private int fState;
55         /** The last significant characters read. */
56         private int fLast;
57         /** The amount of characters already read on first call to nextToken(). */
58         private int fPrefixLength;
59         
60         // emulate JavaPartitionScanner
61         private static final boolean fgEmulate= false;
62         private int fJavaOffset;
63         private int fJavaLength;
64         
65         private final IToken[] fTokens= new IToken[] {
66                 new Token(null),
67                 new Token(JAVA_SINGLE_LINE_COMMENT),
68                 new Token(JAVA_MULTI_LINE_COMMENT),
69                 new Token(JAVA_DOC),
70                 new Token(SKIP),
71                 new Token(JAVA_STRING)
72         };
73
74         /*
75          * @see org.eclipse.jface.text.rules.ITokenScanner#nextToken()
76          */
77         public IToken nextToken() {
78                 
79                 // emulate JavaPartitionScanner
80                 if (fgEmulate) {
81                         if (fJavaOffset != -1 && fTokenOffset + fTokenLength != fJavaOffset + fJavaLength) {
82                                 fTokenOffset += fTokenLength;           
83                                 return fTokens[JAVA];   
84                         } else {
85                                 fJavaOffset= -1;
86                                 fJavaLength= 0; 
87                         }
88                 }               
89
90                 fTokenOffset += fTokenLength;
91                 fTokenLength= fPrefixLength;
92
93                 while (true) {
94                         final int ch= fScanner.read();
95                         
96                         // characters
97                         switch (ch) {
98                         case ICharacterScanner.EOF:
99                                 if (fTokenLength > 0) {
100                                         fLast= NONE; // ignore last
101                                         return preFix(fState, JAVA, NONE, 0);
102
103                                 } else {
104                                         fLast= NONE;
105                                         fPrefixLength= 0;
106                                         return Token.EOF;
107                                 }
108
109                         case '\r':
110                                 // emulate JavaPartitionScanner
111                                 if (!fgEmulate && fLast != CARRIAGE_RETURN) {
112                                                 fLast= CARRIAGE_RETURN;
113                                                 fTokenLength++;
114                                                 continue;
115
116                                 } else {
117                                         
118                                         switch (fState) {
119                                         case SINGLE_LINE_COMMENT:
120                                         case CHARACTER:
121                                         case STRING:
122                                                 if (fTokenLength > 0) {
123                                                         IToken token= fTokens[fState];
124                                                         
125                                                         // emulate JavaPartitionScanner
126                                                         if (fgEmulate) {
127                                                                 fTokenLength++;
128                                                                 fLast= NONE;
129                                                                 fPrefixLength= 0;
130                                                         } else {                                                                
131                                                                 fLast= CARRIAGE_RETURN; 
132                                                                 fPrefixLength= 1;
133                                                         }
134                                                         
135                                                         fState= JAVA;
136                                                         return token;
137
138                                                 } else {
139                                                         consume();
140                                                         continue;       
141                                                 }
142
143                                         default:
144                                                 consume();
145                                                 continue;
146                                         }                                       
147                                 }
148         
149                         case '\n':                              
150                                 switch (fState) {
151                                 case SINGLE_LINE_COMMENT:
152                                 case CHARACTER:
153                                 case STRING:                            
154                                         // assert(fTokenLength > 0);
155                                         return postFix(fState);
156
157                                 default:
158                                         consume();
159                                         continue;
160                                 }
161
162                         default:
163                                 if (!fgEmulate && fLast == CARRIAGE_RETURN) {                   
164                                         switch (fState) {
165                                         case SINGLE_LINE_COMMENT:
166                                         case CHARACTER:
167                                         case STRING:
168
169                                                 int last;
170                                                 int newState;
171                                                 switch (ch) {
172                                                 case '/':
173                                                         last= SLASH;
174                                                         newState= JAVA;
175                                                         break;
176
177                                                 case '*':
178                                                         last= STAR;
179                                                         newState= JAVA;
180                                                         break;
181                                                 
182                                                 case '\'':
183                                                         last= NONE;
184                                                         newState= CHARACTER;
185                                                         break;
186
187                                                 case '"':
188                                                         last= NONE;
189                                                         newState= STRING;
190                                                         break;
191
192                                                 case '\r':
193                                                         last= CARRIAGE_RETURN;
194                                                         newState= JAVA;
195                                                         break;
196
197                                                 case '\\':
198                                                         last= BACKSLASH;
199                                                         newState= JAVA;
200                                                         break;
201
202                                                 default:
203                                                         last= NONE;
204                                                         newState= JAVA;
205                                                         break;
206                                                 }
207                                                 
208                                                 fLast= NONE; // ignore fLast
209                                                 return preFix(fState, newState, last, 1);
210         
211                                         default:
212                                                 break;
213                                         }
214                                 }
215                         }
216
217                         // states        
218                         switch (fState) {
219                         case JAVA:
220                                 switch (ch) {
221                                 case '/':
222                                         if (fLast == SLASH) {
223                                                 if (fTokenLength - getLastLength(fLast) > 0) {
224                                                         return preFix(JAVA, SINGLE_LINE_COMMENT, NONE, 2);
225                                                 } else {                                                        
226                                                         preFix(JAVA, SINGLE_LINE_COMMENT, NONE, 2);
227                                                         fTokenOffset += fTokenLength;
228                                                         fTokenLength= fPrefixLength;
229                                                         break;
230                                                 }
231         
232                                         } else {
233                                                 fTokenLength++;
234                                                 fLast= SLASH;
235                                                 break;
236                                         }
237         
238                                 case '*':
239                                         if (fLast == SLASH) {
240                                                 if (fTokenLength - getLastLength(fLast) > 0)
241                                                         return preFix(JAVA, MULTI_LINE_COMMENT, SLASH_STAR, 2);
242                                                 else {
243                                                         preFix(JAVA, MULTI_LINE_COMMENT, SLASH_STAR, 2);
244                                                         fTokenOffset += fTokenLength;
245                                                         fTokenLength= fPrefixLength;
246                                                         break;
247                                                 }
248
249                                         } else {
250                                                 consume();
251                                                 break;
252                                         }
253                                         
254                                 case '\'':
255                                         fLast= NONE; // ignore fLast
256                                         if (fTokenLength > 0)
257                                                 return preFix(JAVA, CHARACTER, NONE, 1);
258                                         else {                                          
259                                                 preFix(JAVA, CHARACTER, NONE, 1);
260                                                 fTokenOffset += fTokenLength;
261                                                 fTokenLength= fPrefixLength;
262                                                 break;
263                                         }
264
265                                 case '"':
266                                         fLast= NONE; // ignore fLast                            
267                                         if (fTokenLength > 0)
268                                                 return preFix(JAVA, STRING, NONE, 1);
269                                         else {
270                                                 preFix(JAVA, STRING, NONE, 1);
271                                                 fTokenOffset += fTokenLength;
272                                                 fTokenLength= fPrefixLength;
273                                                 break;
274                                         }
275         
276                                 default:
277                                         consume();
278                                         break;
279                                 }
280                                 break;
281         
282                         case SINGLE_LINE_COMMENT:
283                                 consume();
284                                 break;
285                                 
286                         case JAVADOC:
287                                 switch (ch) {
288                                 case '/':
289                                         switch (fLast) {
290                                         case SLASH_STAR_STAR:
291                                                 return postFix(MULTI_LINE_COMMENT);
292         
293                                         case STAR:
294                                                 return postFix(JAVADOC);
295
296                                         default:
297                                                 consume();
298                                                 break;
299                                         }
300                                         break;
301
302                                 case '*':
303                                         fTokenLength++;
304                                         fLast= STAR;
305                                         break;
306
307                                 default:
308                                         consume();
309                                         break;
310                                 }
311                                 break;
312         
313                         case MULTI_LINE_COMMENT:
314                                 switch (ch) {
315                                 case '*':
316                                         if (fLast == SLASH_STAR) {
317                                                 fLast= SLASH_STAR_STAR;
318                                                 fTokenLength++;
319                                                 fState= JAVADOC;
320                                         } else {
321                                                 fTokenLength++;
322                                                 fLast= STAR;
323                                         }
324                                         break;
325         
326                                 case '/':
327                                         if (fLast == STAR) {
328                                                 return postFix(MULTI_LINE_COMMENT);
329                                         } else {
330                                                 consume();
331                                                 break;
332                                         }
333         
334                                 default:
335                                         consume();
336                                         break;                  
337                                 }
338                                 break;
339                                 
340                         case STRING:
341                                 switch (ch) {
342                                 case '\\':
343                                         fLast= (fLast == BACKSLASH) ? NONE : BACKSLASH;
344                                         fTokenLength++;
345                                         break;
346                                         
347                                 case '\"':                                                      
348                                         if (fLast != BACKSLASH) {
349                                                 return postFix(STRING);
350
351                                         } else {
352                                                 consume();
353                                                 break;                                  
354                                         }
355                                 
356                                 default:
357                                         consume();
358                                         break;
359                                 }
360                                 break;
361         
362                         case CHARACTER:
363                                 switch (ch) {
364                                 case '\\':
365                                         fLast= (fLast == BACKSLASH) ? NONE : BACKSLASH;
366                                         fTokenLength++;
367                                         break;
368         
369                                 case '\'':
370                                         if (fLast != BACKSLASH) {
371                                                 return postFix(CHARACTER);
372         
373                                         } else {
374                                                 consume();
375                                                 break;
376                                         }
377         
378                                 default:
379                                         consume();
380                                         break;
381                                 }
382                                 break;
383                         }
384                 } 
385         }               
386
387         private static final int getLastLength(int last) {
388                 switch (last) {
389                 default:
390                         return -1;
391
392                 case NONE:
393                         return 0;
394                         
395                 case CARRIAGE_RETURN:
396                 case BACKSLASH:
397                 case SLASH:
398                 case STAR:
399                         return 1;
400
401                 case SLASH_STAR:
402                         return 2;
403
404                 case SLASH_STAR_STAR:
405                         return 3;
406                 }       
407         }
408
409         private final void consume() {
410                 fTokenLength++;
411                 fLast= NONE;    
412         }
413         
414         private final IToken postFix(int state) {
415                 fTokenLength++;
416                 fLast= NONE;
417                 fState= JAVA;
418                 fPrefixLength= 0;               
419                 return fTokens[state];
420         }
421
422         private final IToken preFix(int state, int newState, int last, int prefixLength) {
423                 // emulate JavaPartitionScanner
424                 if (fgEmulate && state == JAVA && (fTokenLength - getLastLength(fLast) > 0)) {
425                         fTokenLength -= getLastLength(fLast);
426                         fJavaOffset= fTokenOffset;
427                         fJavaLength= fTokenLength;
428                         fTokenLength= 1;
429                         fState= newState;
430                         fPrefixLength= prefixLength;
431                         fLast= last;
432                         return fTokens[state];
433
434                 } else {
435                         fTokenLength -= getLastLength(fLast);
436                         fLast= last;
437                         fPrefixLength= prefixLength;
438                         IToken token= fTokens[state];           
439                         fState= newState;
440                         return token;
441                 }
442         }
443
444         private static int getState(String contentType) {
445
446                 if (contentType == null)
447                         return JAVA;
448
449                 else if (contentType.equals(JAVA_SINGLE_LINE_COMMENT))
450                         return SINGLE_LINE_COMMENT;
451
452                 else if (contentType.equals(JAVA_MULTI_LINE_COMMENT))
453                         return MULTI_LINE_COMMENT;
454
455                 else if (contentType.equals(JAVA_DOC))
456                         return JAVADOC;
457
458                 else if (contentType.equals(JAVA_STRING))
459                         return STRING;
460
461                 else if (contentType.equals(SKIP))
462                         return CHARACTER;
463                         
464                 else
465                         return JAVA;
466         }
467
468         /*
469          * @see IPartitionTokenScanner#setPartialRange(IDocument, int, int, String, int)
470          */
471         public void setPartialRange(IDocument document, int offset, int length, String contentType, int partitionOffset) {
472
473                 fScanner.setRange(document, offset, length);
474                 fTokenOffset= partitionOffset;
475                 fTokenLength= 0;
476                 fPrefixLength= offset - partitionOffset;
477                 fLast= NONE;
478                 
479                 if (offset == partitionOffset) {
480                         // restart at beginning of partition
481                         fState= JAVA;
482                 } else {
483                         fState= getState(contentType);                  
484                 }
485
486                 // emulate JavaPartitionScanner
487                 if (fgEmulate) {
488                         fJavaOffset= -1;
489                         fJavaLength= 0;
490                 }
491         }
492
493         /*
494          * @see ITokenScanner#setRange(IDocument, int, int)
495          */
496         public void setRange(IDocument document, int offset, int length) {
497
498                 fScanner.setRange(document, offset, length);
499                 fTokenOffset= offset;
500                 fTokenLength= 0;                
501                 fPrefixLength= 0;
502                 fLast= NONE;
503                 fState= JAVA;
504
505                 // emulate JavaPartitionScanner
506                 if (fgEmulate) {
507                         fJavaOffset= -1;
508                         fJavaLength= 0;
509                 }
510         }
511
512         /*
513          * @see ITokenScanner#getTokenLength()
514          */
515         public int getTokenLength() {
516                 return fTokenLength;
517         }
518
519         /*
520          * @see ITokenScanner#getTokenOffset()
521          */
522         public int getTokenOffset() {
523                 return fTokenOffset;
524         }
525
526 }