1) Reintroduced some out commented methods. (Did break some preference settings)
[phpeclipse.git] / net.sourceforge.phpeclipse.ui / src / net / sourceforge / phpdt / internal / ui / text / FastJavaPartitionScanner.java
1 /*******************************************************************************
2  * Copyright (c) 2000, 2004 IBM Corporation and others.
3  * All rights reserved. This program and the accompanying materials
4  * are made available under the terms of the Common Public License v1.0
5  * which accompanies this distribution, and is available at
6  * http://www.eclipse.org/legal/cpl-v10.html
7  *
8  * Contributors:
9  *     IBM Corporation - initial API and implementation
10  *******************************************************************************/
11 package net.sourceforge.phpdt.internal.ui.text;
12
13 import net.sourceforge.phpeclipse.ui.text.rules.AbstractPartitioner;
14
15 //incastrix
16 //import org.eclipse.jface.text.Assert;
17 import org.eclipse.core.runtime.Assert;
18 import org.eclipse.jface.text.IDocument;
19 import org.eclipse.jface.text.rules.ICharacterScanner;
20 import org.eclipse.jface.text.rules.IPartitionTokenScanner;
21 import org.eclipse.jface.text.rules.IToken;
22 import org.eclipse.jface.text.rules.Token;
23
24 /**
25  * This scanner recognizes the JavaDoc comments, Java multi line comments, Java
26  * single line comments, Java strings.
27  */
28 public class FastJavaPartitionScanner implements IPartitionTokenScanner,
29                 IPHPPartitions {
30
31         // states
32         private static final int PHP = 0;
33
34         private static final int SINGLE_LINE_COMMENT = 1;
35
36         private static final int MULTI_LINE_COMMENT = 2;
37
38         private static final int PHPDOC = 3;
39
40         private static final int STRING_DQ = 4;
41
42         private static final int STRING_SQ = 5;
43
44         private static final int STRING_HEREDOC = 6;
45
46         // beginning of prefixes and postfixes
47         private static final int NONE = 0;
48
49         private static final int BACKSLASH = 1; // postfix for STRING_DQ and
50                                                                                         // CHARACTER
51
52         private static final int SLASH = 2; // prefix for SINGLE_LINE or MULTI_LINE
53                                                                                 // or
54
55         // JAVADOC
56
57         private static final int SLASH_STAR = 3; // prefix for MULTI_LINE_COMMENT
58                                                                                                 // or
59
60         // JAVADOC
61
62         private static final int SLASH_STAR_STAR = 4; // prefix for
63                                                                                                         // MULTI_LINE_COMMENT
64
65         // or JAVADOC
66
67         private static final int STAR = 5; // postfix for MULTI_LINE_COMMENT or
68
69         // JAVADOC
70
71         private static final int CARRIAGE_RETURN = 6; // postfix for STRING_DQ,
72
73         // CHARACTER and
74         // SINGLE_LINE_COMMENT
75
76         // private static final int HEREDOC = 7;
77
78         /** The scanner. */
79         private final BufferedDocumentScanner fScanner = new BufferedDocumentScanner(
80                         1000); // faster
81
82         // implementation
83
84         /** The offset of the last returned token. */
85         private int fTokenOffset;
86
87         /** The length of the last returned token. */
88         private int fTokenLength;
89
90         /** The state of the scanner. */
91         private int fState;
92
93         /** The last significant characters read. */
94         private int fLast;
95
96         /** The amount of characters already read on first call to nextToken(). */
97         private int fPrefixLength;
98
99         // emulate JavaPartitionScanner
100         private boolean fEmulate = false;
101
102         private int fJavaOffset;
103
104         private int fJavaLength;
105
106         private final IToken[] fTokens = new IToken[] { new Token(null),
107                         new Token(PHP_SINGLELINE_COMMENT),
108                         new Token(PHP_MULTILINE_COMMENT), new Token(PHP_PHPDOC_COMMENT),
109                         new Token(PHP_STRING_DQ), new Token(PHP_STRING_SQ),
110                         new Token(PHP_STRING_HEREDOC) };
111
112         public FastJavaPartitionScanner(boolean emulate) {
113                 fEmulate = emulate;
114         }
115
116         public FastJavaPartitionScanner() {
117                 this(false);
118         }
119
120         /*
121          * @see org.eclipse.jface.text.rules.ITokenScanner#nextToken()
122          */
123         public IToken nextToken() {
124
125                 // emulate JavaPartitionScanner
126                 if (fEmulate) {
127                         if (fJavaOffset != -1
128                                         && fTokenOffset + fTokenLength != fJavaOffset + fJavaLength) {
129                                 fTokenOffset += fTokenLength;
130                                 return fTokens[PHP];
131                         } else {
132                                 fJavaOffset = -1;
133                                 fJavaLength = 0;
134                         }
135                 }
136
137                 fTokenOffset += fTokenLength;
138                 fTokenLength = fPrefixLength;
139
140                 while (true) {
141                         final int ch = fScanner.read();
142
143                         // characters
144                         switch (ch) {
145                         case ICharacterScanner.EOF:
146                                 if (fTokenLength > 0) {
147                                         fLast = NONE; // ignore last
148                                         return preFix(fState, PHP, NONE, 0);
149
150                                 } else {
151                                         fLast = NONE;
152                                         fPrefixLength = 0;
153                                         return Token.EOF;
154                                 }
155
156                         case '\r':
157                                 // emulate JavaPartitionScanner
158                                 if (!fEmulate && fLast != CARRIAGE_RETURN) {
159                                         fLast = CARRIAGE_RETURN;
160                                         fTokenLength++;
161                                         continue;
162
163                                 } else {
164
165                                         switch (fState) {
166                                         case SINGLE_LINE_COMMENT:
167                                                 // case CHARACTER:
168                                                 // case STRING_DQ:
169                                                 // case STRING_SQ:
170                                                 if (fTokenLength > 0) {
171                                                         IToken token = fTokens[fState];
172
173                                                         // emulate JavaPartitionScanner
174                                                         if (fEmulate) {
175                                                                 fTokenLength++;
176                                                                 fLast = NONE;
177                                                                 fPrefixLength = 0;
178                                                         } else {
179                                                                 fLast = CARRIAGE_RETURN;
180                                                                 fPrefixLength = 1;
181                                                         }
182
183                                                         fState = PHP;
184                                                         return token;
185
186                                                 } else {
187                                                         consume();
188                                                         continue;
189                                                 }
190
191                                         default:
192                                                 consume();
193                                                 continue;
194                                         }
195                                 }
196
197                         case '\n':
198                                 switch (fState) {
199                                 case SINGLE_LINE_COMMENT:
200                                         // case CHARACTER:
201                                         // case STRING_DQ:
202                                         // case STRING_SQ:
203                                         // assert(fTokenLength > 0);
204                                         return postFix(fState);
205
206                                 default:
207                                         consume();
208                                         continue;
209                                 }
210
211                         case '?':
212                                 if (fState == SINGLE_LINE_COMMENT) {
213                                         int nextch = fScanner.read();
214                                         if (nextch == '>') {
215                                                 // <h1>This is an <?php # echo 'simple' ?> example.</h1>
216                                                 fTokenLength--;
217                                                 fScanner.unread();
218                                                 fScanner.unread();
219                                                 return postFix(fState);
220                                         } else {
221                                                 // bug #1404228: Crash on <?php // comment ?>
222                                                 if (nextch != ICharacterScanner.EOF) {
223                                                         fScanner.unread();
224                                                 }
225                                         }
226                                 }
227
228                         default:
229                                 if (!fEmulate && fLast == CARRIAGE_RETURN) {
230                                         switch (fState) {
231                                         case SINGLE_LINE_COMMENT:
232                                                 // case CHARACTER:
233                                                 // case STRING_DQ:
234                                                 // case STRING_SQ:
235                                                 int last;
236                                                 int newState;
237                                                 switch (ch) {
238                                                 case '/':
239                                                         last = SLASH;
240                                                         newState = PHP;
241                                                         break;
242
243                                                 case '*':
244                                                         last = STAR;
245                                                         newState = PHP;
246                                                         break;
247
248                                                 case '\'':
249                                                         last = NONE;
250                                                         newState = STRING_SQ;
251                                                         break;
252
253                                                 case '"':
254                                                         last = NONE;
255                                                         newState = STRING_DQ;
256                                                         break;
257
258                                                 case '\r':
259                                                         last = CARRIAGE_RETURN;
260                                                         newState = PHP;
261                                                         break;
262
263                                                 case '\\':
264                                                         last = BACKSLASH;
265                                                         newState = PHP;
266                                                         break;
267
268                                                 default:
269                                                         last = NONE;
270                                                         newState = PHP;
271                                                         break;
272                                                 }
273
274                                                 fLast = NONE; // ignore fLast
275                                                 return preFix(fState, newState, last, 1);
276
277                                         default:
278                                                 break;
279                                         }
280                                 }
281                         }
282
283                         // states
284                         switch (fState) {
285                         case PHP:
286                                 switch (ch) {
287                                 case '#':
288                                         if (fTokenLength > 0) {
289                                                 return preFix(PHP, SINGLE_LINE_COMMENT, NONE, 1);
290                                         } else {
291                                                 preFix(PHP, SINGLE_LINE_COMMENT, NONE, 1);
292                                                 fTokenOffset += fTokenLength;
293                                                 fTokenLength = fPrefixLength;
294                                                 break;
295                                         }
296                                 case '/':
297                                         if (fLast == SLASH) {
298                                                 if (fTokenLength - getLastLength(fLast) > 0) {
299                                                         return preFix(PHP, SINGLE_LINE_COMMENT, NONE, 2);
300                                                 } else {
301                                                         preFix(PHP, SINGLE_LINE_COMMENT, NONE, 2);
302                                                         fTokenOffset += fTokenLength;
303                                                         fTokenLength = fPrefixLength;
304                                                         break;
305                                                 }
306
307                                         } else {
308                                                 fTokenLength++;
309                                                 fLast = SLASH;
310                                                 break;
311                                         }
312
313                                 case '*':
314                                         if (fLast == SLASH) {
315                                                 if (fTokenLength - getLastLength(fLast) > 0)
316                                                         return preFix(PHP, MULTI_LINE_COMMENT, SLASH_STAR,
317                                                                         2);
318                                                 else {
319                                                         preFix(PHP, MULTI_LINE_COMMENT, SLASH_STAR, 2);
320                                                         fTokenOffset += fTokenLength;
321                                                         fTokenLength = fPrefixLength;
322                                                         break;
323                                                 }
324
325                                         } else {
326                                                 consume();
327                                                 break;
328                                         }
329
330                                 case '\'':
331                                         fLast = NONE; // ignore fLast
332                                         if (fTokenLength > 0)
333                                                 return preFix(PHP, STRING_SQ, NONE, 1);
334                                         else {
335                                                 preFix(PHP, STRING_SQ, NONE, 1);
336                                                 fTokenOffset += fTokenLength;
337                                                 fTokenLength = fPrefixLength;
338                                                 break;
339                                         }
340
341                                 case '"':
342                                         fLast = NONE; // ignore fLast
343                                         if (fTokenLength > 0)
344                                                 return preFix(PHP, STRING_DQ, NONE, 1);
345                                         else {
346                                                 preFix(PHP, STRING_DQ, NONE, 1);
347                                                 fTokenOffset += fTokenLength;
348                                                 fTokenLength = fPrefixLength;
349                                                 break;
350                                         }
351
352                                 default:
353                                         consume();
354                                         break;
355                                 }
356                                 break;
357
358                         case SINGLE_LINE_COMMENT:
359                                 consume();
360                                 break;
361
362                         case PHPDOC:
363                                 switch (ch) {
364                                 case '/':
365                                         switch (fLast) {
366                                         case SLASH_STAR_STAR:
367                                                 return postFix(MULTI_LINE_COMMENT);
368
369                                         case STAR:
370                                                 return postFix(PHPDOC);
371
372                                         default:
373                                                 consume();
374                                                 break;
375                                         }
376                                         break;
377
378                                 case '*':
379                                         fTokenLength++;
380                                         fLast = STAR;
381                                         break;
382
383                                 default:
384                                         consume();
385                                         break;
386                                 }
387                                 break;
388
389                         case MULTI_LINE_COMMENT:
390                                 switch (ch) {
391                                 case '*':
392                                         if (fLast == SLASH_STAR) {
393                                                 fLast = SLASH_STAR_STAR;
394                                                 fTokenLength++;
395                                                 fState = PHPDOC;
396                                         } else {
397                                                 fTokenLength++;
398                                                 fLast = STAR;
399                                         }
400                                         break;
401
402                                 case '/':
403                                         if (fLast == STAR) {
404                                                 return postFix(MULTI_LINE_COMMENT);
405                                         } else {
406                                                 consume();
407                                                 break;
408                                         }
409
410                                 default:
411                                         consume();
412                                         break;
413                                 }
414                                 break;
415
416                         case STRING_DQ:
417                                 switch (ch) {
418                                 case '\\':
419                                         fLast = (fLast == BACKSLASH) ? NONE : BACKSLASH;
420                                         fTokenLength++;
421                                         break;
422
423                                 case '\"':
424                                         if (fLast != BACKSLASH) {
425                                                 return postFix(STRING_DQ);
426
427                                         } else {
428                                                 consume();
429                                                 break;
430                                         }
431
432                                 default:
433                                         consume();
434                                         break;
435                                 }
436                                 break;
437                         case STRING_SQ:
438                                 switch (ch) {
439                                 case '\\':
440                                         fLast = (fLast == BACKSLASH) ? NONE : BACKSLASH;
441                                         fTokenLength++;
442                                         break;
443
444                                 case '\'':
445                                         if (fLast != BACKSLASH) {
446                                                 return postFix(STRING_SQ);
447
448                                         } else {
449                                                 consume();
450                                                 break;
451                                         }
452
453                                 default:
454                                         consume();
455                                         break;
456                                 }
457                                 break;
458                         // case CHARACTER:
459                         // switch (ch) {
460                         // case '\\':
461                         // fLast= (fLast == BACKSLASH) ? NONE : BACKSLASH;
462                         // fTokenLength++;
463                         // break;
464                         //
465                         // case '\'':
466                         // if (fLast != BACKSLASH) {
467                         // return postFix(CHARACTER);
468                         //
469                         // } else {
470                         // consume();
471                         // break;
472                         // }
473                         //
474                         // default:
475                         // consume();
476                         // break;
477                         // }
478                         // break;
479                         }
480                 }
481         }
482
483         private static final int getLastLength(int last) {
484                 switch (last) {
485                 default:
486                         return -1;
487
488                 case NONE:
489                         return 0;
490
491                 case CARRIAGE_RETURN:
492                 case BACKSLASH:
493                 case SLASH:
494                 case STAR:
495                         return 1;
496
497                 case SLASH_STAR:
498                         return 2;
499
500                 case SLASH_STAR_STAR:
501                         return 3;
502                 }
503         }
504
505         private final void consume() {
506                 fTokenLength++;
507                 fLast = NONE;
508         }
509
510         private final IToken postFix(int state) {
511                 fTokenLength++;
512                 fLast = NONE;
513                 fState = PHP;
514                 fPrefixLength = 0;
515                 return fTokens[state];
516         }
517
518         private final IToken preFix(int state, int newState, int last,
519                         int prefixLength) {
520                 // emulate JavaPartitionScanner
521                 if (fEmulate && state == PHP
522                                 && (fTokenLength - getLastLength(fLast) > 0)) {
523                         fTokenLength -= getLastLength(fLast);
524                         fJavaOffset = fTokenOffset;
525                         fJavaLength = fTokenLength;
526                         fTokenLength = 1;
527                         fState = newState;
528                         fPrefixLength = prefixLength;
529                         fLast = last;
530                         return fTokens[state];
531
532                 } else {
533                         fTokenLength -= getLastLength(fLast);
534                         fLast = last;
535                         fPrefixLength = prefixLength;
536                         IToken token = fTokens[state];
537                         fState = newState;
538                         return token;
539                 }
540         }
541
542         private static int getState(String contentType) {
543
544                 if (contentType == null)
545                         return PHP;
546
547                 else if (contentType.equals(PHP_SINGLELINE_COMMENT))
548                         return SINGLE_LINE_COMMENT;
549
550                 else if (contentType.equals(PHP_MULTILINE_COMMENT))
551                         return MULTI_LINE_COMMENT;
552
553                 else if (contentType.equals(PHP_PHPDOC_COMMENT))
554                         return PHPDOC;
555
556                 else if (contentType.equals(PHP_STRING_DQ))
557                         return STRING_DQ;
558
559                 else if (contentType.equals(PHP_STRING_SQ))
560                         return STRING_SQ;
561
562                 else if (contentType.equals(PHP_STRING_HEREDOC))
563                         return STRING_HEREDOC;
564
565                 // else if (contentType.equals(JAVA_CHARACTER))
566                 // return CHARACTER;
567
568                 else
569                         return PHP;
570         }
571
572         /*
573          * @see IPartitionTokenScanner#setPartialRange(IDocument, int, int, String,
574          *      int)
575          */
576         public void setPartialRange(IDocument document, int offset, int length,
577                         String contentType, int partitionOffset) {
578                 fScanner.setRange(document, offset, length);
579                 setRange(document, offset, length);
580                 fTokenOffset = partitionOffset;
581                 fTokenLength = 0;
582                 fPrefixLength = offset - partitionOffset;
583                 fLast = NONE;
584
585                 if (offset == partitionOffset) {
586                         // restart at beginning of partition
587                         fState = PHP;
588                 } else {
589                         fState = getState(contentType);
590                 }
591
592                 // emulate JavaPartitionScanner
593                 if (fEmulate) {
594                         fJavaOffset = -1;
595                         fJavaLength = 0;
596                 }
597         }
598
599         /*
600          * @see ITokenScanner#setRange(IDocument, int, int)
601          */
602         public void setRange(IDocument document, int offset, int length) {
603                 fScanner.setRange(document, offset, length);
604                 fTokenOffset = offset;
605                 fTokenLength = 0;
606                 fPrefixLength = 0;
607                 fLast = NONE;
608                 fState = PHP;
609
610                 // emulate JavaPartitionScanner
611                 if (fEmulate) {
612                         fJavaOffset = -1;
613                         fJavaLength = 0;
614                 }
615         }
616
617         /*
618          * @see ITokenScanner#getTokenLength()
619          */
620         public int getTokenLength() {
621                 return fTokenLength;
622         }
623
624         /*
625          * @see ITokenScanner#getTokenOffset()
626          */
627         public int getTokenOffset() {
628                 if (AbstractPartitioner.DEBUG) {
629                         Assert.isTrue(fTokenOffset >= 0, Integer.toString(fTokenOffset));
630                 }
631                 return fTokenOffset;
632         }
633
634 }