1. Fix #1589403 - wrap double/single quoted strings
[phpeclipse.git] / net.sourceforge.phpeclipse / src / net / sourceforge / phpdt / internal / compiler / parser / AbstractCommentParser.java
1 /*******************************************************************************
2  * Copyright (c) 2000, 2004 IBM Corporation and others.
3  * All rights reserved. This program and the accompanying materials 
4  * are made available under the terms of the Common Public License v1.0
5  * which accompanies this distribution, and is available at
6  * http://www.eclipse.org/legal/cpl-v10.html
7  * 
8  * Contributors:
9  *     IBM Corporation - initial API and implementation
10  *******************************************************************************/
11 package net.sourceforge.phpdt.internal.compiler.parser;
12
13 import java.util.ArrayList;
14 import java.util.List;
15
16 import net.sourceforge.phpdt.core.compiler.CharOperation;
17 import net.sourceforge.phpdt.core.compiler.ITerminalSymbols;
18 import net.sourceforge.phpdt.core.compiler.InvalidInputException;
19
20 /**
21  * Parser specialized for decoding javadoc comments
22  */
23 public abstract class AbstractCommentParser {
24
25         // recognized tags
26         public static final char[] TAG_DEPRECATED = "deprecated".toCharArray(); //$NON-NLS-1$
27
28         public static final char[] TAG_PARAM = "param".toCharArray(); //$NON-NLS-1$
29
30         public static final char[] TAG_RETURN = "return".toCharArray(); //$NON-NLS-1$
31
32         public static final char[] TAG_THROWS = "throws".toCharArray(); //$NON-NLS-1$
33
34         public static final char[] TAG_EXCEPTION = "exception".toCharArray(); //$NON-NLS-1$
35
36         public static final char[] TAG_SEE = "see".toCharArray(); //$NON-NLS-1$
37
38         public static final char[] TAG_LINK = "link".toCharArray(); //$NON-NLS-1$
39
40         public static final char[] TAG_LINKPLAIN = "linkplain".toCharArray(); //$NON-NLS-1$
41
42         public static final char[] TAG_INHERITDOC = "inheritDoc".toCharArray(); //$NON-NLS-1$
43
44         // tags expected positions
45         public final static int ORDERED_TAGS_NUMBER = 3;
46
47         public final static int PARAM_TAG_EXPECTED_ORDER = 0;
48
49         public final static int THROWS_TAG_EXPECTED_ORDER = 1;
50
51         public final static int SEE_TAG_EXPECTED_ORDER = 2;
52
53         // Kind of comment parser
54         public final static int COMPIL_PARSER = 0x00000001;
55
56         public final static int DOM_PARSER = 0x00000002;
57
58         // Public fields
59         public Scanner scanner;
60
61         public boolean checkDocComment = false;
62
63         // Protected fields
64         protected boolean inherited, deprecated;
65
66         protected char[] source;
67
68         protected int index, endComment, lineEnd;
69
70         protected int tokenPreviousPosition, lastIdentifierEndPosition,
71                         starPosition;
72
73         protected int textStart, memberStart;
74
75         protected int tagSourceStart, tagSourceEnd;
76
77         protected int inlineTagStart;
78
79         protected Parser sourceParser;
80
81         protected Object returnStatement;
82
83         protected boolean lineStarted = false, inlineTagStarted = false;
84
85         protected int kind;
86
87         protected int[] lineEnds;
88
89         // Private fields
90         private int currentTokenType = -1;
91
92         // Line pointers
93         private int linePtr, lastLinePtr;
94
95         // Identifier stack
96         protected int identifierPtr;
97
98         protected char[][] identifierStack;
99
100         protected int identifierLengthPtr;
101
102         protected int[] identifierLengthStack;
103
104         protected long[] identifierPositionStack;
105
106         // Ast stack
107         protected static int AstStackIncrement = 10;
108
109         protected int astPtr;
110
111         protected Object[] astStack;
112
113         protected int astLengthPtr;
114
115         protected int[] astLengthStack;
116
117         protected AbstractCommentParser(Parser sourceParser) {
118                 this.sourceParser = sourceParser;
119                 this.scanner = new Scanner(false, false, false, false, false, null,
120                                 null, false);
121
122                 this.identifierStack = new char[20][];
123                 this.identifierPositionStack = new long[20];
124                 this.identifierLengthStack = new int[10];
125                 this.astStack = new Object[30];
126                 this.astLengthStack = new int[20];
127         }
128
129         /*
130          * (non-Javadoc) Returns true if tag
131          * 
132          * @deprecated is present in javadoc comment.
133          * 
134          * If javadoc checking is enabled, will also construct an Javadoc node,
135          * which will be stored into Parser.javadoc slot for being consumed later
136          * on.
137          */
138         protected boolean parseComment(int javadocStart, int javadocEnd) {
139
140                 boolean validComment = true;
141                 try {
142                         // Init scanner position
143                         this.scanner.resetTo(javadocStart, javadocEnd);
144                         this.endComment = javadocEnd;
145                         this.index = javadocStart;
146                         readChar(); // starting '/'
147                         int previousPosition = this.index;
148                         readChar(); // first '*'
149                         char nextCharacter = readChar(); // second '*'
150
151                         // Init local variables
152                         this.astLengthPtr = -1;
153                         this.astPtr = -1;
154                         this.currentTokenType = -1;
155                         this.inlineTagStarted = false;
156                         this.inlineTagStart = -1;
157                         this.lineStarted = false;
158                         this.returnStatement = null;
159                         this.inherited = false;
160                         this.deprecated = false;
161                         this.linePtr = getLineNumber(javadocStart);
162                         this.lastLinePtr = getLineNumber(javadocEnd);
163                         this.lineEnd = (this.linePtr == this.lastLinePtr) ? this.endComment
164                                         : this.scanner.getLineEnd(this.linePtr);
165                         this.textStart = -1;
166                         char previousChar = 0;
167                         int invalidTagLineEnd = -1;
168                         int invalidInlineTagLineEnd = -1;
169
170                         // Loop on each comment character
171                         while (this.index < this.endComment) {
172                                 previousPosition = this.index;
173                                 previousChar = nextCharacter;
174
175                                 // Calculate line end (cannot use this.scanner.linePtr as
176                                 // scanner does not parse line ends again)
177                                 if (this.index > (this.lineEnd + 1)) {
178                                         updateLineEnd();
179                                 }
180
181                                 // Read next char only if token was consumed
182                                 if (this.currentTokenType < 0) {
183                                         nextCharacter = readChar(); // consider unicodes
184                                 } else {
185                                         previousPosition = this.scanner
186                                                         .getCurrentTokenStartPosition();
187                                         switch (this.currentTokenType) {
188                                         case ITerminalSymbols.TokenNameRBRACE:
189                                                 nextCharacter = '}';
190                                                 break;
191                                         case ITerminalSymbols.TokenNameMULTIPLY:
192                                                 nextCharacter = '*';
193                                                 break;
194                                         default:
195                                                 nextCharacter = this.scanner.currentCharacter;
196                                         }
197                                         consumeToken();
198                                 }
199
200                                 if (this.index >= this.endComment) {
201                                         break;
202                                 }
203
204                                 switch (nextCharacter) {
205                                 case '@':
206                                         boolean valid = false;
207                                         // Start tag parsing only if we have a java identifier start
208                                         // character and if we are on line beginning or at inline
209                                         // tag beginning
210                                         if ((!this.lineStarted || previousChar == '{')) {
211                                                 this.lineStarted = true;
212                                                 if (this.inlineTagStarted) {
213                                                         this.inlineTagStarted = false;
214                                                         // bug
215                                                         // https://bugs.eclipse.org/bugs/show_bug.cgi?id=53279
216                                                         // Cannot have @ inside inline comment
217                                                         if (this.sourceParser != null) {
218                                                                 int end = previousPosition < invalidInlineTagLineEnd ? previousPosition
219                                                                                 : invalidInlineTagLineEnd;
220                                                                 this.sourceParser.problemReporter()
221                                                                                 .javadocUnterminatedInlineTag(
222                                                                                                 this.inlineTagStart, end);
223                                                         }
224                                                         validComment = false;
225                                                         if (this.lineStarted && this.textStart != -1
226                                                                         && this.textStart < previousPosition) {
227                                                                 pushText(this.textStart, previousPosition);
228                                                         }
229                                                         if (this.kind == DOM_PARSER)
230                                                                 refreshInlineTagPosition(previousPosition);
231                                                 }
232                                                 if (previousChar == '{') {
233                                                         if (this.textStart != -1
234                                                                         && this.textStart < this.inlineTagStart) {
235                                                                 pushText(this.textStart, this.inlineTagStart);
236                                                         }
237                                                         this.inlineTagStarted = true;
238                                                         invalidInlineTagLineEnd = this.lineEnd;
239                                                 } else if (this.textStart != -1
240                                                                 && this.textStart < invalidTagLineEnd) {
241                                                         pushText(this.textStart, invalidTagLineEnd);
242                                                 }
243                                                 this.scanner.resetTo(this.index, this.endComment);
244                                                 this.currentTokenType = -1; // flush token cache at line
245                                                                                                         // begin
246                                                 try {
247                                                         int token = readTokenAndConsume();
248                                                         this.tagSourceStart = this.scanner
249                                                                         .getCurrentTokenStartPosition();
250                                                         this.tagSourceEnd = this.scanner
251                                                                         .getCurrentTokenEndPosition();
252                                                         char[] tag = this.scanner
253                                                                         .getCurrentIdentifierSource(); // first
254                                                                                                                                         // token is
255                                                                                                                                         // either an
256                                                                                                                                         // identifier
257                                                                                                                                         // or a
258                                                                                                                                         // keyword
259                                                         if (this.kind == DOM_PARSER) {
260                                                                 // For DOM parser, try to get tag name other
261                                                                 // than java identifier
262                                                                 // (see bug
263                                                                 // https://bugs.eclipse.org/bugs/show_bug.cgi?id=51660)
264                                                                 int tk = token;
265                                                                 int le = this.lineEnd;
266                                                                 char pc = peekChar();
267                                                                 tagNameToken: while (tk != ITerminalSymbols.TokenNameEOF) {
268                                                                         this.tagSourceEnd = this.scanner
269                                                                                         .getCurrentTokenEndPosition();
270                                                                         token = tk;
271                                                                         // !, ", #, %, &, ', -, :, <, >, * chars and
272                                                                         // spaces are not allowed in tag names
273                                                                         switch (pc) {
274                                                                         case '}':
275                                                                         case '!':
276                                                                         case '#':
277                                                                         case '%':
278                                                                         case '&':
279                                                                         case '\'':
280                                                                         case ':':
281                                                                                 // case '-': allowed in tag names as
282                                                                                 // this character is often used in
283                                                                                 // doclets (bug 68087)
284                                                                         case '<':
285                                                                         case '>':
286                                                                         case '*': // break for '*' as this is
287                                                                                                 // perhaps the end of comment
288                                                                                                 // (bug 65288)
289                                                                                 break tagNameToken;
290                                                                         default:
291                                                                                 if (pc == ' '
292                                                                                                 || Character.isWhitespace(pc))
293                                                                                         break tagNameToken;
294                                                                         }
295                                                                         tk = readTokenAndConsume();
296                                                                         pc = peekChar();
297                                                                 }
298                                                                 int length = this.tagSourceEnd
299                                                                                 - this.tagSourceStart + 1;
300                                                                 tag = new char[length];
301                                                                 System.arraycopy(this.source,
302                                                                                 this.tagSourceStart, tag, 0, length);
303                                                                 this.index = this.tagSourceEnd + 1;
304                                                                 this.scanner.currentPosition = this.tagSourceEnd + 1;
305                                                                 this.tagSourceStart = previousPosition;
306                                                                 this.lineEnd = le;
307                                                         }
308                                                         switch (token) {
309                                                         case ITerminalSymbols.TokenNameIdentifier:
310                                                                 if (CharOperation.equals(tag, TAG_DEPRECATED)) {
311                                                                         this.deprecated = true;
312                                                                         if (this.kind == DOM_PARSER) {
313                                                                                 valid = parseTag();
314                                                                         } else {
315                                                                                 valid = true;
316                                                                         }
317                                                                 } else if (CharOperation.equals(tag,
318                                                                                 TAG_INHERITDOC)) {
319                                                                         // inhibits inherited flag when tags have
320                                                                         // been already stored
321                                                                         // see bug
322                                                                         // https://bugs.eclipse.org/bugs/show_bug.cgi?id=51606
323                                                                         // Note that for DOM_PARSER, nodes stack may
324                                                                         // be not empty even no '@' tag
325                                                                         // was encountered in comment. But it cannot
326                                                                         // be the case for COMPILER_PARSER
327                                                                         // and so is enough as it is only this
328                                                                         // parser which signals the missing tag
329                                                                         // warnings...
330                                                                         this.inherited = this.astPtr == -1;
331                                                                         if (this.kind == DOM_PARSER) {
332                                                                                 valid = parseTag();
333                                                                         } else {
334                                                                                 valid = true;
335                                                                         }
336                                                                 } else if (CharOperation.equals(tag, TAG_PARAM)) {
337                                                                         valid = parseParam();
338                                                                 } else if (CharOperation.equals(tag,
339                                                                                 TAG_EXCEPTION)) {
340                                                                         valid = parseThrows(false);
341                                                                 } else if (CharOperation.equals(tag, TAG_SEE)) {
342                                                                         if (this.inlineTagStarted) {
343                                                                                 // bug
344                                                                                 // https://bugs.eclipse.org/bugs/show_bug.cgi?id=53290
345                                                                                 // Cannot have @see inside inline
346                                                                                 // comment
347                                                                                 valid = false;
348                                                                                 if (this.sourceParser != null)
349                                                                                         this.sourceParser
350                                                                                                         .problemReporter()
351                                                                                                         .javadocUnexpectedTag(
352                                                                                                                         this.tagSourceStart,
353                                                                                                                         this.tagSourceEnd);
354                                                                         } else {
355                                                                                 valid = parseSee(false);
356                                                                         }
357                                                                 } else if (CharOperation.equals(tag, TAG_LINK)) {
358                                                                         if (this.inlineTagStarted) {
359                                                                                 valid = parseSee(false);
360                                                                         } else {
361                                                                                 // bug
362                                                                                 // https://bugs.eclipse.org/bugs/show_bug.cgi?id=53290
363                                                                                 // Cannot have @link outside inline
364                                                                                 // comment
365                                                                                 valid = false;
366                                                                                 if (this.sourceParser != null)
367                                                                                         this.sourceParser
368                                                                                                         .problemReporter()
369                                                                                                         .javadocUnexpectedTag(
370                                                                                                                         this.tagSourceStart,
371                                                                                                                         this.tagSourceEnd);
372                                                                         }
373                                                                 } else if (CharOperation.equals(tag,
374                                                                                 TAG_LINKPLAIN)) {
375                                                                         if (this.inlineTagStarted) {
376                                                                                 valid = parseSee(true);
377                                                                         } else {
378                                                                                 valid = parseTag();
379                                                                         }
380                                                                 } else {
381                                                                         valid = parseTag();
382                                                                 }
383                                                                 break;
384                                                         case ITerminalSymbols.TokenNamereturn:
385                                                                 valid = parseReturn();
386                                                                 // verify characters after return tag (we're
387                                                                 // expecting text description)
388                                                                 if (!verifyCharsAfterReturnTag(this.index)) {
389                                                                         if (this.sourceParser != null) {
390                                                                                 int end = this.starPosition == -1
391                                                                                                 || this.lineEnd < this.starPosition ? this.lineEnd
392                                                                                                 : this.starPosition;
393                                                                                 this.sourceParser.problemReporter()
394                                                                                                 .javadocInvalidTag(
395                                                                                                                 this.tagSourceStart,
396                                                                                                                 end);
397                                                                         }
398                                                                 }
399                                                                 break;
400                                                         // case ITerminalSymbols.TokenNamethrows :
401                                                         // valid = parseThrows(true);
402                                                         // break;
403                                                         default:
404                                                                 if (this.kind == DOM_PARSER) {
405                                                                         switch (token) {
406                                                                         case ITerminalSymbols.TokenNameabstract:
407                                                                                 // case
408                                                                                 // ITerminalSymbols.TokenNameassert:
409                                                                                 // case
410                                                                                 // ITerminalSymbols.TokenNameboolean:
411                                                                         case ITerminalSymbols.TokenNamebreak:
412                                                                                 // case ITerminalSymbols.TokenNamebyte:
413                                                                         case ITerminalSymbols.TokenNamecase:
414                                                                         case ITerminalSymbols.TokenNamecatch:
415                                                                                 // case ITerminalSymbols.TokenNamechar:
416                                                                         case ITerminalSymbols.TokenNameclass:
417                                                                         case ITerminalSymbols.TokenNamecontinue:
418                                                                         case ITerminalSymbols.TokenNamedefault:
419                                                                         case ITerminalSymbols.TokenNamedo:
420                                                                                 // case
421                                                                                 // ITerminalSymbols.TokenNamedouble:
422                                                                         case ITerminalSymbols.TokenNameelse:
423                                                                         case ITerminalSymbols.TokenNameextends:
424                                                                                 // case ITerminalSymbols.TokenNamefalse:
425                                                                         case ITerminalSymbols.TokenNamefinal:
426                                                                         case ITerminalSymbols.TokenNamefinally:
427                                                                                 // case ITerminalSymbols.TokenNamefloat:
428                                                                         case ITerminalSymbols.TokenNamefor:
429                                                                         case ITerminalSymbols.TokenNameif:
430                                                                         case ITerminalSymbols.TokenNameimplements:
431                                                                                 // case
432                                                                                 // ITerminalSymbols.TokenNameimport:
433                                                                         case ITerminalSymbols.TokenNameinstanceof:
434                                                                                 // case ITerminalSymbols.TokenNameint:
435                                                                         case ITerminalSymbols.TokenNameinterface:
436                                                                                 // case ITerminalSymbols.TokenNamelong:
437                                                                                 // case
438                                                                                 // ITerminalSymbols.TokenNamenative:
439                                                                         case ITerminalSymbols.TokenNamenew:
440                                                                                 // case ITerminalSymbols.TokenNamenull:
441                                                                                 // case
442                                                                                 // ITerminalSymbols.TokenNamepackage:
443                                                                         case ITerminalSymbols.TokenNameprivate:
444                                                                         case ITerminalSymbols.TokenNameprotected:
445                                                                         case ITerminalSymbols.TokenNamepublic:
446                                                                                 // case ITerminalSymbols.TokenNameshort:
447                                                                         case ITerminalSymbols.TokenNamestatic:
448                                                                                 // case
449                                                                                 // ITerminalSymbols.TokenNamestrictfp:
450                                                                         case ITerminalSymbols.TokenNamesuper:
451                                                                         case ITerminalSymbols.TokenNameswitch:
452                                                                                 // case
453                                                                                 // ITerminalSymbols.TokenNamesynchronized:
454                                                                                 // case ITerminalSymbols.TokenNamethis:
455                                                                         case ITerminalSymbols.TokenNamethrow:
456                                                                                 // case
457                                                                                 // ITerminalSymbols.TokenNametransient:
458                                                                                 // case ITerminalSymbols.TokenNametrue:
459                                                                         case ITerminalSymbols.TokenNametry:
460                                                                                 // case ITerminalSymbols.TokenNamevoid:
461                                                                                 // case
462                                                                                 // ITerminalSymbols.TokenNamevolatile:
463                                                                         case ITerminalSymbols.TokenNamewhile:
464                                                                                 valid = parseTag();
465                                                                                 break;
466                                                                         }
467                                                                 }
468                                                         }
469                                                         this.textStart = this.index;
470                                                         if (!valid) {
471                                                                 // bug
472                                                                 // https://bugs.eclipse.org/bugs/show_bug.cgi?id=51600
473                                                                 // do not stop the inline tag when error is
474                                                                 // encountered to get text after
475                                                                 validComment = false;
476                                                                 // bug
477                                                                 // https://bugs.eclipse.org/bugs/show_bug.cgi?id=51600
478                                                                 // for DOM AST node, store tag as text in case
479                                                                 // of invalid syntax
480                                                                 if (this.kind == DOM_PARSER) {
481                                                                         parseTag();
482                                                                         this.textStart = this.tagSourceEnd + 1;
483                                                                         invalidTagLineEnd = this.lineEnd;
484                                                                 }
485                                                         }
486                                                 } catch (InvalidInputException e) {
487                                                         consumeToken();
488                                                 }
489                                         }
490                                         break;
491                                 case '\r':
492                                 case '\n':
493                                         if (this.lineStarted && this.textStart < previousPosition) {
494                                                 pushText(this.textStart, previousPosition);
495                                         }
496                                         this.lineStarted = false;
497                                         // Fix bug 51650
498                                         this.textStart = -1;
499                                         break;
500                                 case '}':
501                                         if (this.inlineTagStarted) {
502                                                 if (this.lineStarted && this.textStart != -1
503                                                                 && this.textStart < previousPosition) {
504                                                         pushText(this.textStart, previousPosition);
505                                                 }
506                                                 if (this.kind == DOM_PARSER)
507                                                         refreshInlineTagPosition(previousPosition);
508                                                 this.textStart = this.index;
509                                                 this.inlineTagStarted = false;
510                                         } else {
511                                                 if (!this.lineStarted) {
512                                                         this.textStart = previousPosition;
513                                                 }
514                                         }
515                                         this.lineStarted = true;
516                                         break;
517                                 case '{':
518                                         if (this.inlineTagStarted) {
519                                                 this.inlineTagStarted = false;
520                                                 // bug
521                                                 // https://bugs.eclipse.org/bugs/show_bug.cgi?id=53279
522                                                 // Cannot have opening brace in inline comment
523                                                 if (this.sourceParser != null) {
524                                                         int end = previousPosition < invalidInlineTagLineEnd ? previousPosition
525                                                                         : invalidInlineTagLineEnd;
526                                                         this.sourceParser.problemReporter()
527                                                                         .javadocUnterminatedInlineTag(
528                                                                                         this.inlineTagStart, end);
529                                                 }
530                                                 if (this.lineStarted && this.textStart != -1
531                                                                 && this.textStart < previousPosition) {
532                                                         pushText(this.textStart, previousPosition);
533                                                 }
534                                                 if (this.kind == DOM_PARSER)
535                                                         refreshInlineTagPosition(previousPosition);
536                                         }
537                                         if (!this.lineStarted) {
538                                                 this.textStart = previousPosition;
539                                         }
540                                         this.lineStarted = true;
541                                         this.inlineTagStart = previousPosition;
542                                         break;
543                                 case '*':
544                                 case '\u000c': /* FORM FEED */
545                                 case ' ': /* SPACE */
546                                 case '\t': /* HORIZONTAL TABULATION */
547                                         // do nothing for space or '*' characters
548                                         break;
549                                 default:
550                                         if (!this.lineStarted) {
551                                                 this.textStart = previousPosition;
552                                         }
553                                         this.lineStarted = true;
554                                         break;
555                                 }
556                         }
557                         // bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=53279
558                         // Cannot leave comment inside inline comment
559                         if (this.inlineTagStarted) {
560                                 this.inlineTagStarted = false;
561                                 if (this.sourceParser != null) {
562                                         int end = previousPosition < invalidInlineTagLineEnd ? previousPosition
563                                                         : invalidInlineTagLineEnd;
564                                         if (this.index >= this.endComment)
565                                                 end = invalidInlineTagLineEnd;
566                                         this.sourceParser.problemReporter()
567                                                         .javadocUnterminatedInlineTag(this.inlineTagStart,
568                                                                         end);
569                                 }
570                                 if (this.lineStarted && this.textStart != -1
571                                                 && this.textStart < previousPosition) {
572                                         pushText(this.textStart, previousPosition);
573                                 }
574                                 if (this.kind == DOM_PARSER) {
575                                         refreshInlineTagPosition(previousPosition);
576                                 }
577                         } else if (this.lineStarted && this.textStart < previousPosition) {
578                                 pushText(this.textStart, previousPosition);
579                         }
580                         updateDocComment();
581                 } catch (Exception ex) {
582                         validComment = false;
583                 }
584                 return validComment;
585         }
586
587         private void consumeToken() {
588                 this.currentTokenType = -1; // flush token cache
589                 updateLineEnd();
590         }
591
592         protected abstract Object createArgumentReference(char[] name, int dim,
593                         Object typeRef, long[] dimPos, long argNamePos)
594                         throws InvalidInputException;
595
596         protected abstract Object createFieldReference(Object receiver)
597                         throws InvalidInputException;
598
599         protected abstract Object createMethodReference(Object receiver,
600                         List arguments) throws InvalidInputException;
601
602         protected Object createReturnStatement() {
603                 return null;
604         }
605
606         protected abstract Object createTypeReference(int primitiveToken);
607
608         private int getEndPosition() {
609                 if (this.scanner.getCurrentTokenEndPosition() > this.lineEnd) {
610                         return this.lineEnd;
611                 } else {
612                         return this.scanner.getCurrentTokenEndPosition();
613                 }
614         }
615
616         /*
617          * Search the source position corresponding to the end of a given line
618          * number. Warning: returned position is 1-based index!
619          * 
620          * @see Scanner#getLineEnd(int) We cannot directly use this method when
621          *      linePtr field is not initialized.
622          * 
623          * private int getLineEnd(int lineNumber) {
624          * 
625          * if (this.scanner.linePtr != -1) { return
626          * this.scanner.getLineEnd(lineNumber); } if (this.lineEnds == null) return
627          * -1; if (lineNumber > this.lineEnds.length+1) return -1; if (lineNumber <=
628          * 0) return -1; if (lineNumber == this.lineEnds.length + 1) return
629          * this.scanner.eofPosition; return this.lineEnds[lineNumber-1]; // next
630          * line start one character behind the lineEnd of the previous line }
631          */
632
633         /**
634          * Search the line number corresponding to a specific position. Warning:
635          * returned position is 1-based index!
636          * 
637          * @see Scanner#getLineNumber(int) We cannot directly use this method when
638          *      linePtr field is not initialized.
639          */
640         private int getLineNumber(int position) {
641
642                 if (this.scanner.linePtr != -1) {
643                         return this.scanner.getLineNumber(position);
644                 }
645                 if (this.lineEnds == null)
646                         return 1;
647                 int length = this.lineEnds.length;
648                 if (length == 0)
649                         return 1;
650                 int g = 0, d = length - 1;
651                 int m = 0;
652                 while (g <= d) {
653                         m = (g + d) / 2;
654                         if (position < this.lineEnds[m]) {
655                                 d = m - 1;
656                         } else if (position > this.lineEnds[m]) {
657                                 g = m + 1;
658                         } else {
659                                 return m + 1;
660                         }
661                 }
662                 if (position < this.lineEnds[m]) {
663                         return m + 1;
664                 }
665                 return m + 2;
666         }
667
668         /*
669          * Parse argument in
670          * 
671          * @see tag method reference
672          */
673         private Object parseArguments(Object receiver) throws InvalidInputException {
674
675                 // Init
676                 int modulo = 0; // should be 2 for (Type,Type,...) or 3 for (Type
677                                                 // arg,Type arg,...)
678                 int iToken = 0;
679                 char[] argName = null;
680                 List arguments = new ArrayList(10);
681                 int start = this.scanner.getCurrentTokenStartPosition();
682
683                 // Parse arguments declaration if method reference
684                 nextArg: while (this.index < this.scanner.eofPosition) {
685
686                         // Read argument type reference
687                         Object typeRef;
688                         try {
689                                 typeRef = parseQualifiedName(false);
690                         } catch (InvalidInputException e) {
691                                 break nextArg;
692                         }
693                         boolean firstArg = modulo == 0;
694                         if (firstArg) { // verify position
695                                 if (iToken != 0)
696                                         break nextArg;
697                         } else if ((iToken % modulo) != 0) {
698                                 break nextArg;
699                         }
700                         if (typeRef == null) {
701                                 if (firstArg
702                                                 && this.currentTokenType == ITerminalSymbols.TokenNameRPAREN) {
703                                         // verify characters after arguments declaration (expecting
704                                         // white space or end comment)
705                                         if (!verifySpaceOrEndComment()) {
706                                                 int end = this.starPosition == -1 ? this.lineEnd
707                                                                 : this.starPosition;
708                                                 if (this.source[end] == '\n')
709                                                         end--;
710                                                 if (this.sourceParser != null)
711                                                         this.sourceParser.problemReporter()
712                                                                         .javadocMalformedSeeReference(start, end);
713                                                 return null;
714                                         }
715                                         this.lineStarted = true;
716                                         return createMethodReference(receiver, null);
717                                 }
718                                 break nextArg;
719                         }
720                         iToken++;
721
722                         // Read possible array declaration
723                         int dim = 0;
724                         long[] dimPositions = new long[20]; // assume that there won't be
725                                                                                                 // more than 20 dimensions...
726                         if (readToken() == ITerminalSymbols.TokenNameLBRACKET) {
727                                 int dimStart = this.scanner.getCurrentTokenStartPosition();
728                                 while (readToken() == ITerminalSymbols.TokenNameLBRACKET) {
729                                         consumeToken();
730                                         if (readToken() != ITerminalSymbols.TokenNameRBRACKET) {
731                                                 break nextArg;
732                                         }
733                                         consumeToken();
734                                         dimPositions[dim++] = (((long) dimStart) << 32)
735                                                         + this.scanner.getCurrentTokenEndPosition();
736                                 }
737                         }
738
739                         // Read argument name
740                         long argNamePos = -1;
741                         if (readToken() == ITerminalSymbols.TokenNameIdentifier) {
742                                 consumeToken();
743                                 if (firstArg) { // verify position
744                                         if (iToken != 1)
745                                                 break nextArg;
746                                 } else if ((iToken % modulo) != 1) {
747                                         break nextArg;
748                                 }
749                                 if (argName == null) { // verify that all arguments name are
750                                                                                 // declared
751                                         if (!firstArg) {
752                                                 break nextArg;
753                                         }
754                                 }
755                                 argName = this.scanner.getCurrentIdentifierSource();
756                                 argNamePos = (((long) this.scanner
757                                                 .getCurrentTokenStartPosition()) << 32)
758                                                 + this.scanner.getCurrentTokenEndPosition();
759                                 iToken++;
760                         } else if (argName != null) { // verify that no argument name is
761                                                                                         // declared
762                                 break nextArg;
763                         }
764
765                         // Verify token position
766                         if (firstArg) {
767                                 modulo = iToken + 1;
768                         } else {
769                                 if ((iToken % modulo) != (modulo - 1)) {
770                                         break nextArg;
771                                 }
772                         }
773
774                         // Read separator or end arguments declaration
775                         int token = readToken();
776                         char[] name = argName == null ? new char[0] : argName;
777                         if (token == ITerminalSymbols.TokenNameCOMMA) {
778                                 // Create new argument
779                                 Object argument = createArgumentReference(name, dim, typeRef,
780                                                 dimPositions, argNamePos);
781                                 arguments.add(argument);
782                                 consumeToken();
783                                 iToken++;
784                         } else if (token == ITerminalSymbols.TokenNameRPAREN) {
785                                 // verify characters after arguments declaration (expecting
786                                 // white space or end comment)
787                                 if (!verifySpaceOrEndComment()) {
788                                         int end = this.starPosition == -1 ? this.lineEnd
789                                                         : this.starPosition;
790                                         if (this.source[end] == '\n')
791                                                 end--;
792                                         if (this.sourceParser != null)
793                                                 this.sourceParser.problemReporter()
794                                                                 .javadocMalformedSeeReference(start, end);
795                                         return null;
796                                 }
797                                 // Create new argument
798                                 Object argument = createArgumentReference(name, dim, typeRef,
799                                                 dimPositions, argNamePos);
800                                 arguments.add(argument);
801                                 consumeToken();
802                                 return createMethodReference(receiver, arguments);
803                         } else {
804                                 break nextArg;
805                         }
806                 }
807
808                 // Something wrong happened => Invalid input
809                 throw new InvalidInputException();
810         }
811
812         /*
813          * Parse an URL link reference in
814          * 
815          * @see tag
816          */
817         private boolean parseHref() throws InvalidInputException {
818                 int start = this.scanner.getCurrentTokenStartPosition();
819                 if (Character.toLowerCase(readChar()) == 'a') {
820                         this.scanner.currentPosition = this.index;
821                         if (readToken() == ITerminalSymbols.TokenNameIdentifier) {
822                                 this.currentTokenType = -1; // do not update line end
823                                 try {
824                                         if (CharOperation.equals(this.scanner
825                                                         .getCurrentIdentifierSource(), new char[] { 'h',
826                                                         'r', 'e', 'f' }, false)
827                                                         && readToken() == ITerminalSymbols.TokenNameEQUAL) {
828                                                 this.currentTokenType = -1; // do not update line end
829                                                 if (readToken() == ITerminalSymbols.TokenNameStringDoubleQuote
830                                                                 || readToken() == ITerminalSymbols.TokenNameStringSingleQuote) {
831                                                         this.currentTokenType = -1; // do not update line
832                                                                                                                 // end
833                                                         // Skip all characters after string literal until
834                                                         // closing '>' (see bug 68726)
835                                                         while (this.index <= this.lineEnd
836                                                                         && readToken() != ITerminalSymbols.TokenNameGREATER) {
837                                                                 this.currentTokenType = -1; // do not update
838                                                                                                                         // line end
839                                                         }
840                                                         if (this.currentTokenType == ITerminalSymbols.TokenNameGREATER) {
841                                                                 consumeToken(); // update line end as new lines
842                                                                                                 // are allowed in URL
843                                                                                                 // description
844                                                                 while (readToken() != ITerminalSymbols.TokenNameLESS) {
845                                                                         if (this.scanner.currentPosition >= this.scanner.eofPosition
846                                                                                         || this.scanner.currentCharacter == '@') {
847                                                                                 // Reset position: we want to rescan
848                                                                                 // last token
849                                                                                 this.index = this.tokenPreviousPosition;
850                                                                                 this.scanner.currentPosition = this.tokenPreviousPosition;
851                                                                                 this.currentTokenType = -1;
852                                                                                 // Signal syntax error
853                                                                                 if (this.sourceParser != null)
854                                                                                         this.sourceParser
855                                                                                                         .problemReporter()
856                                                                                                         .javadocInvalidSeeUrlReference(
857                                                                                                                         start, this.lineEnd);
858                                                                                 return false;
859                                                                         }
860                                                                         consumeToken();
861                                                                 }
862                                                                 this.currentTokenType = -1; // do not update
863                                                                                                                         // line end
864                                                                 if (readChar() == '/') {
865                                                                         if (Character.toLowerCase(readChar()) == 'a') {
866                                                                                 if (readChar() == '>') {
867                                                                                         // Valid href
868                                                                                         return true;
869                                                                                 }
870                                                                         }
871                                                                 }
872                                                         }
873                                                 }
874                                         }
875                                 } catch (InvalidInputException ex) {
876                                         // Do nothing as we want to keep positions for error message
877                                 }
878                         }
879                 }
880                 // Reset position: we want to rescan last token
881                 this.index = this.tokenPreviousPosition;
882                 this.scanner.currentPosition = this.tokenPreviousPosition;
883                 this.currentTokenType = -1;
884                 // Signal syntax error
885                 if (this.sourceParser != null)
886                         this.sourceParser.problemReporter().javadocInvalidSeeUrlReference(
887                                         start, this.lineEnd);
888                 return false;
889         }
890
891         /*
892          * Parse a method reference in
893          * 
894          * @see tag
895          */
896         private Object parseMember(Object receiver) throws InvalidInputException {
897                 // Init
898                 this.identifierPtr = -1;
899                 this.identifierLengthPtr = -1;
900                 int start = this.scanner.getCurrentTokenStartPosition();
901                 this.memberStart = start;
902
903                 // Get member identifier
904                 if (readToken() == ITerminalSymbols.TokenNameIdentifier) {
905                         consumeToken();
906                         pushIdentifier(true);
907                         // Look for next token to know whether it's a field or method
908                         // reference
909                         int previousPosition = this.index;
910                         if (readToken() == ITerminalSymbols.TokenNameLPAREN) {
911                                 consumeToken();
912                                 start = this.scanner.getCurrentTokenStartPosition();
913                                 try {
914                                         return parseArguments(receiver);
915                                 } catch (InvalidInputException e) {
916                                         int end = this.scanner.getCurrentTokenEndPosition() < this.lineEnd ? this.scanner
917                                                         .getCurrentTokenEndPosition()
918                                                         : this.scanner.getCurrentTokenStartPosition();
919                                         end = end < this.lineEnd ? end : this.lineEnd;
920                                         if (this.sourceParser != null)
921                                                 this.sourceParser.problemReporter()
922                                                                 .javadocInvalidSeeReferenceArgs(start, end);
923                                 }
924                                 return null;
925                         }
926
927                         // Reset position: we want to rescan last token
928                         this.index = previousPosition;
929                         this.scanner.currentPosition = previousPosition;
930                         this.currentTokenType = -1;
931
932                         // Verify character(s) after identifier (expecting space or end
933                         // comment)
934                         if (!verifySpaceOrEndComment()) {
935                                 int end = this.starPosition == -1 ? this.lineEnd
936                                                 : this.starPosition;
937                                 if (this.source[end] == '\n')
938                                         end--;
939                                 if (this.sourceParser != null)
940                                         this.sourceParser.problemReporter()
941                                                         .javadocMalformedSeeReference(start, end);
942                                 return null;
943                         }
944                         return createFieldReference(receiver);
945                 }
946                 int end = getEndPosition() - 1;
947                 end = start > end ? start : end;
948                 if (this.sourceParser != null)
949                         this.sourceParser.problemReporter().javadocInvalidSeeReference(
950                                         start, end);
951                 // Reset position: we want to rescan last token
952                 this.index = this.tokenPreviousPosition;
953                 this.scanner.currentPosition = this.tokenPreviousPosition;
954                 this.currentTokenType = -1;
955                 return null;
956         }
957
958         /*
959          * Parse @param tag declaration
960          */
961         protected boolean parseParam() {
962
963                 // Store current token state
964                 int start = this.tagSourceStart;
965                 int end = this.tagSourceEnd;
966
967                 try {
968                         // Push identifier next
969                         int token = readToken();
970                         switch (token) {
971                         case ITerminalSymbols.TokenNameIdentifier:
972                                 consumeToken();
973                                 return pushParamName();
974                         case ITerminalSymbols.TokenNameEOF:
975                                 break;
976                         default:
977                                 start = this.scanner.getCurrentTokenStartPosition();
978                                 end = getEndPosition();
979                                 if (end < start)
980                                         start = this.tagSourceStart;
981                                 break;
982                         }
983                 } catch (InvalidInputException e) {
984                         end = getEndPosition();
985                 }
986
987                 // Reset position to avoid missing tokens when new line was encountered
988                 this.index = this.tokenPreviousPosition;
989                 this.scanner.currentPosition = this.tokenPreviousPosition;
990                 this.currentTokenType = -1;
991
992                 // Report problem
993                 if (this.sourceParser != null)
994                         this.sourceParser.problemReporter().javadocMissingParamName(start,
995                                         end);
996                 return false;
997         }
998
999         /*
1000          * Parse a qualified name and built a type reference if the syntax is valid.
1001          */
1002         protected Object parseQualifiedName(boolean reset)
1003                         throws InvalidInputException {
1004
1005                 // Reset identifier stack if requested
1006                 if (reset) {
1007                         this.identifierPtr = -1;
1008                         this.identifierLengthPtr = -1;
1009                 }
1010
1011                 // Scan tokens
1012                 int primitiveToken = -1;
1013                 nextToken: for (int iToken = 0;; iToken++) {
1014                         int token = readToken();
1015                         switch (token) {
1016                         case ITerminalSymbols.TokenNameIdentifier:
1017                                 if (((iToken % 2) > 0)) { // identifiers must be odd tokens
1018                                         break nextToken;
1019                                 }
1020                                 pushIdentifier(iToken == 0);
1021                                 consumeToken();
1022                                 break;
1023
1024                         case ITerminalSymbols.TokenNameDOT:
1025                                 if ((iToken % 2) == 0) { // dots must be even tokens
1026                                         throw new InvalidInputException();
1027                                 }
1028                                 consumeToken();
1029                                 break;
1030
1031                         // case ITerminalSymbols.TokenNamevoid :
1032                         // case ITerminalSymbols.TokenNameboolean :
1033                         // case ITerminalSymbols.TokenNamebyte :
1034                         // case ITerminalSymbols.TokenNamechar :
1035                         // case ITerminalSymbols.TokenNamedouble :
1036                         // case ITerminalSymbols.TokenNamefloat :
1037                         // case ITerminalSymbols.TokenNameint :
1038                         // case ITerminalSymbols.TokenNamelong :
1039                         // case ITerminalSymbols.TokenNameshort :
1040                         // if (iToken > 0) {
1041                         // throw new InvalidInputException();
1042                         // }
1043                         // pushIdentifier(true);
1044                         // primitiveToken = token;
1045                         // consumeToken();
1046                         // break nextToken;
1047
1048                         default:
1049                                 if (iToken == 0) {
1050                                         return null;
1051                                 }
1052                                 if ((iToken % 2) == 0) { // cannot leave on a dot
1053                                         // Reset position: we want to rescan last token
1054                                         if (this.kind == DOM_PARSER && this.currentTokenType != -1) {
1055                                                 this.index = this.tokenPreviousPosition;
1056                                                 this.scanner.currentPosition = this.tokenPreviousPosition;
1057                                                 this.currentTokenType = -1;
1058                                         }
1059                                         throw new InvalidInputException();
1060                                 }
1061                                 break nextToken;
1062                         }
1063                 }
1064                 // Reset position: we want to rescan last token
1065                 if (this.currentTokenType != -1) {
1066                         this.index = this.tokenPreviousPosition;
1067                         this.scanner.currentPosition = this.tokenPreviousPosition;
1068                         this.currentTokenType = -1;
1069                 }
1070                 this.lastIdentifierEndPosition = (int) this.identifierPositionStack[this.identifierPtr];
1071                 return createTypeReference(primitiveToken);
1072         }
1073
1074         /*
1075          * Parse a reference in
1076          * 
1077          * @see tag
1078          */
1079         protected boolean parseReference(boolean plain)
1080                         throws InvalidInputException {
1081                 Object typeRef = null;
1082                 Object reference = null;
1083                 int previousPosition = -1;
1084                 int typeRefStartPosition = -1;
1085                 nextToken: while (this.index < this.scanner.eofPosition) {
1086                         previousPosition = this.index;
1087                         int token = readToken();
1088                         switch (token) {
1089                         case ITerminalSymbols.TokenNameStringDoubleQuote: // @see "string"
1090                         case ITerminalSymbols.TokenNameStringSingleQuote:
1091                                 int start = this.scanner.getCurrentTokenStartPosition();
1092                                 consumeToken();
1093                                 // If typeRef != null we may raise a warning here to let user
1094                                 // know there's an unused reference...
1095                                 // Currently as javadoc 1.4.2 ignore it, we do the same (see bug
1096                                 // 69302)
1097                                 if (typeRef != null) {
1098                                         start = this.tagSourceEnd + 1;
1099                                         previousPosition = start;
1100                                         typeRef = null;
1101                                 }
1102                                 // verify end line (expecting empty or end comment)
1103                                 if (verifyEndLine(previousPosition)) {
1104                                         return true;
1105                                 }
1106                                 if (this.sourceParser != null)
1107                                         this.sourceParser.problemReporter()
1108                                                         .javadocInvalidSeeReference(start, this.lineEnd);
1109                                 return false;
1110                         case ITerminalSymbols.TokenNameLESS: // @see "<a
1111                                                                                                         // href="URL#Value">label</a>
1112                                 consumeToken();
1113                                 start = this.scanner.getCurrentTokenStartPosition();
1114                                 if (parseHref()) {
1115                                         consumeToken();
1116                                         // If typeRef != null we may raise a warning here to let
1117                                         // user know there's an unused reference...
1118                                         // Currently as javadoc 1.4.2 ignore it, we do the same (see
1119                                         // bug 69302)
1120                                         if (typeRef != null) {
1121                                                 start = this.tagSourceEnd + 1;
1122                                                 previousPosition = start;
1123                                                 typeRef = null;
1124                                         }
1125                                         // verify end line (expecting empty or end comment)
1126                                         if (verifyEndLine(previousPosition)) {
1127                                                 return true;
1128                                         }
1129                                         if (this.sourceParser != null)
1130                                                 this.sourceParser
1131                                                                 .problemReporter()
1132                                                                 .javadocInvalidSeeReference(start, this.lineEnd);
1133                                 }
1134                                 return false;
1135                         case ITerminalSymbols.TokenNameERROR:
1136                                 if (this.scanner.currentCharacter == '#') { // @see ...#member
1137                                         consumeToken();
1138                                         reference = parseMember(typeRef);
1139                                         if (reference != null) {
1140                                                 return pushSeeRef(reference, plain);
1141                                         }
1142                                         return false;
1143                                 }
1144                                 break nextToken;
1145                         case ITerminalSymbols.TokenNameIdentifier:
1146                                 if (typeRef == null) {
1147                                         typeRefStartPosition = this.scanner
1148                                                         .getCurrentTokenStartPosition();
1149                                         typeRef = parseQualifiedName(true);
1150                                         break;
1151                                 }
1152                                 break nextToken;
1153                         default:
1154                                 break nextToken;
1155                         }
1156                 }
1157
1158                 // Verify that we got a reference
1159                 if (reference == null)
1160                         reference = typeRef;
1161                 if (reference == null) {
1162                         this.index = this.tokenPreviousPosition;
1163                         this.scanner.currentPosition = this.tokenPreviousPosition;
1164                         this.currentTokenType = -1;
1165                         if (this.sourceParser != null)
1166                                 this.sourceParser.problemReporter().javadocMissingSeeReference(
1167                                                 this.tagSourceStart, this.tagSourceEnd);
1168                         return false;
1169                 }
1170
1171                 // Reset position at the end of type reference
1172                 this.index = this.lastIdentifierEndPosition + 1;
1173                 this.scanner.currentPosition = this.index;
1174                 this.currentTokenType = -1;
1175
1176                 // Verify that line end does not start with an open parenthese (which
1177                 // could be a constructor reference wrongly written...)
1178                 // See bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=47215
1179                 char ch = peekChar();
1180                 if (ch == '(') {
1181                         if (this.sourceParser != null)
1182                                 this.sourceParser.problemReporter().javadocInvalidSeeReference(
1183                                                 typeRefStartPosition, this.lineEnd);
1184                         return false;
1185                 }
1186
1187                 // Verify that we get white space after reference
1188                 if (!verifySpaceOrEndComment()) {
1189                         this.index = this.tokenPreviousPosition;
1190                         this.scanner.currentPosition = this.tokenPreviousPosition;
1191                         this.currentTokenType = -1;
1192                         int end = this.starPosition == -1 ? this.lineEnd
1193                                         : this.starPosition;
1194                         if (this.source[end] == '\n')
1195                                 end--;
1196                         if (this.sourceParser != null)
1197                                 this.sourceParser
1198                                                 .problemReporter()
1199                                                 .javadocMalformedSeeReference(typeRefStartPosition, end);
1200                         return false;
1201                 }
1202
1203                 // Everything is OK, store reference
1204                 return pushSeeRef(reference, plain);
1205         }
1206
1207         /*
1208          * Parse @return tag declaration
1209          */
1210         protected abstract boolean parseReturn();
1211
1212         /*
1213          * Parse
1214          * 
1215          * @see tag declaration
1216          */
1217         protected boolean parseSee(boolean plain) {
1218                 int start = this.scanner.currentPosition;
1219                 try {
1220                         return parseReference(plain);
1221                 } catch (InvalidInputException ex) {
1222                         if (this.sourceParser != null)
1223                                 this.sourceParser.problemReporter().javadocInvalidSeeReference(
1224                                                 start, getEndPosition());
1225                 }
1226                 // Reset position to avoid missing tokens when new line was encountered
1227                 this.index = this.tokenPreviousPosition;
1228                 this.scanner.currentPosition = this.tokenPreviousPosition;
1229                 this.currentTokenType = -1;
1230                 return false;
1231         }
1232
1233         /*
1234          * Parse @return tag declaration
1235          */
1236         protected abstract boolean parseTag();
1237
1238         /*
1239          * Parse @throws tag declaration
1240          */
1241         protected boolean parseThrows(boolean real) {
1242                 int start = this.scanner.currentPosition;
1243                 try {
1244                         Object typeRef = parseQualifiedName(true);
1245                         if (typeRef == null) {
1246                                 if (this.sourceParser != null)
1247                                         this.sourceParser.problemReporter()
1248                                                         .javadocMissingThrowsClassName(this.tagSourceStart,
1249                                                                         this.tagSourceEnd);
1250                         } else {
1251                                 return pushThrowName(typeRef, real);
1252                         }
1253                 } catch (InvalidInputException ex) {
1254                         if (this.sourceParser != null)
1255                                 this.sourceParser.problemReporter().javadocInvalidThrowsClass(
1256                                                 start, getEndPosition());
1257                 }
1258                 return false;
1259         }
1260
1261         /*
1262          * Return current character without move index position.
1263          */
1264         private char peekChar() {
1265                 int idx = this.index;
1266                 char c = this.source[idx++];
1267                 if (c == '\\' && this.source[idx] == 'u') {
1268                         int c1, c2, c3, c4;
1269                         idx++;
1270                         while (this.source[idx] == 'u')
1271                                 idx++;
1272                         if (!(((c1 = Character.getNumericValue(this.source[idx++])) > 15 || c1 < 0)
1273                                         || ((c2 = Character.getNumericValue(this.source[idx++])) > 15 || c2 < 0)
1274                                         || ((c3 = Character.getNumericValue(this.source[idx++])) > 15 || c3 < 0) || ((c4 = Character
1275                                         .getNumericValue(this.source[idx++])) > 15 || c4 < 0))) {
1276                                 c = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1277                         }
1278                 }
1279                 return c;
1280         }
1281
1282         /*
1283          * push the consumeToken on the identifier stack. Increase the total number
1284          * of identifier in the stack.
1285          */
1286         protected void pushIdentifier(boolean newLength) {
1287
1288                 int stackLength = this.identifierStack.length;
1289                 if (++this.identifierPtr >= stackLength) {
1290                         System.arraycopy(this.identifierStack, 0,
1291                                         this.identifierStack = new char[stackLength + 10][], 0,
1292                                         stackLength);
1293                         System.arraycopy(this.identifierPositionStack, 0,
1294                                         this.identifierPositionStack = new long[stackLength + 10],
1295                                         0, stackLength);
1296                 }
1297                 this.identifierStack[this.identifierPtr] = this.scanner
1298                                 .getCurrentIdentifierSource();
1299                 this.identifierPositionStack[this.identifierPtr] = (((long) this.scanner.startPosition) << 32)
1300                                 + (this.scanner.currentPosition - 1);
1301
1302                 if (newLength) {
1303                         stackLength = this.identifierLengthStack.length;
1304                         if (++this.identifierLengthPtr >= stackLength) {
1305                                 System.arraycopy(this.identifierLengthStack, 0,
1306                                                 this.identifierLengthStack = new int[stackLength + 10],
1307                                                 0, stackLength);
1308                         }
1309                         this.identifierLengthStack[this.identifierLengthPtr] = 1;
1310                 } else {
1311                         this.identifierLengthStack[this.identifierLengthPtr]++;
1312                 }
1313         }
1314
1315         /*
1316          * Add a new obj on top of the ast stack. If new length is required, then
1317          * add also a new length in length stack.
1318          */
1319         protected void pushOnAstStack(Object node, boolean newLength) {
1320
1321                 if (node == null) {
1322                         this.astLengthStack[++this.astLengthPtr] = 0;
1323                         return;
1324                 }
1325
1326                 int stackLength = this.astStack.length;
1327                 if (++this.astPtr >= stackLength) {
1328                         System
1329                                         .arraycopy(this.astStack, 0,
1330                                                         this.astStack = new Object[stackLength
1331                                                                         + AstStackIncrement], 0, stackLength);
1332                         this.astPtr = stackLength;
1333                 }
1334                 this.astStack[this.astPtr] = node;
1335
1336                 if (newLength) {
1337                         stackLength = this.astLengthStack.length;
1338                         if (++this.astLengthPtr >= stackLength) {
1339                                 System.arraycopy(this.astLengthStack, 0,
1340                                                 this.astLengthStack = new int[stackLength
1341                                                                 + AstStackIncrement], 0, stackLength);
1342                         }
1343                         this.astLengthStack[this.astLengthPtr] = 1;
1344                 } else {
1345                         this.astLengthStack[this.astLengthPtr]++;
1346                 }
1347         }
1348
1349         /*
1350          * Push a param name in ast node stack.
1351          */
1352         protected abstract boolean pushParamName();
1353
1354         /*
1355          * Push a reference statement in ast node stack.
1356          */
1357         protected abstract boolean pushSeeRef(Object statement, boolean plain);
1358
1359         /*
1360          * Push a text element in ast node stack
1361          */
1362         protected abstract void pushText(int start, int end);
1363
1364         /*
1365          * Push a throws type ref in ast node stack.
1366          */
1367         protected abstract boolean pushThrowName(Object typeRef, boolean real);
1368
1369         /*
1370          * Read current character and move index position. Warning: scanner position
1371          * is unchanged using this method!
1372          */
1373         protected char readChar() {
1374
1375                 char c = this.source[this.index++];
1376                 if (c == '\\' && this.source[this.index] == 'u') {
1377                         int c1, c2, c3, c4;
1378                         int pos = this.index;
1379                         this.index++;
1380                         while (this.source[this.index] == 'u')
1381                                 this.index++;
1382                         if (!(((c1 = Character.getNumericValue(this.source[this.index++])) > 15 || c1 < 0)
1383                                         || ((c2 = Character
1384                                                         .getNumericValue(this.source[this.index++])) > 15 || c2 < 0)
1385                                         || ((c3 = Character
1386                                                         .getNumericValue(this.source[this.index++])) > 15 || c3 < 0) || ((c4 = Character
1387                                         .getNumericValue(this.source[this.index++])) > 15 || c4 < 0))) {
1388                                 c = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1389                         } else {
1390                                 // TODO (frederic) currently reset to previous position, perhaps
1391                                 // signal a syntax error would be more appropriate
1392                                 this.index = pos;
1393                         }
1394                 }
1395                 return c;
1396         }
1397
1398         /*
1399          * Read token only if previous was consumed
1400          */
1401         private int readToken() throws InvalidInputException {
1402                 if (this.currentTokenType < 0) {
1403                         this.tokenPreviousPosition = this.scanner.currentPosition;
1404                         this.currentTokenType = this.scanner.getNextToken();
1405                         if (this.scanner.currentPosition > (this.lineEnd + 1)) { // be
1406                                                                                                                                                 // sure
1407                                                                                                                                                 // to be
1408                                                                                                                                                 // on
1409                                                                                                                                                 // next
1410                                                                                                                                                 // line
1411                                                                                                                                                 // (lineEnd
1412                                                                                                                                                 // is
1413                                                                                                                                                 // still
1414                                                                                                                                                 // on
1415                                                                                                                                                 // the
1416                                                                                                                                                 // same
1417                                                                                                                                                 // line)
1418                                 this.lineStarted = false;
1419                                 while (this.currentTokenType == ITerminalSymbols.TokenNameMULTIPLY) {
1420                                         this.currentTokenType = this.scanner.getNextToken();
1421                                 }
1422                         }
1423                         this.index = this.scanner.currentPosition;
1424                         this.lineStarted = true; // after having read a token, line is
1425                                                                                 // obviously started...
1426                 }
1427                 return this.currentTokenType;
1428         }
1429
1430         private int readTokenAndConsume() throws InvalidInputException {
1431                 int token = readToken();
1432                 consumeToken();
1433                 return token;
1434         }
1435
1436         /*
1437          * Refresh start position and length of an inline tag.
1438          */
1439         protected void refreshInlineTagPosition(int previousPosition) {
1440                 // do nothing by default
1441         }
1442
1443         public String toString() {
1444                 StringBuffer buffer = new StringBuffer();
1445                 int startPos = this.scanner.currentPosition < this.index ? this.scanner.currentPosition
1446                                 : this.index;
1447                 int endPos = this.scanner.currentPosition < this.index ? this.index
1448                                 : this.scanner.currentPosition;
1449                 if (startPos == this.source.length)
1450                         return "EOF\n\n" + new String(this.source); //$NON-NLS-1$
1451                 if (endPos > this.source.length)
1452                         return "behind the EOF\n\n" + new String(this.source); //$NON-NLS-1$
1453
1454                 char front[] = new char[startPos];
1455                 System.arraycopy(this.source, 0, front, 0, startPos);
1456
1457                 int middleLength = (endPos - 1) - startPos + 1;
1458                 char middle[];
1459                 if (middleLength > -1) {
1460                         middle = new char[middleLength];
1461                         System.arraycopy(this.source, startPos, middle, 0, middleLength);
1462                 } else {
1463                         middle = CharOperation.NO_CHAR;
1464                 }
1465
1466                 char end[] = new char[this.source.length - (endPos - 1)];
1467                 System.arraycopy(this.source, (endPos - 1) + 1, end, 0,
1468                                 this.source.length - (endPos - 1) - 1);
1469
1470                 buffer.append(front);
1471                 if (this.scanner.currentPosition < this.index) {
1472                         buffer
1473                                         .append("\n===============================\nScanner current position here -->"); //$NON-NLS-1$
1474                 } else {
1475                         buffer
1476                                         .append("\n===============================\nParser index here -->"); //$NON-NLS-1$
1477                 }
1478                 buffer.append(middle);
1479                 if (this.scanner.currentPosition < this.index) {
1480                         buffer
1481                                         .append("<-- Parser index here\n===============================\n"); //$NON-NLS-1$
1482                 } else {
1483                         buffer
1484                                         .append("<-- Scanner current position here\n===============================\n"); //$NON-NLS-1$
1485                 }
1486                 buffer.append(end);
1487
1488                 return buffer.toString();
1489         }
1490
1491         /*
1492          * Update
1493          */
1494         protected abstract void updateDocComment();
1495
1496         /*
1497          * Update line end
1498          */
1499         protected void updateLineEnd() {
1500                 while (this.index > (this.lineEnd + 1)) { // be sure to be on next
1501                                                                                                         // line (lineEnd is still on
1502                                                                                                         // the same line)
1503                         if (this.linePtr < this.lastLinePtr) {
1504                                 this.lineEnd = this.scanner.getLineEnd(++this.linePtr) - 1;
1505                         } else {
1506                                 this.lineEnd = this.endComment;
1507                                 return;
1508                         }
1509                 }
1510         }
1511
1512         /*
1513          * Verify that end of the line only contains space characters or end of
1514          * comment. Note that end of comment may be preceeding by several contiguous
1515          * '*' chars.
1516          */
1517         private boolean verifyEndLine(int textPosition) {
1518                 int startPosition = this.index;
1519                 int previousPosition = this.index;
1520                 this.starPosition = -1;
1521                 char ch = readChar();
1522                 nextChar: while (true) {
1523                         switch (ch) {
1524                         case '\r':
1525                         case '\n':
1526                                 if (this.kind == DOM_PARSER) {
1527                                         parseTag();
1528                                         pushText(textPosition, previousPosition);
1529                                 }
1530                                 this.index = previousPosition;
1531                                 return true;
1532                         case '\u000c': /* FORM FEED */
1533                         case ' ': /* SPACE */
1534                         case '\t': /* HORIZONTAL TABULATION */
1535                                 if (this.starPosition >= 0)
1536                                         break nextChar;
1537                                 break;
1538                         case '*':
1539                                 this.starPosition = previousPosition;
1540                                 break;
1541                         case '/':
1542                                 if (this.starPosition >= textPosition) {
1543                                         if (this.kind == DOM_PARSER) {
1544                                                 parseTag();
1545                                                 pushText(textPosition, this.starPosition);
1546                                         }
1547                                         return true;
1548                                 }
1549                         default:
1550                                 // leave loop
1551                                 break nextChar;
1552
1553                         }
1554                         previousPosition = this.index;
1555                         ch = readChar();
1556                 }
1557                 this.index = startPosition;
1558                 return false;
1559         }
1560
1561         /*
1562          * Verify that some text exists after a @return tag. Text must be different
1563          * than end of comment which may be preceeding by several '*' chars.
1564          */
1565         private boolean verifyCharsAfterReturnTag(int startPosition) {
1566                 // Whitespace or inline tag closing brace
1567                 int previousPosition = this.index;
1568                 char ch = readChar();
1569                 boolean malformed = true;
1570                 while (Character.isWhitespace(ch)) {
1571                         malformed = false;
1572                         previousPosition = this.index;
1573                         ch = readChar();
1574                 }
1575                 // End of comment
1576                 this.starPosition = -1;
1577                 nextChar: while (this.index < this.source.length) {
1578                         switch (ch) {
1579                         case '*':
1580                                 // valid whatever the number of star before last '/'
1581                                 this.starPosition = previousPosition;
1582                                 break;
1583                         case '/':
1584                                 if (this.starPosition >= startPosition) { // valid only if a
1585                                                                                                                         // star was previous
1586                                                                                                                         // character
1587                                         return false;
1588                                 }
1589                         default:
1590                                 // valid if any other character is encountered, even white
1591                                 // spaces
1592                                 this.index = startPosition;
1593                                 return !malformed;
1594
1595                         }
1596                         previousPosition = this.index;
1597                         ch = readChar();
1598                 }
1599                 this.index = startPosition;
1600                 return false;
1601         }
1602
1603         /*
1604          * Verify characters after a name matches one of following conditions: 1-
1605          * first character is a white space 2- first character is a closing brace
1606          * *and* we're currently parsing an inline tag 3- are the end of comment
1607          * (several contiguous star ('*') characters may be found before the last
1608          * slash ('/') character).
1609          */
1610         private boolean verifySpaceOrEndComment() {
1611                 int startPosition = this.index;
1612                 // Whitespace or inline tag closing brace
1613                 char ch = peekChar();
1614                 switch (ch) {
1615                 case '}':
1616                         return this.inlineTagStarted;
1617                 default:
1618                         if (Character.isWhitespace(ch)) {
1619                                 return true;
1620                         }
1621                 }
1622                 // End of comment
1623                 int previousPosition = this.index;
1624                 this.starPosition = -1;
1625                 ch = readChar();
1626                 nextChar: while (this.index < this.source.length) {
1627                         switch (ch) {
1628                         case '*':
1629                                 // valid whatever the number of star before last '/'
1630                                 this.starPosition = previousPosition;
1631                                 break;
1632                         case '/':
1633                                 if (this.starPosition >= startPosition) { // valid only if a
1634                                                                                                                         // star was previous
1635                                                                                                                         // character
1636                                         return true;
1637                                 }
1638                         default:
1639                                 // invalid whatever other character, even white spaces
1640                                 this.index = startPosition;
1641                                 return false;
1642
1643                         }
1644                         previousPosition = this.index;
1645                         ch = readChar();
1646                 }
1647                 this.index = startPosition;
1648                 return false;
1649         }
1650 }