Created a separated 'externaltools' plugin
[phpeclipse.git] / net.sourceforge.phpeclipse / src / net / sourceforge / phpdt / internal / compiler / parser / AbstractCommentParser.java
1 /*******************************************************************************
2  * Copyright (c) 2000, 2004 IBM Corporation and others.
3  * All rights reserved. This program and the accompanying materials 
4  * are made available under the terms of the Common Public License v1.0
5  * which accompanies this distribution, and is available at
6  * http://www.eclipse.org/legal/cpl-v10.html
7  * 
8  * Contributors:
9  *     IBM Corporation - initial API and implementation
10  *******************************************************************************/
11 package net.sourceforge.phpdt.internal.compiler.parser;
12
13 import java.util.ArrayList;
14 import java.util.List;
15
16 import net.sourceforge.phpdt.core.compiler.CharOperation;
17 import net.sourceforge.phpdt.core.compiler.ITerminalSymbols;
18 import net.sourceforge.phpdt.core.compiler.InvalidInputException;
19
20 /**
21  * Parser specialized for decoding javadoc comments
22  */
23 public abstract class AbstractCommentParser {
24
25         // recognized tags
26         public static final char[] TAG_DEPRECATED = "deprecated".toCharArray(); //$NON-NLS-1$
27         public static final char[] TAG_PARAM = "param".toCharArray(); //$NON-NLS-1$
28         public static final char[] TAG_RETURN = "return".toCharArray(); //$NON-NLS-1$
29         public static final char[] TAG_THROWS = "throws".toCharArray(); //$NON-NLS-1$
30         public static final char[] TAG_EXCEPTION = "exception".toCharArray(); //$NON-NLS-1$
31         public static final char[] TAG_SEE = "see".toCharArray(); //$NON-NLS-1$
32         public static final char[] TAG_LINK = "link".toCharArray(); //$NON-NLS-1$
33         public static final char[] TAG_LINKPLAIN = "linkplain".toCharArray(); //$NON-NLS-1$
34         public static final char[] TAG_INHERITDOC = "inheritDoc".toCharArray(); //$NON-NLS-1$
35         
36         // tags expected positions
37         public final static int ORDERED_TAGS_NUMBER = 3;
38         public final static int PARAM_TAG_EXPECTED_ORDER = 0;
39         public final static int THROWS_TAG_EXPECTED_ORDER = 1;
40         public final static int SEE_TAG_EXPECTED_ORDER = 2;
41         
42         // Kind of comment parser
43         public final static int COMPIL_PARSER = 0x00000001;
44         public final static int DOM_PARSER = 0x00000002;
45         
46         // Public fields
47         public Scanner scanner;
48         public boolean checkDocComment = false;
49         
50         // Protected fields
51         protected boolean inherited, deprecated;
52         protected char[] source;
53         protected int index, endComment, lineEnd;
54         protected int tokenPreviousPosition, lastIdentifierEndPosition, starPosition;
55         protected int textStart, memberStart;
56         protected int tagSourceStart, tagSourceEnd;
57         protected int inlineTagStart;
58         protected Parser sourceParser;
59         protected Object returnStatement;
60         protected boolean lineStarted = false, inlineTagStarted = false;
61         protected int kind;
62         protected int[] lineEnds;
63         
64         // Private fields
65         private int currentTokenType = -1;
66         
67         // Line pointers
68         private int linePtr, lastLinePtr;
69         
70         // Identifier stack
71         protected int identifierPtr;
72         protected char[][] identifierStack;
73         protected int identifierLengthPtr;
74         protected int[] identifierLengthStack;
75         protected long[] identifierPositionStack;
76         // Ast stack
77         protected static int AstStackIncrement = 10;
78         protected int astPtr;
79         protected Object[] astStack;
80         protected int astLengthPtr;
81         protected int[] astLengthStack;
82
83         protected AbstractCommentParser(Parser sourceParser) {
84                 this.sourceParser = sourceParser;
85                 this.scanner = new Scanner(false, false, false, false, false, null, null, false);
86                    
87                 this.identifierStack = new char[20][];
88                 this.identifierPositionStack = new long[20];
89                 this.identifierLengthStack = new int[10];
90                 this.astStack = new Object[30];
91                 this.astLengthStack = new int[20];
92         }
93
94         /* (non-Javadoc)
95          * Returns true if tag @deprecated is present in javadoc comment.
96          * 
97          * If javadoc checking is enabled, will also construct an Javadoc node, which will be stored into Parser.javadoc
98          * slot for being consumed later on.
99          */
100         protected boolean parseComment(int javadocStart, int javadocEnd) {
101
102                 boolean validComment = true;
103                 try {
104                         // Init scanner position
105                         this.scanner.resetTo(javadocStart, javadocEnd);
106                         this.endComment = javadocEnd;
107                         this.index = javadocStart;
108                         readChar(); // starting '/'
109                         int previousPosition = this.index;
110                         readChar(); // first '*'
111                         char nextCharacter= readChar(); // second '*'
112                         
113                         // Init local variables
114                         this.astLengthPtr = -1;
115                         this.astPtr = -1;
116                         this.currentTokenType = -1;
117                         this.inlineTagStarted = false;
118                         this.inlineTagStart = -1;
119                         this.lineStarted = false;
120                         this.returnStatement = null;
121                         this.inherited = false;
122                         this.deprecated = false;
123                         this.linePtr = getLineNumber(javadocStart);
124                         this.lastLinePtr = getLineNumber(javadocEnd);
125                         this.lineEnd = (this.linePtr == this.lastLinePtr) ? this.endComment : this.scanner.getLineEnd(this.linePtr);
126                         this.textStart = -1;
127                         char previousChar = 0;
128                         int invalidTagLineEnd = -1;
129                         int invalidInlineTagLineEnd = -1;
130                         
131                         // Loop on each comment character
132                         while (this.index < this.endComment) {
133                                 previousPosition = this.index;
134                                 previousChar = nextCharacter;
135                                 
136                                 // Calculate line end (cannot use this.scanner.linePtr as scanner does not parse line ends again)
137                                 if (this.index > (this.lineEnd+1)) {
138                                         updateLineEnd();
139                                 }
140                                 
141                                 // Read next char only if token was consumed
142                                 if (this.currentTokenType < 0) {
143                                         nextCharacter = readChar(); // consider unicodes
144                                 } else {
145                                         previousPosition = this.scanner.getCurrentTokenStartPosition();
146                                         switch (this.currentTokenType) {
147                                                 case ITerminalSymbols.TokenNameRBRACE:
148                                                         nextCharacter = '}';
149                                                         break;
150                                                 case ITerminalSymbols.TokenNameMULTIPLY:
151                                                         nextCharacter = '*';
152                                                         break;
153                                         default:
154                                                         nextCharacter = this.scanner.currentCharacter;
155                                         }
156                                         consumeToken();
157                                 }
158                         
159                                 if (this.index >= this.endComment) {
160                                         break;
161                                 }
162                                 
163                                 switch (nextCharacter) {
164                                         case '@' :
165                                                 boolean valid = false;
166                                                 // Start tag parsing only if we have a java identifier start character and if we are on line beginning or at inline tag beginning
167                                                 if ((!this.lineStarted || previousChar == '{')) {
168                                                         this.lineStarted = true;
169                                                         if (this.inlineTagStarted) {
170                                                                 this.inlineTagStarted = false;
171                                                                 // bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=53279
172                                                                 // Cannot have @ inside inline comment
173                                                                 if (this.sourceParser != null) {
174                                                                         int end = previousPosition<invalidInlineTagLineEnd ? previousPosition : invalidInlineTagLineEnd;
175                                                                         this.sourceParser.problemReporter().javadocUnterminatedInlineTag(this.inlineTagStart, end);
176                                                                 }
177                                                                 validComment = false;
178                                                                 if (this.lineStarted && this.textStart != -1 && this.textStart < previousPosition) {
179                                                                         pushText(this.textStart, previousPosition);
180                                                                 }
181                                                                 if (this.kind == DOM_PARSER) refreshInlineTagPosition(previousPosition);
182                                                         }
183                                                         if (previousChar == '{') {
184                                                                 if (this.textStart != -1 && this.textStart < this.inlineTagStart) {
185                                                                         pushText(this.textStart, this.inlineTagStart);
186                                                                 }
187                                                                 this.inlineTagStarted = true;
188                                                                 invalidInlineTagLineEnd = this.lineEnd;
189                                                         } else if (this.textStart != -1 && this.textStart < invalidTagLineEnd) {
190                                                                 pushText(this.textStart, invalidTagLineEnd);
191                                                         }
192                                                         this.scanner.resetTo(this.index, this.endComment);
193                                                         this.currentTokenType = -1; // flush token cache at line begin
194                                                         try {
195                                                                 int token = readTokenAndConsume();
196                                                                 this.tagSourceStart = this.scanner.getCurrentTokenStartPosition();
197                                                                 this.tagSourceEnd = this.scanner.getCurrentTokenEndPosition();
198                                                                 char[] tag = this.scanner.getCurrentIdentifierSource(); // first token is either an identifier or a keyword
199                                                                 if (this.kind == DOM_PARSER) {
200                                                                         // For DOM parser, try to get tag name other than java identifier
201                                                                         // (see bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=51660)
202                                                                         int tk = token;
203                                                                         int le = this.lineEnd;
204                                                                         char pc = peekChar();
205                                                                         tagNameToken: while (tk != ITerminalSymbols.TokenNameEOF) {
206                                                                                 this.tagSourceEnd = this.scanner.getCurrentTokenEndPosition();
207                                                                                 token = tk;
208                                                                                 // !, ", #, %, &, ', -, :, <, >, * chars and spaces are not allowed in tag names
209                                                                                 switch (pc) {
210                                                                                         case '}':
211                                                                                         case '!':
212                                                                                         case '#':
213                                                                                         case '%':
214                                                                                         case '&':
215                                                                                         case '\'':
216                                                                                         case ':':
217                                                                                         // case '-': allowed in tag names as this character is often used in doclets (bug 68087)
218                                                                                         case '<':
219                                                                                         case '>':
220                                                                                         case '*': // break for '*' as this is perhaps the end of comment (bug 65288)
221                                                                                                 break tagNameToken;
222                                                                                         default:
223                                                                                                 if (pc == ' ' || Character.isWhitespace(pc)) break tagNameToken;
224                                                                                 }
225                                                                                 tk = readTokenAndConsume();
226                                                                                 pc = peekChar();
227                                                                         }
228                                                                         int length = this.tagSourceEnd-this.tagSourceStart+1;
229                                                                         tag = new char[length];
230                                                                         System.arraycopy(this.source, this.tagSourceStart, tag, 0, length);
231                                                                         this.index = this.tagSourceEnd+1;
232                                                                         this.scanner.currentPosition = this.tagSourceEnd+1;
233                                                                         this.tagSourceStart = previousPosition;
234                                                                         this.lineEnd = le;
235                                                                 }
236                                                                 switch (token) {
237                                                                         case ITerminalSymbols.TokenNameIdentifier :
238                                                                                 if (CharOperation.equals(tag, TAG_DEPRECATED)) {
239                                                                                         this.deprecated = true;
240                                                                                         if (this.kind == DOM_PARSER) {
241                                                                                                 valid = parseTag();
242                                                                                         } else {
243                                                                                                 valid = true;
244                                                                                         }
245                                                                                 } else if (CharOperation.equals(tag, TAG_INHERITDOC)) {
246                                                                                         // inhibits inherited flag when tags have been already stored
247                                                                                         // see bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=51606
248                                                                                         // Note that for DOM_PARSER, nodes stack may be not empty even no '@' tag
249                                                                                         // was encountered in comment. But it cannot be the case for COMPILER_PARSER
250                                                                                         // and so is enough as it is only this parser which signals the missing tag warnings...
251                                                                                         this.inherited = this.astPtr==-1;
252                                                                                         if (this.kind == DOM_PARSER) {
253                                                                                                 valid = parseTag();
254                                                                                         } else {
255                                                                                                 valid = true;
256                                                                                         }
257                                                                                 } else if (CharOperation.equals(tag, TAG_PARAM)) {
258                                                                                         valid = parseParam();
259                                                                                 } else if (CharOperation.equals(tag, TAG_EXCEPTION)) {
260                                                                                         valid = parseThrows(false);
261                                                                                 } else if (CharOperation.equals(tag, TAG_SEE)) {
262                                                                                         if (this.inlineTagStarted) {
263                                                                                                 // bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=53290
264                                                                                                 // Cannot have @see inside inline comment
265                                                                                                 valid = false;
266                                                                                                 if (this.sourceParser != null)
267                                                                                                         this.sourceParser.problemReporter().javadocUnexpectedTag(this.tagSourceStart, this.tagSourceEnd);
268                                                                                         } else {
269                                                                                                 valid = parseSee(false);
270                                                                                         }
271                                                                                 } else if (CharOperation.equals(tag, TAG_LINK)) {
272                                                                                         if (this.inlineTagStarted) {
273                                                                                                 valid = parseSee(false);
274                                                                                         } else {
275                                                                                                 // bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=53290
276                                                                                                 // Cannot have @link outside inline comment
277                                                                                                 valid = false;
278                                                                                                 if (this.sourceParser != null)
279                                                                                                         this.sourceParser.problemReporter().javadocUnexpectedTag(this.tagSourceStart, this.tagSourceEnd);
280                                                                                         }
281                                                                                 } else if (CharOperation.equals(tag, TAG_LINKPLAIN)) {
282                                                                                         if (this.inlineTagStarted) {
283                                                                                                 valid = parseSee(true);
284                                                                                         } else {
285                                                                                                 valid = parseTag();
286                                                                                         }
287                                                                                 } else {
288                                                                                         valid = parseTag();
289                                                                                 }
290                                                                                 break;
291                                                                         case ITerminalSymbols.TokenNamereturn :
292                                                                                 valid = parseReturn();
293                                                                                 // verify characters after return tag (we're expecting text description)
294                                                                                 if(!verifyCharsAfterReturnTag(this.index)) {
295                                                                                         if (this.sourceParser != null) {
296                                                                                                 int end = this.starPosition == -1 || this.lineEnd<this.starPosition ? this.lineEnd : this.starPosition;
297                                                                                                 this.sourceParser.problemReporter().javadocInvalidTag(this.tagSourceStart, end);
298                                                                                         }
299                                                                                 }
300                                                                                 break;
301 //                                                                      case ITerminalSymbols.TokenNamethrows :
302 //                                                                              valid = parseThrows(true);
303 //                                                                              break;
304                                                                         default:
305                                                                                 if (this.kind == DOM_PARSER) {
306                                                                                         switch (token) {
307                                                                                                 case ITerminalSymbols.TokenNameabstract:
308 //                                                                                              case ITerminalSymbols.TokenNameassert:
309 //                                                                                              case ITerminalSymbols.TokenNameboolean:
310                                                                                                 case ITerminalSymbols.TokenNamebreak:
311 //                                                                                              case ITerminalSymbols.TokenNamebyte:
312                                                                                                 case ITerminalSymbols.TokenNamecase:
313                                                                                                 case ITerminalSymbols.TokenNamecatch:
314 //                                                                                              case ITerminalSymbols.TokenNamechar:
315                                                                                                 case ITerminalSymbols.TokenNameclass:
316                                                                                                 case ITerminalSymbols.TokenNamecontinue:
317                                                                                                 case ITerminalSymbols.TokenNamedefault:
318                                                                                                 case ITerminalSymbols.TokenNamedo:
319 //                                                                                              case ITerminalSymbols.TokenNamedouble:
320                                                                                                 case ITerminalSymbols.TokenNameelse:
321                                                                                                 case ITerminalSymbols.TokenNameextends:
322 //                                                                                              case ITerminalSymbols.TokenNamefalse:
323                                                                                                 case ITerminalSymbols.TokenNamefinal:
324                                                                                                 case ITerminalSymbols.TokenNamefinally:
325 //                                                                                              case ITerminalSymbols.TokenNamefloat:
326                                                                                                 case ITerminalSymbols.TokenNamefor:
327                                                                                                 case ITerminalSymbols.TokenNameif:
328                                                                                                 case ITerminalSymbols.TokenNameimplements:
329 //                                                                                              case ITerminalSymbols.TokenNameimport:
330                                                                                                 case ITerminalSymbols.TokenNameinstanceof:
331 //                                                                                              case ITerminalSymbols.TokenNameint:
332                                                                                                 case ITerminalSymbols.TokenNameinterface:
333 //                                                                                              case ITerminalSymbols.TokenNamelong:
334 //                                                                                              case ITerminalSymbols.TokenNamenative:
335                                                                                                 case ITerminalSymbols.TokenNamenew:
336 //                                                                                              case ITerminalSymbols.TokenNamenull:
337 //                                                                                              case ITerminalSymbols.TokenNamepackage:
338                                                                                                 case ITerminalSymbols.TokenNameprivate:
339                                                                                                 case ITerminalSymbols.TokenNameprotected:
340                                                                                                 case ITerminalSymbols.TokenNamepublic:
341 //                                                                                              case ITerminalSymbols.TokenNameshort:
342                                                                                                 case ITerminalSymbols.TokenNamestatic:
343 //                                                                                              case ITerminalSymbols.TokenNamestrictfp:
344                                                                                                 case ITerminalSymbols.TokenNamesuper:
345                                                                                                 case ITerminalSymbols.TokenNameswitch:
346 //                                                                                              case ITerminalSymbols.TokenNamesynchronized:
347 //                                                                                              case ITerminalSymbols.TokenNamethis:
348                                                                                                 case ITerminalSymbols.TokenNamethrow:
349 //                                                                                              case ITerminalSymbols.TokenNametransient:
350 //                                                                                              case ITerminalSymbols.TokenNametrue:
351                                                                                                 case ITerminalSymbols.TokenNametry:
352 //                                                                                              case ITerminalSymbols.TokenNamevoid:
353 //                                                                                              case ITerminalSymbols.TokenNamevolatile:
354                                                                                                 case ITerminalSymbols.TokenNamewhile:
355                                                                                                         valid = parseTag();
356                                                                                                         break;
357                                                                                         }
358                                                                                 }
359                                                                 }
360                                                                 this.textStart = this.index;
361                                                                 if (!valid) {
362                                                                         // bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=51600
363                                                                         // do not stop the inline tag when error is encountered to get text after
364                                                                         validComment = false;
365                                                                         // bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=51600
366                                                                         // for DOM AST node, store tag as text in case of invalid syntax
367                                                                         if (this.kind == DOM_PARSER) {
368                                                                                 parseTag();
369                                                                                 this.textStart = this.tagSourceEnd+1;
370                                                                                 invalidTagLineEnd  = this.lineEnd;
371                                                                         }
372                                                                 }
373                                                         } catch (InvalidInputException e) {
374                                                                 consumeToken();
375                                                         }
376                                                 }
377                                                 break;
378                                         case '\r':
379                                         case '\n':
380                                                 if (this.lineStarted && this.textStart < previousPosition) {
381                                                         pushText(this.textStart, previousPosition);
382                                                 }
383                                                 this.lineStarted = false;
384                                                 // Fix bug 51650
385                                                 this.textStart = -1;
386                                                 break;
387                                         case '}' :
388                                                 if (this.inlineTagStarted) {
389                                                         if (this.lineStarted && this.textStart != -1 && this.textStart < previousPosition) {
390                                                                 pushText(this.textStart, previousPosition);
391                                                         }
392                                                         if (this.kind == DOM_PARSER) refreshInlineTagPosition(previousPosition);
393                                                         this.textStart = this.index;
394                                                         this.inlineTagStarted = false;
395                                                 } else {
396                                                         if (!this.lineStarted) {
397                                                                 this.textStart = previousPosition;
398                                                         }
399                                                 }
400                                                 this.lineStarted = true;
401                                                 break;
402                                         case '{' :
403                                                 if (this.inlineTagStarted) {
404                                                         this.inlineTagStarted = false;
405                                                         // bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=53279
406                                                         // Cannot have opening brace in inline comment
407                                                         if (this.sourceParser != null) {
408                                                                 int end = previousPosition<invalidInlineTagLineEnd ? previousPosition : invalidInlineTagLineEnd;
409                                                                 this.sourceParser.problemReporter().javadocUnterminatedInlineTag(this.inlineTagStart, end);
410                                                         }
411                                                         if (this.lineStarted && this.textStart != -1 && this.textStart < previousPosition) {
412                                                                 pushText(this.textStart, previousPosition);
413                                                         }
414                                                         if (this.kind == DOM_PARSER) refreshInlineTagPosition(previousPosition);
415                                                 }
416                                                 if (!this.lineStarted) {
417                                                         this.textStart = previousPosition;
418                                                 }
419                                                 this.lineStarted = true;
420                                                 this.inlineTagStart = previousPosition;
421                                                 break;
422                                         case '*' :
423                                         case '\u000c' : /* FORM FEED               */
424                                         case ' ' :                      /* SPACE                   */
425                                         case '\t' :                     /* HORIZONTAL TABULATION   */
426                                                 // do nothing for space or '*' characters
427                                                 break;
428                                         default :
429                                                 if (!this.lineStarted) {
430                                                         this.textStart = previousPosition;
431                                                 }
432                                                 this.lineStarted = true;
433                                                 break;
434                                 }
435                         }
436                         // bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=53279
437                         // Cannot leave comment inside inline comment
438                         if (this.inlineTagStarted) {
439                                 this.inlineTagStarted = false;
440                                 if (this.sourceParser != null) {
441                                         int end = previousPosition<invalidInlineTagLineEnd ? previousPosition : invalidInlineTagLineEnd;
442                                         if (this.index >= this.endComment) end = invalidInlineTagLineEnd;
443                                         this.sourceParser.problemReporter().javadocUnterminatedInlineTag(this.inlineTagStart, end);
444                                 }
445                                 if (this.lineStarted && this.textStart != -1 && this.textStart < previousPosition) {
446                                         pushText(this.textStart, previousPosition);
447                                 }
448                                 if (this.kind == DOM_PARSER) {
449                                         refreshInlineTagPosition(previousPosition);
450                                 }
451                         } else if (this.lineStarted && this.textStart < previousPosition) {
452                                 pushText(this.textStart, previousPosition);
453                         }
454                         updateDocComment();
455                 } catch (Exception ex) {
456                         validComment = false;
457                 }
458                 return validComment;
459         }
460
461         private void consumeToken() {
462                 this.currentTokenType = -1; // flush token cache
463                 updateLineEnd();
464         }
465
466         protected abstract Object createArgumentReference(char[] name, int dim, Object typeRef, long[] dimPos, long argNamePos) throws InvalidInputException;
467         protected abstract Object createFieldReference(Object receiver) throws InvalidInputException;
468         protected abstract Object createMethodReference(Object receiver, List arguments) throws InvalidInputException;
469         protected Object createReturnStatement() { return null; }
470         protected abstract Object createTypeReference(int primitiveToken);
471         
472         private int getEndPosition() {
473                 if (this.scanner.getCurrentTokenEndPosition() > this.lineEnd) {
474                         return this.lineEnd;
475                 } else {
476                         return this.scanner.getCurrentTokenEndPosition();
477                 }
478         }
479
480         /*
481          * Search the source position corresponding to the end of a given line number.
482          * Warning: returned position is 1-based index!
483          * @see Scanner#getLineEnd(int) We cannot directly use this method
484          * when linePtr field is not initialized.
485          *
486         private int getLineEnd(int lineNumber) {
487         
488                 if (this.scanner.linePtr != -1) {
489                         return this.scanner.getLineEnd(lineNumber);
490                 }
491                 if (this.lineEnds == null) 
492                         return -1;
493                 if (lineNumber > this.lineEnds.length+1) 
494                         return -1;
495                 if (lineNumber <= 0) 
496                         return -1;
497                 if (lineNumber == this.lineEnds.length + 1) 
498                         return this.scanner.eofPosition;
499                 return this.lineEnds[lineNumber-1]; // next line start one character behind the lineEnd of the previous line
500         }
501         */
502
503         /**
504          * Search the line number corresponding to a specific position.
505          * Warning: returned position is 1-based index!
506          * @see Scanner#getLineNumber(int) We cannot directly use this method
507          * when linePtr field is not initialized.
508          */
509         private int getLineNumber(int position) {
510         
511                 if (this.scanner.linePtr != -1) {
512                         return this.scanner.getLineNumber(position);
513                 }
514                 if (this.lineEnds == null)
515                         return 1;
516                 int length = this.lineEnds.length;
517                 if (length == 0)
518                         return 1;
519                 int g = 0, d = length - 1;
520                 int m = 0;
521                 while (g <= d) {
522                         m = (g + d) /2;
523                         if (position < this.lineEnds[m]) {
524                                 d = m-1;
525                         } else if (position > this.lineEnds[m]) {
526                                 g = m+1;
527                         } else {
528                                 return m + 1;
529                         }
530                 }
531                 if (position < this.lineEnds[m]) {
532                         return m+1;
533                 }
534                 return m+2;
535         }
536
537         /*
538          * Parse argument in @see tag method reference
539          */
540         private Object parseArguments(Object receiver) throws InvalidInputException {
541
542                 // Init
543                 int modulo = 0; // should be 2 for (Type,Type,...) or 3 for (Type arg,Type arg,...)
544                 int iToken = 0;
545                 char[] argName = null;
546                 List arguments = new ArrayList(10);
547                 int start = this.scanner.getCurrentTokenStartPosition();
548                 
549                 // Parse arguments declaration if method reference
550                 nextArg : while (this.index < this.scanner.eofPosition) {
551
552                         // Read argument type reference
553                         Object typeRef;
554                         try {
555                                 typeRef = parseQualifiedName(false);
556                         } catch (InvalidInputException e) {
557                                 break nextArg;
558                         }
559                         boolean firstArg = modulo == 0;
560                         if (firstArg) { // verify position
561                                 if (iToken != 0)
562                                         break nextArg;
563                         } else if ((iToken % modulo) != 0) {
564                                         break nextArg;
565                         }
566                         if (typeRef == null) {
567                                 if (firstArg && this.currentTokenType == ITerminalSymbols.TokenNameRPAREN) {
568                                         // verify characters after arguments declaration (expecting white space or end comment)
569                                         if (!verifySpaceOrEndComment()) {
570                                                 int end = this.starPosition == -1 ? this.lineEnd : this.starPosition;
571                                                 if (this.source[end]=='\n') end--;
572                                                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocMalformedSeeReference(start, end);
573                                                 return null;
574                                         }
575                                         this.lineStarted = true;
576                                         return createMethodReference(receiver, null);
577                                 }
578                                 break nextArg;
579                         }
580                         iToken++;
581
582                         // Read possible array declaration
583                         int dim = 0;
584                         long[] dimPositions = new long[20]; // assume that there won't be more than 20 dimensions...
585                         if (readToken() == ITerminalSymbols.TokenNameLBRACKET) {
586                                 int dimStart = this.scanner.getCurrentTokenStartPosition();
587                                 while (readToken() == ITerminalSymbols.TokenNameLBRACKET) {
588                                         consumeToken();
589                                         if (readToken() != ITerminalSymbols.TokenNameRBRACKET) {
590                                                 break nextArg;
591                                         }
592                                         consumeToken();
593                                         dimPositions[dim++] = (((long) dimStart) << 32) + this.scanner.getCurrentTokenEndPosition();
594                                 }
595                         }
596
597                         // Read argument name
598                         long argNamePos = -1;
599                         if (readToken() == ITerminalSymbols.TokenNameIdentifier) {
600                                 consumeToken();
601                                 if (firstArg) { // verify position
602                                         if (iToken != 1)
603                                                 break nextArg;
604                                 } else if ((iToken % modulo) != 1) {
605                                                 break nextArg;
606                                 }
607                                 if (argName == null) { // verify that all arguments name are declared
608                                         if (!firstArg) {
609                                                 break nextArg;
610                                         }
611                                 }
612                                 argName = this.scanner.getCurrentIdentifierSource();
613                                 argNamePos = (((long)this.scanner.getCurrentTokenStartPosition())<<32)+this.scanner.getCurrentTokenEndPosition();
614                                 iToken++;
615                         } else if (argName != null) { // verify that no argument name is declared
616                                 break nextArg;
617                         }
618                         
619                         // Verify token position
620                         if (firstArg) {
621                                 modulo = iToken + 1;
622                         } else {
623                                 if ((iToken % modulo) != (modulo - 1)) {
624                                         break nextArg;
625                                 }
626                         }
627
628                         // Read separator or end arguments declaration
629                         int token = readToken();
630                         char[] name = argName == null ? new char[0] : argName;
631                         if (token == ITerminalSymbols.TokenNameCOMMA) {
632                                 // Create new argument
633                                 Object argument = createArgumentReference(name, dim, typeRef, dimPositions, argNamePos);
634                                 arguments.add(argument);
635                                 consumeToken();
636                                 iToken++;
637                         } else if (token == ITerminalSymbols.TokenNameRPAREN) {
638                                 // verify characters after arguments declaration (expecting white space or end comment)
639                                 if (!verifySpaceOrEndComment()) {
640                                         int end = this.starPosition == -1 ? this.lineEnd : this.starPosition;
641                                         if (this.source[end]=='\n') end--;
642                                         if (this.sourceParser != null) this.sourceParser.problemReporter().javadocMalformedSeeReference(start, end);
643                                         return null;
644                                 }
645                                 // Create new argument
646                                 Object argument = createArgumentReference(name, dim, typeRef, dimPositions, argNamePos);
647                                 arguments.add(argument);
648                                 consumeToken();
649                                 return createMethodReference(receiver, arguments);
650                         } else {
651                                 break nextArg;
652                         }
653                 }
654
655                 // Something wrong happened => Invalid input
656                 throw new InvalidInputException();
657         }
658
659         /*
660          * Parse an URL link reference in @see tag
661          */
662         private boolean parseHref() throws InvalidInputException {
663                 int start = this.scanner.getCurrentTokenStartPosition();
664                 if (Character.toLowerCase(readChar()) == 'a') {
665                         this.scanner.currentPosition = this.index;
666                         if (readToken() == ITerminalSymbols.TokenNameIdentifier) {
667                                 this.currentTokenType = -1; // do not update line end
668                                 try {
669                                         if (CharOperation.equals(this.scanner.getCurrentIdentifierSource(), new char[]{'h', 'r', 'e', 'f'}, false) &&
670                                                 readToken() == ITerminalSymbols.TokenNameEQUAL) {
671                                                 this.currentTokenType = -1; // do not update line end
672                                                 if (readToken() == ITerminalSymbols.TokenNameStringDoubleQuote ||
673                                                     readToken() == ITerminalSymbols.TokenNameStringSingleQuote) {
674                                                         this.currentTokenType = -1; // do not update line end
675                                                         // Skip all characters after string literal until closing '>' (see bug 68726)
676                                                         while (this.index <= this.lineEnd && readToken() != ITerminalSymbols.TokenNameGREATER) {
677                                                                 this.currentTokenType = -1; // do not update line end
678                                                         }
679                                                         if (this.currentTokenType == ITerminalSymbols.TokenNameGREATER) {
680                                                                 consumeToken(); // update line end as new lines are allowed in URL description
681                                                                 while (readToken() != ITerminalSymbols.TokenNameLESS) {
682                                                                         if (this.scanner.currentPosition >= this.scanner.eofPosition || this.scanner.currentCharacter == '@') {
683                                                                                 // Reset position: we want to rescan last token
684                                                                                 this.index = this.tokenPreviousPosition;
685                                                                                 this.scanner.currentPosition = this.tokenPreviousPosition;
686                                                                                 this.currentTokenType = -1;
687                                                                                 // Signal syntax error
688                                                                                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidSeeUrlReference(start, this.lineEnd);
689                                                                                 return false;
690                                                                         }
691                                                                         consumeToken();
692                                                                 }
693                                                                 this.currentTokenType = -1; // do not update line end
694                                                                 if (readChar() == '/') {
695                                                                         if (Character.toLowerCase(readChar()) == 'a') {
696                                                                                 if (readChar() == '>') {
697                                                                                         // Valid href
698                                                                                         return true;
699                                                                                 }
700                                                                         }
701                                                                 }
702                                                         }
703                                                 }
704                                         }
705                                 } catch (InvalidInputException ex) {
706                                         // Do nothing as we want to keep positions for error message
707                                 }
708                         }
709                 }
710                 // Reset position: we want to rescan last token
711                 this.index = this.tokenPreviousPosition;
712                 this.scanner.currentPosition = this.tokenPreviousPosition;
713                 this.currentTokenType = -1;
714                 // Signal syntax error
715                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidSeeUrlReference(start, this.lineEnd);
716                 return false;
717         }
718
719         /*
720          * Parse a method reference in @see tag
721          */
722         private Object parseMember(Object receiver) throws InvalidInputException {
723                 // Init
724                 this.identifierPtr = -1;
725                 this.identifierLengthPtr = -1;
726                 int start = this.scanner.getCurrentTokenStartPosition();
727                 this.memberStart = start;
728
729                 // Get member identifier
730                 if (readToken() == ITerminalSymbols.TokenNameIdentifier) {
731                         consumeToken();
732                         pushIdentifier(true);
733                         // Look for next token to know whether it's a field or method reference
734                         int previousPosition = this.index;
735                         if (readToken() == ITerminalSymbols.TokenNameLPAREN) {
736                                 consumeToken();
737                                 start = this.scanner.getCurrentTokenStartPosition();
738                                 try {
739                                         return parseArguments(receiver);
740                                 } catch (InvalidInputException e) {
741                                         int end = this.scanner.getCurrentTokenEndPosition() < this.lineEnd ?
742                                                         this.scanner.getCurrentTokenEndPosition() :
743                                                         this.scanner.getCurrentTokenStartPosition();
744                                         end = end < this.lineEnd ? end : this.lineEnd;
745                                         if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidSeeReferenceArgs(start, end);
746                                 }
747                                 return null;
748                         }
749
750                         // Reset position: we want to rescan last token
751                         this.index = previousPosition;
752                         this.scanner.currentPosition = previousPosition;
753                         this.currentTokenType = -1;
754
755                         // Verify character(s) after identifier (expecting space or end comment)
756                         if (!verifySpaceOrEndComment()) {
757                                 int end = this.starPosition == -1 ? this.lineEnd : this.starPosition;
758                                 if (this.source[end]=='\n') end--;
759                                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocMalformedSeeReference(start, end);
760                                 return null;
761                         }
762                         return createFieldReference(receiver);
763                 }
764                 int end = getEndPosition() - 1;
765                 end = start > end ? start : end;
766                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidSeeReference(start, end);
767                 // Reset position: we want to rescan last token
768                 this.index = this.tokenPreviousPosition;
769                 this.scanner.currentPosition = this.tokenPreviousPosition;
770                 this.currentTokenType = -1;
771                 return null;
772         }
773
774         /*
775          * Parse @param tag declaration
776          */
777         protected boolean parseParam() {
778
779                 // Store current token state
780                 int start = this.tagSourceStart;
781                 int end = this.tagSourceEnd;
782
783                 try {
784                         // Push identifier next
785                         int token = readToken();
786                         switch (token) {
787                                 case ITerminalSymbols.TokenNameIdentifier :
788                                         consumeToken();
789                                         return pushParamName();
790                                 case ITerminalSymbols.TokenNameEOF :
791                                         break;
792                                 default :
793                                         start = this.scanner.getCurrentTokenStartPosition();
794                                         end = getEndPosition();
795                                         if (end < start) start = this.tagSourceStart;
796                                         break;
797                         }
798                 } catch (InvalidInputException e) {
799                         end = getEndPosition();
800                 }
801
802                 // Reset position to avoid missing tokens when new line was encountered
803                 this.index = this.tokenPreviousPosition;
804                 this.scanner.currentPosition = this.tokenPreviousPosition;
805                 this.currentTokenType = -1;
806
807                 // Report problem
808                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocMissingParamName(start, end);
809                 return false;
810         }
811
812         /*
813          * Parse a qualified name and built a type reference if the syntax is valid.
814          */
815         protected Object parseQualifiedName(boolean reset) throws InvalidInputException {
816
817                 // Reset identifier stack if requested
818                 if (reset) {
819                         this.identifierPtr = -1;
820                         this.identifierLengthPtr = -1;
821                 }
822
823                 // Scan tokens
824                 int primitiveToken = -1;
825                 nextToken : for (int iToken = 0; ; iToken++) {
826                         int token = readToken();
827                         switch (token) {
828                                 case ITerminalSymbols.TokenNameIdentifier :
829                                         if (((iToken % 2) > 0)) { // identifiers must be odd tokens
830                                                 break nextToken;
831                                         }
832                                         pushIdentifier(iToken == 0);
833                                         consumeToken();
834                                         break;
835
836                                 case ITerminalSymbols.TokenNameDOT :
837                                         if ((iToken % 2) == 0) { // dots must be even tokens
838                                                 throw new InvalidInputException();
839                                         }
840                                         consumeToken();
841                                         break;
842
843 //                              case ITerminalSymbols.TokenNamevoid :
844 //                              case ITerminalSymbols.TokenNameboolean :
845 //                              case ITerminalSymbols.TokenNamebyte :
846 //                              case ITerminalSymbols.TokenNamechar :
847 //                              case ITerminalSymbols.TokenNamedouble :
848 //                              case ITerminalSymbols.TokenNamefloat :
849 //                              case ITerminalSymbols.TokenNameint :
850 //                              case ITerminalSymbols.TokenNamelong :
851 //                              case ITerminalSymbols.TokenNameshort :
852 //                                      if (iToken > 0) {
853 //                                              throw new InvalidInputException();
854 //                                      }
855 //                                      pushIdentifier(true);
856 //                                      primitiveToken = token;
857 //                                      consumeToken();
858 //                                      break nextToken;
859
860                                 default :
861                                         if (iToken == 0) {
862                                                 return null;
863                                         }
864                                         if ((iToken % 2) == 0) { // cannot leave on a dot
865                                                 // Reset position: we want to rescan last token
866                                                 if (this.kind == DOM_PARSER && this.currentTokenType != -1) {
867                                                         this.index = this.tokenPreviousPosition;
868                                                         this.scanner.currentPosition = this.tokenPreviousPosition;
869                                                         this.currentTokenType = -1;
870                                                 }
871                                                 throw new InvalidInputException();
872                                         }
873                                         break nextToken;
874                         }
875                 }
876                 // Reset position: we want to rescan last token
877                 if (this.currentTokenType != -1) {
878                         this.index = this.tokenPreviousPosition;
879                         this.scanner.currentPosition = this.tokenPreviousPosition;
880                         this.currentTokenType = -1;
881                 }
882                 this.lastIdentifierEndPosition = (int) this.identifierPositionStack[this.identifierPtr];
883                 return createTypeReference(primitiveToken);
884         }
885
886         /*
887          * Parse a reference in @see tag
888          */
889         protected boolean parseReference(boolean plain) throws InvalidInputException {
890                 Object typeRef = null;
891                 Object reference = null;
892                 int previousPosition = -1;
893                 int typeRefStartPosition = -1;
894                 nextToken : while (this.index < this.scanner.eofPosition) {
895                         previousPosition = this.index;
896                         int token = readToken();
897                         switch (token) {
898                                 case ITerminalSymbols.TokenNameStringDoubleQuote : // @see "string"
899                                 case ITerminalSymbols.TokenNameStringSingleQuote :
900                                         int start = this.scanner.getCurrentTokenStartPosition();
901                                         consumeToken();
902                                         // If typeRef != null we may raise a warning here to let user know there's an unused reference...
903                                         // Currently as javadoc 1.4.2 ignore it, we do the same (see bug 69302)
904                                         if (typeRef != null) {
905                                                 start = this.tagSourceEnd+1;
906                                                 previousPosition = start;
907                                                 typeRef = null;
908                                         }
909                                         // verify end line (expecting empty or end comment)
910                                         if (verifyEndLine(previousPosition)) {
911                                                 return true;
912                                         }
913                                         if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidSeeReference(start, this.lineEnd);
914                                         return false;
915                                 case ITerminalSymbols.TokenNameLESS : // @see "<a href="URL#Value">label</a>
916                                         consumeToken();
917                                         start = this.scanner.getCurrentTokenStartPosition();
918                                         if (parseHref()) {
919                                                 consumeToken();
920                                                 // If typeRef != null we may raise a warning here to let user know there's an unused reference...
921                                                 // Currently as javadoc 1.4.2 ignore it, we do the same (see bug 69302)
922                                                 if (typeRef != null) {
923                                                         start = this.tagSourceEnd+1;
924                                                         previousPosition = start;
925                                                         typeRef = null;
926                                                 }
927                                                 // verify end line (expecting empty or end comment)
928                                                 if (verifyEndLine(previousPosition)) {
929                                                         return true;
930                                                 }
931                                                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidSeeReference(start, this.lineEnd);
932                                         }
933                                         return false;
934                                 case ITerminalSymbols.TokenNameERROR :
935                                         if (this.scanner.currentCharacter == '#') { // @see ...#member
936                                                 consumeToken();
937                                                 reference = parseMember(typeRef);
938                                                 if (reference != null) {
939                                                         return pushSeeRef(reference, plain);
940                                                 }
941                                                 return false;
942                                         }
943                                         break nextToken;
944                                 case ITerminalSymbols.TokenNameIdentifier :
945                                         if (typeRef == null) {
946                                                 typeRefStartPosition = this.scanner.getCurrentTokenStartPosition();
947                                                 typeRef = parseQualifiedName(true);
948                                                 break;
949                                         }
950                                         break nextToken;
951                                 default :
952                                         break nextToken;
953                         }
954                 }
955                 
956                 // Verify that we got a reference
957                 if (reference == null) reference = typeRef;
958                 if (reference == null) {
959                         this.index = this.tokenPreviousPosition;
960                         this.scanner.currentPosition = this.tokenPreviousPosition;
961                         this.currentTokenType = -1;
962                         if (this.sourceParser != null) this.sourceParser.problemReporter().javadocMissingSeeReference(this.tagSourceStart, this.tagSourceEnd);
963                         return false;
964                 }
965
966                 // Reset position at the end of type reference
967                 this.index = this.lastIdentifierEndPosition+1;
968                 this.scanner.currentPosition = this.index;
969                 this.currentTokenType = -1;
970
971                 // Verify that line end does not start with an open parenthese (which could be a constructor reference wrongly written...)
972                 // See bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=47215
973                 char ch = peekChar();
974                 if (ch == '(') {
975                         if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidSeeReference(typeRefStartPosition, this.lineEnd);
976                         return false;
977                 }
978
979                 // Verify that we get white space after reference
980                 if (!verifySpaceOrEndComment()) {
981                         this.index = this.tokenPreviousPosition;
982                         this.scanner.currentPosition = this.tokenPreviousPosition;
983                         this.currentTokenType = -1;
984                         int end = this.starPosition == -1 ? this.lineEnd : this.starPosition;
985                         if (this.source[end]=='\n') end--;
986                         if (this.sourceParser != null) this.sourceParser.problemReporter().javadocMalformedSeeReference(typeRefStartPosition, end);
987                         return false;
988                 }
989                 
990                 // Everything is OK, store reference
991                 return pushSeeRef(reference, plain);
992         }
993
994         /*
995          * Parse @return tag declaration
996          */
997         protected abstract boolean parseReturn();
998
999         /*
1000          * Parse @see tag declaration
1001          */
1002         protected boolean parseSee(boolean plain) {
1003                 int start = this.scanner.currentPosition;
1004                 try {
1005                         return parseReference(plain);
1006                 } catch (InvalidInputException ex) {
1007                                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidSeeReference(start, getEndPosition());
1008                 }
1009                 // Reset position to avoid missing tokens when new line was encountered
1010                 this.index = this.tokenPreviousPosition;
1011                 this.scanner.currentPosition = this.tokenPreviousPosition;
1012                 this.currentTokenType = -1;
1013                 return false;
1014         }
1015
1016         /*
1017          * Parse @return tag declaration
1018          */
1019         protected abstract boolean parseTag();
1020
1021         /*
1022          * Parse @throws tag declaration
1023          */
1024         protected boolean parseThrows(boolean real) {
1025                 int start = this.scanner.currentPosition;
1026                 try {
1027                         Object typeRef = parseQualifiedName(true);
1028                         if (typeRef == null) {
1029                                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocMissingThrowsClassName(this.tagSourceStart, this.tagSourceEnd);
1030                         } else {
1031                                 return pushThrowName(typeRef, real);
1032                         }
1033                 } catch (InvalidInputException ex) {
1034                         if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidThrowsClass(start, getEndPosition());
1035                 }
1036                 return false;
1037         }
1038
1039         /*
1040          * Return current character without move index position.
1041          */
1042         private char peekChar() {
1043                 int idx = this.index;
1044                 char c = this.source[idx++];
1045                 if (c == '\\' && this.source[idx] == 'u') {
1046                         int c1, c2, c3, c4;
1047                         idx++;
1048                         while (this.source[idx] == 'u')
1049                                 idx++;
1050                         if (!(((c1 = Character.getNumericValue(this.source[idx++])) > 15 || c1 < 0)
1051                                         || ((c2 = Character.getNumericValue(this.source[idx++])) > 15 || c2 < 0)
1052                                         || ((c3 = Character.getNumericValue(this.source[idx++])) > 15 || c3 < 0) || ((c4 = Character.getNumericValue(this.source[idx++])) > 15 || c4 < 0))) {
1053                                 c = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1054                         }
1055                 }
1056                 return c;
1057         }
1058
1059         /*
1060          * push the consumeToken on the identifier stack. Increase the total number of identifier in the stack.
1061          */
1062         protected void pushIdentifier(boolean newLength) {
1063
1064                 int stackLength = this.identifierStack.length;
1065                 if (++this.identifierPtr >= stackLength) {
1066                         System.arraycopy(
1067                                 this.identifierStack, 0,
1068                                 this.identifierStack = new char[stackLength + 10][], 0,
1069                                 stackLength);
1070                         System.arraycopy(
1071                                 this.identifierPositionStack, 0,
1072                                 this.identifierPositionStack = new long[stackLength + 10], 0,
1073                                 stackLength);
1074                 }
1075                 this.identifierStack[this.identifierPtr] = this.scanner.getCurrentIdentifierSource();
1076                 this.identifierPositionStack[this.identifierPtr] = (((long) this.scanner.startPosition) << 32) + (this.scanner.currentPosition - 1);
1077
1078                 if (newLength) {
1079                         stackLength = this.identifierLengthStack.length;
1080                         if (++this.identifierLengthPtr >= stackLength) {
1081                                 System.arraycopy(
1082                                         this.identifierLengthStack, 0,
1083                                         this.identifierLengthStack = new int[stackLength + 10], 0,
1084                                         stackLength);
1085                         }
1086                         this.identifierLengthStack[this.identifierLengthPtr] = 1;
1087                 } else {
1088                         this.identifierLengthStack[this.identifierLengthPtr]++;
1089                 }
1090         }
1091
1092         /*
1093          * Add a new obj on top of the ast stack.
1094          * If new length is required, then add also a new length in length stack.
1095          */
1096         protected void pushOnAstStack(Object node, boolean newLength) {
1097
1098                 if (node == null) {
1099                         this.astLengthStack[++this.astLengthPtr] = 0;
1100                         return;
1101                 }
1102
1103                 int stackLength = this.astStack.length;
1104                 if (++this.astPtr >= stackLength) {
1105                         System.arraycopy(
1106                                 this.astStack, 0,
1107                                 this.astStack = new Object[stackLength + AstStackIncrement], 0,
1108                                 stackLength);
1109                         this.astPtr = stackLength;
1110                 }
1111                 this.astStack[this.astPtr] = node;
1112
1113                 if (newLength) {
1114                         stackLength = this.astLengthStack.length;
1115                         if (++this.astLengthPtr >= stackLength) {
1116                                 System.arraycopy(
1117                                         this.astLengthStack, 0,
1118                                         this.astLengthStack = new int[stackLength + AstStackIncrement], 0,
1119                                         stackLength);
1120                         }
1121                         this.astLengthStack[this.astLengthPtr] = 1;
1122                 } else {
1123                         this.astLengthStack[this.astLengthPtr]++;
1124                 }
1125         }
1126
1127         /*
1128          * Push a param name in ast node stack.
1129          */
1130         protected abstract boolean pushParamName();
1131
1132         /*
1133          * Push a reference statement in ast node stack.
1134          */
1135         protected abstract boolean pushSeeRef(Object statement, boolean plain);
1136
1137         /*
1138          * Push a text element in ast node stack
1139          */
1140         protected abstract void pushText(int start, int end);
1141
1142         /*
1143          * Push a throws type ref in ast node stack.
1144          */
1145         protected abstract boolean pushThrowName(Object typeRef, boolean real);
1146
1147         /*
1148          * Read current character and move index position.
1149          * Warning: scanner position is unchanged using this method!
1150          */
1151         protected char readChar() {
1152         
1153                 char c = this.source[this.index++];
1154                 if (c == '\\' && this.source[this.index] == 'u') {
1155                         int c1, c2, c3, c4;
1156                         int pos = this.index;
1157                         this.index++;
1158                         while (this.source[this.index] == 'u')
1159                                 this.index++;
1160                         if (!(((c1 = Character.getNumericValue(this.source[this.index++])) > 15 || c1 < 0)
1161                                         || ((c2 = Character.getNumericValue(this.source[this.index++])) > 15 || c2 < 0)
1162                                         || ((c3 = Character.getNumericValue(this.source[this.index++])) > 15 || c3 < 0) || ((c4 = Character.getNumericValue(this.source[this.index++])) > 15 || c4 < 0))) {
1163                                 c = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1164                         } else {
1165                                 // TODO (frederic) currently reset to previous position, perhaps signal a syntax error would be more appropriate
1166                                 this.index = pos;
1167                         }
1168                 }
1169                 return c;
1170         }
1171
1172         /*
1173          * Read token only if previous was consumed
1174          */
1175         private int readToken() throws InvalidInputException {
1176                 if (this.currentTokenType < 0) {
1177                         this.tokenPreviousPosition = this.scanner.currentPosition;
1178                         this.currentTokenType = this.scanner.getNextToken();
1179                         if (this.scanner.currentPosition > (this.lineEnd+1)) { // be sure to be on next line (lineEnd is still on the same line)
1180                                 this.lineStarted = false;
1181                                 while (this.currentTokenType == ITerminalSymbols.TokenNameMULTIPLY) {
1182                                         this.currentTokenType = this.scanner.getNextToken();
1183                                 }
1184                         }
1185                         this.index = this.scanner.currentPosition;
1186                         this.lineStarted = true; // after having read a token, line is obviously started...
1187                 }
1188                 return this.currentTokenType;
1189         }
1190
1191         private int readTokenAndConsume() throws InvalidInputException {
1192                 int token = readToken();
1193                 consumeToken();
1194                 return token;
1195         }
1196         
1197         /*
1198          * Refresh start position and length of an inline tag.
1199          */
1200         protected void refreshInlineTagPosition(int previousPosition) {
1201                 // do nothing by default
1202         }
1203
1204         public String toString() {
1205                 StringBuffer buffer = new StringBuffer();
1206                 int startPos = this.scanner.currentPosition<this.index ? this.scanner.currentPosition : this.index;
1207                 int endPos = this.scanner.currentPosition<this.index ? this.index : this.scanner.currentPosition;
1208                 if (startPos == this.source.length)
1209                         return "EOF\n\n" + new String(this.source); //$NON-NLS-1$
1210                 if (endPos > this.source.length)
1211                         return "behind the EOF\n\n" + new String(this.source); //$NON-NLS-1$
1212         
1213                 char front[] = new char[startPos];
1214                 System.arraycopy(this.source, 0, front, 0, startPos);
1215         
1216                 int middleLength = (endPos - 1) - startPos + 1;
1217                 char middle[];
1218                 if (middleLength > -1) {
1219                         middle = new char[middleLength];
1220                         System.arraycopy(
1221                                 this.source, 
1222                                 startPos, 
1223                                 middle, 
1224                                 0, 
1225                                 middleLength);
1226                 } else {
1227                         middle = CharOperation.NO_CHAR;
1228                 }
1229                 
1230                 char end[] = new char[this.source.length - (endPos - 1)];
1231                 System.arraycopy(
1232                         this.source, 
1233                         (endPos - 1) + 1, 
1234                         end, 
1235                         0, 
1236                         this.source.length - (endPos - 1) - 1);
1237                 
1238                 buffer.append(front);
1239                 if (this.scanner.currentPosition<this.index) {
1240                         buffer.append("\n===============================\nScanner current position here -->"); //$NON-NLS-1$
1241                 } else {
1242                         buffer.append("\n===============================\nParser index here -->"); //$NON-NLS-1$
1243                 }
1244                 buffer.append(middle);
1245                 if (this.scanner.currentPosition<this.index) {
1246                         buffer.append("<-- Parser index here\n===============================\n"); //$NON-NLS-1$
1247                 } else {
1248                         buffer.append("<-- Scanner current position here\n===============================\n"); //$NON-NLS-1$
1249                 }
1250                 buffer.append(end);
1251
1252                 return buffer.toString();
1253         }
1254
1255         /*
1256          * Update 
1257          */
1258         protected abstract void updateDocComment();
1259
1260         /*
1261          * Update line end
1262          */
1263         protected void updateLineEnd() {
1264                 while (this.index > (this.lineEnd+1)) { // be sure to be on next line (lineEnd is still on the same line)
1265                         if (this.linePtr < this.lastLinePtr) {
1266                                 this.lineEnd = this.scanner.getLineEnd(++this.linePtr) - 1;
1267                         } else {
1268                                 this.lineEnd = this.endComment;
1269                                 return;
1270                         }
1271                 }
1272         }
1273
1274         /*
1275          * Verify that end of the line only contains space characters or end of comment.
1276          * Note that end of comment may be preceeding by several contiguous '*' chars.
1277          */
1278         private boolean verifyEndLine(int textPosition) {
1279                 int startPosition = this.index;
1280                 int previousPosition = this.index;
1281                 this.starPosition = -1;
1282                 char ch = readChar();
1283                 nextChar: while (true) {
1284                         switch (ch) {
1285                                 case '\r':
1286                                 case '\n':
1287                                         if (this.kind == DOM_PARSER) {
1288                                                 parseTag();
1289                                                 pushText(textPosition, previousPosition);
1290                                         }
1291                                         this.index = previousPosition;
1292                                         return true;
1293                                 case '\u000c' : /* FORM FEED               */
1294                                 case ' ' :                      /* SPACE                   */
1295                                 case '\t' :                     /* HORIZONTAL TABULATION   */
1296                                         if (this.starPosition >= 0) break nextChar;
1297                                         break;
1298                                 case '*':
1299                                         this.starPosition = previousPosition;
1300                                         break;
1301                                 case '/':
1302                                         if (this.starPosition >= textPosition) {
1303                                                 if (this.kind == DOM_PARSER) {
1304                                                         parseTag();
1305                                                         pushText(textPosition, this.starPosition);
1306                                                 }
1307                                                 return true;
1308                                         }
1309                                 default :
1310                                         // leave loop
1311                                         break nextChar;
1312                                 
1313                         }
1314                         previousPosition = this.index;
1315                         ch = readChar();
1316                 }
1317                 this.index = startPosition;
1318                 return false;
1319         }
1320
1321         /*
1322          * Verify that some text exists after a @return tag. Text must be different than
1323          * end of comment which may be preceeding by several '*' chars.
1324          */
1325         private boolean verifyCharsAfterReturnTag(int startPosition) {
1326                 // Whitespace or inline tag closing brace
1327                 int previousPosition = this.index;
1328                 char ch = readChar();
1329                 boolean malformed = true;
1330                 while (Character.isWhitespace(ch)) {
1331                         malformed = false;
1332                         previousPosition = this.index;
1333                         ch = readChar();        
1334                 }
1335                 // End of comment
1336                 this.starPosition = -1;
1337                 nextChar: while (this.index<this.source.length) {
1338                         switch (ch) {
1339                                 case '*':
1340                                         // valid whatever the number of star before last '/'
1341                                         this.starPosition = previousPosition;
1342                                         break;
1343                                 case '/':
1344                                         if (this.starPosition >= startPosition) { // valid only if a star was previous character
1345                                                 return false;
1346                                         }
1347                                 default :
1348                                         // valid if any other character is encountered, even white spaces
1349                                         this.index = startPosition;
1350                                         return !malformed;
1351                                 
1352                         }
1353                         previousPosition = this.index;
1354                         ch = readChar();
1355                 }
1356                 this.index = startPosition;
1357                 return false;
1358         }
1359
1360         /*
1361          * Verify characters after a name matches one of following conditions:
1362          *      1- first character is a white space
1363          *      2- first character is a closing brace *and* we're currently parsing an inline tag
1364          *      3- are the end of comment (several contiguous star ('*') characters may be
1365          *          found before the last slash ('/') character).
1366          */
1367         private boolean verifySpaceOrEndComment() {
1368                 int startPosition = this.index;
1369                 // Whitespace or inline tag closing brace
1370                 char ch = peekChar();
1371                 switch (ch) {
1372                         case '}':
1373                                 return this.inlineTagStarted;
1374                         default:
1375                                 if (Character.isWhitespace(ch)) {
1376                                         return true;
1377                                 }
1378                 }
1379                 // End of comment
1380                 int previousPosition = this.index;
1381                 this.starPosition = -1;
1382                 ch = readChar();
1383                 nextChar: while (this.index<this.source.length) {
1384                         switch (ch) {
1385                                 case '*':
1386                                         // valid whatever the number of star before last '/'
1387                                         this.starPosition = previousPosition;
1388                                         break;
1389                                 case '/':
1390                                         if (this.starPosition >= startPosition) { // valid only if a star was previous character
1391                                                 return true;
1392                                         }
1393                                 default :
1394                                         // invalid whatever other character, even white spaces
1395                                         this.index = startPosition;
1396                                         return false;
1397                                 
1398                         }
1399                         previousPosition = this.index;
1400                         ch = readChar();
1401                 }
1402                 this.index = startPosition;
1403                 return false;
1404         }
1405 }