net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/AbstractCommentParser.java

   1 /*******************************************************************************
   2  * Copyright (c) 2000, 2004 IBM Corporation and others.
   3  * All rights reserved. This program and the accompanying materials
   4  * are made available under the terms of the Common Public License v1.0
   5  * which accompanies this distribution, and is available at
   6  * http://www.eclipse.org/legal/cpl-v10.html
   7  *
   8  * Contributors:
   9  *     IBM Corporation - initial API and implementation
  10  *******************************************************************************/
  11 package net.sourceforge.phpdt.internal.compiler.parser;
  12
  13 import java.util.ArrayList;
  14 import java.util.List;
  15
  16 import net.sourceforge.phpdt.core.compiler.CharOperation;
  17 import net.sourceforge.phpdt.core.compiler.ITerminalSymbols;
  18 import net.sourceforge.phpdt.core.compiler.InvalidInputException;
  19
  20 /**
  21  * Parser specialized for decoding javadoc comments
  22  */
  23 public abstract class AbstractCommentParser {
  24
  25         // recognized tags
  26         public static final char[] TAG_DEPRECATED = "deprecated".toCharArray(); //$NON-NLS-1$
  27         public static final char[] TAG_PARAM = "param".toCharArray(); //$NON-NLS-1$
  28         public static final char[] TAG_RETURN = "return".toCharArray(); //$NON-NLS-1$
  29         public static final char[] TAG_THROWS = "throws".toCharArray(); //$NON-NLS-1$
  30         public static final char[] TAG_EXCEPTION = "exception".toCharArray(); //$NON-NLS-1$
  31         public static final char[] TAG_SEE = "see".toCharArray(); //$NON-NLS-1$
  32         public static final char[] TAG_LINK = "link".toCharArray(); //$NON-NLS-1$
  33         public static final char[] TAG_LINKPLAIN = "linkplain".toCharArray(); //$NON-NLS-1$
  34         public static final char[] TAG_INHERITDOC = "inheritDoc".toCharArray(); //$NON-NLS-1$
  35
  36         // tags expected positions
  37         public final static int ORDERED_TAGS_NUMBER = 3;
  38         public final static int PARAM_TAG_EXPECTED_ORDER = 0;
  39         public final static int THROWS_TAG_EXPECTED_ORDER = 1;
  40         public final static int SEE_TAG_EXPECTED_ORDER = 2;
  41
  42         // Kind of comment parser
  43         public final static int COMPIL_PARSER = 0x00000001;
  44         public final static int DOM_PARSER = 0x00000002;
  45
  46         // Public fields
  47         public Scanner scanner;
  48         public boolean checkDocComment = false;
  49
  50         // Protected fields
  51         protected boolean inherited, deprecated;
  52         protected char[] source;
  53         protected int index, endComment, lineEnd;
  54         protected int tokenPreviousPosition, lastIdentifierEndPosition, starPosition;
  55         protected int textStart, memberStart;
  56         protected int tagSourceStart, tagSourceEnd;
  57         protected int inlineTagStart;
  58         protected Parser sourceParser;
  59         protected Object returnStatement;
  60         protected boolean lineStarted = false, inlineTagStarted = false;
  61         protected int kind;
  62         protected int[] lineEnds;
  63
  64         // Private fields
  65         private int currentTokenType = -1;
  66
  67         // Line pointers
  68         private int linePtr, lastLinePtr;
  69
  70         // Identifier stack
  71         protected int identifierPtr;
  72         protected char[][] identifierStack;
  73         protected int identifierLengthPtr;
  74         protected int[] identifierLengthStack;
  75         protected long[] identifierPositionStack;
  76         // Ast stack
  77         protected static int AstStackIncrement = 10;
  78         protected int astPtr;
  79         protected Object[] astStack;
  80         protected int astLengthPtr;
  81         protected int[] astLengthStack;
  82
  83         protected AbstractCommentParser(Parser sourceParser) {
  84                 this.sourceParser = sourceParser;
  85                 this.scanner = new Scanner(false, false, false, false, false, null, null, false);
  86
  87                 this.identifierStack = new char[20][];
  88                 this.identifierPositionStack = new long[20];
  89                 this.identifierLengthStack = new int[10];
  90                 this.astStack = new Object[30];
  91                 this.astLengthStack = new int[20];
  92         }
  93
  94         /* (non-Javadoc)
  95          * Returns true if tag @deprecated is present in javadoc comment.
  96          *
  97          * If javadoc checking is enabled, will also construct an Javadoc node, which will be stored into Parser.javadoc
  98          * slot for being consumed later on.
  99          */
 100         protected boolean parseComment(int javadocStart, int javadocEnd) {
 101
 102                 boolean validComment = true;
 103                 try {
 104                         // Init scanner position
 105                         this.scanner.resetTo(javadocStart, javadocEnd);
 106                         this.endComment = javadocEnd;
 107                         this.index = javadocStart;
 108                         readChar(); // starting '/'
 109                         int previousPosition = this.index;
 110                         readChar(); // first '*'
 111                         char nextCharacter= readChar(); // second '*'
 112
 113                         // Init local variables
 114                         this.astLengthPtr = -1;
 115                         this.astPtr = -1;
 116                         this.currentTokenType = -1;
 117                         this.inlineTagStarted = false;
 118                         this.inlineTagStart = -1;
 119                         this.lineStarted = false;
 120                         this.returnStatement = null;
 121                         this.inherited = false;
 122                         this.deprecated = false;
 123                         this.linePtr = getLineNumber(javadocStart);
 124                         this.lastLinePtr = getLineNumber(javadocEnd);
 125                         this.lineEnd = (this.linePtr == this.lastLinePtr) ? this.endComment : this.scanner.getLineEnd(this.linePtr);
 126                         this.textStart = -1;
 127                         char previousChar = 0;
 128                         int invalidTagLineEnd = -1;
 129                         int invalidInlineTagLineEnd = -1;
 130
 131                         // Loop on each comment character
 132                         while (this.index < this.endComment) {
 133                                 previousPosition = this.index;
 134                                 previousChar = nextCharacter;
 135
 136                                 // Calculate line end (cannot use this.scanner.linePtr as scanner does not parse line ends again)
 137                                 if (this.index > (this.lineEnd+1)) {
 138                                         updateLineEnd();
 139                                 }
 140
 141                                 // Read next char only if token was consumed
 142                                 if (this.currentTokenType < 0) {
 143                                         nextCharacter = readChar(); // consider unicodes
 144                                 } else {
 145                                         previousPosition = this.scanner.getCurrentTokenStartPosition();
 146                                         switch (this.currentTokenType) {
 147                                                 case ITerminalSymbols.TokenNameRBRACE:
 148                                                         nextCharacter = '}';
 149                                                         break;
 150                                                 case ITerminalSymbols.TokenNameMULTIPLY:
 151                                                         nextCharacter = '*';
 152                                                         break;
 153                                         default:
 154                                                         nextCharacter = this.scanner.currentCharacter;
 155                                         }
 156                                         consumeToken();
 157                                 }
 158
 159                                 if (this.index >= this.endComment) {
 160                                         break;
 161                                 }
 162
 163                                 switch (nextCharacter) {
 164                                         case '@' :
 165                                                 boolean valid = false;
 166                                                 // Start tag parsing only if we have a java identifier start character and if we are on line beginning or at inline tag beginning
 167                                                 if ((!this.lineStarted || previousChar == '{')) {
 168                                                         this.lineStarted = true;
 169                                                         if (this.inlineTagStarted) {
 170                                                                 this.inlineTagStarted = false;
 171                                                                 // bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=53279
 172                                                                 // Cannot have @ inside inline comment
 173                                                                 if (this.sourceParser != null) {
 174                                                                         int end = previousPosition<invalidInlineTagLineEnd ? previousPosition : invalidInlineTagLineEnd;
 175                                                                         this.sourceParser.problemReporter().javadocUnterminatedInlineTag(this.inlineTagStart, end);
 176                                                                 }
 177                                                                 validComment = false;
 178                                                                 if (this.lineStarted && this.textStart != -1 && this.textStart < previousPosition) {
 179                                                                         pushText(this.textStart, previousPosition);
 180                                                                 }
 181                                                                 if (this.kind == DOM_PARSER) refreshInlineTagPosition(previousPosition);
 182                                                         }
 183                                                         if (previousChar == '{') {
 184                                                                 if (this.textStart != -1 && this.textStart < this.inlineTagStart) {
 185                                                                         pushText(this.textStart, this.inlineTagStart);
 186                                                                 }
 187                                                                 this.inlineTagStarted = true;
 188                                                                 invalidInlineTagLineEnd = this.lineEnd;
 189                                                         } else if (this.textStart != -1 && this.textStart < invalidTagLineEnd) {
 190                                                                 pushText(this.textStart, invalidTagLineEnd);
 191                                                         }
 192                                                         this.scanner.resetTo(this.index, this.endComment);
 193                                                         this.currentTokenType = -1; // flush token cache at line begin
 194                                                         try {
 195                                                                 int token = readTokenAndConsume();
 196                                                                 this.tagSourceStart = this.scanner.getCurrentTokenStartPosition();
 197                                                                 this.tagSourceEnd = this.scanner.getCurrentTokenEndPosition();
 198                                                                 char[] tag = this.scanner.getCurrentIdentifierSource(); // first token is either an identifier or a keyword
 199                                                                 if (this.kind == DOM_PARSER) {
 200                                                                         // For DOM parser, try to get tag name other than java identifier
 201                                                                         // (see bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=51660)
 202                                                                         int tk = token;
 203                                                                         int le = this.lineEnd;
 204                                                                         char pc = peekChar();
 205                                                                         tagNameToken: while (tk != ITerminalSymbols.TokenNameEOF) {
 206                                                                                 this.tagSourceEnd = this.scanner.getCurrentTokenEndPosition();
 207                                                                                 token = tk;
 208                                                                                 // !, ", #, %, &, ', -, :, <, >, * chars and spaces are not allowed in tag names
 209                                                                                 switch (pc) {
 210                                                                                         case '}':
 211                                                                                         case '!':
 212                                                                                         case '#':
 213                                                                                         case '%':
 214                                                                                         case '&':
 215                                                                                         case '\'':
 216                                                                                         case ':':
 217                                                                                         // case '-': allowed in tag names as this character is often used in doclets (bug 68087)
 218                                                                                         case '<':
 219                                                                                         case '>':
 220                                                                                         case '*': // break for '*' as this is perhaps the end of comment (bug 65288)
 221                                                                                                 break tagNameToken;
 222                                                                                         default:
 223                                                                                                 if (pc == ' ' || Character.isWhitespace(pc)) break tagNameToken;
 224                                                                                 }
 225                                                                                 tk = readTokenAndConsume();
 226                                                                                 pc = peekChar();
 227                                                                         }
 228                                                                         int length = this.tagSourceEnd-this.tagSourceStart+1;
 229                                                                         tag = new char[length];
 230                                                                         System.arraycopy(this.source, this.tagSourceStart, tag, 0, length);
 231                                                                         this.index = this.tagSourceEnd+1;
 232                                                                         this.scanner.currentPosition = this.tagSourceEnd+1;
 233                                                                         this.tagSourceStart = previousPosition;
 234                                                                         this.lineEnd = le;
 235                                                                 }
 236                                                                 switch (token) {
 237                                                                         case ITerminalSymbols.TokenNameIdentifier :
 238                                                                                 if (CharOperation.equals(tag, TAG_DEPRECATED)) {
 239                                                                                         this.deprecated = true;
 240                                                                                         if (this.kind == DOM_PARSER) {
 241                                                                                                 valid = parseTag();
 242                                                                                         } else {
 243                                                                                                 valid = true;
 244                                                                                         }
 245                                                                                 } else if (CharOperation.equals(tag, TAG_INHERITDOC)) {
 246                                                                                         // inhibits inherited flag when tags have been already stored
 247                                                                                         // see bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=51606
 248                                                                                         // Note that for DOM_PARSER, nodes stack may be not empty even no '@' tag
 249                                                                                         // was encountered in comment. But it cannot be the case for COMPILER_PARSER
 250                                                                                         // and so is enough as it is only this parser which signals the missing tag warnings...
 251                                                                                         this.inherited = this.astPtr==-1;
 252                                                                                         if (this.kind == DOM_PARSER) {
 253                                                                                                 valid = parseTag();
 254                                                                                         } else {
 255                                                                                                 valid = true;
 256                                                                                         }
 257                                                                                 } else if (CharOperation.equals(tag, TAG_PARAM)) {
 258                                                                                         valid = parseParam();
 259                                                                                 } else if (CharOperation.equals(tag, TAG_EXCEPTION)) {
 260                                                                                         valid = parseThrows(false);
 261                                                                                 } else if (CharOperation.equals(tag, TAG_SEE)) {
 262                                                                                         if (this.inlineTagStarted) {
 263                                                                                                 // bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=53290
 264                                                                                                 // Cannot have @see inside inline comment
 265                                                                                                 valid = false;
 266                                                                                                 if (this.sourceParser != null)
 267                                                                                                         this.sourceParser.problemReporter().javadocUnexpectedTag(this.tagSourceStart, this.tagSourceEnd);
 268                                                                                         } else {
 269                                                                                                 valid = parseSee(false);
 270                                                                                         }
 271                                                                                 } else if (CharOperation.equals(tag, TAG_LINK)) {
 272                                                                                         if (this.inlineTagStarted) {
 273                                                                                                 valid = parseSee(false);
 274                                                                                         } else {
 275                                                                                                 // bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=53290
 276                                                                                                 // Cannot have @link outside inline comment
 277                                                                                                 valid = false;
 278                                                                                                 if (this.sourceParser != null)
 279                                                                                                         this.sourceParser.problemReporter().javadocUnexpectedTag(this.tagSourceStart, this.tagSourceEnd);
 280                                                                                         }
 281                                                                                 } else if (CharOperation.equals(tag, TAG_LINKPLAIN)) {
 282                                                                                         if (this.inlineTagStarted) {
 283                                                                                                 valid = parseSee(true);
 284                                                                                         } else {
 285                                                                                                 valid = parseTag();
 286                                                                                         }
 287                                                                                 } else {
 288                                                                                         valid = parseTag();
 289                                                                                 }
 290                                                                                 break;
 291                                                                         case ITerminalSymbols.TokenNamereturn :
 292                                                                                 valid = parseReturn();
 293                                                                                 // verify characters after return tag (we're expecting text description)
 294                                                                                 if(!verifyCharsAfterReturnTag(this.index)) {
 295                                                                                         if (this.sourceParser != null) {
 296                                                                                                 int end = this.starPosition == -1 || this.lineEnd<this.starPosition ? this.lineEnd : this.starPosition;
 297                                                                                                 this.sourceParser.problemReporter().javadocInvalidTag(this.tagSourceStart, end);
 298                                                                                         }
 299                                                                                 }
 300                                                                                 break;
 301 //                                                                      case ITerminalSymbols.TokenNamethrows :
 302 //                                                                              valid = parseThrows(true);
 303 //                                                                              break;
 304                                                                         default:
 305                                                                                 if (this.kind == DOM_PARSER) {
 306                                                                                         switch (token) {
 307                                                                                                 case ITerminalSymbols.TokenNameabstract:
 308 //                                                                                              case ITerminalSymbols.TokenNameassert:
 309 //                                                                                              case ITerminalSymbols.TokenNameboolean:
 310                                                                                                 case ITerminalSymbols.TokenNamebreak:
 311 //                                                                                              case ITerminalSymbols.TokenNamebyte:
 312                                                                                                 case ITerminalSymbols.TokenNamecase:
 313                                                                                                 case ITerminalSymbols.TokenNamecatch:
 314 //                                                                                              case ITerminalSymbols.TokenNamechar:
 315                                                                                                 case ITerminalSymbols.TokenNameclass:
 316                                                                                                 case ITerminalSymbols.TokenNamecontinue:
 317                                                                                                 case ITerminalSymbols.TokenNamedefault:
 318                                                                                                 case ITerminalSymbols.TokenNamedo:
 319 //                                                                                              case ITerminalSymbols.TokenNamedouble:
 320                                                                                                 case ITerminalSymbols.TokenNameelse:
 321                                                                                                 case ITerminalSymbols.TokenNameextends:
 322 //                                                                                              case ITerminalSymbols.TokenNamefalse:
 323                                                                                                 case ITerminalSymbols.TokenNamefinal:
 324                                                                                                 case ITerminalSymbols.TokenNamefinally:
 325 //                                                                                              case ITerminalSymbols.TokenNamefloat:
 326                                                                                                 case ITerminalSymbols.TokenNamefor:
 327                                                                                                 case ITerminalSymbols.TokenNameif:
 328                                                                                                 case ITerminalSymbols.TokenNameimplements:
 329 //                                                                                              case ITerminalSymbols.TokenNameimport:
 330                                                                                                 case ITerminalSymbols.TokenNameinstanceof:
 331 //                                                                                              case ITerminalSymbols.TokenNameint:
 332                                                                                                 case ITerminalSymbols.TokenNameinterface:
 333 //                                                                                              case ITerminalSymbols.TokenNamelong:
 334 //                                                                                              case ITerminalSymbols.TokenNamenative:
 335                                                                                                 case ITerminalSymbols.TokenNamenew:
 336 //                                                                                              case ITerminalSymbols.TokenNamenull:
 337 //                                                                                              case ITerminalSymbols.TokenNamepackage:
 338                                                                                                 case ITerminalSymbols.TokenNameprivate:
 339                                                                                                 case ITerminalSymbols.TokenNameprotected:
 340                                                                                                 case ITerminalSymbols.TokenNamepublic:
 341 //                                                                                              case ITerminalSymbols.TokenNameshort:
 342                                                                                                 case ITerminalSymbols.TokenNamestatic:
 343 //                                                                                              case ITerminalSymbols.TokenNamestrictfp:
 344                                                                                                 case ITerminalSymbols.TokenNamesuper:
 345                                                                                                 case ITerminalSymbols.TokenNameswitch:
 346 //                                                                                              case ITerminalSymbols.TokenNamesynchronized:
 347 //                                                                                              case ITerminalSymbols.TokenNamethis:
 348                                                                                                 case ITerminalSymbols.TokenNamethrow:
 349 //                                                                                              case ITerminalSymbols.TokenNametransient:
 350 //                                                                                              case ITerminalSymbols.TokenNametrue:
 351                                                                                                 case ITerminalSymbols.TokenNametry:
 352 //                                                                                              case ITerminalSymbols.TokenNamevoid:
 353 //                                                                                              case ITerminalSymbols.TokenNamevolatile:
 354                                                                                                 case ITerminalSymbols.TokenNamewhile:
 355                                                                                                         valid = parseTag();
 356                                                                                                         break;
 357                                                                                         }
 358                                                                                 }
 359                                                                 }
 360                                                                 this.textStart = this.index;
 361                                                                 if (!valid) {
 362                                                                         // bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=51600
 363                                                                         // do not stop the inline tag when error is encountered to get text after
 364                                                                         validComment = false;
 365                                                                         // bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=51600
 366                                                                         // for DOM AST node, store tag as text in case of invalid syntax
 367                                                                         if (this.kind == DOM_PARSER) {
 368                                                                                 parseTag();
 369                                                                                 this.textStart = this.tagSourceEnd+1;
 370                                                                                 invalidTagLineEnd  = this.lineEnd;
 371                                                                         }
 372                                                                 }
 373                                                         } catch (InvalidInputException e) {
 374                                                                 consumeToken();
 375                                                         }
 376                                                 }
 377                                                 break;
 378                                         case '\r':
 379                                         case '\n':
 380                                                 if (this.lineStarted && this.textStart < previousPosition) {
 381                                                         pushText(this.textStart, previousPosition);
 382                                                 }
 383                                                 this.lineStarted = false;
 384                                                 // Fix bug 51650
 385                                                 this.textStart = -1;
 386                                                 break;
 387                                         case '}' :
 388                                                 if (this.inlineTagStarted) {
 389                                                         if (this.lineStarted && this.textStart != -1 && this.textStart < previousPosition) {
 390                                                                 pushText(this.textStart, previousPosition);
 391                                                         }
 392                                                         if (this.kind == DOM_PARSER) refreshInlineTagPosition(previousPosition);
 393                                                         this.textStart = this.index;
 394                                                         this.inlineTagStarted = false;
 395                                                 } else {
 396                                                         if (!this.lineStarted) {
 397                                                                 this.textStart = previousPosition;
 398                                                         }
 399                                                 }
 400                                                 this.lineStarted = true;
 401                                                 break;
 402                                         case '{' :
 403                                                 if (this.inlineTagStarted) {
 404                                                         this.inlineTagStarted = false;
 405                                                         // bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=53279
 406                                                         // Cannot have opening brace in inline comment
 407                                                         if (this.sourceParser != null) {
 408                                                                 int end = previousPosition<invalidInlineTagLineEnd ? previousPosition : invalidInlineTagLineEnd;
 409                                                                 this.sourceParser.problemReporter().javadocUnterminatedInlineTag(this.inlineTagStart, end);
 410                                                         }
 411                                                         if (this.lineStarted && this.textStart != -1 && this.textStart < previousPosition) {
 412                                                                 pushText(this.textStart, previousPosition);
 413                                                         }
 414                                                         if (this.kind == DOM_PARSER) refreshInlineTagPosition(previousPosition);
 415                                                 }
 416                                                 if (!this.lineStarted) {
 417                                                         this.textStart = previousPosition;
 418                                                 }
 419                                                 this.lineStarted = true;
 420                                                 this.inlineTagStart = previousPosition;
 421                                                 break;
 422                                         case '*' :
 423                                         case '\u000c' : /* FORM FEED               */
 424                                         case ' ' :                      /* SPACE                   */
 425                                         case '\t' :                     /* HORIZONTAL TABULATION   */
 426                                                 // do nothing for space or '*' characters
 427                                                 break;
 428                                         default :
 429                                                 if (!this.lineStarted) {
 430                                                         this.textStart = previousPosition;
 431                                                 }
 432                                                 this.lineStarted = true;
 433                                                 break;
 434                                 }
 435                         }
 436                         // bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=53279
 437                         // Cannot leave comment inside inline comment
 438                         if (this.inlineTagStarted) {
 439                                 this.inlineTagStarted = false;
 440                                 if (this.sourceParser != null) {
 441                                         int end = previousPosition<invalidInlineTagLineEnd ? previousPosition : invalidInlineTagLineEnd;
 442                                         if (this.index >= this.endComment) end = invalidInlineTagLineEnd;
 443                                         this.sourceParser.problemReporter().javadocUnterminatedInlineTag(this.inlineTagStart, end);
 444                                 }
 445                                 if (this.lineStarted && this.textStart != -1 && this.textStart < previousPosition) {
 446                                         pushText(this.textStart, previousPosition);
 447                                 }
 448                                 if (this.kind == DOM_PARSER) {
 449                                         refreshInlineTagPosition(previousPosition);
 450                                 }
 451                         } else if (this.lineStarted && this.textStart < previousPosition) {
 452                                 pushText(this.textStart, previousPosition);
 453                         }
 454                         updateDocComment();
 455                 } catch (Exception ex) {
 456                         validComment = false;
 457                 }
 458                 return validComment;
 459         }
 460
 461         private void consumeToken() {
 462                 this.currentTokenType = -1; // flush token cache
 463                 updateLineEnd();
 464         }
 465
 466         protected abstract Object createArgumentReference(char[] name, int dim, Object typeRef, long[] dimPos, long argNamePos) throws InvalidInputException;
 467         protected abstract Object createFieldReference(Object receiver) throws InvalidInputException;
 468         protected abstract Object createMethodReference(Object receiver, List arguments) throws InvalidInputException;
 469         protected Object createReturnStatement() { return null; }
 470         protected abstract Object createTypeReference(int primitiveToken);
 471
 472         private int getEndPosition() {
 473                 if (this.scanner.getCurrentTokenEndPosition() > this.lineEnd) {
 474                         return this.lineEnd;
 475                 } else {
 476                         return this.scanner.getCurrentTokenEndPosition();
 477                 }
 478         }
 479
 480         /*
 481          * Search the source position corresponding to the end of a given line number.
 482          * Warning: returned position is 1-based index!
 483          * @see Scanner#getLineEnd(int) We cannot directly use this method
 484          * when linePtr field is not initialized.
 485          *
 486         private int getLineEnd(int lineNumber) {
 487
 488                 if (this.scanner.linePtr != -1) {
 489                         return this.scanner.getLineEnd(lineNumber);
 490                 }
 491                 if (this.lineEnds == null)
 492                         return -1;
 493                 if (lineNumber > this.lineEnds.length+1)
 494                         return -1;
 495                 if (lineNumber <= 0)
 496                         return -1;
 497                 if (lineNumber == this.lineEnds.length + 1)
 498                         return this.scanner.eofPosition;
 499                 return this.lineEnds[lineNumber-1]; // next line start one character behind the lineEnd of the previous line
 500         }
 501         */
 502
 503         /**
 504          * Search the line number corresponding to a specific position.
 505          * Warning: returned position is 1-based index!
 506          * @see Scanner#getLineNumber(int) We cannot directly use this method
 507          * when linePtr field is not initialized.
 508          */
 509         private int getLineNumber(int position) {
 510
 511                 if (this.scanner.linePtr != -1) {
 512                         return this.scanner.getLineNumber(position);
 513                 }
 514                 if (this.lineEnds == null)
 515                         return 1;
 516                 int length = this.lineEnds.length;
 517                 if (length == 0)
 518                         return 1;
 519                 int g = 0, d = length - 1;
 520                 int m = 0;
 521                 while (g <= d) {
 522                         m = (g + d) /2;
 523                         if (position < this.lineEnds[m]) {
 524                                 d = m-1;
 525                         } else if (position > this.lineEnds[m]) {
 526                                 g = m+1;
 527                         } else {
 528                                 return m + 1;
 529                         }
 530                 }
 531                 if (position < this.lineEnds[m]) {
 532                         return m+1;
 533                 }
 534                 return m+2;
 535         }
 536
 537         /*
 538          * Parse argument in @see tag method reference
 539          */
 540         private Object parseArguments(Object receiver) throws InvalidInputException {
 541
 542                 // Init
 543                 int modulo = 0; // should be 2 for (Type,Type,...) or 3 for (Type arg,Type arg,...)
 544                 int iToken = 0;
 545                 char[] argName = null;
 546                 List arguments = new ArrayList(10);
 547                 int start = this.scanner.getCurrentTokenStartPosition();
 548
 549                 // Parse arguments declaration if method reference
 550                 nextArg : while (this.index < this.scanner.eofPosition) {
 551
 552                         // Read argument type reference
 553                         Object typeRef;
 554                         try {
 555                                 typeRef = parseQualifiedName(false);
 556                         } catch (InvalidInputException e) {
 557                                 break nextArg;
 558                         }
 559                         boolean firstArg = modulo == 0;
 560                         if (firstArg) { // verify position
 561                                 if (iToken != 0)
 562                                         break nextArg;
 563                         } else if ((iToken % modulo) != 0) {
 564                                         break nextArg;
 565                         }
 566                         if (typeRef == null) {
 567                                 if (firstArg && this.currentTokenType == ITerminalSymbols.TokenNameRPAREN) {
 568                                         // verify characters after arguments declaration (expecting white space or end comment)
 569                                         if (!verifySpaceOrEndComment()) {
 570                                                 int end = this.starPosition == -1 ? this.lineEnd : this.starPosition;
 571                                                 if (this.source[end]=='\n') end--;
 572                                                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocMalformedSeeReference(start, end);
 573                                                 return null;
 574                                         }
 575                                         this.lineStarted = true;
 576                                         return createMethodReference(receiver, null);
 577                                 }
 578                                 break nextArg;
 579                         }
 580                         iToken++;
 581
 582                         // Read possible array declaration
 583                         int dim = 0;
 584                         long[] dimPositions = new long[20]; // assume that there won't be more than 20 dimensions...
 585                         if (readToken() == ITerminalSymbols.TokenNameLBRACKET) {
 586                                 int dimStart = this.scanner.getCurrentTokenStartPosition();
 587                                 while (readToken() == ITerminalSymbols.TokenNameLBRACKET) {
 588                                         consumeToken();
 589                                         if (readToken() != ITerminalSymbols.TokenNameRBRACKET) {
 590                                                 break nextArg;
 591                                         }
 592                                         consumeToken();
 593                                         dimPositions[dim++] = (((long) dimStart) << 32) + this.scanner.getCurrentTokenEndPosition();
 594                                 }
 595                         }
 596
 597                         // Read argument name
 598                         long argNamePos = -1;
 599                         if (readToken() == ITerminalSymbols.TokenNameIdentifier) {
 600                                 consumeToken();
 601                                 if (firstArg) { // verify position
 602                                         if (iToken != 1)
 603                                                 break nextArg;
 604                                 } else if ((iToken % modulo) != 1) {
 605                                                 break nextArg;
 606                                 }
 607                                 if (argName == null) { // verify that all arguments name are declared
 608                                         if (!firstArg) {
 609                                                 break nextArg;
 610                                         }
 611                                 }
 612                                 argName = this.scanner.getCurrentIdentifierSource();
 613                                 argNamePos = (((long)this.scanner.getCurrentTokenStartPosition())<<32)+this.scanner.getCurrentTokenEndPosition();
 614                                 iToken++;
 615                         } else if (argName != null) { // verify that no argument name is declared
 616                                 break nextArg;
 617                         }
 618
 619                         // Verify token position
 620                         if (firstArg) {
 621                                 modulo = iToken + 1;
 622                         } else {
 623                                 if ((iToken % modulo) != (modulo - 1)) {
 624                                         break nextArg;
 625                                 }
 626                         }
 627
 628                         // Read separator or end arguments declaration
 629                         int token = readToken();
 630                         char[] name = argName == null ? new char[0] : argName;
 631                         if (token == ITerminalSymbols.TokenNameCOMMA) {
 632                                 // Create new argument
 633                                 Object argument = createArgumentReference(name, dim, typeRef, dimPositions, argNamePos);
 634                                 arguments.add(argument);
 635                                 consumeToken();
 636                                 iToken++;
 637                         } else if (token == ITerminalSymbols.TokenNameRPAREN) {
 638                                 // verify characters after arguments declaration (expecting white space or end comment)
 639                                 if (!verifySpaceOrEndComment()) {
 640                                         int end = this.starPosition == -1 ? this.lineEnd : this.starPosition;
 641                                         if (this.source[end]=='\n') end--;
 642                                         if (this.sourceParser != null) this.sourceParser.problemReporter().javadocMalformedSeeReference(start, end);
 643                                         return null;
 644                                 }
 645                                 // Create new argument
 646                                 Object argument = createArgumentReference(name, dim, typeRef, dimPositions, argNamePos);
 647                                 arguments.add(argument);
 648                                 consumeToken();
 649                                 return createMethodReference(receiver, arguments);
 650                         } else {
 651                                 break nextArg;
 652                         }
 653                 }
 654
 655                 // Something wrong happened => Invalid input
 656                 throw new InvalidInputException();
 657         }
 658
 659         /*
 660          * Parse an URL link reference in @see tag
 661          */
 662         private boolean parseHref() throws InvalidInputException {
 663                 int start = this.scanner.getCurrentTokenStartPosition();
 664                 if (Character.toLowerCase(readChar()) == 'a') {
 665                         this.scanner.currentPosition = this.index;
 666                         if (readToken() == ITerminalSymbols.TokenNameIdentifier) {
 667                                 this.currentTokenType = -1; // do not update line end
 668                                 try {
 669                                         if (CharOperation.equals(this.scanner.getCurrentIdentifierSource(), new char[]{'h', 'r', 'e', 'f'}, false) &&
 670                                                 readToken() == ITerminalSymbols.TokenNameEQUAL) {
 671                                                 this.currentTokenType = -1; // do not update line end
 672                                                 if (readToken() == ITerminalSymbols.TokenNameStringDoubleQuote ||
 673                                                     readToken() == ITerminalSymbols.TokenNameStringSingleQuote) {
 674                                                         this.currentTokenType = -1; // do not update line end
 675                                                         // Skip all characters after string literal until closing '>' (see bug 68726)
 676                                                         while (this.index <= this.lineEnd && readToken() != ITerminalSymbols.TokenNameGREATER) {
 677                                                                 this.currentTokenType = -1; // do not update line end
 678                                                         }
 679                                                         if (this.currentTokenType == ITerminalSymbols.TokenNameGREATER) {
 680                                                                 consumeToken(); // update line end as new lines are allowed in URL description
 681                                                                 while (readToken() != ITerminalSymbols.TokenNameLESS) {
 682                                                                         if (this.scanner.currentPosition >= this.scanner.eofPosition || this.scanner.currentCharacter == '@') {
 683                                                                                 // Reset position: we want to rescan last token
 684                                                                                 this.index = this.tokenPreviousPosition;
 685                                                                                 this.scanner.currentPosition = this.tokenPreviousPosition;
 686                                                                                 this.currentTokenType = -1;
 687                                                                                 // Signal syntax error
 688                                                                                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidSeeUrlReference(start, this.lineEnd);
 689                                                                                 return false;
 690                                                                         }
 691                                                                         consumeToken();
 692                                                                 }
 693                                                                 this.currentTokenType = -1; // do not update line end
 694                                                                 if (readChar() == '/') {
 695                                                                         if (Character.toLowerCase(readChar()) == 'a') {
 696                                                                                 if (readChar() == '>') {
 697                                                                                         // Valid href
 698                                                                                         return true;
 699                                                                                 }
 700                                                                         }
 701                                                                 }
 702                                                         }
 703                                                 }
 704                                         }
 705                                 } catch (InvalidInputException ex) {
 706                                         // Do nothing as we want to keep positions for error message
 707                                 }
 708                         }
 709                 }
 710                 // Reset position: we want to rescan last token
 711                 this.index = this.tokenPreviousPosition;
 712                 this.scanner.currentPosition = this.tokenPreviousPosition;
 713                 this.currentTokenType = -1;
 714                 // Signal syntax error
 715                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidSeeUrlReference(start, this.lineEnd);
 716                 return false;
 717         }
 718
 719         /*
 720          * Parse a method reference in @see tag
 721          */
 722         private Object parseMember(Object receiver) throws InvalidInputException {
 723                 // Init
 724                 this.identifierPtr = -1;
 725                 this.identifierLengthPtr = -1;
 726                 int start = this.scanner.getCurrentTokenStartPosition();
 727                 this.memberStart = start;
 728
 729                 // Get member identifier
 730                 if (readToken() == ITerminalSymbols.TokenNameIdentifier) {
 731                         consumeToken();
 732                         pushIdentifier(true);
 733                         // Look for next token to know whether it's a field or method reference
 734                         int previousPosition = this.index;
 735                         if (readToken() == ITerminalSymbols.TokenNameLPAREN) {
 736                                 consumeToken();
 737                                 start = this.scanner.getCurrentTokenStartPosition();
 738                                 try {
 739                                         return parseArguments(receiver);
 740                                 } catch (InvalidInputException e) {
 741                                         int end = this.scanner.getCurrentTokenEndPosition() < this.lineEnd ?
 742                                                         this.scanner.getCurrentTokenEndPosition() :
 743                                                         this.scanner.getCurrentTokenStartPosition();
 744                                         end = end < this.lineEnd ? end : this.lineEnd;
 745                                         if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidSeeReferenceArgs(start, end);
 746                                 }
 747                                 return null;
 748                         }
 749
 750                         // Reset position: we want to rescan last token
 751                         this.index = previousPosition;
 752                         this.scanner.currentPosition = previousPosition;
 753                         this.currentTokenType = -1;
 754
 755                         // Verify character(s) after identifier (expecting space or end comment)
 756                         if (!verifySpaceOrEndComment()) {
 757                                 int end = this.starPosition == -1 ? this.lineEnd : this.starPosition;
 758                                 if (this.source[end]=='\n') end--;
 759                                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocMalformedSeeReference(start, end);
 760                                 return null;
 761                         }
 762                         return createFieldReference(receiver);
 763                 }
 764                 int end = getEndPosition() - 1;
 765                 end = start > end ? start : end;
 766                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidSeeReference(start, end);
 767                 // Reset position: we want to rescan last token
 768                 this.index = this.tokenPreviousPosition;
 769                 this.scanner.currentPosition = this.tokenPreviousPosition;
 770                 this.currentTokenType = -1;
 771                 return null;
 772         }
 773
 774         /*
 775          * Parse @param tag declaration
 776          */
 777         protected boolean parseParam() {
 778
 779                 // Store current token state
 780                 int start = this.tagSourceStart;
 781                 int end = this.tagSourceEnd;
 782
 783                 try {
 784                         // Push identifier next
 785                         int token = readToken();
 786                         switch (token) {
 787                                 case ITerminalSymbols.TokenNameIdentifier :
 788                                         consumeToken();
 789                                         return pushParamName();
 790                                 case ITerminalSymbols.TokenNameEOF :
 791                                         break;
 792                                 default :
 793                                         start = this.scanner.getCurrentTokenStartPosition();
 794                                         end = getEndPosition();
 795                                         if (end < start) start = this.tagSourceStart;
 796                                         break;
 797                         }
 798                 } catch (InvalidInputException e) {
 799                         end = getEndPosition();
 800                 }
 801
 802                 // Reset position to avoid missing tokens when new line was encountered
 803                 this.index = this.tokenPreviousPosition;
 804                 this.scanner.currentPosition = this.tokenPreviousPosition;
 805                 this.currentTokenType = -1;
 806
 807                 // Report problem
 808                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocMissingParamName(start, end);
 809                 return false;
 810         }
 811
 812         /*
 813          * Parse a qualified name and built a type reference if the syntax is valid.
 814          */
 815         protected Object parseQualifiedName(boolean reset) throws InvalidInputException {
 816
 817                 // Reset identifier stack if requested
 818                 if (reset) {
 819                         this.identifierPtr = -1;
 820                         this.identifierLengthPtr = -1;
 821                 }
 822
 823                 // Scan tokens
 824                 int primitiveToken = -1;
 825                 nextToken : for (int iToken = 0; ; iToken++) {
 826                         int token = readToken();
 827                         switch (token) {
 828                                 case ITerminalSymbols.TokenNameIdentifier :
 829                                         if (((iToken % 2) > 0)) { // identifiers must be odd tokens
 830                                                 break nextToken;
 831                                         }
 832                                         pushIdentifier(iToken == 0);
 833                                         consumeToken();
 834                                         break;
 835
 836                                 case ITerminalSymbols.TokenNameDOT :
 837                                         if ((iToken % 2) == 0) { // dots must be even tokens
 838                                                 throw new InvalidInputException();
 839                                         }
 840                                         consumeToken();
 841                                         break;
 842
 843 //                              case ITerminalSymbols.TokenNamevoid :
 844 //                              case ITerminalSymbols.TokenNameboolean :
 845 //                              case ITerminalSymbols.TokenNamebyte :
 846 //                              case ITerminalSymbols.TokenNamechar :
 847 //                              case ITerminalSymbols.TokenNamedouble :
 848 //                              case ITerminalSymbols.TokenNamefloat :
 849 //                              case ITerminalSymbols.TokenNameint :
 850 //                              case ITerminalSymbols.TokenNamelong :
 851 //                              case ITerminalSymbols.TokenNameshort :
 852 //                                      if (iToken > 0) {
 853 //                                              throw new InvalidInputException();
 854 //                                      }
 855 //                                      pushIdentifier(true);
 856 //                                      primitiveToken = token;
 857 //                                      consumeToken();
 858 //                                      break nextToken;
 859
 860                                 default :
 861                                         if (iToken == 0) {
 862                                                 return null;
 863                                         }
 864                                         if ((iToken % 2) == 0) { // cannot leave on a dot
 865                                                 // Reset position: we want to rescan last token
 866                                                 if (this.kind == DOM_PARSER && this.currentTokenType != -1) {
 867                                                         this.index = this.tokenPreviousPosition;
 868                                                         this.scanner.currentPosition = this.tokenPreviousPosition;
 869                                                         this.currentTokenType = -1;
 870                                                 }
 871                                                 throw new InvalidInputException();
 872                                         }
 873                                         break nextToken;
 874                         }
 875                 }
 876                 // Reset position: we want to rescan last token
 877                 if (this.currentTokenType != -1) {
 878                         this.index = this.tokenPreviousPosition;
 879                         this.scanner.currentPosition = this.tokenPreviousPosition;
 880                         this.currentTokenType = -1;
 881                 }
 882                 this.lastIdentifierEndPosition = (int) this.identifierPositionStack[this.identifierPtr];
 883                 return createTypeReference(primitiveToken);
 884         }
 885
 886         /*
 887          * Parse a reference in @see tag
 888          */
 889         protected boolean parseReference(boolean plain) throws InvalidInputException {
 890                 Object typeRef = null;
 891                 Object reference = null;
 892                 int previousPosition = -1;
 893                 int typeRefStartPosition = -1;
 894                 nextToken : while (this.index < this.scanner.eofPosition) {
 895                         previousPosition = this.index;
 896                         int token = readToken();
 897                         switch (token) {
 898                                 case ITerminalSymbols.TokenNameStringDoubleQuote : // @see "string"
 899                                 case ITerminalSymbols.TokenNameStringSingleQuote :
 900                                         int start = this.scanner.getCurrentTokenStartPosition();
 901                                         consumeToken();
 902                                         // If typeRef != null we may raise a warning here to let user know there's an unused reference...
 903                                         // Currently as javadoc 1.4.2 ignore it, we do the same (see bug 69302)
 904                                         if (typeRef != null) {
 905                                                 start = this.tagSourceEnd+1;
 906                                                 previousPosition = start;
 907                                                 typeRef = null;
 908                                         }
 909                                         // verify end line (expecting empty or end comment)
 910                                         if (verifyEndLine(previousPosition)) {
 911                                                 return true;
 912                                         }
 913                                         if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidSeeReference(start, this.lineEnd);
 914                                         return false;
 915                                 case ITerminalSymbols.TokenNameLESS : // @see "<a href="URL#Value">label</a>
 916                                         consumeToken();
 917                                         start = this.scanner.getCurrentTokenStartPosition();
 918                                         if (parseHref()) {
 919                                                 consumeToken();
 920                                                 // If typeRef != null we may raise a warning here to let user know there's an unused reference...
 921                                                 // Currently as javadoc 1.4.2 ignore it, we do the same (see bug 69302)
 922                                                 if (typeRef != null) {
 923                                                         start = this.tagSourceEnd+1;
 924                                                         previousPosition = start;
 925                                                         typeRef = null;
 926                                                 }
 927                                                 // verify end line (expecting empty or end comment)
 928                                                 if (verifyEndLine(previousPosition)) {
 929                                                         return true;
 930                                                 }
 931                                                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidSeeReference(start, this.lineEnd);
 932                                         }
 933                                         return false;
 934                                 case ITerminalSymbols.TokenNameERROR :
 935                                         if (this.scanner.currentCharacter == '#') { // @see ...#member
 936                                                 consumeToken();
 937                                                 reference = parseMember(typeRef);
 938                                                 if (reference != null) {
 939                                                         return pushSeeRef(reference, plain);
 940                                                 }
 941                                                 return false;
 942                                         }
 943                                         break nextToken;
 944                                 case ITerminalSymbols.TokenNameIdentifier :
 945                                         if (typeRef == null) {
 946                                                 typeRefStartPosition = this.scanner.getCurrentTokenStartPosition();
 947                                                 typeRef = parseQualifiedName(true);
 948                                                 break;
 949                                         }
 950                                         break nextToken;
 951                                 default :
 952                                         break nextToken;
 953                         }
 954                 }
 955
 956                 // Verify that we got a reference
 957                 if (reference == null) reference = typeRef;
 958                 if (reference == null) {
 959                         this.index = this.tokenPreviousPosition;
 960                         this.scanner.currentPosition = this.tokenPreviousPosition;
 961                         this.currentTokenType = -1;
 962                         if (this.sourceParser != null) this.sourceParser.problemReporter().javadocMissingSeeReference(this.tagSourceStart, this.tagSourceEnd);
 963                         return false;
 964                 }
 965
 966                 // Reset position at the end of type reference
 967                 this.index = this.lastIdentifierEndPosition+1;
 968                 this.scanner.currentPosition = this.index;
 969                 this.currentTokenType = -1;
 970
 971                 // Verify that line end does not start with an open parenthese (which could be a constructor reference wrongly written...)
 972                 // See bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=47215
 973                 char ch = peekChar();
 974                 if (ch == '(') {
 975                         if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidSeeReference(typeRefStartPosition, this.lineEnd);
 976                         return false;
 977                 }
 978
 979                 // Verify that we get white space after reference
 980                 if (!verifySpaceOrEndComment()) {
 981                         this.index = this.tokenPreviousPosition;
 982                         this.scanner.currentPosition = this.tokenPreviousPosition;
 983                         this.currentTokenType = -1;
 984                         int end = this.starPosition == -1 ? this.lineEnd : this.starPosition;
 985                         if (this.source[end]=='\n') end--;
 986                         if (this.sourceParser != null) this.sourceParser.problemReporter().javadocMalformedSeeReference(typeRefStartPosition, end);
 987                         return false;
 988                 }
 989
 990                 // Everything is OK, store reference
 991                 return pushSeeRef(reference, plain);
 992         }
 993
 994         /*
 995          * Parse @return tag declaration
 996          */
 997         protected abstract boolean parseReturn();
 998
 999         /*
1000          * Parse @see tag declaration
1001          */
1002         protected boolean parseSee(boolean plain) {
1003                 int start = this.scanner.currentPosition;
1004                 try {
1005                         return parseReference(plain);
1006                 } catch (InvalidInputException ex) {
1007                                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidSeeReference(start, getEndPosition());
1008                 }
1009                 // Reset position to avoid missing tokens when new line was encountered
1010                 this.index = this.tokenPreviousPosition;
1011                 this.scanner.currentPosition = this.tokenPreviousPosition;
1012                 this.currentTokenType = -1;
1013                 return false;
1014         }
1015
1016         /*
1017          * Parse @return tag declaration
1018          */
1019         protected abstract boolean parseTag();
1020
1021         /*
1022          * Parse @throws tag declaration
1023          */
1024         protected boolean parseThrows(boolean real) {
1025                 int start = this.scanner.currentPosition;
1026                 try {
1027                         Object typeRef = parseQualifiedName(true);
1028                         if (typeRef == null) {
1029                                 if (this.sourceParser != null) this.sourceParser.problemReporter().javadocMissingThrowsClassName(this.tagSourceStart, this.tagSourceEnd);
1030                         } else {
1031                                 return pushThrowName(typeRef, real);
1032                         }
1033                 } catch (InvalidInputException ex) {
1034                         if (this.sourceParser != null) this.sourceParser.problemReporter().javadocInvalidThrowsClass(start, getEndPosition());
1035                 }
1036                 return false;
1037         }
1038
1039         /*
1040          * Return current character without move index position.
1041          */
1042         private char peekChar() {
1043                 int idx = this.index;
1044                 char c = this.source[idx++];
1045                 if (c == '\\' && this.source[idx] == 'u') {
1046                         int c1, c2, c3, c4;
1047                         idx++;
1048                         while (this.source[idx] == 'u')
1049                                 idx++;
1050                         if (!(((c1 = Character.getNumericValue(this.source[idx++])) > 15 || c1 < 0)
1051                                         || ((c2 = Character.getNumericValue(this.source[idx++])) > 15 || c2 < 0)
1052                                         || ((c3 = Character.getNumericValue(this.source[idx++])) > 15 || c3 < 0) || ((c4 = Character.getNumericValue(this.source[idx++])) > 15 || c4 < 0))) {
1053                                 c = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1054                         }
1055                 }
1056                 return c;
1057         }
1058
1059         /*
1060          * push the consumeToken on the identifier stack. Increase the total number of identifier in the stack.
1061          */
1062         protected void pushIdentifier(boolean newLength) {
1063
1064                 int stackLength = this.identifierStack.length;
1065                 if (++this.identifierPtr >= stackLength) {
1066                         System.arraycopy(
1067                                 this.identifierStack, 0,
1068                                 this.identifierStack = new char[stackLength + 10][], 0,
1069                                 stackLength);
1070                         System.arraycopy(
1071                                 this.identifierPositionStack, 0,
1072                                 this.identifierPositionStack = new long[stackLength + 10], 0,
1073                                 stackLength);
1074                 }
1075                 this.identifierStack[this.identifierPtr] = this.scanner.getCurrentIdentifierSource();
1076                 this.identifierPositionStack[this.identifierPtr] = (((long) this.scanner.startPosition) << 32) + (this.scanner.currentPosition - 1);
1077
1078                 if (newLength) {
1079                         stackLength = this.identifierLengthStack.length;
1080                         if (++this.identifierLengthPtr >= stackLength) {
1081                                 System.arraycopy(
1082                                         this.identifierLengthStack, 0,
1083                                         this.identifierLengthStack = new int[stackLength + 10], 0,
1084                                         stackLength);
1085                         }
1086                         this.identifierLengthStack[this.identifierLengthPtr] = 1;
1087                 } else {
1088                         this.identifierLengthStack[this.identifierLengthPtr]++;
1089                 }
1090         }
1091
1092         /*
1093          * Add a new obj on top of the ast stack.
1094          * If new length is required, then add also a new length in length stack.
1095          */
1096         protected void pushOnAstStack(Object node, boolean newLength) {
1097
1098                 if (node == null) {
1099                         this.astLengthStack[++this.astLengthPtr] = 0;
1100                         return;
1101                 }
1102
1103                 int stackLength = this.astStack.length;
1104                 if (++this.astPtr >= stackLength) {
1105                         System.arraycopy(
1106                                 this.astStack, 0,
1107                                 this.astStack = new Object[stackLength + AstStackIncrement], 0,
1108                                 stackLength);
1109                         this.astPtr = stackLength;
1110                 }
1111                 this.astStack[this.astPtr] = node;
1112
1113                 if (newLength) {
1114                         stackLength = this.astLengthStack.length;
1115                         if (++this.astLengthPtr >= stackLength) {
1116                                 System.arraycopy(
1117                                         this.astLengthStack, 0,
1118                                         this.astLengthStack = new int[stackLength + AstStackIncrement], 0,
1119                                         stackLength);
1120                         }
1121                         this.astLengthStack[this.astLengthPtr] = 1;
1122                 } else {
1123                         this.astLengthStack[this.astLengthPtr]++;
1124                 }
1125         }
1126
1127         /*
1128          * Push a param name in ast node stack.
1129          */
1130         protected abstract boolean pushParamName();
1131
1132         /*
1133          * Push a reference statement in ast node stack.
1134          */
1135         protected abstract boolean pushSeeRef(Object statement, boolean plain);
1136
1137         /*
1138          * Push a text element in ast node stack
1139          */
1140         protected abstract void pushText(int start, int end);
1141
1142         /*
1143          * Push a throws type ref in ast node stack.
1144          */
1145         protected abstract boolean pushThrowName(Object typeRef, boolean real);
1146
1147         /*
1148          * Read current character and move index position.
1149          * Warning: scanner position is unchanged using this method!
1150          */
1151         protected char readChar() {
1152
1153                 char c = this.source[this.index++];
1154                 if (c == '\\' && this.source[this.index] == 'u') {
1155                         int c1, c2, c3, c4;
1156                         int pos = this.index;
1157                         this.index++;
1158                         while (this.source[this.index] == 'u')
1159                                 this.index++;
1160                         if (!(((c1 = Character.getNumericValue(this.source[this.index++])) > 15 || c1 < 0)
1161                                         || ((c2 = Character.getNumericValue(this.source[this.index++])) > 15 || c2 < 0)
1162                                         || ((c3 = Character.getNumericValue(this.source[this.index++])) > 15 || c3 < 0) || ((c4 = Character.getNumericValue(this.source[this.index++])) > 15 || c4 < 0))) {
1163                                 c = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1164                         } else {
1165                                 // TODO (frederic) currently reset to previous position, perhaps signal a syntax error would be more appropriate
1166                                 this.index = pos;
1167                         }
1168                 }
1169                 return c;
1170         }
1171
1172         /*
1173          * Read token only if previous was consumed
1174          */
1175         private int readToken() throws InvalidInputException {
1176                 if (this.currentTokenType < 0) {
1177                         this.tokenPreviousPosition = this.scanner.currentPosition;
1178                         this.currentTokenType = this.scanner.getNextToken();
1179                         if (this.scanner.currentPosition > (this.lineEnd+1)) { // be sure to be on next line (lineEnd is still on the same line)
1180                                 this.lineStarted = false;
1181                                 while (this.currentTokenType == ITerminalSymbols.TokenNameMULTIPLY) {
1182                                         this.currentTokenType = this.scanner.getNextToken();
1183                                 }
1184                         }
1185                         this.index = this.scanner.currentPosition;
1186                         this.lineStarted = true; // after having read a token, line is obviously started...
1187                 }
1188                 return this.currentTokenType;
1189         }
1190
1191         private int readTokenAndConsume() throws InvalidInputException {
1192                 int token = readToken();
1193                 consumeToken();
1194                 return token;
1195         }
1196
1197         /*
1198          * Refresh start position and length of an inline tag.
1199          */
1200         protected void refreshInlineTagPosition(int previousPosition) {
1201                 // do nothing by default
1202         }
1203
1204         public String toString() {
1205                 StringBuffer buffer = new StringBuffer();
1206                 int startPos = this.scanner.currentPosition<this.index ? this.scanner.currentPosition : this.index;
1207                 int endPos = this.scanner.currentPosition<this.index ? this.index : this.scanner.currentPosition;
1208                 if (startPos == this.source.length)
1209                         return "EOF\n\n" + new String(this.source); //$NON-NLS-1$
1210                 if (endPos > this.source.length)
1211                         return "behind the EOF\n\n" + new String(this.source); //$NON-NLS-1$
1212
1213                 char front[] = new char[startPos];
1214                 System.arraycopy(this.source, 0, front, 0, startPos);
1215
1216                 int middleLength = (endPos - 1) - startPos + 1;
1217                 char middle[];
1218                 if (middleLength > -1) {
1219                         middle = new char[middleLength];
1220                         System.arraycopy(
1221                                 this.source,
1222                                 startPos,
1223                                 middle,
1224                                 0,
1225                                 middleLength);
1226                 } else {
1227                         middle = CharOperation.NO_CHAR;
1228                 }
1229
1230                 char end[] = new char[this.source.length - (endPos - 1)];
1231                 System.arraycopy(
1232                         this.source,
1233                         (endPos - 1) + 1,
1234                         end,
1235                         0,
1236                         this.source.length - (endPos - 1) - 1);
1237
1238                 buffer.append(front);
1239                 if (this.scanner.currentPosition<this.index) {
1240                         buffer.append("\n===============================\nScanner current position here -->"); //$NON-NLS-1$
1241                 } else {
1242                         buffer.append("\n===============================\nParser index here -->"); //$NON-NLS-1$
1243                 }
1244                 buffer.append(middle);
1245                 if (this.scanner.currentPosition<this.index) {
1246                         buffer.append("<-- Parser index here\n===============================\n"); //$NON-NLS-1$
1247                 } else {
1248                         buffer.append("<-- Scanner current position here\n===============================\n"); //$NON-NLS-1$
1249                 }
1250                 buffer.append(end);
1251
1252                 return buffer.toString();
1253         }
1254
1255         /*
1256          * Update
1257          */
1258         protected abstract void updateDocComment();
1259
1260         /*
1261          * Update line end
1262          */
1263         protected void updateLineEnd() {
1264                 while (this.index > (this.lineEnd+1)) { // be sure to be on next line (lineEnd is still on the same line)
1265                         if (this.linePtr < this.lastLinePtr) {
1266                                 this.lineEnd = this.scanner.getLineEnd(++this.linePtr) - 1;
1267                         } else {
1268                                 this.lineEnd = this.endComment;
1269                                 return;
1270                         }
1271                 }
1272         }
1273
1274         /*
1275          * Verify that end of the line only contains space characters or end of comment.
1276          * Note that end of comment may be preceeding by several contiguous '*' chars.
1277          */
1278         private boolean verifyEndLine(int textPosition) {
1279                 int startPosition = this.index;
1280                 int previousPosition = this.index;
1281                 this.starPosition = -1;
1282                 char ch = readChar();
1283                 nextChar: while (true) {
1284                         switch (ch) {
1285                                 case '\r':
1286                                 case '\n':
1287                                         if (this.kind == DOM_PARSER) {
1288                                                 parseTag();
1289                                                 pushText(textPosition, previousPosition);
1290                                         }
1291                                         this.index = previousPosition;
1292                                         return true;
1293                                 case '\u000c' : /* FORM FEED               */
1294                                 case ' ' :                      /* SPACE                   */
1295                                 case '\t' :                     /* HORIZONTAL TABULATION   */
1296                                         if (this.starPosition >= 0) break nextChar;
1297                                         break;
1298                                 case '*':
1299                                         this.starPosition = previousPosition;
1300                                         break;
1301                                 case '/':
1302                                         if (this.starPosition >= textPosition) {
1303                                                 if (this.kind == DOM_PARSER) {
1304                                                         parseTag();
1305                                                         pushText(textPosition, this.starPosition);
1306                                                 }
1307                                                 return true;
1308                                         }
1309                                 default :
1310                                         // leave loop
1311                                         break nextChar;
1312
1313                         }
1314                         previousPosition = this.index;
1315                         ch = readChar();
1316                 }
1317                 this.index = startPosition;
1318                 return false;
1319         }
1320
1321         /*
1322          * Verify that some text exists after a @return tag. Text must be different than
1323          * end of comment which may be preceeding by several '*' chars.
1324          */
1325         private boolean verifyCharsAfterReturnTag(int startPosition) {
1326                 // Whitespace or inline tag closing brace
1327                 int previousPosition = this.index;
1328                 char ch = readChar();
1329                 boolean malformed = true;
1330                 while (Character.isWhitespace(ch)) {
1331                         malformed = false;
1332                         previousPosition = this.index;
1333                         ch = readChar();
1334                 }
1335                 // End of comment
1336                 this.starPosition = -1;
1337                 nextChar: while (this.index<this.source.length) {
1338                         switch (ch) {
1339                                 case '*':
1340                                         // valid whatever the number of star before last '/'
1341                                         this.starPosition = previousPosition;
1342                                         break;
1343                                 case '/':
1344                                         if (this.starPosition >= startPosition) { // valid only if a star was previous character
1345                                                 return false;
1346                                         }
1347                                 default :
1348                                         // valid if any other character is encountered, even white spaces
1349                                         this.index = startPosition;
1350                                         return !malformed;
1351
1352                         }
1353                         previousPosition = this.index;
1354                         ch = readChar();
1355                 }
1356                 this.index = startPosition;
1357                 return false;
1358         }
1359
1360         /*
1361          * Verify characters after a name matches one of following conditions:
1362          *      1- first character is a white space
1363          *      2- first character is a closing brace *and* we're currently parsing an inline tag
1364          *      3- are the end of comment (several contiguous star ('*') characters may be
1365          *          found before the last slash ('/') character).
1366          */
1367         private boolean verifySpaceOrEndComment() {
1368                 int startPosition = this.index;
1369                 // Whitespace or inline tag closing brace
1370                 char ch = peekChar();
1371                 switch (ch) {
1372                         case '}':
1373                                 return this.inlineTagStarted;
1374                         default:
1375                                 if (Character.isWhitespace(ch)) {
1376                                         return true;
1377                                 }
1378                 }
1379                 // End of comment
1380                 int previousPosition = this.index;
1381                 this.starPosition = -1;
1382                 ch = readChar();
1383                 nextChar: while (this.index<this.source.length) {
1384                         switch (ch) {
1385                                 case '*':
1386                                         // valid whatever the number of star before last '/'
1387                                         this.starPosition = previousPosition;
1388                                         break;
1389                                 case '/':
1390                                         if (this.starPosition >= startPosition) { // valid only if a star was previous character
1391                                                 return true;
1392                                         }
1393                                 default :
1394                                         // invalid whatever other character, even white spaces
1395                                         this.index = startPosition;
1396                                         return false;
1397
1398                         }
1399                         previousPosition = this.index;
1400                         ch = readChar();
1401                 }
1402                 this.index = startPosition;
1403                 return false;
1404         }
1405 }