package org.asciidoc.intellij.lexer;

import com.intellij.codeInsight.completion.CompletionUtilCore;
import com.intellij.lexer.Lexer;
import com.intellij.testFramework.LexerTestCase;
import org.intellij.lang.annotations.Language;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.Nullable;

/**
 * Test cases covering lexing rules in asciidoc.flex.
 * To learn more about lexing, visit the contributor's guide for coders:
 * <a href="https://intellij-asciidoc-plugin.ahus1.de/docs/contributors-guide/coder/lexing-and-parsing.html">Lexing and parsing AsciiDoc files</a>.
 *
 * @author yole
 * @author Alexander Schwartz (alexander.schwartz@gmx.net)
 */
@SuppressWarnings({"AsciiDocHeadingStyle", "AsciiDocLinkResolve", "AsciiDocAttributeContinuation", "AsciiDocReferenceResolve", "AsciiDocHorizontalRule", "AsciiDocXrefWithNaturalCrossReference", "AsciiDocAttributeShouldBeDefined"})
public class AsciiDocLexerTest extends LexerTestCase {
  public void testSimple() {
    doTest("abc\ndef",
      "AsciiDoc:TEXT ('abc')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('def')");
  }

  public void testLineComment() {
    doTest("// foo\n// bar", "AsciiDoc:LINE_COMMENT ('// foo')\n" +
      "AsciiDoc:LINE_BREAK ('\\n')\n" +
      "AsciiDoc:LINE_COMMENT ('// bar')");
  }

  public void testLineCommentAfterEnumeration() {
    doTest("* item\n// comment",
      "AsciiDoc:BULLET ('*')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('item')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LINE_COMMENT ('// comment')");
  }

  public void testLineCommentWithCellCharacterInTable() {
    doTest("|===\n" +
        "//|comment\n" +
        "|===\n",
      "AsciiDoc:BLOCK_DELIMITER ('|===')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LINE_COMMENT ('//|comment')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('|===')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testNoLineCommentWithCellCharacterInTable() {
    doTest("|===\n" +
        "|//comment\n" +
        "|===\n",
      "AsciiDoc:BLOCK_DELIMITER ('|===')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CELLSEPARATOR ('|')\n" +
        "AsciiDoc:TEXT ('//comment')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('|===')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testLineCommentAndFormattingInCell() {
    doTest("|===\n" +
        "| _x_x_x\n" +
        "// comment\n" +
        "|===\n",
      "AsciiDoc:BLOCK_DELIMITER ('|===')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CELLSEPARATOR ('|')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('_x_x_x')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LINE_COMMENT ('// comment')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('|===')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testPassthroughStartButThatIsText() {
    doTest("+ ",
      "AsciiDoc:TEXT ('+')\n" +
        "AsciiDoc:WHITE_SPACE (' ')");
  }

  public void testPassthroughEscaped() {
    doTest("\\+test *bold* test+",
      "AsciiDoc:TEXT ('\\+test')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:BOLD_START ('*')\n" +
        "AsciiDoc:BOLD ('bold')\n" +
        "AsciiDoc:BOLD_END ('*')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('test+')");
  }

  public void testListing() {
    doTest("some text at start\n----\nbbbb\n----\ncccc",
      "AsciiDoc:TEXT ('some')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('text')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('at')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('start')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_BLOCK_DELIMITER ('----')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_TEXT ('bbbb')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_BLOCK_DELIMITER ('----')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('cccc')");
  }

  public void testListingWithSpaceDash() {
    doTest("----\nbbbb\n---- --\ncccc\n----\nText",
      "AsciiDoc:LISTING_BLOCK_DELIMITER ('----')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_TEXT ('bbbb')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_TEXT ('---- --')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_TEXT ('cccc')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_BLOCK_DELIMITER ('----')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testTextLookingLikeListing() {
    doTest("---- --\nnolisting",
      "AsciiDoc:TEXT ('----')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('--')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('nolisting')");
  }

  public void testListingAtEndOfFile() {
    doTest("----\nlisting\n----",
      "AsciiDoc:LISTING_BLOCK_DELIMITER ('----')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_TEXT ('listing')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_BLOCK_DELIMITER ('----')");
  }

  public void testListingWithInclude() {
    doTest("----\ninclude::file.adoc[]\n----\n",
      "AsciiDoc:LISTING_BLOCK_DELIMITER ('----')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_MACRO_ID ('include::')\n" +
        "AsciiDoc:BLOCK_MACRO_BODY ('file.adoc')\n" +
        "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTRS_END (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_BLOCK_DELIMITER ('----')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testIncludeWithTags() {
    doTest("include::file.adoc[tags=tag1;tag2]",
      "AsciiDoc:BLOCK_MACRO_ID ('include::')\n" +
        "AsciiDoc:BLOCK_MACRO_BODY ('file.adoc')\n" +
        "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('tags')\n" +
        "AsciiDoc:ASSIGNMENT ('=')\n" +
        "AsciiDoc:ATTR_VALUE ('tag1')\n" +
        "AsciiDoc:ATTR_LIST_SEP (';')\n" +
        "AsciiDoc:ATTR_VALUE ('tag2')\n" +
        "AsciiDoc:ATTRS_END (']')");
  }

  public void testIncludeWithIncompleteQuote() {
    doTest("include::file.adoc[tags='xx]",
      "AsciiDoc:BLOCK_MACRO_ID ('include::')\n" +
        "AsciiDoc:BLOCK_MACRO_BODY ('file.adoc')\n" +
        "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('tags')\n" +
        "AsciiDoc:ASSIGNMENT ('=')\n" +
        "AsciiDoc:SINGLE_QUOTE (''')\n" +
        "AsciiDoc:ATTR_VALUE ('xx')\n" +
        "AsciiDoc:ATTRS_END (']')");
  }

  public void testIncludeWithBlankAndIncompleteQuote() {
    doTest("include::file.adoc[tags= 'xx]",
      "AsciiDoc:BLOCK_MACRO_ID ('include::')\n" +
        "AsciiDoc:BLOCK_MACRO_BODY ('file.adoc')\n" +
        "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('tags')\n" +
        "AsciiDoc:ASSIGNMENT ('=')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:SINGLE_QUOTE (''')\n" +
        "AsciiDoc:ATTR_VALUE ('xx')\n" +
        "AsciiDoc:ATTRS_END (']')");
  }

  public void testIncludeWithTagsQuotedWithBlank() {
    doTest("include::file.adoc[tags=\"tag1; tag2\"]",
      "AsciiDoc:BLOCK_MACRO_ID ('include::')\n" +
        "AsciiDoc:BLOCK_MACRO_BODY ('file.adoc')\n" +
        "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('tags')\n" +
        "AsciiDoc:ASSIGNMENT ('=')\n" +
        "AsciiDoc:DOUBLE_QUOTE ('\"')\n" +
        "AsciiDoc:ATTR_VALUE ('tag1')\n" +
        "AsciiDoc:ATTR_LIST_SEP (';')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:ATTR_VALUE ('tag2')\n" +
        "AsciiDoc:DOUBLE_QUOTE ('\"')\n" +
        "AsciiDoc:ATTRS_END (']')");
  }

  public void testIncludeWithNegatedTag() {
    doTest("include::file.adoc[tags=\" !tag1!\"]",
      "AsciiDoc:BLOCK_MACRO_ID ('include::')\n" +
        "AsciiDoc:BLOCK_MACRO_BODY ('file.adoc')\n" +
        "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('tags')\n" +
        "AsciiDoc:ASSIGNMENT ('=')\n" +
        "AsciiDoc:DOUBLE_QUOTE ('\"')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:ATTR_LIST_OP ('!')\n" +
        "AsciiDoc:ATTR_VALUE ('tag1!')\n" +
        "AsciiDoc:DOUBLE_QUOTE ('\"')\n" +
        "AsciiDoc:ATTRS_END (']')");
  }

  public void testHeading() {
    doTest("= Abc\n\nabc\n== Def\ndef",
      "AsciiDoc:HEADING_TOKEN ('= Abc')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:EMPTY_LINE ('\\n')\n" +
        "AsciiDoc:TEXT ('abc')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('==')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Def')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('def')");
  }

  public void testDoctitleWithBlockId() {
    doTest("= [[id]] Title",
      "AsciiDoc:HEADING_TOKEN ('= ')\n" +
        "AsciiDoc:INLINEIDSTART ('[[')\n" +
        "AsciiDoc:BLOCKID ('id')\n" +
        "AsciiDoc:INLINEIDEND (']]')\n" +
        "AsciiDoc:HEADING_TOKEN (' Title')");
  }

  public void testBibBlock() {
    doTest("= [[id]] Title",
      "AsciiDoc:HEADING_TOKEN ('= ')\n" +
        "AsciiDoc:INLINEIDSTART ('[[')\n" +
        "AsciiDoc:BLOCKID ('id')\n" +
        "AsciiDoc:INLINEIDEND (']]')\n" +
        "AsciiDoc:HEADING_TOKEN (' Title')");
  }

  public void testSectionWithBlockId() {
    doTest("== [[id]] Section",
      "AsciiDoc:HEADING_TOKEN ('== ')\n" +
        "AsciiDoc:INLINEIDSTART ('[[')\n" +
        "AsciiDoc:BLOCKID ('id')\n" +
        "AsciiDoc:INLINEIDEND (']]')\n" +
        "AsciiDoc:HEADING_TOKEN (' Section')");
  }

  public void testSectionWithBlockIdAndText() {
    doTest("== A [[id,text]] Section",
      "AsciiDoc:HEADING_TOKEN ('== A ')\n" +
        "AsciiDoc:INLINEIDSTART ('[[')\n" +
        "AsciiDoc:BLOCKID ('id')\n" +
        "AsciiDoc:SEPARATOR (',')\n" +
        "AsciiDoc:BLOCKREFTEXT ('text')\n" +
        "AsciiDoc:INLINEIDEND (']]')\n" +
        "AsciiDoc:HEADING_TOKEN (' Section')");
  }

  public void testHeaderIfDef() {
    doTest("= Abc\nifdef::hi[]\nxx\nendif::[]",
      "AsciiDoc:HEADING_TOKEN ('= Abc')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_MACRO_ID ('ifdef::')\n" +
        "AsciiDoc:ATTRIBUTE_REF ('hi')\n" +
        "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTRS_END (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:HEADER ('xx')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_MACRO_ID ('endif::')\n" +
        "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTRS_END (']')");
  }

  public void testTable() {
    doTest("|====\n" +
        "|1|2|3\n\n" +
        "|4|5|6\n" +
        "|====",
      "AsciiDoc:BLOCK_DELIMITER ('|====')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CELLSEPARATOR ('|')\n" +
        "AsciiDoc:TEXT ('1')\n" +
        "AsciiDoc:CELLSEPARATOR ('|')\n" +
        "AsciiDoc:TEXT ('2')\n" +
        "AsciiDoc:CELLSEPARATOR ('|')\n" +
        "AsciiDoc:TEXT ('3')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:EMPTY_LINE ('\\n')\n" +
        "AsciiDoc:CELLSEPARATOR ('|')\n" +
        "AsciiDoc:TEXT ('4')\n" +
        "AsciiDoc:CELLSEPARATOR ('|')\n" +
        "AsciiDoc:TEXT ('5')\n" +
        "AsciiDoc:CELLSEPARATOR ('|')\n" +
        "AsciiDoc:TEXT ('6')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('|====')");
  }

  public void testTableWithVerticalAlignment() {
    doTest("|====\n" +
        ".^|1\n" +
        "|====",
      "AsciiDoc:BLOCK_DELIMITER ('|====')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CELLSEPARATOR ('.^|')\n" +
        "AsciiDoc:TEXT ('1')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('|====')");
  }

  public void testTableWithColAndRowSpanAlignmentAndFormatting() {
    doTest("|====\n" +
        "2.2+^.^h|1\n" +
        "|====",
      "AsciiDoc:BLOCK_DELIMITER ('|====')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CELLSEPARATOR ('2.2+^.^h|')\n" +
        "AsciiDoc:TEXT ('1')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('|====')");
  }

  public void testTableWithColSpanningAlignmentAndFormatting() {
    doTest("|====\n" +
        "2+^.^h|1\n" +
        "|====",
      "AsciiDoc:BLOCK_DELIMITER ('|====')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CELLSEPARATOR ('2+^.^h|')\n" +
        "AsciiDoc:TEXT ('1')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('|====')");
  }

  public void testTableWithRowSpanningAlignmentAndFormatting() {
    doTest("|====\n" +
        ".2+^.^h|1\n" +
        "|====",
      "AsciiDoc:BLOCK_DELIMITER ('|====')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CELLSEPARATOR ('.2+^.^h|')\n" +
        "AsciiDoc:TEXT ('1')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('|====')");
  }

  public void testTableWithDuplicationAlignmentAndFormatting() {
    doTest("|====\n" +
        "2*^.^h|1\n" +
        "|====",
      "AsciiDoc:BLOCK_DELIMITER ('|====')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CELLSEPARATOR ('2*^.^h|')\n" +
        "AsciiDoc:TEXT ('1')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('|====')");
  }

  public void testCellWithTitle() {
    doTest("|===\n" +
        "a| .Title\n" +
        "|===",
      "AsciiDoc:BLOCK_DELIMITER ('|===')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CELLSEPARATOR ('a|')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TITLE_TOKEN ('.')\n" +
        "AsciiDoc:TEXT ('Title')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('|===')");
  }

  public void testCellWithBlock() {
    doTest("|===\n" +
        "a|\n" +
        ".Title\n" +
        "| Cell\n" +
        "|===",
      "AsciiDoc:BLOCK_DELIMITER ('|===')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CELLSEPARATOR ('a|')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TITLE_TOKEN ('.')\n" +
        "AsciiDoc:TEXT ('Title')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CELLSEPARATOR ('|')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Cell')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('|===')");
  }

  public void testTableCellWithLeadingBlanks() {
    doTest("|====\n" +
        "|  1\n" +
        "|====",
      "AsciiDoc:BLOCK_DELIMITER ('|====')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CELLSEPARATOR ('|')\n" +
        "AsciiDoc:WHITE_SPACE ('  ')\n" +
        "AsciiDoc:TEXT ('1')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('|====')");
  }

  public void testTableWithNoBlanks() {
    doTest("|===\n" +
        "|Name|Type|\n" +
        "|===",
      "AsciiDoc:BLOCK_DELIMITER ('|===')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CELLSEPARATOR ('|')\n" +
        "AsciiDoc:TEXT ('Name')\n" +
        "AsciiDoc:CELLSEPARATOR ('|')\n" +
        "AsciiDoc:TEXT ('Type')\n" +
        "AsciiDoc:CELLSEPARATOR ('|')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('|===')");
  }

  public void testHeadingOldStyle() {
    doTest("Abc\n===\n\ndef",
      "AsciiDoc:HEADING_OLDSTYLE ('Abc\\n===')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:EMPTY_LINE ('\\n')\n" +
        "AsciiDoc:TEXT ('def')");
  }

  public void testHeadingOldStyleWithHeaderSeparatedByBlankLine() {
    doTest("Abc\n===\nHeader\n\ndef",
      "AsciiDoc:HEADING_OLDSTYLE ('Abc\\n===')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:HEADER ('Header')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:EMPTY_LINE ('\\n')\n" +
        "AsciiDoc:TEXT ('def')");
  }

  public void testHeadingOldStyleWithHeaderTwoLines() {
    doTest("Abc\n===\nHeader1\nHeader2\ndef",
      "AsciiDoc:HEADING_OLDSTYLE ('Abc\\n===')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:HEADER ('Header1')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:HEADER ('Header2')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('def')");
  }

  public void testNonHeadingAsItHasNoAlphanumericCharacter() {
    doTest("****\n" +
        "++++\n",
      "AsciiDoc:BLOCK_DELIMITER ('****')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:PASSTRHOUGH_BLOCK_DELIMITER ('++++')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testNonHeadingAsItIsAComment() {
    doTest("// c\n" +
        "++++\n",
      "AsciiDoc:LINE_COMMENT ('// c')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:PASSTRHOUGH_BLOCK_DELIMITER ('++++')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testNoLineCommentWithThreeSlashes() {
    doTest("/// notacomment",
      "AsciiDoc:TEXT ('///')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('notacomment')");
  }

  public void testVerySimpleComment() {
    doTest("//\n",
      "AsciiDoc:LINE_COMMENT ('//')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testHeadingNewStyleWithHeaderTwoLines() {
    doTest("= Abc\nHeader1\nHeader2\ndef",
      "AsciiDoc:HEADING_TOKEN ('= Abc')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:HEADER ('Header1')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:HEADER ('Header2')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('def')");
  }

  public void testHeadingNewStyleWithHeaderTwoLinesAndLineComment() {
    doTest("= Abc\nHeader1 // no comment\n// Comment\nHeader2\ndef",
      "AsciiDoc:HEADING_TOKEN ('= Abc')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:HEADER ('Header1 // no comment')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LINE_COMMENT ('// Comment')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:HEADER ('Header2')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('def')");
  }

  public void testHeadingNewStyleWithHeaderTwoLinesAndBlockComment() {
    doTest("= Abc\nHeader1\n////\nHiHo\n////\nHeader2\ndef",
      "AsciiDoc:HEADING_TOKEN ('= Abc')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:HEADER ('Header1')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_COMMENT ('////')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_COMMENT ('HiHo')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_COMMENT ('////')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:HEADER ('Header2')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('def')");
  }

  public void testHeadingNewStyleWithInclude() {
    doTest("= Abc\ninclude::test.adoc[]\n",
      "AsciiDoc:HEADING_TOKEN ('= Abc')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_MACRO_ID ('include::')\n" +
        "AsciiDoc:BLOCK_MACRO_BODY ('test.adoc')\n" +
        "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTRS_END (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testHeadingMarkdownStyleWithHeaderTwoLines() {
    //noinspection AsciiDocHeadingStyleInspection
    doTest("# Abc\nHeader1\nHeader2\ndef",
      "AsciiDoc:HEADING_TOKEN ('# Abc')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:HEADER ('Header1')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:HEADER ('Header2')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('def')");
  }

  public void testHeadingNewStyleWithAppendixStyle() {
    doTest("[appendix]\n= Abc\nText\n",
      "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('appendix')\n" +
        "AsciiDoc:ATTRS_END (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:HEADING_TOKEN ('= Abc')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('Text')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testHeadingOldStyleWithHeaderTwoLinesAndAttribute() {
    doTest("Abc\n===\nHeader1\n:attr: val\nHeader2\ndef",
      "AsciiDoc:HEADING_OLDSTYLE ('Abc\\n===')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:HEADER ('Header1')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:ATTRIBUTE_NAME_START (':')\n" +
        "AsciiDoc:ATTRIBUTE_NAME ('attr')\n" +
        "AsciiDoc:ATTRIBUTE_NAME_END (':')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:ATTRIBUTE_VAL ('val')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:HEADER ('Header2')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('def')");
  }

  public void testCommentBlock() {
    doTest("////\nfoo bar\n////\nabc",
      "AsciiDoc:BLOCK_COMMENT ('////')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_COMMENT ('foo bar')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_COMMENT ('////')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('abc')");
  }

  public void testNonCommentBlock() {
    doTest("////\\nfoo bar\\n////\\nabc",
      "AsciiDoc:TEXT ('////\\nfoo')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('bar\\n////\\nabc')");
  }


  public void testNonCommentBlockCell() {
    doTest("|===\n|////\n|===",
      "AsciiDoc:BLOCK_DELIMITER ('|===')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CELLSEPARATOR ('|')\n" +
        "AsciiDoc:TEXT ('////')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('|===')");
  }

  public void testBlockMacro() {
    doTest("image::foo.png[Caption]\nabc",
      "AsciiDoc:BLOCK_MACRO_ID ('image::')\n" +
        "AsciiDoc:BLOCK_MACRO_BODY ('foo.png')\n" +
        "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('Caption')\n" +
        "AsciiDoc:ATTRS_END (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('abc')");
  }

  public void testBlockMacroWithBlockAttributesShouldClearStyle() {
    doTest("[image]\n" +
        "image::test.png[]\n" +
        "\n" +
        "Example Query\n" +
        "[source]\n" +
        "sql",
      "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('image')\n" +
        "AsciiDoc:ATTRS_END (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_MACRO_ID ('image::')\n" +
        "AsciiDoc:BLOCK_MACRO_BODY ('test.png')\n" +
        "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTRS_END (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:EMPTY_LINE ('\\n')\n" +
        "AsciiDoc:TEXT ('Example')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Query')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('source')\n" +
        "AsciiDoc:ATTRS_END (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_TEXT ('sql')");
  }

  public void testInlineMacroUrl() {
    doTest("image:http://image.com[Caption]\nabc",
      "AsciiDoc:INLINE_MACRO_ID ('image:')\n" +
        "AsciiDoc:URL_LINK ('http://image.com')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('Caption')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('abc')");
  }

  public void testBlockMacroWithAttribute() {
    doTest("macro::foo[key=value]",
      "AsciiDoc:BLOCK_MACRO_ID ('macro::')\n" +
        "AsciiDoc:BLOCK_MACRO_BODY ('foo')\n" +
        "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('key')\n" +
        "AsciiDoc:ASSIGNMENT ('=')\n" +
        "AsciiDoc:ATTR_VALUE ('value')\n" +
        "AsciiDoc:ATTRS_END (']')");
  }

  public void testBlockMacroWithSingleQuotedAttribute() {
    doTest("macro::foo[key='value']",
      "AsciiDoc:BLOCK_MACRO_ID ('macro::')\n" +
        "AsciiDoc:BLOCK_MACRO_BODY ('foo')\n" +
        "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('key')\n" +
        "AsciiDoc:ASSIGNMENT ('=')\n" +
        "AsciiDoc:SINGLE_QUOTE (''')\n" +
        "AsciiDoc:ATTR_VALUE ('value')\n" +
        "AsciiDoc:SINGLE_QUOTE (''')\n" +
        "AsciiDoc:ATTRS_END (']')");
  }

  public void testBlockMacroWithDoubleQuotedAttribute() {
    doTest("macro::foo[key=\"value\"]",
      "AsciiDoc:BLOCK_MACRO_ID ('macro::')\n" +
        "AsciiDoc:BLOCK_MACRO_BODY ('foo')\n" +
        "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('key')\n" +
        "AsciiDoc:ASSIGNMENT ('=')\n" +
        "AsciiDoc:DOUBLE_QUOTE ('\"')\n" +
        "AsciiDoc:ATTR_VALUE ('value')\n" +
        "AsciiDoc:DOUBLE_QUOTE ('\"')\n" +
        "AsciiDoc:ATTRS_END (']')");
  }

  public void testExample() {
    doTest("====\nFoo\n====\n",
      "AsciiDoc:BLOCK_DELIMITER ('====')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('Foo')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('====')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testTitle() {
    doTest(".Foo bar baz\nFoo bar baz",
      "AsciiDoc:TITLE_TOKEN ('.')\n" +
        "AsciiDoc:TEXT ('Foo')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('bar')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('baz')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('Foo')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('bar')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('baz')");
  }

  public void testBlockAttrs() {
    doTest("[NOTE]\n",
      "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('NOTE')\n" +
        "AsciiDoc:ATTRS_END (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testAdmonition() {
    doTest("[NOTE]\n" +
      "====\n" +
      "TTT\n" +
      "====",
      "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('NOTE')\n" +
        "AsciiDoc:ATTRS_END (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('====')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('TTT')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('====')");
  }

  public void testNoBlockAttrs() {
    doTest("[nolink]:: Term",
      "AsciiDoc:LBRACKET ('[')\n" +
        "AsciiDoc:DESCRIPTION ('nolink')\n" +
        "AsciiDoc:RBRACKET (']')\n" +
        "AsciiDoc:DESCRIPTION_END ('::')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Term')");
  }

  public void testUnclosedBlockAttrs() {
    doTest("[\nfoo",
      "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('foo')");
  }

  public void testOldStyleHeading() {
    doTest("Hi\n--\n",
      "AsciiDoc:HEADING_OLDSTYLE ('Hi\\n--')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testBlockTypeLookingLikeAHeading() {
    doTest("[TIP]\n=====\nTip\n=====\n",
      "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('TIP')\n" +
        "AsciiDoc:ATTRS_END (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('=====')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('Tip')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('=====')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }


  public void testAttributeUsage() {
    doTest("This is an {attribute} more text.",
      "AsciiDoc:TEXT ('This')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('is')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('an')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:ATTRIBUTE_REF_START ('{')\n" +
        "AsciiDoc:ATTRIBUTE_REF ('attribute')\n" +
        "AsciiDoc:ATTRIBUTE_REF_END ('}')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('more')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('text.')");
  }

  public void testEscapedAttribute() {
    doTest("\\{ ",
      "AsciiDoc:TEXT ('\\{')\n" +
        "AsciiDoc:WHITE_SPACE (' ')");
  }

  public void testEscapedInHeading() {
    doTest("== Heading \\{esc} word",
      "AsciiDoc:HEADING_TOKEN ('== Heading \\{esc} word')");
  }

  public void testAttributeWithoutValue() {
    doTest(":attribute:",
      "AsciiDoc:ATTRIBUTE_NAME_START (':')\n" +
        "AsciiDoc:ATTRIBUTE_NAME ('attribute')\n" +
        "AsciiDoc:ATTRIBUTE_NAME_END (':')");
  }

  public void testAttributeEmptyAtEnd() {
    doTest(":attribute!:",
      "AsciiDoc:ATTRIBUTE_NAME_START (':')\n" +
        "AsciiDoc:ATTRIBUTE_NAME ('attribute')\n" +
        "AsciiDoc:ATTRIBUTE_UNSET ('!')\n" +
        "AsciiDoc:ATTRIBUTE_NAME_END (':')");
  }

  public void testAttributeWithSoftSet() {
    doTest(":attribute@:",
      "AsciiDoc:ATTRIBUTE_NAME_START (':')\n" +
        "AsciiDoc:ATTRIBUTE_NAME ('attribute')\n" +
        "AsciiDoc:ATTRIBUTE_SOFTSET ('@')\n" +
        "AsciiDoc:ATTRIBUTE_NAME_END (':')");
  }

  public void testAttributeWithBlanks() {
    doTest(":attri b ute :",
      "AsciiDoc:ATTRIBUTE_NAME_START (':')\n" +
        "AsciiDoc:ATTRIBUTE_NAME ('attri b ute ')\n" +
        "AsciiDoc:ATTRIBUTE_NAME_END (':')");
  }


  public void testAttributeEmptyAtStart() {
    doTest(":attribute!:",
      "AsciiDoc:ATTRIBUTE_NAME_START (':')\n" +
        "AsciiDoc:ATTRIBUTE_NAME ('attribute')\n" +
        "AsciiDoc:ATTRIBUTE_UNSET ('!')\n" +
        "AsciiDoc:ATTRIBUTE_NAME_END (':')");
  }

  public void testAttributeWithBracket() {
    doTest(":attr: icon:check[]",
      "AsciiDoc:ATTRIBUTE_NAME_START (':')\n" +
        "AsciiDoc:ATTRIBUTE_NAME ('attr')\n" +
        "AsciiDoc:ATTRIBUTE_NAME_END (':')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:ATTRIBUTE_VAL ('icon:check[]')");
  }

  public void testAttributeInTitle() {
    doTest(".xx{hi}xx",
      "AsciiDoc:TITLE_TOKEN ('.')\n" +
        "AsciiDoc:TEXT ('xx')\n" +
        "AsciiDoc:ATTRIBUTE_REF_START ('{')\n" +
        "AsciiDoc:ATTRIBUTE_REF ('hi')\n" +
        "AsciiDoc:ATTRIBUTE_REF_END ('}')\n" +
        "AsciiDoc:TEXT ('xx')");
  }

  public void testBracketInBlockAttributes() {
    doTest("[val=\"{attr}[xx]\"]",
      "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('val')\n" +
        "AsciiDoc:ASSIGNMENT ('=')\n" +
        "AsciiDoc:DOUBLE_QUOTE ('\"')\n" +
        "AsciiDoc:ATTRIBUTE_REF_START ('{')\n" +
        "AsciiDoc:ATTRIBUTE_REF ('attr')\n" +
        "AsciiDoc:ATTRIBUTE_REF_END ('}')\n" +
        "AsciiDoc:ATTR_VALUE ('[xx]')\n" +
        "AsciiDoc:DOUBLE_QUOTE ('\"')\n" +
        "AsciiDoc:ATTRS_END (']')");
  }

  public void testAttributeEscaped() {
    doTest("\\:attribute:",
      "AsciiDoc:TEXT ('\\:attribute:')");
  }

  public void testAttributeWithValue() {
    doTest(":attribute: value",
      "AsciiDoc:ATTRIBUTE_NAME_START (':')\n" +
        "AsciiDoc:ATTRIBUTE_NAME ('attribute')\n" +
        "AsciiDoc:ATTRIBUTE_NAME_END (':')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:ATTRIBUTE_VAL ('value')");
  }

  public void testAttributeWithNestedAttributeAndValue() {
    doTest(":attribute: {otherattr}value",
      "AsciiDoc:ATTRIBUTE_NAME_START (':')\n" +
        "AsciiDoc:ATTRIBUTE_NAME ('attribute')\n" +
        "AsciiDoc:ATTRIBUTE_NAME_END (':')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:ATTRIBUTE_REF_START ('{')\n" +
        "AsciiDoc:ATTRIBUTE_REF ('otherattr')\n" +
        "AsciiDoc:ATTRIBUTE_REF_END ('}')\n" +
        "AsciiDoc:ATTRIBUTE_VAL ('value')");
  }

  /**
   * Value continue on the next line if the line is ended by a space followed by a backslash.
   */
  public void testAttributeMultiline() {
    doTest(":attribute: value \\\n continue on the next line\nMore text",
      "AsciiDoc:ATTRIBUTE_NAME_START (':')\n" +
        "AsciiDoc:ATTRIBUTE_NAME ('attribute')\n" +
        "AsciiDoc:ATTRIBUTE_NAME_END (':')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:ATTRIBUTE_VAL ('value')\n" +
        "AsciiDoc:ATTRIBUTE_CONTINUATION (' \\\\n ')\n" +
        "AsciiDoc:ATTRIBUTE_VAL ('continue on the next line')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('More')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('text')");
  }

  /**
   * Value continue on the next line if the line is ended by a space followed by a backslash.
   */
  public void testAttributeMultilineWithPlus() {
    //noinspection AsciiDocAttributeContinuationInspection
    doTest(":attribute: value +\n continue on the next line\nMore text",
      "AsciiDoc:ATTRIBUTE_NAME_START (':')\n" +
        "AsciiDoc:ATTRIBUTE_NAME ('attribute')\n" +
        "AsciiDoc:ATTRIBUTE_NAME_END (':')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:ATTRIBUTE_VAL ('value')\n" +
        "AsciiDoc:ATTRIBUTE_CONTINUATION_LEGACY (' +\\n ')\n" +
        "AsciiDoc:ATTRIBUTE_VAL ('continue on the next line')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('More')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('text')");
  }

  public void testTwoConsecutiveAttributes() {
    doTest("Text\n\n:attribute1:\n:attribute2:",
      "AsciiDoc:TEXT ('Text')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:EMPTY_LINE ('\\n')\n" +
        "AsciiDoc:ATTRIBUTE_NAME_START (':')\n" +
        "AsciiDoc:ATTRIBUTE_NAME ('attribute1')\n" +
        "AsciiDoc:ATTRIBUTE_NAME_END (':')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:ATTRIBUTE_NAME_START (':')\n" +
        "AsciiDoc:ATTRIBUTE_NAME ('attribute2')\n" +
        "AsciiDoc:ATTRIBUTE_NAME_END (':')");
  }

  public void testNoAttributeAfterText() {
    doTest("Text\n:attribute:\n",
      "AsciiDoc:TEXT ('Text')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT (':attribute')\n" +
        "AsciiDoc:END_OF_SENTENCE (':')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testContinuation() {
    doTest("+\n--\n",
      "AsciiDoc:CONTINUATION ('+')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('--')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testContinuationInList() {
    doTest("* Hi\n" +
        "+\n" +
        "image::animage.png[]",
      "AsciiDoc:BULLET ('*')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Hi')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CONTINUATION ('+')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_MACRO_ID ('image::')\n" +
        "AsciiDoc:BLOCK_MACRO_BODY ('animage.png')\n" +
        "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTRS_END (']')");
  }

  public void testContinuationsThatLookLikeAHeading() {
    doTest("1. one\n" +
        "+\n" +
        "to\n" +
        "+\n" +
        "2. two",
      "AsciiDoc:ENUMERATION ('1.')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('one')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CONTINUATION ('+')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('to')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CONTINUATION ('+')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:ENUMERATION ('2.')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('two')");
  }

  public void testContinuationAfter() {
    doTest("--\n+\n",
      "AsciiDoc:BLOCK_DELIMITER ('--')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CONTINUATION ('+')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testContinuationThenTitle() {
    doTest("+\n.Title",
      "AsciiDoc:CONTINUATION ('+')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TITLE_TOKEN ('.')\n" +
        "AsciiDoc:TEXT ('Title')");
  }

  public void testAnchorThenTitle() {
    doTest("[#anchor]\n.Title",
      "AsciiDoc:BLOCKIDSTART ('[#')\n" +
        "AsciiDoc:BLOCKID ('anchor')\n" +
        "AsciiDoc:BLOCKIDEND (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TITLE_TOKEN ('.')\n" +
        "AsciiDoc:TEXT ('Title')");
  }

  public void testAnchorWithStyle() {
    doTest("[#anchor%style]\nText",
      "AsciiDoc:BLOCKIDSTART ('[#')\n" +
        "AsciiDoc:BLOCKID ('anchor')\n" +
        "AsciiDoc:SEPARATOR ('%style')\n" +
        "AsciiDoc:BLOCKIDEND (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testBoldSimple() {
    doTest("Hello *bold* world",
      "AsciiDoc:TEXT ('Hello')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:BOLD_START ('*')\n" +
        "AsciiDoc:BOLD ('bold')\n" +
        "AsciiDoc:BOLD_END ('*')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('world')");
  }

  public void testBoldDouble() {
    doTest("Hello **bold** world",
      "AsciiDoc:TEXT ('Hello')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:DOUBLEBOLD_START ('**')\n" +
        "AsciiDoc:BOLD ('bold')\n" +
        "AsciiDoc:DOUBLEBOLD_END ('**')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('world')");
  }

  public void testBoldDoubleMultiple() {
    doTest("**E**quivalent **M**odulo",
      "AsciiDoc:DOUBLEBOLD_START ('**')\n" +
        "AsciiDoc:BOLD ('E')\n" +
        "AsciiDoc:DOUBLEBOLD_END ('**')\n" +
        "AsciiDoc:TEXT ('quivalent')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:DOUBLEBOLD_START ('**')\n" +
        "AsciiDoc:BOLD ('M')\n" +
        "AsciiDoc:DOUBLEBOLD_END ('**')\n" +
        "AsciiDoc:TEXT ('odulo')");
  }

  public void testItalicDoubleMultiple() {
    doTest("__E__quivalent __M__odulo",
      "AsciiDoc:DOUBLEITALIC_START ('__')\n" +
        "AsciiDoc:ITALIC ('E')\n" +
        "AsciiDoc:DOUBLEITALIC_END ('__')\n" +
        "AsciiDoc:TEXT ('quivalent')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:DOUBLEITALIC_START ('__')\n" +
        "AsciiDoc:ITALIC ('M')\n" +
        "AsciiDoc:DOUBLEITALIC_END ('__')\n" +
        "AsciiDoc:TEXT ('odulo')");
  }

  public void testNonBoldWithBlockBreak() {
    doTest("Hello **bold\n\n** world",
      "AsciiDoc:TEXT ('Hello')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('**bold')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:EMPTY_LINE ('\\n')\n" +
        "AsciiDoc:BULLET ('**')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('world')");
  }

  public void testBoldAtBeginningAndEndOfLineSingle() {
    doTest("*bold*",
      "AsciiDoc:BOLD_START ('*')\n" +
        "AsciiDoc:BOLD ('bold')\n" +
        "AsciiDoc:BOLD_END ('*')");
  }

  public void testSingleQuote() {
    doTest("'single'",
      "AsciiDoc:SINGLE_QUOTE (''')\n" +
        "AsciiDoc:TEXT ('single')\n" +
        "AsciiDoc:SINGLE_QUOTE (''')");
  }

  public void testNoSingleQuoteJustText() {
    doTest("don't",
      "AsciiDoc:TEXT ('don't')");
  }

  public void testItalicBlankAtEndOfFirstLine() {
    doTest("_test \ntest_",
      "AsciiDoc:ITALIC_START ('_')\n" +
        "AsciiDoc:ITALIC ('test')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:ITALIC ('test')\n" +
        "AsciiDoc:ITALIC_END ('_')");
  }

  public void testNonItalicAsPrecededByNewline() {
    doTest("_test\n_",
      "AsciiDoc:TEXT ('_test')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('_')");
  }

  public void testBoldMultipleInSingleLine() {
    doTest("bold *constrained* & **un**constrained",
      "AsciiDoc:TEXT ('bold')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:BOLD_START ('*')\n" +
        "AsciiDoc:BOLD ('constrained')\n" +
        "AsciiDoc:BOLD_END ('*')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('&')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:DOUBLEBOLD_START ('**')\n" +
        "AsciiDoc:BOLD ('un')\n" +
        "AsciiDoc:DOUBLEBOLD_END ('**')\n" +
        "AsciiDoc:TEXT ('constrained')");
  }

  public void testConstrainedNumber() {
    doTest("11_11_11",
      "AsciiDoc:TEXT ('11_11_11')");
  }

  public void testSingleClosingQuoteInMonospace() {
    doTest("`test`'s formatting`",
      "AsciiDoc:MONO_START ('`')\n" +
        "AsciiDoc:MONO ('test')\n" +
        "AsciiDoc:TYPOGRAPHIC_SINGLE_QUOTE_END ('`'')\n" +
        "AsciiDoc:MONO ('s')\n" +
        "AsciiDoc:WHITE_SPACE_MONO (' ')\n" +
        "AsciiDoc:MONO ('formatting')\n" +
        "AsciiDoc:MONO_END ('`')");
  }

  public void testItalicMultipleInSingleLine() {
    doTest("italic _constrained_ & __un__constrained",
      "AsciiDoc:TEXT ('italic')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:ITALIC_START ('_')\n" +
        "AsciiDoc:ITALIC ('constrained')\n" +
        "AsciiDoc:ITALIC_END ('_')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('&')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:DOUBLEITALIC_START ('__')\n" +
        "AsciiDoc:ITALIC ('un')\n" +
        "AsciiDoc:DOUBLEITALIC_END ('__')\n" +
        "AsciiDoc:TEXT ('constrained')");
  }

  public void testItalicWithTempltingUnderscore() {
    doTest("text `_t` text `_t`",
      "AsciiDoc:TEXT ('text')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:MONO_START ('`')\n" +
        "AsciiDoc:MONO ('_t')\n" +
        "AsciiDoc:MONO_END ('`')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('text')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:MONO_START ('`')\n" +
        "AsciiDoc:MONO ('_t')\n" +
        "AsciiDoc:MONO_END ('`')");
  }

  public void testItalicWithTwoTableCells() {
    doTest("|===\n|_text | text_\n|===",
      "AsciiDoc:BLOCK_DELIMITER ('|===')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CELLSEPARATOR ('|')\n" +
        "AsciiDoc:TEXT ('_text')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:CELLSEPARATOR ('|')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('text_')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('|===')");
  }

  public void testDoubleItalicWithTwoTableCells() {
    doTest("|===\n|__text | text__\n|===",
      "AsciiDoc:BLOCK_DELIMITER ('|===')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CELLSEPARATOR ('|')\n" +
        "AsciiDoc:TEXT ('__text')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:CELLSEPARATOR ('|')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('text__')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('|===')");
  }

  public void testWithItalicInTwoParagraphs() {
    doTest("_text_text\n\ntext_",
      "AsciiDoc:TEXT ('_text_text')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:EMPTY_LINE ('\\n')\n" +
        "AsciiDoc:TEXT ('text_')");
  }

  public void testMonoMultipleInSingleLine() {
    doTest("mono `constrained` & ``un``constrained",
      "AsciiDoc:TEXT ('mono')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:MONO_START ('`')\n" +
        "AsciiDoc:MONO ('constrained')\n" +
        "AsciiDoc:MONO_END ('`')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('&')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:DOUBLEMONO_START ('``')\n" +
        "AsciiDoc:MONO ('un')\n" +
        "AsciiDoc:DOUBLEMONO_END ('``')\n" +
        "AsciiDoc:TEXT ('constrained')");
  }

  public void testMonoItalicBold() {
    doTest("``**__un__**``constrained",
      "AsciiDoc:DOUBLEMONO_START ('``')\n" +
        "AsciiDoc:DOUBLEBOLD_START ('**')\n" +
        "AsciiDoc:DOUBLEITALIC_START ('__')\n" +
        "AsciiDoc:MONOBOLDITALIC ('un')\n" +
        "AsciiDoc:DOUBLEITALIC_END ('__')\n" +
        "AsciiDoc:DOUBLEBOLD_END ('**')\n" +
        "AsciiDoc:DOUBLEMONO_END ('``')\n" +
        "AsciiDoc:TEXT ('constrained')");
  }

  public void testArrow() {
    doTest("-> => <- <=",
      "AsciiDoc:ARROW ('->')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:ARROW ('=>')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:ARROW ('<-')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:ARROW ('<=')");
  }

  public void testBoldAtBeginningAndEndOfLineDouble() {
    doTest("**bold**",
      "AsciiDoc:DOUBLEBOLD_START ('**')\n" +
        "AsciiDoc:BOLD ('bold')\n" +
        "AsciiDoc:DOUBLEBOLD_END ('**')");
  }

  public void testNonMatchingBoldHead() {
    doTest("**bold*",
      "AsciiDoc:BOLD_START ('*')\n" +
        "AsciiDoc:BOLD ('*bold')\n" +
        "AsciiDoc:BOLD_END ('*')");
  }

  public void testNonMatchingBoldTail() {
    doTest("*bold**",
      "AsciiDoc:BOLD_START ('*')\n" +
        "AsciiDoc:BOLD ('bold*')\n" +
        "AsciiDoc:BOLD_END ('*')");
  }

  public void testUnconstrainedNonBold() {
    doTest("x*nonbold*x",
      "AsciiDoc:TEXT ('x*nonbold*x')");
  }

  public void testUnconstrainedNonItalic() {
    doTest("x_nonitalic_x",
      "AsciiDoc:TEXT ('x_nonitalic_x')");
  }

  public void testUnconstrainedNonMono() {
    doTest("x`nonmono`x",
      "AsciiDoc:TEXT ('x`nonmono`x')");
  }

  public void testSpecialUnderscore() {
    doTest("x__*italiconly*__x",
      "AsciiDoc:TEXT ('x')\n" +
        "AsciiDoc:DOUBLEITALIC_START ('__')\n" +
        "AsciiDoc:ITALIC ('*italiconly*')\n" +
        "AsciiDoc:DOUBLEITALIC_END ('__')\n" +
        "AsciiDoc:TEXT ('x')");
  }

  public void testBoldItalic() {
    doTest("*_bolditalic_*",
      "AsciiDoc:BOLD_START ('*')\n" +
        "AsciiDoc:ITALIC_START ('_')\n" +
        "AsciiDoc:BOLDITALIC ('bolditalic')\n" +
        "AsciiDoc:ITALIC_END ('_')\n" +
        "AsciiDoc:BOLD_END ('*')");
  }

  public void testConstrainedMustNotEndWithBlankBold() {
    doTest("*test * test*",
      "AsciiDoc:BOLD_START ('*')\n" +
        "AsciiDoc:BOLD ('test')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:BOLD ('*')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:BOLD ('test')\n" +
        "AsciiDoc:BOLD_END ('*')");
  }

  public void testConstrainedMustNotEndWithBlankItalic() {
    doTest("_test _ test_",
      "AsciiDoc:ITALIC_START ('_')\n" +
        "AsciiDoc:ITALIC ('test')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:ITALIC ('_')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:ITALIC ('test')\n" +
        "AsciiDoc:ITALIC_END ('_')");
  }

  public void testConstrainedMustNotEndWithBlankMono() {
    doTest("`test ` test`",
      "AsciiDoc:MONO_START ('`')\n" +
        "AsciiDoc:MONO ('test')\n" +
        "AsciiDoc:WHITE_SPACE_MONO (' ')\n" +
        "AsciiDoc:MONO ('`')\n" +
        "AsciiDoc:WHITE_SPACE_MONO (' ')\n" +
        "AsciiDoc:MONO ('test')\n" +
        "AsciiDoc:MONO_END ('`')");
  }

  public void testBullet() {
    doTest("* bullet",
      "AsciiDoc:BULLET ('*')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('bullet')");
  }

  public void testBulletWithBlanksInFront() {
    doTest("  * bullet",
      "AsciiDoc:WHITE_SPACE ('  ')\n" +
        "AsciiDoc:BULLET ('*')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('bullet')");
  }

  public void testMultipleBullets() {
    doTest("* bullet1\n* bullet2",
      "AsciiDoc:BULLET ('*')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('bullet1')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BULLET ('*')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('bullet2')");
  }

  public void testMultipleBulletsLevel2() {
    doTest("** bullet1\n** bullet2",
      "AsciiDoc:BULLET ('**')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('bullet1')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BULLET ('**')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('bullet2')");
  }

  public void testThreeBulletItems() {
    doTest("* abc\n" +
        "* def\n" +
        "* ghi\n",
      "AsciiDoc:BULLET ('*')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('abc')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BULLET ('*')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('def')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BULLET ('*')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('ghi')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testSidebar() {
    doTest("****\nFoo\n****\n",
      "AsciiDoc:BLOCK_DELIMITER ('****')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('Foo')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('****')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testRef() {
    doTest("Text <<REF>> More Text",
      "AsciiDoc:TEXT ('Text')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:REFSTART ('<<')\n" +
        "AsciiDoc:REF ('REF')\n" +
        "AsciiDoc:REFEND ('>>')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('More')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testEscapedRef() {
    doTest("\\<<REF>>",
      "AsciiDoc:TEXT ('\\')\n" +
        "AsciiDoc:LT ('<')\n" +
        "AsciiDoc:LT ('<')\n" +
        "AsciiDoc:TEXT ('REF')\n" +
        "AsciiDoc:GT ('>')\n" +
        "AsciiDoc:GT ('>')");
  }

  public void testRefWithFile() {
    doTest("Text <<FILE#REF>> More Text",
      "AsciiDoc:TEXT ('Text')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:REFSTART ('<<')\n" +
        "AsciiDoc:REF ('FILE#REF')\n" +
        "AsciiDoc:REFEND ('>>')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('More')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testRefWithRefText() {
    doTest("Text <<REF,Text>> More Text",
      "AsciiDoc:TEXT ('Text')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:REFSTART ('<<')\n" +
        "AsciiDoc:REF ('REF')\n" +
        "AsciiDoc:SEPARATOR (',')\n" +
        "AsciiDoc:REFTEXT ('Text')\n" +
        "AsciiDoc:REFEND ('>>')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('More')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testRefWithRefTextWithClosingPointyBracket() {
    doTest("Text <<REF,Text > More>> Text",
      "AsciiDoc:TEXT ('Text')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:REFSTART ('<<')\n" +
        "AsciiDoc:REF ('REF')\n" +
        "AsciiDoc:SEPARATOR (',')\n" +
        "AsciiDoc:REFTEXT ('Text > More')\n" +
        "AsciiDoc:REFEND ('>>')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testEscapedLink() {
    doTest("\\link:FILE[Text]",
      "AsciiDoc:TEXT ('\\link:FILE')\n" +
        "AsciiDoc:LBRACKET ('[')\n" +
        "AsciiDoc:TEXT ('Text')\n" +
        "AsciiDoc:RBRACKET (']')");
  }

  public void testEscapedLinkText() {
    doTest("link:FILE[T\\]ext]",
      "AsciiDoc:LINKSTART ('link:')\n" +
        "AsciiDoc:LINKFILE ('FILE')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:MACROTEXT ('T\\]ext')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')");
  }

  public void testAttrInUrl() {
    doTest("http://url.com{path}[text]",
      "AsciiDoc:URL_LINK ('http://url.com')\n" +
        "AsciiDoc:ATTRIBUTE_REF_START ('{')\n" +
        "AsciiDoc:ATTRIBUTE_REF ('path')\n" +
        "AsciiDoc:ATTRIBUTE_REF_END ('}')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:MACROTEXT ('text')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')");
  }

  public void testHtmlEntity() {
    doTest("Hi&amp;Ho",
      "AsciiDoc:TEXT ('Hi')\n" +
        "AsciiDoc:HTML_ENTITY ('&amp;')\n" +
        "AsciiDoc:TEXT ('Ho')");
  }

  public void testUnicodeDecimalEntity() {
    doTest("Hi&#123;Ho",
      "AsciiDoc:TEXT ('Hi')\n" +
        "AsciiDoc:HTML_ENTITY ('&#123;')\n" +
        "AsciiDoc:TEXT ('Ho')");
  }

  public void testUnicodeHexEntity() {
    doTest("Hi&#x123Af;Ho",
      "AsciiDoc:TEXT ('Hi')\n" +
        "AsciiDoc:HTML_ENTITY ('&#x123Af;')\n" +
        "AsciiDoc:TEXT ('Ho')");
  }

  public void testHtmlEntityEscaped() {
    doTest("Hi\\&amp;Ho",
      "AsciiDoc:TEXT ('Hi\\&amp;Ho')");
  }

  public void testUnicodeDecimalEntityEscaped() {
    doTest("Hi\\&#123;Ho",
      "AsciiDoc:TEXT ('Hi\\&#123;Ho')");
  }

  public void testUnicodeHexEntityEscaped() {
    doTest("Hi\\&#x123Af;Ho",
      "AsciiDoc:TEXT ('Hi\\&#x123Af;Ho')");
  }

  public void testAttrInLink() {
    doTest("link:http://url.com{path}[text]",
      "AsciiDoc:LINKSTART ('link:')\n" +
        "AsciiDoc:URL_LINK ('http://url.com')\n" +
        "AsciiDoc:ATTRIBUTE_REF_START ('{')\n" +
        "AsciiDoc:ATTRIBUTE_REF ('path')\n" +
        "AsciiDoc:ATTRIBUTE_REF_END ('}')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:MACROTEXT ('text')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')");
  }

  public void testLinkWithAttributeAutocomplete() {
    doTest("link:IntellijIdeaRulezzz test.adoc[]\n",
      "AsciiDoc:LINKSTART ('link:')\n" +
        "AsciiDoc:LINKFILE ('IntellijIdeaRulezzz test.adoc')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testLinkInFormatting() {
    doTest("`http://localhost:8080/`",
      "AsciiDoc:MONO_START ('`')\n" +
        "AsciiDoc:URL_LINK ('http://localhost:8080/')\n" +
        "AsciiDoc:MONO_END ('`')");

  }

  public void testLinkCharLikeFormattingAtEnd() {
    doTest("http://localhost:8080/`",
      "AsciiDoc:URL_LINK ('http://localhost:8080/`')\n");
  }

  public void testLinkWithTitleAndContinuation() {
    doTest("link:test.adoc[Title +\nContinuing]\n",
      "AsciiDoc:LINKSTART ('link:')\n" +
        "AsciiDoc:LINKFILE ('test.adoc')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:MACROTEXT ('Title')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:CONTINUATION ('+')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:MACROTEXT ('Continuing')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testLinkWithAnchor() {
    doTest("Text link:FILE#ANCHOR[Text] More Text",
      "AsciiDoc:TEXT ('Text')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:LINKSTART ('link:')\n" +
        "AsciiDoc:LINKFILE ('FILE')\n" +
        "AsciiDoc:SEPARATOR ('#')\n" +
        "AsciiDoc:LINKANCHOR ('ANCHOR')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:MACROTEXT ('Text')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('More')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testLinkWithQuotes() {
    doTest("Text link:++https://example.org/?q=[a b]++[URL with special characters] Text",
      "AsciiDoc:TEXT ('Text')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:LINKSTART ('link:')\n" +
        "AsciiDoc:URL_LINK ('++https://example.org/?q=[a b]++')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:MACROTEXT ('URL')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:MACROTEXT ('with')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:MACROTEXT ('special')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:MACROTEXT ('characters')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testLinkForAutocomplete() {
    doTest("Text link:FILEIntellijIdeaRulezzz More Text",
      "AsciiDoc:TEXT ('Text')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:LINKSTART ('link:')\n" +
        "AsciiDoc:LINKFILE ('FILEIntellijIdeaRulezzz More')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testLinkForAutocompleteWithBrackets() {
    doTest("link:IntellijIdeaRulezzz []",
      "AsciiDoc:LINKSTART ('link:')\n" +
        "AsciiDoc:LINKFILE ('IntellijIdeaRulezzz ')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')");
  }

  public void testBlockid() {
    doTest("[[BLOCKID]] Text",
      "AsciiDoc:BLOCKIDSTART ('[[')\n" +
        "AsciiDoc:BLOCKID ('BLOCKID')\n" +
        "AsciiDoc:BLOCKIDEND (']]')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testBlockidWithRefText() {
    doTest("[[BLOCKID,name]] Text",
      "AsciiDoc:BLOCKIDSTART ('[[')\n" +
        "AsciiDoc:BLOCKID ('BLOCKID')\n" +
        "AsciiDoc:SEPARATOR (',')\n" +
        "AsciiDoc:BLOCKREFTEXT ('name')\n" +
        "AsciiDoc:BLOCKIDEND (']]')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testAnchorid() {
    doTest("[#BLOCKID]Text",
      "AsciiDoc:BLOCKIDSTART ('[#')\n" +
        "AsciiDoc:BLOCKID ('BLOCKID')\n" +
        "AsciiDoc:BLOCKIDEND (']')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testAnchoridWithStyle() {
    doTest("[#BLOCKID]Text",
      "AsciiDoc:BLOCKIDSTART ('[#')\n" +
        "AsciiDoc:BLOCKID ('BLOCKID')\n" +
        "AsciiDoc:BLOCKIDEND (']')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testAnchorWithRefText() {
    doTest("[#BLOCKID,name]Text",
      "AsciiDoc:BLOCKIDSTART ('[#')\n" +
        "AsciiDoc:BLOCKID ('BLOCKID')\n" +
        "AsciiDoc:SEPARATOR (',')\n" +
        "AsciiDoc:BLOCKREFTEXT ('name')\n" +
        "AsciiDoc:BLOCKIDEND (']')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testAnchorWithClass() {
    doTest("[#BLOCKID.class]Text",
      "AsciiDoc:BLOCKIDSTART ('[#')\n" +
        "AsciiDoc:BLOCKID ('BLOCKID')\n" +
        "AsciiDoc:SEPARATOR ('.')\n" +
        "AsciiDoc:BLOCKREFTEXT ('class')\n" +
        "AsciiDoc:BLOCKIDEND (']')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testEscapedBold() {
    doTest("Text \\*nonbold* Text",
      "AsciiDoc:TEXT ('Text')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('\\*nonbold*')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testTypographicDoubleQuotes() {
    doTest("\"`typoquote`\"",
      "AsciiDoc:TYPOGRAPHIC_DOUBLE_QUOTE_START ('\"`')\n" +
        "AsciiDoc:TEXT ('typoquote')\n" +
        "AsciiDoc:TYPOGRAPHIC_DOUBLE_QUOTE_END ('`\"')");
  }

  public void testTypographicSingleQuotes() {
    doTest("'`typoquote`'",
      "AsciiDoc:TYPOGRAPHIC_SINGLE_QUOTE_START (''`')\n" +
        "AsciiDoc:TEXT ('typoquote')\n" +
        "AsciiDoc:TYPOGRAPHIC_SINGLE_QUOTE_END ('`'')");
  }

  public void testMultipleDoubleTypographicQuotes() {
    doTest("\"`test?`\" \"`test?`\"",
      "AsciiDoc:TYPOGRAPHIC_DOUBLE_QUOTE_START ('\"`')\n" +
        "AsciiDoc:TEXT ('test?')\n" +
        "AsciiDoc:TYPOGRAPHIC_DOUBLE_QUOTE_END ('`\"')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TYPOGRAPHIC_DOUBLE_QUOTE_START ('\"`')\n" +
        "AsciiDoc:TEXT ('test?')\n" +
        "AsciiDoc:TYPOGRAPHIC_DOUBLE_QUOTE_END ('`\"')");
  }

  public void testMultiplSingleTypographicQuotes() {
    doTest("'`test?`' '`test?`'",
      "AsciiDoc:TYPOGRAPHIC_SINGLE_QUOTE_START (''`')\n" +
        "AsciiDoc:TEXT ('test?')\n" +
        "AsciiDoc:TYPOGRAPHIC_SINGLE_QUOTE_END ('`'')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TYPOGRAPHIC_SINGLE_QUOTE_START (''`')\n" +
        "AsciiDoc:TEXT ('test?')\n" +
        "AsciiDoc:TYPOGRAPHIC_SINGLE_QUOTE_END ('`'')");
  }

  public void testMonospaceWithQuotes() {
    doTest("`\"initial value\"`",
      "AsciiDoc:MONO_START ('`')\n" +
        "AsciiDoc:DOUBLE_QUOTE ('\"')\n" +
        "AsciiDoc:MONO ('initial')\n" +
        "AsciiDoc:WHITE_SPACE_MONO (' ')\n" +
        "AsciiDoc:MONO ('value')\n" +
        "AsciiDoc:DOUBLE_QUOTE ('\"')\n" +
        "AsciiDoc:MONO_END ('`')");
  }

  public void testNoTypographicQuotes() {
    doTest("\"` test `\"",
      "AsciiDoc:DOUBLE_QUOTE ('\"')\n" +
        "AsciiDoc:TEXT ('`')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('test')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('`')\n" +
        "AsciiDoc:DOUBLE_QUOTE ('\"')");
  }

  public void testTwoTypographicQuotesThatMightBeConsideredAMonospace() {
    doTest("\"`Test?`\", and \"`What?`\"",
      "AsciiDoc:TYPOGRAPHIC_DOUBLE_QUOTE_START ('\"`')\n" +
        "AsciiDoc:TEXT ('Test?')\n" +
        "AsciiDoc:TYPOGRAPHIC_DOUBLE_QUOTE_END ('`\"')\n" +
        "AsciiDoc:TEXT (',')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('and')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TYPOGRAPHIC_DOUBLE_QUOTE_START ('\"`')\n" +
        "AsciiDoc:TEXT ('What?')\n" +
        "AsciiDoc:TYPOGRAPHIC_DOUBLE_QUOTE_END ('`\"')");
  }

  public void testNoTypographicQuotesNonMatching() {
    doTest("\"`test",
      "AsciiDoc:DOUBLE_QUOTE ('\"')\n" +
        "AsciiDoc:TEXT ('`test')");
  }

  public void testPassThroughWithSomethingLookingLikeAnAttribute() {
    doTest("pass:[{attr}]",
      "AsciiDoc:INLINE_MACRO_ID ('pass:')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:MACROTEXT ('{attr}')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')");
  }

  public void testPassWithEscapedContent() {
    doTest("pass:[\\]]",
      "AsciiDoc:INLINE_MACRO_ID ('pass:')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:MACROTEXT ('\\]')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')");
  }

  public void testStemWithSomethingLookingLikeAnAttribute() {
    doTest("stem:[{attr}]",
      "AsciiDoc:INLINE_MACRO_ID ('stem:')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:MACROTEXT ('{attr}')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')");
  }


  public void testPassThroughInlineThreePlus() {
    doTest("+++pt\npt2+++",
      "AsciiDoc:PASSTRHOUGH_INLINE_START ('+++')\n" +
        "AsciiDoc:PASSTRHOUGH_CONTENT ('pt\\npt2')\n" +
        "AsciiDoc:PASSTRHOUGH_INLINE_END ('+++')");
  }

  public void testPassThroughInlineThreePlusWithPlusInside() {
    doTest("+++ ++ +++",
      "AsciiDoc:PASSTRHOUGH_INLINE_START ('+++')\n" +
        "AsciiDoc:PASSTRHOUGH_CONTENT (' ++ ')\n" +
        "AsciiDoc:PASSTRHOUGH_INLINE_END ('+++')");
  }

  public void testSuperscriptSimple() {
    doTest("^super^",
      "AsciiDoc:SUPERSCRIPT_START ('^')\n" +
        "AsciiDoc:TEXT ('super')\n" +
        "AsciiDoc:SUPERSCRIPT_END ('^')");
  }

  public void testSuperscriptInside() {
    doTest("outside^inside^outside",
      "AsciiDoc:TEXT ('outside')\n" +
        "AsciiDoc:SUPERSCRIPT_START ('^')\n" +
        "AsciiDoc:TEXT ('inside')\n" +
        "AsciiDoc:SUPERSCRIPT_END ('^')\n" +
        "AsciiDoc:TEXT ('outside')");
  }

  public void testNotSuperscriptStart() {
    doTest("^ inside^",
      "AsciiDoc:TEXT ('^')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('inside^')");
  }

  public void testNotSuperscriptEnd() {
    doTest("^inside ^",
      "AsciiDoc:TEXT ('^inside')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('^')");
  }

  public void testSubscriptSimple() {
    doTest("~sub~",
      "AsciiDoc:SUBSCRIPT_START ('~')\n" +
        "AsciiDoc:TEXT ('sub')\n" +
        "AsciiDoc:SUBSCRIPT_END ('~')");
  }

  public void testSubscriptInside() {
    doTest("outside~inside~outside",
      "AsciiDoc:TEXT ('outside')\n" +
        "AsciiDoc:SUBSCRIPT_START ('~')\n" +
        "AsciiDoc:TEXT ('inside')\n" +
        "AsciiDoc:SUBSCRIPT_END ('~')\n" +
        "AsciiDoc:TEXT ('outside')");
  }

  public void testNotSubscriptStart() {
    doTest("~ inside~",
      "AsciiDoc:TEXT ('~')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('inside~')");
  }

  public void testNotSubscriptEnd() {
    doTest("~inside ~",
      "AsciiDoc:TEXT ('~inside')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('~')");
  }

  public void testPassThroughInlineDollars() {
    doTest("$$pt\npt2$$",
      "AsciiDoc:PASSTRHOUGH_INLINE_START ('$$')\n" +
        "AsciiDoc:PASSTRHOUGH_CONTENT ('pt\\npt2')\n" +
        "AsciiDoc:PASSTRHOUGH_INLINE_END ('$$')");
  }
  public void testPassThroughInlineOnePlus() {
    doTest("+pt\np+t2+",
      "AsciiDoc:PASSTRHOUGH_INLINE_START ('+')\n" +
        "AsciiDoc:PASSTRHOUGH_CONTENT ('pt\\np+t2')\n" +
        "AsciiDoc:PASSTRHOUGH_INLINE_END ('+')");
  }

  public void testPassThroughInlineTwoPlus() {
    doTest("++pt\npt2++",
      "AsciiDoc:PASSTRHOUGH_INLINE_START ('++')\n" +
        "AsciiDoc:PASSTRHOUGH_CONTENT ('pt\\npt2')\n" +
        "AsciiDoc:PASSTRHOUGH_INLINE_END ('++')");
  }

  public void testPassThroughInlineTwoPlusEscaped() {
    doTest("\\++npt++",
      "AsciiDoc:TEXT ('\\+')\n" +
        "AsciiDoc:PASSTRHOUGH_INLINE_START ('+')\n" +
        "AsciiDoc:PASSTRHOUGH_CONTENT ('npt')\n" +
        "AsciiDoc:PASSTRHOUGH_INLINE_END ('+')\n" +
        "AsciiDoc:TEXT ('+')");
  }

  public void testPassThroughDoublePlusAndSingle() {
    doTest("++text++and +some+ other",
      "AsciiDoc:PASSTRHOUGH_INLINE_START ('++')\n" +
        "AsciiDoc:PASSTRHOUGH_CONTENT ('text')\n" +
        "AsciiDoc:PASSTRHOUGH_INLINE_END ('++')\n" +
        "AsciiDoc:TEXT ('and')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:PASSTRHOUGH_INLINE_START ('+')\n" +
        "AsciiDoc:PASSTRHOUGH_CONTENT ('some')\n" +
        "AsciiDoc:PASSTRHOUGH_INLINE_END ('+')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('other')");
  }

  public void testLiteralBlock() {
    doTest("....\nliteral\n....\n",
      "AsciiDoc:LITERAL_BLOCK_DELIMITER ('....')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LITERAL_BLOCK ('literal')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LITERAL_BLOCK_DELIMITER ('....')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testQuotedBlock() {
    doTest("____\nQuoted with *bold*\n____\n",
      "AsciiDoc:BLOCK_DELIMITER ('____')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('Quoted')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('with')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:BOLD_START ('*')\n" +
        "AsciiDoc:BOLD ('bold')\n" +
        "AsciiDoc:BOLD_END ('*')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('____')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testNestedList() {
    doTest("* item\n" +
        "** item\n" +
        "\n" +
        "== Section",
      "AsciiDoc:BULLET ('*')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('item')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BULLET ('**')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('item')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:EMPTY_LINE ('\\n')\n" +
        "AsciiDoc:HEADING_TOKEN ('== Section')");
  }

  public void testNestedQuotedBlock() {
    doTest("____\nQuoted\n_____\nDoubleQuote\n_____\n____\n",
      "AsciiDoc:BLOCK_DELIMITER ('____')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('Quoted')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('_____')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('DoubleQuote')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('_____')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('____')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testListingNestedInExample() {
    doTest("====\n----\n----\n====\n",
      "AsciiDoc:BLOCK_DELIMITER ('====')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_BLOCK_DELIMITER ('----')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_BLOCK_DELIMITER ('----')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('====')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testListingWithCallout() {
    doTest("----\n----\n<1> Callout 1\n<.> Callout 2",
      "AsciiDoc:LISTING_BLOCK_DELIMITER ('----')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_BLOCK_DELIMITER ('----')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CALLOUT ('<1>')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Callout')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('1')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CALLOUT ('<.>')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Callout')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('2')");
  }

  public void testNoCallout() {
    doTest("Test\n" +
      "<1> Test", "AsciiDoc:TEXT ('Test')\n" +
      "AsciiDoc:LINE_BREAK ('\\n')\n" +
      "AsciiDoc:LT ('<')\n" +
      "AsciiDoc:TEXT ('1')\n" +
      "AsciiDoc:GT ('>')\n" +
      "AsciiDoc:WHITE_SPACE (' ')\n" +
      "AsciiDoc:TEXT ('Test')");
  }

  public void testNoTitle() {
    doTest("text\n.notitle",
      "AsciiDoc:TEXT ('text')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('.notitle')");
  }

  public void testTitleThatLooksLikeATableCell() {
    doTest(".|notatable",
      "AsciiDoc:TITLE_TOKEN ('.')\n" +
        "AsciiDoc:TEXT ('|notatable')");
  }

  public void testTitleAfterId() {
    doTest("[[id]]\n.Title\n====\nExample\n====",
      "AsciiDoc:BLOCKIDSTART ('[[')\n" +
        "AsciiDoc:BLOCKID ('id')\n" +
        "AsciiDoc:BLOCKIDEND (']]')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TITLE_TOKEN ('.')\n" +
        "AsciiDoc:TEXT ('Title')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('====')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('Example')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('====')");
  }

  public void testTitleSTartingWithADot() {
    doTest("..gitignore\n----\nExample\n----",
      "AsciiDoc:TITLE_TOKEN ('.')\n" +
        "AsciiDoc:TEXT ('.gitignore')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_BLOCK_DELIMITER ('----')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_TEXT ('Example')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_BLOCK_DELIMITER ('----')");
  }

  public void testDoubleColonNotEndOfSentence() {
    doTest("::\n",
      "AsciiDoc:TEXT ('::')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testInitialNotEndOfSentenceMiddleOfLine() {
    doTest("Wolfgang A. Mozart",
      "AsciiDoc:TEXT ('Wolfgang')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('A.')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Mozart')");
  }

  public void testHardBreakWithContinuation() {
    doTest("* Test +\n+\nsecond line",
      "AsciiDoc:BULLET ('*')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Test')\n" +
        "AsciiDoc:HARD_BREAK (' +')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CONTINUATION ('+')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('second')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('line')");
  }

  public void testHardBreakAtBlockEnd() {
    doTest("|===\n" +
        "|XX +\n" +
        "|===\n" +
        "\n" +
        "== Title",
      "AsciiDoc:BLOCK_DELIMITER ('|===')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CELLSEPARATOR ('|')\n" +
        "AsciiDoc:TEXT ('XX')\n" +
        "AsciiDoc:HARD_BREAK (' +')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('|===')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:EMPTY_LINE ('\\n')\n" +
        "AsciiDoc:HEADING_TOKEN ('== Title')");
  }

  public void testInitialEndOfSentenceAtEndOfLineSoThatItKeepsExistingWraps() {
    doTest("Wolfgang A.\nMozart",
      "AsciiDoc:TEXT ('Wolfgang')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('A')\n" +
        "AsciiDoc:END_OF_SENTENCE ('.')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('Mozart')");
  }

  public void testDontWrapIfFollowedByNumberInsideLine() {
    doTest("Ch. 9 important",
      "AsciiDoc:TEXT ('Ch.')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('9')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('important')");
  }

  public void testDontWrapWhenNumberWithDotInsideLine() {
    doTest("CSS3. Text",
      "AsciiDoc:TEXT ('CSS3.')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testKeepWrapWithNumberAndDotAtEndOfLine() {
    doTest("CSS3.\nText",
      "AsciiDoc:TEXT ('CSS3')\n" +
        "AsciiDoc:END_OF_SENTENCE ('.')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testDonWrapIfFollowedByNumberNextLineLine() {
    doTest("Ch.\n9 important",
      "AsciiDoc:TEXT ('Ch')\n" +
        "AsciiDoc:END_OF_SENTENCE ('.')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('9')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('important')");
  }

  public void testInitialEndOfSentenceAtEndOfLineSoThatItKeepsExistingWrapsEvenIfThereIsABlankAtTheEndOfTheLine() {
    doTest("Wolfgang A. \nMozart",
      "AsciiDoc:TEXT ('Wolfgang')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('A')\n" +
        "AsciiDoc:END_OF_SENTENCE ('.')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('Mozart')");
  }

  public void testExampleWithBlankLine() {
    doTest("====\nTest\n\n====\n",
      "AsciiDoc:BLOCK_DELIMITER ('====')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('Test')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:EMPTY_LINE ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('====')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testExampleWithListing() {
    doTest("====\n.Title\n[source]\n----\nSource\n----\n====\n",
      "AsciiDoc:BLOCK_DELIMITER ('====')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TITLE_TOKEN ('.')\n" +
        "AsciiDoc:TEXT ('Title')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('source')\n" +
        "AsciiDoc:ATTRS_END (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_BLOCK_DELIMITER ('----')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_TEXT ('Source')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_BLOCK_DELIMITER ('----')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('====')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testBlockEndingOverOldStyleHeader() {
    doTest("--\nS\n--\n",
      "AsciiDoc:BLOCK_DELIMITER ('--')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('S')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('--')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testContinuationWithBlockAndEnumeration() {
    doTest("* Test\n" +
        "+\n" +
        "--\n" +
        "* Test\n" +
        "--\n" +
        "\n" +
        "== Section\n",
      "AsciiDoc:BULLET ('*')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Test')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CONTINUATION ('+')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('--')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BULLET ('*')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Test')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('--')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:EMPTY_LINE ('\\n')\n" +
        "AsciiDoc:HEADING_TOKEN ('== Section')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testHorizontalRule() {
    doTest("'''\n",
      "AsciiDoc:HORIZONTALRULE (''''')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testMarkdownHorizontalRuleDash() {
    //noinspection AsciiDocHorizontalRuleInspection
    doTest("---\n",
      "AsciiDoc:HORIZONTALRULE ('---')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testMarkdownHorizontalRuleStar() {
    //noinspection AsciiDocHorizontalRuleInspection
    doTest("***\n",
      "AsciiDoc:HORIZONTALRULE ('***')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testMarkdownHorizontalRuleUnderscore() {
    //noinspection AsciiDocHorizontalRuleInspection
    doTest("___\n",
      "AsciiDoc:HORIZONTALRULE ('___')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testPagebreak() {
    doTest("<<<\n",
      "AsciiDoc:PAGEBREAK ('<<<')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testPagebreakWithAutocomplete() {
    doTest("<<<IntellijIdeaRulezzz\n",
            "AsciiDoc:PAGEBREAK ('<<<IntellijIdeaRulezzz')\n" +
                    "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testEscapedBlockId() {
    doTest("\\[[id]]",
      "AsciiDoc:TEXT ('\\')\n" +
        "AsciiDoc:LBRACKET ('[')\n" +
        "AsciiDoc:LBRACKET ('[')\n" +
        "AsciiDoc:TEXT ('id')\n" +
        "AsciiDoc:RBRACKET (']')\n" +
        "AsciiDoc:RBRACKET (']')");
  }

  public void testEndOfSentence() {
    doTest("End. Of Sentence",
      "AsciiDoc:TEXT ('End')\n" +
        "AsciiDoc:END_OF_SENTENCE ('.')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Of')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Sentence')");
  }

  public void testEndOfSentenceWithUmlaut() {
    doTest("End. Öf Sentence",
      "AsciiDoc:TEXT ('End')\n" +
        "AsciiDoc:END_OF_SENTENCE ('.')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Öf')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Sentence')");
  }

  public void testNoEndOfSentence() {
    doTest("End.No Sentence",
      "AsciiDoc:TEXT ('End.No')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Sentence')");
  }

  public void testNoEndOfSentenceAfterNumber() {
    doTest("After 1. Number",
      "AsciiDoc:TEXT ('After')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('1.')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Number')");
  }

  public void testNoEndOfSentenceAfterColon() {
    doTest("Colon: Word",
      "AsciiDoc:TEXT ('Colon:')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Word')");
  }

  public void testEndOfSentenceAfterColonAndNewline() {
    doTest("Colon:\nWord",
      "AsciiDoc:TEXT ('Colon')\n" +
        "AsciiDoc:END_OF_SENTENCE (':')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('Word')");
  }

  public void testNoEndOfSentenceAgain() {
    doTest("End. no Sentence",
      "AsciiDoc:TEXT ('End.')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('no')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Sentence')");
  }

  public void testNoEndOfSentenceAdExemplar() {
    doTest("e.g. No Sentence",
      "AsciiDoc:TEXT ('e.g.')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('No')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Sentence')");
  }

  public void testDescription() {
    doTest("a property:: description",
      "AsciiDoc:DESCRIPTION ('a')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:DESCRIPTION ('property')\n" +
        "AsciiDoc:DESCRIPTION_END ('::')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('description')");
  }

  public void testCommentForDescription() {
    doTest("// a property:: description",
      "AsciiDoc:LINE_COMMENT ('// a property:: description')");
  }

  public void testDescriptionLong() {
    doTest("A:: B\nC::: D",
      "AsciiDoc:DESCRIPTION ('A')\n" +
        "AsciiDoc:DESCRIPTION_END ('::')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('B')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:DESCRIPTION ('C')\n" +
        "AsciiDoc:DESCRIPTION_END (':::')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('D')");
  }

  public void testDescriptionWithContinuationAndListing() {
    doTest("X:: Y\n" +
        "+\n" +
        "----\n" +
        "----\n" +
        "== Hi",
      "AsciiDoc:DESCRIPTION ('X')\n" +
        "AsciiDoc:DESCRIPTION_END ('::')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Y')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CONTINUATION ('+')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_BLOCK_DELIMITER ('----')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_BLOCK_DELIMITER ('----')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('==')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Hi')");
  }

  public void testDescriptionWithContinuationAndExample() {
    doTest("X:: Y\n" +
        "+\n" +
        "====\n" +
        "====\n" +
        "== Hi",
      "AsciiDoc:DESCRIPTION ('X')\n" +
        "AsciiDoc:DESCRIPTION_END ('::')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Y')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CONTINUATION ('+')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('====')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('====')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('==')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Hi')");
  }

  public void testHeadingAfterListing() {
    doTest("----\n" +
        "----\n" +
        "== Hi",
      "AsciiDoc:LISTING_BLOCK_DELIMITER ('----')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_BLOCK_DELIMITER ('----')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:HEADING_TOKEN ('== Hi')");
  }

  public void testDescriptionWithMultipleColons() {
    doTest("a property::ext:: description",
      "AsciiDoc:DESCRIPTION ('a')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:DESCRIPTION ('property::ext')\n" +
        "AsciiDoc:DESCRIPTION_END ('::')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('description')");
  }

  public void testDescriptionWithFormatting() {
    doTest("`property`:: description",
      "AsciiDoc:MONO_START ('`')\n" +
        "AsciiDoc:DESCRIPTION ('property')\n" +
        "AsciiDoc:MONO_END ('`')\n" +
        "AsciiDoc:DESCRIPTION_END ('::')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('description')");
  }

  public void testDescriptionWithLink() {
    doTest("link:http://www.example.com[Example]:: description",
      "AsciiDoc:LINKSTART ('link:')\n" +
        "AsciiDoc:URL_LINK ('http://www.example.com')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:MACROTEXT ('Example')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')\n" +
        "AsciiDoc:DESCRIPTION_END ('::')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('description')");
  }

  public void testDescriptionWithAttribute() {
    doTest("{attr}:: description",
      "AsciiDoc:ATTRIBUTE_REF_START ('{')\n" +
        "AsciiDoc:ATTRIBUTE_REF ('attr')\n" +
        "AsciiDoc:ATTRIBUTE_REF_END ('}')\n" +
        "AsciiDoc:DESCRIPTION_END ('::')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('description')");
  }

  public void testIndentedListing() {
    doTest("   Listing\nMore\n\nText",
      "AsciiDoc:LISTING_TEXT ('   Listing')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_TEXT ('More')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:EMPTY_LINE ('\\n')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testListingWithNoDelimiters() {
    doTest("[source]\nListing\n\nText",
      "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('source')\n" +
        "AsciiDoc:ATTRS_END (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_TEXT ('Listing')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:EMPTY_LINE ('\\n')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testListingNoStyle() {
    doTest("  Listing\n+\nText",
      "AsciiDoc:LISTING_TEXT ('  Listing')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:CONTINUATION ('+')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testListingWithLanguage() {
    doTest("[source,php]\nListing\n\nText",
      "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('source')\n" +
        "AsciiDoc:SEPARATOR (',')\n" +
        "AsciiDoc:ATTR_NAME ('php')\n" +
        "AsciiDoc:ATTRS_END (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_TEXT ('Listing')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:EMPTY_LINE ('\\n')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testListingWithRef() {
    doTest("[source#ref]\nListing\n\nText",
      "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('source')\n" +
        "AsciiDoc:SEPARATOR ('#')\n" +
        "AsciiDoc:BLOCKID ('ref')\n" +
        "AsciiDoc:ATTRS_END (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_TEXT ('Listing')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:EMPTY_LINE ('\\n')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testListingWithOpenBlock() {
    doTest("[source]\n--\nListing\n--\nText",
      "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('source')\n" +
        "AsciiDoc:ATTRS_END (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_BLOCK_DELIMITER ('--')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_TEXT ('Listing')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_BLOCK_DELIMITER ('--')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testPassthroughWithNoDelimiters() {
    doTest("[pass]\nPas**ss**ss\n\nText",
      "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('pass')\n" +
        "AsciiDoc:ATTRS_END (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:PASSTRHOUGH_CONTENT ('Pas**ss**ss')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:EMPTY_LINE ('\\n')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testListingWithAttributeAndDelimiter() {
    doTest("[source]\n----\nListing\n----\nText",
      "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('source')\n" +
        "AsciiDoc:ATTRS_END (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_BLOCK_DELIMITER ('----')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_TEXT ('Listing')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_BLOCK_DELIMITER ('----')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testVerseWithCommentNoDelimiters() {
    doTest("[verse]\n" +
        "// test\n" +
        " Verse\n",
      "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('verse')\n" +
        "AsciiDoc:ATTRS_END (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LINE_COMMENT ('// test')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Verse')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testBlockWithTitleInsideExample() {
    doTest("====\n" +
        "Text\n" +
        "\n" +
        ".Title\n" +
        "----\n" +
        "Hi\n" +
        "----\n" +
        "====",
      "AsciiDoc:BLOCK_DELIMITER ('====')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('Text')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:EMPTY_LINE ('\\n')\n" +
        "AsciiDoc:TITLE_TOKEN ('.')\n" +
        "AsciiDoc:TEXT ('Title')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_BLOCK_DELIMITER ('----')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_TEXT ('Hi')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_BLOCK_DELIMITER ('----')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('====')");
  }

  public void testVerseWithSomethingLookingLikeBlock() {
    doTest("[verse]\n" +
        "V1\n" +
        "----\n" +
        "V2\n" +
        "\n" +
        "[source]\n" +
        "Hi",
      "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('verse')\n" +
        "AsciiDoc:ATTRS_END (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('V1')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('----')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('V2')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:EMPTY_LINE ('\\n')\n" +
        "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('source')\n" +
        "AsciiDoc:ATTRS_END (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_TEXT ('Hi')");
  }

  public void testEnumeration() {
    doTest(". Item",
      "AsciiDoc:ENUMERATION ('.')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Item')");
  }

  public void testEnumerationWithBlank() {
    doTest("* item 1\n" +
      " * item 2\n", "AsciiDoc:BULLET ('*')\n" +
      "AsciiDoc:WHITE_SPACE (' ')\n" +
      "AsciiDoc:TEXT ('item')\n" +
      "AsciiDoc:WHITE_SPACE (' ')\n" +
      "AsciiDoc:TEXT ('1')\n" +
      "AsciiDoc:LINE_BREAK ('\\n')\n" +
      "AsciiDoc:WHITE_SPACE (' ')\n" +
      "AsciiDoc:BULLET ('*')\n" +
      "AsciiDoc:WHITE_SPACE (' ')\n" +
      "AsciiDoc:TEXT ('item')\n" +
      "AsciiDoc:WHITE_SPACE (' ')\n" +
      "AsciiDoc:TEXT ('2')\n" +
      "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testNoEnumeration() {
    doTest("Das\n" +
      "* Test", "AsciiDoc:TEXT ('Das')\n" +
      "AsciiDoc:LINE_BREAK ('\\n')\n" +
      "AsciiDoc:TEXT ('*')\n" +
      "AsciiDoc:WHITE_SPACE (' ')\n" +
      "AsciiDoc:TEXT ('Test')");
  }

  public void testItemWithListing() {
    doTest("* item\n" +
        "----\n" +
        "----",
      "AsciiDoc:BULLET ('*')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('item')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_BLOCK_DELIMITER ('----')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_BLOCK_DELIMITER ('----')");
  }

  public void testEnumerationSecondLevel() {
    doTest(".. Item",
      "AsciiDoc:ENUMERATION ('..')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Item')");
  }

  public void testEnumerationNumber() {
    doTest("1. Item",
      "AsciiDoc:ENUMERATION ('1.')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Item')");
  }

  public void testEnumerationCharacter() {
    doTest("a. Item",
      "AsciiDoc:ENUMERATION ('a.')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Item')");
  }

  public void testEndingBlockWithNoDelimiterInsideBlockWithDelimiter() {
    doTest("====\n" +
        "[verse]\n" +
        "test\n" +
        "----\n" +
        "====\n",
      "AsciiDoc:BLOCK_DELIMITER ('====')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('verse')\n" +
        "AsciiDoc:ATTRS_END (']')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('test')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('----')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('====')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testInlineMacro() {
    doTest("Text image:image.png[] text",
      "AsciiDoc:TEXT ('Text')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:INLINE_MACRO_ID ('image:')\n" +
        "AsciiDoc:INLINE_MACRO_BODY ('image.png')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('text')");
  }

  public void testInlineMacroWithBlank() {
    doTest("pass:[ blank ]",
      "AsciiDoc:INLINE_MACRO_ID ('pass:')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:MACROTEXT ('blank')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')");
  }

  public void testInlineMacroWithAttributeInBody() {
    doTest("image:{url}/image.png[]",
      "AsciiDoc:INLINE_MACRO_ID ('image:')\n" +
        "AsciiDoc:ATTRIBUTE_REF_START ('{')\n" +
        "AsciiDoc:ATTRIBUTE_REF ('url')\n" +
        "AsciiDoc:ATTRIBUTE_REF_END ('}')\n" +
        "AsciiDoc:INLINE_MACRO_BODY ('/image.png')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')");
  }

  public void testInlineMacroWithPassthrough() {
    doTest("kbd:[+]+]",
      "AsciiDoc:INLINE_MACRO_ID ('kbd:')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:PASSTRHOUGH_INLINE_START ('+')\n" +
        "AsciiDoc:PASSTRHOUGH_CONTENT (']')\n" +
        "AsciiDoc:PASSTRHOUGH_INLINE_END ('+')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')");
  }

  public void testInlineMacroWithPassthroughAndSpaces() {
    doTest("kbd:[X + ++ + ++]",
      "AsciiDoc:INLINE_MACRO_ID ('kbd:')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:MACROTEXT ('X')\n" +
        "AsciiDoc:SEPARATOR (' + ')\n" +
        "AsciiDoc:PASSTRHOUGH_INLINE_START ('++')\n" +
        "AsciiDoc:PASSTRHOUGH_CONTENT (' + ')\n" +
        "AsciiDoc:PASSTRHOUGH_INLINE_END ('++')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')");
  }

  public void testInlineMacroWithNoPassthrough() {
    doTest("kbd:[⌃+X+X]+",
      "AsciiDoc:INLINE_MACRO_ID ('kbd:')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:MACROTEXT ('⌃')\n" +
        "AsciiDoc:SEPARATOR ('+')\n" +
        "AsciiDoc:MACROTEXT ('X')\n" +
        "AsciiDoc:SEPARATOR ('+')\n" +
        "AsciiDoc:MACROTEXT ('X')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')\n" +
        "AsciiDoc:TEXT ('+')");
  }

  public void testInlineMacroWithTextAndPassthrough() {
    doTest("kbd:[text++]++]",
      "AsciiDoc:INLINE_MACRO_ID ('kbd:')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:MACROTEXT ('text')\n" +
        "AsciiDoc:PASSTRHOUGH_INLINE_START ('++')\n" +
        "AsciiDoc:PASSTRHOUGH_CONTENT (']')\n" +
        "AsciiDoc:PASSTRHOUGH_INLINE_END ('++')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')");
  }

  public void testInlineMacroMultiLine() {
    doTest("image:image.png[Text\nText]",
      "AsciiDoc:INLINE_MACRO_ID ('image:')\n" +
        "AsciiDoc:INLINE_MACRO_BODY ('image.png')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('Text')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:ATTR_NAME ('Text')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')");
  }

  public void testInlineMacroWithAttribute() {
    doTest("Text image:image.png[link=http://www.gmx.net] text",
      "AsciiDoc:TEXT ('Text')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:INLINE_MACRO_ID ('image:')\n" +
        "AsciiDoc:INLINE_MACRO_BODY ('image.png')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('link')\n" +
        "AsciiDoc:ASSIGNMENT ('=')\n" +
        "AsciiDoc:URL_LINK ('http://www.gmx.net')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('text')");
  }

  public void testInlineMacroWithAttributeRef() {
    doTest("Text image:image.png[link={url}] text",
      "AsciiDoc:TEXT ('Text')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:INLINE_MACRO_ID ('image:')\n" +
        "AsciiDoc:INLINE_MACRO_BODY ('image.png')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('link')\n" +
        "AsciiDoc:ASSIGNMENT ('=')\n" +
        "AsciiDoc:ATTRIBUTE_REF_START ('{')\n" +
        "AsciiDoc:ATTRIBUTE_REF ('url')\n" +
        "AsciiDoc:ATTRIBUTE_REF_END ('}')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('text')");
  }

  public void testInlineMacroWithBracketsInside() {
    doTest("Text footnote:[some macro:text[About]] text",
      "AsciiDoc:TEXT ('Text')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:INLINE_MACRO_ID ('footnote:')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:MACROTEXT ('some')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:MACROTEXT ('macro:text[About]')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('text')");
  }

  public void testFootnoteAdjacentToText() {
    doTest("prefootnote:[xx] xx",
      "AsciiDoc:TEXT ('pre')\n" +
        "AsciiDoc:INLINE_MACRO_ID ('footnote:')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:MACROTEXT ('xx')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('xx')");
  }

  public void testFootnoteAfterEscape() {
    doTest("\\escapedfootnote:[xx] xx",
      "AsciiDoc:TEXT ('\\escaped')\n" +
        "AsciiDoc:INLINE_MACRO_ID ('footnote:')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:MACROTEXT ('xx')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('xx')");
  }

  public void testInlineMacroThatIsIncompleteAndHasAnInlineMacroOnTheSameLine() {
    doTest("weight:120kg label:procedure[]",
      "AsciiDoc:TEXT ('weight:120kg')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:INLINE_MACRO_ID ('label:')\n" +
        "AsciiDoc:INLINE_MACRO_BODY ('procedure')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')");
  }

  public void testInlineMacroThatSupportsBlanks() {
    doTest("xref:This has Blanks[]",
      "AsciiDoc:LINKSTART ('xref:')\n" +
        "AsciiDoc:LINKFILE ('This has Blanks')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')");
  }

  public void testInlineMacroThatHasAttributeWhenThereIsAnEqualSign() {
    doTest("xref:file.adoc[attr1=a,attr2=b]",
      "AsciiDoc:LINKSTART ('xref:')\n" +
        "AsciiDoc:LINKFILE ('file.adoc')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('attr1')\n" +
        "AsciiDoc:ASSIGNMENT ('=')\n" +
        "AsciiDoc:ATTR_VALUE ('a')\n" +
        "AsciiDoc:SEPARATOR (',')\n" +
        "AsciiDoc:ATTR_NAME ('attr2')\n" +
        "AsciiDoc:ASSIGNMENT ('=')\n" +
        "AsciiDoc:ATTR_VALUE ('b')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')");
  }

  public void testInlineMacroThatDoesntSupportBlanks() {
    doTest("label:This is a label[]",
      "AsciiDoc:TEXT ('label:This')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('is')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('a')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('label')\n" +
        "AsciiDoc:LBRACKET ('[')\n" +
        "AsciiDoc:RBRACKET (']')");
  }

  public void testInlineMacroThatSupportFileNames() {
    doTest("image:file with blank.png[]",
      "AsciiDoc:INLINE_MACRO_ID ('image:')\n" +
        "AsciiDoc:INLINE_MACRO_BODY ('file with blank.png')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')");
  }

  public void testBlockMacroWithBracketsInside() {
    doTest("macro::text[other:[hi]]",
      "AsciiDoc:BLOCK_MACRO_ID ('macro::')\n" +
        "AsciiDoc:BLOCK_MACRO_BODY ('text')\n" +
        "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('other:[hi]')\n" +
        "AsciiDoc:ATTRS_END (']')");
  }

  public void testBlockMacroAutocompleteWithOtherMacroInSameLine() {
    doTest("xref:" + CompletionUtilCore.DUMMY_IDENTIFIER + " and other xref:complete[] normal text",
      "AsciiDoc:LINKSTART ('xref:')\n" +
        "AsciiDoc:LINKFILE ('IntellijIdeaRulezzz ')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('and')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('other')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:LINKSTART ('xref:')\n" +
        "AsciiDoc:LINKFILE ('complete')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('normal')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('text')");
  }

  public void testBlockMacroAutocompleteAtEndOfLine() {
    doTest("xref:" + CompletionUtilCore.DUMMY_IDENTIFIER_TRIMMED,
      "AsciiDoc:LINKSTART ('xref:')\n" +
        "AsciiDoc:LINKFILE ('IntellijIdeaRulezzz')");
  }

  public void testBlockMacroAutocompleteCompleteMacroWithOtherMacroInSameLine() {
    doTest("xref:" + CompletionUtilCore.DUMMY_IDENTIFIER + "complete[] and other xref:complete[] normal text",
      "AsciiDoc:LINKSTART ('xref:')\n" +
        "AsciiDoc:LINKFILE ('IntellijIdeaRulezzz complete')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('and')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('other')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:LINKSTART ('xref:')\n" +
        "AsciiDoc:LINKFILE ('complete')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('normal')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('text')");
  }

  public void testBlockMacroAfterList() {
    doTest("* Item\n" +
      "\n" +
      "blockmacro::content[]", "AsciiDoc:BULLET ('*')\n" +
      "AsciiDoc:WHITE_SPACE (' ')\n" +
      "AsciiDoc:TEXT ('Item')\n" +
      "AsciiDoc:LINE_BREAK ('\\n')\n" +
      "AsciiDoc:EMPTY_LINE ('\\n')\n" +
      "AsciiDoc:BLOCK_MACRO_ID ('blockmacro::')\n" +
      "AsciiDoc:BLOCK_MACRO_BODY ('content')\n" +
      "AsciiDoc:ATTRS_START ('[')\n" +
      "AsciiDoc:ATTRS_END (']')");
  }

  public void testExampleWithListingNoDelimiter() {
    doTest("====\n" +
        " Test\n" +
        "====\n",
      "AsciiDoc:BLOCK_DELIMITER ('====')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:LISTING_TEXT (' Test')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:BLOCK_DELIMITER ('====')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')");
  }

  public void testEllipseInsideLIne() {
    doTest("Text... Text",
      "AsciiDoc:TEXT ('Text...')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testResetFormatting() {
    doTest("`Mono Text++`++\n\nText",
      "AsciiDoc:MONO_START ('`')\n" +
        "AsciiDoc:MONO ('Mono')\n" +
        "AsciiDoc:WHITE_SPACE_MONO (' ')\n" +
        "AsciiDoc:MONO ('Text')\n" +
        "AsciiDoc:PASSTRHOUGH_INLINE_START ('++')\n" +
        "AsciiDoc:PASSTRHOUGH_CONTENT ('`')\n" +
        "AsciiDoc:PASSTRHOUGH_INLINE_END ('++')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:EMPTY_LINE ('\\n')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testIfDef() {
    doTest("ifdef::attr[]",
      "AsciiDoc:BLOCK_MACRO_ID ('ifdef::')\n" +
        "AsciiDoc:ATTRIBUTE_REF ('attr')\n" +
        "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTRS_END (']')");
  }

  public void testIfDefWithAttributeInBody() {
    doTest("ifdef::attr[:other: val]",
      "AsciiDoc:BLOCK_MACRO_ID ('ifdef::')\n" +
        "AsciiDoc:ATTRIBUTE_REF ('attr')\n" +
        "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTRIBUTE_NAME_START (':')\n" +
        "AsciiDoc:ATTRIBUTE_NAME ('other')\n" +
        "AsciiDoc:ATTRIBUTE_NAME_END (':')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:ATTRIBUTE_VAL ('val')\n" +
        "AsciiDoc:ATTRS_END (']')");
  }

  public void testLinkAttribute() {
    doTest("image::file.png[link='http://www.gmx.net']",
      "AsciiDoc:BLOCK_MACRO_ID ('image::')\n" +
        "AsciiDoc:BLOCK_MACRO_BODY ('file.png')\n" +
        "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('link')\n" +
        "AsciiDoc:ASSIGNMENT ('=')\n" +
        "AsciiDoc:SINGLE_QUOTE (''')\n" +
        "AsciiDoc:URL_LINK ('http://www.gmx.net')\n" +
        "AsciiDoc:SINGLE_QUOTE (''')\n" +
        "AsciiDoc:ATTRS_END (']')");
  }

  public void testEndifSpecial() {
    doTest("endif::[]",
      "AsciiDoc:BLOCK_MACRO_ID ('endif::')\n" +
        "AsciiDoc:ATTRS_START ('[')\n" +
        "AsciiDoc:ATTRS_END (']')");
  }

  public void testSimpleUrl() {
    doTest("http://www.gmx.net",
      "AsciiDoc:URL_LINK ('http://www.gmx.net')");
  }

  public void testSimpleUrlWithAttributesInBrackets() {
    doTest("http://www.gmx.net[attr=val]",
      "AsciiDoc:URL_LINK ('http://www.gmx.net')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:ATTR_NAME ('attr')\n" +
        "AsciiDoc:ASSIGNMENT ('=')\n" +
        "AsciiDoc:ATTR_VALUE ('val')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')");
  }

  public void testSimpleUrlWithTextInBrackets() {
    doTest("http://www.gmx.net[text]",
      "AsciiDoc:URL_LINK ('http://www.gmx.net')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:MACROTEXT ('text')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')");
  }

  public void testSimpleUrlAtEndOfSentence() {
    doTest("http://www.gmx.net.",
      "AsciiDoc:URL_LINK ('http://www.gmx.net')\n" +
        "AsciiDoc:TEXT ('.')");
  }

  public void testSimpleUrlInParentheses() {
    doTest("(http://www.gmx.net)",
      "AsciiDoc:LPAREN ('(')\n" +
        "AsciiDoc:URL_LINK ('http://www.gmx.net')\n" +
        "AsciiDoc:RPAREN (')')");
  }

  public void testSimpleUrlInParenthesesWithColon() {
    doTest("(http://www.gmx.net):",
      "AsciiDoc:LPAREN ('(')\n" +
        "AsciiDoc:URL_LINK ('http://www.gmx.net')\n" +
        "AsciiDoc:RPAREN (')')\n" +
        "AsciiDoc:TEXT (':')");
  }

  public void testSimpleUrlInParenthesesAndText() {
    doTest("(http://www.gmx.net) Text",
      "AsciiDoc:LPAREN ('(')\n" +
        "AsciiDoc:URL_LINK ('http://www.gmx.net')\n" +
        "AsciiDoc:RPAREN (')')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testSimpleUrlInParenthesesWithColonAndText() {
    doTest("(http://www.gmx.net): Text",
      "AsciiDoc:LPAREN ('(')\n" +
        "AsciiDoc:URL_LINK ('http://www.gmx.net')\n" +
        "AsciiDoc:RPAREN (')')\n" +
        "AsciiDoc:TEXT (':')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testSimpleUrlInParenthesesAtEndOfLine() {
    doTest("(http://www.gmx.net)\nText",
      "AsciiDoc:LPAREN ('(')\n" +
        "AsciiDoc:URL_LINK ('http://www.gmx.net')\n" +
        "AsciiDoc:RPAREN (')')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testSimpleUrlInParenthesesWithColonAtEndOfLine() {
    doTest("(http://www.gmx.net):\nText",
      "AsciiDoc:LPAREN ('(')\n" +
        "AsciiDoc:URL_LINK ('http://www.gmx.net')\n" +
        "AsciiDoc:RPAREN (')')\n" +
        "AsciiDoc:END_OF_SENTENCE (':')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('Text')");
  }

  public void testUrlInBrackets() {
    doTest("<http://www.gmx.net>",
      "AsciiDoc:URL_START ('<')\n" +
        "AsciiDoc:URL_LINK ('http://www.gmx.net')\n" +
        "AsciiDoc:URL_END ('>')");
  }

  public void testUrlInQuotes() {
    doTest("\"`http://www.gmx.net`\"",
      "AsciiDoc:TYPOGRAPHIC_DOUBLE_QUOTE_START ('\"`')\n" +
        "AsciiDoc:URL_LINK ('http://www.gmx.net')\n" +
        "AsciiDoc:TYPOGRAPHIC_DOUBLE_QUOTE_END ('`\"')");
  }

  public void testUrlEscapedInMonoSpace() {
    doTest("`\\http://www.gmx.net` text",
      "AsciiDoc:MONO_START ('`')\n" +
        "AsciiDoc:MONO ('\\http://www.gmx.net')\n" +
        "AsciiDoc:MONO_END ('`')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('text')");
  }

  public void testUrlInBracketsWithSpace() {
    doTest("<http://www.gmx.net >",
      "AsciiDoc:LT ('<')\n" +
        "AsciiDoc:URL_LINK ('http://www.gmx.net')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:GT ('>')");
  }

  public void testUrlInBracketsWithSquareBracket() {
    doTest("<http://www.gmx.net[Hi]>",
      "AsciiDoc:LT ('<')\n" +
        "AsciiDoc:URL_LINK ('http://www.gmx.net')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:MACROTEXT ('Hi')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')\n" +
        "AsciiDoc:GT ('>')");
  }

  public void testUrlWithLinkPrefix() {
    doTest("link:http://www.gmx.net[Hi]",
      "AsciiDoc:LINKSTART ('link:')\n" +
        "AsciiDoc:URL_LINK ('http://www.gmx.net')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:MACROTEXT ('Hi')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')");
  }

  public void testEmail() {
    doTest("doc.writer@example.com",
      "AsciiDoc:URL_EMAIL ('doc.writer@example.com')");
  }

  public void testEmailWithPrefix() {
    doTest("mailto:doc.writer@example.com[]",
      "AsciiDoc:URL_PREFIX ('mailto:')\n" +
        "AsciiDoc:URL_EMAIL ('doc.writer@example.com')\n" +
        "AsciiDoc:INLINE_ATTRS_START ('[')\n" +
        "AsciiDoc:INLINE_ATTRS_END (']')");
  }

  public void testEmailWithPrefixButNoSquareBrackets() {
    doTest("mailto:doc.writer@example.com",
      "AsciiDoc:TEXT ('mailto:doc.writer@example.com')");
  }

  public void testFrontmatter() {
    doTest("---\nhi: ho\n---",
      "AsciiDoc:FRONTMATTER_DELIMITER ('---')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:FRONTMATTER ('hi: ho')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:FRONTMATTER_DELIMITER ('---')");
  }

  public void testBibliography() {
    doTest("* [[[bib,2]]] Book",
      "AsciiDoc:BULLET ('*')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:BIBSTART ('[[[')\n" +
        "AsciiDoc:BLOCKID ('bib')\n" +
        "AsciiDoc:SEPARATOR (',')\n" +
        "AsciiDoc:BLOCKREFTEXT ('2')\n" +
        "AsciiDoc:BIBEND (']]]')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('Book')");
  }

  public void testBibliographyOnNewLine() {
    doTest("* [[[bib,2]]]\nBook",
      "AsciiDoc:BULLET ('*')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:BIBSTART ('[[[')\n" +
        "AsciiDoc:BLOCKID ('bib')\n" +
        "AsciiDoc:SEPARATOR (',')\n" +
        "AsciiDoc:BLOCKREFTEXT ('2')\n" +
        "AsciiDoc:BIBEND (']]]')\n" +
        "AsciiDoc:LINE_BREAK ('\\n')\n" +
        "AsciiDoc:TEXT ('Book')");
  }

  public void testInlineId() {
    doTest("The [[id]] word",
      "AsciiDoc:TEXT ('The')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:INLINEIDSTART ('[[')\n" +
        "AsciiDoc:BLOCKID ('id')\n" +
        "AsciiDoc:INLINEIDEND (']]')\n" +
        "AsciiDoc:WHITE_SPACE (' ')\n" +
        "AsciiDoc:TEXT ('word')");
  }

  @Override
  protected void doTest(@Language("asciidoc") @NonNls String text, @Nullable String expected) {
    super.doTest(text, expected);
  }

  @Override
  protected Lexer createLexer() {
    return new AsciiDocLexer();
  }

  @Override
  protected String getDirPath() {
    return null;
  }
}
