Skip to content

Initial support for fenced code blocks in Markdown Javadoc. #1368

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
copybara-service merged 1 commit into from
Apr 7, 2026
Merged

Initial support for fenced code blocks in Markdown Javadoc. #1368

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions core/src/main/java/com/google/googlejavaformat/java/javadoc/JavadocFormatter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import com.google.googlejavaformat.java.javadoc.Token.ListItemOpenTag;
import com.google.googlejavaformat.java.javadoc.Token.ListOpenTag;
import com.google.googlejavaformat.java.javadoc.Token.Literal;
import com.google.googlejavaformat.java.javadoc.Token.MarkdownFencedCodeBlock;
import com.google.googlejavaformat.java.javadoc.Token.MoeBeginStripComment;
import com.google.googlejavaformat.java.javadoc.Token.MoeEndStripComment;
import com.google.googlejavaformat.java.javadoc.Token.OptionalLineBreak;
Expand Down Expand Up @@ -134,6 +135,7 @@ private static String render(List<Token> input, int blockIndent, boolean classic
case ParagraphCloseTag unused -> {}
case ListItemCloseTag unused -> {}
case OptionalLineBreak unused -> {}
case MarkdownFencedCodeBlock t -> output.writeMarkdownFencedCodeBlock(t);
}
}
throw new AssertionError();
Expand Down
25 changes: 20 additions & 5 deletions core/src/main/java/com/google/googlejavaformat/java/javadoc/JavadocLexer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,11 +127,26 @@ private ImmutableList<Token> generateTokens() throws LexException {
tokens.add(token);

while (!input.isExhausted()) {
for (Token markdownToken : markdownPositions.tokensAt(input.position())) {
boolean consumed = input.tryConsume(markdownToken.value());
verify(consumed, "Did not consume markdown token: %s", markdownToken);
var unused = input.readAndResetRecorded();
tokens.add(markdownToken);
boolean moreMarkdown;
do {
moreMarkdown = false;
// If there are one or more markdown tokens at the current position, consume their text and
// add them to the token list. If a token has non-empty text, consuming its text changes the
// position, so we need to start looking for markdown tokens at the new position. It is
// assumed that there are no other tokens (markdown or otherwise) in a non-empty text span
// covered by a markdown token.
for (Token markdownToken : markdownPositions.tokensAt(input.position())) {
tokens.add(markdownToken);
if (!markdownToken.value().isEmpty()) {
boolean consumed = input.tryConsume(markdownToken.value());
verify(consumed, "Did not consume markdown token: %s", markdownToken);
var unused = input.readAndResetRecorded();
moreMarkdown = true;
}
}
} while (moreMarkdown);
if (input.isExhausted()) {
break;
}
token = readToken();
tokens.add(token);
Expand Down
33 changes: 30 additions & 3 deletions core/src/main/java/com/google/googlejavaformat/java/javadoc/JavadocWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import static com.google.googlejavaformat.java.javadoc.JavadocWriter.RequestedWhitespace.NONE;
import static com.google.googlejavaformat.java.javadoc.JavadocWriter.RequestedWhitespace.WHITESPACE;

import com.google.googlejavaformat.java.javadoc.Token.BrTag;
import com.google.googlejavaformat.java.javadoc.Token.CodeCloseTag;
import com.google.googlejavaformat.java.javadoc.Token.CodeOpenTag;
import com.google.googlejavaformat.java.javadoc.Token.FooterJavadocTagStart;
Expand All @@ -33,6 +34,7 @@
import com.google.googlejavaformat.java.javadoc.Token.ListItemOpenTag;
import com.google.googlejavaformat.java.javadoc.Token.ListOpenTag;
import com.google.googlejavaformat.java.javadoc.Token.Literal;
import com.google.googlejavaformat.java.javadoc.Token.MarkdownFencedCodeBlock;
import com.google.googlejavaformat.java.javadoc.Token.MoeBeginStripComment;
import com.google.googlejavaformat.java.javadoc.Token.MoeEndStripComment;
import com.google.googlejavaformat.java.javadoc.Token.PreCloseTag;
Expand Down Expand Up @@ -310,7 +312,7 @@ void writeHtmlComment(HtmlComment token) {
requestNewline();
}

void writeBr(Token token) {
void writeBr(BrTag token) {
writeToken(token);

requestNewline();
Expand All @@ -324,6 +326,22 @@ void writeLiteral(Literal token) {
writeToken(token);
}

void writeMarkdownFencedCodeBlock(MarkdownFencedCodeBlock token) {
flushWhitespace();
output.append(token.start());
token
.literal()
.lines()
.forEach(
line -> {
writeNewline();
output.append(line);
});
writeNewline();
output.append(token.end());
requestBlankLine();
}

@Override
public String toString() {
return output.toString();
Expand All @@ -350,12 +368,13 @@ enum RequestedWhitespace {
BLANK_LINE,
}

private void writeToken(Token token) {
private void flushWhitespace() {
if (requestedMoeBeginStripComment != null) {
requestNewline();
}

if (requestedWhitespace == BLANK_LINE
if (classicJavadoc
&& requestedWhitespace == BLANK_LINE
&& (!postWriteModifiedContinuingListStack.isEmpty() || continuingFooterTag)) {
/*
* We don't write blank lines inside lists or footer tags, even in cases where we otherwise
Expand All @@ -374,6 +393,14 @@ private void writeToken(Token token) {
writeNewline();
requestedWhitespace = NONE;
}
}

private void writeToken(Token token) {
if (token.value().isEmpty()) {
return;
}

flushWhitespace();
boolean needWhitespace = (requestedWhitespace == WHITESPACE);

/*
Expand Down
36 changes: 31 additions & 5 deletions core/src/main/java/com/google/googlejavaformat/java/javadoc/MarkdownPositions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,13 @@
import com.google.googlejavaformat.java.javadoc.Token.ListItemCloseTag;
import com.google.googlejavaformat.java.javadoc.Token.ListItemOpenTag;
import com.google.googlejavaformat.java.javadoc.Token.ListOpenTag;
import com.google.googlejavaformat.java.javadoc.Token.MarkdownFencedCodeBlock;
import com.google.googlejavaformat.java.javadoc.Token.ParagraphCloseTag;
import com.google.googlejavaformat.java.javadoc.Token.ParagraphOpenTag;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.commonmark.node.BulletList;
import org.commonmark.node.FencedCodeBlock;
import org.commonmark.node.Heading;
import org.commonmark.node.ListItem;
import org.commonmark.node.Node;
Expand Down Expand Up @@ -102,6 +104,25 @@ void visit(Node node) {
visitNodeList(paragraph.getNext());
}
}
case FencedCodeBlock fencedCodeBlock -> {
// Any indentation before the code block is part of FencedCodeBlock. This makes sense
// because the lines inside the code block must also be indented by that amount. That
// indentation gets subtracted from FencedCodeBlock.getLiteral(), which is the actual text
// represented by the code block.
int start = startPosition(fencedCodeBlock) + fencedCodeBlock.getFenceIndent();
MarkdownFencedCodeBlock token =
new MarkdownFencedCodeBlock(
input.substring(start, endPosition(fencedCodeBlock)),
fencedCodeBlock
.getFenceCharacter()
.repeat(fencedCodeBlock.getOpeningFenceLength())
+ fencedCodeBlock.getInfo(),
fencedCodeBlock
.getFenceCharacter()
.repeat(fencedCodeBlock.getClosingFenceLength()),
fencedCodeBlock.getLiteral());
positionToToken.get(start).addLast(token);
}
// TODO: others
default -> {}
}
Expand Down Expand Up @@ -131,12 +152,17 @@ private void visitNodeList(Node node) {
*/
private void addSpan(Node node, Token startToken, Token endToken) {
// We could write the first part more simply as a `put`, but we do it this way for symmetry.
var first = node.getSourceSpans().getFirst();
int startPosition = first.getInputIndex();
positionToToken.get(startPosition).addLast(startToken);
positionToToken.get(startPosition(node)).addLast(startToken);
positionToToken.get(endPosition(node)).addFirst(endToken);
}

private int startPosition(Node node) {
return node.getSourceSpans().getFirst().getInputIndex();
}

private int endPosition(Node node) {
var last = node.getSourceSpans().getLast();
int endPosition = last.getInputIndex() + last.getLength();
positionToToken.get(endPosition).addFirst(endToken);
return last.getInputIndex() + last.getLength();
}
}

Expand Down
21 changes: 21 additions & 0 deletions core/src/main/java/com/google/googlejavaformat/java/javadoc/Token.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,27 @@ record HtmlComment(String value) implements Token {}

record BrTag(String value) implements Token {}

/**
* A fenced code block, like:
*
* <pre>
* ```java
* code block
* with an info string ("java")
* ```
* </pre>
*
* @param value the full text of the code block as it appeared in the input, including the start
* and end fences and the literal content.
* @param start the start fence, including the info string if any ({@code ```java} in the
* example).
* @param end the end fence.
* @param literal the text that the code block represents. This does not include the start and end
* fences, nor any indentation that precedes these fences and every intervening line.
*/
record MarkdownFencedCodeBlock(String value, String start, String end, String literal)
implements Token {}

/**
* Whitespace that is not in a {@code <pre>} or {@code <table>} section. Whitespace includes
* leading newlines, asterisks, and tabs and spaces. In the output, it is translated to newlines
Expand Down
54 changes: 54 additions & 0 deletions core/src/test/java/com/google/googlejavaformat/java/JavadocFormattingTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1717,6 +1717,60 @@ class Test {}
///
/// A following paragraph.
class Test {}
""";
doFormatTest(input, expected);
}

@Test
public void markdownFencedCodeBlocks() {
assume().that(MARKDOWN_JAVADOC_SUPPORTED).isTrue();
// If fenced code blocks are not supported correctly, the contents of each one will be joined.
// If the input lines survive as separate lines, that means we identified the code block.
String input =
"""
/// ```
/// foo
/// bar
/// ```
///
/// - ```
/// code block
/// in a list
/// ```
///
/// ~~~java
/// code block
/// with tildes and an info string ("java")
/// ~~~
///
/// ````
/// code block
/// with more than three backticks and an extra leading space
/// ````
class Test {}
""";
String expected =
"""
/// ```
/// foo
/// bar
/// ```
///
/// - ```
/// code block
/// in a list
/// ```
///
/// ~~~java
/// code block
/// with tildes and an info string ("java")
/// ~~~
///
/// ````
/// code block
/// with more than three backticks and an extra leading space
/// ````
class Test {}
""";
doFormatTest(input, expected);
}
Expand Down
57 changes: 57 additions & 0 deletions core/src/test/java/com/google/googlejavaformat/java/javadoc/MarkdownPositionsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import com.google.googlejavaformat.java.javadoc.Token.ListItemCloseTag;
import com.google.googlejavaformat.java.javadoc.Token.ListItemOpenTag;
import com.google.googlejavaformat.java.javadoc.Token.ListOpenTag;
import com.google.googlejavaformat.java.javadoc.Token.MarkdownFencedCodeBlock;
import com.google.googlejavaformat.java.javadoc.Token.ParagraphCloseTag;
import com.google.googlejavaformat.java.javadoc.Token.ParagraphOpenTag;
import java.util.Map;
Expand Down Expand Up @@ -100,6 +101,62 @@ public void heading() {
assertThat(map).isEqualTo(expected);
}

@Test
public void codeBlock() {
String text =
"""
- ```
foo
bar
```

~~~java
code
with tildes
~~~

````
indented code
with more than three backticks
````
""";
var positions = MarkdownPositions.parse(text);
ImmutableListMultimap<Integer, Token> map = positionToToken(positions, text);
int bullet = text.indexOf('-');
int firstCodeStart = text.indexOf("```");
int firstCodeEnd = text.indexOf("```", firstCodeStart + 3) + 3;
int secondCodeStart = text.indexOf("~~~", firstCodeEnd);
int secondCodeEnd = text.indexOf("~~~", secondCodeStart + 3) + 3;
int thirdCodeStart = text.indexOf("````", secondCodeEnd);
int thirdCodeEnd = text.indexOf("````", thirdCodeStart + 4) + 4;
ImmutableListMultimap<Integer, Token> expected =
ImmutableListMultimap.<Integer, Token>builder()
.put(bullet, new ListOpenTag(""))
.put(bullet, new ListItemOpenTag("- "))
.put(
firstCodeStart,
new MarkdownFencedCodeBlock(
text.substring(firstCodeStart, firstCodeEnd), "```", "```", "foo\nbar\n"))
.put(firstCodeEnd, new ListItemCloseTag(""))
.put(firstCodeEnd, new ListCloseTag(""))
.put(
secondCodeStart,
new MarkdownFencedCodeBlock(
text.substring(secondCodeStart, secondCodeEnd),
"~~~java",
"~~~",
"code\nwith tildes\n"))
.put(
thirdCodeStart,
new MarkdownFencedCodeBlock(
text.substring(thirdCodeStart, thirdCodeEnd),
"````",
"````",
"indented code\nwith more than three backticks\n"))
.build();
assertThat(map).isEqualTo(expected);
}

private static ImmutableListMultimap<Integer, Token> positionToToken(
MarkdownPositions positions, String input) {
return IntStream.rangeClosed(0, input.length())
Expand Down
Loading