Strip markdown HTML comments within paragraph (#1698)

Standalone HTML comments is converted to `RDoc::Markup::Raw`. However,
HTML comments in a paragraph were not handled correctly.
Since RDoc paragraph can't contain Raw, inline HTML comments should be
stripped. HTML comments in a paragraph `"\n<!-- -->\n"` will generate
double newline `"\n\n"`, so ToHtml also needs to handle this double
newline.

Fix this markdown comment rendering
```ruby
##
# :markup: markdown
# <!-- comment1 -->
# hello
# <!-- comment2 -->
# world
```

Before
```html
<!-- comment1 -->
<p>hello &lt;!– comment2 –&gt; world</p>
```
After
```html
<!-- comment1 -->
<p>hello world</p>
```

Author: tompng

lib/rdoc/markdown.kpeg

@@ -975,6 +975,7 @@ Inline  = Str
         | InlineNote
         | Code
         | RawHtml
+        | StrippedComment
         | Entity
         | EscapedChar
         | Symbol
@@ -1152,8 +1153,8 @@ Code = ( Ticks1 < (
        )
        { code text }
 
-RawHtml = < (HtmlComment | HtmlBlockScript | HtmlTag) >
-          { if html? then text else '' end }
+RawHtml = < HtmlBlockScript | HtmlTag > { html? ? text : '' }
+StrippedComment = < HtmlComment > { '' }
 
 BlankLine =     @Sp @Newline { "\n" }
 

lib/rdoc/markdown.rb

@@ -9787,7 +9787,7 @@ def _Inlines
     return _tmp
   end
 
-  # Inline = (Str | @Endline | UlOrStarLine | @Space | Strong | Emph | Strike | Image | Link | NoteReference | InlineNote | Code | RawHtml | Entity | EscapedChar | Symbol)
+  # Inline = (Str | @Endline | UlOrStarLine | @Space | Strong | Emph | Strike | Image | Link | NoteReference | InlineNote | Code | RawHtml | StrippedComment | Entity | EscapedChar | Symbol)
   def _Inline
 
     _save = self.pos
@@ -9831,6 +9831,9 @@ def _Inline
       _tmp = apply(:_RawHtml)
       break if _tmp
       self.pos = _save
+      _tmp = apply(:_StrippedComment)
+      break if _tmp
+      self.pos = _save
       _tmp = apply(:_Entity)
       break if _tmp
       self.pos = _save
@@ -14131,7 +14134,7 @@ def _Code
     return _tmp
   end
 
-  # RawHtml = < (HtmlComment | HtmlBlockScript | HtmlTag) > { if html? then text else '' end }
+  # RawHtml = < (HtmlBlockScript | HtmlTag) > { html? ? text : '' }
   def _RawHtml
 
     _save = self.pos
@@ -14140,9 +14143,6 @@ def _RawHtml
 
       _save1 = self.pos
       while true # choice
-        _tmp = apply(:_HtmlComment)
-        break if _tmp
-        self.pos = _save1
         _tmp = apply(:_HtmlBlockScript)
         break if _tmp
         self.pos = _save1
@@ -14159,7 +14159,7 @@ def _RawHtml
         self.pos = _save
         break
       end
-      @result = begin;  if html? then text else '' end ; end
+      @result = begin;  html? ? text : '' ; end
       _tmp = true
       unless _tmp
         self.pos = _save
@@ -14171,6 +14171,32 @@ def _RawHtml
     return _tmp
   end
 
+  # StrippedComment = < HtmlComment > { '' }
+  def _StrippedComment
+
+    _save = self.pos
+    while true # sequence
+      _text_start = self.pos
+      _tmp = apply(:_HtmlComment)
+      if _tmp
+        text = get_text(_text_start)
+      end
+      unless _tmp
+        self.pos = _save
+        break
+      end
+      @result = begin;  '' ; end
+      _tmp = true
+      unless _tmp
+        self.pos = _save
+      end
+      break
+    end # end sequence
+
+    set_failed_rule :_StrippedComment unless _tmp
+    return _tmp
+  end
+
   # BlankLine = @Sp @Newline { "\n" }
   def _BlankLine
 
@@ -16733,7 +16759,7 @@ def _DefinitionListDefinition
   Rules[:_InStyleTags] = rule_info("InStyleTags", "StyleOpen (!StyleClose .)* StyleClose")
   Rules[:_StyleBlock] = rule_info("StyleBlock", "< InStyleTags > @BlankLine* { if css? then                     RDoc::Markup::Raw.new text                   end }")
   Rules[:_Inlines] = rule_info("Inlines", "(!@Endline Inline:i { i } | @Endline:c !(&{ github? } Ticks3 /[^`\\n]*$/) &Inline { c })+:chunks @Endline? { chunks }")
-  Rules[:_Inline] = rule_info("Inline", "(Str | @Endline | UlOrStarLine | @Space | Strong | Emph | Strike | Image | Link | NoteReference | InlineNote | Code | RawHtml | Entity | EscapedChar | Symbol)")
+  Rules[:_Inline] = rule_info("Inline", "(Str | @Endline | UlOrStarLine | @Space | Strong | Emph | Strike | Image | Link | NoteReference | InlineNote | Code | RawHtml | StrippedComment | Entity | EscapedChar | Symbol)")
   Rules[:_Space] = rule_info("Space", "@Spacechar+ { \" \" }")
   Rules[:_Str] = rule_info("Str", "@StartList:a < @NormalChar+ > { a = text } (StrChunk:c { a << c })* { rdoc_escape(a) }")
   Rules[:_StrChunk] = rule_info("StrChunk", "< (@NormalChar | /_+/ &Alphanumeric)+ > { text }")
@@ -16785,7 +16811,8 @@ def _DefinitionListDefinition
   Rules[:_Ticks4] = rule_info("Ticks4", "\"````\" !\"`\"")
   Rules[:_Ticks5] = rule_info("Ticks5", "\"`````\" !\"`\"")
   Rules[:_Code] = rule_info("Code", "(Ticks1 < ((!\"`\" Nonspacechar)+ | !Ticks1 /`+/ | !Ticks1 (@Spacechar | @Newline !@BlankLine))+ > Ticks1 | Ticks2 < ((!\"`\" Nonspacechar)+ | !Ticks2 /`+/ | !Ticks2 (@Spacechar | @Newline !@BlankLine))+ > Ticks2 | Ticks3 < ((!\"`\" Nonspacechar)+ | !Ticks3 /`+/ | !Ticks3 (@Spacechar | @Newline !@BlankLine))+ > Ticks3 | Ticks4 < ((!\"`\" Nonspacechar)+ | !Ticks4 /`+/ | !Ticks4 (@Spacechar | @Newline !@BlankLine))+ > Ticks4 | Ticks5 < ((!\"`\" Nonspacechar)+ | !Ticks5 /`+/ | !Ticks5 (@Spacechar | @Newline !@BlankLine))+ > Ticks5) { code text }")
-  Rules[:_RawHtml] = rule_info("RawHtml", "< (HtmlComment | HtmlBlockScript | HtmlTag) > { if html? then text else '' end }")
+  Rules[:_RawHtml] = rule_info("RawHtml", "< (HtmlBlockScript | HtmlTag) > { html? ? text : '' }")
+  Rules[:_StrippedComment] = rule_info("StrippedComment", "< HtmlComment > { '' }")
   Rules[:_BlankLine] = rule_info("BlankLine", "@Sp @Newline { \"\\n\" }")
   Rules[:_Quoted] = rule_info("Quoted", "(\"\\\"\" (!\"\\\"\" .)* \"\\\"\" | \"'\" (!\"'\" .)* \"'\")")
   Rules[:_HtmlAttribute] = rule_info("HtmlAttribute", "(AlphanumericAscii | \"-\")+ Spnl (\"=\" Spnl (Quoted | (!\">\" Nonspacechar)+))? Spnl")

lib/rdoc/markup/to_html.rb

@@ -314,7 +314,7 @@ def accept_block_quote(block_quote)
   def accept_paragraph(paragraph)
     @res << "\n<p>"
     text = paragraph.text @hard_break
-    text = text.gsub(/(#{SPACE_SEPARATED_LETTER_CLASS})?\K\r?\n(?=(?(1)(#{SPACE_SEPARATED_LETTER_CLASS})?))/o) {
+    text = text.gsub(/(#{SPACE_SEPARATED_LETTER_CLASS})?\K(?:\r?\n)+(?=(?(1)(#{SPACE_SEPARATED_LETTER_CLASS})?))/o) {
       defined?($2) && ' '
     }
     @res << to_html(text)

test/rdoc/rdoc_markdown_test.rb

@@ -1590,4 +1590,21 @@ def test_escape_tilde_not_supported
     assert_match(/\\~/, html)
   end
 
+  def test_html_comment_suppressed
+    # Standalone HTML block tags and comments are preserved as raw text.
+    # But if comments are inside a paragraph, they should be stripped
+    # because raw inside para is not permitted and RDoc format doesn't have comment nodes.
+    markdown = <<~MARKDOWN
+      <!-- comment -->
+      a<!-- comment -->b
+      <!-- comment -->
+      c
+    MARKDOWN
+
+    expected_doc = doc(raw('<!-- comment -->'), para("ab\n\nc"))
+    assert_equal expected_doc, parse(markdown)
+
+    expected_html = "<!-- comment -->\n<p>ab c</p>\n"
+    assert_equal expected_html, render(markdown)
+  end
 end