fix: granite33 citation spans wrong for duplicate sentences (#851) (#872)

str.find() without an offset always returned the first occurrence, so
when the same sentence appeared more than once in the response, every
citation after the first got response_begin/response_end pointing at
the first occurrence.

Introduce search_offset that advances past each consumed sentence.
When find() misses after search_offset the sentence appears only once
(multiple citations on one sentence); fall back to searching from the
start so those citations share the correct span.

Adds two regression tests to TestAddCitationResponseSpans:
- test_duplicate_sentences_each_get_own_span
- test_multiple_citations_on_same_sentence_share_span

Assisted-by: Claude Code

Signed-off-by: Nigel Jones <jonesn@uk.ibm.com>

Author: planetf1

mellea/formatters/granite/granite3/granite33/output.py

@@ -211,16 +211,28 @@ def _add_citation_response_spans(
                 )
                 continue
 
-    # For each citation bring the response sentence to which it refers and its
-    # begin/end spans
+    # For each citation, find its span in the clean response.
+    # Citations are indexed in left-to-right response order (citation_idx above),
+    # so search_offset can advance monotonically (fixes issue #851 — str.find()
+    # without an offset always returned the first occurrence for duplicate sentences).
+    # If find() misses after search_offset the sentence appears only once in the
+    # response; fall back to the start so multi-citation sentences share their span.
+    search_offset = 0
     for i, citation in enumerate(augmented_citation_info):
         response_text = response_sents_by_citation_id.get(str(i), "")
-        index = response_text_without_citations.find(response_text)
+
+        index = response_text_without_citations.find(response_text, search_offset)
         if index < 0:
-            logger.warning(
-                "Error in extracting the response sentence of a citation: Unexpected error."
-            )
-            continue
+            # Single occurrence: fall back so every citation on this sentence
+            # gets the same span; do not advance search_offset.
+            index = response_text_without_citations.find(response_text)
+            if index < 0:
+                logger.warning(
+                    "Error in extracting the response sentence of a citation: Unexpected error."
+                )
+                continue
+        else:
+            search_offset = index + len(response_text)
 
         citation["response_text"] = response_text
         citation["response_begin"] = index

test/formatters/granite/test_granite33_output.py

@@ -292,6 +292,60 @@ def test_single_sentence_response(self):
         end = citation["response_end"]
         assert response_without[begin:end] == citation["response_text"]
 
+    def test_duplicate_sentences_each_get_own_span(self):
+        """Regression (#851): duplicate sentence text must map to distinct occurrences.
+
+        Without this fix, str.find() always returns the first occurrence, so both
+        citations end up with the same span pointing at the first sentence.
+        """
+        sent = "The sky is blue."
+        cite1 = f'{CITE_START}{{"document_id": "1"}}{CITE_END}'
+        cite2 = f'{CITE_START}{{"document_id": "2"}}{CITE_END}'
+        # Two identical sentences, each followed by a separate citation tag.
+        response_with = f"{sent} {cite1} {sent} {cite2}"
+        # Clean response has both sentences, separated by a space.
+        response_without = f"{sent} {sent}"
+
+        result = _add_citation_response_spans(
+            [self._make_citation(), self._make_citation()],
+            response_with,
+            response_without,
+        )
+
+        assert len(result) == 2
+        spans = [(c["response_begin"], c["response_end"]) for c in result]
+        # Both spans must be valid slices of the clean response.
+        for begin, end in spans:
+            assert response_without[begin:end] == sent
+        # The two spans must be different (pointing at the two different occurrences).
+        assert spans[0] != spans[1]
+        # They must not overlap.
+        spans_sorted = sorted(spans)
+        assert spans_sorted[0][1] <= spans_sorted[1][0]
+
+    def test_multiple_citations_on_same_sentence_share_span(self):
+        """Two citations on a single occurrence of a sentence must share the same span."""
+        sent = "The sky is blue."
+        cite1 = f'{CITE_START}{{"document_id": "1"}}{CITE_END}'
+        cite2 = f'{CITE_START}{{"document_id": "2"}}{CITE_END}'
+        # Single sentence with two citation tags; clean response has one occurrence.
+        response_with = f"{sent} {cite1} {cite2}"
+        response_without = sent
+
+        result = _add_citation_response_spans(
+            [self._make_citation(), self._make_citation()],
+            response_with,
+            response_without,
+        )
+
+        assert len(result) == 2
+        # Both citations reference the one occurrence — spans must be identical.
+        assert result[0]["response_begin"] == result[1]["response_begin"]
+        assert result[0]["response_end"] == result[1]["response_end"]
+        begin = result[0]["response_begin"]
+        end = result[0]["response_end"]
+        assert response_without[begin:end] == sent
+
 
 # ---------------------------------------------------------------------------
 # Granite33OutputProcessor.transform