filter NOP stanzas

Author: lws-team

CMakeLists.txt

@@ -15,7 +15,7 @@ install(PROGRAMS tools/concat.sh DESTINATION bin)
 
 # single patched file, directly taken from Gemini 2.5 copy button
 
-add_test(NAME lws1
+add_test(NAME fixdiff1
 	 COMMAND ${CMAKE_COMMAND}
 	 	-DCMD=$<TARGET_FILE:${PROJECT_NAME}>
 		-DSRC=client-parser-ws.c
@@ -27,7 +27,7 @@ add_test(NAME lws1
 
  # two concatenated single file patches
 
-add_test(NAME lws2
+add_test(NAME fixdiff2
 	 COMMAND ${CMAKE_COMMAND}
 	 	-DCMD=$<TARGET_FILE:${PROJECT_NAME}>
 		-DSRC=deaddrop.js
@@ -42,7 +42,7 @@ add_test(NAME lws2
 
  # two concatenated patches using git style diff and index lines (otherwise same as above)
 
- add_test(NAME lws3
+ add_test(NAME fixdiff3
 	 COMMAND ${CMAKE_COMMAND}
 	 	-DCMD=$<TARGET_FILE:${PROJECT_NAME}>
 		-DSRC=deaddrop.js
@@ -58,7 +58,7 @@ add_test(NAME lws2
  # two concatenated patches using git style diff and index lines with extra newline at end
  # (otherwise same as above)
 
- add_test(NAME lws4
+ add_test(NAME fixdiff4
 	 COMMAND ${CMAKE_COMMAND}
 	 	-DCMD=$<TARGET_FILE:${PROJECT_NAME}>
 		-DSRC=deaddrop.js
@@ -74,7 +74,7 @@ add_test(NAME lws2
  # Same as test1 but with 3 x examples of whitespace-only lines added that should be
  # reduced down to empty lines
 
-add_test(NAME lws5
+add_test(NAME fixdiff5
 	 COMMAND ${CMAKE_COMMAND}
 	 	-DCMD=$<TARGET_FILE:${PROJECT_NAME}>
 		-DSRC=client-parser-ws.c
@@ -84,7 +84,7 @@ add_test(NAME lws5
 		-P ${CMAKE_CURRENT_SOURCE_DIR}/tests/runtest.cmake
 	 WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/5)
 
-add_test(NAME lws6
+add_test(NAME fixdiff6
 	 COMMAND ${CMAKE_COMMAND}
 	 	-DCMD=$<TARGET_FILE:${PROJECT_NAME}>
 		-DSRC=b-comms.c
@@ -94,4 +94,14 @@ add_test(NAME lws6
 		-P ${CMAKE_CURRENT_SOURCE_DIR}/tests/runtest.cmake
 	 WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/6)
 
+add_test(NAME fixdiff7
+	 COMMAND ${CMAKE_COMMAND}
+	 	-DCMD=$<TARGET_FILE:${PROJECT_NAME}>
+		-DSRC=deaddrop.js
+		-DPATCH=gemini.patch
+		-DEXPSHA=1fc2b5b927ee6f4b3ee43d6d5db02c8f00322ea7b6b2a6e346d08faae82b4d3a
+		-DEXPSHA_WIN=2e6b9b12ae0128c9edfc109744b9c67848712b0521c322a45104895aa4cbc3b1
+		-P ${CMAKE_CURRENT_SOURCE_DIR}/tests/runtest.cmake
+	 WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/7)
+
 

README.md

@@ -32,17 +32,21 @@ headers with accurate line counts on stdout.
 
 It silently repairs:
 
- 1. new empty lines with only whitespace, by rewriting to blank lines
- 2. original lines in diff that differ from real line in file only by
+ 1. New empty + lines with only whitespace are rewritten to be empty blank lines
+ 2. Diff stanzas that do not contain any +/- lines are removed
+ 3. Original lines in diff that differ from real line in file only by
     whitespace are rewritten to contain the correct whitespace
- 3. wrong "before" line in original stanza header
- 4. wrong "before" line count in original stanza header
- 5. wrong "after" line in original stanza header
- 6. wrong "after" line count in original stanza header
- 7. extra lead-in context lines to stanza by removing until only 3
- 8. diffs adding to end of file with missing or wrong context caused by
+ 4. All stanza header line offsets and counts are recomputed from the actual
+    match in the original source and counting before and after lines in the diff,
+    the incoming @@ line is completely ignored and rewritten with actual info
+ 5. Extra lead-in context lines to stanza by removing until only 3
+ 6. Excessive lead-out-context is removed, missing lead-out context is added.
+    Diffs adding to EOF with missing or wrong context caused by
     LLM losing blank lines at the original EOF are rewritten by checking
-    the original source file for extra lines and adding them to the stanza as context)
+    the original source file for extra lines and adding them as needed.
+ 7. Unexpected blank lines in a stanza (without space, + or -) are either ignored
+    if happening at the end of the stanza, or rewritten to be context by adding
+    a space at the beginning, if the normal diff resumes.
 
 It finds and scans the sources the patches apply to and uses the diff stanza to
 find the original line it applied to by itself, along with the original line

fixdiff.c

@@ -114,10 +114,13 @@ typedef struct {
 
 	int		li_out;
 
+	int		pending_empty_lines;
+
 	char		ongoing;
 	char		skip_this_one;
 	char		lead_in_active;
 	char		cx_active;
+	char		have_seen_delta;
 
 	char		osh[128];
 	char		temp[64];
@@ -286,15 +289,17 @@ _mkstemp(const char *base, char *tmp, size_t len)
 static int
 fixdiff_stanza_start(dp_t *pdp, char *sh, size_t len)
 {
-	pdp->pre		= 0;
-	pdp->post		= 0;
-	pdp->lead_in		= 0;
-	pdp->lead_in_active	= 1;
-	pdp->lead_out		= 0;
-	pdp->cx_active		= 1;
-	pdp->lead_in_corrected	= 0;
-	pdp->d			= DSS_PMSAD;
-	pdp->ongoing		= 1;
+	pdp->pre			= 0;
+	pdp->post			= 0;
+	pdp->lead_in			= 0;
+	pdp->lead_in_active		= 1;
+	pdp->lead_out			= 0;
+	pdp->cx_active			= 1;
+	pdp->lead_in_corrected		= 0;
+	pdp->d				= DSS_PMSAD;
+	pdp->ongoing			= 1;
+	pdp->have_seen_delta		= 0;
+	pdp->pending_empty_lines	= 0;
 
 	pdp->stanzas++;
 
@@ -432,7 +437,7 @@ fixdiff_find_original(dp_t *pdp, int *line_start)
 				break;
 
 			if (!ls) {
-				elog("failed to match, best chunk %d lines at %s:%d (tabs shown below as >)\n",
+				elog("Failed to match, best chunk %d lines started at %s:%d (tabs shown below as >)\n",
 				     lmc, pdp->pf, lg_lis);
 				elog("last match: patch = '%s"
 				     "',         source = '%s'\n", b1, b2);
@@ -628,6 +633,17 @@ fixdiff_stanza_end(dp_t *pdp)
 	if (!pdp->ongoing)
 		return 0;
 
+	if (!pdp->have_seen_delta) {
+		pdp->ongoing = 0;
+		close(pdp->fd_temp);
+		elog("  - stanza %d: (filtered out due to no delta inside)\n", pdp->stanzas);
+
+		return 0;
+	}
+
+	if (dp.pending_empty_lines)
+		elog("  Dropped %d unexpected empty lines\n", dp.pending_empty_lines);
+
 	if (fixdiff_find_original(pdp, &orig)) {
 		elog("Unable to find original stanza in source\n");
 		goto probs;
@@ -653,7 +669,7 @@ fixdiff_stanza_end(dp_t *pdp)
 	/* is that what we already had? */
 
 	if (strcmp(buf, pdp->osh)) {
-		elog("  - (lead_in %d, lead_out %d) %s", pdp->lead_in, pdp->lead_out, buf);
+		elog("  - stanza %d: %s", pdp->stanzas, buf);
 		pdp->bad++;
 	}
 
@@ -865,6 +881,32 @@ main(int argc, char *argv[])
 				goto bail;
 			}
 
+			if (dp.pending_empty_lines &&
+			    (in[0] == ' ' || in[0] == '-' || in[0] == '+')) {
+				char ctx[3];
+
+				elog("  Treating %d unexpected newline(s) as context\n", dp.pending_empty_lines);
+
+				ctx[0] = ' ';
+				ctx[1] = '\n';
+				ctx[2] = '\0';
+
+				while (dp.pending_empty_lines > 0) {
+					dp.pending_empty_lines--;
+					dp.pre++;
+					dp.post++;
+					if (dp.lead_in_active)
+						dp.lead_in++;
+					dp.cx_active++;
+
+					w = write(dp.fd_temp != -1 ? dp.fd_temp : 1, ctx, TO_POSLEN(2));
+					if (w < 0) {
+						elog("write to stdout failed: %d\n", errno);
+						goto bail;
+					}
+				}
+			}
+
 			if (in[0] == ' ') { /* Space */
 				dp.pre++;
 				dp.post++;
@@ -888,6 +930,7 @@ main(int argc, char *argv[])
 					dp.pre++;
 					dp.lead_in_active = 0;
 					dp.cx_active = 0;
+					dp.have_seen_delta = 1;
 					break;
 				} else
 					if (in[0] == '+') { /* Plus */
@@ -930,6 +973,7 @@ main(int argc, char *argv[])
 						dp.post++;
 						dp.lead_in_active = 0;
 						dp.cx_active = 0;
+						dp.have_seen_delta = 1;
 						break;
 					}
 
@@ -957,15 +1001,29 @@ main(int argc, char *argv[])
 			}
 
 			if (in[0] == 0xa) {
-				/* we can often find this from extra lines at EOT */
-				elog("  Skipping unexpected newline\n");
+				/*
+				 * We can find this blank diff line illegally generated by the LLM:
+				 *
+				 * 1) from extra lines at diff EOT, maybe the user
+				 *    picked them up from screenscraping too (tests/4)
+				 * 2) because there was an empty line there,
+				 *    but the LLM did not prepend it with a
+				 *    character indicating what action to
+				 *    take with it (tests/7)
+				 *
+				 * We can distinguish what to (for these cases anyway) by waiting
+				 * to see if there are any more lines in the stanza that have the
+				 * +/-/space, if not, just drop the CR-only line
+				 */
+
+				dp.pending_empty_lines++;
 				continue;
 			}
 
 			elog("'%c' (0x%x)\n", in[0], in[0]);
 			dp.reason = "unexpected character in stanza";
 			goto bail;
-		}
+		} /* switch */
 
 		if (dp.skip_this_one) {
 			dp.skip_this_one = 0;