Skip to content

Commit e05107c

Browse files
aksOpsclaude
andcommitted
fix: batched DETACH DELETE to avoid Neo4j memory pool limit on large graphs
MATCH (n) DETACH DELETE n in a single transaction tries to delete 140K nodes + 130K edges at once, exceeding the memory pool. Now deletes in batches of 5000 nodes per transaction. Applied to both EnrichCommand and GraphStore.bulkSave(). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent bb05efd commit e05107c

2 files changed

Lines changed: 21 additions & 11 deletions

File tree

src/main/java/io/github/randomcodespace/iq/cli/EnrichCommand.java

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -168,11 +168,17 @@ private int enrichFromCache(AnalysisCache cache, Path root, NumberFormat nf, Ins
168168
dbms = new DatabaseManagementServiceBuilder(graphPath).build();
169169
GraphDatabaseService db = dbms.database("neo4j");
170170

171-
// Clear existing data
172-
try (Transaction tx = db.beginTx()) {
173-
tx.execute("MATCH (n) DETACH DELETE n");
174-
tx.commit();
175-
}
171+
// Clear existing data in batches to avoid memory pool limit on large graphs
172+
CliOutput.info(" Clearing existing graph...");
173+
int deleted;
174+
do {
175+
try (Transaction tx = db.beginTx()) {
176+
var result = tx.execute(
177+
"MATCH (n) WITH n LIMIT 5000 DETACH DELETE n RETURN count(*) AS cnt");
178+
deleted = result.hasNext() ? ((Number) result.next().get("cnt")).intValue() : 0;
179+
tx.commit();
180+
}
181+
} while (deleted > 0);
176182

177183
// Bulk-load nodes in batches using UNWIND
178184
// Smaller batches to avoid Neo4j memory pool limit (nodes carry prop_* properties)

src/main/java/io/github/randomcodespace/iq/graph/GraphStore.java

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -70,13 +70,17 @@ public void bulkSave(List<CodeNode> nodes) {
7070
if (nodes.isEmpty()) return;
7171
long start = System.currentTimeMillis();
7272

73-
// 1. Clear existing data
73+
// 1. Clear existing data in batches to avoid memory pool limit
7474
log.info("Neo4j: clearing existing graph...");
75-
try (Transaction tx = graphDb.beginTx()) {
76-
// Batch delete to avoid OOM on huge graphs
77-
tx.execute("MATCH (n) DETACH DELETE n");
78-
tx.commit();
79-
}
75+
int deleted;
76+
do {
77+
try (Transaction tx = graphDb.beginTx()) {
78+
var result = tx.execute(
79+
"MATCH (n) WITH n LIMIT 5000 DETACH DELETE n RETURN count(*) AS cnt");
80+
deleted = result.hasNext() ? ((Number) result.next().get("cnt")).intValue() : 0;
81+
tx.commit();
82+
}
83+
} while (deleted > 0);
8084

8185
// 2. Create index on id property for fast MATCH during edge creation
8286
try (Transaction tx = graphDb.beginTx()) {

0 commit comments

Comments
 (0)