test(pipeline): end-to-end integration test over markdown corpus

New integration test drives pipeline.New().IndexPath().Finalize() over
5 small markdown files with the mock LLM provider, then asserts:

- Document count is exactly 5.
- Chunk count is in the 5..50 band.
- Embedding count equals chunk count (Phase 2 invariant).
- Entity count is in the 2..2*chunks band (mock returns 2 entities
  per extraction prompt; dedup collapses duplicates).
- Relationship count is >=1.
- LocalSearch("Apollo program", topK=5) returns >=1 chunk containing
  "Apollo".

Gated by //go:build integration && sqlite_fts5 so the default
`go test ./...` path is unaffected. The test-integration CI job picks
it up automatically via its existing -tags "sqlite_fts5 integration"
invocation. No CI workflow change needed.

Local run: 0.05s without -race, 1.07s with -race.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: aksOps

internal/pipeline/integration_test.go

@@ -0,0 +1,156 @@
+//go:build integration && sqlite_fts5
+
+package pipeline_test
+
+import (
+	"context"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/RandomCodeSpace/docsiq/internal/config"
+	"github.com/RandomCodeSpace/docsiq/internal/embedder"
+	"github.com/RandomCodeSpace/docsiq/internal/llm/mock"
+	"github.com/RandomCodeSpace/docsiq/internal/pipeline"
+	"github.com/RandomCodeSpace/docsiq/internal/search"
+	"github.com/RandomCodeSpace/docsiq/internal/store"
+)
+
+// TestPipeline_IndexAndSearch_EndToEnd drives pipeline.New().IndexPath()
+// followed by Finalize() against a 5-file markdown corpus using a
+// deterministic mock LLM provider, then asserts:
+//   - SQLite documents, chunks, embeddings row counts are in the
+//     expected bands,
+//   - entity / relationship counts reflect mock extraction,
+//   - a LocalSearch for a known substring returns >=1 hit from the
+//     correct document.
+//
+// Runs under the integration build tag so it stays out of the default
+// `go test ./...` path; the CI test-integration job runs it with -race.
+func TestPipeline_IndexAndSearch_EndToEnd(t *testing.T) {
+	t.Parallel()
+
+	// 1. Temp dir for the SQLite DB (OpenForProject constructs
+	// <dataDir>/projects/<slug>/docsiq.db for us).
+	dataDir := t.TempDir()
+	st, err := store.OpenForProject(dataDir, "itest")
+	if err != nil {
+		t.Fatalf("store.OpenForProject: %v", err)
+	}
+	t.Cleanup(func() { _ = st.Close() })
+
+	// 2. Resolve the corpus relative to this test file; filepath.Abs
+	// resolves against the test binary's cwd which is the package dir
+	// (internal/pipeline), so ../../testdata/pipeline is correct.
+	corpus, err := filepath.Abs(filepath.Join("..", "..", "testdata", "pipeline"))
+	if err != nil {
+		t.Fatalf("resolve corpus: %v", err)
+	}
+
+	// 3. Minimal config; values low to keep wall-clock predictable
+	// under -race.
+	cfg := &config.Config{
+		DataDir:        dataDir,
+		DefaultProject: "itest",
+		LLM:            config.LLMConfig{Provider: "none"}, // unused; we inject the mock directly
+		Indexing: config.IndexingConfig{
+			BatchSize:     4,
+			ChunkSize:     512,
+			ChunkOverlap:  64,
+			ExtractGraph:  true,
+			ExtractClaims: false,
+			MaxGleanings:  0,
+		},
+		Community: config.CommunityConfig{
+			MinCommunitySize: 1,
+			MaxLevels:        2,
+		},
+	}
+
+	// 4. Mock provider — no network, deterministic.
+	prov := mock.New(mock.DefaultDims)
+	pl := pipeline.New(st, prov, cfg)
+
+	// 5. Drive the indexer with a 120s deadline; a real deadlock will
+	// blow past this and fail loud.
+	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	defer cancel()
+
+	if err := pl.IndexPath(ctx, corpus, pipeline.IndexOptions{
+		Workers: 2,
+		Verbose: false,
+	}); err != nil {
+		t.Fatalf("IndexPath: %v", err)
+	}
+
+	// 6. Run Finalize (Phases 3-4: community detection + summaries).
+	if err := pl.Finalize(ctx, false); err != nil {
+		t.Fatalf("Finalize: %v", err)
+	}
+
+	// 7. Row-count assertions — bands, not exact values, so chunker
+	// re-tuning doesn't break this test.
+	docCount := countRows(ctx, t, st, `SELECT count(*) FROM documents WHERE is_latest = 1`)
+	if docCount != 5 {
+		t.Errorf("document count: want 5, got %d", docCount)
+	}
+
+	chunkCount := countRows(ctx, t, st, `SELECT count(*) FROM chunks`)
+	if chunkCount < 5 || chunkCount > 50 {
+		t.Errorf("chunk count: want 5..50, got %d", chunkCount)
+	}
+
+	embCount := countRows(ctx, t, st, `SELECT count(*) FROM embeddings`)
+	if embCount != chunkCount {
+		t.Errorf("embedding count: want %d (= chunk count), got %d", chunkCount, embCount)
+	}
+
+	// Mock returns 2 entities per extraction prompt; extractor runs
+	// at least once per chunk; dedup collapses duplicates. Lower
+	// bound 2, upper bound 2*chunkCount is a generous band.
+	entityCount := countRows(ctx, t, st, `SELECT count(*) FROM entities`)
+	if entityCount < 2 || entityCount > 2*chunkCount {
+		t.Errorf("entity count: want 2..%d, got %d", 2*chunkCount, entityCount)
+	}
+
+	relCount := countRows(ctx, t, st, `SELECT count(*) FROM relationships`)
+	if relCount < 1 {
+		t.Errorf("relationship count: want >=1, got %d", relCount)
+	}
+
+	// 8. Search assertion: "Apollo" appears in alpha.md, beta.md,
+	// gamma.md (indirectly via missions), and epsilon.md. LocalSearch
+	// must return >=1 chunk whose content references the corpus.
+	emb := embedder.New(prov, cfg.Indexing.BatchSize)
+	if emb == nil {
+		t.Fatal("embedder.New returned nil for non-nil provider")
+	}
+	result, err := search.LocalSearch(ctx, st, emb, nil, "Apollo program", 5, 0)
+	if err != nil {
+		t.Fatalf("LocalSearch: %v", err)
+	}
+	if len(result.Chunks) == 0 {
+		t.Fatal("LocalSearch returned 0 chunks; expected >=1")
+	}
+	var gotApollo bool
+	for _, c := range result.Chunks {
+		if strings.Contains(strings.ToLower(c.Chunk.Content), "apollo") {
+			gotApollo = true
+			break
+		}
+	}
+	if !gotApollo {
+		t.Errorf("LocalSearch returned %d chunks but none mentioned Apollo", len(result.Chunks))
+	}
+}
+
+// countRows runs a `SELECT count(*) ...` and fails the test on error.
+func countRows(ctx context.Context, t *testing.T, st *store.Store, q string) int {
+	t.Helper()
+	var n int
+	if err := st.DB().QueryRowContext(ctx, q).Scan(&n); err != nil {
+		t.Fatalf("countRows %q: %v", q, err)
+	}
+	return n
+}

testdata/pipeline/alpha.md

@@ -0,0 +1,10 @@
+# Alpha Doc
+
+Alpha is the first document in the fixture corpus. It mentions
+Project Apollo, which was a spaceflight program run by NASA from 1961
+to 1972. Apollo 11 landed humans on the Moon for the first time.
+
+## Background
+
+The program was led by administrator James Webb, named after the
+Greek god Apollo. The Saturn V rocket launched every mission.

testdata/pipeline/beta.md

@@ -0,0 +1,7 @@
+# Beta Doc
+
+Beta covers the Apollo program's missions in more detail. Apollo 11,
+Apollo 12, and Apollo 13 are the most famous flights. Neil Armstrong
+was the commander of Apollo 11 and the first human on the Moon.
+
+Apollo 13 suffered an oxygen tank explosion but returned safely.

testdata/pipeline/delta.md

@@ -0,0 +1,5 @@
+# Delta Doc
+
+Delta is a short document about unrelated topics. The moon is a
+celestial body orbiting Earth. The Earth orbits the Sun. Neither is
+particularly relevant to the Apollo program except by coincidence.

testdata/pipeline/epsilon.md

@@ -0,0 +1,5 @@
+# Epsilon Doc
+
+Epsilon is the last fixture. It mentions James Webb again — the
+James Webb Space Telescope was named after the Apollo-era NASA
+administrator.

testdata/pipeline/gamma.md

@@ -0,0 +1,11 @@
+# Gamma Doc
+
+Gamma is about the Saturn V rocket. Saturn V was the largest rocket
+ever flown successfully. It was designed by Wernher von Braun and his
+team at the Marshall Space Flight Center. The rocket had three
+stages.
+
+## Notable Flights
+
+All crewed Apollo missions used Saturn V. Skylab was launched on a
+modified Saturn V.