Merge branch 'main' into claude/practical-mclaren-4da7df

Author: tannerlinsley

netlify.toml

@@ -16,6 +16,7 @@ directory = "netlify/functions"
 included_files = [
   "public/fonts/Inter-Regular.ttf",
   "public/fonts/Inter-ExtraBold.ttf",
+  "public/fonts/Inter-Black.ttf",
   "public/images/logos/splash-dark.png",
   "node_modules/.pnpm/@takumi-rs+wasm@*/node_modules/@takumi-rs/wasm/pkg/takumi_wasm_bg.wasm",
 ]

netlify/functions/prune-content-cache-background.ts

@@ -0,0 +1,47 @@
+import type { Config } from '@netlify/functions'
+import { pruneStaleCacheRows } from '~/utils/github-content-cache.server'
+
+/**
+ * Netlify Scheduled Function - Prune stale GitHub content cache rows
+ *
+ * Deletes rows in `github_content_cache` and `docs_artifact_cache` whose
+ * `updatedAt` is older than the configured TTL. Anything that's still being
+ * read regularly is touched on refresh and stays put; the rest (old gitRefs,
+ * deleted paths, abandoned branches) gets reclaimed.
+ *
+ * Scheduled: Daily at 09:00 UTC.
+ */
+const handler = async (req: Request) => {
+  const { next_run } = await req.json()
+
+  const startTime = Date.now()
+  console.log('[prune-content-cache] Starting prune...')
+
+  try {
+    const result = await pruneStaleCacheRows()
+    const duration = Date.now() - startTime
+
+    console.log(
+      `[prune-content-cache] ✓ Completed in ${duration}ms - deleted ${result.githubContentDeleted} content rows (${result.githubContentNegativesDeleted} negatives), ${result.docsArtifactDeleted} artifact rows (cutoff: ${result.cutoff.toISOString()}, negativeCutoff: ${result.negativeCutoff.toISOString()})`,
+    )
+    console.log('[prune-content-cache] Next invocation at:', next_run)
+  } catch (error) {
+    const duration = Date.now() - startTime
+    const errorMessage = error instanceof Error ? error.message : String(error)
+    const errorStack = error instanceof Error ? error.stack : undefined
+
+    console.error(
+      `[prune-content-cache] ✗ Failed after ${duration}ms:`,
+      errorMessage,
+    )
+    if (errorStack) {
+      console.error('[prune-content-cache] Stack:', errorStack)
+    }
+  }
+}
+
+export default handler
+
+export const config: Config = {
+  schedule: '0 9 * * *', // Daily at 09:00 UTC
+}

package.json

@@ -12,6 +12,7 @@
     "build": "vite build",
     "start": "vite start",
     "start:prod": "pnpm run with-env node scripts/run-built-server.mjs",
+    "content:build": "node scripts/build-content-collections.mjs",
     "lint": "oxlint --type-aware --disable-nested-config",
     "format": "oxfmt --write",
     "db:generate": "drizzle-kit generate",
@@ -21,6 +22,7 @@
     "docs:webhooks:plan": "tsx scripts/sync-docs-webhooks.ts --dry-run",
     "docs:webhooks:sync": "tsx scripts/sync-docs-webhooks.ts",
     "husky": "pnpm run format && pnpm run test && pnpm run test:smoke",
+    "pretest": "pnpm run content:build",
     "test": "run-p test:tsc test:lint",
     "test:tsc": "tsc",
     "test:lint": "pnpm run lint",
@@ -125,9 +127,9 @@
     "@playwright/test": "^1.59.0",
     "@shikijs/transformers": "^4.0.2",
     "@tanstack/devtools-vite": "^0.6.0",
-    "@tanstack/redact": "^0.0.3",
     "@tanstack/react-devtools": "^0.10.2",
     "@tanstack/react-query-devtools": "^5.99.0",
+    "@tanstack/redact": "^0.0.6",
     "@types/hast": "^3.0.4",
     "@types/node": "^25.5.0",
     "@types/pg": "^8.20.0",

pnpm-lock.yaml

@@ -0,0 +1,5 @@
+import { createBuilder } from '@content-collections/core'
+import path from 'node:path'
+
+const builder = await createBuilder(path.resolve('content-collections.ts'))
+await builder.build()

public/fonts/Inter-Black.ttf

@@ -0,0 +1,114 @@
+import { and, eq, sql } from 'drizzle-orm'
+import { db } from '../src/db/client'
+import { githubContentCache } from '../src/db/schema'
+
+async function main() {
+  const repo = 'tanstack/router'
+  const gitRef = 'main'
+
+  console.log(`\n=== ${repo}@${gitRef} ===`)
+
+  const [{ total }] = await db
+    .select({ total: sql<number>`count(*)::int` })
+    .from(githubContentCache)
+    .where(
+      and(
+        eq(githubContentCache.repo, repo),
+        eq(githubContentCache.gitRef, gitRef),
+      ),
+    )
+  console.log(`total: ${total}`)
+
+  console.log('\nby contentKind:')
+  const byKind = await db
+    .select({
+      kind: githubContentCache.contentKind,
+      rows: sql<number>`count(*)::int`,
+      present: sql<number>`count(*) filter (where ${githubContentCache.isPresent})::int`,
+      absent: sql<number>`count(*) filter (where not ${githubContentCache.isPresent})::int`,
+    })
+    .from(githubContentCache)
+    .where(
+      and(
+        eq(githubContentCache.repo, repo),
+        eq(githubContentCache.gitRef, gitRef),
+      ),
+    )
+    .groupBy(githubContentCache.contentKind)
+  console.table(byKind)
+
+  console.log('\ntop-level path segment distribution:')
+  const byTopSegment = await db.execute(sql`
+    select
+      content_kind,
+      split_part(path, '/', 1) as top,
+      count(*)::int as rows
+    from github_content_cache
+    where repo = ${repo} and git_ref = ${gitRef}
+    group by content_kind, top
+    order by count(*) desc
+    limit 30
+  `)
+  console.table(byTopSegment)
+
+  console.log('\nfile extension distribution (file kind only):')
+  const byExt = await db.execute(sql`
+    select
+      case
+        when path ~ '\\.[a-zA-Z0-9]+$' then regexp_replace(path, '.*\\.([a-zA-Z0-9]+)$', '\\1')
+        else '(none)'
+      end as ext,
+      count(*)::int as rows
+    from github_content_cache
+    where repo = ${repo} and git_ref = ${gitRef} and content_kind = 'file'
+    group by ext
+    order by count(*) desc
+    limit 30
+  `)
+  console.table(byExt)
+
+  console.log('\nsample 20 random file paths:')
+  const samples = await db.execute(sql`
+    select path, is_present, length(coalesce(text_content, '')) as text_len
+    from github_content_cache
+    where repo = ${repo} and git_ref = ${gitRef} and content_kind = 'file'
+    order by random()
+    limit 20
+  `)
+  console.table(samples)
+
+  console.log('\nsample 10 random dir paths:')
+  const dirSamples = await db.execute(sql`
+    select path, is_present, jsonb_array_length(coalesce(json_content, '[]'::jsonb)) as entries
+    from github_content_cache
+    where repo = ${repo} and git_ref = ${gitRef} and content_kind = 'dir'
+    order by random()
+    limit 10
+  `)
+  console.table(dirSamples)
+
+  console.log('\npath length distribution:')
+  const lenBuckets = await db.execute(sql`
+    select
+      case
+        when length(path) < 50 then '< 50'
+        when length(path) < 100 then '50-100'
+        when length(path) < 200 then '100-200'
+        when length(path) < 500 then '200-500'
+        else '500+'
+      end as bucket,
+      count(*)::int as rows
+    from github_content_cache
+    where repo = ${repo} and git_ref = ${gitRef}
+    group by bucket
+    order by min(length(path))
+  `)
+  console.table(lenBuckets)
+}
+
+main()
+  .then(() => process.exit(0))
+  .catch((err) => {
+    console.error(err)
+    process.exit(1)
+  })

scripts/build-content-collections.mjs

@@ -0,0 +1,71 @@
+import { sql } from 'drizzle-orm'
+import { db } from '../src/db/client'
+import { docsArtifactCache, githubContentCache } from '../src/db/schema'
+
+async function main() {
+  console.log('\n=== github_content_cache ===')
+
+  const [{ total }] = await db
+    .select({ total: sql<number>`count(*)::int` })
+    .from(githubContentCache)
+  console.log(`total rows: ${total}`)
+
+  console.log('\nby repo:')
+  const byRepo = await db
+    .select({
+      repo: githubContentCache.repo,
+      rows: sql<number>`count(*)::int`,
+      refs: sql<number>`count(distinct ${githubContentCache.gitRef})::int`,
+    })
+    .from(githubContentCache)
+    .groupBy(githubContentCache.repo)
+    .orderBy(sql`count(*) desc`)
+  console.table(byRepo)
+
+  console.log('\ntop 20 (repo, gitRef) by row count:')
+  const byRef = await db
+    .select({
+      repo: githubContentCache.repo,
+      gitRef: githubContentCache.gitRef,
+      rows: sql<number>`count(*)::int`,
+      oldestCreated: sql<Date>`min(${githubContentCache.createdAt})`,
+      newestCreated: sql<Date>`max(${githubContentCache.createdAt})`,
+      newestUpdated: sql<Date>`max(${githubContentCache.updatedAt})`,
+    })
+    .from(githubContentCache)
+    .groupBy(githubContentCache.repo, githubContentCache.gitRef)
+    .orderBy(sql`count(*) desc`)
+    .limit(20)
+  console.table(byRef)
+
+  console.log('\ncreatedAt age buckets:')
+  const ageBuckets = await db.execute(sql`
+    select
+      case
+        when created_at > now() - interval '1 day' then '0-1d'
+        when created_at > now() - interval '7 days' then '1-7d'
+        when created_at > now() - interval '30 days' then '7-30d'
+        when created_at > now() - interval '90 days' then '30-90d'
+        when created_at > now() - interval '180 days' then '90-180d'
+        else '180d+'
+      end as bucket,
+      count(*)::int as rows
+    from github_content_cache
+    group by bucket
+    order by min(created_at)
+  `)
+  console.table(ageBuckets)
+
+  console.log('\n=== docs_artifact_cache ===')
+  const [{ totalArt }] = await db
+    .select({ totalArt: sql<number>`count(*)::int` })
+    .from(docsArtifactCache)
+  console.log(`total rows: ${totalArt}`)
+}
+
+main()
+  .then(() => process.exit(0))
+  .catch((err) => {
+    console.error(err)
+    process.exit(1)
+  })

scripts/inspect-content-cache-paths.ts

@@ -0,0 +1,35 @@
+import { sql } from 'drizzle-orm'
+import { db } from '../src/db/client'
+
+async function main() {
+  console.log('\n=== preview prune impact ===')
+
+  const before = await db.execute(sql`
+    select
+      count(*)::int as total_rows,
+      count(*) filter (where is_present)::int as present_rows,
+      count(*) filter (where not is_present)::int as absent_rows,
+      count(*) filter (where not is_present and updated_at < now() - interval '1 day')::int as absent_older_than_1d,
+      count(*) filter (where updated_at < now() - interval '30 days')::int as any_older_than_30d
+    from github_content_cache
+  `)
+  console.table(before)
+
+  console.log('\nbreakdown of absent rows by repo (top 20):')
+  const byRepo = await db.execute(sql`
+    select repo, count(*)::int as absent_rows
+    from github_content_cache
+    where not is_present and updated_at < now() - interval '1 day'
+    group by repo
+    order by count(*) desc
+    limit 20
+  `)
+  console.table(byRepo)
+}
+
+main()
+  .then(() => process.exit(0))
+  .catch((err) => {
+    console.error(err)
+    process.exit(1)
+  })

scripts/inspect-content-cache.ts

@@ -0,0 +1,42 @@
+import { eq, sql } from 'drizzle-orm'
+import { db } from '../src/db/client'
+import { githubContentCache } from '../src/db/schema'
+
+/**
+ * One-shot cleanup: delete every negative cache row (is_present = false).
+ *
+ * The daily prune fn keeps negatives for 1 day going forward, but the
+ * existing rows had their updated_at bumped by mark*Stale so they all
+ * look fresh. They're safe to drop wholesale — anything still being
+ * requested will repopulate within 15 minutes (the negative TTL), and
+ * most are bogus paths that won't be requested again.
+ */
+async function main() {
+  const startTime = Date.now()
+
+  const [{ before }] = await db
+    .select({ before: sql<number>`count(*)::int` })
+    .from(githubContentCache)
+  console.log(`[prune-negatives] rows before: ${before}`)
+
+  const deleted = await db
+    .delete(githubContentCache)
+    .where(eq(githubContentCache.isPresent, false))
+    .returning({ id: githubContentCache.id })
+
+  const [{ after }] = await db
+    .select({ after: sql<number>`count(*)::int` })
+    .from(githubContentCache)
+
+  const duration = Date.now() - startTime
+  console.log(
+    `[prune-negatives] ✓ Deleted ${deleted.length} negative rows in ${duration}ms (rows after: ${after})`,
+  )
+}
+
+main()
+  .then(() => process.exit(0))
+  .catch((err) => {
+    console.error('[prune-negatives] ✗ Failed:', err)
+    process.exit(1)
+  })