feat(e2e): lazy app init + playwright sharding for integration tests

.changeset/lazy-app-init-sharding.md

@@ -0,0 +1,2 @@
+---
+---

.github/workflows/ci.yml

@@ -275,7 +275,7 @@ jobs:
   integration-tests:
     needs: [check-permissions, build-packages]
     if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
-    name: Integration Tests (${{ matrix.test-name }}, ${{ matrix.test-project }}${{ matrix.next-version && format(', {0}', matrix.next-version) || '' }})
+    name: Integration Tests (${{ matrix.test-name }}, ${{ matrix.test-project }}${{ matrix.next-version && format(', {0}', matrix.next-version) || '' }}${{ matrix.shard && format(', shard {0}', matrix.shard) || '' }})
     permissions:
       contents: read
       actions: write # needed for actions/upload-artifact
@@ -315,9 +315,33 @@ jobs:
           - test-name: "nextjs"
             test-project: "chrome"
             next-version: "15"
+            shard: "1/3"
+            shard-label: "1-of-3"
+          - test-name: "nextjs"
+            test-project: "chrome"
+            next-version: "15"
+            shard: "2/3"
+            shard-label: "2-of-3"
+          - test-name: "nextjs"
+            test-project: "chrome"
+            next-version: "15"
+            shard: "3/3"
+            shard-label: "3-of-3"
+          - test-name: "nextjs"
+            test-project: "chrome"
+            next-version: "16"
+            shard: "1/3"
+            shard-label: "1-of-3"
+          - test-name: "nextjs"
+            test-project: "chrome"
+            next-version: "16"
+            shard: "2/3"
+            shard-label: "2-of-3"
           - test-name: "nextjs"
             test-project: "chrome"
             next-version: "16"
+            shard: "3/3"
+            shard-label: "3-of-3"
           - test-name: "quickstart"
             test-project: "chrome"
             next-version: "15"
@@ -365,6 +389,7 @@ jobs:
           E2E_NEXTJS_VERSION: ${{ matrix.next-version }}
           E2E_PROJECT: ${{ matrix.test-project }}
           INTEGRATION_INSTANCE_KEYS: ${{ secrets.INTEGRATION_INSTANCE_KEYS }}
+          PLAYWRIGHT_SHARD: ${{ matrix.shard || '' }}
         run: |
           # Use turbo's built-in --affected flag to detect changes
           # This automatically uses GITHUB_BASE_REF in GitHub Actions
@@ -449,13 +474,14 @@ jobs:
           E2E_CLERK_ENCRYPTION_KEY: ${{ matrix.clerk-encryption-key }}
           INTEGRATION_INSTANCE_KEYS: ${{ secrets.INTEGRATION_INSTANCE_KEYS }}
           NODE_EXTRA_CA_CERTS: ${{ github.workspace }}/integration/certs/rootCA.pem
+          PLAYWRIGHT_SHARD: ${{ matrix.shard || '' }}
           VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }}
 
       - name: Upload test-results
         if: ${{ cancelled() || failure() }}
         uses: actions/upload-artifact@v4
         with:
-          name: playwright-traces-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.test-name }}${{ matrix.next-version && format('-next{0}', matrix.next-version) || '' }}
+          name: playwright-traces-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.test-name }}${{ matrix.next-version && format('-next{0}', matrix.next-version) || '' }}${{ matrix.shard-label && format('-shard-{0}', matrix.shard-label) || '' }}
           path: integration/test-results
           retention-days: 1
 

integration/models/longRunningApplication.ts

@@ -1,7 +1,7 @@
 import { parsePublishableKey } from '@clerk/shared/keys';
 import { clerkSetup } from '@clerk/testing/playwright';
 
-import { awaitableTreekill, fs } from '../scripts';
+import { acquireProcessLock, awaitableTreekill, fs } from '../scripts';
 import type { Application } from './application';
 import type { ApplicationConfig } from './applicationConfig';
 import type { EnvironmentConfig } from './environment';
@@ -16,6 +16,18 @@ const getPort = (_url: string) => {
   return Number.parseInt(url.port || (url.protocol === 'https:' ? '443' : '80'));
 };
 
+/**
+ * Check if a server is responding at the given URL.
+ */
+const isServerReady = async (url: string): Promise<boolean> => {
+  try {
+    const res = await fetch(url);
+    return res.ok;
+  } catch {
+    return false;
+  }
+};
+
 export type LongRunningApplication = ReturnType<typeof longRunningApplication>;
 export type LongRunningApplicationParams = {
   id: string;
@@ -29,7 +41,8 @@ export type LongRunningApplicationParams = {
  * Its interface is the same as the Application and the ApplicationConfig interface,
  * making it interchangeable with the Application and ApplicationConfig.
  *
- * After init() is called, all mutating methods on the config are ignored.
+ * init() is lazy and idempotent: it checks the state file first, and uses
+ * file-based locking to ensure only one process initializes each app.
  */
 export const longRunningApplication = (params: LongRunningApplicationParams) => {
   const { id } = params;
@@ -54,92 +67,195 @@ export const longRunningApplication = (params: LongRunningApplicationParams) =>
     env ||= environmentConfig().fromJson(data.env);
   };
 
+  /**
+   * Try to adopt an already-running app from the state file.
+   * Returns true if the app is running and state was loaded.
+   */
+  const tryAdoptFromStateFile = async (): Promise<boolean> => {
+    try {
+      const apps = stateFile.getLongRunningApps();
+      const data = apps?.[id];
+      if (!data?.serverUrl) {
+        return false;
+      }
+      const ready = await isServerReady(data.serverUrl);
+      if (ready) {
+        port = data.port;
+        serverUrl = data.serverUrl;
+        pid = data.pid;
+        appDir = data.appDir;
+        env = params.env;
+        // Propagate testing tokens to this worker process so that
+        // setupClerkTestingToken() can bypass bot protection.
+        if (data.clerkFapi) {
+          process.env.CLERK_FAPI = data.clerkFapi;
+        }
+        if (data.clerkTestingToken) {
+          process.env.CLERK_TESTING_TOKEN = data.clerkTestingToken;
+        }
+        return true;
+      }
+      return false;
+    } catch {
+      // State file may be partially written by another process — not an error
+      return false;
+    }
+  };
+
+  /**
+   * Perform the full app initialization: testing tokens, commit, install, build, serve.
+   */
+  const doFullInit = async () => {
+    const log = (msg: string) => console.log(`[${name}] ${msg}`);
+    log('Starting full init...');
+
+    try {
+      const publishableKey = params.env.publicVariables.get('CLERK_PUBLISHABLE_KEY');
+      const secretKey = params.env.privateVariables.get('CLERK_SECRET_KEY');
+      const apiUrl = params.env.privateVariables.get('CLERK_API_URL');
+      const { instanceType, frontendApi: frontendApiUrl } = parsePublishableKey(publishableKey);
+
+      if (instanceType !== 'development') {
+        log('Skipping setup of testing tokens for non-development instance');
+      } else {
+        log('Setting up testing tokens...');
+        await clerkSetup({
+          publishableKey,
+          frontendApiUrl,
+          secretKey,
+          // @ts-expect-error apiUrl is not a typed option for clerkSetup, but it is accepted at runtime.
+          apiUrl,
+          dotenv: false,
+        });
+        log('Testing tokens setup complete');
+      }
+    } catch (error) {
+      console.error('Error setting up testing tokens:', error);
+      throw error;
+    }
+
+    try {
+      log('Committing config...');
+      app = await config.commit();
+      log(`Config committed, appDir: ${app.appDir}`);
+    } catch (error) {
+      console.error('Error committing config:', error);
+      throw error;
+    }
+
+    try {
+      await app.withEnv(params.env);
+    } catch (error) {
+      console.error('Error setting up environment:', error);
+      throw error;
+    }
+
+    try {
+      log('Running setup (pnpm install)...');
+      await app.setup();
+      log('Setup complete');
+    } catch (error) {
+      console.error('Error during app setup:', error);
+      throw error;
+    }
+
+    try {
+      log('Building app...');
+      const buildTimeout = new Promise((_, reject) =>
+        setTimeout(() => reject(new Error(`Build timed out after 120s for ${name}`)), 120_000),
+      );
+      await Promise.race([app.build(), buildTimeout]);
+      log('Build complete');
+    } catch (error) {
+      console.error('Error during app build:', error);
+      throw error;
+    }
+
+    try {
+      log('Starting serve (detached)...');
+      const serveResult = await app.serve({ detached: true });
+      port = serveResult.port;
+      serverUrl = serveResult.serverUrl;
+      pid = serveResult.pid;
+      appDir = app.appDir;
+      log(`Serve complete: port=${port}, serverUrl=${serverUrl}, pid=${pid}`);
+      // Serialize state file writes across all apps to prevent concurrent
+      // read-modify-write from clobbering entries written by other workers.
+      const releaseStateFileLock = await acquireProcessLock('__state-file__');
+      try {
+        stateFile.addLongRunningApp({
+          port,
+          serverUrl,
+          pid,
+          id,
+          appDir,
+          env: params.env.toJson(),
+          clerkFapi: process.env.CLERK_FAPI,
+          clerkTestingToken: process.env.CLERK_TESTING_TOKEN,
+        });
+      } finally {
+        releaseStateFileLock();
+      }
+    } catch (error) {
+      console.error('Error during app serve:', error);
+      throw error;
+    }
+  };
+
   const self = new Proxy(
     {
-      // will be called by global.setup.ts and by the test runner
-      // the first time this is called, the app starts and the state is persisted in the state file
+      /**
+       * Lazy, idempotent init. Safe to call from multiple Playwright workers.
+       * - If the app is already running (found in state file + server responds), reuses it.
+       * - Otherwise, acquires a file lock and initializes. Other workers wait for the lock.
+       */
       init: async () => {
         const log = (msg: string) => console.log(`[${name}] ${msg}`);
-        log('Starting init...');
-        try {
-          const publishableKey = params.env.publicVariables.get('CLERK_PUBLISHABLE_KEY');
-          const secretKey = params.env.privateVariables.get('CLERK_SECRET_KEY');
-          const apiUrl = params.env.privateVariables.get('CLERK_API_URL');
-          const { instanceType, frontendApi: frontendApiUrl } = parsePublishableKey(publishableKey);
-
-          if (instanceType !== 'development') {
-            log('Skipping setup of testing tokens for non-development instance');
-          } else {
-            log('Setting up testing tokens...');
-            await clerkSetup({
-              publishableKey,
-              frontendApiUrl,
-              secretKey,
-              // @ts-expect-error apiUrl is not a typed option for clerkSetup, but it is accepted at runtime.
-              apiUrl,
-              dotenv: false,
-            });
-            log('Testing tokens setup complete');
-          }
-        } catch (error) {
-          console.error('Error setting up testing tokens:', error);
-          throw error;
-        }
-        try {
-          log('Committing config...');
-          app = await config.commit();
-          log(`Config committed, appDir: ${app.appDir}`);
-        } catch (error) {
-          console.error('Error committing config:', error);
-          throw error;
-        }
-        try {
-          await app.withEnv(params.env);
-        } catch (error) {
-          console.error('Error setting up environment:', error);
-          throw error;
-        }
-        try {
-          log('Running setup (pnpm install)...');
-          await app.setup();
-          log('Setup complete');
-        } catch (error) {
-          console.error('Error during app setup:', error);
-          throw error;
+
+        // Fast path: already initialized in this process
+        if (serverUrl && (await isServerReady(serverUrl))) {
+          log('Already initialized in this process');
+          return;
         }
-        try {
-          log('Building app...');
-          const buildTimeout = new Promise((_, reject) =>
-            setTimeout(() => reject(new Error(`Build timed out after 120s for ${name}`)), 120_000),
-          );
-          await Promise.race([app.build(), buildTimeout]);
-          log('Build complete');
-        } catch (error) {
-          console.error('Error during app build:', error);
-          throw error;
+
+        // Check if another process already initialized this app
+        if (await tryAdoptFromStateFile()) {
+          log(`Adopted from state file: ${serverUrl}`);
+          return;
         }
+
+        // Need to initialize — acquire lock to prevent duplicate work
+        log('Acquiring init lock...');
+        const releaseLock = await acquireProcessLock(id);
         try {
-          log('Starting serve (detached)...');
-          const serveResult = await app.serve({ detached: true });
-          port = serveResult.port;
-          serverUrl = serveResult.serverUrl;
-          pid = serveResult.pid;
-          appDir = app.appDir;
-          log(`Serve complete: port=${port}, serverUrl=${serverUrl}, pid=${pid}`);
-          stateFile.addLongRunningApp({ port, serverUrl, pid, id, appDir, env: params.env.toJson() });
-        } catch (error) {
-          console.error('Error during app serve:', error);
-          throw error;
+          // Double-check after acquiring lock (another process may have finished while we waited)
+          if (await tryAdoptFromStateFile()) {
+            log(`Adopted from state file after lock: ${serverUrl}`);
+            return;
+          }
+
+          // We hold the lock and the app is not running — do full init
+          await doFullInit();
+        } finally {
+          releaseLock();
         }
       },
       // will be called by global.teardown.ts
       destroy: async () => {
         readFromStateFile();
+        if (!pid && !appDir) {
+          console.log(`Skipping destroy for ${name}: no pid or appDir`);
+          return;
+        }
         console.log(`Destroying ${serverUrl}`);
-        await awaitableTreekill(pid, 'SIGKILL');
-        // TODO: Test whether this is necessary now that we have awaitableTreekill
-        await new Promise(res => setTimeout(res, 2000));
-        await fs.rm(appDir, { recursive: true, force: true });
+        if (pid) {
+          await awaitableTreekill(pid, 'SIGKILL');
+          // TODO: Test whether this is necessary now that we have awaitableTreekill
+          await new Promise(res => setTimeout(res, 2000));
+        }
+        if (appDir) {
+          await fs.rm(appDir, { recursive: true, force: true });
+        }
       },
       // read the persisted state and behave like an app
       commit: () => {

integration/models/stateFile.ts

@@ -9,6 +9,8 @@ export type AppParams = {
   pid?: number;
   appDir: string;
   env: ReturnType<EnvironmentConfig['toJson']>;
+  clerkFapi?: string;
+  clerkTestingToken?: string;
 };
 
 type StandaloneAppParams = {