Restore test reporter and fix flaky e2e test issues (#97)

* restore test reporter and fix flaky e2e test issues

* restore test reporter

* fixing the flaky test

* fixing error

* restore test reporter

Author: charithaT07

.github/workflows/e2e-tests.yml

@@ -38,7 +38,7 @@ jobs:
         node-version: [22.x]
     runs-on: ubuntu-latest
     environment: e2e-dev
-    timeout-minutes: 25
+    timeout-minutes: 40
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
@@ -110,8 +110,8 @@ jobs:
           echo "Running E2E tests with realm: ${TEST_REALM}"
           echo "Node version: $(node --version)"
           
-          # Run E2E tests with JSON reporter for test results
-          pnpm run test:e2e && pnpm run lint
+          # Run E2E tests with CI reporter (outputs test-results.json)
+          pnpm run test:e2e:ci && pnpm run lint
 
       - name: E2E Test Report
         uses: dorny/test-reporter@fe45e9537387dac839af0d33ba56eed8e24189e8  # v2.3.0

packages/b2c-cli/package.json

@@ -265,7 +265,8 @@
     "test:ci": "c8 env OCLIF_TEST_ROOT=. mocha --forbid-only --exclude \"test/functional/e2e/**\" --reporter json --reporter-option output=test-results.json \"test/**/*.test.ts\"",
     "test:unit": "env OCLIF_TEST_ROOT=. mocha --forbid-only --exclude \"test/functional/e2e/**\" \"test/**/*.test.ts\"",
     "test:agent": "env OCLIF_TEST_ROOT=. mocha --forbid-only --reporter min --exclude \"test/functional/e2e/**\" \"test/**/*.test.ts\"",
-    "test:e2e": "env TEST_USE_SHARED_SANDBOX=true OCLIF_TEST_ROOT=. mocha --forbid-only --config test/functional/e2e/.mocharc.json \"test/functional/e2e/**/*.test.ts\"",
+    "test:e2e": "env TEST_USE_SHARED_SANDBOX=true OCLIF_TEST_ROOT=. mocha --forbid-only --require test/functional/e2e/hooks.ts --node-option import=tsx --timeout 30000 --retries 2 --reporter spec \"test/functional/e2e/**/*.test.ts\"",
+    "test:e2e:ci": "env TEST_USE_SHARED_SANDBOX=true OCLIF_TEST_ROOT=. mocha --forbid-only --require test/functional/e2e/hooks.ts --node-option import=tsx --timeout 30000 --retries 2 --reporter json --reporter-option output=test-results.json \"test/functional/e2e/**/*.test.ts\"",
     "test:e2e:auth": "env OCLIF_TEST_ROOT=. mocha --forbid-only \"test/functional/e2e/auth-token.test.ts\"",
     "test:e2e:code": "env OCLIF_TEST_ROOT=. mocha --forbid-only \"test/functional/e2e/code-lifecycle.test.ts\"",
     "test:e2e:jobs": "env OCLIF_TEST_ROOT=. mocha --forbid-only \"test/functional/e2e/job-execution.test.ts\"",

packages/b2c-cli/test/functional/e2e/.mocharc.json

@@ -5,40 +5,21 @@
  */
 
 import {expect} from 'chai';
-import {execa} from 'execa';
-import path from 'node:path';
-import {fileURLToPath} from 'node:url';
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
+import {parseJSONOutput, runCLIWithRetry, TIMEOUTS, toString} from './test-utils.js';
 
 /**
  * E2E Tests for Authentication Token Generation
  */
 describe('Auth Token E2E Tests', function () {
-  this.timeout(120_000); // 2 minutes
+  this.timeout(TIMEOUTS.AUTH * 8); // 2 minutes
   this.retries(2);
 
-  const CLI_BIN = path.resolve(__dirname, '../../../bin/run.js');
-
   before(function () {
     if (!process.env.SFCC_CLIENT_ID || !process.env.SFCC_CLIENT_SECRET) {
       this.skip();
     }
   });
 
-  async function runCLI(args: string[], env?: Record<string, string>) {
-    const result = await execa('node', [CLI_BIN, ...args], {
-      env: {
-        ...process.env,
-        ...env,
-        SFCC_LOG_LEVEL: 'silent',
-      },
-      reject: false,
-    });
-    return result;
-  }
-
   function decodeJWT(token: string): Record<string, unknown> {
     const parts = token.split('.');
     if (parts.length !== 3) {
@@ -49,11 +30,11 @@ describe('Auth Token E2E Tests', function () {
   }
 
   it('should generate a valid OAuth token with correct format, scopes, and expiration', async function () {
-    const result = await runCLI(['auth:token', '--json']);
-    expect(result.exitCode).to.equal(0, `Token generation failed: ${result.stderr}`);
+    const result = await runCLIWithRetry(['auth:token', '--json'], {timeout: TIMEOUTS.AUTH, verbose: true});
+    expect(result.exitCode, `Token generation failed: ${toString(result.stderr)}`).to.equal(0);
     expect(result.stdout).to.not.be.empty;
 
-    const response = JSON.parse(result.stdout);
+    const response = parseJSONOutput(result);
     expect(response).to.be.an('object');
     expect(response.accessToken).to.be.a('string').and.not.be.empty;
     expect(response.expires).to.be.a('string');
@@ -86,11 +67,13 @@ describe('Auth Token E2E Tests', function () {
       // Use only scopes your client actually has
       const extraScopes = ['profile', 'roles'];
 
-      const result = await runCLI(['auth:token', '--scope', extraScopes.join(','), '--json']);
+      const result = await runCLIWithRetry(['auth:token', '--scope', extraScopes.join(','), '--json'], {
+        timeout: TIMEOUTS.AUTH,
+      });
 
-      expect(result.exitCode).to.equal(0, `Token generation with extra scopes failed: ${result.stderr}`);
+      expect(result.exitCode, `Token generation with extra scopes failed: ${toString(result.stderr)}`).to.equal(0);
 
-      const response = JSON.parse(result.stdout);
+      const response = parseJSONOutput(result);
       const accessToken = response.accessToken as string;
       expect(accessToken).to.be.a('string').and.not.be.empty;
       expect(response.scopes).to.include.members(extraScopes);
@@ -110,9 +93,12 @@ describe('Auth Token E2E Tests', function () {
 
   describe('Invalid Credentials', function () {
     it('should fail with invalid client credentials', async function () {
-      const result = await runCLI(['auth:token', '--json'], {
-        SFCC_CLIENT_ID: 'invalid-client-id',
-        SFCC_CLIENT_SECRET: 'invalid-client-secret',
+      const result = await runCLIWithRetry(['auth:token', '--json'], {
+        timeout: TIMEOUTS.AUTH,
+        env: {
+          SFCC_CLIENT_ID: 'invalid-client-id',
+          SFCC_CLIENT_SECRET: 'invalid-client-secret',
+        },
       });
 
       expect(result.exitCode).to.not.equal(0);
@@ -123,23 +109,23 @@ describe('Auth Token E2E Tests', function () {
 
   describe('JSON Output Structure', function () {
     it('should return correct JSON keys', async function () {
-      const result = await runCLI(['auth:token', '--json']);
-      const response = JSON.parse(result.stdout);
+      const result = await runCLIWithRetry(['auth:token', '--json'], {timeout: TIMEOUTS.AUTH});
+      const response = parseJSONOutput(result);
       expect(response).to.have.all.keys('accessToken', 'expires', 'scopes');
     });
   });
 
   describe('Default Scopes', function () {
     it('should return default scopes when no scopes are requested', async function () {
-      const result = await runCLI(['auth:token', '--json']);
-      const response = JSON.parse(result.stdout);
+      const result = await runCLIWithRetry(['auth:token', '--json'], {timeout: TIMEOUTS.AUTH});
+      const response = parseJSONOutput(result) as {scopes: string[]};
       expect(response.scopes.length).to.be.greaterThan(0);
     });
   });
 
   describe('Non-JSON Output', function () {
     it('should output raw token in non-JSON mode', async function () {
-      const result = await runCLI(['auth:token']);
+      const result = await runCLIWithRetry(['auth:token'], {timeout: TIMEOUTS.AUTH});
       expect(result.exitCode).to.equal(0);
       expect(result.stdout).to.match(/^ey[A-Za-z0-9-_]+\.[A-Za-z0-9-_]+\.[A-Za-z0-9-_]+$/); // JWT regex
     });

packages/b2c-cli/test/functional/e2e/auth-token.test.ts

@@ -10,12 +10,13 @@ import * as fs from 'node:fs/promises';
 import path from 'node:path';
 import {fileURLToPath} from 'node:url';
 import {getSharedContext, hasSharedSandbox} from './shared-context.js';
+import {getSandboxId, getHostname, parseJSONOutput, runCLIWithRetry, TIMEOUTS, toString} from './test-utils.js';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 
 describe('Code Lifecycle E2E Tests', function () {
-  this.timeout(900_000);
+  this.timeout(TIMEOUTS.CODE_DEPLOY * 3); // 15 minutes
   this.retries(2);
 
   const CLI_BIN = path.resolve(__dirname, '../../../bin/run.js');
@@ -42,33 +43,25 @@ describe('Code Lifecycle E2E Tests', function () {
     } else {
       // Fallback: Create own sandbox
       console.log('No shared sandbox available, creating dedicated sandbox for Code tests...');
-      this.timeout(720_000); // 12 minutes for sandbox creation
+      this.timeout(TIMEOUTS.ODS_OPERATION);
 
       if (!process.env.TEST_REALM) {
         throw new Error('TEST_REALM required to create sandbox');
       }
 
-      const result = await runCLI(
+      const result = await runCLIWithRetry(
         ['ods', 'create', '--realm', process.env.TEST_REALM, '--ttl', '4', '--wait', '--set-permissions', '--json'],
-        {timeout: 720_000},
+        {timeout: TIMEOUTS.ODS_OPERATION, verbose: true},
       );
 
-      expect(result.exitCode).to.equal(0, `Failed to create sandbox: ${result.stderr}`);
-      const sandbox = JSON.parse(result.stdout);
-      ownSandboxId = sandbox.id;
-      serverHostname = sandbox.hostName;
-      console.log(`Created dedicated sandbox ${ownSandboxId} at ${serverHostname}`);
+      expect(result.exitCode, `Failed to create sandbox: ${toString(result.stderr)}`).to.equal(0);
+      const sandbox = parseJSONOutput(result);
+      ownSandboxId = getSandboxId(sandbox);
+      serverHostname = getHostname(sandbox);
+      console.log(`  ✓ Created dedicated sandbox ${ownSandboxId} at ${serverHostname}`);
     }
   });
 
-  async function runCLI(args: string[], options: {timeout?: number} = {}) {
-    return execa('node', [CLI_BIN, ...args], {
-      env: {...process.env, SFCC_LOG_LEVEL: 'silent'},
-      reject: false,
-      timeout: options.timeout,
-    });
-  }
-
   after(async function () {
     this.timeout(180_000); // 3 minutes for cleanup
 
@@ -78,13 +71,13 @@ describe('Code Lifecycle E2E Tests', function () {
 
     // Delete remaining code versions
     if (codeVersionB && serverHostname) {
-      await runCLI(['code', 'delete', codeVersionB, '--server', serverHostname, '--force']);
+      await runCLIWithRetry(['code', 'delete', codeVersionB, '--server', serverHostname, '--force']);
     }
 
     // Delete own sandbox if we created one
     if (ownSandboxId) {
       console.log(`Cleaning up dedicated sandbox ${ownSandboxId}...`);
-      await runCLI(['ods', 'delete', ownSandboxId, '--force']);
+      await runCLIWithRetry(['ods', 'delete', ownSandboxId, '--force']);
       console.log('Dedicated sandbox deleted');
     }
   });
@@ -94,7 +87,7 @@ describe('Code Lifecycle E2E Tests', function () {
     it('should deploy first code version', async function () {
       codeVersionA = `e2e-a-${Date.now()}`;
 
-      const result = await runCLI([
+      const result = await runCLIWithRetry([
         'code',
         'deploy',
         CARTRIDGES_DIR,
@@ -105,24 +98,24 @@ describe('Code Lifecycle E2E Tests', function () {
         '--json',
       ]);
 
-      expect(result.exitCode).to.equal(0, result.stderr);
+      expect(result.exitCode).to.equal(0, toString(result.stderr));
     });
   });
 
   describe('Step 2: Verify Code Version A in List', function () {
     it('should find code version A in list', async function () {
-      const result = await runCLI(['code', 'list', '--server', serverHostname, '--json']);
+      const result = await runCLIWithRetry(['code', 'list', '--server', serverHostname, '--json']);
       expect(result.exitCode).to.equal(0);
 
-      const response = JSON.parse(result.stdout);
+      const response = parseJSONOutput(result);
       const found = response.data.find((v: any) => v.id === codeVersionA);
       expect(found).to.exist;
     });
   });
 
   describe('Step 3: Activate Code Version A', function () {
     it('should activate version A', async function () {
-      const result = await runCLI(['code', 'activate', codeVersionA, '--server', serverHostname, '--json']);
+      const result = await runCLIWithRetry(['code', 'activate', codeVersionA, '--server', serverHostname, '--json']);
 
       expect(result.exitCode).to.equal(0);
     });
@@ -132,7 +125,7 @@ describe('Code Lifecycle E2E Tests', function () {
     it('should deploy second code version', async function () {
       codeVersionB = `e2e-b-${Date.now()}`;
 
-      const result = await runCLI([
+      const result = await runCLIWithRetry([
         'code',
         'deploy',
         CARTRIDGES_DIR,
@@ -149,27 +142,27 @@ describe('Code Lifecycle E2E Tests', function () {
 
   describe('Step 5: Verify Code Version B in List', function () {
     it('should find code version B in list', async function () {
-      const result = await runCLI(['code', 'list', '--server', serverHostname, '--json']);
+      const result = await runCLIWithRetry(['code', 'list', '--server', serverHostname, '--json']);
       expect(result.exitCode).to.equal(0);
 
-      const response = JSON.parse(result.stdout);
+      const response = parseJSONOutput(result);
       const found = response.data.find((v: any) => v.id === codeVersionB);
       expect(found).to.exist;
     });
   });
 
   describe('Step 6: Activate Code Version B', function () {
     it('should activate version B (A becomes inactive)', async function () {
-      const result = await runCLI(['code', 'activate', codeVersionB, '--server', serverHostname, '--json']);
+      const result = await runCLIWithRetry(['code', 'activate', codeVersionB, '--server', serverHostname, '--json']);
 
       expect(result.exitCode).to.equal(0);
     });
   });
 
   describe('Step 7: Verify Active Code Version', function () {
     it('should show version B as active', async function () {
-      const result = await runCLI(['code', 'list', '--server', serverHostname, '--json']);
-      const response = JSON.parse(result.stdout);
+      const result = await runCLIWithRetry(['code', 'list', '--server', serverHostname, '--json']);
+      const response = parseJSONOutput(result);
 
       const active = response.data.find((v: any) => v.active === true);
       expect(active.id).to.equal(codeVersionB);
@@ -208,21 +201,24 @@ describe('Code Lifecycle E2E Tests', function () {
     it('should delete inactive version A', async function () {
       console.log(`Starting deletion of code version: ${codeVersionA}`);
 
-      const result = await runCLI(['code', 'delete', codeVersionA, '--server', serverHostname, '--force', '--json'], {
-        timeout: 120_000,
-      }); // 2 minutes timeout
+      const result = await runCLIWithRetry(
+        ['code', 'delete', codeVersionA, '--server', serverHostname, '--force', '--json'],
+        {
+          timeout: 120_000,
+        },
+      ); // 2 minutes timeout
 
       console.log(`Deletion finished with exit code: ${result.exitCode}`);
 
-      expect(result.exitCode).to.equal(0, `Delete failed: ${result.stderr}`);
+      expect(result.exitCode).to.equal(0, `Delete failed: ${toString(result.stderr)}`);
       codeVersionA = '';
     });
   });
 
   describe('Step 10: Verify Code Version A Removed', function () {
     it('should not find deleted version A', async function () {
-      const result = await runCLI(['code', 'list', '--server', serverHostname, '--json']);
-      const response = JSON.parse(result.stdout);
+      const result = await runCLIWithRetry(['code', 'list', '--server', serverHostname, '--json']);
+      const response = parseJSONOutput(result);
 
       const found = response.data.find((v: any) => v.id === codeVersionA);
       expect(found).to.not.exist;