Merge pull request #19815 from Microsoft/add-definitely-typed-runner

Add DefinitelyTyped runner

Author: sandersn

Jakefile.js

@@ -105,7 +105,7 @@ var harnessCoreSources = [
     "projectsRunner.ts",
     "loggedIO.ts",
     "rwcRunner.ts",
-    "userRunner.ts",
+    "externalCompileRunner.ts",
     "test262Runner.ts",
     "./parallel/shared.ts",
     "./parallel/host.ts",

src/harness/externalCompileRunner.ts

@@ -0,0 +1,67 @@
+/// <reference path="harness.ts"/>
+/// <reference path="runnerbase.ts" />
+abstract class ExternalCompileRunnerBase extends RunnerBase {
+    abstract testDir: string;
+    public enumerateTestFiles() {
+        return Harness.IO.getDirectories(this.testDir);
+    }
+    /** Setup the runner's tests so that they are ready to be executed by the harness
+     *  The first test should be a describe/it block that sets up the harness's compiler instance appropriately
+     */
+    public initializeTests(): void {
+        // Read in and evaluate the test list
+        const testList = this.tests && this.tests.length ? this.tests : this.enumerateTestFiles();
+
+        describe(`${this.kind()} code samples`, () => {
+            for (const test of testList) {
+                this.runTest(test);
+            }
+        });
+    }
+    private runTest(directoryName: string) {
+        describe(directoryName, () => {
+            const cp = require("child_process");
+            const path = require("path");
+            const fs = require("fs");
+
+            it("should build successfully", () => {
+                const cwd = path.join(__dirname, "../../", this.testDir, directoryName);
+                const timeout = 600000; // 600s = 10 minutes
+                if (fs.existsSync(path.join(cwd, "package.json"))) {
+                    if (fs.existsSync(path.join(cwd, "package-lock.json"))) {
+                        fs.unlinkSync(path.join(cwd, "package-lock.json"));
+                    }
+                    const stdio = isWorker ? "pipe" : "inherit";
+                    const install = cp.spawnSync(`npm`, ["i"], { cwd, timeout, shell: true, stdio });
+                    if (install.status !== 0) throw new Error(`NPM Install for ${directoryName} failed!`);
+                }
+                Harness.Baseline.runBaseline(`${this.kind()}/${directoryName}.log`, () => {
+                    const result = cp.spawnSync(`node`, [path.join(__dirname, "tsc.js")], { cwd, timeout, shell: true });
+                    // tslint:disable-next-line:no-null-keyword
+                    return result.status === 0 && !result.stdout.length && !result.stderr.length ? null : `Exit Code: ${result.status}
+Standard output:
+${result.stdout.toString().replace(/\r\n/g, "\n")}
+
+
+Standard error:
+${result.stderr.toString().replace(/\r\n/g, "\n")}`;
+                });
+            });
+        });
+    }
+}
+
+class UserCodeRunner extends ExternalCompileRunnerBase {
+    public readonly testDir = "tests/cases/user/";
+    public kind(): TestRunnerKind {
+        return "user";
+    }
+}
+
+class DefinitelyTypedRunner extends ExternalCompileRunnerBase {
+    public readonly testDir = "../DefinitelyTyped/types/";
+    public workingDirectory = this.testDir;
+    public kind(): TestRunnerKind {
+        return "dt";
+    }
+}

src/harness/parallel/host.ts

@@ -77,18 +77,18 @@ namespace Harness.Parallel.Host {
         console.log("Discovering runner-based tests...");
         const discoverStart = +(new Date());
         const { statSync }: { statSync(path: string): { size: number }; } = require("fs");
+        const path: { join: (...args: string[]) => string } = require("path");
         for (const runner of runners) {
-            const files = runner.enumerateTestFiles();
-            for (const file of files) {
+            for (const file of runner.enumerateTestFiles()) {
                 let size: number;
                 if (!perfData) {
                     try {
-                        size = statSync(file).size;
+                        size = statSync(path.join(runner.workingDirectory, file)).size;
                     }
                     catch {
                         // May be a directory
                         try {
-                            size = Harness.IO.listFiles(file, /.*/g, { recursive: true }).reduce((acc, elem) => acc + statSync(elem).size, 0);
+                            size = Harness.IO.listFiles(path.join(runner.workingDirectory, file), /.*/g, { recursive: true }).reduce((acc, elem) => acc + statSync(elem).size, 0);
                         }
                         catch {
                             // Unknown test kind, just return 0 and let the historical analysis take over after one run

src/harness/runner.ts

@@ -18,7 +18,7 @@
 /// <reference path="fourslashRunner.ts" />
 /// <reference path="projectsRunner.ts" />
 /// <reference path="rwcRunner.ts" />
-/// <reference path="userRunner.ts" />
+/// <reference path="externalCompileRunner.ts" />
 /// <reference path="harness.ts" />
 /// <reference path="./parallel/shared.ts" />
 
@@ -62,6 +62,8 @@ function createRunner(kind: TestRunnerKind): RunnerBase {
             return new Test262BaselineRunner();
         case "user":
             return new UserCodeRunner();
+        case "dt":
+            return new DefinitelyTypedRunner();
     }
     ts.Debug.fail(`Unknown runner kind ${kind}`);
 }
@@ -183,6 +185,9 @@ function handleTestConfig() {
                     case "user":
                         runners.push(new UserCodeRunner());
                         break;
+                    case "dt":
+                        runners.push(new DefinitelyTypedRunner());
+                        break;
                 }
             }
         }

src/harness/runnerbase.ts

@@ -1,7 +1,7 @@
 /// <reference path="harness.ts" />
 
 
-type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | "test262" | "user";
+type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | "test262" | "user" | "dt";
 type CompilerTestKind = "conformance" | "compiler";
 type FourslashTestKind = "fourslash" | "fourslash-shims" | "fourslash-shims-pp" | "fourslash-server";
 
@@ -22,6 +22,9 @@ abstract class RunnerBase {
 
     abstract enumerateTestFiles(): string[];
 
+    /** The working directory where tests are found. Needed for batch testing where the input path will differ from the output path inside baselines */
+    public workingDirectory = "";
+
     /** Setup the runner's tests so that they are ready to be executed by the harness
      *  The first test should be a describe/it block that sets up the harness's compiler instance appropriately
      */

src/harness/tsconfig.json

@@ -83,7 +83,7 @@
         "projectsRunner.ts",
         "loggedIO.ts",
         "rwcRunner.ts",
-        "userRunner.ts",
+        "externalCompileRunner.ts",
         "test262Runner.ts",
         "./parallel/shared.ts",
         "./parallel/host.ts",