General fixes (#546)

- Rename `maxAsyncE2EResponseWaitTime` to `maxResponseWaitTime`
- `virtualDeviceConfig` over `extraParameters`, merging props having `virtualDeviceConfig` as priority
- HTML-report under timestamp output when `traceOutput` is true
- Report dir name default to test_file_name if `description` is empty

Author: german1311

docs/api/TestSuite.html

@@ -28,7 +28,7 @@ module.exports = class Configuration {
         Configuration.singleton = undefined;
     }
 
-    static instance()  {
+    static instance() {
         return Configuration.singleton;
     }
 
@@ -143,12 +143,12 @@ module.exports = class Configuration {
         if (cliOverrides && (cliOverrides["context"] || cliOverrides["config"])) {
             const contextOverride = cliOverrides["context"] ||
                 path.dirname(cliOverrides["config"]);
-            
+
             let contextPath = "";
             if (contextOverride) {
                 contextPath = path.isAbsolute(contextOverride) ? contextOverride : path.join(process.cwd(), contextOverride);
-            } 
-            
+            }
+
             if (contextPath) {
                 jestConfig.roots = [contextPath];
             }
@@ -188,7 +188,7 @@ module.exports = class Configuration {
         return path.dirname(this.configurationJSON.configurationPath);
     }
 
-    htmlReporterConfiguration () {
+    htmlReporterConfiguration() {
         let pathToModule = path.join(__dirname, "../../node_modules/bespoken-jest-stare");
         // If we do not find jest-stare in this directory, means this module (skill-testing-ml) is a dependency
         // In that case, we keep going up to the node_modules that contains skill-testing-ml - jest-stare will be at that same level
@@ -267,10 +267,17 @@ module.exports = class Configuration {
     }
 
     resultsDirectory() {
-        const testResultsDir = path.join(process.cwd(), "test_output");
+        let testResultsDir = path.join(process.cwd(), "test_output");
+        const testTimestamp = this.value("runTimestamp", undefined, undefined);
+
+        if (this.value("traceOutput", undefined, false) && testTimestamp) {
+            testResultsDir = path.resolve(`./test_output/trace_output/${Util.formatTimeStamp(testTimestamp)}`);
+        }
+
         if (!fs.existsSync(testResultsDir)) {
-            fs.mkdirSync(testResultsDir);
+            fs.mkdirSync(testResultsDir, { recursive: true });
         }
+
         return testResultsDir;
     }
 

docs/api/TestSuite.js.html

@@ -80,7 +80,7 @@ const ConfigurationKeys = [
         text: "don't run the indicated tags",
     },
     {
-        key: "maxAsyncE2EResponseWaitTime",
+        key: "maxResponseWaitTime",
         text: "set the max time we wait for a single response in a e2e async interaction in ms, defaults to 15000",
     },
     {

lib/runner/Configuration.js

@@ -1,5 +1,6 @@
 const fs = require("fs");
 const path = require("path");
+const Util = require("../util/Util");
 const { camelCase, padStart, upperFirst } = require("lodash");
 
 const createDirectoriesForFile = (fileName) => {
@@ -15,14 +16,6 @@ const shortName = longName => longName.split(/[^a-z0-9]/ig)
     .map(p => upperFirst(camelCase(p)))
     .reduce((p, c) => p.length <= 200 ? p.concat(c) : p, "");
 
-const formatTimeStamp = (milliseconds) => {
-    const timestamp = new Date(milliseconds);
-    const date = `${timestamp.getFullYear()}${formatTwoDigits(timestamp.getMonth() + 1)}${formatTwoDigits(timestamp.getDate())}`;
-    const time = `${formatTwoDigits(timestamp.getHours())}${formatTwoDigits(timestamp.getMinutes())}${formatTwoDigits(timestamp.getSeconds())}`;
-
-    return `${date}_${time}`;
-};
-
 const formatTwoDigits = number => padStart(number, 2, "0");
 
 class TraceOutputWriter {
@@ -35,9 +28,11 @@ class TraceOutputWriter {
             const { description: testSuiteDescription, runTimestamp: timestamp } = testSuite;
             const utteranceOrder = allInteractions.findIndex(({ utterance }) => utterance === utteranceText);
 
+            const suite = testSuiteDescription || path.parse(testSuite.fileName).name;
+
             const requestPath = testSuite.resolvePath(
-                `./test_output/trace_output/${formatTimeStamp(timestamp)}`
-                + `/${shortName(testSuiteDescription)}`
+                `./test_output/trace_output/${Util.formatTimeStamp(timestamp)}`
+                + `/${shortName(suite)}`
                 + `/${shortName(testName)}`
                 + `/${formatTwoDigits(utteranceOrder)}-${shortName(utteranceText)}-res.json`
             );
@@ -55,9 +50,11 @@ class TraceOutputWriter {
             const { description: testSuiteDescription, runTimestamp: timestamp } = testSuite;
             const { test: { description: testName } } = interaction;
 
+            const suite = testSuiteDescription || path.parse(testSuite.fileName).name;
+
             const responsePath = testSuite.resolvePath(
-                `./test_output/trace_output/${formatTimeStamp(timestamp)}`
-                + `/${shortName(testSuiteDescription)}`
+                `./test_output/trace_output/${Util.formatTimeStamp(timestamp)}`
+                + `/${shortName(suite)}`
                 + `/${shortName(testName)}/req.json`
             );
 
@@ -71,7 +68,6 @@ class TraceOutputWriter {
     }
 }
 
-
 module.exports = {
     traceOutput: new TraceOutputWriter(),
 };

lib/runner/ConfigurationKeys.js

@@ -80,7 +80,7 @@ module.exports = class VirtualDeviceInvoker extends Invoker {
                 }
             }
         }
-        maxResponseWaitTime = testSuite.maxAsyncE2EResponseWaitTime;
+        maxResponseWaitTime = testSuite.maxResponseWaitTime;
         waitInterval = testSuite.asyncE2EWaitInterval;
 
         if (!virtualDeviceToken) {
@@ -93,7 +93,7 @@ module.exports = class VirtualDeviceInvoker extends Invoker {
         }
 
         if (replyTimeout * 1000 > maxResponseWaitTime) {
-            throw new FrameworkError("The replyTimeout property must be less than or equal to the maxAsyncE2EResponseWaitTime property in the " +
+            throw new FrameworkError("The replyTimeout property must be less than or equal to the maxResponseWaitTime property in the " +
                 "testing.json or the YML test file under the config element");
         }
 
@@ -368,7 +368,7 @@ module.exports = class VirtualDeviceInvoker extends Invoker {
             } while (!rawVirtualDeviceResponse && !isCurrentInteractionTimeoutExceed && !isCompleted);
 
             if (isCurrentInteractionTimeoutExceed && !rawVirtualDeviceResponse) {
-                const message = "Timeout exceeded while waiting for the interaction response. Increase the maxAsyncE2EResponseWaitTime value to fix this issue. " +
+                const message = "Timeout exceeded while waiting for the interaction response. Increase the maxResponseWaitTime value to fix this issue. " +
                     "More info at: https://read.bespoken.io/end-to-end/guide/.";
                 errorObject = {
                     error_category: "system",

lib/runner/TraceOutput.js

@@ -507,14 +507,29 @@ class TestSuite {
     }
 
     /**
+     * @alias from {maxResponseWaitTime}
+     * @deprecated
      * Indicates what is the maximum time e2e test can wait for a single utterance response to come back
+     * @todo These whole maxAsyncE2EResponseWaitTime prop will be removed in next versions
      * @return {number} max wait time for a single utterance
      */
     get maxAsyncE2EResponseWaitTime() {
+        return Configuration.instance().value("maxAsyncE2EResponseWaitTime", this.configuration, undefined);
+    }
+
+    /**
+     * Indicates what is the maximum time e2e test can wait for a single utterance response to come back
+     * @return {number} max wait time for a single utterance
+     */
+    get maxResponseWaitTime() {
         const defaultValue = ["phone", "twilio", "sms", "whatsapp"].includes(this.platform)
             ? 60000
             : 15000;
-        return Configuration.instance().value("maxAsyncE2EResponseWaitTime", this.configuration, defaultValue);
+
+        const newPropValue = Configuration.instance().value("maxResponseWaitTime", this.configuration, undefined);
+        const oldPropValue = this.maxAsyncE2EResponseWaitTime;
+
+        return newPropValue || oldPropValue || defaultValue;
     }
 
     /**
@@ -599,13 +614,13 @@ class TestSuite {
     }
 
     /**
-     * extra parameters for virtual device
+     * Extra parameters for virtual device, `virtualDeviceConfig` has the preference
      * @return {string} extraParameters
      */
     get extraParameters() {
         const extraParameters = Configuration.instance().value("extraParameters", this.configuration);
         const virtualDeviceConfig = Configuration.instance().value("virtualDeviceConfig", this.configuration);
-        return virtualDeviceConfig || extraParameters;
+        return _.assign(extraParameters, virtualDeviceConfig);
     }
 
     /**

lib/runner/VirtualDeviceInvoker.js

@@ -2,6 +2,7 @@ const _ = require("lodash");
 const fs = require("fs");
 const os = require("os");
 const path = require("path");
+const { padStart } = require("lodash");
 
 module.exports = class Util {
     static cleanString(s) {
@@ -253,4 +254,14 @@ module.exports = class Util {
 
         return result;
     }
+
+    static formatTimeStamp(milliseconds) {
+        const formatTwoDigits = number => padStart(number, 2, "0");
+
+        const timestamp = new Date(milliseconds);
+        const date = `${timestamp.getFullYear()}${formatTwoDigits(timestamp.getMonth() + 1)}${formatTwoDigits(timestamp.getDate())}`;
+        const time = `${formatTwoDigits(timestamp.getHours())}${formatTwoDigits(timestamp.getMinutes())}${formatTwoDigits(timestamp.getSeconds())}`;
+    
+        return `${date}_${time}`;
+    }
 };

lib/test/TestSuite.js

@@ -1,6 +1,6 @@
 {
   "name": "skill-testing-ml",
-  "version": "1.4.7",
+  "version": "1.4.8-alpha.1",
   "description": "Skill Testing Markup Language",
   "main": "dist/lib/Index.js",
   "bin": {
@@ -96,6 +96,7 @@
     "ask-sdk-model": "^1.0.0",
     "body-parser": "^1.18.3",
     "codecov": "^3.0.2",
+    "debug": "^4.3.4",
     "eslint": "^5.16.0",
     "eslint-config-google": "^0.12.0",
     "eslint-config-standard": "^12.0.0",