[Lubuild] fix multiple bugs: case sensitive issue, default version issue and exception not throw out issue (#576)

* fix case issue of utterances and patterns

* remove lower function for entities

* adjust test cases

* use a better solution to compare utterances

* add tests to verify the fixes

* move new added tests to luis package from lu package

* add test cases for luConfig parameter in lubuild CLI

* fix error not throw out bug

* resolve comments of CLIError

* remove uncessary semicolon

* Updates and fixes

* Updates

* updating tests.

* Updates for PR feedback, fix settings out path

* updates

* fix build errors

* fix command parameter issue in test cases

* fix tslint errors

* remove lubuild.js which is not necessary

* fixing bad merge

* fix tslint and expose more in index.js and composerindex.js

* remove lubuild module in index.js

Co-authored-by: Emilio Munoz <munoz_emilio@hotmail.com>
Co-authored-by: Vishwac Sena Kannan <vishwack@hotmail.com>

Author: 3 people

common/config/rush/pnpm-lock.yaml

@@ -1,4 +1,11 @@
 module.exports = {
-    parseFile: require('./lufile/parseFileContents').parseFile,
-    validateLUISBlob: require('./luis/luisValidator')
+    parser: {
+        parseFile: require('./lufile/parseFileContents').parseFile,
+        validateLUISBlob: require('./luis/luisValidator')
+    },
+    sectionHandler: {
+        luParser: require('./lufile/luParser'),
+        sectionOperator: require('./lufile/sectionOperator'),
+        luSectionTypes: require('./utils/enums/lusectiontypes')
+    },
 }

packages/lu/src/parser/composerindex.js

@@ -27,11 +27,6 @@ const modules = {
 
         Parser: require('./lufile/classes/parserObject')
     },
-    sectionHandler: {
-        luParser: require('./lufile/luParser'),
-        sectionOperator: require('./lufile/sectionOperator'),
-        luSectionTypes: require('./utils/enums/lusectiontypes')
-    },
     V2 : {
         Luis: require('./luis/luis'),
         LU: require('./lu/lu'),

packages/lu/src/parser/index.js

@@ -12,6 +12,8 @@ const fs = require('fs-extra')
 const delay = require('delay')
 const fileHelper = require('./../../utils/filehelper')
 const fileExtEnum = require('./../utils/helpers').FileExtTypeEnum
+const retCode = require('./../utils/enums/CLI-errors')
+const exception = require('./../utils/exception')
 const LuisBuilderVerbose = require('./../luis/luisCollate')
 const LuisBuilder = require('./../luis/luisBuilder')
 const LUOptions = require('./../lu/luOptions')
@@ -38,16 +40,25 @@ export class Builder {
       let fileCulture: string
       let fileName: string
       const luFiles = await fileHelper.getLuObjects(undefined, file, true, fileExtEnum.LUFile)
-      const result = await LuisBuilderVerbose.build(luFiles, true, culture)
-      const fileContent = result.parseToLuContent()
+
+      let fileContent = ''
+      let result
+      try {
+        result = await LuisBuilderVerbose.build(luFiles, true, culture)
+        fileContent = result.parseToLuContent()
+      } catch (err) {
+        err.text = `Invalid LU file ${file}: ${err.text}`
+        throw(new exception(retCode.errorCode.INVALID_INPUT_FILE, err.text))
+      }
+
       this.handler(`${file} loaded\n`)
       let cultureFromPath = fileHelper.getCultureFromPath(file)
       if (cultureFromPath) {
         fileCulture = cultureFromPath
         let fileNameWithCulture = path.basename(file, path.extname(file))
         fileName = fileNameWithCulture.substring(0, fileNameWithCulture.length - fileCulture.length - 1)
       } else {
-        fileCulture = culture
+        fileCulture = result.culture !== 'en-us' ? result.culture : culture
         fileName = path.basename(file, path.extname(file))
       }
 
@@ -95,7 +106,7 @@ export class Builder {
     })
 
     if (hasDuplicates) {
-      throw new Error('Files with same name and locale are found.')
+      throw(new exception(retCode.errorCode.INVALID_INPUT_FILE, 'Files with same name and locale are found.'))
     }
 
     return {luContents, recognizers, multiRecognizers, settings}
@@ -200,16 +211,23 @@ export class Builder {
     return dialogContents
   }
 
-  async writeDialogAssets(contents: any[], force: boolean, out: string) {
+  async writeDialogAssets(contents: any[], force: boolean, out: string, luconfig: string) {
     let writeDone = false
-    if (out) {
-      let settingsContents = contents.filter(c => c.id.endsWith('.json'))
-      let writeContents = contents.filter(c => c.id.endsWith('.dialog'))
 
-      if (settingsContents && settingsContents.length > 0) {
-        writeContents.push(this.mergeSettingsContent(path.join(path.resolve(out), settingsContents[0].id), settingsContents))
+    let writeContents = contents.filter(c => c.id.endsWith('.dialog'))
+    let settingsContents = contents.filter(c => c.id.endsWith('.json'))
+
+    if (settingsContents && settingsContents.length > 0) {
+      let outPath
+      if (luconfig) {
+        outPath = path.join(path.resolve(path.dirname(luconfig)), settingsContents[0].id)
+      } else if (out) {
+        outPath = path.join(path.resolve(out), settingsContents[0].id)
       }
+      writeContents.push(this.mergeSettingsContent(outPath, settingsContents))
+    }
 
+    if (out) {
       for (const content of writeContents) {
         const outFilePath = path.join(path.resolve(out), path.basename(content.path))
         if (force || !fs.existsSync(outFilePath)) {
@@ -219,7 +237,7 @@ export class Builder {
         }
       }
     } else {
-      for (const content of contents) {
+      for (const content of writeContents) {
         if (force || !fs.existsSync(content.path)) {
           this.handler(`Writing to ${content.path}\n`)
           await fs.writeFile(content.path, content.content, 'utf-8')
@@ -269,7 +287,7 @@ export class Builder {
         const versionObjs = await luBuildCore.listApplicationVersions(recognizer.getAppId())
         for (const versionObj of versionObjs) {
           if (versionObj.version !== newVersionId) {
-            this.handler(`deleting old version=${versionObj.version}`)
+            this.handler(`${recognizer.getLuPath()} deleting old version=${versionObj.version}`)
             await luBuildCore.deleteVersion(recognizer.getAppId(), versionObj.version)
           }
         }

packages/lu/src/parser/lubuild.js

@@ -49,13 +49,7 @@ export class LuBuildCore {
   public compareApplications(currentApp: any, existingApp: any) {
     currentApp.desc = currentApp.desc && currentApp.desc !== '' && currentApp.desc !== existingApp.desc ? currentApp.desc : existingApp.desc
     currentApp.culture = currentApp.culture && currentApp.culture !== '' && currentApp.culture !== existingApp.culture ? currentApp.culture : existingApp.culture
-    currentApp.versionId = currentApp.versionId && currentApp.versionId !== '' && currentApp.versionId !== existingApp.versionId ? currentApp.versionId : existingApp.versionId;
-
-    // convert utterance text from lu file to lower case
-    // as utterances from luis api are all converted to lower case automatically
-    (currentApp.utterances || []).forEach((u: any) => {
-      u.text = u.text.toLowerCase()
-    });
+    currentApp.versionId = currentApp.versionId && currentApp.versionId !== '' && currentApp.versionId > existingApp.versionId ? currentApp.versionId : existingApp.versionId;
 
     // convert list entities to remove synonyms word in list which is same with canonicalForm
     (currentApp.closedLists || []).forEach((c: any) => {
@@ -85,7 +79,7 @@ export class LuBuildCore {
 
   public updateVersion(currentApp: any, existingApp: any) {
     let newVersionId: string
-    if (currentApp.versionId && currentApp.versionId !== existingApp.versionId) {
+    if (currentApp.versionId > existingApp.versionId) {
       newVersionId = currentApp.versionId
     } else {
       newVersionId = this.updateVersionValue(existingApp.versionId)
@@ -178,12 +172,12 @@ export class LuBuildCore {
     equal = equal && this.isArrayEqual(appA.closedLists, appB.closedLists)
     equal = equal && this.isArrayEqual(appA.composites, appB.composites)
     equal = equal && this.isArrayEqual(appA.entities, appB.entities)
-    equal = equal && this.isArrayEqual(appA.modelFeatures, appB.modelFeatures)
+    equal = equal && this.isArrayEqual(appA.model_features, appB.modelFeatures)
     equal = equal && this.isArrayEqual(appA.patternAnyEntities, appB.patternAnyEntities)
     equal = equal && this.isArrayEqual(appA.patterns, appB.patterns)
     equal = equal && this.isArrayEqual(appA.prebuiltEntities, appB.prebuiltEntities)
-    equal = equal && this.isArrayEqual(appA.regexEntities, appB.regexEntities)
-    equal = equal && this.isArrayEqual(appA.regexFeatures, appB.regexFeatures)
+    equal = equal && this.isArrayEqual(appA.regex_entities, appB.regexEntities)
+    equal = equal && this.isArrayEqual(appA.regex_features, appB.regexFeatures)
     equal = equal && this.isArrayEqual(appA.utterances, appB.utterances)
 
     // handle exception for none intent which is default added in luis portal
@@ -205,15 +199,14 @@ export class LuBuildCore {
     let yObj = []
 
     if (x && x.length > 0) {
-      xObj = JSON.parse(JSON.stringify(x))
+      xObj = JSON.parse(JSON.stringify(x).toLowerCase().replace(/ {2}/g, ' '))
     }
 
     if (y && y.length > 0) {
-      yObj = JSON.parse(JSON.stringify(y))
+      yObj = JSON.parse(JSON.stringify(y).toLowerCase().replace(/ {2}/g, ' '))
     }
 
     if (xObj.length !== yObj.length) return false
-
     if (differenceWith(xObj, yObj, isEqual).length > 0) return false
 
     return true