[Cherry Pick 4.13] Updates to Recognizers and unit tests. (#1171)

* [SDK][Recognizers-Text] Add Recognizer-Text unit tests into the temporal Recognizer-Text folder in bot-dialogs library (#1168)

* Add timex-expression unit tests

* Add Specs folder

* Add datetime unit tests

* Add sequence unit tests

* Add number with unit unit tests

* Add number unit tests

* Add choice unit tests

* Add text unit tests

* Update bot-dialogs pom to use UTF-8 to execute unit tests correctly

Co-authored-by: Martin Battaglino <martinbatta32@gmail.com>

* [SDK][Recognizers-Text] Update temporal folder with latest changes (#1164)

* Update Datetime extractors

* Update Datetime resources

* Update Number resources

* Update NumberWithUnit resources

* Update NumberWithUnit extractors

* Update NumberWithUnit parsers

Co-authored-by: Franco Alvarez <51216149+fran893@users.noreply.github.com>
Co-authored-by: Martin Battaglino <martinbatta32@gmail.com>

Author: 3 people

libraries/bot-dialogs/pom.xml

@@ -158,4 +158,15 @@
     </plugins>
   </reporting>
 
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <argLine>-Dfile.encoding=UTF-8</argLine>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
 </project>

libraries/bot-dialogs/src/main/java/com/microsoft/recognizers/text/datetime/extractors/BaseDateExtractor.java

@@ -108,6 +108,25 @@ private boolean validateMatch(Match match, String text) {
                     isValidMatch = startsWithBasicDate(subText);
                 }
             }
+            
+            // Expressions with mixed separators are not considered valid dates e.g. "30/4.85" (unless one is a comma "30/4, 2016")
+            MatchGroup dayGroup = match.getGroup("day");
+            MatchGroup monthGroup = match.getGroup("month");
+            if (!StringUtility.isNullOrEmpty(dayGroup.value) && !StringUtility.isNullOrEmpty(monthGroup.value)) {
+                String noDateText = match.value.replace(yearGroup.value, "")
+                    .replace(monthGroup.value, "").replace(dayGroup.value, "");
+                String[] separators = {"/", "\\", "-", "."};
+                int separatorCount = 0;
+                for (String separator : separators) {
+                    if (noDateText.contains(separator)) {
+                        separatorCount++;
+                    }
+                    if (separatorCount > 1) {
+                        isValidMatch = false;
+                        break;
+                    }
+                }
+            }
         }
 
         return isValidMatch;

libraries/bot-dialogs/src/main/java/com/microsoft/recognizers/text/datetime/extractors/BaseDateTimeExtractor.java

@@ -128,11 +128,27 @@ public List<Token> timeOfTodayAfter(String input, LocalDateTime reference) {
 
         Match[] matches = RegExpUtility.getMatches(this.config.getSimpleTimeOfTodayAfterRegex(), input);
         for (Match match : matches) {
+            // @TODO Remove when lookbehinds are handled correctly
+            if (isDecimal(match, input)) {
+                continue;
+            }
+            
             ret.add(new Token(match.index, match.index + match.length));
         }
 
         return ret;
     }
+    
+    // Check if the match is part of a decimal number (e.g. 123.24)
+    private boolean isDecimal(Match match, String text) {
+        boolean isDecimal = false;
+        if (match.index > 1 && (text.charAt(match.index - 1) == ',' ||
+                text.charAt(match.index - 1) == '.') && Character.isDigit(text.charAt(match.index - 2)) && Character.isDigit(match.value.charAt(0))) {
+            isDecimal = true;
+        }
+        
+        return isDecimal;
+    }
 
     public List<Token> timeOfTodayBefore(String input, LocalDateTime reference) {
         List<Token> ret = new ArrayList<>();

libraries/bot-dialogs/src/main/java/com/microsoft/recognizers/text/datetime/extractors/BaseTimeExtractor.java

@@ -88,6 +88,11 @@ public final List<Token> basicRegexMatch(String text) {
 
             Match[] matches = RegExpUtility.getMatches(regex, text);
             for (Match match : matches) {
+                
+                // @TODO Remove when lookbehinds are handled correctly
+                if (isDecimal(match, text)) {
+                    continue;
+                }
 
                 // @TODO Workaround to avoid incorrect partial-only matches. Remove after time regex reviews across languages.
                 String lth = match.getGroup("lth").value;
@@ -102,6 +107,17 @@ public final List<Token> basicRegexMatch(String text) {
 
         return ret;
     }
+    
+    // Check if the match is part of a decimal number (e.g. 123.24)
+    private boolean isDecimal(Match match, String text) {
+        boolean isDecimal = false;
+        if (match.index > 1 && (text.charAt(match.index - 1) == ',' ||
+                text.charAt(match.index - 1) == '.') && Character.isDigit(text.charAt(match.index - 2)) && Character.isDigit(match.value.charAt(0))) {
+            isDecimal = true;
+        }
+        
+        return isDecimal;
+    }
 
     private List<Token> atRegexMatch(String text) {
         List<Token> ret = new ArrayList<>();

libraries/bot-dialogs/src/main/java/com/microsoft/recognizers/text/datetime/resources/BaseDateTime.java

@@ -17,7 +17,7 @@
 
 public class BaseDateTime {
 
-    public static final String HourRegex = "(?<hour>2[0-4]|[0-1]?\\d)(h)?";
+    public static final String HourRegex = "(?<!\\d[,.])(?<hour>2[0-4]|[0-1]?\\d)(h)?";
 
     public static final String TwoDigitHourRegex = "(?<hour>[0-1]\\d|2[0-4])(h)?";
 
@@ -36,6 +36,8 @@ public class BaseDateTime {
     public static final String IllegalYearRegex = "([-])({FourDigitYearRegex})([-])"
             .replace("{FourDigitYearRegex}", FourDigitYearRegex);
 
+    public static final String CheckDecimalRegex = "(?![,.]\\d)";
+
     public static final String RangeConnectorSymbolRegex = "(--|-|—|——|~|–)";
 
     public static final String BaseAmDescRegex = "(am\\b|a\\s*\\.\\s*m\\s*\\.|a[\\.]?\\s*m\\b)";