validate ssv files against schema (#1261)

* validate ssv files 
* validate ssm files
* validate modelDescription.xml in FMU

Author: arun3688

src/OMSimulatorLib/Scope.cpp

@@ -198,7 +198,7 @@ oms_status_enu_t oms::Scope::importModel(const std::string& filename, char** _cr
     return logError("failed to extract \"SystemStructure.ssd\" from \"" + std::string(filename) + "\"");
 
   XercesValidator xercesValidator;
-  xercesValidator.validateSSD(systemStructure, filename);
+  xercesValidator.validateSSP(systemStructure, filename);
 
   Snapshot snapshot;
   oms_status_enu_t status = snapshot.importResourceMemory("SystemStructure.ssd", systemStructure);

src/OMSimulatorLib/Values.cpp

@@ -35,6 +35,7 @@
 #include "Logging.h"
 #include "ssd/Tags.h"
 #include "Util.h"
+#include "XercesValidator.h"
 
 #include <iostream>
 #include <map>
@@ -841,6 +842,14 @@ oms_status_enu_t oms::Values::importFromSnapshot(const pugi::xml_node& node, con
     std::string ssvFile = parameterBindingNode.attribute("source").as_string();
     if (!ssvFile.empty()) // parameter binding provided with .ssv file
     {
+      // validate ssv files against SSP schem, only if the file exists, because it is possible we use importSnapShot API
+      // to go back to old states and in this case we should not validate those ssv files in memory until it is exported to a ssp
+      if (filesystem::exists(ssvFile))
+      {
+        XercesValidator xercesValidator;
+        xercesValidator.validateSSP("", ssvFile);
+      }
+
       //resourceFiles.push_back(ssvFile);
       pugi::xml_node parameterSet = snapshot.getResourceNode(ssvFile);
       if (!parameterSet)
@@ -857,6 +866,12 @@ oms_status_enu_t oms::Values::importFromSnapshot(const pugi::xml_node& node, con
         std::string ssmFileSource = ssd_parameterMapping.attribute("source").as_string();
         if (!ssmFileSource.empty())
         {
+          // validate ssm file only if it exists
+          if (filesystem::exists(ssmFileSource))
+          {
+            XercesValidator xercesValidator;
+            xercesValidator.validateSSP("", ssmFileSource);
+          }
           pugi::xml_node ssm_parameterMapping = snapshot.getResourceNode(ssmFileSource);
           if (!ssm_parameterMapping)
             return logError("loading <oms:file> \"" + ssmFileSource + "\" from <oms:snapshot> failed");
@@ -1476,6 +1491,10 @@ oms_status_enu_t oms::Values::parseModelDescription(const filesystem::path& root
 
   const char* modelDescription = ::miniunz_onefile_to_memory(root.generic_string().c_str(), "modelDescription.xml");
 
+  // validate modeldescription.xml against schema fmi2ModelDescription.xsd
+  XercesValidator xercesValidator;
+  xercesValidator.validateFMU(modelDescription, root.generic_string());
+
   Snapshot snapshot;
   oms_status_enu_t status = snapshot.importResourceMemory("modelDescription.xml", modelDescription);
   ::miniunz_free(modelDescription);

src/OMSimulatorLib/XercesValidator.cpp

@@ -35,6 +35,7 @@
 #include "Logging.h"
 #include "OMSFileSystem.h"
 #include "OMSString.h"
+#include <iostream>
 
 #include <xercesc/parsers/XercesDOMParser.hpp>
 #include <xercesc/dom/DOM.hpp>
@@ -122,8 +123,14 @@ std::string oms::XercesValidator::getExecutablePath()
   return executablePath;
 }
 
-oms_status_enu_t oms::XercesValidator::validateSSD(const char *ssd, const std::string& filePath)
+oms_status_enu_t oms::XercesValidator::validateSSP(const char *ssd, const std::string& filePath)
 {
+
+  std::string extension = filesystem::path(filePath).extension().generic_string();
+
+  if (extension != ".ssp" && extension != ".ssv" && extension != ".ssm")
+    return logWarning("filename extension must be \".ssp\" or \".ssv\" or \".ssm\" ; no other formats are supported for SSP validation");
+
   try
   {
     XMLPlatformUtils::Initialize();
@@ -142,24 +149,40 @@ oms_status_enu_t oms::XercesValidator::validateSSD(const char *ssd, const std::s
     return logError("executable path could not be found");
 
   filesystem::path schemaRootPath (path);
-  filesystem::path schemaFilePath;
+  filesystem::path schemaSSDPath, schemaSSVPath, schemaSSMPath;
+
+  schemaSSDPath = schemaRootPath / "../share/OMSimulator/schema/ssp/SystemStructureDescription.xsd";
+  schemaSSVPath = schemaRootPath / "../share/OMSimulator/schema/ssp/SystemStructureParameterValues.xsd";
+  schemaSSMPath = schemaRootPath / "../share/OMSimulator/schema/ssp/SystemStructureParameterMapping.xsd";
 
-  schemaFilePath = schemaRootPath / "../share/OMSimulator/schema/ssp/SystemStructureDescription.xsd";
   // std::cout << "schemaPath: " << schemaFilePath.generic_string() << "\n" << filesystem::absolute(schemaFilePath).generic_string() << "\n";
 
   // this is done if we run the OMSimulator using python extension (e.g) python3 test.py and in this case the executable path is the dll path
   // for windows and mingw "libOMSimulator.dll" is put in "install/bin" directory and for linux and other platforms
   // the shared libraries are put in "install/lib/x86_64-linux-gnu/", so to find the schema location we have to move two directories back
-  if (!filesystem::exists(schemaFilePath))
-    schemaFilePath = schemaRootPath / "../../share/OMSimulator/schema/ssp/SystemStructureDescription.xsd";
+  if (!filesystem::exists(schemaSSDPath))
+  {
+    schemaSSDPath = schemaRootPath / "../../share/OMSimulator/schema/ssp/SystemStructureDescription.xsd";
+    schemaSSVPath = schemaRootPath / "../../share/OMSimulator/schema/ssp/SystemStructureParameterValues.xsd";
+    schemaSSMPath = schemaRootPath / "../../share/OMSimulator/schema/ssp/SystemStructureParameterMapping.xsd";
+  }
 
   XercesDOMParser domParser;
 
   // load the schema
-  if (domParser.loadGrammar(schemaFilePath.generic_string().c_str(), Grammar::SchemaGrammarType) == NULL)
-      return logError("could not load the ssd schema file: " + filesystem::absolute(schemaFilePath).generic_string());
+  if (domParser.loadGrammar(schemaSSDPath.generic_string().c_str(), Grammar::SchemaGrammarType) == NULL)
+    return logError("could not load the ssd schema file: " + filesystem::absolute(schemaSSDPath).generic_string());
+
+  std::string sspVariant = "";
+
+  if (extension == ".ssp")
+    sspVariant = "SystemStructureDescription";
+  else if (extension == ".ssv")
+    sspVariant = "SystemStructureParameterValues";
+  else if (extension == ".ssm")
+    sspVariant = "SystemStructureParameterMapping";
 
-  ParserErrorHandler parserErrorHandler("SystemStructure.ssd", filePath.c_str());
+  ParserErrorHandler parserErrorHandler(sspVariant.c_str(), filePath.c_str());
 
   domParser.setErrorHandler(&parserErrorHandler);
   domParser.cacheGrammarFromParse(true);
@@ -169,14 +192,88 @@ oms_status_enu_t oms::XercesValidator::validateSSD(const char *ssd, const std::s
   domParser.setValidationSchemaFullChecking(true);
   domParser.setValidationConstraintFatal(true);
   // domParser.setExternalNoNamespaceSchemaLocation(schemaFilePath); // set this for noNameSpace
-  std::string ssdTargetNameSpacePath = "http://ssp-standard.org/SSP1/SystemStructureDescription " + schemaFilePath.generic_string();
+  std::string ssdTargetNameSpacePath = "http://ssp-standard.org/SSP1/SystemStructureDescription " + schemaSSDPath.generic_string();
+  ssdTargetNameSpacePath = ssdTargetNameSpacePath + " http://ssp-standard.org/SSP1/SystemStructureParameterValues " + schemaSSVPath.generic_string();
+  ssdTargetNameSpacePath = ssdTargetNameSpacePath + " http://ssp-standard.org/SSP1/SystemStructureParameterMapping " + schemaSSMPath.generic_string();
+
   domParser.setExternalSchemaLocation(ssdTargetNameSpacePath.c_str());
 
-  xercesc::MemBufInputSource pMemBufIS((const XMLByte*)ssd, std::string(ssd).size() , "ssdfile");
+  if (strlen(ssd) > 0)
+  {
+    xercesc::MemBufInputSource pMemBufIS((const XMLByte *)ssd, std::string(ssd).size(), "ssdfile");
+    domParser.parse(pMemBufIS);
+  }
+  else
+  {
+    domParser.parse(filePath.c_str());
+  }
+
+  if (domParser.getErrorCount() > 0)
+    return logWarning( "\"" + sspVariant + "\"" + " does not conform to the SSP standard schema");
+
+  return oms_status_ok;
+}
+
+oms_status_enu_t oms::XercesValidator::validateFMU(const char *modeldescription, const std::string& filePath)
+{
+
+  std::string extension = filesystem::path(filePath).extension().generic_string();
+
+  if (extension != ".fmu")
+    return logWarning("filename extension must be \".fmu\" ; no other formats are supported for fmu validation");
+
+  try
+  {
+    XMLPlatformUtils::Initialize();
+  }
+  catch (const XMLException &toCatch)
+  {
+    char *message = XMLString::transcode(toCatch.getMessage());
+    logError("Xerces error during initialization: "+ std::string(message));
+    XMLString::release(&message);
+    return oms_status_error;
+  }
+
+  std::string path = getExecutablePath();
+
+  if (path.empty())
+    return logError("executable path could not be found");
+
+  filesystem::path schemaRootPath (path);
+  filesystem::path schemaFmiModeldescriptionPath;
+
+  schemaFmiModeldescriptionPath = schemaRootPath / "../share/OMSimulator/schema/fmi2/fmi2ModelDescription.xsd";
+
+  // this is done if we run the OMSimulator using python extension (e.g) python3 test.py and in this case the executable path is the dll path
+  // for windows and mingw "libOMSimulator.dll" is put in "install/bin" directory and for linux and other platforms
+  // the shared libraries are put in "install/lib/x86_64-linux-gnu/", so to find the schema location we have to move two directories back
+  if (!filesystem::exists(schemaFmiModeldescriptionPath))
+  {
+    schemaFmiModeldescriptionPath = schemaRootPath / "../../share/OMSimulator/schema/fmi2/fmi2ModelDescription.xsd";
+  }
+
+  XercesDOMParser domParser;
+
+  // load the schema
+  if (domParser.loadGrammar(schemaFmiModeldescriptionPath.generic_string().c_str(), Grammar::SchemaGrammarType) == NULL)
+    return logError("could not load the ssd schema file: " + filesystem::absolute(schemaFmiModeldescriptionPath).generic_string());
+
+  ParserErrorHandler parserErrorHandler("modeldescription.xml", filePath.c_str());
+
+  domParser.setErrorHandler(&parserErrorHandler);
+  domParser.cacheGrammarFromParse(true);
+  domParser.setValidationScheme(XercesDOMParser::Val_Always);
+  domParser.setDoNamespaces(true);
+  domParser.setDoSchema(true);
+  domParser.setValidationSchemaFullChecking(true);
+  domParser.setValidationConstraintFatal(true);
+  domParser.setExternalNoNamespaceSchemaLocation(schemaFmiModeldescriptionPath.generic_string().c_str()); // set this for noNameSpace
+
+  xercesc::MemBufInputSource pMemBufIS((const XMLByte *)modeldescription, std::string(modeldescription).size(), "modeldescriptionfile");
   domParser.parse(pMemBufIS);
 
   if (domParser.getErrorCount() > 0)
-      return logWarning("\"SystemStructure.ssd\" does not conform to the SSP standard schema");
+    return logWarning("\"modeldescription.xml\" does not conform to the FMI-2.0 standard schema");
 
   return oms_status_ok;
-}
+}

src/OMSimulatorLib/XercesValidator.h

@@ -46,7 +46,8 @@ namespace oms
   public:
     XercesValidator();
     ~XercesValidator();
-    oms_status_enu_t validateSSD(const char * ssd, const std::string& filePath);
+    oms_status_enu_t validateSSP(const char * ssd, const std::string& filePath);
+    oms_status_enu_t validateFMU(const char * modeldescription, const std::string& filePath);
     std::string getExecutablePath();
   };
 }

testsuite/AircraftVehicleDemonstrator/embrace.py

@@ -67,15 +67,17 @@
 oms.delete("embrace")
 
 ## Result:
-## warning: invalid "SystemStructure.ssd" detected in file "../../resources/embrace.ssp" at line: 17 column: 26, missing required attribute 'name'
-## warning: "SystemStructure.ssd" does not conform to the SSP standard schema
+## warning: invalid "SystemStructureDescription" detected in file "../../resources/embrace.ssp" at line: 17 column: 26, missing required attribute 'name'
+## warning: "SystemStructureDescription" does not conform to the SSP standard schema
+## warning: invalid "SystemStructureParameterValues" detected in file "resources/RAPID_Systems_2021-03-29_Test_1.ssv" at line: 3 column: 10, element 'Units' must be qualified
+## warning: "SystemStructureParameterValues" does not conform to the SSP standard schema
 ## info:    Result file: sim_results.mat (bufferSize=1)
 ## info:    Initialize:
 ## info:      embrace.root.ECS_HW.coolinPackAir.looptype:  2
 ## info:      embrace.root.ECS_HW.eCS.MaxCoolPower.k    :  5.0
 ## info:      embrace.root.ECS_HW.pipeC.L               :  1.0
 ## info:      embrace.root.ECS_HW.pipeB.L               :  0.976535328081166
 ## info:      embrace.root.ECS_HW.pipeA.L               :  0.976535328081166
-## info:    2 warnings
+## info:    4 warnings
 ## info:    0 errors
 ## endResult

testsuite/AircraftVehicleDemonstrator/embrace_test2Conf.py

@@ -76,15 +76,17 @@
 oms.delete("model")
 
 ## Result:
-## warning: invalid "SystemStructure.ssd" detected in file "../../resources/embrace_TwoConf.ssp" at line: 17 column: 26, missing required attribute 'name'
-## warning: "SystemStructure.ssd" does not conform to the SSP standard schema
+## warning: invalid "SystemStructureDescription" detected in file "../../resources/embrace_TwoConf.ssp" at line: 17 column: 26, missing required attribute 'name'
+## warning: "SystemStructureDescription" does not conform to the SSP standard schema
+## warning: invalid "SystemStructureParameterValues" detected in file "resources/ECS_Conf2.ssv" at line: 3 column: 10, element 'Units' must be qualified
+## warning: "SystemStructureParameterValues" does not conform to the SSP standard schema
 ## info:    Result file: sim_results_ECS_Conf1.mat (bufferSize=1)
 ## info:    Initialize:
 ## info:      model.root.ECS_HW.coolinPackAir.looptype:  2
 ## info:      model.root.ECS_HW.eCS.MaxCoolPower.k    :  5.0
 ## info:      model.root.ECS_HW.pipeC.L               :  1.0
 ## info:      model.root.ECS_HW.pipeB.L               :  7.41248272578546
 ## info:      model.root.ECS_HW.pipeA.L               :  7.39290438403619
-## info:    2 warnings
+## info:    4 warnings
 ## info:    0 errors
 ## endResult

testsuite/resources/Makefile

@@ -21,6 +21,7 @@ fmi_attributes_19 \
 HelloWorld \
 HelloWorldWithInput \
 Int1 \
+invalidModeldescription \
 Lin2DimODE \
 Modelica.Blocks.Continuous.Integrator \
 Modelica.Blocks.Math.Add \
@@ -73,7 +74,9 @@ import_export_parameters1 \
 import_export_parameters2 \
 import_export_parameters3 \
 importExportAllResources \
-invalidSSP \
+invalidSSD \
+invalidSSV \
+invalidSSM \
 replaceSubmodel4 \
 replaceSubmodel5 \
 

testsuite/resources/embrace/resources/RAPID_Systems_2021-03-29_Test_1.ssv

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<ssv:ParameterSet xmlns:ssv="http://ssp-standard.org/SSP1/SystemStructureParameterValues" xmlns:ssc="http://ssp-standard.org/SSP1/SystemStructureCommon" name="product_mainAssembly">
+<ssv:ParameterSet xmlns:ssv="http://ssp-standard.org/SSP1/SystemStructureParameterValues" xmlns:ssc="http://ssp-standard.org/SSP1/SystemStructureCommon" version="1.0" name="product_mainAssembly">
   <Units>
     <Unit name="m">
       <BaseUnit m="1"/>

testsuite/resources/embrace_TwoConf/resources/ECS_Conf1.ssv

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<ssv:ParameterSet xmlns:ssv="http://ssp-standard.org/SSP1/SystemStructureParameterValues" xmlns:ssc="http://ssp-standard.org/SSP1/SystemStructureCommon" name="product_ECS">
+<ssv:ParameterSet xmlns:ssv="http://ssp-standard.org/SSP1/SystemStructureParameterValues" xmlns:ssc="http://ssp-standard.org/SSP1/SystemStructureCommon" version="1.0" name="product_ECS">
   <Units>
     <Unit name="m">
       <BaseUnit m="1"/>

testsuite/resources/embrace_TwoConf/resources/ECS_Conf2.ssv

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<ssv:ParameterSet xmlns:ssv="http://ssp-standard.org/SSP1/SystemStructureParameterValues" xmlns:ssc="http://ssp-standard.org/SSP1/SystemStructureCommon" name="product_ECS2">
+<ssv:ParameterSet xmlns:ssv="http://ssp-standard.org/SSP1/SystemStructureParameterValues" xmlns:ssc="http://ssp-standard.org/SSP1/SystemStructureCommon" version="1.0" name="product_ECS2">
   <Units>
     <Unit name="m">
       <BaseUnit m="1"/>