Multiple improvements for the scenario import panel (#1203)

* Deleted scenario's now actually update the scenario dataframe

* Logging changes to fix #1086

* Adding and removing scenario tables now quicker

* Changed combination chooser to radio buttons

* The combination type can now be succesfully switched after scenario files have been loaded

* Add statistiscs to scenario import panel

* Improvements to scenario toolbar layout

* Stats now also update on empty scenarios or deletions

* Fixed tooltip bug on scenario tables

* Align scenariotables at the top

* Resolve bug with duplicates where 1) DF could not be created 2) file numbering was broken

* Minor documentation + code changes

* Fix scenario exporter saving with useful file extension by default

* Fix scenario exporter saving with useful file extension by default

* Set Excel default separator (;) for CSV export

* Fix ABPopup default DF export file format

* Move cursor override + add todo

---------

Co-authored-by: marc-vdm <m.t.van.der.meide@cml.leidenuniv.nl>

Author: mrvisscher

activity_browser/bwutils/superstructure/file_dialogs.py

@@ -175,15 +175,16 @@ def save_dataframe(self):
             return True
         filepath, _ = QtWidgets.QFileDialog.getSaveFileName(
             parent=self, caption="Choose the location to save the dataframe",
-            filter="All Files (*.*);; CSV (*.csv);; Excel (*.xlsx)",
+            filter="Excel (*.xlsx *.xls);; CSV (*.csv)",
         )
         QtWidgets.QApplication.setOverrideCursor(QtCore.Qt.WaitCursor)
-        if not filepath.strip():
-            return False
         if filepath.endswith('.xlsx') or filepath.endswith('.xls'):
             self._dataframe.to_excel(filepath, index=False)
-        else:
-            self._dataframe.to_csv(filepath, index=False, sep=';')
+            QtWidgets.QApplication.restoreOverrideCursor()
+            return True
+        elif not filepath.endswith('.csv'):
+            filepath += '.csv'
+        self._dataframe.to_csv(filepath, index=False, sep=';')
         QtWidgets.QApplication.restoreOverrideCursor()
         return True
 

activity_browser/bwutils/superstructure/manager.py

@@ -3,7 +3,7 @@
 from typing import List
 import numpy as np
 import pandas as pd
-from pandas.api.types import is_numeric_dtype, is_number
+from pandas.api.types import is_numeric_dtype
 from PySide2.QtWidgets import QApplication, QPushButton
 from PySide2.QtCore import Qt
 from typing import Union, Optional
@@ -12,7 +12,7 @@
 
 from .activities import fill_df_keys_with_fields, get_activities_from_keys
 from .dataframe import scenario_columns
-from .utils import guess_flow_type, SUPERSTRUCTURE, _time_it_, edit_superstructure_for_string
+from .utils import guess_flow_type, SUPERSTRUCTURE, _time_it_
 
 from .file_dialogs import ABPopup
 from ..errors import (CriticalScenarioExtensionError, ScenarioExchangeNotFoundError,
@@ -465,10 +465,10 @@ def check_scenario_exchange_values(df: pd.DataFrame, cols: pd.Index):
                 QPushButton('Save'),
                 QPushButton('Cancel')
             )
+            QApplication.restoreOverrideCursor()
             critical.dataframe(df[bad_entries.isna().any(axis=1)], SUPERSTRUCTURE)
             critical.save_options()
             critical.dataframe_to_file(df, bad_entries.isna().any(axis=1))
-            QApplication.restoreOverrideCursor()
             critical.exec_()
             raise ScenarioExchangeDataNonNumericError()
 
@@ -507,18 +507,16 @@ def check_duplicates(data: Optional[Union[pd.DataFrame, list]],
             while count < len(data):
                 count += 1
                 popped = data[-count]
-                duplicates = SuperstructureManager._check_duplicates(df, popped)
+                duplicates = SuperstructureManager._check_duplicates(df, popped, count)
                 if not duplicates.empty:
                     duplicated[count] = duplicates
-            if duplicated:
 
+            if duplicated:
                 msg = "<p>Duplicates have been found, meaning that there are several rows in the scenario file describing "\
                 "scenarios for the same flow. The AB can deal with this by discarding all but the last row for this "\
                 "exchange.</p> <p>Press 'Ok' to proceed, press 'Cancel' to abort.</p>"
-                for file, frame in duplicated.items():
-                    frame.insert(0, 'File', file, allow_duplicates=True)
                 warning = ABPopup.abWarning('Duplicate flow exchanges', msg, QPushButton('Ok'), QPushButton('Cancel'))
-                warning.dataframe(pd.concat([file for file in duplicated.values()]), ['File'] + SUPERSTRUCTURE)
+                warning.dataframe(pd.concat([file for file in duplicated.values()]), ['file'] + SUPERSTRUCTURE.tolist())
                 QApplication.restoreOverrideCursor()
                 response = warning.exec_()
                 QApplication.setOverrideCursor(Qt.WaitCursor)
@@ -527,23 +525,27 @@ def check_duplicates(data: Optional[Union[pd.DataFrame, list]],
             return data
 
     @staticmethod
-    def _check_duplicates(dfp: pd.DataFrame, pdf: pd.DataFrame,
+    def _check_duplicates(dfp: pd.DataFrame, pdf: pd.DataFrame, count: int,
                           index: list = ['to key', 'from key', 'flow type']) -> pd.DataFrame:
+        # TODO fix variable names 'dfp' & 'pdf' to something undearstandeable
+        # TODO create useful docstring, already clear this is a private method from '_' prefix
         """ NOT TO BE USED OUTSIDE OF CALLING METHOD check_duplicates """
         # First save the original index and create a new one that can help the user identify duplicates in their files
         d_idx = dfp.index
+        dfp.insert(0, 'file', '1', allow_duplicates=True)  # add the file number
         dfp.index = pd.Index([str(i) for i in range(dfp.shape[0])])
         p_idx = pdf.index
+        pdf.insert(0, 'file', str(count), allow_duplicates=True)  # add the file number
         pdf.index = pd.Index([str(i) for i in range(pdf.shape[0])])
         df = pd.concat([dfp, pdf], ignore_index=True)
         dfp.index = d_idx
         pdf.index = p_idx
         dfp.drop_duplicates(index, keep='last', inplace=True)
-        #        pdf.drop_duplicates(index, keep='last', inplace=True)
         return df.loc[df.duplicated(index, keep=False)]
 
     @staticmethod
     def _check_duplicate(data: pd.DataFrame, index: list = ['to key', 'from key', 'flow type']) -> pd.DataFrame:
+        # TODO create useful docstring, already clear this is a private method from '_' prefix
         """ NOT TO BE USED OUTSIDE OF CALLING METHOD check_duplicates """
         df = data.copy()
         df.index = pd.Index([str(i) for i in range(df.shape[0])])

activity_browser/bwutils/superstructure/utils.py

@@ -11,6 +11,7 @@
 
 # Different kinds of indexes, to allow for quick selection of data from
 # the Superstructure DataFrame.
+#TODO review if this can be made a list instead of pd.Index
 SUPERSTRUCTURE = pd.Index([
     "from activity name",
     "from reference product",