MDS settings and living without a searcher

Author: mrvisscher

activity_browser/app/pages/settings/__init__.py

@@ -4,11 +4,13 @@
 from .startup import StartupSettingsChapter
 from .appearance import AppearanceSettingsChapter
 from .project_manager import ProjectManagerSettingsChapter
+from .metadatastore import MetadataStoreSettingsChapter
 
 __all__ = [
     "SettingsPage", 
     "BaseSettingsChapter",
     "StartupSettingsChapter",
     "AppearanceSettingsChapter",
     "ProjectManagerSettingsChapter",
+    "MetadataStoreSettingsChapter",
 ]

activity_browser/app/pages/settings/metadatastore.py

@@ -0,0 +1,111 @@
+# -*- coding: utf-8 -*-
+from loguru import logger
+from qtpy import QtWidgets
+
+from activity_browser.app import settings
+from activity_browser.app.pages.settings.base import BaseSettingsChapter
+
+
+class MetadataStoreSettingsChapter(BaseSettingsChapter):
+    """Chapter for metadatastore-related settings."""
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+
+        # Caching enabled checkbox
+        self.caching_checkbox = QtWidgets.QCheckBox("Enable caching")
+        self.caching_checkbox.setToolTip(
+            "Enable caching for faster data access. "
+            "Disable if you experience memory issues or want to force fresh data loading."
+        )
+
+        # Searcher enabled checkbox
+        self.searcher_checkbox = QtWidgets.QCheckBox("Enable searcher")
+        self.searcher_checkbox.setToolTip(
+            "Enable the full-text search functionality for activities and metadata. "
+            "Disable if you experience performance issues with large databases."
+        )
+
+        self.build_layout()
+        self.connect_signals()
+        self.reset()
+
+    def connect_signals(self):
+        """Connect signals and slots."""
+        # Emit changed signal when settings change
+        self.caching_checkbox.stateChanged.connect(lambda: self.changed.emit())
+        self.searcher_checkbox.stateChanged.connect(lambda: self.changed.emit())
+
+    def build_layout(self):
+        """Build the chapter layout."""
+        layout = QtWidgets.QVBoxLayout()
+
+        # Metadata store group
+        metadatastore_group = QtWidgets.QGroupBox("Metadata Store Options")
+        metadatastore_layout = QtWidgets.QVBoxLayout()
+
+        metadatastore_layout.addWidget(self.caching_checkbox)
+        metadatastore_layout.addWidget(self.searcher_checkbox)
+
+        # Add description label
+        description = QtWidgets.QLabel(
+            "These settings control the behavior of the metadata store, "
+            "which manages activity and exchange metadata for improved performance."
+        )
+        description.setWordWrap(True)
+        description.setStyleSheet("color: gray; font-size: 10pt;")
+        metadatastore_layout.addWidget(description)
+
+        metadatastore_group.setLayout(metadatastore_layout)
+
+        layout.addWidget(metadatastore_group)
+        layout.addStretch()
+
+        self.setLayout(layout)
+
+    # --- Settings management methods --- #
+    def reset(self):
+        """(Re)set to initial values."""
+        try:
+            self.caching_checkbox.setChecked(
+                settings["metadatastore"]["caching_enabled"]
+            )
+            self.searcher_checkbox.setChecked(
+                settings["metadatastore"]["searcher_enabled"]
+            )
+        except (KeyError, TypeError):
+            # Use defaults if settings don't exist yet
+            self.caching_checkbox.setChecked(True)
+            self.searcher_checkbox.setChecked(True)
+
+    def has_changes(self):
+        """Check if there are unsaved changes."""
+        try:
+            current_state = {
+                'caching_enabled': self.caching_checkbox.isChecked(),
+                'searcher_enabled': self.searcher_checkbox.isChecked(),
+            }
+            initial_state = {
+                'caching_enabled': settings["metadatastore"]["caching_enabled"],
+                'searcher_enabled': settings["metadatastore"]["searcher_enabled"],
+            }
+            return current_state != initial_state
+        except (KeyError, TypeError):
+            # If settings don't exist, check against defaults
+            return (self.caching_checkbox.isChecked() != True or
+                    self.searcher_checkbox.isChecked() != True)
+
+    def set_settings(self):
+        """Save metadatastore settings."""
+        if "metadatastore" not in settings.global_config:
+            settings.global_config["metadatastore"] = {}
+
+        settings.global_config["metadatastore"]["caching_enabled"] = self.caching_checkbox.isChecked()
+        settings.global_config["metadatastore"]["searcher_enabled"] = self.searcher_checkbox.isChecked()
+
+        logger.info(
+            f"Metadatastore settings saved: "
+            f"caching={self.caching_checkbox.isChecked()}, "
+            f"searcher={self.searcher_checkbox.isChecked()}"
+        )
+

activity_browser/app/pages/settings/settings_page.py

@@ -11,6 +11,7 @@
 from .startup import StartupSettingsChapter
 from .appearance import AppearanceSettingsChapter
 from .project_manager import ProjectManagerSettingsChapter
+from .metadatastore import MetadataStoreSettingsChapter
 
 
 class SettingsPage(QtWidgets.QWidget):
@@ -37,12 +38,14 @@ def __init__(self, parent=None):
         self.startup_chapter = StartupSettingsChapter(self)
         self.appearance_chapter = AppearanceSettingsChapter(self)
         self.project_manager_chapter = ProjectManagerSettingsChapter(self)
-        
+        self.metadatastore_chapter = MetadataStoreSettingsChapter(self)
+
         # Add chapters to the stack
         self.chapters = [
             ("Startup", self.startup_chapter),
             ("Appearance", self.appearance_chapter),
             ("Projects", self.project_manager_chapter),
+            ("Metadata Store", self.metadatastore_chapter),
         ]
 
         for name, widget in self.chapters:

activity_browser/bwutils/metadata/loader.py

@@ -8,6 +8,8 @@
 
 from qtpy.QtCore import QObject, QThread, Signal, SignalInstance
 
+from activity_browser.bwutils.settings import Settings
+
 from .metadata import MetaDataStore
 from .fields import secondary_types, primary, secondary, search_engine_whitelist, all_fields
 
@@ -42,7 +44,7 @@ def load_project(self):
         self.secondary_status = "loading"
 
         # check for valid cache and load from it if available
-        if self._has_cache():
+        if self._has_cache() and Settings()["metadatastore"]["caching_enabled"]:
             self.cache_load_project()
             return
 
@@ -266,6 +268,10 @@ def run(self):
             logger.debug("Skipping searcher initialization due to AB_NO_SEARCHER environment variable")
             return
 
+        if Settings()["metadatastore"]["searcher_enabled"] is False:
+            logger.debug("Skipping searcher initialization due to settings")
+            return
+
         if self.mds.searcher is not None:
             old_searcher = self.mds.searcher
             self.mds.searcher = None

activity_browser/bwutils/metadata/metadata.py

@@ -5,6 +5,7 @@
 
 import pandas as pd
 
+from activity_browser.bwutils.settings import Settings
 from .fields import all_fields, all_types
 
 
@@ -101,22 +102,22 @@ def flush_mutations(self) -> tuple[set[tuple[str, str]], set[tuple[str, str]], s
         self._updated.clear()
         self._deleted.clear()
 
-        cache_path = filesystem.get_project_ab_path() / "metadatastore_cache.pkl"
-        self._dataframe.to_pickle(cache_path)
+        if Settings()["metadatastore"]["caching_enabled"]:
+            cache_path = filesystem.get_project_ab_path() / "metadatastore_cache.pkl"
+            self._dataframe.to_pickle(cache_path)
 
         return added, updated, deleted
 
     def match(self, **kwargs: dict[str, str]) -> pd.DataFrame:
         """Return a slice of the dataframe matching the criteria.
         """
-        with self._df_lock:
-            df = self._dataframe.query(
-                " and ".join(
-                    [
-                        f"`{key}`.astype('str') == {str(value)!r}" if not pd.isna(value) else f"`{key}`.isnull()"
-                        for key, value in kwargs.items()
-                    ])
-            )
+        df = self._dataframe.query(
+            " and ".join(
+                [
+                    f"`{key}`.astype('str') == {str(value)!r}" if not pd.isna(value) else f"`{key}`.isnull()"
+                    for key, value in kwargs.items()
+                ])
+        )
 
         return df
 
@@ -142,23 +143,85 @@ def get_database_metadata(self, db_name: str, columns: list = None) -> pd.DataFr
         df = self._dataframe.loc[[db_name], columns]
         return df.reindex(columns, axis="columns")
 
+    def _pandas_search(self, query: str, database: str = None, columns: list = None) -> pd.DataFrame:
+        """Fallback pandas-based search when searcher is not initialized.
+
+        Args:
+            query: Search query string, may contain key:value parameters
+            database: Optional database name to restrict search
+            columns: Optional list of columns to return
+
+        Returns:
+            DataFrame with matching results
+        """
+        params, clean_query = get_query_parameters(query)
+        columns = columns if columns is not None else all_fields
+
+        # Start with the full dataframe or database subset
+        if database and database in self.databases:
+            df = self._dataframe.loc[[database]]
+        else:
+            df = self._dataframe
+
+        if not clean_query.strip():
+            # If no search query, just filter by parameters
+            if params:
+                extra_query = " & ".join(
+                    [
+                        f"`{key}`.astype('str').str.contains('{value}', case=False)"
+                        for key, value in params.items()
+                        if key in df.columns
+                    ]
+                )
+                if extra_query:
+                    df = df.query(extra_query)
+            return df[columns]
+
+        # Search across text fields: name, product, synonyms, categories, unit, location
+        search_fields = ['name', 'product', 'synonyms', 'categories', 'unit', 'location', 'CAS number']
+        mask = pd.Series([False] * len(df), index=df.index)
+
+        for field in search_fields:
+            if field in df.columns:
+                # Case-insensitive search
+                mask |= df[field].astype(str).str.contains(clean_query, case=False, na=False)
+
+        df = df[mask]
+
+        # Apply additional parameter filters if any
+        if params:
+            extra_query = " & ".join(
+                [
+                    f"`{key}`.astype('str').str.contains('{value}', case=False)"
+                    for key, value in params.items()
+                    if key in df.columns
+                ]
+            )
+            if extra_query:
+                df = df.query(extra_query)
+
+        return df[columns] if columns else df
+
     def search(self, query: str, columns: list = None) -> pd.DataFrame:
-        if not self.searcher:
-            logger.warning(f"Attempted to search metadata before searcher was initialized.")
-            return pd.DataFrame(columns=columns or all_fields)
+        if self.searcher:
+            # Advanced searcher is initialized, so use that
+            params, query = get_query_parameters(query)
+            result = self.searcher.search(query)
+            return self._meta_from_result(params, result, columns)
 
-        params, query = get_query_parameters(query)
-        result = self.searcher.search(query)
-        return self._meta_from_result(params, result, columns)
+        # Fallback to simple pandas search
+        logger.debug("Using simple pandas search as searcher is not initialized.")
+        return self._pandas_search(query, columns=columns)
 
     def search_database(self, query: str, database: str, columns: list = None) -> pd.DataFrame:
-        if not self.searcher:
-            logger.warning(f"Attempted to search metadata before searcher was initialized.")
-            return pd.DataFrame(columns=columns or all_fields)
-
-        params, query = get_query_parameters(query)
-        result = self.searcher.fuzzy_search(query, database=database)
-        return self._meta_from_result(params, result, columns)
+        if self.searcher:
+            params, query = get_query_parameters(query)
+            result = self.searcher.fuzzy_search(query, database=database)
+            return self._meta_from_result(params, result, columns)
+
+        # Fallback to simple pandas search
+        logger.debug(f"Using simple pandas search for database '{database}' as searcher is not initialized.")
+        return self._pandas_search(query, database=database, columns=columns)
 
     def _meta_from_result(self, params: dict, result: list[int], columns: list = None) -> pd.DataFrame:
         df = self._dataframe.loc[self.dataframe["id"].isin(result), columns or all_fields]

activity_browser/bwutils/settings.py

@@ -17,6 +17,10 @@
     "appearance": {
         "theme": "default",
         "pane_tab_position": "bottom",
+    },
+    "metadatastore": {
+        "caching_enabled": True,
+        "searcher_enabled": True,
     }
 }
 
@@ -52,7 +56,11 @@ def __getitem__(self, key):
             return self.virtual_config[key]
         if key in self.project_config:
             return self.project_config[key]
-        return self.global_config[key]
+        if key in self.global_config:
+            return self.global_config[key]
+        if key in defaults:
+            return defaults[key]
+        raise KeyError(f"Setting '{key}' not found in any configuration level.")
 
     def __setitem__(self, key, value):
         if isinstance(key, tuple):

activity_browser/ui/widgets/text_edit.py

@@ -174,6 +174,9 @@ def __init__(self, parent=None):
         self.database_name = ""
 
     def _sanitize_input(self):
+        if not self.mds.searcher:
+            return
+
         self._debounce_timer.stop()
         text = self.toPlainText()
         clean_text = self.mds.searcher.ONE_SPACE_PATTERN.sub(" ", text)
@@ -198,6 +201,9 @@ def _sanitize_input(self):
         self._set_debounce()
 
     def _set_autocomplete_items(self):
+        if not self.mds.searcher:
+            return
+
         text = self.toPlainText()
         if text.startswith("="):
             self.model.setStringList([])