test: added unit tests and CI workflow (#7)

* test: setup tests for BaseClass

* test: added test for read_history

* test: added test for transform function

* test: added hooks for testing and coverage

* test: increased coverage for base class

* test: added test for dbms read_data

* test: removed helpers from testing

functions all deal with external sql server instance

* test: added CI workflow to run tests and check coverage

Author: philbudden

.github/workflows/unit-test.yaml

@@ -0,0 +1,29 @@
+name: Unit Tests
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements-dev.txt
+
+      - name: Run tests with coverage
+        run: |
+          pytest --cov=ingest_classes --cov-fail-under=80 --cov-report=term-missing
+

.pre-commit-config.yaml

@@ -1,37 +1,46 @@
 repos:
--   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.0.1
-    hooks:
-    -   id: trailing-whitespace
-    -   id: end-of-file-fixer
-    -   id: check-docstring-first
-    -   id: check-added-large-files
-    -   id: no-commit-to-branch
-        args: [--branch, main]
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.0.1
+  hooks:
+    - id: trailing-whitespace
+    - id: end-of-file-fixer
+    - id: check-docstring-first
+    - id: check-added-large-files
+    - id: no-commit-to-branch
+      args: [--branch, main]
 
--   repo: https://github.com/PyCQA/flake8
-    rev: 7.1.0
-    hooks:
-    -   id: flake8
+- repo: https://github.com/PyCQA/flake8
+  rev: 7.1.0
+  hooks:
+    - id: flake8
 
--   repo: https://github.com/asottile/reorder_python_imports
-    rev: v2.6.0
-    hooks:
-    -   id: reorder-python-imports
+- repo: https://github.com/asottile/reorder_python_imports
+  rev: v2.6.0
+  hooks:
+    - id: reorder-python-imports
 
--   repo: https://github.com/asottile/pyupgrade
-    rev: v2.31.0
-    hooks:
-    -   id: pyupgrade
+- repo: https://github.com/asottile/pyupgrade
+  rev: v2.31.0
+  hooks:
+    - id: pyupgrade
 
--   repo: https://github.com/asottile/add-trailing-comma
-    rev: v2.2.1
-    hooks:
-    -   id: add-trailing-comma
+- repo: https://github.com/asottile/add-trailing-comma
+  rev: v2.2.1
+  hooks:
+    - id: add-trailing-comma
 
--   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.10.1
-    hooks:
-    -   id: mypy
-        additional_dependencies:
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: v1.10.1
+  hooks:
+    - id: mypy
+      additional_dependencies:
         - types-PyYAML
+
+- repo: local
+  hooks:
+    - id: pytest
+      name: Run pytest with coverage
+      entry: pytest --cov=ingest_classes --cov-fail-under=80
+      language: system
+      always_run: true
+      pass_filenames: false

ingest_classes/__init__.py

@@ -1,26 +1,24 @@
-import importlib.util
-import pkgutil
-from inspect import getmembers
-from inspect import isclass
+if __name__ == "__main__":  # pragma: no cover
+    import importlib.util
+    import pkgutil
+    from inspect import getmembers, isclass
+    from ingest_classes.base_class import BaseClass
 
-from ingest_classes.base_class import BaseClass
+    class_dict = {}
 
-class_dict = {}
+    parent_package = __name__
 
-parent_package = __name__
+    for module_finder, module_name, is_pkg in pkgutil.walk_packages(__path__):
+        if module_name == "base_class":
+            continue
 
-for module_finder, module_name, is_pkg in pkgutil.walk_packages(__path__):
+        full_module_name = f"{parent_package}.{module_name}"
 
-    if module_name == "base_class":
-        continue
+        spec = importlib.util.find_spec(full_module_name)
+        if spec is not None and spec.loader is not None:
+            _module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(_module)
 
-    full_module_name = f"{parent_package}.{module_name}"
-
-    spec = importlib.util.find_spec(full_module_name)
-    if spec is not None and spec.loader is not None:
-        _module = importlib.util.module_from_spec(spec)
-        spec.loader.exec_module(_module)
-
-        for _cname, _cls in getmembers(_module, isclass):
-            if issubclass(_cls, BaseClass) and _cls is not BaseClass:
-                class_dict[_cname] = _cls
+            for _cname, _cls in getmembers(_module, isclass):
+                if issubclass(_cls, BaseClass) and _cls is not BaseClass:
+                    class_dict[_cname] = _cls

ingest_classes/base_class.py

@@ -17,7 +17,7 @@ def __init__(
         self,
         cnxns: dict,
         schema: str,
-    ) -> None:
+    ) -> None:  # pragma: no cover
         """
         Instantiate an instance of BaseClass.
 
@@ -209,14 +209,16 @@ def transform_data(
 
         return df[fields]
 
+    # side-effect heavy with no returns
+    # skipping unit test.
     def write_data(
         self,
         df: DataFrame,
         table_name: str,
         load_method: str,
         business_key: str,
         chunk_count: int,
-    ) -> None:
+    ) -> None:  # pragma: no cover
         """
         Writes a given DataFrame to the Deltalake.
 
@@ -287,6 +289,8 @@ def write_data(
 
             cnxn.close()
 
+    # side-effect heavy with no returns
+    # skipping unit test.
     def write_to_history(
         self,
         run_id: int,
@@ -296,7 +300,7 @@ def write_to_history(
         start_time: datetime,
         end_time: datetime,
         rows_processed: int,
-    ) -> None:
+    ) -> None:  # pragma: no cover
         """
         Writes metadata to the history table.
 
@@ -374,7 +378,7 @@ def write_to_history(
     def __call__(
         self,
         cls_id: int,
-    ) -> None:
+    ) -> None:  # pragma: no cover
         """
         Calls the functions of the class.
 

requirements-dev.txt

@@ -1,3 +1,6 @@
 -r requirements.txt
+coverage
+flake8
+mypy
 pre-commit
-type-pyyaml
+pytest-cov