Merge branch 'fix_min_epoch_time' of 'https://github.com/sduenas/sortinghat-eclipse-foundation'

Merges #1
Closes #1

Author: jjmerchante

releases/unreleased/minimum-value-for-from_date-error.yml

@@ -0,0 +1,9 @@
+---
+title: Minimum value for 'from_date' error
+category: fixed
+author: Santiago Dueñas <sduenas@bitergia.com>
+issue: null
+notes: >
+  EclipseFoundation API only allows to fetch identities
+  updated since the last year. The backend allowed dates
+  before that but the server raised BadRequest error.

sortinghat/core/importer/backends/eclipse.py

@@ -17,6 +17,7 @@
 
 import logging
 
+import dateutil.relativedelta
 import requests
 
 from django.conf import settings
@@ -27,7 +28,10 @@
     OAuth2Client
 )
 
-from grimoirelab_toolkit.datetime import str_to_datetime
+from grimoirelab_toolkit.datetime import (
+    str_to_datetime,
+    datetime_utcnow
+)
 from sortinghat.core.importer.backend import IdentitiesImporter
 from sortinghat.core.importer.models import (
     Individual,
@@ -53,24 +57,43 @@ class EclipseFoundationAccountsImporter(IdentitiesImporter):
 
     The importer fetches and stores in the database identities
     created or updated after the given date (`from_date`) parameter.
-    When no date is given, it will import all the identities available
-    in the platform.
+    Currently, it can only import identities updated since a year ago.
+    When no date is given, it will import all the identities updated
+    since last year.
 
     Each individual created after importing will have two identities:
     one with source set as 'eclipsefdn' that includes their name, email
     and username as it comes from the platform, and a second one with
     source 'github' only when the github user was set by the identity
     on the Eclipse profile.
+
+    :param ctx: SortingHat context
+    :param url: not used on this importer
+    :param from_date: start fetching identities updated from this date
+
+    :raises ValueError: when the date is older than one year ago
     """
     NAME = "EclipseFoundation"
 
     def __init__(self, ctx, url, from_date=None):
         super().__init__(ctx, url)
 
-        if isinstance(from_date, str):
-            from_date = str_to_datetime(from_date)
+        min_date = datetime_utcnow() - dateutil.relativedelta.relativedelta(years=1)
 
-        self.from_date = from_date
+        if not from_date:
+            self.from_date = min_date
+        elif isinstance(from_date, str):
+            self.from_date = str_to_datetime(from_date)
+        else:
+            self.from_date = from_date
+
+        if self.from_date < min_date:
+            msg = (
+                "Invalid 'from_date' value. It can only import identities updated since a year ago."
+                "from_date=" + from_date
+            )
+            logger.error(msg)
+            raise ValueError(msg)
 
     def get_individuals(self):
         """Get the individuals from the Eclipse Foundation platform."""
@@ -81,7 +104,7 @@ def get_individuals(self):
         client = EclipseFoundationAPIClient()
         client.login(user_id, password)
 
-        epoch = int(self.from_date.timestamp()) if self.from_date else 1
+        epoch = int(self.from_date.timestamp())
 
         # Fetch accounts pages
         for account in client.fetch_accounts(epoch=epoch):

tests/test_eclipse.py

@@ -4,6 +4,8 @@
 from unittest.mock import patch
 
 import httpretty
+
+from dateutil.relativedelta import relativedelta
 from dateutil.tz import tzutc
 from django.test import TestCase
 from django.contrib.auth import get_user_model
@@ -20,6 +22,9 @@
 OAUTH_TOKEN_ENDPOINT = "https://accounts.eclipse.org/oauth2/token"
 
 
+MOCK_DATETIME_NOW = datetime.datetime(2025, 1, 1, tzinfo=tzutc())
+
+
 def read_file(filename, mode='r'):
     with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), filename), mode) as f:
         content = f.read()
@@ -95,38 +100,59 @@ def setUp(self):
         self.user = get_user_model().objects.create(username='test')
         self.ctx = SortingHatContext(self.user)
 
-    def test_initialization(self):
+    @patch('sortinghat.core.importer.backends.eclipse.datetime_utcnow', return_value=MOCK_DATETIME_NOW)
+    def test_initialization(self, mock_datetime_now):
         """Test whether attributes are initialized"""
 
         url = "https://test-url.com/"
 
-        importer = EclipseFoundationAccountsImporter(ctx=self.ctx,
-                                       url=url)
+        importer = EclipseFoundationAccountsImporter(
+            ctx=self.ctx,
+            url=url
+        )
 
         self.assertEqual(importer.url, url)
         self.assertEqual(importer.ctx, self.ctx)
         self.assertEqual(importer.NAME, "EclipseFoundation")
-        self.assertIsNone(importer.from_date)
+
+        # 'from_date' is 1 year before the current date (MOCK_DATETIME_NOW)
+        self.assertEqual(importer.from_date, mock_datetime_now.return_value - relativedelta(years=1))
+
+    @patch('sortinghat.core.importer.backends.eclipse.datetime_utcnow', return_value=MOCK_DATETIME_NOW)
+    def test_parse_from_date(self, mock_datetime_now):
+        """Check if from_date parameter is parsed correctly"""
 
         # Check from_date is parsed correctly
         importer = EclipseFoundationAccountsImporter(
             ctx=self.ctx,
-            url=url,
-            from_date="2023-12-01"
+            url="https://test-url.com/",
+            from_date="2025-12-01"
         )
-        self.assertEqual(importer.from_date, datetime.datetime(year=2023,
+        self.assertEqual(importer.from_date, datetime.datetime(year=2025,
                                                                month=12,
                                                                day=1,
                                                                tzinfo=tzutc()))
 
+    @patch('sortinghat.core.importer.backends.eclipse.datetime_utcnow', return_value=MOCK_DATETIME_NOW)
+    def test_from_date_older_than_one_year(self, mock_datetime_now):
+        """Check if an error is raised when the from_date is invalid"""
+
+        with self.assertRaises(ValueError):
+            _ = EclipseFoundationAccountsImporter(
+                ctx=self.ctx,
+                url="https://test-url.com/",
+                from_date="2000-01-01"
+            )
+
     def test_backend_name(self):
         """Test whether the NAME of the backend is right"""
 
         self.assertEqual(EclipseFoundationAccountsImporter.NAME, "EclipseFoundation")
 
     @httpretty.activate
     @patch('sortinghat.core.importer.backends.eclipse.EclipseFoundationAPIClient.login', return_value="mocked_login")
-    def test_import_identities(self, mock_login):
+    @patch('sortinghat.core.importer.backends.eclipse.datetime_utcnow', return_value=MOCK_DATETIME_NOW)
+    def test_import_identities(self, mock_login, mock_datetime_now):
 
         # Set up a mock HTTP server
         requests, bodies = setup_mock_server()
@@ -194,14 +220,15 @@ def test_import_identities(self, mock_login):
 
     @httpretty.activate
     @patch('sortinghat.core.importer.backends.eclipse.EclipseFoundationAPIClient.login', return_value="mocked_login")
-    def test_import_no_identities(self, mock_login):
+    @patch('sortinghat.core.importer.backends.eclipse.datetime_utcnow', return_value=MOCK_DATETIME_NOW)
+    def test_import_no_identities(self, mock_login, mock_datetime_now):
         # Set up a mock HTTP server
         requests, bodies = setup_mock_server()
 
         importer = EclipseFoundationAccountsImporter(
             ctx=self.ctx,
             url=None,
-            from_date=datetime.datetime(2100, 1, 1, 0, 0, 0, tzinfo=tzutc())
+            from_date="2100-1-1"
         )
 
         n = importer.import_identities()
@@ -211,7 +238,8 @@ def test_import_no_identities(self, mock_login):
 
     @httpretty.activate
     @patch('sortinghat.core.importer.backends.eclipse.EclipseFoundationAPIClient.login', return_value="mocked_login")
-    def test_import_merge_identities(self, mock_login):
+    @patch('sortinghat.core.importer.backends.eclipse.datetime_utcnow', return_value=MOCK_DATETIME_NOW)
+    def test_import_merge_identities(self, mock_login, mock_datetime_now):
 
         # Add individuals that share email and github handle
         api.add_identity(self.ctx, source='github', username='jsmith')
@@ -231,7 +259,8 @@ def test_import_merge_identities(self, mock_login):
 
     @httpretty.activate
     @patch('sortinghat.core.importer.backends.eclipse.EclipseFoundationAPIClient.login', return_value="mocked_login")
-    def test_import_enrollments(self, mock_login):
+    @patch('sortinghat.core.importer.backends.eclipse.datetime_utcnow', return_value=MOCK_DATETIME_NOW)
+    def test_import_enrollments(self, mock_login, mock_datetime_now):
 
         # Set up a mock HTTP server
         setup_mock_server()