diff --git a/releases/unreleased/minimum-value-for-from_date-error.yml b/releases/unreleased/minimum-value-for-from_date-error.yml new file mode 100644 index 0000000..5dccc15 --- /dev/null +++ b/releases/unreleased/minimum-value-for-from_date-error.yml @@ -0,0 +1,9 @@ +--- +title: Minimum value for 'from_date' error +category: fixed +author: Santiago Dueñas +issue: null +notes: > + EclipseFoundation API only allows to fetch identities + updated since the last year. The backend allowed dates + before that but the server raised BadRequest error. diff --git a/sortinghat/core/importer/backends/eclipse.py b/sortinghat/core/importer/backends/eclipse.py index da0f7a3..c665508 100644 --- a/sortinghat/core/importer/backends/eclipse.py +++ b/sortinghat/core/importer/backends/eclipse.py @@ -17,6 +17,7 @@ import logging +import dateutil.relativedelta import requests from django.conf import settings @@ -27,7 +28,10 @@ OAuth2Client ) -from grimoirelab_toolkit.datetime import str_to_datetime +from grimoirelab_toolkit.datetime import ( + str_to_datetime, + datetime_utcnow +) from sortinghat.core.importer.backend import IdentitiesImporter from sortinghat.core.importer.models import ( Individual, @@ -53,24 +57,43 @@ class EclipseFoundationAccountsImporter(IdentitiesImporter): The importer fetches and stores in the database identities created or updated after the given date (`from_date`) parameter. - When no date is given, it will import all the identities available - in the platform. + Currently, it can only import identities updated since a year ago. + When no date is given, it will import all the identities updated + since last year. Each individual created after importing will have two identities: one with source set as 'eclipsefdn' that includes their name, email and username as it comes from the platform, and a second one with source 'github' only when the github user was set by the identity on the Eclipse profile. + + :param ctx: SortingHat context + :param url: not used on this importer + :param from_date: start fetching identities updated from this date + + :raises ValueError: when the date is older than one year ago """ NAME = "EclipseFoundation" def __init__(self, ctx, url, from_date=None): super().__init__(ctx, url) - if isinstance(from_date, str): - from_date = str_to_datetime(from_date) + min_date = datetime_utcnow() - dateutil.relativedelta.relativedelta(years=1) - self.from_date = from_date + if not from_date: + self.from_date = min_date + elif isinstance(from_date, str): + self.from_date = str_to_datetime(from_date) + else: + self.from_date = from_date + + if self.from_date < min_date: + msg = ( + "Invalid 'from_date' value. It can only import identities updated since a year ago." + "from_date=" + from_date + ) + logger.error(msg) + raise ValueError(msg) def get_individuals(self): """Get the individuals from the Eclipse Foundation platform.""" @@ -81,7 +104,7 @@ def get_individuals(self): client = EclipseFoundationAPIClient() client.login(user_id, password) - epoch = int(self.from_date.timestamp()) if self.from_date else 1 + epoch = int(self.from_date.timestamp()) # Fetch accounts pages for account in client.fetch_accounts(epoch=epoch): diff --git a/tests/test_eclipse.py b/tests/test_eclipse.py index 2782a26..1283304 100644 --- a/tests/test_eclipse.py +++ b/tests/test_eclipse.py @@ -4,6 +4,8 @@ from unittest.mock import patch import httpretty + +from dateutil.relativedelta import relativedelta from dateutil.tz import tzutc from django.test import TestCase from django.contrib.auth import get_user_model @@ -20,6 +22,9 @@ OAUTH_TOKEN_ENDPOINT = "https://accounts.eclipse.org/oauth2/token" +MOCK_DATETIME_NOW = datetime.datetime(2025, 1, 1, tzinfo=tzutc()) + + def read_file(filename, mode='r'): with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), filename), mode) as f: content = f.read() @@ -95,30 +100,50 @@ def setUp(self): self.user = get_user_model().objects.create(username='test') self.ctx = SortingHatContext(self.user) - def test_initialization(self): + @patch('sortinghat.core.importer.backends.eclipse.datetime_utcnow', return_value=MOCK_DATETIME_NOW) + def test_initialization(self, mock_datetime_now): """Test whether attributes are initialized""" url = "https://test-url.com/" - importer = EclipseFoundationAccountsImporter(ctx=self.ctx, - url=url) + importer = EclipseFoundationAccountsImporter( + ctx=self.ctx, + url=url + ) self.assertEqual(importer.url, url) self.assertEqual(importer.ctx, self.ctx) self.assertEqual(importer.NAME, "EclipseFoundation") - self.assertIsNone(importer.from_date) + + # 'from_date' is 1 year before the current date (MOCK_DATETIME_NOW) + self.assertEqual(importer.from_date, mock_datetime_now.return_value - relativedelta(years=1)) + + @patch('sortinghat.core.importer.backends.eclipse.datetime_utcnow', return_value=MOCK_DATETIME_NOW) + def test_parse_from_date(self, mock_datetime_now): + """Check if from_date parameter is parsed correctly""" # Check from_date is parsed correctly importer = EclipseFoundationAccountsImporter( ctx=self.ctx, - url=url, - from_date="2023-12-01" + url="https://test-url.com/", + from_date="2025-12-01" ) - self.assertEqual(importer.from_date, datetime.datetime(year=2023, + self.assertEqual(importer.from_date, datetime.datetime(year=2025, month=12, day=1, tzinfo=tzutc())) + @patch('sortinghat.core.importer.backends.eclipse.datetime_utcnow', return_value=MOCK_DATETIME_NOW) + def test_from_date_older_than_one_year(self, mock_datetime_now): + """Check if an error is raised when the from_date is invalid""" + + with self.assertRaises(ValueError): + _ = EclipseFoundationAccountsImporter( + ctx=self.ctx, + url="https://test-url.com/", + from_date="2000-01-01" + ) + def test_backend_name(self): """Test whether the NAME of the backend is right""" @@ -126,7 +151,8 @@ def test_backend_name(self): @httpretty.activate @patch('sortinghat.core.importer.backends.eclipse.EclipseFoundationAPIClient.login', return_value="mocked_login") - def test_import_identities(self, mock_login): + @patch('sortinghat.core.importer.backends.eclipse.datetime_utcnow', return_value=MOCK_DATETIME_NOW) + def test_import_identities(self, mock_login, mock_datetime_now): # Set up a mock HTTP server requests, bodies = setup_mock_server() @@ -194,14 +220,15 @@ def test_import_identities(self, mock_login): @httpretty.activate @patch('sortinghat.core.importer.backends.eclipse.EclipseFoundationAPIClient.login', return_value="mocked_login") - def test_import_no_identities(self, mock_login): + @patch('sortinghat.core.importer.backends.eclipse.datetime_utcnow', return_value=MOCK_DATETIME_NOW) + def test_import_no_identities(self, mock_login, mock_datetime_now): # Set up a mock HTTP server requests, bodies = setup_mock_server() importer = EclipseFoundationAccountsImporter( ctx=self.ctx, url=None, - from_date=datetime.datetime(2100, 1, 1, 0, 0, 0, tzinfo=tzutc()) + from_date="2100-1-1" ) n = importer.import_identities() @@ -211,7 +238,8 @@ def test_import_no_identities(self, mock_login): @httpretty.activate @patch('sortinghat.core.importer.backends.eclipse.EclipseFoundationAPIClient.login', return_value="mocked_login") - def test_import_merge_identities(self, mock_login): + @patch('sortinghat.core.importer.backends.eclipse.datetime_utcnow', return_value=MOCK_DATETIME_NOW) + def test_import_merge_identities(self, mock_login, mock_datetime_now): # Add individuals that share email and github handle api.add_identity(self.ctx, source='github', username='jsmith') @@ -231,7 +259,8 @@ def test_import_merge_identities(self, mock_login): @httpretty.activate @patch('sortinghat.core.importer.backends.eclipse.EclipseFoundationAPIClient.login', return_value="mocked_login") - def test_import_enrollments(self, mock_login): + @patch('sortinghat.core.importer.backends.eclipse.datetime_utcnow', return_value=MOCK_DATETIME_NOW) + def test_import_enrollments(self, mock_login, mock_datetime_now): # Set up a mock HTTP server setup_mock_server()