diff --git a/releases/unreleased/5xx-errors-ignored.yml b/releases/unreleased/5xx-errors-ignored.yml new file mode 100644 index 0000000..e01ffaa --- /dev/null +++ b/releases/unreleased/5xx-errors-ignored.yml @@ -0,0 +1,9 @@ +--- +title: 5xx errors ignored +category: added +author: Santiago Dueñas +issue: null +notes: > + The importer won't fail when 5xx errors are returned + by the server. These errors are normally due to invalid + identities stored in the platform. diff --git a/sortinghat/core/importer/backends/eclipse.py b/sortinghat/core/importer/backends/eclipse.py index c665508..295c96e 100644 --- a/sortinghat/core/importer/backends/eclipse.py +++ b/sortinghat/core/importer/backends/eclipse.py @@ -108,7 +108,20 @@ def get_individuals(self): # Fetch accounts pages for account in client.fetch_accounts(epoch=epoch): - ef_profile = client.fetch_account_profile(account['name']) + try: + ef_profile = client.fetch_account_profile(account['name']) + except requests.exceptions.HTTPError as error: + # Ignore 5xx errors + if 500 <= error.response.status_code < 600: + msg = ( + f"Unable to fetch {account['name']} profile." + f"Server error: {error.response.status_code} - {error.response.reason}." + "Skipping" + ) + logger.error(msg) + continue + else: + raise error if not ef_profile['eca']['signed']: continue diff --git a/tests/test_eclipse.py b/tests/test_eclipse.py index 1283304..1ed0dd7 100644 --- a/tests/test_eclipse.py +++ b/tests/test_eclipse.py @@ -31,7 +31,7 @@ def read_file(filename, mode='r'): return content -def setup_mock_server(): +def setup_mock_server(raise_5xx_errors=False): """Configure a Mock server""" def request_callback(request, uri, headers): @@ -88,6 +88,17 @@ def request_callback(request, uri, headers): ECLIPSE_API_URL + "/account/profile/jrae/employment-history", body=read_file('data/eclipse_jrae_employment.json')) + # Overwrite jsmith and jdoe profile replies to raise 5xx errors + if raise_5xx_errors: + httpretty.register_uri(httpretty.GET, + ECLIPSE_API_URL + "/account/profile/jsmith", + body="", + status=500) + httpretty.register_uri(httpretty.GET, + ECLIPSE_API_URL + "/account/profile/jdoe", + body="", + status=504) + return requests, bodies @@ -153,6 +164,7 @@ def test_backend_name(self): @patch('sortinghat.core.importer.backends.eclipse.EclipseFoundationAPIClient.login', return_value="mocked_login") @patch('sortinghat.core.importer.backends.eclipse.datetime_utcnow', return_value=MOCK_DATETIME_NOW) def test_import_identities(self, mock_login, mock_datetime_now): + """Check if identities are imported""" # Set up a mock HTTP server requests, bodies = setup_mock_server() @@ -222,6 +234,8 @@ def test_import_identities(self, mock_login, mock_datetime_now): @patch('sortinghat.core.importer.backends.eclipse.EclipseFoundationAPIClient.login', return_value="mocked_login") @patch('sortinghat.core.importer.backends.eclipse.datetime_utcnow', return_value=MOCK_DATETIME_NOW) def test_import_no_identities(self, mock_login, mock_datetime_now): + """Check if all goes ok when there aren't identities to import""" + # Set up a mock HTTP server requests, bodies = setup_mock_server() @@ -236,10 +250,49 @@ def test_import_no_identities(self, mock_login, mock_datetime_now): # No identities self.assertEqual(n, 0) + @httpretty.activate + @patch('sortinghat.core.importer.backends.eclipse.EclipseFoundationAPIClient.login', return_value="mocked_login") + @patch('sortinghat.core.importer.backends.eclipse.datetime_utcnow', return_value=MOCK_DATETIME_NOW) + def test_ignore_5xx_errors(self, mock_login, mock_datetime_now): + """Check if the importer ignores 5xx errors""" + + # Set up a mock HTTP server + requests, bodies = setup_mock_server(raise_5xx_errors=True) + + importer = EclipseFoundationAccountsImporter(ctx=self.ctx, url=None, from_date=None) + n = importer.import_identities() + + # In total, only 1 individual and 2 identities will be imported. + # Individuals 'jsmith' and 'jdoe' profiles return 5xx errors. + self.assertEqual(n, 2) + + individuals = Individual.objects.order_by('mk').all() + + self.assertEqual(len(individuals), 1) + + # Jane Rae + jrae = individuals[0] + self.assertEqual(jrae.profile.name, 'Jane Rae') + self.assertEqual(jrae.profile.email, 'jrae@example.com') + + ids = jrae.identities.order_by('uuid').all() + + self.assertEqual(len(ids), 2) + self.assertEqual(ids[0].name, 'Jane Rae') + self.assertEqual(ids[0].email, 'jrae@example.com') + self.assertEqual(ids[0].username, 'jrae') + self.assertEqual(ids[0].source, 'eclipsefdn') + + self.assertEqual(ids[1].name, 'Jane Rae') + self.assertEqual(ids[1].email, 'jrae@example.com') + self.assertEqual(ids[1].username, 'jrae') + self.assertEqual(ids[1].source, 'github') + @httpretty.activate @patch('sortinghat.core.importer.backends.eclipse.EclipseFoundationAPIClient.login', return_value="mocked_login") @patch('sortinghat.core.importer.backends.eclipse.datetime_utcnow', return_value=MOCK_DATETIME_NOW) def test_import_merge_identities(self, mock_login, mock_datetime_now): + """Check if existing identities are merged""" # Add individuals that share email and github handle api.add_identity(self.ctx, source='github', username='jsmith') @@ -261,6 +314,7 @@ def test_import_merge_identities(self, mock_login, mock_datetime_now): @patch('sortinghat.core.importer.backends.eclipse.EclipseFoundationAPIClient.login', return_value="mocked_login") @patch('sortinghat.core.importer.backends.eclipse.datetime_utcnow', return_value=MOCK_DATETIME_NOW) def test_import_enrollments(self, mock_login, mock_datetime_now): + """Check if enrolments are imported""" # Set up a mock HTTP server setup_mock_server()