diff --git a/releases/unreleased/handle-5xx-errors-on-every-request.yml b/releases/unreleased/handle-5xx-errors-on-every-request.yml new file mode 100644 index 0000000..a8b0cf1 --- /dev/null +++ b/releases/unreleased/handle-5xx-errors-on-every-request.yml @@ -0,0 +1,9 @@ +--- +title: 5xx errors handled for every request +category: added +author: Santiago DueƱas +issue: null +notes: > + Errors were only handled when profiles were fetched + and not in other cases. + diff --git a/sortinghat/core/importer/backends/eclipse.py b/sortinghat/core/importer/backends/eclipse.py index 295c96e..2ff7f1f 100644 --- a/sortinghat/core/importer/backends/eclipse.py +++ b/sortinghat/core/importer/backends/eclipse.py @@ -108,22 +108,9 @@ def get_individuals(self): # Fetch accounts pages for account in client.fetch_accounts(epoch=epoch): - try: - ef_profile = client.fetch_account_profile(account['name']) - except requests.exceptions.HTTPError as error: - # Ignore 5xx errors - if 500 <= error.response.status_code < 600: - msg = ( - f"Unable to fetch {account['name']} profile." - f"Server error: {error.response.status_code} - {error.response.reason}." - "Skipping" - ) - logger.error(msg) - continue - else: - raise error - - if not ef_profile['eca']['signed']: + ef_profile = client.fetch_account_profile(account['name']) + + if not ef_profile: continue individual = Individual(uuid=ef_profile['uid']) @@ -154,21 +141,22 @@ def get_individuals(self): ) individual.identities.append(idt) - employment_history = client.fetch_employment_history(account['name']) - # Fetch enrollments for the identity. If no enrollment is set # use the organization field from the profile, if set. - for employment in employment_history: - org = Organization(name=employment['organization_name']) - start, end = None, None + employment_history = client.fetch_employment_history(account['name']) - if employment['start']: - start = str_to_datetime(employment['start']) - if employment['end']: - end = str_to_datetime(employment['end']) + if employment_history: + for employment in employment_history: + org = Organization(name=employment['organization_name']) + start, end = None, None - enr = Enrollment(org, start=start, end=end) - individual.enrollments.append(enr) + if employment['start']: + start = str_to_datetime(employment['start']) + if employment['end']: + end = str_to_datetime(employment['end']) + + enr = Enrollment(org, start=start, end=end) + individual.enrollments.append(enr) if not individual.enrollments: company = ef_profile.get('org', None) @@ -292,8 +280,21 @@ def fetch_employment_history(self, eclipsefdn_id): def _fetch(self, url, params=None): """Generic query to Eclipse usr API.""" - response = requests.get(url, params=params, auth=self.token) - response.raise_for_status() + try: + response = requests.get(url, params=params, auth=self.token) + response.raise_for_status() + except requests.exceptions.HTTPError as error: + # Ignore 5xx errors + if 500 <= error.response.status_code < 600: + msg = ( + f"Unable to fetch {url}" + f"Server error: {error.response.status_code} - {error.response.reason}." + "Skipping" + ) + logger.error(msg) + return None + else: + raise error return response.json() diff --git a/tests/test_eclipse.py b/tests/test_eclipse.py index 1ed0dd7..007ecdf 100644 --- a/tests/test_eclipse.py +++ b/tests/test_eclipse.py @@ -98,6 +98,10 @@ def request_callback(request, uri, headers): ECLIPSE_API_URL + "/account/profile/jdoe", body="", status=504) + httpretty.register_uri(httpretty.GET, + ECLIPSE_API_URL + "/account/profile/jrae/employment-history", + body="", + status=504) return requests, bodies @@ -264,6 +268,8 @@ def test_ignore_5xx_errors(self, mock_login, mock_datetime_now): # In total, only 1 individual and 2 identities will be imported. # Individuals 'jsmith' and 'jdoe' profiles return 5xx errors. + # Employment info for 'jrae' also returns an error, so the affiliation + # info is taken from the profile page setting default dates. self.assertEqual(n, 2) individuals = Individual.objects.order_by('mk').all() @@ -288,6 +294,14 @@ def test_ignore_5xx_errors(self, mock_login, mock_datetime_now): self.assertEqual(ids[1].username, 'jrae') self.assertEqual(ids[1].source, 'github') + enrollments = jrae.enrollments.all() + self.assertEqual(len(enrollments), 1) + + rol = enrollments[0] + self.assertEqual(rol.group.name, "ACME") + self.assertEqual(rol.start, MIN_PERIOD_DATE) + self.assertEqual(rol.end, MAX_PERIOD_DATE) + @httpretty.activate @patch('sortinghat.core.importer.backends.eclipse.EclipseFoundationAPIClient.login', return_value="mocked_login") @patch('sortinghat.core.importer.backends.eclipse.datetime_utcnow', return_value=MOCK_DATETIME_NOW)