Handle 5xx errors for every requests

HTTP 5xx errors were only handle when fetching profile data.
With this change, these errors are handle for every HTTP
request.

Signed-off-by: Santiago Dueñas <sduenas@bitergia.com>

Author: sduenas

releases/unreleased/handle-5xx-errors-on-every-request.yml

@@ -0,0 +1,9 @@
+---
+title: 5xx errors handled for every request
+category: added
+author: Santiago Dueñas <sduenas@bitergia.com>
+issue: null
+notes: >
+  Errors were only handled when profiles were fetched
+  and not in other cases.
+

sortinghat/core/importer/backends/eclipse.py

@@ -108,22 +108,9 @@ def get_individuals(self):
 
         # Fetch accounts pages
         for account in client.fetch_accounts(epoch=epoch):
-            try:
-                ef_profile = client.fetch_account_profile(account['name'])
-            except requests.exceptions.HTTPError as error:
-                # Ignore 5xx errors
-                if 500 <= error.response.status_code < 600:
-                    msg = (
-                        f"Unable to fetch {account['name']} profile."
-                        f"Server error: {error.response.status_code} - {error.response.reason}."
-                        "Skipping"
-                    )
-                    logger.error(msg)
-                    continue
-                else:
-                    raise error
-
-            if not ef_profile['eca']['signed']:
+            ef_profile = client.fetch_account_profile(account['name'])
+
+            if not ef_profile:
                 continue
 
             individual = Individual(uuid=ef_profile['uid'])
@@ -154,21 +141,22 @@ def get_individuals(self):
                 )
                 individual.identities.append(idt)
 
-            employment_history = client.fetch_employment_history(account['name'])
-
             # Fetch enrollments for the identity. If no enrollment is set
             # use the organization field from the profile, if set.
-            for employment in employment_history:
-                org = Organization(name=employment['organization_name'])
-                start, end = None, None
+            employment_history = client.fetch_employment_history(account['name'])
 
-                if employment['start']:
-                    start = str_to_datetime(employment['start'])
-                if employment['end']:
-                    end = str_to_datetime(employment['end'])
+            if employment_history:
+                for employment in employment_history:
+                    org = Organization(name=employment['organization_name'])
+                    start, end = None, None
 
-                enr = Enrollment(org, start=start, end=end)
-                individual.enrollments.append(enr)
+                    if employment['start']:
+                        start = str_to_datetime(employment['start'])
+                    if employment['end']:
+                        end = str_to_datetime(employment['end'])
+
+                    enr = Enrollment(org, start=start, end=end)
+                    individual.enrollments.append(enr)
 
             if not individual.enrollments:
                 company = ef_profile.get('org', None)
@@ -292,8 +280,21 @@ def fetch_employment_history(self, eclipsefdn_id):
     def _fetch(self, url, params=None):
         """Generic query to Eclipse usr API."""
 
-        response = requests.get(url, params=params, auth=self.token)
-        response.raise_for_status()
+        try:
+            response = requests.get(url, params=params, auth=self.token)
+            response.raise_for_status()
+        except requests.exceptions.HTTPError as error:
+            # Ignore 5xx errors
+            if 500 <= error.response.status_code < 600:
+                msg = (
+                    f"Unable to fetch {url}"
+                    f"Server error: {error.response.status_code} - {error.response.reason}."
+                    "Skipping"
+                )
+                logger.error(msg)
+                return None
+            else:
+                raise error
 
         return response.json()
 

tests/test_eclipse.py

@@ -98,6 +98,10 @@ def request_callback(request, uri, headers):
                                ECLIPSE_API_URL + "/account/profile/jdoe",
                                body="",
                                status=504)
+        httpretty.register_uri(httpretty.GET,
+                               ECLIPSE_API_URL + "/account/profile/jrae/employment-history",
+                               body="",
+                               status=504)
 
     return requests, bodies
 
@@ -264,6 +268,8 @@ def test_ignore_5xx_errors(self, mock_login, mock_datetime_now):
 
         # In total, only 1 individual and 2 identities will be imported.
         # Individuals 'jsmith' and 'jdoe' profiles return 5xx errors.
+        # Employment info for 'jrae' also returns an error, so the affiliation
+        # info is taken from the profile page setting default dates.
         self.assertEqual(n, 2)
 
         individuals = Individual.objects.order_by('mk').all()
@@ -288,6 +294,14 @@ def test_ignore_5xx_errors(self, mock_login, mock_datetime_now):
         self.assertEqual(ids[1].username, 'jrae')
         self.assertEqual(ids[1].source, 'github')
 
+        enrollments = jrae.enrollments.all()
+        self.assertEqual(len(enrollments), 1)
+
+        rol = enrollments[0]
+        self.assertEqual(rol.group.name, "ACME")
+        self.assertEqual(rol.start, MIN_PERIOD_DATE)
+        self.assertEqual(rol.end, MAX_PERIOD_DATE)
+
     @httpretty.activate
     @patch('sortinghat.core.importer.backends.eclipse.EclipseFoundationAPIClient.login', return_value="mocked_login")
     @patch('sortinghat.core.importer.backends.eclipse.datetime_utcnow', return_value=MOCK_DATETIME_NOW)