Add objects-user fixture and cleanup

Author: href

api.py

@@ -1,17 +1,20 @@
+import boto3
 import re
 import requests
 import time
 
 from constants import API_TOKEN
 from constants import API_URL
 from constants import LOCKS_PATH
+from constants import OBJECTS_URL
 from constants import PROCESS_ID
 from constants import RUNNER_ID
 from errors import Timeout
 from events import trigger
 from filelock import FileLock
 from requests.adapters import HTTPAdapter
 from requests.packages.urllib3.util.retry import Retry
+from resources import ObjectsUser
 from urllib.parse import urlparse
 
 
@@ -81,11 +84,13 @@ def __init__(self, scope, zone=None, read_only=False):
             agent_suffix = ''
 
         self.api_url = API_URL
+        self.objects_url = OBJECTS_URL
         self.headers['Authorization'] = f'Bearer {API_TOKEN}'
         self.headers['User-Agent'] = f'Acceptance Tests{agent_suffix}'
         self.hooks = {'response': self.on_response}
         self.scope = scope
         self.read_only = read_only
+        self.zone = zone
 
         # 8 Retries @ 2.5 backoff_factor = 10.6 minutes
         retry_strategy = RetryStrategy(
@@ -99,6 +104,10 @@ def __init__(self, scope, zone=None, read_only=False):
 
         self.mount("https://", adapter)
 
+        # This is None, when running "invoke cleanup"
+        if self.zone:
+            self.objects_endpoint = self.objects_endpoint_for(self.zone)
+
     def post(self, url, data=None, json=None, add_tags=True, **kwargs):
         assert not data, "Please only use json, not data"
 
@@ -107,6 +116,7 @@ def post(self, url, data=None, json=None, add_tags=True, **kwargs):
                 'runner': RUNNER_ID,
                 'process': PROCESS_ID,
                 'scope': self.scope,
+                'zone': self.zone,
             }
 
         return super().post(url, data=data, json=json, **kwargs)
@@ -156,6 +166,7 @@ def resources(path):
         yield from resources('/networks')
         yield from resources('/server-groups')
         yield from resources('/custom-images')
+        yield from resources('/objects-users')
 
     def cleanup(self, limit_to_scope=True, limit_to_process=True):
         """ Deletes resources created by this API object. """
@@ -179,6 +190,9 @@ def cleanup(self, limit_to_scope=True, limit_to_process=True):
         if exceptions:
             raise ExceptionGroup("Failures during cleanup.", exceptions)
 
+    def objects_endpoint_for(self, zone):
+        return self.objects_url.format(region=zone.rstrip('0123456789'))
+
 
 def delete_handler(path):
     """ Registers the decorated function as delete handler for the given
@@ -244,3 +258,25 @@ def delete_volume_snapshots(api, url):
             f'Snapshot failed to delete within 60 seconds. Status '
             f'is still "{snapshot.json()["status"]}".'
         )
+
+
+@delete_handler(path='/v1/objects-users/.+')
+def delete_objects_users(api, url):
+    """ Before deleting an objects user, we have to delete owned buckets. """
+
+    user = ObjectsUser.from_href(None, api, url, name="")
+    user.wait_for_access()
+
+    session = boto3.Session(
+        aws_access_key_id=user.keys[0]['access_key'],
+        aws_secret_access_key=user.keys[0]['secret_key'],
+    )
+
+    objects_endpoint = api.objects_endpoint_for(zone=user.tags['zone'])
+    s3 = session.resource('s3', endpoint_url=objects_endpoint)
+
+    for bucket in s3.buckets.all():
+        bucket.objects.all().delete()
+        bucket.delete()
+
+    api.request("DELETE", url)

conftest.py

@@ -1,3 +1,4 @@
+import boto3
 import os
 import pytest
 import random
@@ -21,6 +22,7 @@
 from resources import FloatingIP
 from resources import LoadBalancer
 from resources import Network
+from resources import ObjectsUser
 from resources import Server
 from resources import ServerGroup
 from resources import Volume
@@ -829,3 +831,51 @@ def factory(num_backends,
         return load_balancer, listener, pool, backends, private_network
 
     return factory
+
+
+@pytest.fixture(scope='session')
+def create_objects_user(request, session_api, objects_endpoint):
+    """ Factory to create an objects user. """
+
+    factory = ObjectsUser.factory(
+        request=request,
+        api=session_api,
+    )
+
+    def wrapper(*args, **kwargs):
+        user = factory(*args, **kwargs)
+
+        # We need to wait for a moment for the key to become available.
+        user.wait_for_access()
+
+        return user
+
+    return wrapper
+
+
+@pytest.fixture(scope='session')
+def objects_user(create_objects_user):
+    """ An object user that can be used with objects_endpoint. """
+
+    return create_objects_user(name=f'at-{secrets.token_hex(8)}')
+
+
+@pytest.fixture(scope='session')
+def objects_endpoint(session_api):
+    """ An objects endpoint of the given zone. """
+
+    return session_api.objects_endpoint
+
+
+@pytest.fixture(scope='session')
+def access_key(objects_user):
+    """ An S3 access key for the objects endpoint. """
+
+    return objects_user.keys[0]["access_key"]
+
+
+@pytest.fixture(scope='session')
+def secret_key(objects_user):
+    """ An S3 secret key for the objects endpoint. """
+
+    return objects_user.keys[0]["secret_key"]

constants.py

@@ -9,7 +9,15 @@
 if not os.environ.get('CLOUDSCALE_API_TOKEN'):
     raise RuntimeError(
         "No valid API token found in the CLOUDSCALE_API_TOKEN "
-        "environment variable"
+        "environment variable."
+    )
+
+if (('CLOUDSCALE_API_URL' in os.environ)
+        != ('CLOUDSCALE_OBJECTS_URL' in os.environ)):
+    raise RuntimeError(
+        "Either the CLOUDSCALE_API_URL or the CLOUDSCALE_OBJECTS_URL "
+        "environment variable is set, but not both. Set both variables or "
+        "use the defaults."
     )
 
 # The API token is used to distinguish tests from various runners. If you have
@@ -24,6 +32,15 @@
 API_URL = os.environ.get('CLOUDSCALE_API_URL', 'https://api.cloudscale.ch/v1')
 API_URL = API_URL.rstrip('/')
 
+OBJECTS_URL = os.environ.get(
+    'CLOUDSCALE_OBJECTS_URL', 'https://objects.{region}.cloudscale.ch')
+OBJECTS_URL = OBJECTS_URL.rstrip('/')
+if '{region}' not in OBJECTS_URL:
+    raise RuntimeError(
+        'The CLOUDSCALE_OBJECTS_URL variable must contain the "{region}" '
+        'template.'
+    )
+
 # One external ping target per IP version, that is assumed to be online
 PUBLIC_PING_TARGETS = {
     4: '8.8.8.8',

requirements.txt

@@ -1,3 +1,4 @@
+boto3
 dnspython
 filelock
 flake8

resources.py

@@ -1,3 +1,5 @@
+import boto3
+import botocore
 import re
 import secrets
 import textwrap
@@ -1454,3 +1456,40 @@ def verify_backend(self, prober, backend, count=1, port=None):
         for i in range(count):
             assert (prober.http_get(self.build_url(url='/hostname', port=port))
                     == backend.name)
+
+
+class ObjectsUser(CloudscaleResource):
+
+    def __init__(self, request, api, name):
+        super().__init__(request, api)
+
+        self.spec = {
+            'display_name': f'{RESOURCE_NAME_PREFIX}-{name}',
+        }
+
+    @with_trigger('objects-user.create')
+    def create(self):
+        self.info = self.api.post('/objects-users', json=self.spec).json()
+
+    def wait_for_access(self, timeout=30):
+        objects_endpoint = self.api.objects_endpoint_for(self.tags['zone'])
+
+        s3 = boto3.client(
+            's3',
+            endpoint_url=objects_endpoint,
+            aws_access_key_id=self.keys[0]["access_key"],
+            aws_secret_access_key=self.keys[0]["secret_key"],
+        )
+
+        timeout = time.monotonic() + 30
+        exception = None
+
+        while time.monotonic() < timeout:
+            try:
+                s3.list_buckets()
+                break
+            except botocore.exceptions.ClientError as e:
+                exception = e
+                time.sleep(1)
+        else:
+            raise TimeoutError from exception