Merge pull request #20 from keboola/erik-ci-tmp-fix

AZ pipelines fixup

Author: ErikZigo

.env

@@ -1,4 +1,4 @@
-STORAGE_API_URL=
+HOSTNAME_SUFFIX=
 TEST_MANAGE_API_APPLICATION_TOKEN=
 TEST_NOTIFICATION_API_URL=
 TEST_STORAGE_API_PROJECT_ID=

README.md

@@ -68,7 +68,7 @@ $clientFactory->getEventsClient('xxx-xxxxx-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
     ```
 
 - Set the following environment variables in `set-env.sh` file (use `set-env.template.sh` as sample):
-    - `STORAGE_API_URL` - Keboola Connection URL to arbitrary stack where the notification service is registered.
+    - `HOSTNAME_SUFFIX` - Suffix for the hostname, e.g. `eu-west-1.aws.keboola.dev`.
     - `TEST_STORAGE_API_TOKEN` - Token to a test project. 
     - `TEST_STORAGE_API_PROJECT_ID` - Project ID of the test project.
     - `TEST_MANAGE_API_APPLICATION_TOKEN` - Application token with scope `notifications:push-event`.
@@ -82,7 +82,6 @@ $clientFactory->getEventsClient('xxx-xxxxx-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
         - `TEST_AWS_ACCESS_KEY` - The created security credentials for the `JobQueueApiPhpClient` user.
         - `TEST_AWS_SECRET_ACCESS_KEY` - The created security credentials for the `JobQueueApiPhpClient` user.
         - `AWS_REGION` - `Region` output of the above stack.
-        - `AWS_LOGS_S3_BUCKET` - `S3LogsBucket` output of the above stackk.
 
 ### Azure Setup
 
@@ -123,18 +122,13 @@ $clientFactory->getEventsClient('xxx-xxxxx-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
     az storage account show-connection-string -g testing-notification-api-php-client -n mirontfcnacc2 --query "connectionString" --output tsv
     ```
 
-Set the connection string and container name you provided as parameter to the create command to following environment variables in the `set-env.sh` file:
- - AZURE_LOGS_ABS_CONTAINER 
- - AZURE_LOGS_ABS_CONNECTION_STRING
-
 ## Generate environment configuration
 
 ```bash
 export DATABASE_URL_BASE64=$(printf "mysql://root:root@dev-mysql-service:3310/notifications?serverVersion=8.0" | base64 --wrap=0)
 export DATABASE_PASSWORD_BASE64=$(printf "root" | base64 --wrap=0)
 export TEST_AZURE_CLIENT_SECRET_BASE64=$(printf "%s" "$TEST_AZURE_CLIENT_SECRET"| base64 --wrap=0)
 export TEST_AWS_SECRET_ACCESS_KEY_BASE64=$(printf "%s" "$TEST_AWS_SECRET_ACCESS_KEY"| base64 --wrap=0)
-export AZURE_LOGS_ABS_CONNECTION_STRING_BASE64=$(printf "%s" "$AZURE_LOGS_ABS_CONNECTION_STRING"| base64 --wrap=0)
 
 ./set-env.sh
 envsubst < provisioning/environments.yaml.template > provisioning/environments.yaml
@@ -152,15 +146,15 @@ Store the result `TEST_NOTIFICATION_API_URL` in `set-env.sh`.
 - With the above setup, you can run tests:
 
     ```bash
-    docker-compose build
-    source ./set-env.sh && docker-compose run tests
+    docker compose build
+    source ./set-env.sh && docker compose run tests
     ```
 
 - To run tests with local code use:
 
     ```bash
-    docker-compose run tests-local composer install
-    source ./set-env.sh && docker-compose run tests-local
+    docker compose run tests-local composer install
+    source ./set-env.sh && docker compose run tests-local
     ```
 
 ## License

azure-pipelines.yml

@@ -43,11 +43,11 @@ stages:
             name: prepare_cluster
             env:
               KUBERNETES_NAMESPACE: $(k8sNamespace)
-              STORAGE_API_URL: $(STORAGE_API_URL)
-              AWS_LOGS_S3_BUCKET: $(AWS_LOGS_S3_BUCKET)
               AWS_REGION: $(AWS_REGION)
               TEST_AWS_ACCESS_KEY_ID: $(TEST_AWS_ACCESS_KEY_ID)
               TEST_AWS_SECRET_ACCESS_KEY: $(TEST_AWS_SECRET_ACCESS_KEY)
+              HOSTNAME_SUFFIX: $(HOSTNAME_SUFFIX)
+              AWS_SQS_QUEUE_URL: $(AWS_SQS_QUEUE_URL)
 
   - stage: tests
     dependsOn: prepare
@@ -63,15 +63,15 @@ stages:
 
               echo "Notification API URL: $NOTIFICATION_API_IP"
 
-              docker-compose build --pull
-              docker-compose run --rm tests
+              docker compose build --pull
+              docker compose run --rm tests
             displayName: 'Build & Run Tests'
             env:
               TEST_STORAGE_API_TOKEN: $(TEST_STORAGE_API_TOKEN)
               TEST_MANAGE_API_APPLICATION_TOKEN: $(TEST_MANAGE_API_APPLICATION_TOKEN)
               TEST_NOTIFICATION_API_URL: http://$(NOTIFICATION_API_IP):8181
-              STORAGE_API_URL: $(STORAGE_API_URL)
               TEST_STORAGE_API_PROJECT_ID: $(TEST_STORAGE_API_PROJECT_ID)
+              HOSTNAME_SUFFIX: $(HOSTNAME_SUFFIX)
 
           - task: PublishTestResults@2
             displayName: 'Publish Test results'

docker-compose.yml

@@ -10,8 +10,8 @@ services:
       TEST_NOTIFICATION_API_URL:
       TEST_STORAGE_API_TOKEN:
       TEST_MANAGE_API_APPLICATION_TOKEN:
-      STORAGE_API_URL:
       TEST_STORAGE_API_PROJECT_ID:
+      HOSTNAME_SUFFIX:
     command: ['composer', 'ci']
 
   tests-local:

provisioning/aws.json

@@ -3,19 +3,10 @@
     "Description": "Resources for Notification Tests",
     "Parameters": {},
     "Resources": {
-        "NotificationS3LogsBucket": {
-            "Type": "AWS::S3::Bucket",
+        "NotificationQueue": {
+            "Type": "AWS::SQS::Queue",
             "Properties": {
-                "LifecycleConfiguration": {
-                    "Rules": [
-                        {
-                            "Id": "Delete debug files",
-                            "Prefix": "debug-files/",
-                            "ExpirationInDays": 7,
-                            "Status": "Enabled"
-                        }
-                    ]
-                },
+                "VisibilityTimeout": 30,
                 "Tags": [
                     {
                         "Key": "KeboolaStack",
@@ -33,37 +24,23 @@
                 "Path": "/",
                 "Policies": [
                     {
-                        "PolicyName": "UseS3BucketPolicy",
+                        "PolicyName": "UseSQSQueuePolicy",
                         "PolicyDocument": {
                             "Statement": [
                                 {
-                                    "Sid": "UseS3Bucket",
+                                    "Sid": "UseSQSQueue",
                                     "Effect": "Allow",
                                     "Action": [
-                                        "s3:PutObject",
-                                        "s3:GetObject",
-                                        "s3:ListBucket",
-                                        "s3:DeleteObject"
+                                        "sqs:GetQueueAttributes",
+                                        "sqs:SendMessage",
+                                        "sqs:ReceiveMessage",
+                                        "sqs:DeleteMessage",
+                                        "sqs:ChangeMessageVisibility"
                                     ],
                                     "Resource": [
-                                        {
-                                            "Fn::Join": [
-                                                "",
-                                                [
-                                                    "",
-                                                    {
-                                                        "Fn::GetAtt": [
-                                                            "NotificationS3LogsBucket",
-                                                            "Arn"
-                                                        ]
-                                                    },
-                                                    "/*"
-                                                ]
-                                            ]
-                                        },
                                         {
                                             "Fn::GetAtt": [
-                                                "NotificationS3LogsBucket",
+                                                "NotificationQueue",
                                                 "Arn"
                                             ]
                                         }
@@ -75,7 +52,7 @@
                 ]
             },
             "DependsOn": [
-                "NotificationS3LogsBucket"
+                "NotificationQueue"
             ]
         }
     },
@@ -90,9 +67,17 @@
                 "Ref": "AWS::Region"
             }
         },
-        "S3LogsBucket": {
+        "SQSQueueUrl": {
             "Value": {
-                "Ref": "NotificationS3LogsBucket"
+                "Ref": "NotificationQueue"
+            }
+        },
+        "SQSQueueArn": {
+            "Value": {
+                "Fn::GetAtt": [
+                    "NotificationQueue",
+                    "Arn"
+                ]
             }
         }
     }

provisioning/ci-setup-cluster.sh

@@ -28,6 +28,8 @@ export DATABASE_URL_BASE64
 export DATABASE_PASSWORD_BASE64
 export TEST_AWS_SECRET_ACCESS_KEY_BASE64
 
+export MESSENGER_TRANSPORT_DSN="${AWS_SQS_QUEUE_URL}?auto_setup=false"
+
 envsubst < ./provisioning/environments.yaml.template > ./provisioning/environments.yaml
 kubectl apply -f ./provisioning/environments.yaml
 kubectl apply -f ./provisioning/notification.yaml

provisioning/environments.yaml.template

@@ -12,7 +12,6 @@ metadata:
 type: Opaque
 data:
   awsSecretAccessKey: '$TEST_AWS_SECRET_ACCESS_KEY_BASE64'
-  azureLogsAbsConnectionString: '$AZURE_LOGS_ABS_CONNECTION_STRING_BASE64'
   azureClientSecret: '$TEST_AZURE_CLIENT_SECRET_BASE64'
   databaseUrl: '$DATABASE_URL_BASE64'
   databasePassword: '$DATABASE_PASSWORD_BASE64'
@@ -26,12 +25,10 @@ data:
   appEnv: 'dev'
   keboolaStack: 'local'
   awsAccessKeyId: '$TEST_AWS_ACCESS_KEY_ID'
-  awsLogsS3Bucket: '$AWS_LOGS_S3_BUCKET'
   awsRegion: '$AWS_REGION'
-  azureLogsAbsContainer: '$AZURE_LOGS_ABS_CONTAINER'
   azureClientId: '$TEST_AZURE_CLIENT_ID'
   azureTenantId: '$TEST_AZURE_TENANT_ID'
-  storageApiUrl: '$STORAGE_API_URL'
   databaseSslVerifyEnabled: '0'
-  messengerTransportDsn: 'doctrine://default'
+  messengerTransportDsn: '$MESSENGER_TRANSPORT_DSN'
   mailerDsn: 'null://null'
+  hostnameSufix: '$HOSTNAME_SUFFIX'

provisioning/notification.yaml

@@ -20,7 +20,7 @@ spec:
     spec:
       containers:
         - name: dev-mysql
-          image: mysql:8
+          image: mysql:8.0
           resources:
             requests:
               cpu: "100m"
@@ -90,11 +90,6 @@ spec:
                   name: dev-notification-environments
                   key: keboolaStack
 
-            - name: AWS_LOGS_S3_BUCKET
-              valueFrom:
-                configMapKeyRef:
-                  name: dev-notification-environments
-                  key: awsLogsS3Bucket
             - name: AWS_REGION
               valueFrom:
                 configMapKeyRef:
@@ -111,16 +106,6 @@ spec:
                   name: dev-notification-environments
                   key: awsSecretAccessKey
 
-            - name: AZURE_LOGS_ABS_CONTAINER
-              valueFrom:
-                configMapKeyRef:
-                  name: dev-notification-environments
-                  key: azureLogsAbsContainer
-            - name: AZURE_LOGS_ABS_CONNECTION_STRING
-              valueFrom:
-                secretKeyRef:
-                  name: dev-notification-environments
-                  key: azureLogsAbsConnectionString
             - name: AZURE_CLIENT_ID
               valueFrom:
                 configMapKeyRef:
@@ -138,11 +123,11 @@ spec:
                   key: azureTenantId
 
 
-            - name: STORAGE_API_URL
+            - name: HOSTNAME_SUFFIX
               valueFrom:
                 configMapKeyRef:
                   name: dev-notification-environments
-                  key: storageApiUrl
+                  key: hostnameSufix
             - name: DATABASE_URL
               valueFrom:
                 secretKeyRef:

set-env.template.sh

@@ -1,13 +1,10 @@
 #!/bin/bash
 
-export STORAGE_API_URL=https://connection.keboola.com/
 export DATABASE_URL=mysql://root:root@dev-mysql-service:3310/notifications?serverVersion=8.0
 export DATABASE_PASSWORD=root
 
-export AWS_LOGS_S3_BUCKET=
 export AWS_REGION=us-east-1
-export AZURE_LOGS_ABS_CONTAINER=
-export AZURE_LOGS_ABS_CONNECTION_STRING=
+export MESSENGER_TRANSPORT_DSN="${AWS_SQS_QUEUE_URL}?auto_setup=false"
 
 export TEST_AWS_ACCESS_KEY_ID=
 export TEST_AWS_SECRET_ACCESS_KEY=
@@ -20,3 +17,4 @@ export TEST_NOTIFICATION_API_URL=https://localhost:8181/
 export TEST_STORAGE_API_TOKEN=
 export TEST_MANAGE_API_APPLICATION_TOKEN=
 export TEST_STORAGE_API_PROJECT_ID=
+export HOSTNAME_SUFFIX=keboola.com

src/Client.php

@@ -49,17 +49,17 @@ public function __construct(
         $errors->addAll($validator->validate(
             // @phpstan-ignore-next-line
             $options['backoffMaxTries'] ?? null,
-            [new NotBlank(null, '"backoffMaxTries" option must be provided')]
+            [new NotBlank(null, '"backoffMaxTries" option must be provided')],
         ));
         $errors->addAll($validator->validate(
             // @phpstan-ignore-next-line
             $options['backoffMaxTries'] ?? null,
-            [new Range(['min' => 0, 'max' => 100])]
+            [new Range(['min' => 0, 'max' => 100])],
         ));
         $errors->addAll($validator->validate(
             // @phpstan-ignore-next-line
             $options['userAgent'] ?? null,
-            [new NotBlank(null, '"userAgent" option must be provided')]
+            [new NotBlank(null, '"userAgent" option must be provided')],
         ));
         if ($errors->count() !== 0) {
             $messages = '';
@@ -91,14 +91,14 @@ private function createDefaultDecider(int $maxRetries, LoggerInterface $logger):
                 $logger->notice(sprintf(
                     'Got a %s response for this reason: %s, retrying.',
                     $response->getStatusCode(),
-                    $response->getReasonPhrase()
+                    $response->getReasonPhrase(),
                 ));
                 return true;
             } elseif ($error && $error->getCode() >= 500) {
                 $logger->notice(sprintf(
                     'Got a %s error with this message: %s, retrying.',
                     $error->getCode(),
-                    $error->getMessage()
+                    $error->getMessage(),
                 ));
                 return true;
             } else {
@@ -126,9 +126,9 @@ private function initClient(string $url, ?string $token, array $options): Guzzle
                 $options['logger'],
                 new MessageFormatter(
                     '{hostname} {req_header_User-Agent} - [{ts}] "{method} {resource} {protocol}/{version}"' .
-                    ' {code} {res_header_Content-Length}'
+                    ' {code} {res_header_Content-Length}',
                 ),
-                'debug'
+                'debug',
             ));
             $logger = $options['logger'];
         } else {