Merge pull request #227 from keboola/zajca/inc_25140/retry

INC 25140 retry on case of specific snowflake error

Author: zajca

src/Connection/Snowflake/SnowflakeStatement.php

@@ -7,6 +7,12 @@
 use Doctrine\DBAL\Driver\Statement;
 use Doctrine\DBAL\ParameterType;
 use Keboola\TableBackendUtils\Connection\Exception\DriverException;
+use Retry\BackOff\ExponentialBackOffPolicy;
+use Retry\BackOff\ExponentialRandomBackOffPolicy;
+use Retry\BackOff\UniformRandomBackOffPolicy;
+use Retry\Policy\CallableRetryPolicy;
+use Retry\Policy\SimpleRetryPolicy;
+use Retry\RetryProxy;
 use Throwable;
 
 class SnowflakeStatement implements Statement
@@ -16,6 +22,8 @@ class SnowflakeStatement implements Statement
      */
     private $dbh;
 
+    private RetryProxy $retry;
+
     /**
      * @var resource
      */
@@ -31,8 +39,30 @@ class SnowflakeStatement implements Statement
     /**
      * @param resource $dbh database handle
      */
-    public function __construct($dbh, string $query)
+    public function __construct($dbh, string $query, RetryProxy|null $retry = null)
     {
+        if ($retry === null) {
+            $this->retry = new RetryProxy(
+                new CallableRetryPolicy(
+                    function (Throwable $e): bool {
+                        if (str_contains($e->getMessage(), 'SYSTEM$ALLOWLIST')) {
+                            // Retry in case of SYSTEM$ALLOWLIST error #prod_24_7___inc_25140
+                            // this is usually accompanied with SNFLK incidents
+                            // or can happen in case hostname is wrong
+                            return true;
+                        }
+                        return false;
+                    },
+                    5, // 5 attempts
+                ),
+                new UniformRandomBackOffPolicy(
+                    3_000, // 3 seconds
+                    10_000, // 10 seconds
+                ),
+            );
+        } else {
+            $this->retry = $retry;
+        }
         $this->dbh = $dbh;
         $this->query = $query;
         $this->stmt = $this->prepare();
@@ -43,7 +73,10 @@ public function __construct($dbh, string $query)
      */
     private function prepare()
     {
-        $stmt = @odbc_prepare($this->dbh, $this->query);
+        /** @var resource|false $stmt */
+        $stmt = $this->retry->call(function () {
+            return @odbc_prepare($this->dbh, $this->query);
+        });
         if (!$stmt) {
             throw DriverException::newFromHandle($this->dbh);
         }
@@ -82,8 +115,13 @@ public function execute($params = null): Result
         }
 
         try {
-            odbc_execute($this->stmt, $this->repairBinding($this->params));
-        } catch (Throwable $e) {
+            $this->retry->call(function () {
+                odbc_execute(
+                    $this->stmt,
+                    $this->repairBinding($this->params),
+                );
+            });
+        } catch (Throwable) {
             throw DriverException::newFromHandle($this->dbh);
         }