diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 2868e8e..82d8435 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -11,6 +11,7 @@ env: KBC_DEVELOPERPORTAL_PASSWORD: ${{ secrets.KBC_DEVELOPERPORTAL_PASSWORD }} ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} REFRESH_TOKEN: ${{ secrets.REFRESH_TOKEN }} + SERVICE_ACCOUNT_JSON: ${{ secrets.SERVICE_ACCOUNT_JSON }} KBC_URL: "https://connection.keboola.com" KBC_TOKEN: ${{ secrets.KBC_TOKEN }} KBC_COMPONENTID: "keboola.ex-google-analytics-v4" @@ -26,7 +27,7 @@ jobs: - name: Build image and run tests run: | docker build -t $APP_IMAGE . - docker run -e KBC_DATA_TYPE_SUPPORT=none -e CLIENT_ID -e CLIENT_SECRET -e ACCESS_TOKEN -e REFRESH_TOKEN -e VIEW_ID -e KBC_URL -e KBC_TOKEN -e KBC_COMPONENTID $APP_IMAGE composer ci + docker run -e KBC_DATA_TYPE_SUPPORT=none -e CLIENT_ID -e CLIENT_SECRET -e ACCESS_TOKEN -e REFRESH_TOKEN -e VIEW_ID -e KBC_URL -e KBC_TOKEN -e KBC_COMPONENTID -e SERVICE_ACCOUNT_JSON $APP_IMAGE composer ci - name: Push image to ECR run: | docker pull quay.io/keboola/developer-portal-cli-v2:latest diff --git a/composer.json b/composer.json index b0eabcb..04b0cc2 100644 --- a/composer.json +++ b/composer.json @@ -14,7 +14,7 @@ "ext-json": "*", "keboola/common-exceptions": "^1.1", "keboola/csv": "^1.1", - "keboola/google-client-bundle": "^5.4", + "keboola/google-client-bundle": "^6.0", "keboola/php-component": "^10.1", "keboola/storage-api-client": "^12.9", "symfony/config": "^5.2", diff --git a/composer.lock b/composer.lock index 750b13b..7493127 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "4a971a5d90416c56702fa62bc982e191", + "content-hash": "123125c873c2dcab15230f5d7221a585", "packages": [ { "name": "aws/aws-crt-php", @@ -158,18 +158,141 @@ }, "time": "2024-08-21T18:14:31+00:00" }, + { + "name": "firebase/php-jwt", + "version": "v6.11.1", + "source": { + "type": "git", + "url": "https://github.com/firebase/php-jwt.git", + "reference": "d1e91ecf8c598d073d0995afa8cd5c75c6e19e66" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/firebase/php-jwt/zipball/d1e91ecf8c598d073d0995afa8cd5c75c6e19e66", + "reference": "d1e91ecf8c598d073d0995afa8cd5c75c6e19e66", + "shasum": "" + }, + "require": { + "php": "^8.0" + }, + "require-dev": { + "guzzlehttp/guzzle": "^7.4", + "phpspec/prophecy-phpunit": "^2.0", + "phpunit/phpunit": "^9.5", + "psr/cache": "^2.0||^3.0", + "psr/http-client": "^1.0", + "psr/http-factory": "^1.0" + }, + "suggest": { + "ext-sodium": "Support EdDSA (Ed25519) signatures", + "paragonie/sodium_compat": "Support EdDSA (Ed25519) signatures when libsodium is not present" + }, + "type": "library", + "autoload": { + "psr-4": { + "Firebase\\JWT\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Neuman Vong", + "email": "neuman+pear@twilio.com", + "role": "Developer" + }, + { + "name": "Anant Narayanan", + "email": "anant@php.net", + "role": "Developer" + } + ], + "description": "A simple library to encode and decode JSON Web Tokens (JWT) in PHP. Should conform to the current spec.", + "homepage": "https://github.com/firebase/php-jwt", + "keywords": [ + "jwt", + "php" + ], + "support": { + "issues": "https://github.com/firebase/php-jwt/issues", + "source": "https://github.com/firebase/php-jwt/tree/v6.11.1" + }, + "time": "2025-04-09T20:32:01+00:00" + }, + { + "name": "google/auth", + "version": "v1.44.0", + "source": { + "type": "git", + "url": "https://github.com/googleapis/google-auth-library-php.git", + "reference": "5670e56307d7a2eac931f677c0e59a4f8abb2e43" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/googleapis/google-auth-library-php/zipball/5670e56307d7a2eac931f677c0e59a4f8abb2e43", + "reference": "5670e56307d7a2eac931f677c0e59a4f8abb2e43", + "shasum": "" + }, + "require": { + "firebase/php-jwt": "^6.0", + "guzzlehttp/guzzle": "^7.4.5", + "guzzlehttp/psr7": "^2.4.5", + "php": "^8.1", + "psr/cache": "^2.0||^3.0", + "psr/http-message": "^1.1||^2.0" + }, + "require-dev": { + "guzzlehttp/promises": "^2.0", + "kelvinmo/simplejwt": "0.7.1", + "phpseclib/phpseclib": "^3.0.35", + "phpspec/prophecy-phpunit": "^2.1", + "phpunit/phpunit": "^9.6", + "sebastian/comparator": ">=1.2.3", + "squizlabs/php_codesniffer": "^3.5", + "symfony/process": "^6.0||^7.0", + "webmozart/assert": "^1.11" + }, + "suggest": { + "phpseclib/phpseclib": "May be used in place of OpenSSL for signing strings or for token management. Please require version ^2." + }, + "type": "library", + "autoload": { + "psr-4": { + "Google\\Auth\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "description": "Google Auth Library for PHP", + "homepage": "http://github.com/google/google-auth-library-php", + "keywords": [ + "Authentication", + "google", + "oauth2" + ], + "support": { + "docs": "https://googleapis.github.io/google-auth-library-php/main/", + "issues": "https://github.com/googleapis/google-auth-library-php/issues", + "source": "https://github.com/googleapis/google-auth-library-php/tree/v1.44.0" + }, + "time": "2024-12-04T15:34:58+00:00" + }, { "name": "guzzlehttp/guzzle", - "version": "7.9.2", + "version": "7.9.3", "source": { "type": "git", "url": "https://github.com/guzzle/guzzle.git", - "reference": "d281ed313b989f213357e3be1a179f02196ac99b" + "reference": "7b2f29fe81dc4da0ca0ea7d42107a0845946ea77" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/guzzle/zipball/d281ed313b989f213357e3be1a179f02196ac99b", - "reference": "d281ed313b989f213357e3be1a179f02196ac99b", + "url": "https://api.github.com/repos/guzzle/guzzle/zipball/7b2f29fe81dc4da0ca0ea7d42107a0845946ea77", + "reference": "7b2f29fe81dc4da0ca0ea7d42107a0845946ea77", "shasum": "" }, "require": { @@ -266,7 +389,7 @@ ], "support": { "issues": "https://github.com/guzzle/guzzle/issues", - "source": "https://github.com/guzzle/guzzle/tree/7.9.2" + "source": "https://github.com/guzzle/guzzle/tree/7.9.3" }, "funding": [ { @@ -282,20 +405,20 @@ "type": "tidelift" } ], - "time": "2024-07-24T11:22:20+00:00" + "time": "2025-03-27T13:37:11+00:00" }, { "name": "guzzlehttp/promises", - "version": "2.0.3", + "version": "2.2.0", "source": { "type": "git", "url": "https://github.com/guzzle/promises.git", - "reference": "6ea8dd08867a2a42619d65c3deb2c0fcbf81c8f8" + "reference": "7c69f28996b0a6920945dd20b3857e499d9ca96c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/promises/zipball/6ea8dd08867a2a42619d65c3deb2c0fcbf81c8f8", - "reference": "6ea8dd08867a2a42619d65c3deb2c0fcbf81c8f8", + "url": "https://api.github.com/repos/guzzle/promises/zipball/7c69f28996b0a6920945dd20b3857e499d9ca96c", + "reference": "7c69f28996b0a6920945dd20b3857e499d9ca96c", "shasum": "" }, "require": { @@ -349,7 +472,7 @@ ], "support": { "issues": "https://github.com/guzzle/promises/issues", - "source": "https://github.com/guzzle/promises/tree/2.0.3" + "source": "https://github.com/guzzle/promises/tree/2.2.0" }, "funding": [ { @@ -365,20 +488,20 @@ "type": "tidelift" } ], - "time": "2024-07-18T10:29:17+00:00" + "time": "2025-03-27T13:27:01+00:00" }, { "name": "guzzlehttp/psr7", - "version": "2.7.0", + "version": "2.7.1", "source": { "type": "git", "url": "https://github.com/guzzle/psr7.git", - "reference": "a70f5c95fb43bc83f07c9c948baa0dc1829bf201" + "reference": "c2270caaabe631b3b44c85f99e5a04bbb8060d16" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/psr7/zipball/a70f5c95fb43bc83f07c9c948baa0dc1829bf201", - "reference": "a70f5c95fb43bc83f07c9c948baa0dc1829bf201", + "url": "https://api.github.com/repos/guzzle/psr7/zipball/c2270caaabe631b3b44c85f99e5a04bbb8060d16", + "reference": "c2270caaabe631b3b44c85f99e5a04bbb8060d16", "shasum": "" }, "require": { @@ -465,7 +588,7 @@ ], "support": { "issues": "https://github.com/guzzle/psr7/issues", - "source": "https://github.com/guzzle/psr7/tree/2.7.0" + "source": "https://github.com/guzzle/psr7/tree/2.7.1" }, "funding": [ { @@ -481,7 +604,7 @@ "type": "tidelift" } ], - "time": "2024-07-18T11:15:46+00:00" + "time": "2025-03-27T12:30:47+00:00" }, { "name": "keboola/common-exceptions", @@ -560,19 +683,20 @@ }, { "name": "keboola/google-client-bundle", - "version": "5.5.0", + "version": "6.0.1", "source": { "type": "git", "url": "https://github.com/keboola/google-client-bundle.git", - "reference": "feb67714b3399c0ba9aa7d8649d95115e2dd335e" + "reference": "e077a3e55eaa2ea4211d8cdbda4de22ac51307c9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/keboola/google-client-bundle/zipball/feb67714b3399c0ba9aa7d8649d95115e2dd335e", - "reference": "feb67714b3399c0ba9aa7d8649d95115e2dd335e", + "url": "https://api.github.com/repos/keboola/google-client-bundle/zipball/e077a3e55eaa2ea4211d8cdbda4de22ac51307c9", + "reference": "e077a3e55eaa2ea4211d8cdbda4de22ac51307c9", "shasum": "" }, "require": { + "google/auth": "^1.26", "guzzlehttp/guzzle": "^7.0", "monolog/monolog": "^2.1", "php": "^8.1" @@ -580,7 +704,7 @@ "require-dev": { "keboola/coding-standard": "^15.0", "php-parallel-lint/php-parallel-lint": "^1.2", - "phpstan/phpstan": "^1.4", + "phpstan/phpstan": "^2.1", "phpunit/phpunit": "^9.5" }, "type": "library", @@ -609,9 +733,9 @@ ], "support": { "issues": "https://github.com/keboola/google-client-bundle/issues", - "source": "https://github.com/keboola/google-client-bundle/tree/5.5.0" + "source": "https://github.com/keboola/google-client-bundle/tree/6.0.1" }, - "time": "2023-10-12T09:39:07+00:00" + "time": "2025-07-29T19:55:25+00:00" }, { "name": "keboola/php-component", @@ -997,6 +1121,55 @@ }, "time": "2023-08-25T10:54:48+00:00" }, + { + "name": "psr/cache", + "version": "3.0.0", + "source": { + "type": "git", + "url": "https://github.com/php-fig/cache.git", + "reference": "aa5030cfa5405eccfdcb1083ce040c2cb8d253bf" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/cache/zipball/aa5030cfa5405eccfdcb1083ce040c2cb8d253bf", + "reference": "aa5030cfa5405eccfdcb1083ce040c2cb8d253bf", + "shasum": "" + }, + "require": { + "php": ">=8.0.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Cache\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "https://www.php-fig.org/" + } + ], + "description": "Common interface for caching libraries", + "keywords": [ + "cache", + "psr", + "psr-6" + ], + "support": { + "source": "https://github.com/php-fig/cache/tree/3.0.0" + }, + "time": "2021-02-03T23:26:27+00:00" + }, { "name": "psr/container", "version": "2.0.2", @@ -1385,16 +1558,16 @@ }, { "name": "symfony/deprecation-contracts", - "version": "v3.5.0", + "version": "v3.6.0", "source": { "type": "git", "url": "https://github.com/symfony/deprecation-contracts.git", - "reference": "0e0d29ce1f20deffb4ab1b016a7257c4f1e789a1" + "reference": "63afe740e99a13ba87ec199bb07bbdee937a5b62" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/0e0d29ce1f20deffb4ab1b016a7257c4f1e789a1", - "reference": "0e0d29ce1f20deffb4ab1b016a7257c4f1e789a1", + "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/63afe740e99a13ba87ec199bb07bbdee937a5b62", + "reference": "63afe740e99a13ba87ec199bb07bbdee937a5b62", "shasum": "" }, "require": { @@ -1402,12 +1575,12 @@ }, "type": "library", "extra": { - "branch-alias": { - "dev-main": "3.5-dev" - }, "thanks": { - "name": "symfony/contracts", - "url": "https://github.com/symfony/contracts" + "url": "https://github.com/symfony/contracts", + "name": "symfony/contracts" + }, + "branch-alias": { + "dev-main": "3.6-dev" } }, "autoload": { @@ -1432,7 +1605,7 @@ "description": "A generic function and convention to trigger deprecation notices", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/deprecation-contracts/tree/v3.5.0" + "source": "https://github.com/symfony/deprecation-contracts/tree/v3.6.0" }, "funding": [ { @@ -1448,7 +1621,7 @@ "type": "tidelift" } ], - "time": "2024-04-18T09:32:20+00:00" + "time": "2024-09-25T14:21:43+00:00" }, { "name": "symfony/filesystem", @@ -1738,20 +1911,20 @@ }, { "name": "symfony/polyfill-intl-normalizer", - "version": "v1.30.0", + "version": "v1.32.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-intl-normalizer.git", - "reference": "a95281b0be0d9ab48050ebd988b967875cdb9fdb" + "reference": "3833d7255cc303546435cb650316bff708a1c75c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-intl-normalizer/zipball/a95281b0be0d9ab48050ebd988b967875cdb9fdb", - "reference": "a95281b0be0d9ab48050ebd988b967875cdb9fdb", + "url": "https://api.github.com/repos/symfony/polyfill-intl-normalizer/zipball/3833d7255cc303546435cb650316bff708a1c75c", + "reference": "3833d7255cc303546435cb650316bff708a1c75c", "shasum": "" }, "require": { - "php": ">=7.1" + "php": ">=7.2" }, "suggest": { "ext-intl": "For best performance" @@ -1759,8 +1932,8 @@ "type": "library", "extra": { "thanks": { - "name": "symfony/polyfill", - "url": "https://github.com/symfony/polyfill" + "url": "https://github.com/symfony/polyfill", + "name": "symfony/polyfill" } }, "autoload": { @@ -1799,7 +1972,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.30.0" + "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.32.0" }, "funding": [ { @@ -1815,24 +1988,25 @@ "type": "tidelift" } ], - "time": "2024-05-31T15:07:36+00:00" + "time": "2024-09-09T11:45:10+00:00" }, { "name": "symfony/polyfill-mbstring", - "version": "v1.30.0", + "version": "v1.32.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "fd22ab50000ef01661e2a31d850ebaa297f8e03c" + "reference": "6d857f4d76bd4b343eac26d6b539585d2bc56493" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/fd22ab50000ef01661e2a31d850ebaa297f8e03c", - "reference": "fd22ab50000ef01661e2a31d850ebaa297f8e03c", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/6d857f4d76bd4b343eac26d6b539585d2bc56493", + "reference": "6d857f4d76bd4b343eac26d6b539585d2bc56493", "shasum": "" }, "require": { - "php": ">=7.1" + "ext-iconv": "*", + "php": ">=7.2" }, "provide": { "ext-mbstring": "*" @@ -1843,8 +2017,8 @@ "type": "library", "extra": { "thanks": { - "name": "symfony/polyfill", - "url": "https://github.com/symfony/polyfill" + "url": "https://github.com/symfony/polyfill", + "name": "symfony/polyfill" } }, "autoload": { @@ -1879,7 +2053,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.30.0" + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.32.0" }, "funding": [ { @@ -1895,7 +2069,7 @@ "type": "tidelift" } ], - "time": "2024-06-19T12:30:46+00:00" + "time": "2024-12-23T08:48:59+00:00" }, { "name": "symfony/polyfill-php80", diff --git a/docker-compose.yml b/docker-compose.yml index 0326641..b7b8cae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,7 @@ services: - CLIENT_SECRET - ACCESS_TOKEN - REFRESH_TOKEN + - SERVICE_ACCOUNT_JSON - VIEW_ID - KBC_URL - KBC_TOKEN diff --git a/phpstan-baseline.neon b/phpstan-baseline.neon index 0d514c7..2539de7 100644 --- a/phpstan-baseline.neon +++ b/phpstan-baseline.neon @@ -1,15 +1,5 @@ parameters: ignoreErrors: - - - message: "#^Cannot access offset 'access_token' on mixed\\.$#" - count: 1 - path: src/Component.php - - - - message: "#^Cannot access offset 'refresh_token' on mixed\\.$#" - count: 1 - path: src/Component.php - - message: "#^Method Keboola\\\\GoogleAnalyticsExtractor\\\\Component\\:\\:getProfilesPropertiesAction\\(\\) return type has no value type specified in iterable type array\\.$#" count: 1 @@ -55,21 +45,6 @@ parameters: count: 1 path: src/Component.php - - - message: "#^Parameter \\#3 \\$accessToken of class Keboola\\\\Google\\\\ClientBundle\\\\Google\\\\RestApi constructor expects string, mixed given\\.$#" - count: 1 - path: src/Component.php - - - - message: "#^Parameter \\#4 \\$refreshToken of class Keboola\\\\Google\\\\ClientBundle\\\\Google\\\\RestApi constructor expects string, mixed given\\.$#" - count: 1 - path: src/Component.php - - - - message: "#^Parameter \\#5 \\$logger of class Keboola\\\\Google\\\\ClientBundle\\\\Google\\\\RestApi constructor expects Monolog\\\\Logger\\|null, Psr\\\\Log\\\\LoggerInterface given\\.$#" - count: 1 - path: src/Component.php - - message: "#^Cannot access offset 'enabled' on mixed\\.$#" count: 1 @@ -135,6 +110,11 @@ parameters: count: 1 path: src/Configuration/Config.php + - + message: "#^Method Keboola\\\\GoogleAnalyticsExtractor\\\\Configuration\\\\Config\\:\\:getServiceAccount\\(\\) return type has no value type specified in iterable type array\\.$#" + count: 1 + path: src/Configuration/Config.php + - message: "#^Method Keboola\\\\GoogleAnalyticsExtractor\\\\Configuration\\\\Config\\:\\:skipGenerateSystemTables\\(\\) should return bool but returns mixed\\.$#" count: 1 diff --git a/src/Component.php b/src/Component.php index c80e97f..fb56be5 100644 --- a/src/Component.php +++ b/src/Component.php @@ -298,18 +298,35 @@ private function getExtractor(): Extractor private function getGoogleRestApi(): RestApi { - $tokenData = json_decode($this->getConfig()->getOAuthApiData(), true); - if (!isset($tokenData['access_token'], $tokenData['refresh_token'])) { - throw new UserException('The token data are broken. Please try to reauthorize.'); - } + $serviceAccount = $this->getConfig()->getServiceAccount(); + if ($serviceAccount) { + $this->getLogger()->info(sprintf( + 'Login with service account: "%s"', + $serviceAccount['client_email'], + )); + $client = RestApi::createWithServiceAccount( + $serviceAccount, + [ + 'https://www.googleapis.com/auth/analytics.readonly', + ], + $this->getLogger(), + ); + } else { + $this->getLogger()->info('Login with OAuth'); + /** @var array{access_token?: string, refresh_token?: string}|null $tokenData */ + $tokenData = json_decode($this->getConfig()->getOAuthApiData(), true); + if (!isset($tokenData['access_token'], $tokenData['refresh_token'])) { + throw new UserException('The token data are broken. Please try to reauthorize.'); + } - $client = new RestApi( - $this->getConfig()->getOAuthApiAppKey(), - $this->getConfig()->getOAuthApiAppSecret(), - $tokenData['access_token'], - $tokenData['refresh_token'], - $this->getLogger(), - ); + $client = RestApi::createWithOAuth( + $this->getConfig()->getOAuthApiAppKey(), + $this->getConfig()->getOAuthApiAppSecret(), + $tokenData['access_token'], + $tokenData['refresh_token'], + $this->getLogger(), + ); + } $client->setBackoffsCount($this->getConfig()->getRetries()); return $client; diff --git a/src/Configuration/Config.php b/src/Configuration/Config.php index b294bf8..71bc4ec 100644 --- a/src/Configuration/Config.php +++ b/src/Configuration/Config.php @@ -122,4 +122,17 @@ public function processProperties(string $configDefinition): bool return in_array($this->getEndpoint(), [Config::ENDPOINT_DATA_API]); } + + public function getServiceAccount(): ?array + { + $serviceAccount = $this->getArrayValue(['parameters', 'service_account'], []); + if (empty($serviceAccount)) { + return null; + } + + $serviceAccount['private_key'] = $serviceAccount['#private_key']; + unset($serviceAccount['#private_key']); + + return $serviceAccount; + } } diff --git a/src/Configuration/ConfigDefinition.php b/src/Configuration/ConfigDefinition.php index df96050..e61927c 100644 --- a/src/Configuration/ConfigDefinition.php +++ b/src/Configuration/ConfigDefinition.php @@ -6,6 +6,7 @@ use DateTime; use Keboola\Component\Config\BaseConfigDefinition; +use Keboola\GoogleAnalyticsExtractor\Configuration\Node\ServiceAccountNode; use Symfony\Component\Config\Definition\Builder\ArrayNodeDefinition; use Symfony\Component\Config\Definition\Exception\InvalidConfigurationException; @@ -50,6 +51,7 @@ public function getParametersDefinition(): ArrayNodeDefinition ->end() ->ignoreExtraKeys(true) ->children() + ->append(new ServiceAccountNode()) ->booleanNode('skipGenerateSystemTables')->defaultFalse()->end() ->scalarNode('outputBucket')->isRequired()->cannotBeEmpty()->end() ->integerNode('nonConflictPrimaryKey') diff --git a/src/Configuration/ConfigGetProfilesPropertiesDefinition.php b/src/Configuration/ConfigGetProfilesPropertiesDefinition.php index c89ff93..9e920d3 100644 --- a/src/Configuration/ConfigGetProfilesPropertiesDefinition.php +++ b/src/Configuration/ConfigGetProfilesPropertiesDefinition.php @@ -5,7 +5,17 @@ namespace Keboola\GoogleAnalyticsExtractor\Configuration; use Keboola\Component\Config\BaseConfigDefinition; +use Keboola\GoogleAnalyticsExtractor\Configuration\Node\ServiceAccountNode; +use Symfony\Component\Config\Definition\Builder\ArrayNodeDefinition; class ConfigGetProfilesPropertiesDefinition extends BaseConfigDefinition { + public function getParametersDefinition(): ArrayNodeDefinition + { + $parametersNode = parent::getParametersDefinition(); + + $parametersNode->children()->append(new ServiceAccountNode())->end(); + + return $parametersNode; + } } diff --git a/src/Configuration/ConfigGetPropertiesMetadataDefinition.php b/src/Configuration/ConfigGetPropertiesMetadataDefinition.php index 69d3b54..e1ca189 100644 --- a/src/Configuration/ConfigGetPropertiesMetadataDefinition.php +++ b/src/Configuration/ConfigGetPropertiesMetadataDefinition.php @@ -5,6 +5,7 @@ namespace Keboola\GoogleAnalyticsExtractor\Configuration; use Keboola\Component\Config\BaseConfigDefinition; +use Keboola\GoogleAnalyticsExtractor\Configuration\Node\ServiceAccountNode; use Symfony\Component\Config\Definition\Builder\ArrayNodeDefinition; class ConfigGetPropertiesMetadataDefinition extends BaseConfigDefinition @@ -15,6 +16,7 @@ public function getParametersDefinition(): ArrayNodeDefinition $parametersNode ->children() + ->append(new ServiceAccountNode()) ->arrayNode('properties') ->arrayPrototype() ->children() diff --git a/src/Configuration/Node/ServiceAccountNode.php b/src/Configuration/Node/ServiceAccountNode.php new file mode 100644 index 0000000..8646213 --- /dev/null +++ b/src/Configuration/Node/ServiceAccountNode.php @@ -0,0 +1,36 @@ +init($this->children()); + } + + protected function init(NodeBuilder $builder): void + { + $builder + ->scalarNode('#private_key')->isRequired()->cannotBeEmpty()->end() + ->scalarNode('type')->isRequired()->cannotBeEmpty()->end() + ->scalarNode('project_id')->isRequired()->cannotBeEmpty()->end() + ->scalarNode('client_email')->isRequired()->cannotBeEmpty()->end() + ->scalarNode('private_key_id')->end() + ->scalarNode('client_id')->end() + ->scalarNode('auth_uri')->end() + ->scalarNode('token_uri')->end() + ->scalarNode('universe_domain')->end() + ->scalarNode('auth_provider_x509_cert_url')->end() + ->scalarNode('client_x509_cert_url')->end() + ; + } +} diff --git a/src/Configuration/OldConfigDefinition.php b/src/Configuration/OldConfigDefinition.php index 213cd61..5c21595 100644 --- a/src/Configuration/OldConfigDefinition.php +++ b/src/Configuration/OldConfigDefinition.php @@ -5,6 +5,7 @@ namespace Keboola\GoogleAnalyticsExtractor\Configuration; use Keboola\Component\Config\BaseConfigDefinition; +use Keboola\GoogleAnalyticsExtractor\Configuration\Node\ServiceAccountNode; use Symfony\Component\Config\Definition\Builder\ArrayNodeDefinition; class OldConfigDefinition extends BaseConfigDefinition @@ -16,6 +17,7 @@ public function getParametersDefinition(): ArrayNodeDefinition $parametersNode ->ignoreExtraKeys() ->children() + ->append(new ServiceAccountNode()) ->scalarNode('outputBucket') ->isRequired() ->cannotBeEmpty() diff --git a/tests/Keboola/GoogleAnalyticsExtractor/Extractor/ExtractorTest.php b/tests/Keboola/GoogleAnalyticsExtractor/Extractor/ExtractorTest.php index 873add6..d6bd030 100644 --- a/tests/Keboola/GoogleAnalyticsExtractor/Extractor/ExtractorTest.php +++ b/tests/Keboola/GoogleAnalyticsExtractor/Extractor/ExtractorTest.php @@ -37,7 +37,7 @@ public function setUp(): void $this->config = $this->getConfig(); $this->logger = new TestLogger(); $client = new Client( - new RestApi( + RestApi::createWithOAuth( (string) getenv('CLIENT_ID'), (string) getenv('CLIENT_SECRET'), (string) getenv('ACCESS_TOKEN'), diff --git a/tests/Keboola/GoogleAnalyticsExtractor/GoogleAnalytics/ClientTest.php b/tests/Keboola/GoogleAnalyticsExtractor/GoogleAnalytics/ClientTest.php index 1353e53..ad32edc 100644 --- a/tests/Keboola/GoogleAnalyticsExtractor/GoogleAnalytics/ClientTest.php +++ b/tests/Keboola/GoogleAnalyticsExtractor/GoogleAnalytics/ClientTest.php @@ -25,7 +25,7 @@ public function setUp(): void $this->logger->pushHandler($testHandler); $this->client = new Client( - new RestApi( + RestApi::createWithOAuth( (string) getenv('CLIENT_ID'), (string) getenv('CLIENT_SECRET'), (string) getenv('ACCESS_TOKEN'), diff --git a/tests/functional/DatadirTest.php b/tests/functional/DatadirTest.php index c703b38..1ec8cf5 100644 --- a/tests/functional/DatadirTest.php +++ b/tests/functional/DatadirTest.php @@ -22,4 +22,39 @@ public function __construct(?string $name = null, array $data = [], string $data parent::__construct($name, $data, $dataName); } + + /** + * @dataProvider provideDatadirSpecifications + */ + public function testDatadir(DatadirTestSpecificationInterface $specification): void + { + $tempDatadir = $this->getTempDatadir($specification); + + /** + * @var array{ + * "parameters": ?array{ + * "service_account": string, + * } + * } $config + */ + $config = json_decode( + (string) file_get_contents($tempDatadir->getTmpFolder() . '/config.json'), + true, + ); + if (array_key_exists('service_account', $config['parameters'] ?? [])) { + /** + * @var array{ + * "private_key": string, + * } $serviceAccount + */ + $serviceAccount = json_decode($config['parameters']['service_account'], true); + $serviceAccount['#private_key'] = $serviceAccount['private_key']; + unset($serviceAccount['private_key']); + $config['parameters']['service_account'] = $serviceAccount; + } + file_put_contents($tempDatadir->getTmpFolder() . '/config.json', json_encode($config)); + $process = $this->runScript($tempDatadir->getTmpFolder()); + + $this->assertMatchesSpecification($specification, $process, $tempDatadir->getTmpFolder()); + } } diff --git a/tests/functional/service-account/expected/data/out/files/.gitkeep b/tests/functional/service-account/expected/data/out/files/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/functional/service-account/expected/data/out/tables/properties.csv b/tests/functional/service-account/expected/data/out/tables/properties.csv new file mode 100644 index 0000000..fea70bf --- /dev/null +++ b/tests/functional/service-account/expected/data/out/tables/properties.csv @@ -0,0 +1 @@ +"properties/255885884","users","accounts/185283969","Keboola" diff --git a/tests/functional/service-account/expected/data/out/tables/properties.csv.manifest b/tests/functional/service-account/expected/data/out/tables/properties.csv.manifest new file mode 100644 index 0000000..0be3875 --- /dev/null +++ b/tests/functional/service-account/expected/data/out/tables/properties.csv.manifest @@ -0,0 +1 @@ +{"destination":"in.c-ex-google-analytics-cfg.properties","incremental":true,"columns":["propertyKey","propertyName","accountKey","accountName"],"primary_key":["propertyKey"],"column_metadata":{"propertyKey":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"propertyName":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"accountKey":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"accountName":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}]}} \ No newline at end of file diff --git a/tests/functional/service-account/expected/data/out/tables/users.csv b/tests/functional/service-account/expected/data/out/tables/users.csv new file mode 100644 index 0000000..1d1db1c --- /dev/null +++ b/tests/functional/service-account/expected/data/out/tables/users.csv @@ -0,0 +1,70 @@ +"72194b333e93bc548ef559a2ae554d7a6ee50947","255885884","2023-05-16","2","3","0" +"6a0c94663242fa1b581c0f9d9f61a6460a1c3a81","255885884","2023-03-07","2","2","0" +"4a1cf07a609a881015969ed288e8d50a8010bedd","255885884","2023-03-16","4","2","0" +"4025b8904b24bd6f7a71ce5b9f2805c46b739788","255885884","2023-03-17","5","2","0" +"be35d9bfa1a2286d60aba94c5dfd280bd99adbd4","255885884","2023-03-20","1","2","0" +"79e0eebf5e6d9e439e36c463a503cc77c940d986","255885884","2023-03-27","1","2","0" +"dc1edce9a87d775bf3b4743244729a9d996c5168","255885884","2023-04-01","6","2","0" +"6aa5858739621460c312fc61bafc36182740241e","255885884","2023-04-03","1","2","0" +"20915d9158fa409723d6750a34b58519a42f39d9","255885884","2023-04-05","3","2","0" +"3479b554e95332620756a7f417ad4d6acbfaea54","255885884","2023-04-22","6","2","0" +"690f7f1a963817cd620701844580433c405fa019","255885884","2023-04-24","1","2","0" +"932b7cdb4b09d42e15c61bc24e4a07db07568fd0","255885884","2023-04-28","5","2","0" +"372bc1dab0440089873e6925879ea196d9a4a8b3","255885884","2023-05-04","4","2","0" +"a79ab9ae69bc1ab6e6765f638f5dd8ec079caa56","255885884","2023-05-06","6","2","0" +"bc752f7f08fff8c16580a7eb36f10d445eb70c98","255885884","2023-05-08","1","2","0" +"810a089ab9052b31a47dfcd15d46d5adbe0072b4","255885884","2023-03-09","4","1","0" +"ab2276e5c4ecc29ab91f06196eaa28ebe398e067","255885884","2023-03-10","5","1","0" +"a1f21a0a372fa2dc2487010142d0025238a57b68","255885884","2023-03-11","6","1","0" +"8d8f65e630ffd865d4b3bab29ad6031459c6194f","255885884","2023-03-12","0","1","0" +"d2df6e3f288747c9f59bf2caad54ea9a02b93b81","255885884","2023-03-13","1","1","0" +"f7630c8c0629aa9ea00f93da0c706ea893606177","255885884","2023-03-14","2","1","0" +"0b9f5dfd5e36f7a5ba2bfac9bf6e0a4dfba1a1a1","255885884","2023-03-18","6","1","0" +"a7ccfc291b472d36e4462b9dd46963223f62e283","255885884","2023-03-22","3","1","0" +"c9222c5c237e0dc866c0816d685bb4601dddc99a","255885884","2023-03-23","4","1","0" +"bcbdc989391e5436abe566b641f50d2bccc6f49d","255885884","2023-03-24","5","1","0" +"6c72b35050d28080006e988597bf4e601479c532","255885884","2023-03-25","6","1","0" +"0fea5a4587633b25ebf748b2203d5fa46d53324d","255885884","2023-03-26","0","1","0" +"c171f9b873901812c083f13fde11aafd67407bc6","255885884","2023-03-29","3","1","0" +"483cbd89ceb001246a61bddd340f7cfb8a7d6ecd","255885884","2023-03-31","5","1","0" +"0ccd7dad4a4de6b6e7bea51daf2abb7f29f1b320","255885884","2023-04-02","0","1","0" +"df787f933da2e54e832d9dd7f212371bef8e6751","255885884","2023-04-04","2","1","0" +"7abe9295662dc06fadecfee024439ef0e328e66b","255885884","2023-04-06","4","1","0" +"4b2a0f97ba3b982481da174f2497c6f3cd463045","255885884","2023-04-07","5","1","0" +"497224d74eb00c6c9f46abe18a049115674c04b6","255885884","2023-04-08","6","1","0" +"0717f614f328d701340c5638ff4b79448f31cdd8","255885884","2023-04-10","1","1","0" +"36d4537d931d1e614cf6874099f5661bc30927df","255885884","2023-04-12","3","1","0" +"207ab77dd04c89d937d03d37474a66aac126d0b0","255885884","2023-04-13","4","1","0" +"f16537004f8b68e9da694295b65da7f9c97ef9be","255885884","2023-04-14","5","1","0" +"542273b2db7a35d316ba112a17bebbc14eaa5694","255885884","2023-04-16","0","1","0" +"e287321e33f1ae47e175fa99b82e3a4ffa7b4149","255885884","2023-04-17","1","1","0" +"0163c6a7f236f9b805059f93556ba584dbda59e6","255885884","2023-04-19","3","1","0" +"52edb76dbd1c590a772504a3e9099a36c699ca5c","255885884","2023-04-21","5","1","0" +"f99ea979482c8cb8f8a1ddfdd5417e4776f46f12","255885884","2023-04-23","0","1","0" +"7388dc29d0ce8fb9836a5b66f48ad0f47ff5efbc","255885884","2023-04-25","2","1","0" +"f748841f37163c1d38419cb47b9b5de7f505f987","255885884","2023-04-29","6","1","0" +"ea8de439e65f1b77359449586efbd6b7e49501e8","255885884","2023-04-30","0","1","0" +"2a63d921bf25df05d90b9cb0b3fc4d54dee99876","255885884","2023-05-01","1","1","0" +"14f5cba68cf4083d017cb033223c0515932dc8f6","255885884","2023-05-02","2","1","0" +"bb72ce4b6f330dab42b05cfde3f4ff4b93aee838","255885884","2023-05-03","3","1","0" +"522b3e8fd76cf283eba1fdea3965928221f6a213","255885884","2023-05-05","5","1","0" +"6e419d6b5654e1b4fb358b83395cd8f649caa876","255885884","2023-05-07","0","1","0" +"6519f3e86e6cdc6d00c20a671b6dbc0b7eef54d7","255885884","2023-05-09","2","1","0" +"7def282a1baed45e70f526dad58b9291e4861f25","255885884","2023-05-11","4","1","0" +"93057eb54d75c6f8a31087b44cf95ceddc06d5c2","255885884","2023-05-13","6","1","0" +"67c36c3bdb64d7796afdb44c93e0fb0d24fb25b2","255885884","2023-05-15","1","1","0" +"1c61f9de1e727acc7941ee673eb1f3e0d9a1a718","255885884","2023-05-17","3","1","0" +"c22ac60c4b33dac1d14bfc6e539fd261ad88be26","255885884","2023-05-18","4","1","0" +"101d4a664e920f8b500110e25ba3d6f180c1b534","255885884","2023-05-19","5","1","0" +"b84933e2cf39678c5d0b42ccb561edc47444714a","255885884","2023-05-20","6","1","0" +"c11c529174eeaf88e18df71dfd00cd5bcf1e2a84","255885884","2023-05-21","0","1","0" +"2163cb803564c4da700a33273aaf0242e1da5514","255885884","2023-05-24","3","1","0" +"997485c79412e5669cb7a1d09e3228749fc3f8f9","255885884","2023-05-25","4","1","0" +"fa24140c14dc60d91a9d6ba509185b59c4e9f41d","255885884","2023-05-26","5","1","0" +"5a8b5d6a390626e151fa0985311b3890205b6004","255885884","2023-05-27","6","1","0" +"03f1a10f322b33a21c66eee4d66e9b2de7ad62f2","255885884","2023-05-29","1","1","0" +"4b662e07e0a4427599761fc58bc1906a4a43ad6a","255885884","2023-05-31","3","1","0" +"6fe8664c240f430f15265df8ebf3e3c1232eb34c","255885884","2023-06-01","4","1","0" +"ef2fec98b7b5c83446388ea7ace4923393894d6b","255885884","2023-06-02","5","1","0" +"c98224c25fff2c406794f27dec67d03db7581bc7","255885884","2023-06-03","6","1","0" +"36d6b7a9b0f5c12aa70b64972afc2f54e7096e28","255885884","2023-06-06","2","1","0" diff --git a/tests/functional/service-account/expected/data/out/tables/users.csv.manifest b/tests/functional/service-account/expected/data/out/tables/users.csv.manifest new file mode 100644 index 0000000..85f9eb9 --- /dev/null +++ b/tests/functional/service-account/expected/data/out/tables/users.csv.manifest @@ -0,0 +1 @@ +{"destination":"in.c-ex-google-analytics-cfg.users","incremental":true,"columns":["id","idProperty","date","dayOfWeek","totalUsers","itemViews"],"primary_key":["id"],"column_metadata":{"id":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"idProperty":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"date":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"dayOfWeek":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"totalUsers":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"itemViews":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}]}} \ No newline at end of file diff --git a/tests/functional/service-account/expected/data/out/usage.json b/tests/functional/service-account/expected/data/out/usage.json new file mode 100644 index 0000000..7121ffb --- /dev/null +++ b/tests/functional/service-account/expected/data/out/usage.json @@ -0,0 +1 @@ +[{"metric":"API Calls","value":1}] \ No newline at end of file diff --git a/tests/functional/service-account/source/data/config.json b/tests/functional/service-account/source/data/config.json new file mode 100644 index 0000000..5a68cf2 --- /dev/null +++ b/tests/functional/service-account/source/data/config.json @@ -0,0 +1,42 @@ +{ + "parameters": { + "service_account": "%env(string:SERVICE_ACCOUNT_JSON)%", + "outputBucket": "in.c-ex-google-analytics-cfg", + "retriesCount": 1, + "properties": [ + { + "accountKey": "accounts/185283969", + "accountName": "Keboola", + "propertyKey": "properties/255885884", + "propertyName": "users" + } + ], + "outputTable": "users", + "endpoint": "data-api", + "query": { + "metrics": [ + { + "name": "totalUsers" + }, + { + "name": "itemViews" + } + ], + "dimensions": [ + { + "name": "date" + }, + { + "name": "dayOfWeek" + } + ], + "filtersExpression": "", + "dateRanges": [ + { + "startDate": "2023-03-07", + "endDate": "2023-06-06" + } + ] + } + } +}