Merge pull request #262 from keboola/zajca/utils-cleanup

Author: zajca

.env.dist

@@ -11,26 +11,9 @@
 #SNOWFLAKE_DATABASE=USER_IMPORT
 #SNOWFLAKE_WAREHOUSE=DEV
 
-SYNAPSE_UID=UID
-SYNAPSE_PWD=xxxx
-SYNAPSE_DATABASE=USER_IMPORT
-SYNAPSE_SERVER=tcp:<synapse>.database.windows.net,1433
-
-# your local DB prefix to avoid using same DB on the shared server. Probably your name
-TERADATA_DB_PREFIX=
-TERADATA_HOST=
-TERADATA_PORT=1025
-TERADATA_USERNAME=
-TERADATA_PASSWORD=
-TERADATA_DATABASE=
-
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
 
-EXASOL_HOST=exasol:8563
-EXASOL_USERNAME=sys
-EXASOL_PASSWORD=exasol
-
 TEST_PREFIX=change_me_
 
 BQ_KEY_FILE=

.github/workflows/tests.yml

@@ -16,27 +16,6 @@ env:
   SNOWFLAKE_PRIVATE_KEY: ${{ secrets.SNOWFLAKE_PRIVATE_KEY }}
   SNOWFLAKE_DATABASE: KEBOOLA_CI_TABLE_UTILS
   SNOWFLAKE_WAREHOUSE: DEV
-  # Synapse
-  SYNAPSE_UID: ci
-  SYNAPSE_PWD: ${{ secrets.SYNAPSE_PWD }}
-  SYNAPSE_DATABASE: ci-github-actions
-  SYNAPSE_SERVER: ci-github-actions.database.windows.net
-  SYNAPSE_PRINCIPAL: 9a024e58-4bc6-403a-b6a0-78b1ffdaefd6
-  SYNAPSE_PRINCIPAL_PASSWORD: ${{ secrets.SYNAPSE_PRINCIPAL_PASSWORD }}
-  SYNAPSE_PRINCIPAL_TENANT: 9b85ee6f-4fb0-4a46-8cb7-4dcc6b262a89
-  SYNAPSE_RESOURCE_GROUP: ci-php-table-backend-utils
-  SYNAPSE_SQL_SERVER_NAME: ci-github-actions
-  SYNAPSE_DW_SERVER_NAME: ci-github-actions
-  # Teradata
-  TERADATA_HOST: 20.105.40.100
-  TERADATA_USERNAME: dbc
-  TERADATA_PASSWORD: ${{ secrets.TERADATA_PASSWORD }}
-  TERADATA_PORT: 1025
-  TERADATA_DATABASE: ci_table_utils
-  # Exasol
-  EXASOL_HOST: exasol:8563
-  EXASOL_USERNAME: sys
-  EXASOL_PASSWORD: exasol
   # Bigquery
   BQ_KEY_FILE: ${{ secrets.BQ_KEY_FILE }}
 
@@ -74,7 +53,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        suite: [exasol, snowflake, synapse, teradata, bigquery]
+        suite: [snowflake, bigquery]
     steps:
       -
         name: Checkout
@@ -89,20 +68,7 @@ jobs:
         name: Load Docker image
         run: |
           docker load --input /tmp/image.tar
-#
-# Before tests, start services
-      -
-        name: Synapse start
-        if: ${{ matrix.suite == 'synapse' }}
-        run: |
-          bash ./ci/synapse.sh -r
-          bash ./ci/synapse.sh -w
-      -
-        name: Exasol start
-        if: ${{ matrix.suite == 'exasol' }}
-        run: |
-          docker compose up -d exasol
-          sleep 60
+
 #
 # TESTS
 #
@@ -111,14 +77,3 @@ jobs:
         env:
           SUITE: ${{ matrix.suite }}
         run: docker compose run production composer tests-${{ matrix.suite }}
-#
-# After tests, stop services
-      -
-        name: Exasol stop
-        if: ${{ always() && matrix.suite == 'exasol' }}
-        run: docker compose stop exasol
-      -
-        name: Synapse stop
-        if: ${{ always() && matrix.suite == 'synapse' }}
-        run: | 
-          bash ./ci/synapse.sh -p

Dockerfile

@@ -17,8 +17,6 @@ COPY docker/composer-install.sh /tmp/composer-install.sh
 
 RUN apt-get update -q \
     && apt-get install gnupg -y --no-install-recommends \
-    && curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - \
-    && curl https://packages.microsoft.com/config/debian/11/prod.list > /etc/apt/sources.list.d/mssql-release.list \
     && apt-get update -q \
     && ACCEPT_EULA=Y apt-get install -y --no-install-recommends\
         git \
@@ -31,7 +29,6 @@ RUN apt-get update -q \
         debsig-verify \
         dirmngr \
         gpg-agent \
-        msodbcsql17 \
         libonig-dev \
         libxml2-dev \
 	&& rm -r /var/lib/apt/lists/* \
@@ -59,14 +56,6 @@ RUN set -ex; \
     docker-php-ext-install odbc; \
     docker-php-source delete
 
-
-
-#Synapse ODBC
-RUN set -ex; \
-    pecl install sqlsrv-$SQLSRV_VERSION pdo_sqlsrv-$SQLSRV_VERSION; \
-    docker-php-ext-enable sqlsrv pdo_sqlsrv; \
-    docker-php-source delete
-
 ## Snowflake
 COPY ./docker/snowflake/generic.pol /etc/debsig/policies/$SNOWFLAKE_GPG_KEY/generic.pol
 COPY ./docker/snowflake/simba.snowflake.ini /usr/lib/snowflake/odbc/lib/simba.snowflake.ini

README.md

@@ -1,6 +1,6 @@
 # Table backend utils
 
-Common stuff for table backends (Snowflake|Synapse|Redshift) shared between apps.
+Common stuff for table backends (Snowflake|Redshift) shared between apps.
 
 
 ## Interfaces
@@ -149,38 +149,6 @@ cp .env.dist .env
 
 Set `TEST_PREFIX=` ENV
 
-#### SYNAPSE
-
-Create synapse server on Azure portal or using CLI.
-
-set up env variables:
-```bash
-SYNAPSE_UID=UID
-SYNAPSE_PWD=xxxx
-SYNAPSE_DATABASE=synapse_db
-SYNAPSE_SERVER=<synapse>.database.windows.net
-```
-
-#### Teradata
-
-Prepare Teradata servers on AWS/Azure and set following properties. See 
-
-create new database for tests:
-```sql
-CREATE DATABASE <nick>_utils_tests FROM dbc
-    AS PERMANENT = 1e8,
-       SPOOL = 1e8;
-```
-
-set up env variables:
-```bash
-TERADATA_HOST=
-TERADATA_PORT=1025
-TERADATA_USERNAME=
-TERADATA_PASSWORD=
-TERADATA_DATABASE=
-```
-
 #### AWS
 
 In order to download TD odbc driver, create a user who can access S3 bucket with the driver package. 
@@ -191,30 +159,6 @@ AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
 ```
 
-#### Exasol
-
-Run Exasol on your local machine in docker (for this case .env is preconfigured)
-```
-docker compose up -d exasol
-```
-
-Run Exasol server somewhere else and set up env variables:
-```bash
-EXASOL_HOST=
-EXASOL_USERNAME=
-EXASOL_PASSWORD=
-```
-
-_issues:_
-
-- on slower machines Exasol can take a lot of resources. Run `docker compose -f docker compose.yml -f docker compose.limits.yml up exasol` to limit memory and cpu 
-
-- If you are getting error `exadt::ERROR: the private interface with address '<ip>/16' either does not exist or is down.` https://github.com/exasol/docker-db/issues/45
-Exasol saves you (container) current ip address and docker daemon is changing default bridge range. There are two ways to fix this:
-1. each reboot set you current ip range to exasol `docket-compose run --rm exasol exaconf modify-node -n 11 -p '<ip>/16'` this is have to be done each time when ip addresses are not align.
-2. set docker default bridge to some fixed range: edit or create `/etc/docker/daemon.json` and set `{"bip":"172.0.0.1/24"}` (or different range that suites you)
-
-
 #### Snowflake
 
 Prepare credentials for Snowflake access

composer.json

@@ -39,14 +39,10 @@
     },
     "scripts": {
         "tests-unit": "phpunit --colors=always --testsuite unit",
-        "tests-exasol": "phpunit --colors=always --testsuite Exasol",
         "tests-snowflake": "phpunit --colors=always --testsuite Snowflake",
-        "tests-synapse": "phpunit --colors=always --testsuite Synapse",
         "tests-bigquery": "phpunit --colors=always --testsuite Bigquery",
         "tests-functional": [
-            "@tests-exasol",
             "@tests-snowflake",
-            "@tests-synapse",
             "@tests-bigquery"
         ],
         "tests": [

docker-compose.limits.yml

@@ -18,18 +18,6 @@ services:
       - SNOWFLAKE_PRIVATE_KEY
       - SNOWFLAKE_DATABASE
       - SNOWFLAKE_WAREHOUSE
-      - SYNAPSE_UID
-      - SYNAPSE_PWD
-      - SYNAPSE_DATABASE
-      - SYNAPSE_SERVER
-      - TERADATA_HOST
-      - TERADATA_PORT
-      - TERADATA_USERNAME
-      - TERADATA_PASSWORD
-      - TERADATA_DATABASE
-      - EXASOL_HOST
-      - EXASOL_USERNAME
-      - EXASOL_PASSWORD
       - BQ_KEY_FILE
   dev: &dev
     <<: *prod
@@ -40,10 +28,3 @@ services:
   dev-xdebug:
     <<: *dev
     build: docker/xdebug
-  exasol:
-    image: exasol/docker-db:latest-7.1
-    privileged: true
-    volumes:
-      - exa-volume:/exa
-volumes:
-  exa-volume: