Skip to content

Commit 9323bbe

Browse files
vojtabiberlevojtabiberle
authored
Merge pull request #188 from keboola/vb-DMD-58-snowflake-datatype-test
DMD-58 - Snowflake datatype test
2 parents e872a5e + d50864e commit 9323bbe

2 files changed

Lines changed: 328 additions & 1 deletion

File tree

‎src/Schema/Snowflake/SnowflakeSchemaReflection.php‎

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,27 @@ final class SnowflakeSchemaReflection implements SchemaReflectionInterface
2121

2222
private string $schemaName;
2323

24+
private const TYPE_NEED_FALLBACK = [
25+
Snowflake::TYPE_BINARY,
26+
Snowflake::TYPE_VARBINARY,
27+
Snowflake::TYPE_VECTOR,
28+
];
29+
/**
30+
* @var array<string, array<string, array{name: string,
31+
* type: string,
32+
* kind: string,
33+
* null?: string,
34+
* default: ?string,
35+
* "primary key": string,
36+
* "unique key": string,
37+
* check: ?string,
38+
* expression: ?string,
39+
* comment: ?string,
40+
* "policy name": ?string,
41+
* "privacy domain": ?string}>>
42+
*/
43+
private array $fallbackTableCache = [];
44+
2445
public function __construct(Connection $connection, string $schemaName)
2546
{
2647
$this->schemaName = $schemaName;
@@ -176,7 +197,12 @@ public function getDefinitions(): array
176197
// array{TABLE_NAME: string, name: string, type: string, default: string, null?: string}.
177198
// @phpstan-ignore-next-line
178199
$column['null?'] = ($column['null?'] === 'YES' ? 'Y' : 'N');
179-
$tables[$tableKey]['COLUMNS'][] = SnowflakeColumn::createFromDB($column);
200+
201+
if (in_array($column['type'], self::TYPE_NEED_FALLBACK, true)) {
202+
$tables[$tableKey]['COLUMNS'][] = $this->fallbackColumnType($column['TABLE_NAME'], $column['name']);
203+
} else {
204+
$tables[$tableKey]['COLUMNS'][] = SnowflakeColumn::createFromDB($column);
205+
}
180206
}
181207

182208
foreach ($primaryKeys as $primaryKey) {
@@ -206,4 +232,42 @@ public function getDefinitions(): array
206232
}
207233
return $definitions;
208234
}
235+
236+
/**
237+
* Snowflake does not provide length information for the datatypes listed in
238+
* self::TYPE_NEED_FALLBACK in INFORMATION_SCHEMA.COLUMNS (or anywhere else in INFORMATION_SCHEMA).
239+
* As a result, we must run DESC TABLE on tables that contain columns of these types in order
240+
* to retrieve all the necessary information to properly construct the SnowflakeColumn class.
241+
*/
242+
private function fallbackColumnType(string $tableName, string $columnName): SnowflakeColumn
243+
{
244+
$tableKey = md5($tableName);
245+
if (!array_key_exists($tableKey, $this->fallbackTableCache)) {
246+
/**
247+
* @var array<array{
248+
* name: string,
249+
* type: string,
250+
* kind: string,
251+
* "null?": string,
252+
* default: ?string,
253+
* "primary key": string,
254+
* "unique key": string,
255+
* check: ?string,
256+
* expression: ?string,
257+
* comment: ?string,
258+
* "policy name": ?string,
259+
* "privacy domain": ?string}> $tableDesc
260+
*/
261+
$tableDesc = $this->connection->fetchAllAssociative(sprintf(
262+
'DESC TABLE %s.%s',
263+
SnowflakeQuote::quoteSingleIdentifier($this->schemaName),
264+
SnowflakeQuote::quoteSingleIdentifier($tableName),
265+
));
266+
$this->fallbackTableCache[$tableKey] = array_column($tableDesc, null, 'name');
267+
}
268+
269+
// Offset 'null?' does not exist on
270+
// @phpstan-ignore-next-line
271+
return SnowflakeColumn::createFromDB($this->fallbackTableCache[$tableKey][$columnName]);
272+
}
209273
}

‎tests/Functional/Snowflake/Schema/SnowflakeSchemaReflectionTest.php‎

Lines changed: 263 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,15 @@
44

55
namespace Tests\Keboola\TableBackendUtils\Functional\Snowflake\Schema;
66

7+
use Keboola\Datatype\Definition\Snowflake;
8+
use Keboola\TableBackendUtils\Column\Snowflake\SnowflakeColumn;
79
use Keboola\TableBackendUtils\Escaping\Snowflake\SnowflakeQuote;
810
use Keboola\TableBackendUtils\Schema\Snowflake\SnowflakeSchemaReflection;
11+
use Keboola\TableBackendUtils\Table\Snowflake\SnowflakeTableDefinition;
12+
use RuntimeException;
913
use Tests\Keboola\TableBackendUtils\Functional\Snowflake\SnowflakeBaseCase;
1014
use function PHPUnit\Framework\assertEquals;
15+
use const PHP_EOL;
1116

1217
class SnowflakeSchemaReflectionTest extends SnowflakeBaseCase
1318
{
@@ -163,4 +168,262 @@ public function testGetDefinitionsWithEmptySchema(): void
163168

164169
self::assertCount(0, $definitions);
165170
}
171+
172+
public function testGetDefinitionsCoversAllTypes(): void
173+
{
174+
$this->createSchema(self::TEST_SCHEMA);
175+
176+
$createTableQuery = $this->createTableWithAllSupportedTypes('all_types_table');
177+
$this->connection->executeQuery($createTableQuery);
178+
179+
$definitions = $this->schemaRef->getDefinitions();
180+
181+
$columnsReflection = [];
182+
/** @var SnowflakeColumn $columnReflection */
183+
foreach ($definitions['all_types_table']->getColumnsDefinitions() as $columnReflection) {
184+
$columnName = $columnReflection->getColumnName();
185+
$columnSql = $columnReflection->getColumnDefinition()->getTypeOnlySQLDefinition();
186+
$columnsReflection[$columnName] = $columnSql;
187+
}
188+
189+
$this->assertEquals(
190+
[
191+
'col_number_default' => 'NUMBER (38,0)',
192+
'col_number_9' => 'NUMBER (9,0)',
193+
'col_number_9_2' => 'NUMBER (9,2)',
194+
'col_dec_default' => 'NUMBER (38,0)',
195+
'col_decimal_default' => 'NUMBER (38,0)',
196+
'col_numeric_default' => 'NUMBER (38,0)',
197+
'col_int_default' => 'NUMBER (38,0)',
198+
'col_integer_default' => 'NUMBER (38,0)',
199+
'col_bigint_default' => 'NUMBER (38,0)',
200+
'col_smallint_default' => 'NUMBER (38,0)',
201+
'col_tinyint_default' => 'NUMBER (38,0)',
202+
'col_byteint_default' => 'NUMBER (38,0)',
203+
'col_float_default' => 'FLOAT',
204+
'col_float4_default' => 'FLOAT',
205+
'col_float8_default' => 'FLOAT',
206+
'col_double_default' => 'FLOAT',
207+
'col_double_precision_default'=> 'FLOAT',
208+
'col_real_default' => 'FLOAT',
209+
'col_varchar_default' => 'VARCHAR (16777216)',
210+
'col_varchar_50' => 'VARCHAR (50)',
211+
'col_varchar_100' => 'VARCHAR (100)',
212+
'col_char_default' => 'VARCHAR (1)',
213+
'col_char_10' => 'VARCHAR (10)',
214+
'col_char_20' => 'VARCHAR (20)',
215+
'col_character_default' => 'VARCHAR (1)',
216+
'col_character_20' => 'VARCHAR (20)',
217+
'col_char_varying_default' => 'VARCHAR (16777216)',
218+
'col_character_varying_default'=> 'VARCHAR (16777216)',
219+
'col_string_default' => 'VARCHAR (16777216)',
220+
'col_text_default' => 'VARCHAR (16777216)',
221+
'col_nchar_varying_default' => 'VARCHAR (16777216)',
222+
'col_nchar_default' => 'VARCHAR (1)',
223+
'col_nvarchar_default' => 'VARCHAR (16777216)',
224+
'col_nvarchar2_default' => 'VARCHAR (16777216)',
225+
'col_boolean_default' => 'BOOLEAN',
226+
'col_date_default' => 'DATE',
227+
'col_datetime_default' => 'TIMESTAMP_NTZ (9)',
228+
'col_time_default' => 'TIME (9)',
229+
'col_time_3' => 'TIME (3)',
230+
'col_timestamp_default' => 'TIMESTAMP_NTZ (9)',
231+
'col_timestamp_ntz_default' => 'TIMESTAMP_NTZ (9)',
232+
'col_timestamp_ntz_6' => 'TIMESTAMP_NTZ (6)',
233+
'col_timestamp_ltz_default' => 'TIMESTAMP_LTZ (9)',
234+
'col_timestamp_ltz_6' => 'TIMESTAMP_LTZ (6)',
235+
'col_timestamp_tz_default' => 'TIMESTAMP_TZ (9)',
236+
'col_timestamp_tz_6' => 'TIMESTAMP_TZ (6)',
237+
'col_variant_default' => 'VARIANT',
238+
'col_binary_default' => 'BINARY (8388608)',
239+
'col_varbinary_default' => 'BINARY (8388608)',
240+
'col_object_default' => 'OBJECT',
241+
'col_array_default' => 'ARRAY',
242+
'col_geography_default' => 'GEOGRAPHY',
243+
'col_geometry_default' => 'GEOMETRY',
244+
'col_vector_int_10' => 'VECTOR (INT, 10)',
245+
'col_vector_float_15' => 'VECTOR (FLOAT, 15)',
246+
],
247+
$columnsReflection,
248+
);
249+
}
250+
251+
private function createTableWithAllSupportedTypes(string $tableName): string
252+
{
253+
$typesDefinitions = [];
254+
foreach (Snowflake::TYPES as $type) {
255+
$typesDefinitions += $this->createTypeDefinition($type);
256+
}
257+
258+
return sprintf(
259+
'CREATE TABLE %s (%s);',
260+
SnowflakeQuote::quoteSingleIdentifier($tableName),
261+
PHP_EOL . implode(', ' . PHP_EOL, $typesDefinitions),
262+
);
263+
}
264+
265+
/**
266+
* @return array<string, string>
267+
*/
268+
private function createTypeDefinition(string $typeName): array
269+
{
270+
$definitions = [];
271+
272+
switch ($typeName) {
273+
// Numeric types
274+
case Snowflake::TYPE_NUMBER:
275+
$definitions['col_number_default'] = '"col_number_default" NUMBER';
276+
$definitions['col_number_9'] = '"col_number_9" NUMBER(9)';
277+
$definitions['col_number_9_2'] = '"col_number_9_2" NUMBER(9,2)';
278+
break;
279+
case Snowflake::TYPE_DEC:
280+
$definitions['col_dec_default'] = '"col_dec_default" DEC';
281+
break;
282+
case Snowflake::TYPE_DECIMAL:
283+
$definitions['col_decimal_default'] = '"col_decimal_default" DECIMAL';
284+
break;
285+
case Snowflake::TYPE_NUMERIC:
286+
$definitions['col_numeric_default'] = '"col_numeric_default" NUMERIC';
287+
break;
288+
case Snowflake::TYPE_INT:
289+
$definitions['col_int_default'] = '"col_int_default" INT';
290+
break;
291+
case Snowflake::TYPE_INTEGER:
292+
$definitions['col_integer_default'] = '"col_integer_default" INTEGER';
293+
break;
294+
case Snowflake::TYPE_BIGINT:
295+
$definitions['col_bigint_default'] = '"col_bigint_default" BIGINT';
296+
break;
297+
case Snowflake::TYPE_SMALLINT:
298+
$definitions['col_smallint_default'] = '"col_smallint_default" SMALLINT';
299+
break;
300+
case Snowflake::TYPE_TINYINT:
301+
$definitions['col_tinyint_default'] = '"col_tinyint_default" TINYINT';
302+
break;
303+
case Snowflake::TYPE_BYTEINT:
304+
$definitions['col_byteint_default'] = '"col_byteint_default" BYTEINT';
305+
break;
306+
case Snowflake::TYPE_FLOAT:
307+
$definitions['col_float_default'] = '"col_float_default" FLOAT';
308+
break;
309+
case Snowflake::TYPE_FLOAT4:
310+
$definitions['col_float4_default'] = '"col_float4_default" FLOAT4';
311+
break;
312+
case Snowflake::TYPE_FLOAT8:
313+
$definitions['col_float8_default'] = '"col_float8_default" FLOAT8';
314+
break;
315+
case Snowflake::TYPE_DOUBLE:
316+
$definitions['col_double_default'] = '"col_double_default" DOUBLE';
317+
break;
318+
case Snowflake::TYPE_DOUBLE_PRECISION:
319+
$definitions['col_double_precision_default'] = '"col_double_precision_default" DOUBLE PRECISION';
320+
break;
321+
case Snowflake::TYPE_REAL:
322+
$definitions['col_real_default'] = '"col_real_default" REAL';
323+
break;
324+
325+
// String types
326+
case Snowflake::TYPE_VARCHAR:
327+
$definitions['col_varchar_default'] = '"col_varchar_default" VARCHAR';
328+
$definitions['col_varchar_50'] = '"col_varchar_50" VARCHAR(50)';
329+
$definitions['col_varchar_100'] = '"col_varchar_100" VARCHAR(100)';
330+
break;
331+
case Snowflake::TYPE_CHAR:
332+
$definitions['col_char_default'] = '"col_char_default" CHAR';
333+
$definitions['col_char_10'] = '"col_char_10" CHAR(10)';
334+
$definitions['col_char_20'] = '"col_char_20" CHAR(20)';
335+
break;
336+
case Snowflake::TYPE_CHARACTER:
337+
$definitions['col_character_default'] = '"col_character_default" CHARACTER';
338+
$definitions['col_character_20'] = '"col_character_20" CHARACTER(20)';
339+
break;
340+
case Snowflake::TYPE_CHAR_VARYING:
341+
$definitions['col_char_varying_default'] = '"col_char_varying_default" CHAR VARYING';
342+
break;
343+
case Snowflake::TYPE_CHARACTER_VARYING:
344+
$definitions['col_character_varying_default'] = '"col_character_varying_default" CHARACTER VARYING';
345+
break;
346+
case Snowflake::TYPE_STRING:
347+
$definitions['col_string_default'] = '"col_string_default" STRING';
348+
break;
349+
case Snowflake::TYPE_TEXT:
350+
$definitions['col_text_default'] = '"col_text_default" TEXT';
351+
break;
352+
case Snowflake::TYPE_NCHAR_VARYING:
353+
$definitions['col_nchar_varying_default'] = '"col_nchar_varying_default" NCHAR VARYING';
354+
break;
355+
case Snowflake::TYPE_NCHAR:
356+
$definitions['col_nchar_default'] = '"col_nchar_default" NCHAR';
357+
break;
358+
case Snowflake::TYPE_NVARCHAR:
359+
$definitions['col_nvarchar_default'] = '"col_nvarchar_default" NVARCHAR';
360+
break;
361+
case Snowflake::TYPE_NVARCHAR2:
362+
$definitions['col_nvarchar2_default'] = '"col_nvarchar2_default" NVARCHAR2';
363+
break;
364+
365+
// Boolean
366+
case Snowflake::TYPE_BOOLEAN:
367+
$definitions['col_boolean_default'] = '"col_boolean_default" BOOLEAN';
368+
break;
369+
370+
// Date/Time types
371+
case Snowflake::TYPE_DATE:
372+
$definitions['col_date_default'] = '"col_date_default" DATE';
373+
break;
374+
case Snowflake::TYPE_DATETIME:
375+
$definitions['col_datetime_default'] = '"col_datetime_default" DATETIME';
376+
break;
377+
case Snowflake::TYPE_TIME:
378+
$definitions['col_time_default'] = '"col_time_default" TIME';
379+
$definitions['col_time_3'] = '"col_time_3" TIME(3)';
380+
break;
381+
case Snowflake::TYPE_TIMESTAMP:
382+
$definitions['col_timestamp_default'] = '"col_timestamp_default" TIMESTAMP';
383+
break;
384+
case Snowflake::TYPE_TIMESTAMP_NTZ:
385+
$definitions['col_timestamp_ntz_default'] = '"col_timestamp_ntz_default" TIMESTAMP_NTZ';
386+
$definitions['col_timestamp_ntz_6'] = '"col_timestamp_ntz_6" TIMESTAMP_NTZ(6)';
387+
break;
388+
case Snowflake::TYPE_TIMESTAMP_LTZ:
389+
$definitions['col_timestamp_ltz_default'] = '"col_timestamp_ltz_default" TIMESTAMP_LTZ';
390+
$definitions['col_timestamp_ltz_6'] = '"col_timestamp_ltz_6" TIMESTAMP_LTZ(6)';
391+
break;
392+
case Snowflake::TYPE_TIMESTAMP_TZ:
393+
$definitions['col_timestamp_tz_default'] = '"col_timestamp_tz_default" TIMESTAMP_TZ';
394+
$definitions['col_timestamp_tz_6'] = '"col_timestamp_tz_6" TIMESTAMP_TZ(6)';
395+
break;
396+
397+
// Semi-structured & Other types
398+
case Snowflake::TYPE_VARIANT:
399+
$definitions['col_variant_default'] = '"col_variant_default" VARIANT';
400+
break;
401+
case Snowflake::TYPE_BINARY:
402+
$definitions['col_binary_default'] = '"col_binary_default" BINARY';
403+
break;
404+
case Snowflake::TYPE_VARBINARY:
405+
$definitions['col_varbinary_default'] = '"col_varbinary_default" VARBINARY';
406+
break;
407+
case Snowflake::TYPE_OBJECT:
408+
$definitions['col_object_default'] = '"col_object_default" OBJECT';
409+
break;
410+
case Snowflake::TYPE_ARRAY:
411+
$definitions['col_array_default'] = '"col_array_default" ARRAY';
412+
break;
413+
case Snowflake::TYPE_GEOGRAPHY:
414+
$definitions['col_geography_default'] = '"col_geography_default" GEOGRAPHY';
415+
break;
416+
case Snowflake::TYPE_GEOMETRY:
417+
$definitions['col_geometry_default'] = '"col_geometry_default" GEOMETRY';
418+
break;
419+
case Snowflake::TYPE_VECTOR:
420+
$definitions['col_vector_int_10'] = '"col_vector_int_10" VECTOR(INT, 10)';
421+
$definitions['col_vector_float_15'] = '"col_vector_float_15" VECTOR(FLOAT, 15)';
422+
break;
423+
default:
424+
throw new RuntimeException(sprintf('Unsupported Snowflake type "%s"', $typeName));
425+
}
426+
427+
return $definitions;
428+
}
166429
}

0 commit comments

Comments
 (0)