Merge pull request #126 from keboola/ondra-fix-nullable-columns

ondrajodas · web-flow · commit 85ff382e2a67 · 2025-01-08T13:58:36.000+01:00
fix columns nullability
diff --git a/src/Extractor/Output.php b/src/Extractor/Output.php
@@ -186,7 +186,7 @@ public function createManifest(
             $manifestOptions->addSchema(new ManifestOptionsSchema(
                 $column,
                 ['base' => ['type' => 'STRING']],
-                false,
+                true,
                 isset($primaryKey) && in_array($column, $primaryKey, true),
             ));
         }
diff --git a/tests/functional/data-api-filter-dimension/expected/data/out/tables/users.csv.manifest b/tests/functional/data-api-filter-dimension/expected/data/out/tables/users.csv.manifest
@@ -1 +1 @@
-{"destination":"in.c-ex-google-analytics-cfg.users","incremental":true,"columns":["id","idProperty","date","dayOfWeek","totalUsers","itemViews"],"primary_key":["id"],"column_metadata":{"id":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}],"idProperty":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}],"date":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}],"dayOfWeek":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}],"totalUsers":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}],"itemViews":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}]}}
+{"destination":"in.c-ex-google-analytics-cfg.users","incremental":true,"columns":["id","idProperty","date","dayOfWeek","totalUsers","itemViews"],"primary_key":["id"],"column_metadata":{"id":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"idProperty":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"date":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"dayOfWeek":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"totalUsers":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"itemViews":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}]}}
diff --git a/tests/functional/data-api-filter-metrics-native-types-manifest/expected/data/out/tables/users.csv.manifest b/tests/functional/data-api-filter-metrics-native-types-manifest/expected/data/out/tables/users.csv.manifest
@@ -1 +1 @@
-{"destination":"in.c-ex-google-analytics-cfg.users","incremental":true,"schema":[{"nullable":false,"primary_key":true,"name":"id","data_type":{"base":{"type":"STRING"}}},{"nullable":false,"primary_key":false,"name":"idProperty","data_type":{"base":{"type":"STRING"}}},{"nullable":false,"primary_key":false,"name":"date","data_type":{"base":{"type":"STRING"}}},{"nullable":false,"primary_key":false,"name":"totalUsers","data_type":{"base":{"type":"STRING"}}},{"nullable":false,"primary_key":false,"name":"itemViews","data_type":{"base":{"type":"STRING"}}}]}
+{"destination":"in.c-ex-google-analytics-cfg.users","incremental":true,"schema":[{"nullable":true,"primary_key":true,"name":"id","data_type":{"base":{"type":"STRING"}}},{"nullable":true,"primary_key":false,"name":"idProperty","data_type":{"base":{"type":"STRING"}}},{"nullable":true,"primary_key":false,"name":"date","data_type":{"base":{"type":"STRING"}}},{"nullable":true,"primary_key":false,"name":"totalUsers","data_type":{"base":{"type":"STRING"}}},{"nullable":true,"primary_key":false,"name":"itemViews","data_type":{"base":{"type":"STRING"}}}]}
diff --git a/tests/functional/data-api-filter-metrics/expected/data/out/tables/users.csv.manifest b/tests/functional/data-api-filter-metrics/expected/data/out/tables/users.csv.manifest
@@ -1 +1 @@
-{"destination":"in.c-ex-google-analytics-cfg.users","incremental":true,"columns":["id","idProperty","date","totalUsers","itemViews"],"primary_key":["id"],"column_metadata":{"id":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}],"idProperty":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}],"date":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}],"totalUsers":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}],"itemViews":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}]}}
+{"destination":"in.c-ex-google-analytics-cfg.users","incremental":true,"columns":["id","idProperty","date","totalUsers","itemViews"],"primary_key":["id"],"column_metadata":{"id":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"idProperty":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"date":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"totalUsers":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"itemViews":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}]}}
diff --git a/tests/functional/skip-generate-properties/expected/data/out/tables/users.csv.manifest b/tests/functional/skip-generate-properties/expected/data/out/tables/users.csv.manifest
@@ -1 +1 @@
-{"destination":"out.outputBucket.users","incremental":true,"columns":["id","idProperty","date","totalUsers","itemViews"],"primary_key":["id"],"column_metadata":{"id":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}],"idProperty":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}],"date":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}],"totalUsers":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}],"itemViews":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}]}}
+{"destination":"out.outputBucket.users","incremental":true,"columns":["id","idProperty","date","totalUsers","itemViews"],"primary_key":["id"],"column_metadata":{"id":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"idProperty":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"date":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"totalUsers":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"itemViews":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}]}}

Original file line number	Diff line number	Diff line change
`@@ -186,7 +186,7 @@ public function createManifest(`
`186`	`186`	`$manifestOptions->addSchema(new ManifestOptionsSchema(`
`187`	`187`	`$column,`
`188`	`188`	`['base' => ['type' => 'STRING']],`
`189`		`- false,`
	`189`	`+ true,`
`190`	`190`	`isset($primaryKey) && in_array($column, $primaryKey, true),`
`191`	`191`	`));`
`192`	`192`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		-{"destination":"in.c-ex-google-analytics-cfg.users","incremental":true,"columns":["id","idProperty","date","dayOfWeek","totalUsers","itemViews"],"primary_key":["id"],"column_metadata":{"id":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}],"idProperty":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}],"date":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}],"dayOfWeek":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}],"totalUsers":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}],"itemViews":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}]}}
	`1`	+{"destination":"in.c-ex-google-analytics-cfg.users","incremental":true,"columns":["id","idProperty","date","dayOfWeek","totalUsers","itemViews"],"primary_key":["id"],"column_metadata":{"id":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"idProperty":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"date":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"dayOfWeek":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"totalUsers":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"itemViews":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}]}}
Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		-{"destination":"out.outputBucket.users","incremental":true,"columns":["id","idProperty","date","totalUsers","itemViews"],"primary_key":["id"],"column_metadata":{"id":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}],"idProperty":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}],"date":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}],"totalUsers":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}],"itemViews":[{"key":"KBC.datatype.nullable","value":false},{"key":"KBC.datatype.basetype","value":"STRING"}]}}
	`1`	+{"destination":"out.outputBucket.users","incremental":true,"columns":["id","idProperty","date","totalUsers","itemViews"],"primary_key":["id"],"column_metadata":{"id":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"idProperty":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"date":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"totalUsers":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}],"itemViews":[{"key":"KBC.datatype.nullable","value":true},{"key":"KBC.datatype.basetype","value":"STRING"}]}}