Merge pull request #2495 from keboola/fix-skip-transformation-schema-validation

fix: Skip transformation schema validation

Author: Matovidlo

.gitignore

@@ -51,6 +51,7 @@ vendor/*
 
 # CPU profiles
 /**/*.prof
+/.claude
 
 # Certificates
 /ca

internal/pkg/diff/reporter.go

@@ -99,7 +99,7 @@ func (r *Reporter) Paths() []string {
 }
 
 func (r *Reporter) relationsDiff(remoteValue, localValue reflect.Value) ([]string, bool) {
-	relationsType := reflect.TypeOf((*model.Relations)(nil)).Elem()
+	relationsType := reflect.TypeFor[model.Relations]()
 	if remoteValue.IsValid() && localValue.IsValid() && remoteValue.Type().ConvertibleTo(relationsType) && localValue.Type().ConvertibleTo(relationsType) {
 		onlyInRemote, onlyInLocal := remoteValue.Interface().(model.Relations).Diff(localValue.Interface().(model.Relations))
 		var out []string

internal/pkg/encoding/json/schema/schema.go

@@ -68,6 +68,10 @@ func ValidateConfig(component *keboola.Component, config *model.Config) error {
 	if component.IsDeprecated() {
 		return nil
 	}
+	// Skip components with custom schema handling
+	if skipSchemaValidation(component.ID) {
+		return nil
+	}
 	return ValidateContent(component.Schema, config.Content)
 }
 
@@ -76,6 +80,10 @@ func ValidateConfigRow(component *keboola.Component, configRow *model.ConfigRow)
 	if component.IsDeprecated() {
 		return nil
 	}
+	// Skip components with custom schema handling
+	if skipSchemaValidation(component.ID) {
+		return nil
+	}
 	return ValidateContent(component.SchemaRow, configRow.Content)
 }
 
@@ -249,3 +257,15 @@ func compileSchema(s []byte, savePropertyOrder bool) (*jsonschema.Schema, error)
 
 	return schema, nil
 }
+
+// skipSchemaValidation returns true for components where schema validation should be skipped.
+func skipSchemaValidation(componentID keboola.ComponentID) bool {
+	switch componentID {
+	case "keboola.python-transformation-v2",
+		"keboola.snowflake-transformation",
+		"keboola.google-bigquery-transformation":
+		return true
+	default:
+		return false
+	}
+}

internal/pkg/encoding/json/schema/schema_test.go

@@ -384,6 +384,55 @@ func testInvalidComponentSchema(t *testing.T, invalidSchema []byte, expectedLogs
 	assert.Equal(t, strings.TrimLeft(expectedLogs, "\n"), logger.AllMessagesTxt())
 }
 
+func TestValidateConfig_SkipTransformationComponents(t *testing.T) {
+	t.Parallel()
+
+	// Schema that requires "firstName" field
+	schema := getTestSchema()
+
+	// Content that violates the schema (missing required "firstName")
+	invalidContent := orderedmap.FromPairs([]orderedmap.Pair{
+		{
+			Key: "parameters",
+			Value: orderedmap.FromPairs([]orderedmap.Pair{
+				{Key: "lastName", Value: "Brown"},
+			}),
+		},
+	})
+
+	// Test that validation is skipped for transformation components
+	transformationComponents := []keboola.ComponentID{
+		"keboola.python-transformation-v2",
+		"keboola.snowflake-transformation",
+		"keboola.google-bigquery-transformation",
+	}
+
+	for _, componentID := range transformationComponents {
+		component := &keboola.Component{
+			ComponentKey: keboola.ComponentKey{ID: componentID},
+			Type:         "transformation",
+			Name:         "Test Transformation",
+			Schema:       schema,
+		}
+		config := &model.Config{Content: invalidContent}
+
+		// Should return nil because validation is skipped for transformation components
+		err := ValidateConfig(component, config)
+		require.NoError(t, err, "validation should be skipped for component %s", componentID)
+	}
+
+	// Verify that a regular component still gets validated
+	regularComponent := &keboola.Component{
+		ComponentKey: keboola.ComponentKey{ID: "keboola.ex-generic"},
+		Type:         "extractor",
+		Name:         "Generic Extractor",
+		Schema:       schema,
+	}
+	config := &model.Config{Content: invalidContent}
+	err := ValidateConfig(regularComponent, config)
+	require.Error(t, err, "validation should NOT be skipped for regular component")
+}
+
 func TestNormalizeSchema_RequiredTrue(t *testing.T) {
 	t.Parallel()
 

internal/pkg/filesystem/aferofs/filesystem_test.go

@@ -97,7 +97,7 @@ type testCases struct {
 func (tc *testCases) runTests(t *testing.T) {
 	t.Helper()
 	// Call all Test* methods
-	tp := reflect.TypeOf(tc)
+	tp := reflect.TypeFor[*testCases]()
 	prefix := `Test`
 	for i := range tp.NumMethod() {
 		method := tp.Method(i)

internal/pkg/filesystem/fileloader/fileloader_test.go

@@ -57,7 +57,7 @@ type testCases struct {
 func (tc *testCases) runTests(t *testing.T) {
 	t.Helper()
 	// Call all Test* methods
-	tp := reflect.TypeOf(tc)
+	tp := reflect.TypeFor[*testCases]()
 	prefix := `Test`
 	for i := range tp.NumMethod() {
 		method := tp.Method(i)

internal/pkg/log/debug.go

@@ -142,7 +142,7 @@ func (l *debugLogger) AllMessagesTxt() string {
 	allMessages := l.all.String()
 	scanner := bufio.NewScanner(strings.NewReader(strings.Trim(allMessages, "\n")))
 
-	output := ""
+	var output strings.Builder
 	for scanner.Scan() {
 		message := scanner.Text()
 		var messageData map[string]any
@@ -161,10 +161,10 @@ func (l *debugLogger) AllMessagesTxt() string {
 			panic(errors.New("log message is a json but does not have a \"level\" field"))
 		}
 
-		output += strings.ToUpper(level) + "  " + message + "\n"
+		output.WriteString(strings.ToUpper(level) + "  " + message + "\n")
 	}
 
-	return output
+	return output.String()
 }
 
 // CompareJSONMessages checks that expected json messages appear in actual in the same order.

internal/pkg/service/cli/dialog/inputs_detail.go

@@ -181,7 +181,8 @@ func (d *inputsDetailDialog) parse(ctx context.Context, result string) (input.St
 
 func (d *inputsDetailDialog) defaultValue() string {
 	// File header - info for user
-	fileHeader := `
+	var fileHeader strings.Builder
+	fileHeader.WriteString(`
 <!--
 Please complete definition of the user inputs.
 Edit lines below "## Input ...".
@@ -227,26 +228,26 @@ Options format:
      options: {"value1":"Label 1","value2":"Label 2","value3":"Label 3"}
 
 Preview of steps and groups you created:
-`
+`)
 	var defaultStepID string
 	for _, group := range d.stepsGroups {
-		fileHeader += fmt.Sprintf("- Group %d: %s\n", group.GroupIndex+1, group.Description)
+		fileHeader.WriteString(fmt.Sprintf("- Group %d: %s\n", group.GroupIndex+1, group.Description))
 		for _, step := range group.Steps {
-			fileHeader += fmt.Sprintf("  - Step \"%s\": %s - %s\n", step.ID, step.Name, step.Description)
+			fileHeader.WriteString(fmt.Sprintf("  - Step \"%s\": %s - %s\n", step.ID, step.Name, step.Description))
 			if step.GroupIndex == 0 && step.StepIndex == 0 {
 				defaultStepID = step.ID
 			}
 		}
 	}
-	fileHeader += `
+	fileHeader.WriteString(`
 -->
 
 
-`
+`)
 
 	// Add definitions
 	var lines strings.Builder
-	lines.WriteString(fileHeader)
+	lines.WriteString(fileHeader.String())
 	for _, inputID := range d.inputs.Ids() {
 		i, _ := d.inputs.Get(inputID)
 		lines.WriteString(fmt.Sprintf("## Input \"%s\" (%s)\n", i.ID, i.Type))

pkg/README.md

@@ -1,6 +1,6 @@
 # `/pkg`
 
-Library code that's ok to use by external applications (e.g., `/pkg/mypubliclib`). Other projects will import these libraries expecting them to work, so think twice before you put something here :-) Note that the `internal` directory is a better way to ensure your private packages are not importable because it's enforced by Go. The `/pkg` directory is still a good way to explicitly communicate that the code in that directory is safe for use by others. The [`I'll take pkg over internal`](https://travisjeffery.com/posts/2019-11-01-pkg/) blog post by Travis Jeffery provides a good overview of the `pkg` and `internal` directories and when it might make sense to use them.
+Library code that's ok to use by external applications (e.g., `/pkg/mypubliclib`). Other projects will import these libraries expecting them to work, so think twice before you put something here :-) Note that the `internal` directory is a better way to ensure your private packages are not importable because it's enforced by Go. The `/pkg` directory is still a good way to explicitly communicate that the code in that directory is safe for use by others. The blog post by Travis Jeffery provides a good overview of the `pkg` and `internal` directories and when it might make sense to use them.
 
 It's also a way to group Go code in one place when your root directory contains lots of non-Go components and directories making it easier to run various Go tools (as mentioned in these talks: [`Best Practices for Industrial Programming`](https://www.youtube.com/watch?v=PTE4VJIdHPg) from GopherCon EU 2018, [GopherCon 2018: Kat Zien - How Do You Structure Your Go Apps](https://www.youtube.com/watch?v=oL6JBUk6tj0) and [GoLab 2018 - Massimiliano Pippi - Project layout patterns in Go](https://www.youtube.com/watch?v=3gQa1LWwuzk)).
 

test/stream/bridge/keboola/keboola_test.go

@@ -816,7 +816,7 @@ func (ts *testState) testSlicesUpload(t *testing.T, ctx context.Context, expecta
 
 	// Check content of slices in the staging storage
 	ts.logSection(t, "checking slices content in the staging storage")
-	var allSlicesContent string
+	var allSlicesContent strings.Builder
 	for _, slice := range slicesList {
 		rawReader, err := keboola.DownloadSliceReader(ctx, downloadCred, slice)
 		require.NoError(t, err)
@@ -826,12 +826,12 @@ func (ts *testState) testSlicesUpload(t *testing.T, ctx context.Context, expecta
 		require.NoError(t, err)
 		sliceContent := string(sliceContentBytes)
 		assert.Equal(t, expectations.records.count/2, strings.Count(sliceContent, "\n"))
-		allSlicesContent += sliceContent
+		allSlicesContent.WriteString(sliceContent)
 		require.NoError(t, gzipReader.Close())
 		require.NoError(t, rawReader.Close())
 	}
 	for i := range expectations.records.count {
-		assert.Contains(t, allSlicesContent, fmt.Sprintf(`,"foo%d"`, expectations.records.startID+i))
+		assert.Contains(t, allSlicesContent.String(), fmt.Sprintf(`,"foo%d"`, expectations.records.startID+i))
 	}
 }