Skip to content

Commit bb6e0b0

Browse files
vojtabiberleclaude
andcommitted
DMD-920 - Simplify processor code
- Remove unnecessary jobOutputCount variable that was only used for logging - Use strings.Builder in buildTableUID and buildTransformationUID to prevent string reallocations Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1 parent 380500b commit bb6e0b0

2 files changed

Lines changed: 12 additions & 207 deletions

File tree

internal/pkg/llm/twinformat/fetcher_test.go

Lines changed: 0 additions & 203 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import (
55
"testing"
66

77
"github.com/jarcoal/httpmock"
8-
"github.com/keboola/go-utils/pkg/orderedmap"
98
"github.com/keboola/keboola-sdk-go/v2/pkg/client"
109
"github.com/keboola/keboola-sdk-go/v2/pkg/keboola"
1110
"github.com/stretchr/testify/assert"
@@ -223,208 +222,6 @@ func TestFetchJobsQueue(t *testing.T) {
223222
assert.Equal(t, "error", jobs[1].Status)
224223
}
225224

226-
func TestFetchTransformationConfigs(t *testing.T) {
227-
t.Parallel()
228-
229-
fetcher, transport := newTestFetcher(t)
230-
branchID := keboola.BranchID(123)
231-
232-
// Create config content using orderedmap
233-
configContent := orderedmap.New()
234-
storageSection := orderedmap.New()
235-
inputSection := orderedmap.New()
236-
inputTables := []any{
237-
map[string]any{
238-
"source": "in.c-bucket.source-table",
239-
"destination": "source_table",
240-
},
241-
}
242-
inputSection.Set("tables", inputTables)
243-
storageSection.Set("input", inputSection.ToMap())
244-
245-
outputSection := orderedmap.New()
246-
outputTables := []any{
247-
map[string]any{
248-
"source": "result_table",
249-
"destination": "out.c-bucket.result",
250-
},
251-
}
252-
outputSection.Set("tables", outputTables)
253-
storageSection.Set("output", outputSection.ToMap())
254-
configContent.Set("storage", storageSection.ToMap())
255-
256-
paramsSection := orderedmap.New()
257-
blocks := []any{
258-
map[string]any{
259-
"name": "Block 1",
260-
"codes": []any{
261-
map[string]any{
262-
"name": "Code 1",
263-
"script": "SELECT * FROM source_table;",
264-
},
265-
},
266-
},
267-
}
268-
paramsSection.Set("blocks", blocks)
269-
configContent.Set("parameters", paramsSection.ToMap())
270-
271-
// Mock components response - must include transformation and non-transformation components
272-
transport.RegisterResponder(
273-
http.MethodGet,
274-
`=~/v2/storage/branch/123/components`,
275-
httpmock.NewJsonResponderOrPanic(200, []*keboola.ComponentWithConfigs{
276-
{
277-
Component: keboola.Component{
278-
ComponentKey: keboola.ComponentKey{ID: "keboola.snowflake-transformation"},
279-
Type: "transformation",
280-
Name: "Snowflake Transformation",
281-
Flags: []string{"genericCodeBlocksUI"},
282-
},
283-
Configs: []*keboola.ConfigWithRows{
284-
{
285-
Config: &keboola.Config{
286-
ConfigKey: keboola.ConfigKey{
287-
BranchID: branchID,
288-
ComponentID: "keboola.snowflake-transformation",
289-
ID: "config-1",
290-
},
291-
Name: "Test Transformation",
292-
Description: "A test transformation",
293-
Content: configContent,
294-
},
295-
},
296-
},
297-
},
298-
{
299-
Component: keboola.Component{
300-
ComponentKey: keboola.ComponentKey{ID: "keboola.ex-db-mysql"},
301-
Type: "extractor",
302-
Name: "MySQL Extractor",
303-
},
304-
Configs: []*keboola.ConfigWithRows{
305-
{
306-
Config: &keboola.Config{
307-
ConfigKey: keboola.ConfigKey{
308-
BranchID: branchID,
309-
ComponentID: "keboola.ex-db-mysql",
310-
ID: "config-extractor",
311-
},
312-
Name: "MySQL Extraction",
313-
Content: orderedmap.New(),
314-
},
315-
},
316-
},
317-
},
318-
}),
319-
)
320-
321-
configs, err := fetcher.FetchTransformationConfigs(t.Context(), branchID)
322-
require.NoError(t, err)
323-
324-
// Should only have transformation configs, not extractors
325-
assert.Len(t, configs, 1)
326-
assert.Equal(t, "Test Transformation", configs[0].Name)
327-
assert.Equal(t, "keboola.snowflake-transformation", configs[0].ComponentID)
328-
assert.Len(t, configs[0].InputTables, 1)
329-
assert.Equal(t, "in.c-bucket.source-table", configs[0].InputTables[0].Source)
330-
assert.Len(t, configs[0].OutputTables, 1)
331-
assert.Equal(t, "out.c-bucket.result", configs[0].OutputTables[0].Destination)
332-
assert.Len(t, configs[0].Blocks, 1)
333-
assert.Equal(t, "Block 1", configs[0].Blocks[0].Name)
334-
}
335-
336-
func TestFetchComponentConfigs(t *testing.T) {
337-
t.Parallel()
338-
339-
fetcher, transport := newTestFetcher(t)
340-
branchID := keboola.BranchID(123)
341-
342-
// Create extractor config content
343-
extractorConfig := orderedmap.New()
344-
params := orderedmap.New()
345-
params.Set("host", "db.example.com")
346-
extractorConfig.Set("parameters", params.ToMap())
347-
348-
// Mock components response
349-
transport.RegisterResponder(
350-
http.MethodGet,
351-
`=~/v2/storage/branch/123/components`,
352-
httpmock.NewJsonResponderOrPanic(200, []*keboola.ComponentWithConfigs{
353-
{
354-
Component: keboola.Component{
355-
ComponentKey: keboola.ComponentKey{ID: "keboola.snowflake-transformation"},
356-
Type: "transformation",
357-
Name: "Snowflake Transformation",
358-
Flags: []string{"genericCodeBlocksUI"},
359-
},
360-
Configs: []*keboola.ConfigWithRows{
361-
{
362-
Config: &keboola.Config{
363-
ConfigKey: keboola.ConfigKey{
364-
BranchID: branchID,
365-
ComponentID: "keboola.snowflake-transformation",
366-
ID: "config-1",
367-
},
368-
Name: "Test Transformation",
369-
Content: orderedmap.New(),
370-
},
371-
},
372-
},
373-
},
374-
{
375-
Component: keboola.Component{
376-
ComponentKey: keboola.ComponentKey{ID: "keboola.ex-db-mysql"},
377-
Type: "extractor",
378-
Name: "MySQL Extractor",
379-
},
380-
Configs: []*keboola.ConfigWithRows{
381-
{
382-
Config: &keboola.Config{
383-
ConfigKey: keboola.ConfigKey{
384-
BranchID: branchID,
385-
ComponentID: "keboola.ex-db-mysql",
386-
ID: "config-extractor",
387-
},
388-
Name: "MySQL Extraction",
389-
Description: "Extract from MySQL",
390-
Content: extractorConfig,
391-
},
392-
},
393-
},
394-
},
395-
{
396-
Component: keboola.Component{
397-
ComponentKey: keboola.ComponentKey{ID: keboola.SchedulerComponentID},
398-
Type: "other",
399-
Name: "Scheduler",
400-
},
401-
Configs: []*keboola.ConfigWithRows{
402-
{
403-
Config: &keboola.Config{
404-
ConfigKey: keboola.ConfigKey{
405-
BranchID: branchID,
406-
ComponentID: keboola.SchedulerComponentID,
407-
ID: "scheduler-1",
408-
},
409-
Name: "Scheduler Config",
410-
Content: orderedmap.New(),
411-
},
412-
},
413-
},
414-
},
415-
}),
416-
)
417-
418-
configs, err := fetcher.FetchComponentConfigs(t.Context(), branchID)
419-
require.NoError(t, err)
420-
421-
// Should only have non-transformation, non-scheduler configs
422-
assert.Len(t, configs, 1)
423-
assert.Equal(t, "MySQL Extraction", configs[0].Name)
424-
assert.Equal(t, "keboola.ex-db-mysql", configs[0].ComponentID)
425-
assert.Equal(t, "extractor", configs[0].ComponentType)
426-
}
427-
428225
func TestFetchAll(t *testing.T) {
429226
t.Parallel()
430227

internal/pkg/llm/twinformat/processor.go

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -229,9 +229,9 @@ func (p *Processor) buildTableSourceRegistry(ctx context.Context, transformConfi
229229
}
230230

231231
// PRIORITY 3: Job outputs (fallback for tables not declared in configurations)
232-
jobOutputCount := p.registerTablesFromJobs(jobs, componentRegistry, registry)
232+
p.registerTablesFromJobs(jobs, componentRegistry, registry)
233233

234-
p.logger.Infof(ctx, "Built table source registry with %d table mappings (%d from jobs as fallback)", len(registry.tableToSource), jobOutputCount)
234+
p.logger.Infof(ctx, "Built table source registry with %d table mappings", len(registry.tableToSource))
235235
return registry
236236
}
237237

@@ -632,10 +632,18 @@ func formatJobTimePtr(t *iso8601.Time) string {
632632

633633
// buildTableUID builds a table UID from bucket and table name.
634634
func buildTableUID(bucket, table string) string {
635-
return "table:" + bucket + "/" + table
635+
var b strings.Builder
636+
b.WriteString("table:")
637+
b.WriteString(bucket)
638+
b.WriteByte('/')
639+
b.WriteString(table)
640+
return b.String()
636641
}
637642

638643
// buildTransformationUID builds a transformation UID from a name.
639644
func buildTransformationUID(name string) string {
640-
return "transform:" + name
645+
var b strings.Builder
646+
b.WriteString("transform:")
647+
b.WriteString(name)
648+
return b.String()
641649
}

0 commit comments

Comments
 (0)