Merge pull request #2501 from keboola/vb/DMD-921/llm-export-generator

DMD-921 - Add generator for twin format output

Author: vojtabiberle

CLAUDE.md

@@ -226,6 +226,37 @@ func buildUID(prefix, name string) string {
 - **Dependency management**: Constructor-based DI; interface segregation (small interfaces)
 - **Observability**: Structured logging; OpenTelemetry integration; metrics for critical paths
 - **Early returns**: Prefer early `return` / `continue` to reduce nesting
+- **Default-first assignment**: Set default value first, then override if present. Avoid resetting nil values after assignment.
+
+Example - instead of:
+```go
+value := someMap[key]
+if value == nil {
+    value = defaultValue
+}
+```
+Use:
+```go
+value := defaultValue
+if v, ok := someMap[key]; ok {
+    value = v
+}
+```
+
+Similarly for struct fields:
+```go
+// Good: set default, override if not nil
+items := []string{}
+if data.Items != nil {
+    items = data.Items
+}
+
+// Bad: assign then reset if nil
+items := data.Items
+if items == nil {
+    items = []string{}
+}
+```
 
 ### Testing
 - Test files use `*_test.go` suffix and are located next to implementation

internal/pkg/llm/twinformat/docfields.go

@@ -0,0 +1,108 @@
+package twinformat
+
+// DocFieldsConfig contains the documentation fields for JSON output files.
+// Every JSON file in the twin format output must have these fields.
+type DocFieldsConfig struct {
+	Comment         string
+	Purpose         string
+	UpdateFrequency string
+	Security        string
+	Retention       string
+}
+
+// DefaultDocFields returns the default documentation fields configuration.
+func DefaultDocFields() *DocFieldsConfig {
+	return &DocFieldsConfig{
+		UpdateFrequency: "Every sync",
+	}
+}
+
+// BucketsIndexDocFields returns documentation fields for buckets/index.json.
+func BucketsIndexDocFields() *DocFieldsConfig {
+	return &DocFieldsConfig{
+		Comment:         "GENERATION: GET /v2/storage/buckets - aggregate all buckets",
+		Purpose:         "Catalog of all buckets for fast lookup without scanning directories",
+		UpdateFrequency: "Every sync",
+	}
+}
+
+// TableMetadataDocFields returns documentation fields for table metadata.json files.
+func TableMetadataDocFields(source string) *DocFieldsConfig {
+	return &DocFieldsConfig{
+		Comment:         "GENERATION: GET /v2/storage/tables/{table_id}?include=columns,metadata + computed dependencies",
+		Purpose:         "Table metadata for " + source,
+		UpdateFrequency: "On table structure changes",
+	}
+}
+
+// TransformationsIndexDocFields returns documentation fields for transformations/index.json.
+func TransformationsIndexDocFields() *DocFieldsConfig {
+	return &DocFieldsConfig{
+		Comment:         "GENERATION: Scan transformation configs + group by platform",
+		Purpose:         "Catalog of all transformations grouped by platform",
+		UpdateFrequency: "Every sync",
+	}
+}
+
+// TransformationMetadataDocFields returns documentation fields for transformation metadata.json files.
+func TransformationMetadataDocFields() *DocFieldsConfig {
+	return &DocFieldsConfig{
+		Comment:         "GENERATION: From transformation config + platform detection + computed dependencies + job queue status",
+		Purpose:         "Complete transformation configuration and data flow dependencies",
+		UpdateFrequency: "On transformation config changes and job completion",
+	}
+}
+
+// JobsIndexDocFields returns documentation fields for jobs/index.json.
+func JobsIndexDocFields() *DocFieldsConfig {
+	return &DocFieldsConfig{
+		Comment:         "GENERATION: GET /search/jobs + aggregate statistics",
+		Purpose:         "Job execution statistics and summary",
+		UpdateFrequency: "Every hour or on job completion",
+	}
+}
+
+// JobMetadataDocFields returns documentation fields for individual job files.
+func JobMetadataDocFields() *DocFieldsConfig {
+	return &DocFieldsConfig{
+		Comment:         "GENERATION: GET /search/jobs/{job_id}",
+		Purpose:         "Individual job execution details",
+		UpdateFrequency: "On job completion",
+	}
+}
+
+// ManifestExtendedDocFields returns documentation fields for manifest-extended.json.
+func ManifestExtendedDocFields() *DocFieldsConfig {
+	return &DocFieldsConfig{
+		Comment:         "GENERATION: GET /v2/storage/tokens/verify + computed statistics",
+		Purpose:         "Complete project overview in one file for fast AI analysis",
+		UpdateFrequency: "Every sync",
+	}
+}
+
+// SourcesIndexDocFields returns documentation fields for indices/sources.json.
+func SourcesIndexDocFields() *DocFieldsConfig {
+	return &DocFieldsConfig{
+		Comment:         "GENERATION: Inferred from bucket names + GET /v2/storage components",
+		Purpose:         "Registry of data sources with bucket and table counts",
+		UpdateFrequency: "Every sync",
+	}
+}
+
+// ComponentsIndexDocFields returns documentation fields for components/index.json.
+func ComponentsIndexDocFields() *DocFieldsConfig {
+	return &DocFieldsConfig{
+		Comment:         "GENERATION: Scan component configs + group by type",
+		Purpose:         "Catalog of all components grouped by type",
+		UpdateFrequency: "Every sync",
+	}
+}
+
+// ComponentMetadataDocFields returns documentation fields for component metadata.json files.
+func ComponentMetadataDocFields() *DocFieldsConfig {
+	return &DocFieldsConfig{
+		Comment:         "GENERATION: From component config + job queue status",
+		Purpose:         "Complete component configuration and execution status",
+		UpdateFrequency: "On component config changes and job completion",
+	}
+}

internal/pkg/llm/twinformat/docfields_test.go

@@ -0,0 +1,76 @@
+package twinformat
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDefaultDocFields(t *testing.T) {
+	t.Parallel()
+
+	fields := DefaultDocFields()
+	assert.NotNil(t, fields)
+	assert.Equal(t, "Every sync", fields.UpdateFrequency)
+}
+
+func TestBucketsIndexDocFields(t *testing.T) {
+	t.Parallel()
+
+	fields := BucketsIndexDocFields()
+	assert.NotNil(t, fields)
+	assert.Contains(t, fields.Comment, "buckets")
+	assert.Contains(t, fields.Purpose, "Catalog")
+	assert.Equal(t, "Every sync", fields.UpdateFrequency)
+}
+
+func TestTableMetadataDocFields(t *testing.T) {
+	t.Parallel()
+
+	fields := TableMetadataDocFields("shopify")
+	assert.NotNil(t, fields)
+	assert.Contains(t, fields.Comment, "tables")
+	assert.Contains(t, fields.Purpose, "shopify")
+}
+
+func TestTransformationsIndexDocFields(t *testing.T) {
+	t.Parallel()
+
+	fields := TransformationsIndexDocFields()
+	assert.NotNil(t, fields)
+	assert.Contains(t, fields.Comment, "transformation")
+	assert.Contains(t, fields.Purpose, "Catalog")
+}
+
+func TestJobsIndexDocFields(t *testing.T) {
+	t.Parallel()
+
+	fields := JobsIndexDocFields()
+	assert.NotNil(t, fields)
+	assert.Contains(t, fields.Comment, "jobs")
+	assert.Contains(t, fields.Purpose, "Job execution")
+}
+
+func TestManifestExtendedDocFields(t *testing.T) {
+	t.Parallel()
+
+	fields := ManifestExtendedDocFields()
+	assert.NotNil(t, fields)
+	assert.Contains(t, fields.Purpose, "project overview")
+}
+
+func TestSourcesIndexDocFields(t *testing.T) {
+	t.Parallel()
+
+	fields := SourcesIndexDocFields()
+	assert.NotNil(t, fields)
+	assert.Contains(t, fields.Purpose, "data sources")
+}
+
+func TestComponentsIndexDocFields(t *testing.T) {
+	t.Parallel()
+
+	fields := ComponentsIndexDocFields()
+	assert.NotNil(t, fields)
+	assert.Contains(t, fields.Comment, "component")
+}

internal/pkg/llm/twinformat/fetcher.go

@@ -271,6 +271,9 @@ func (f *Fetcher) FetchTableLastImporter(ctx context.Context, tableID keboola.Ta
 // Format: "Keboola Storage API PHP Client/14 kds-team.app-custom-python"
 // Returns the last space-separated part which is the component ID.
 func extractComponentFromUserAgent(userAgent string) string {
+	if userAgent == "" {
+		return ""
+	}
 	parts := strings.Split(userAgent, " ")
 	return parts[len(parts)-1]
 }