Merge pull request #2503 from keboola/vb/DMD-942/llm-export-polish-performance

DMD-942 - Add progress reporting and polish for llm export

Author: vojtabiberle

CLAUDE.md

@@ -227,8 +227,13 @@ func buildUID(prefix, name string) string {
 - **Observability**: Structured logging; OpenTelemetry integration; metrics for critical paths
 - **Early returns**: Prefer early `return` / `continue` to reduce nesting
 - **Default-first assignment**: Set default value first, then override if present. Avoid resetting nil values after assignment.
+- **Deterministic output**: When iterating over maps, always sort results before output to ensure consistent ordering across runs
+- **Avoid else-if chains**: Prefer early returns, switch statements, or restructured logic
+- **Type assertions with nil checks**: After type assertions, always check for nil before using the value
+- **nolint directives**: Use `//nolint:directive` format (no space after //)
+- **Parser/processor pattern**: Use struct with logger as constructor dependency for components that need logging; keep utility functions that don't need logger as standalone
 
-Example - instead of:
+Example of default-first assignment - instead of:
 ```go
 value := someMap[key]
 if value == nil {
@@ -258,6 +263,11 @@ if items == nil {
 }
 ```
 
+### Commit Practices
+- **Make small, focused commits** - One logical change per commit
+- **Don't bundle multiple tasks** into a single commit; split them for easier review and rebasing
+- **Example**: If implementing column metadata, transformation code output, and component configs, create 3 separate commits rather than one large commit
+
 ### Testing
 - Test files use `*_test.go` suffix and are located next to implementation
 - Use `testify/assert` for assertions

internal/pkg/llm/twinformat/configparser/parser.go

@@ -344,21 +344,25 @@ func ParseComponentConfig(comp *keboola.ComponentWithConfigs, cfg *keboola.Confi
 		Created:       cfg.Created.String(),
 	}
 
+	if cfg.Content == nil {
+		return config
+	}
+
 	// Parse storage.output.tables for output mappings (applications/extractors write to tables too)
-	if cfg.Content != nil {
-		if storage, ok := cfg.Content.Get("storage"); ok {
-			if storageMap := toStringMap(storage); storageMap != nil {
-				config.OutputTables = ParseStorageMappings(storageMap, "output")
-			}
+	if storage, ok := cfg.Content.Get("storage"); ok {
+		if storageMap := toStringMap(storage); storageMap != nil {
+			config.OutputTables = ParseStorageMappings(storageMap, "output")
 		}
+	}
 
-		// Convert configuration content to map
-		config.Configuration = make(map[string]any)
-		for _, key := range cfg.Content.Keys() {
-			if val, ok := cfg.Content.Get(key); ok {
-				config.Configuration[key] = val
-			}
+	// Convert configuration content to map
+	config.Configuration = make(map[string]any)
+	for _, key := range cfg.Content.Keys() {
+		val, ok := cfg.Content.Get(key)
+		if !ok {
+			continue
 		}
+		config.Configuration[key] = val
 	}
 
 	return config

internal/pkg/llm/twinformat/generator.go

@@ -7,6 +7,8 @@ import (
 	"sort"
 	"time"
 
+	"github.com/keboola/keboola-sdk-go/v2/pkg/keboola"
+
 	"github.com/keboola/keboola-as-code/internal/pkg/filesystem"
 	"github.com/keboola/keboola-as-code/internal/pkg/llm/twinformat/templates"
 	"github.com/keboola/keboola-as-code/internal/pkg/llm/twinformat/writer"
@@ -95,6 +97,10 @@ func (g *Generator) Generate(ctx context.Context, data *ProcessedData) error {
 		return errors.Errorf("failed to generate AI guide: %w", err)
 	}
 
+	if err := g.generateDataDictionary(ctx, data); err != nil {
+		return errors.Errorf("failed to generate data dictionary: %w", err)
+	}
+
 	g.logger.Infof(ctx, "Twin format output generated successfully")
 	return nil
 }
@@ -276,35 +282,10 @@ func (g *Generator) generateTableMetadata(ctx context.Context, table *ProcessedT
 // buildColumnDetails builds detailed column information including metadata.
 func (g *Generator) buildColumnDetails(table *ProcessedTable) []map[string]any {
 	columns := make([]map[string]any, 0, len(table.Columns))
-
 	for _, colName := range table.Columns {
-		col := map[string]any{
-			"name": colName,
-		}
-
-		// Extract column metadata if available
-		if table.Table != nil && table.ColumnMetadata != nil {
-			if colMeta, ok := table.ColumnMetadata[colName]; ok {
-				for _, meta := range colMeta {
-					switch meta.Key {
-					case "KBC.datatype.basetype":
-						col["base_type"] = meta.Value
-					case "KBC.datatype.type":
-						col["type"] = meta.Value
-					case "KBC.datatype.nullable":
-						col["nullable"] = meta.Value == "1" || meta.Value == "true"
-					case "KBC.datatype.length":
-						col["length"] = meta.Value
-					case "KBC.description":
-						col["description"] = meta.Value
-					}
-				}
-			}
-		}
-
+		col := buildColumnDetailEntry(colName, table.ColumnMetadata)
 		columns = append(columns, col)
 	}
-
 	return columns
 }
 
@@ -835,6 +816,11 @@ func (g *Generator) generateSourcesIndex(ctx context.Context, data *ProcessedDat
 	docFields := SourcesIndexDocFields()
 	sources := buildSourcesList(data.Buckets)
 
+	// Sort sources by ID for deterministic output.
+	sort.Slice(sources, func(i, j int) bool {
+		return sources[i]["id"].(string) < sources[j]["id"].(string)
+	})
+
 	sourcesIndex := map[string]any{
 		"_comment":          docFields.Comment,
 		"_purpose":          docFields.Purpose,
@@ -878,6 +864,8 @@ func buildSourcesList(buckets []*ProcessedBucket) []map[string]any {
 
 	sources := make([]map[string]any, 0, len(sourceMap))
 	for _, info := range sourceMap {
+		// Sort buckets for deterministic output.
+		sort.Strings(info.Buckets)
 		sources = append(sources, map[string]any{
 			"id":           info.ID,
 			"name":         info.Name,
@@ -890,6 +878,60 @@ func buildSourcesList(buckets []*ProcessedBucket) []map[string]any {
 	return sources
 }
 
+// buildColumnDetailEntry builds a detailed column entry map for table metadata.
+// Extracts all available metadata fields (base_type, type, nullable, length, description).
+func buildColumnDetailEntry(colName string, columnMetadata keboola.ColumnsMetadata) map[string]any {
+	col := map[string]any{"name": colName}
+	if columnMetadata == nil {
+		return col
+	}
+
+	colMeta, ok := columnMetadata[colName]
+	if !ok {
+		return col
+	}
+
+	for _, meta := range colMeta {
+		switch meta.Key {
+		case "KBC.datatype.basetype":
+			col["base_type"] = meta.Value
+		case "KBC.datatype.type":
+			col["type"] = meta.Value
+		case "KBC.datatype.nullable":
+			col["nullable"] = meta.Value == "1" || meta.Value == "true"
+		case "KBC.datatype.length":
+			col["length"] = meta.Value
+		case "KBC.description":
+			col["description"] = meta.Value
+		}
+	}
+	return col
+}
+
+// buildColumnEntry builds a column entry map for data dictionary.
+// Extracts type and description from column metadata if available.
+func buildColumnEntry(colName string, columnMetadata keboola.ColumnsMetadata) map[string]any {
+	col := map[string]any{"name": colName}
+	if columnMetadata == nil {
+		return col
+	}
+
+	colMeta, ok := columnMetadata[colName]
+	if !ok {
+		return col
+	}
+
+	for _, meta := range colMeta {
+		switch meta.Key {
+		case "KBC.datatype.basetype":
+			col["type"] = meta.Value
+		case "KBC.description":
+			col["description"] = meta.Value
+		}
+	}
+	return col
+}
+
 // formatSourceName converts a source ID to a human-readable name.
 func formatSourceName(source string) string {
 	names := map[string]string{
@@ -1050,7 +1092,11 @@ func (g *Generator) generateMostConnectedNodes(ctx context.Context, data *Proces
 	}
 
 	sort.Slice(nodes, func(i, j int) bool {
-		return nodes[i].Connections > nodes[j].Connections
+		if nodes[i].Connections != nodes[j].Connections {
+			return nodes[i].Connections > nodes[j].Connections
+		}
+		// Secondary sort by UID for deterministic order when connections are equal.
+		return nodes[i].UID < nodes[j].UID
 	})
 
 	// Limit to top 20 nodes.
@@ -1105,6 +1151,11 @@ func (g *Generator) generateManifestExtended(ctx context.Context, data *Processe
 	docFields := ManifestExtendedDocFields()
 	sources := buildSourcesList(data.Buckets)
 
+	// Sort sources by ID for deterministic output.
+	sort.Slice(sources, func(i, j int) bool {
+		return sources[i]["id"].(string) < sources[j]["id"].(string)
+	})
+
 	// Build platform counts.
 	platformCounts := make(map[string]int)
 	for _, transform := range data.Transformations {
@@ -1209,6 +1260,97 @@ func (g *Generator) generateAIGuide(ctx context.Context, _ *ProcessedData) error
 	return nil
 }
 
+// generateDataDictionary generates a data dictionary from the exported data.
+func (g *Generator) generateDataDictionary(ctx context.Context, data *ProcessedData) error {
+	g.logger.Debugf(ctx, "Generating data dictionary")
+
+	// Create documentation directory.
+	docDir := filesystem.Join(g.outputDir, "documentation")
+	if err := g.fs.Mkdir(ctx, docDir); err != nil {
+		return errors.Errorf("failed to create documentation directory: %w", err)
+	}
+
+	// Build tables section from actual data.
+	tables := make(map[string]map[string]any)
+	for _, table := range data.Tables {
+		tableEntry := map[string]any{
+			"name":         table.Name,
+			"bucket":       table.BucketName,
+			"source":       table.Source,
+			"rows_count":   table.RowsCount,
+			"column_count": len(table.Columns),
+		}
+
+		if table.DisplayName != "" {
+			tableEntry["display_name"] = table.DisplayName
+		}
+
+		// Add column details.
+		columns := make([]map[string]any, 0, len(table.Columns))
+		for _, colName := range table.Columns {
+			col := buildColumnEntry(colName, table.ColumnMetadata)
+			columns = append(columns, col)
+		}
+		tableEntry["columns"] = columns
+
+		tables[table.UID] = tableEntry
+	}
+
+	// Build transformations section.
+	transformations := make(map[string]map[string]any)
+	for _, transform := range data.Transformations {
+		entry := map[string]any{
+			"name":        transform.Name,
+			"platform":    transform.Platform,
+			"is_disabled": transform.IsDisabled,
+		}
+		if transform.Description != "" {
+			entry["description"] = transform.Description
+		}
+		if transform.Dependencies != nil {
+			entry["inputs"] = transform.Dependencies.Consumes
+			entry["outputs"] = transform.Dependencies.Produces
+		}
+		transformations[transform.UID] = entry
+	}
+
+	// Build components section.
+	components := make(map[string]map[string]any)
+	for _, config := range data.ComponentConfigs {
+		entry := map[string]any{
+			"name":           config.Name,
+			"component_id":   config.ComponentID,
+			"component_type": config.ComponentType,
+			"is_disabled":    config.IsDisabled,
+		}
+		if config.Description != "" {
+			entry["description"] = config.Description
+		}
+		components[config.ID] = entry
+	}
+
+	// Build the data dictionary.
+	dictionary := map[string]any{
+		"_comment":          "Auto-generated data dictionary from Keboola project",
+		"_purpose":          "Comprehensive reference of all tables, transformations, and components",
+		"_update_frequency": "Generated on each export",
+		"project_id":        data.ProjectID.String(),
+		"generated_at":      data.ProcessedAt.UTC().Format(time.RFC3339),
+		"summary": map[string]any{
+			"total_tables":          len(data.Tables),
+			"total_transformations": len(data.Transformations),
+			"total_components":      len(data.ComponentConfigs),
+			"total_buckets":         len(data.Buckets),
+		},
+		"tables":          tables,
+		"transformations": transformations,
+		"components":      components,
+	}
+
+	dictPath := filesystem.Join(docDir, "data-dictionary.json")
+	return g.jsonWriter.Write(ctx, dictPath, dictionary)
+}
+
 // GenerateSamples generates sample CSV files for tables.
 func (g *Generator) GenerateSamples(ctx context.Context, data *ProcessedData, samples []*TableSample) error {
 	g.logger.Infof(ctx, "Generating samples for %d tables", len(samples))