Merge pull request #2498 from keboola/vb/DMD-918/llm-export-scaffolding

DMD-918 - Add CLI scaffolding for kbc llm export command

Author: vojtabiberle

internal/pkg/service/cli/cmd/llm/cmd.go

@@ -3,6 +3,7 @@ package llm
 import (
 	"github.com/spf13/cobra"
 
+	llmExport "github.com/keboola/keboola-as-code/internal/pkg/service/cli/cmd/llm/export"
 	llmInit "github.com/keboola/keboola-as-code/internal/pkg/service/cli/cmd/llm/init"
 	"github.com/keboola/keboola-as-code/internal/pkg/service/cli/dependencies"
 	"github.com/keboola/keboola-as-code/internal/pkg/service/cli/helpmsg"
@@ -15,6 +16,7 @@ func Commands(p dependencies.Provider) *cobra.Command {
 		Long:  helpmsg.Read(`llm/long`),
 	}
 	cmd.AddCommand(
+		llmExport.Command(p),
 		llmInit.Command(p),
 	)
 	return cmd

internal/pkg/service/cli/cmd/llm/export/cmd.go

@@ -0,0 +1,61 @@
+package export
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/keboola/keboola-as-code/internal/pkg/service/cli/dependencies"
+	"github.com/keboola/keboola-as-code/internal/pkg/service/cli/helpmsg"
+	"github.com/keboola/keboola-as-code/internal/pkg/service/common/configmap"
+	exportOp "github.com/keboola/keboola-as-code/pkg/lib/operation/llm/export"
+)
+
+type Flags struct {
+	StorageAPIHost  configmap.Value[string] `configKey:"storage-api-host" configShorthand:"H" configUsage:"storage API host, eg. \"connection.keboola.com\""`
+	StorageAPIToken configmap.Value[string] `configKey:"storage-api-token" configShorthand:"t" configUsage:"storage API token from your project"`
+	Force           configmap.Value[bool]   `configKey:"force" configShorthand:"f" configUsage:"skip confirmation when directory contains existing files"`
+}
+
+func DefaultFlags() Flags {
+	return Flags{}
+}
+
+func Command(p dependencies.Provider) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "export",
+		Short: helpmsg.Read(`llm/export/short`),
+		Long:  helpmsg.Read(`llm/export/long`),
+		RunE: func(cmd *cobra.Command, args []string) (cmdErr error) {
+			f := Flags{}
+			if err := p.BaseScope().ConfigBinder().Bind(cmd.Context(), cmd.Flags(), args, &f); err != nil {
+				return err
+			}
+
+			// Get dependencies
+			d, err := p.RemoteCommandScope(cmd.Context(), f.StorageAPIHost, f.StorageAPIToken)
+			if err != nil {
+				return err
+			}
+
+			// Validate directory
+			if err := validateDirectory(cmd.Context(), d, f); err != nil {
+				return err
+			}
+
+			// Build options
+			options := exportOp.Options{
+				Force: f.Force.Value,
+			}
+
+			// Send cmd successful/failed event
+			defer d.EventSender().SendCmdEvent(cmd.Context(), d.Clock().Now(), &cmdErr, "llm-export")
+
+			// Export
+			return exportOp.Run(cmd.Context(), options, d)
+		},
+	}
+
+	// Flags
+	configmap.MustGenerateFlags(cmd.Flags(), DefaultFlags())
+
+	return cmd
+}

internal/pkg/service/cli/cmd/llm/export/validate.go

@@ -0,0 +1,70 @@
+package export
+
+import (
+	"context"
+	"strings"
+
+	"github.com/keboola/keboola-as-code/internal/pkg/filesystem"
+	"github.com/keboola/keboola-as-code/internal/pkg/service/cli/dialog"
+	"github.com/keboola/keboola-as-code/internal/pkg/service/cli/prompt"
+	"github.com/keboola/keboola-as-code/internal/pkg/utils/errors"
+)
+
+type validateDependencies interface {
+	Dialogs() *dialog.Dialogs
+	Fs() filesystem.Fs
+}
+
+// isAllowedFile checks if a file is allowed to exist in the directory without prompting.
+func isAllowedFile(name string) bool {
+	switch name {
+	case ".keboola", ".gitignore", ".git":
+		return true
+	}
+	return strings.HasPrefix(name, ".env")
+}
+
+// validateDirectory checks if the current directory is suitable for export.
+// It allows .keboola/, .env*, .gitignore, and .git to exist.
+// If other files exist, it prompts for confirmation unless --force is set.
+func validateDirectory(ctx context.Context, d validateDependencies, f Flags) error {
+	fs := d.Fs()
+
+	// List files in the current directory
+	entries, err := fs.ReadDir(ctx, ".")
+	if err != nil {
+		return errors.Errorf("cannot read directory: %w", err)
+	}
+
+	// Check for files that are not in the allowed list
+	conflicts := make([]string, 0, len(entries))
+	for _, entry := range entries {
+		name := entry.Name()
+		if !isAllowedFile(name) {
+			conflicts = append(conflicts, name)
+		}
+	}
+
+	// If no conflicts, proceed
+	if len(conflicts) == 0 {
+		return nil
+	}
+
+	// If --force is set, proceed with warning
+	if f.Force.Value {
+		return nil
+	}
+
+	// Prompt for confirmation, including the list of conflicting files
+	label := "Directory contains existing files: " + strings.Join(conflicts, ", ") + ". Do you want to continue?"
+	confirmed := d.Dialogs().Confirm(&prompt.Confirm{
+		Label:   label,
+		Default: false,
+	})
+
+	if !confirmed {
+		return errors.New("export cancelled by user")
+	}
+
+	return nil
+}

internal/pkg/service/cli/cmd/llm/export/validate_test.go

@@ -0,0 +1,192 @@
+package export
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/keboola/keboola-as-code/internal/pkg/filesystem"
+	"github.com/keboola/keboola-as-code/internal/pkg/filesystem/aferofs"
+	"github.com/keboola/keboola-as-code/internal/pkg/service/cli/dialog"
+	"github.com/keboola/keboola-as-code/internal/pkg/service/cli/prompt"
+	"github.com/keboola/keboola-as-code/internal/pkg/service/common/configmap"
+)
+
+func TestIsAllowedFile(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name     string
+		filename string
+		expected bool
+	}{
+		// Allowed files
+		{name: "keboola dir", filename: ".keboola", expected: true},
+		{name: "env file", filename: ".env", expected: true},
+		{name: "env local", filename: ".env.local", expected: true},
+		{name: "env dist", filename: ".env.dist", expected: true},
+		{name: "env custom", filename: ".env.production", expected: true},
+		{name: "gitignore", filename: ".gitignore", expected: true},
+		{name: "git dir", filename: ".git", expected: true},
+
+		// Not allowed files
+		{name: "readme", filename: "README.md", expected: false},
+		{name: "src dir", filename: "src", expected: false},
+		{name: "go mod", filename: "go.mod", expected: false},
+		{name: "twin format", filename: "twin_format", expected: false},
+		{name: "main dir", filename: "main", expected: false},
+		{name: "hidden file", filename: ".hidden", expected: false},
+		{name: "env without dot prefix", filename: "env", expected: false},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			result := isAllowedFile(tc.filename)
+			assert.Equal(t, tc.expected, result, "isAllowedFile(%q) should be %v", tc.filename, tc.expected)
+		})
+	}
+}
+
+// mockPrompt implements prompt.Prompt for testing.
+type mockPrompt struct {
+	confirmResult bool
+}
+
+func (m *mockPrompt) IsInteractive() bool                                { return true }
+func (m *mockPrompt) Printf(_ string, _ ...any)                          {}
+func (m *mockPrompt) Confirm(_ *prompt.Confirm) bool                     { return m.confirmResult }
+func (m *mockPrompt) Ask(q *prompt.Question) (string, bool)              { return q.Default, true }
+func (m *mockPrompt) Select(s *prompt.Select) (string, bool)             { return s.Default, true }
+func (m *mockPrompt) SelectIndex(s *prompt.SelectIndex) (int, bool)      { return s.Default, true }
+func (m *mockPrompt) MultiSelect(s *prompt.MultiSelect) ([]string, bool) { return s.Default, true }
+func (m *mockPrompt) MultiSelectIndex(s *prompt.MultiSelectIndex) ([]int, bool) {
+	return s.Default, true
+}
+func (m *mockPrompt) Multiline(q *prompt.Question) (string, bool)        { return q.Default, true }
+func (m *mockPrompt) Editor(_ string, q *prompt.Question) (string, bool) { return q.Default, true }
+
+// mockValidateDeps implements validateDependencies for testing.
+type mockValidateDeps struct {
+	fs      filesystem.Fs
+	dialogs *dialog.Dialogs
+}
+
+func (m *mockValidateDeps) Fs() filesystem.Fs        { return m.fs }
+func (m *mockValidateDeps) Dialogs() *dialog.Dialogs { return m.dialogs }
+
+func TestValidateDirectory(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name          string
+		files         []string
+		dirs          []string
+		force         bool
+		confirmResult bool
+		expectError   bool
+		errorContains string
+	}{
+		{
+			name:        "empty directory",
+			files:       nil,
+			dirs:        nil,
+			force:       false,
+			expectError: false,
+		},
+		{
+			name:        "only allowed files",
+			files:       []string{".env", ".env.local", ".gitignore"},
+			dirs:        []string{".keboola", ".git"},
+			force:       false,
+			expectError: false,
+		},
+		{
+			name:        "conflicts with force flag",
+			files:       []string{"README.md", "src"},
+			dirs:        nil,
+			force:       true,
+			expectError: false,
+		},
+		{
+			name:          "conflicts without force, user confirms",
+			files:         []string{"README.md"},
+			dirs:          nil,
+			force:         false,
+			confirmResult: true,
+			expectError:   false,
+		},
+		{
+			name:          "conflicts without force, user rejects",
+			files:         []string{"README.md"},
+			dirs:          nil,
+			force:         false,
+			confirmResult: false,
+			expectError:   true,
+			errorContains: "export cancelled by user",
+		},
+		{
+			name:        "mixed allowed and not allowed files with force",
+			files:       []string{".env", ".gitignore", "main.go", "go.mod"},
+			dirs:        []string{".keboola"},
+			force:       true,
+			expectError: false,
+		},
+		{
+			name:          "mixed files without force, user rejects",
+			files:         []string{".env", "main.go"},
+			dirs:          nil,
+			force:         false,
+			confirmResult: false,
+			expectError:   true,
+			errorContains: "export cancelled by user",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			ctx := context.Background()
+
+			// Create memory filesystem
+			fs := aferofs.NewMemoryFs()
+
+			// Create test files
+			for _, f := range tc.files {
+				require.NoError(t, fs.WriteFile(ctx, filesystem.NewRawFile(f, "test content")))
+			}
+
+			// Create test directories
+			for _, d := range tc.dirs {
+				require.NoError(t, fs.Mkdir(ctx, d))
+			}
+
+			// Create mock dependencies
+			mockP := &mockPrompt{confirmResult: tc.confirmResult}
+			deps := &mockValidateDeps{
+				fs:      fs,
+				dialogs: dialog.New(mockP),
+			}
+
+			// Create flags
+			flags := Flags{
+				Force: configmap.Value[bool]{Value: tc.force},
+			}
+
+			// Run validation
+			err := validateDirectory(ctx, deps, flags)
+
+			// Check result
+			if tc.expectError {
+				require.Error(t, err)
+				if tc.errorContains != "" {
+					assert.Contains(t, err.Error(), tc.errorContains)
+				}
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}

internal/pkg/service/cli/helpmsg/msg/llm/export/long.txt

@@ -0,0 +1,21 @@
+Export project data to an AI-optimized "twin format" directory structure.
+
+The twin format is designed for AI assistants to understand and work with
+Keboola projects directly from Git repositories. It includes:
+
+  - Bucket and table metadata with schema information
+  - Transformation configurations with platform detection
+  - Component configurations organized by type
+  - Job execution history and statistics
+  - Lineage graph showing data flow dependencies
+  - Optional data samples (controlled by flags)
+
+The export creates a "twin_format/" directory containing JSON files with
+inline documentation (_comment, _purpose, _update_frequency fields).
+
+Security features:
+  - Automatic detection of public Git repositories
+  - Sample export disabled by default for public repos
+  - Secrets (fields starting with #) are encrypted
+
+Use "kbc llm init" first to initialize the project directory.

internal/pkg/service/cli/helpmsg/msg/llm/export/short.txt

@@ -0,0 +1 @@
+Export project data to AI-optimized twin format directory structure.