Skip to content

Commit f577aff

Browse files
MatovidloMatovidlo
authored
Merge pull request #2547 from keboola/mvasko/PSGO-201-kbcignore-field-ignore
feat(cli): kbcignore field-level ignore
2 parents 72bb99c + 159b059 commit f577aff

108 files changed

Lines changed: 1267 additions & 1 deletion

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/test-lint.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ jobs:
4444
args: |
4545
'./**/*.md'
4646
--verbose
47+
--accept 200..=299,429
4748
--exclude-path 'vendor'
4849
--exclude-path 'test'
4950
--exclude '^http://localhost.*'

‎internal/pkg/project/ignore/fields.go‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
package ignore
2+
3+
import (
4+
"github.com/keboola/go-utils/pkg/orderedmap"
5+
6+
"github.com/keboola/keboola-as-code/internal/pkg/model"
7+
)
8+
9+
// SyncDirection indicates which side's value to keep for field-level ignores.
10+
type SyncDirection int
11+
12+
const (
13+
// SyncDirectionPush keeps the remote value (copies remote → local) so the field is not pushed.
14+
SyncDirectionPush SyncDirection = iota
15+
// SyncDirectionPull keeps the local value (copies local → remote) so the field is not pulled.
16+
SyncDirectionPull
17+
)
18+
19+
// IgnoreFields applies field-level ignore rules to the state before diffing.
20+
// For each ignored field, the "authority" side's value is copied to the "edited" side
21+
// so the diff sees no change for that field.
22+
func (f *File) IgnoreFields(direction SyncDirection) error {
23+
for _, ignored := range f.state.IgnoredFields() {
24+
for _, configState := range f.state.Configs() {
25+
if configState.ComponentID.String() != ignored.ComponentID {
26+
continue
27+
}
28+
if configState.ID.String() != ignored.ConfigID {
29+
continue
30+
}
31+
if err := applyFieldOverride(configState, ignored.FieldName, direction); err != nil {
32+
return err
33+
}
34+
}
35+
}
36+
return nil
37+
}
38+
39+
func applyFieldOverride(config *model.ConfigState, fieldName string, direction SyncDirection) error {
40+
local := config.Local
41+
remote := config.Remote
42+
if local == nil || remote == nil {
43+
return nil
44+
}
45+
switch fieldName {
46+
case "isDisabled":
47+
if direction == SyncDirectionPush {
48+
local.IsDisabled = remote.IsDisabled
49+
} else {
50+
remote.IsDisabled = local.IsDisabled
51+
}
52+
return nil
53+
default:
54+
// Treat as dot-notation content key.
55+
return applyContentKeyOverride(local.Content, remote.Content, fieldName, direction)
56+
}
57+
}
58+
59+
func applyContentKeyOverride(localContent, remoteContent *orderedmap.OrderedMap, fieldPath string, direction SyncDirection) error {
60+
if localContent == nil || remoteContent == nil {
61+
return nil
62+
}
63+
if direction == SyncDirectionPush {
64+
// Keep remote: copy remote value → local.
65+
value, found, err := remoteContent.GetNested(fieldPath)
66+
if err != nil {
67+
return err
68+
}
69+
if !found {
70+
return nil
71+
}
72+
return localContent.SetNested(fieldPath, value)
73+
}
74+
// Keep local: copy local value → remote.
75+
value, found, err := localContent.GetNested(fieldPath)
76+
if err != nil {
77+
return err
78+
}
79+
if !found {
80+
return nil
81+
}
82+
return remoteContent.SetNested(fieldPath, value)
83+
}

‎internal/pkg/project/ignore/fields_test.go‎

Lines changed: 234 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,234 @@
1+
package ignore
2+
3+
import (
4+
"testing"
5+
6+
"github.com/keboola/go-utils/pkg/orderedmap"
7+
"github.com/stretchr/testify/assert"
8+
"github.com/stretchr/testify/require"
9+
10+
"github.com/keboola/keboola-as-code/internal/pkg/filesystem"
11+
"github.com/keboola/keboola-as-code/internal/pkg/filesystem/aferofs"
12+
"github.com/keboola/keboola-as-code/internal/pkg/model"
13+
"github.com/keboola/keboola-as-code/internal/pkg/state/registry"
14+
)
15+
16+
func newTestRegistryWithScheduler(t *testing.T) *registry.Registry {
17+
t.Helper()
18+
r := newTestRegistry(t)
19+
20+
localContent := orderedmap.New()
21+
localContent.Set("schedule", orderedmap.FromPairs([]orderedmap.Pair{
22+
{Key: "cronTab", Value: "*/5 * * * *"},
23+
{Key: "timezone", Value: "UTC"},
24+
}))
25+
26+
remoteContent := orderedmap.New()
27+
remoteContent.Set("schedule", orderedmap.FromPairs([]orderedmap.Pair{
28+
{Key: "cronTab", Value: "*/10 * * * *"},
29+
{Key: "timezone", Value: "Europe/Prague"},
30+
}))
31+
32+
configKey := model.ConfigKey{BranchID: 123, ComponentID: "keboola.scheduler", ID: "456"}
33+
configState := &model.ConfigState{
34+
ConfigManifest: &model.ConfigManifest{ConfigKey: configKey},
35+
Local: &model.Config{
36+
ConfigKey: configKey,
37+
Name: "My Schedule",
38+
IsDisabled: true,
39+
Content: localContent,
40+
},
41+
Remote: &model.Config{
42+
ConfigKey: configKey,
43+
Name: "My Schedule",
44+
IsDisabled: false,
45+
Content: remoteContent,
46+
},
47+
}
48+
require.NoError(t, r.Set(configState))
49+
return r
50+
}
51+
52+
func findSchedulerConfig(t *testing.T, r *registry.Registry) *model.ConfigState {
53+
t.Helper()
54+
for _, c := range r.Configs() {
55+
if c.ComponentID.String() == "keboola.scheduler" && c.ID.String() == "456" {
56+
return c
57+
}
58+
}
59+
t.Fatal("scheduler config not found")
60+
return nil
61+
}
62+
63+
func TestIgnoreFields_IsDisabled_Push(t *testing.T) {
64+
t.Parallel()
65+
66+
ctx := t.Context()
67+
r := newTestRegistryWithScheduler(t)
68+
fs := aferofs.NewMemoryFs()
69+
70+
require.NoError(t, fs.WriteFile(ctx, filesystem.NewRawFile("kbcignore", "keboola.scheduler/456:isDisabled")))
71+
72+
file, err := LoadFile(ctx, fs, r, "kbcignore")
73+
require.NoError(t, err)
74+
require.NoError(t, file.IgnoreConfigsOrRows())
75+
76+
// SyncDirectionPush: copy remote.IsDisabled → local.IsDisabled
77+
require.NoError(t, file.IgnoreFields(SyncDirectionPush))
78+
79+
c := findSchedulerConfig(t, r)
80+
// Local isDisabled should match remote (false), not the original local value (true).
81+
assert.False(t, c.Local.IsDisabled)
82+
assert.False(t, c.Remote.IsDisabled)
83+
}
84+
85+
func TestIgnoreFields_IsDisabled_Pull(t *testing.T) {
86+
t.Parallel()
87+
88+
ctx := t.Context()
89+
r := newTestRegistryWithScheduler(t)
90+
fs := aferofs.NewMemoryFs()
91+
92+
require.NoError(t, fs.WriteFile(ctx, filesystem.NewRawFile("kbcignore", "keboola.scheduler/456:isDisabled")))
93+
94+
file, err := LoadFile(ctx, fs, r, "kbcignore")
95+
require.NoError(t, err)
96+
require.NoError(t, file.IgnoreConfigsOrRows())
97+
98+
// SyncDirectionPull: copy local.IsDisabled → remote.IsDisabled
99+
require.NoError(t, file.IgnoreFields(SyncDirectionPull))
100+
101+
c := findSchedulerConfig(t, r)
102+
// Remote isDisabled should match local (true), not the original remote value (false).
103+
assert.True(t, c.Local.IsDisabled)
104+
assert.True(t, c.Remote.IsDisabled)
105+
}
106+
107+
func TestIgnoreFields_ContentKey_Push(t *testing.T) {
108+
t.Parallel()
109+
110+
ctx := t.Context()
111+
r := newTestRegistryWithScheduler(t)
112+
fs := aferofs.NewMemoryFs()
113+
114+
require.NoError(t, fs.WriteFile(ctx, filesystem.NewRawFile("kbcignore", "keboola.scheduler/456:schedule")))
115+
116+
file, err := LoadFile(ctx, fs, r, "kbcignore")
117+
require.NoError(t, err)
118+
require.NoError(t, file.IgnoreConfigsOrRows())
119+
120+
// SyncDirectionPush: copy remote schedule → local schedule
121+
require.NoError(t, file.IgnoreFields(SyncDirectionPush))
122+
123+
c := findSchedulerConfig(t, r)
124+
// Local cronTab should now be the remote value.
125+
localCronTab, found, err := c.Local.Content.GetNested("schedule.cronTab")
126+
require.NoError(t, err)
127+
require.True(t, found)
128+
assert.Equal(t, "*/10 * * * *", localCronTab)
129+
}
130+
131+
func TestIgnoreFields_ContentKey_Pull(t *testing.T) {
132+
t.Parallel()
133+
134+
ctx := t.Context()
135+
r := newTestRegistryWithScheduler(t)
136+
fs := aferofs.NewMemoryFs()
137+
138+
require.NoError(t, fs.WriteFile(ctx, filesystem.NewRawFile("kbcignore", "keboola.scheduler/456:schedule")))
139+
140+
file, err := LoadFile(ctx, fs, r, "kbcignore")
141+
require.NoError(t, err)
142+
require.NoError(t, file.IgnoreConfigsOrRows())
143+
144+
// SyncDirectionPull: copy local schedule → remote schedule
145+
require.NoError(t, file.IgnoreFields(SyncDirectionPull))
146+
147+
c := findSchedulerConfig(t, r)
148+
// Remote cronTab should now be the local value.
149+
remoteCronTab, found, err := c.Remote.Content.GetNested("schedule.cronTab")
150+
require.NoError(t, err)
151+
require.True(t, found)
152+
assert.Equal(t, "*/5 * * * *", remoteCronTab)
153+
}
154+
155+
func TestIgnoreFields_NestedContentKey_Push(t *testing.T) {
156+
t.Parallel()
157+
158+
ctx := t.Context()
159+
r := newTestRegistryWithScheduler(t)
160+
fs := aferofs.NewMemoryFs()
161+
162+
// Only ignore the nested leaf "schedule.cronTab", not the whole "schedule" object.
163+
require.NoError(t, fs.WriteFile(ctx, filesystem.NewRawFile("kbcignore", "keboola.scheduler/456:schedule.cronTab")))
164+
165+
file, err := LoadFile(ctx, fs, r, "kbcignore")
166+
require.NoError(t, err)
167+
require.NoError(t, file.IgnoreConfigsOrRows())
168+
169+
// Push: copy remote cronTab → local cronTab; timezone must remain unchanged.
170+
require.NoError(t, file.IgnoreFields(SyncDirectionPush))
171+
172+
c := findSchedulerConfig(t, r)
173+
// Local cronTab should match remote ("*/10 * * * *").
174+
localCronTab, found, err := c.Local.Content.GetNested("schedule.cronTab")
175+
require.NoError(t, err)
176+
require.True(t, found)
177+
assert.Equal(t, "*/10 * * * *", localCronTab)
178+
// Local timezone must be untouched (still "UTC").
179+
localTZ, found, err := c.Local.Content.GetNested("schedule.timezone")
180+
require.NoError(t, err)
181+
require.True(t, found)
182+
assert.Equal(t, "UTC", localTZ)
183+
}
184+
185+
func TestIgnoreFields_NoMatchingConfig(t *testing.T) {
186+
t.Parallel()
187+
188+
ctx := t.Context()
189+
r := newTestRegistryWithScheduler(t)
190+
fs := aferofs.NewMemoryFs()
191+
192+
// Pattern references a non-existent config ID.
193+
require.NoError(t, fs.WriteFile(ctx, filesystem.NewRawFile("kbcignore", "keboola.scheduler/999:isDisabled")))
194+
195+
file, err := LoadFile(ctx, fs, r, "kbcignore")
196+
require.NoError(t, err)
197+
require.NoError(t, file.IgnoreConfigsOrRows())
198+
require.NoError(t, file.IgnoreFields(SyncDirectionPush))
199+
200+
// Nothing should change.
201+
c := findSchedulerConfig(t, r)
202+
assert.True(t, c.Local.IsDisabled)
203+
assert.False(t, c.Remote.IsDisabled)
204+
}
205+
206+
func TestIgnoreFields_MissingRemote(t *testing.T) {
207+
t.Parallel()
208+
209+
ctx := t.Context()
210+
r := newTestRegistry(t)
211+
fs := aferofs.NewMemoryFs()
212+
213+
// Config with only local state (no remote).
214+
configKey := model.ConfigKey{BranchID: 123, ComponentID: "keboola.scheduler", ID: "456"}
215+
configState := &model.ConfigState{
216+
ConfigManifest: &model.ConfigManifest{ConfigKey: configKey},
217+
Local: &model.Config{
218+
ConfigKey: configKey,
219+
IsDisabled: true,
220+
Content: orderedmap.New(),
221+
},
222+
}
223+
require.NoError(t, r.Set(configState))
224+
225+
require.NoError(t, fs.WriteFile(ctx, filesystem.NewRawFile("kbcignore", "keboola.scheduler/456:isDisabled")))
226+
227+
file, err := LoadFile(ctx, fs, r, "kbcignore")
228+
require.NoError(t, err)
229+
require.NoError(t, file.IgnoreConfigsOrRows())
230+
231+
// Should not panic, just skip silently.
232+
require.NoError(t, file.IgnoreFields(SyncDirectionPush))
233+
assert.True(t, configState.Local.IsDisabled)
234+
}

‎internal/pkg/project/ignore/ignore.go‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ func (f *File) IgnoreConfigsOrRows() error {
1414
func (f *File) applyIgnoredPatterns() error {
1515
for _, pattern := range f.parseIgnoredPatterns() {
1616
if err := f.applyIgnorePattern(pattern); err != nil {
17-
continue
17+
return err
1818
}
1919
}
2020
return nil
@@ -43,6 +43,22 @@ func (f *File) applyIgnorePattern(ignoreConfig string) error {
4343
return nil
4444
}
4545

46+
// Field-level ignore: "componentID/configID:fieldName"
47+
if colonIdx := strings.Index(ignoreConfig, ":"); colonIdx != -1 {
48+
objectPath := ignoreConfig[:colonIdx]
49+
fieldName := ignoreConfig[colonIdx+1:]
50+
if fieldName == "" || strings.HasPrefix(fieldName, ".") || strings.HasSuffix(fieldName, ".") {
51+
return errors.Errorf("invalid field-ignore format %q, expected componentID/configID:fieldName", ignoreConfig)
52+
}
53+
parts := strings.Split(objectPath, "/")
54+
if len(parts) == 2 {
55+
componentID, configID := parts[0], parts[1]
56+
f.state.IgnoreConfigField(componentID, configID, fieldName)
57+
return nil
58+
}
59+
return errors.Errorf("invalid field-ignore format %q, expected componentID/configID:fieldName", ignoreConfig)
60+
}
61+
4662
parts := strings.Split(ignoreConfig, "/")
4763
switch len(parts) {
4864
case 2:

0 commit comments

Comments
 (0)