diff --git a/docs/pages/configuration/deployments/basics.mdx b/docs/pages/configuration/deployments/basics.mdx index bc80d1e7af..0447586dfc 100644 --- a/docs/pages/configuration/deployments/basics.mdx +++ b/docs/pages/configuration/deployments/basics.mdx @@ -107,8 +107,8 @@ deployments: -:::info Sequential Deployment -Unlike images which are build in parallel, deployments will be deployed sequentially following the order in which they are specified in the `devspace.yaml`. +:::info Sequential and Concurrent Deployment +Unlike images, which are built in parallel by default, deployments will be deployed sequentially following the order in which they are specified in the `devspace.yaml`. If a deployment has `concurrent: true` set, then it will run before any sequential deployments and in parallel with other deployments that also have concurrency enabled. ::: ## Run Deployments @@ -123,8 +123,11 @@ The following flags are available for all commands that trigger the deployment p ## Deployment Process -DevSpace loads the `deployments` configuration from `devspace.yaml` and builds one deployment after another in the order that they are specified in the `deployments` array. Additionally, DevSpace also deploys related projects speficied in `dependencies`. +DevSpace loads the `deployments` configuration from `devspace.yaml` and by default deploys each deployment sequentially in the order that they are specified in the `deployments` array. Alternatively, some or all deployments can be configured to deploy in parallel by setting `concurrent: true`. Deployments with concurrency enabled will deploy before sequential deployments. Additionally, DevSpace also deploys related projects specified in `dependencies`. +:::warning Helm hooks and concurrency +When using concurrency for Helm deployments that have Helm hooks, be cautious if those hooks depend on resources created by other deployments. You may want such a deployments to be run sequentially after concurrent deployments are completed. Otherwise, appropriate retry logic will be necessary for the affected Helm hook in order to avoid deployment failure. +::: ### 1. Deploy Dependencies diff --git a/docs/pages/fragments/config-deployments.mdx b/docs/pages/fragments/config-deployments.mdx index 962c769cd8..fa9d711ebd 100644 --- a/docs/pages/fragments/config-deployments.mdx +++ b/docs/pages/fragments/config-deployments.mdx @@ -3,6 +3,7 @@ deployments: # struct[] | Array of deployments - name: my-deployment # string | Name of the deployment namespace: "" # string | Namespace to deploy to (Default: "" = namespace of the active namespace/Space) disabled: false # bool | If true, the deployment will not be deployed, purged or rendered + concurrent: false # bool | Deploy concurrently with other deployments that also have concurrency enabled (Default: false) helm: ... # struct | Use Helm as deployment tool and set options for Helm kubectl: ... # struct | Use "kubectl apply" as deployment tool and set options for kubectl ``` diff --git a/e2e/tests/deploy/deploy.go b/e2e/tests/deploy/deploy.go index 231d88235a..e40efa5003 100644 --- a/e2e/tests/deploy/deploy.go +++ b/e2e/tests/deploy/deploy.go @@ -194,4 +194,78 @@ var _ = DevSpaceDescribe("deploy", func() { framework.ExpectNoError(err) framework.ExpectEqual(out, "test") }) + + ginkgo.It("should deploy applications concurrently", func() { + tempDir, err := framework.CopyToTempDir("tests/deploy/testdata/helm_concurrent_all") + framework.ExpectNoError(err) + defer framework.CleanupTempDir(initialDir, tempDir) + + ns, err := kubeClient.CreateNamespace("deploy") + framework.ExpectNoError(err) + defer func() { + err := kubeClient.DeleteNamespace(ns) + framework.ExpectNoError(err) + }() + + // create a new dev command + deployCmd := &cmd.DeployCmd{ + GlobalFlags: &flags.GlobalFlags{ + NoWarn: true, + Namespace: ns, + }, + } + + // run the command + err = deployCmd.Run(f) + framework.ExpectNoError(err) + + // wait until nginx pod is reachable + out, err := kubeClient.ExecByContainer("app.kubernetes.io/component=test", "container-0", ns, []string{"echo", "-n", "test"}) + framework.ExpectNoError(err) + out2, err := kubeClient.ExecByContainer("app.kubernetes.io/component=test-2", "container-0", ns, []string{"echo", "-n", "test"}) + framework.ExpectNoError(err) + out3, err := kubeClient.ExecByContainer("app.kubernetes.io/component=test-3", "container-0", ns, []string{"echo", "-n", "test"}) + framework.ExpectNoError(err) + + framework.ExpectEqual(out, "test") + framework.ExpectEqual(out2, "test") + framework.ExpectEqual(out3, "test") + }) + + ginkgo.It("should deploy applications mixed concurrently and sequentially", func() { + tempDir, err := framework.CopyToTempDir("tests/deploy/testdata/helm_concurrent_sequential") + framework.ExpectNoError(err) + defer framework.CleanupTempDir(initialDir, tempDir) + + ns, err := kubeClient.CreateNamespace("deploy") + framework.ExpectNoError(err) + defer func() { + err := kubeClient.DeleteNamespace(ns) + framework.ExpectNoError(err) + }() + + // create a new dev command + deployCmd := &cmd.DeployCmd{ + GlobalFlags: &flags.GlobalFlags{ + NoWarn: true, + Namespace: ns, + }, + } + + // run the command + err = deployCmd.Run(f) + framework.ExpectNoError(err) + + // wait until nginx pod is reachable + out, err := kubeClient.ExecByContainer("app.kubernetes.io/component=test", "container-0", ns, []string{"echo", "-n", "test"}) + framework.ExpectNoError(err) + out2, err := kubeClient.ExecByContainer("app.kubernetes.io/component=test-2", "container-0", ns, []string{"echo", "-n", "test"}) + framework.ExpectNoError(err) + out3, err := kubeClient.ExecByContainer("app.kubernetes.io/component=test-3", "container-0", ns, []string{"echo", "-n", "test"}) + framework.ExpectNoError(err) + + framework.ExpectEqual(out, "test") + framework.ExpectEqual(out2, "test") + framework.ExpectEqual(out3, "test") + }) }) diff --git a/e2e/tests/deploy/testdata/helm_concurrent_all/devspace.yaml b/e2e/tests/deploy/testdata/helm_concurrent_all/devspace.yaml new file mode 100644 index 0000000000..4d0b32ebc3 --- /dev/null +++ b/e2e/tests/deploy/testdata/helm_concurrent_all/devspace.yaml @@ -0,0 +1,29 @@ +version: v1beta11 +vars: + - name: IMAGE + value: nginx +deployments: + - name: test + concurrent: true + helm: + componentChart: true + values: + containers: + - image: ${IMAGE} + env: $(cat env.yaml) + - name: test-2 + concurrent: true + helm: + componentChart: true + values: + containers: + - image: ${IMAGE} + env: $(cat env.yaml) + - name: test-3 + concurrent: true + helm: + componentChart: true + values: + containers: + - image: ${IMAGE} + env: $(cat env.yaml) diff --git a/e2e/tests/deploy/testdata/helm_concurrent_all/env.yaml b/e2e/tests/deploy/testdata/helm_concurrent_all/env.yaml new file mode 100644 index 0000000000..cd9057a7d8 --- /dev/null +++ b/e2e/tests/deploy/testdata/helm_concurrent_all/env.yaml @@ -0,0 +1,4 @@ +- name: TEST + value: value +- name: TEST2 + value: value2 \ No newline at end of file diff --git a/e2e/tests/deploy/testdata/helm_concurrent_sequential/devspace.yaml b/e2e/tests/deploy/testdata/helm_concurrent_sequential/devspace.yaml new file mode 100644 index 0000000000..2581b8ad02 --- /dev/null +++ b/e2e/tests/deploy/testdata/helm_concurrent_sequential/devspace.yaml @@ -0,0 +1,28 @@ +version: v1beta11 +vars: + - name: IMAGE + value: nginx +deployments: + - name: test + concurrent: true + helm: + componentChart: true + values: + containers: + - image: ${IMAGE} + env: $(cat env.yaml) + - name: test-2 + concurrent: true + helm: + componentChart: true + values: + containers: + - image: ${IMAGE} + env: $(cat env.yaml) + - name: test-3 + helm: + componentChart: true + values: + containers: + - image: ${IMAGE} + env: $(cat env.yaml) diff --git a/e2e/tests/deploy/testdata/helm_concurrent_sequential/env.yaml b/e2e/tests/deploy/testdata/helm_concurrent_sequential/env.yaml new file mode 100644 index 0000000000..cd9057a7d8 --- /dev/null +++ b/e2e/tests/deploy/testdata/helm_concurrent_sequential/env.yaml @@ -0,0 +1,4 @@ +- name: TEST + value: value +- name: TEST2 + value: value2 \ No newline at end of file diff --git a/examples/quickstart-concurrent/.dockerignore b/examples/quickstart-concurrent/.dockerignore new file mode 100644 index 0000000000..d8e9c24cca --- /dev/null +++ b/examples/quickstart-concurrent/.dockerignore @@ -0,0 +1,6 @@ +Dockerfile +.devspace/ +chart/ +node_modules/ +test/ +devspace.yaml \ No newline at end of file diff --git a/examples/quickstart-concurrent/.gitignore b/examples/quickstart-concurrent/.gitignore new file mode 100644 index 0000000000..3dbf23ac16 --- /dev/null +++ b/examples/quickstart-concurrent/.gitignore @@ -0,0 +1,5 @@ +Dockerfile +.devspace/ +chart/ +node_modules/ +test/ diff --git a/examples/quickstart-concurrent/devspace.yaml b/examples/quickstart-concurrent/devspace.yaml new file mode 100755 index 0000000000..f66e604ae0 --- /dev/null +++ b/examples/quickstart-concurrent/devspace.yaml @@ -0,0 +1,58 @@ +version: v1beta11 +vars: # `vars` specifies variables which may be used as ${VAR_NAME} in devspace.yaml +- name: IMAGE + value: loftsh/javascript:latest +deployments: # `deployments` tells DevSpace how to deploy this project +- name: quickstart-con-1 + concurrent: true + helm: # This deployment uses `helm` but you can also define `kubectl` deployments or kustomizations + componentChart: true + displayOutput: true # We are deploying the so-called Component Chart: https://devspace.sh/component-chart/docs + values: # Under `values` we can define the values for this Helm chart used during `helm install/upgrade` + containers: + - image: ${IMAGE} # Use the value of our `${IMAGE}` variable here (see vars above) + command: ["sleep", "infinity"] +- name: quickstart-con-2 + concurrent: true + helm: # This deployment uses `helm` but you can also define `kubectl` deployments or kustomizations + componentChart: true + displayOutput: true # We are deploying the so-called Component Chart: https://devspace.sh/component-chart/docs + values: # Under `values` we can define the values for this Helm chart used during `helm install/upgrade` + containers: + - image: ${IMAGE} # Use the value of our `${IMAGE}` variable here (see vars above) + command: ["sleep", "infinity"] +- name: quickstart-con-3 + concurrent: true + helm: # This deployment uses `helm` but you can also define `kubectl` deployments or kustomizations + componentChart: true + displayOutput: true # We are deploying the so-called Component Chart: https://devspace.sh/component-chart/docs + values: # Under `values` we can define the values for this Helm chart used during `helm install/upgrade` + containers: + - image: ${IMAGE} # Use the value of our `${IMAGE}` variable here (see vars above) + command: ["sleep", "infinity"] +- name: quickstart-con-4 + concurrent: true + helm: # This deployment uses `helm` but you can also define `kubectl` deployments or kustomizations + componentChart: true + displayOutput: true # We are deploying the so-called Component Chart: https://devspace.sh/component-chart/docs + values: # Under `values` we can define the values for this Helm chart used during `helm install/upgrade` + containers: + - image: ${IMAGE} # Use the value of our `${IMAGE}` variable here (see vars above) + command: ["sleep", "infinity"] +- name: quickstart-con-5 + concurrent: true + helm: # This deployment uses `helm` but you can also define `kubectl` deployments or kustomizations + componentChart: true + displayOutput: true # We are deploying the so-called Component Chart: https://devspace.sh/component-chart/docs + values: # Under `values` we can define the values for this Helm chart used during `helm install/upgrade` + containers: + - image: ${IMAGE} # Use the value of our `${IMAGE}` variable here (see vars above) + command: ["sleep", "infinity"] +- name: quickstart-seq-1 + helm: # This deployment uses `helm` but you can also define `kubectl` deployments or kustomizations + componentChart: true + displayOutput: true # We are deploying the so-called Component Chart: https://devspace.sh/component-chart/docs + values: # Under `values` we can define the values for this Helm chart used during `helm install/upgrade` + containers: + - image: ${IMAGE} # Use the value of our `${IMAGE}` variable here (see vars above) + command: ["sleep", "infinity"] diff --git a/pkg/devspace/config/versions/latest/schema.go b/pkg/devspace/config/versions/latest/schema.go index ff91614c63..e20808e073 100644 --- a/pkg/devspace/config/versions/latest/schema.go +++ b/pkg/devspace/config/versions/latest/schema.go @@ -446,11 +446,12 @@ type BuildOptions struct { // DeploymentConfig defines the configuration how the devspace should be deployed type DeploymentConfig struct { - Name string `yaml:"name" json:"name"` - Namespace string `yaml:"namespace,omitempty" json:"namespace,omitempty"` - Disabled bool `yaml:"disabled,omitempty" json:"disabled,omitempty"` - Helm *HelmConfig `yaml:"helm,omitempty" json:"helm,omitempty"` - Kubectl *KubectlConfig `yaml:"kubectl,omitempty" json:"kubectl,omitempty"` + Name string `yaml:"name" json:"name"` + Namespace string `yaml:"namespace,omitempty" json:"namespace,omitempty"` + Disabled bool `yaml:"disabled,omitempty" json:"disabled,omitempty"` + Helm *HelmConfig `yaml:"helm,omitempty" json:"helm,omitempty"` + Kubectl *KubectlConfig `yaml:"kubectl,omitempty" json:"kubectl,omitempty"` + Concurrent bool `yaml:"concurrent,omitempty" json:"concurrent,omitempty"` } // ComponentConfig holds the component information diff --git a/pkg/devspace/deploy/deploy.go b/pkg/devspace/deploy/deploy.go index ad87ab123e..3d38e93220 100644 --- a/pkg/devspace/deploy/deploy.go +++ b/pkg/devspace/deploy/deploy.go @@ -1,11 +1,13 @@ package deploy import ( + "fmt" "io" "strings" config2 "github.com/loft-sh/devspace/pkg/devspace/config" "github.com/loft-sh/devspace/pkg/devspace/dependency/types" + "github.com/sirupsen/logrus" "github.com/loft-sh/devspace/pkg/devspace/config/versions/latest" "github.com/loft-sh/devspace/pkg/devspace/deploy/deployer" @@ -16,6 +18,7 @@ import ( "github.com/loft-sh/devspace/pkg/devspace/hook" kubectlclient "github.com/loft-sh/devspace/pkg/devspace/kubectl" "github.com/loft-sh/devspace/pkg/util/log" + "github.com/loft-sh/devspace/pkg/util/scanner" "github.com/pkg/errors" ) @@ -157,124 +160,184 @@ func (c *controller) getDeployClient(deployConfig *latest.DeploymentConfig, helm } // Deploy deploys all deployments in the config -func (c *controller) Deploy(options *Options, log log.Logger) error { +func (c *controller) Deploy(options *Options, logLogger log.Logger) error { config := c.config.Config() if config.Deployments != nil && len(config.Deployments) > 0 { helmV2Clients := map[string]helmtypes.Client{} // Execute before deployments deploy hook - err := hook.ExecuteHooks(c.client, c.config, c.dependencies, nil, log, "before:deploy") + err := hook.ExecuteHooks(c.client, c.config, c.dependencies, nil, logLogger, "before:deploy") if err != nil { return err } + var ( + concurrentDeployments []*latest.DeploymentConfig + sequentialDeployments []*latest.DeploymentConfig + ) + for _, deployConfig := range config.Deployments { - if deployConfig.Disabled { - log.Debugf("Skip deployment %s, because it is disabled", deployConfig.Name) - continue + if deployConfig.Concurrent { + concurrentDeployments = append(concurrentDeployments, deployConfig) + } else { + sequentialDeployments = append(sequentialDeployments, deployConfig) } + } - if len(options.Deployments) > 0 { - shouldSkip := true - - for _, deployment := range options.Deployments { - if deployment == strings.TrimSpace(deployConfig.Name) { - shouldSkip = false - break + var ( + errChan = make(chan error) + deployedChan = make(chan bool) + ) + + for i, deployConfig := range concurrentDeployments { + go func(deployConfig *latest.DeploymentConfig, deployNumber int) { + // Create new logger to allow concurrent logging. + reader, writer := io.Pipe() + streamLog := log.NewStreamLogger(writer, logrus.InfoLevel) + logsLog := log.NewPrefixLogger("["+deployConfig.Name+"] ", log.Colors[(len(log.Colors)-1)-(deployNumber%len(log.Colors))], logLogger) + go func() { + scanner := scanner.NewScanner(reader) + for scanner.Scan() { + logsLog.Info(scanner.Text()) } - } + }() - if shouldSkip { - continue - } - } - - var ( - deployClient deployer.Interface - err error - method string - ) - - if deployConfig.Kubectl != nil { - deployClient, err = kubectl.New(c.config, c.dependencies, c.client, deployConfig, log) + wasDeployed, err := c.deployOne(deployConfig, streamLog, options, helmV2Clients) + _ = writer.Close() if err != nil { - return errors.Errorf("error deploying: deployment %s error: %v", deployConfig.Name, err) + errChan <- err + } else { + deployedChan <- wasDeployed } + }(deployConfig, i) + } - method = "kubectl" - } else if deployConfig.Helm != nil { - // Get helm client - helmClient, err := GetCachedHelmClient(c.config.Config(), deployConfig, c.client, helmV2Clients, false, log) - if err != nil { - return err - } + logLogger.StartWait(fmt.Sprintf("Deploying %d deployments concurrently", len(concurrentDeployments))) - deployClient, err = helm.New(c.config, c.dependencies, helmClient, c.client, deployConfig, log) - if err != nil { - return errors.Errorf("error deploying: deployment %s error: %v", deployConfig.Name, err) - } + // Wait for concurrent deployments to complete before starting sequential deployments. + for i := 0; i < len(concurrentDeployments); i++ { + select { + case err := <-errChan: + return err + case <-deployedChan: + logLogger.StartWait(fmt.Sprintf("Deploying %d deployments concurrently", len(concurrentDeployments)-i-1)) - method = "helm" - } else { - return errors.Errorf("error deploying: deployment %s has no deployment method", deployConfig.Name) } + } + logLogger.StopWait() - // Execute before deployment deploy hook - err = hook.ExecuteHooks(c.client, c.config, c.dependencies, map[string]interface{}{ - "DEPLOY_NAME": deployConfig.Name, - "DEPLOY_CONFIG": deployConfig, - }, log, hook.EventsForSingle("before:deploy", deployConfig.Name).With("deploy.beforeDeploy")...) + for _, deployConfig := range sequentialDeployments { + _, err := c.deployOne(deployConfig, logLogger, options, helmV2Clients) if err != nil { return err } + } - wasDeployed, err := deployClient.Deploy(options.ForceDeploy, options.BuiltImages) - if err != nil { - hookErr := hook.ExecuteHooks(c.client, c.config, c.dependencies, map[string]interface{}{ - "DEPLOY_NAME": deployConfig.Name, - "DEPLOY_CONFIG": deployConfig, - "ERROR": err, - }, log, hook.EventsForSingle("error:deploy", deployConfig.Name).With("deploy.errorDeploy")...) - if hookErr != nil { - return hookErr - } + // Execute after deployments deploy hook + err = hook.ExecuteHooks(c.client, c.config, c.dependencies, nil, logLogger, "after:deploy") + if err != nil { + return err + } + } - return errors.Errorf("error deploying %s: %v", deployConfig.Name, err) - } + return nil +} - if wasDeployed { - log.Donef("Successfully deployed %s with %s", deployConfig.Name, method) +func (c *controller) deployOne(deployConfig *latest.DeploymentConfig, log log.Logger, options *Options, helmV2Clients map[string]helmtypes.Client) (bool, error) { + if deployConfig.Disabled { + log.Debugf("Skip deployment %s, because it is disabled", deployConfig.Name) + return true, nil + } - // Execute after deployment deploy hook - err = hook.ExecuteHooks(c.client, c.config, c.dependencies, map[string]interface{}{ - "DEPLOY_NAME": deployConfig.Name, - "DEPLOY_CONFIG": deployConfig, - }, log, hook.EventsForSingle("after:deploy", deployConfig.Name).With("deploy.afterDeploy")...) - if err != nil { - return err - } - } else { - log.Infof("Skipping deployment %s", deployConfig.Name) + if len(options.Deployments) > 0 { + shouldSkip := true - // Execute after deployment deploy hook - err = hook.ExecuteHooks(c.client, c.config, c.dependencies, map[string]interface{}{ - "DEPLOY_NAME": deployConfig.Name, - "DEPLOY_CONFIG": deployConfig, - }, log, hook.EventsForSingle("skip:deploy", deployConfig.Name)...) - if err != nil { - return err - } + for _, deployment := range options.Deployments { + if deployment == strings.TrimSpace(deployConfig.Name) { + shouldSkip = false + break } } - // Execute after deployments deploy hook - err = hook.ExecuteHooks(c.client, c.config, c.dependencies, nil, log, "after:deploy") + if shouldSkip { + return true, nil + } + } + + var ( + deployClient deployer.Interface + err error + method string + ) + + if deployConfig.Kubectl != nil { + deployClient, err = kubectl.New(c.config, c.dependencies, c.client, deployConfig, log) if err != nil { - return err + return true, errors.Errorf("error deploying: deployment %s error: %v", deployConfig.Name, err) + } + + method = "kubectl" + } else if deployConfig.Helm != nil { + // Get helm client + helmClient, err := GetCachedHelmClient(c.config.Config(), deployConfig, c.client, helmV2Clients, false, log) + if err != nil { + return true, err } + + deployClient, err = helm.New(c.config, c.dependencies, helmClient, c.client, deployConfig, log) + if err != nil { + return true, errors.Errorf("error deploying: deployment %s error: %v", deployConfig.Name, err) + } + + method = "helm" + } else { + return true, errors.Errorf("error deploying: deployment %s has no deployment method", deployConfig.Name) + } + // Execute before deployment deploy hook + err = hook.ExecuteHooks(c.client, c.config, c.dependencies, map[string]interface{}{ + "DEPLOY_NAME": deployConfig.Name, + "DEPLOY_CONFIG": deployConfig, + }, log, hook.EventsForSingle("before:deploy", deployConfig.Name).With("deploy.beforeDeploy")...) + if err != nil { + return true, err } - return nil + wasDeployed, err := deployClient.Deploy(options.ForceDeploy, options.BuiltImages) + if err != nil { + hookErr := hook.ExecuteHooks(c.client, c.config, c.dependencies, map[string]interface{}{ + "DEPLOY_NAME": deployConfig.Name, + "DEPLOY_CONFIG": deployConfig, + "ERROR": err, + }, log, hook.EventsForSingle("error:deploy", deployConfig.Name).With("deploy.errorDeploy")...) + if hookErr != nil { + return true, hookErr + } + + return true, errors.Errorf("error deploying %s: %v", deployConfig.Name, err) + } + + if wasDeployed { + log.Donef("Successfully deployed %s with %s", deployConfig.Name, method) + // Execute after deployment deploy hook + err = hook.ExecuteHooks(c.client, c.config, c.dependencies, map[string]interface{}{ + "DEPLOY_NAME": deployConfig.Name, + "DEPLOY_CONFIG": deployConfig, + }, log, hook.EventsForSingle("after:deploy", deployConfig.Name).With("deploy.afterDeploy")...) + if err != nil { + return true, err + } + } else { + log.Infof("Skipping deployment %s", deployConfig.Name) + // Execute skip deploy hook + err = hook.ExecuteHooks(c.client, c.config, c.dependencies, map[string]interface{}{ + "DEPLOY_NAME": deployConfig.Name, + "DEPLOY_CONFIG": deployConfig, + }, log, hook.EventsForSingle("skip:deploy", deployConfig.Name)...) + if err != nil { + return true, err + } + } + return false, nil } // Purge removes all deployments or a set of deployments from the cluster diff --git a/pkg/devspace/deploy/deploy_test.go b/pkg/devspace/deploy/deploy_test.go index e3123b1d1c..f590a8698f 100644 --- a/pkg/devspace/deploy/deploy_test.go +++ b/pkg/devspace/deploy/deploy_test.go @@ -137,6 +137,21 @@ func TestDeploy(t *testing.T) { Deployments: map[string]*generated.DeploymentCache{}, }, }, + { + name: "Deploy concurrently", + deployments: []*latest.DeploymentConfig{ + { + Name: "concurrentDeploy", + Kubectl: &latest.KubectlConfig{ + Manifests: []string{}, + }, + Concurrent: true, + }, + }, + cache: &generated.CacheConfig{ + Deployments: map[string]*generated.DeploymentCache{}, + }, + }, } for _, testCase := range testCases {