refactor: pass environment to pipeline execution

Author: FabianKramm

cmd/flags/flags.go

@@ -15,7 +15,6 @@ type GlobalFlags struct {
 	Namespace                string
 	KubeContext              string
 	Profiles                 []string
-	ProfileRefresh           bool
 	DisableProfileActivation bool
 	SwitchContext            bool
 	ConfigPath               string
@@ -33,7 +32,6 @@ func (gf *GlobalFlags) ToConfigOptions() *loader.ConfigOptions {
 	return &loader.ConfigOptions{
 		OverrideName:             gf.OverrideName,
 		Profiles:                 profiles,
-		ProfileRefresh:           gf.ProfileRefresh,
 		DisableProfileActivation: gf.DisableProfileActivation,
 		Vars:                     gf.Vars,
 	}
@@ -52,7 +50,6 @@ func SetGlobalFlags(flags *flag.FlagSet) *GlobalFlags {
 	flags.BoolVar(&globalFlags.Silent, "silent", false, "Run in silent mode and prevents any devspace log output except panics & fatals")
 
 	flags.StringSliceVarP(&globalFlags.Profiles, "profile", "p", []string{}, "The DevSpace profiles to apply. Multiple profiles are applied in the order they are specified")
-	flags.BoolVar(&globalFlags.ProfileRefresh, "profile-refresh", false, "If true will pull and re-download profile parent sources")
 	flags.BoolVar(&globalFlags.DisableProfileActivation, "disable-profile-activation", false, "If true will ignore all profile activations")
 	flags.BoolVarP(&globalFlags.SwitchContext, "switch-context", "s", false, "Switches and uses the last kube context and namespace that was used to deploy the DevSpace project")
 	flags.StringVarP(&globalFlags.Namespace, "namespace", "n", "", "The kubernetes namespace to use")

pkg/devspace/config/versions/latest/schema.go

@@ -143,6 +143,28 @@ type Pipeline struct {
 	Run string `yaml:"run,omitempty" json:"run,omitempty"`
 }
 
+func (p *Pipeline) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	pipelineString := ""
+	err := unmarshal(&pipelineString)
+	if err != nil {
+		m := map[string]interface{}{}
+		err := unmarshal(m)
+		if err != nil {
+			return err
+		}
+
+		out, err := json.Marshal(m)
+		if err != nil {
+			return err
+		}
+
+		return json.Unmarshal(out, p)
+	}
+
+	p.Run = pipelineString
+	return nil
+}
+
 type RequireConfig struct {
 	// DevSpace specifies the DevSpace version constraint that is needed to use this config
 	DevSpace string `yaml:"devspace,omitempty" json:"devspace,omitempty"`

pkg/devspace/kubectl/util.go

@@ -168,7 +168,7 @@ func EnsureNamespace(ctx context.Context, client Client, namespace string, log l
 			return err
 		}
 
-		log.Donef("Created namespace: %s", namespace)
+		log.WithPrefixColor("info ", "cyan+b").Donef("Created namespace: %s", namespace)
 	}
 
 	return nil

pkg/devspace/pipeline/engine/pipelinehandler/commands/run_pipelines.go

@@ -7,14 +7,15 @@ import (
 	devspacecontext "github.com/loft-sh/devspace/pkg/devspace/context"
 	"github.com/loft-sh/devspace/pkg/devspace/pipeline/types"
 	"github.com/pkg/errors"
+	"mvdan.cc/sh/v3/expand"
 	"strings"
 )
 
 type RunPipelineOptions struct {
 	types.PipelineOptions
 }
 
-func RunPipelines(ctx devspacecontext.Context, pipeline types.Pipeline, args []string) error {
+func RunPipelines(ctx devspacecontext.Context, pipeline types.Pipeline, args []string, environ expand.Environ) error {
 	ctx.Log().Debugf("run_pipelines %s", strings.Join(args, " "))
 	options := &RunPipelineOptions{}
 	args, err := flags.ParseArgs(options, args)
@@ -39,5 +40,6 @@ func RunPipelines(ctx devspacecontext.Context, pipeline types.Pipeline, args []s
 		return fmt.Errorf("no pipeline to run specified")
 	}
 
+	options.Environ = environ
 	return pipeline.StartNewPipelines(ctx, pipelines, options.PipelineOptions)
 }

pkg/devspace/pipeline/pipeline_job.go pkg/devspace/pipeline/job.gopkg/devspace/pipeline/pipeline_job.go renamed to pkg/devspace/pipeline/job.go

@@ -10,14 +10,14 @@ import (
 	"github.com/loft-sh/devspace/pkg/util/scanner"
 	"github.com/loft-sh/devspace/pkg/util/tomb"
 	"io"
+	"mvdan.cc/sh/v3/expand"
 	"os"
 	"sync"
 )
 
 type Job struct {
 	Pipeline types.Pipeline
 	Config   *latest.Pipeline
-	ExtraEnv map[string]string
 
 	m sync.Mutex
 	t *tomb.Tomb
@@ -47,7 +47,7 @@ func (j *Job) Stop() error {
 	return t.Wait()
 }
 
-func (j *Job) Run(ctx devspacecontext.Context) error {
+func (j *Job) Run(ctx devspacecontext.Context, environ expand.Environ) error {
 	if ctx.IsDone() {
 		return ctx.Context().Err()
 	}
@@ -64,7 +64,7 @@ func (j *Job) Run(ctx devspacecontext.Context) error {
 	t.Go(func() error {
 		// start the actual job
 		done := t.NotifyGo(func() error {
-			return j.execute(ctx, t)
+			return j.execute(ctx, t, environ)
 		})
 
 		// wait until job is dying
@@ -85,7 +85,7 @@ func (j *Job) Run(ctx devspacecontext.Context) error {
 	return t.Wait()
 }
 
-func (j *Job) execute(ctx devspacecontext.Context, parent *tomb.Tomb) error {
+func (j *Job) execute(ctx devspacecontext.Context, parent *tomb.Tomb, environ expand.Environ) error {
 	ctx = ctx.WithLogger(ctx.Log())
 	stdoutReader, stdoutWriter := io.Pipe()
 	defer stdoutWriter.Close()
@@ -110,6 +110,6 @@ func (j *Job) execute(ctx devspacecontext.Context, parent *tomb.Tomb) error {
 	})
 
 	handler := pipelinehandler.NewPipelineExecHandler(ctx, stdoutWriter, stderrWriter, j.Pipeline)
-	_, err := engine.ExecutePipelineShellCommand(ctx.Context(), j.Config.Run, os.Args[1:], ctx.WorkingDir(), j.Config.ContinueOnError, stdoutWriter, stderrWriter, os.Stdin, ctx.Environ(), handler)
+	_, err := engine.ExecutePipelineShellCommand(ctx.Context(), j.Config.Run, os.Args[1:], ctx.WorkingDir(), j.Config.ContinueOnError, stdoutWriter, stderrWriter, os.Stdin, environ, handler)
 	return err
 }

pkg/devspace/pipeline/pipeline.go

@@ -6,6 +6,7 @@ import (
 	"github.com/loft-sh/devspace/pkg/devspace/context/values"
 	"github.com/loft-sh/devspace/pkg/devspace/kubectl"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	"mvdan.cc/sh/v3/expand"
 	"os"
 	"strings"
 	"sync"
@@ -167,7 +168,7 @@ func (p *pipeline) Dependencies() map[string]types.Pipeline {
 }
 
 func (p *pipeline) Run(ctx devspacecontext.Context) error {
-	return p.executeJob(ctx, p.main)
+	return p.executeJob(ctx, p.main, ctx.Environ())
 }
 
 func (p *pipeline) StartNewDependencies(ctx devspacecontext.Context, dependencies []types2.Dependency, options types.DependencyOptions) error {
@@ -193,12 +194,14 @@ func (p *pipeline) StartNewDependencies(ctx devspacecontext.Context, dependencie
 			continue
 		} else if !deployableDependencies[dependency.Name()] {
 			ctx.Log().Infof("Skipping dependency %s as it was either already deployed or is currently in use by another DevSpace instance in the same namespace", dependency.Name())
+
 			continue
 		}
 
 		deployDependencies = append(deployDependencies, dependency)
 	}
 
+	// Start sequentially
 	if options.Sequential {
 		for _, dependency := range deployDependencies {
 			err := p.startNewDependency(ctx, dependency, options)
@@ -307,37 +310,22 @@ func (p *pipeline) StartNewPipelines(ctx devspacecontext.Context, pipelines []*l
 }
 
 func (p *pipeline) startNewPipeline(ctx devspacecontext.Context, configPipeline *latest.Pipeline, id string, options types.PipelineOptions) error {
-	// parse env
-	envMap := map[string]string{}
-	for _, s := range options.Env {
-		if s == "" {
-			continue
-		}
-
-		splitted := strings.Split(s, "=")
-		if len(splitted) <= 1 {
-			return fmt.Errorf("invalid environment variable format. Has to be KEY=VALUE")
-		}
-
-		envMap[splitted[0]] = strings.Join(splitted[1:], "=")
-	}
-
 	// exchange job if it's not alive anymore
-	j, err := p.createJob(configPipeline, envMap, id)
+	j, err := p.createJob(configPipeline, id)
 	if err != nil {
 		return err
 	}
 	defer p.removeJob(j, id)
 
-	err = p.executeJob(ctx, j)
+	err = p.executeJob(ctx, j, options.Environ)
 	if err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func (p *pipeline) createJob(configPipeline *latest.Pipeline, envMap map[string]string, id string) (job *Job, err error) {
+func (p *pipeline) createJob(configPipeline *latest.Pipeline, id string) (job *Job, err error) {
 	p.m.Lock()
 	defer p.m.Unlock()
 
@@ -349,7 +337,6 @@ func (p *pipeline) createJob(configPipeline *latest.Pipeline, envMap map[string]
 	j = &Job{
 		Pipeline: p,
 		Config:   configPipeline,
-		ExtraEnv: envMap,
 	}
 	p.jobs[id] = j
 	return j, nil
@@ -367,13 +354,13 @@ func (p *pipeline) removeJob(j *Job, id string) {
 	}
 }
 
-func (p *pipeline) executeJob(ctx devspacecontext.Context, j *Job) error {
+func (p *pipeline) executeJob(ctx devspacecontext.Context, j *Job, environ expand.Environ) error {
 	// don't start jobs on a cancelled context
 	if ctx.IsDone() {
 		return nil
 	}
 
-	err := j.Run(ctx)
+	err := j.Run(ctx, environ)
 	if err != nil {
 		return err
 	}

pkg/devspace/pipeline/types/pipeline.go

@@ -8,6 +8,7 @@ import (
 	types2 "github.com/loft-sh/devspace/pkg/devspace/dependency/types"
 	"github.com/loft-sh/devspace/pkg/devspace/deploy"
 	"github.com/loft-sh/devspace/pkg/devspace/devpod"
+	"mvdan.cc/sh/v3/expand"
 )
 
 type Options struct {
@@ -27,9 +28,10 @@ type DependencyOptions struct {
 
 // PipelineOptions describe how pipelines should be run
 type PipelineOptions struct {
-	Env        []string `long:"env" description:"Pass the following environment variable to the pipelines"`
-	Background bool     `long:"background" description:"Run the pipeline in the background"`
-	Sequential bool     `long:"sequential" description:"Run pipelines one after another"`
+	Background bool `long:"background" description:"Run the pipeline in the background"`
+	Sequential bool `long:"sequential" description:"Run pipelines one after another"`
+
+	Environ expand.Environ
 }
 
 type Pipeline interface {