Skip to content

Commit 91358c8

Browse files
motatoesmotatoes
authored
rely on variables spec within orchestrator, not on workflow config (#1681)
* rely purely on env spec * fix json for spec
1 parent ce37338 commit 91358c8

File tree

21 files changed

+103
-43
lines changed

21 files changed

+103
-43
lines changed

‎backend/controllers/github.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -519,7 +519,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
519519
return fmt.Errorf("error processing event")
520520
}
521521

522-
jobsForImpactedProjects, _, err := dg_github.ConvertGithubPullRequestEventToJobs(payload, impactedProjects, nil, *config)
522+
jobsForImpactedProjects, _, err := dg_github.ConvertGithubPullRequestEventToJobs(payload, impactedProjects, nil, *config, false)
523523
if err != nil {
524524
log.Printf("Error converting event to jobsForImpactedProjects: %v", err)
525525
utils.InitCommentReporter(ghService, prNumber, fmt.Sprintf(":x: Error converting event to jobsForImpactedProjects: %v", err))

‎backend/services/spec.go‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,11 @@ import (
77
"github.com/diggerhq/digger/backend/utils"
88
"github.com/diggerhq/digger/libs/scheduler"
99
"github.com/diggerhq/digger/libs/spec"
10+
"github.com/samber/lo"
1011
"log"
1112
"os"
1213
"strconv"
14+
"strings"
1315
)
1416

1517
func GetVCSTokenFromJob(job models.DiggerJob, gh utils.GithubClientProvider) (*string, error) {
@@ -57,6 +59,30 @@ func GetRunNameFromJob(job models.DiggerJob) (*string, error) {
5759
return &runName, nil
5860
}
5961

62+
func getVariablesSpecFromEnvMap(envVars map[string]string) []spec.VariableSpec {
63+
variablesSpec := make([]spec.VariableSpec, 0)
64+
for k, v := range envVars {
65+
if strings.HasPrefix(v, "$DIGGER_") {
66+
val := strings.ReplaceAll(v, "$DIGGER_", "")
67+
variablesSpec = append(variablesSpec, spec.VariableSpec{
68+
Name: k,
69+
Value: val,
70+
IsSecret: false,
71+
IsInterpolated: true,
72+
})
73+
} else {
74+
variablesSpec = append(variablesSpec, spec.VariableSpec{
75+
Name: k,
76+
Value: v,
77+
IsSecret: false,
78+
IsInterpolated: false,
79+
})
80+
81+
}
82+
}
83+
return variablesSpec
84+
}
85+
6086
func GetSpecFromJob(job models.DiggerJob) (*spec.Spec, error) {
6187
var jobSpec scheduler.JobJson
6288
err := json.Unmarshal([]byte(job.SerializedJobSpec), &jobSpec)
@@ -65,6 +91,23 @@ func GetSpecFromJob(job models.DiggerJob) (*spec.Spec, error) {
6591
return nil, fmt.Errorf("could not marshal json string: %v", err)
6692
}
6793

94+
variablesSpec := make([]spec.VariableSpec, 0)
95+
stateVariables := getVariablesSpecFromEnvMap(jobSpec.StateEnvVars)
96+
commandVariables := getVariablesSpecFromEnvMap(jobSpec.CommandEnvVars)
97+
runVariables := getVariablesSpecFromEnvMap(jobSpec.RunEnvVars)
98+
variablesSpec = append(variablesSpec, stateVariables...)
99+
variablesSpec = append(variablesSpec, commandVariables...)
100+
variablesSpec = append(variablesSpec, runVariables...)
101+
102+
// check for duplicates in list of variablesSpec
103+
justNames := lo.Map(variablesSpec, func(item spec.VariableSpec, i int) string {
104+
return item.Name
105+
})
106+
hasDuplicates := len(justNames) != len(lo.Uniq(justNames))
107+
if hasDuplicates {
108+
return nil, fmt.Errorf("could not load variables due to duplicates: %v", err)
109+
}
110+
68111
batch := job.Batch
69112

70113
spec := spec.Spec{
@@ -93,6 +136,7 @@ func GetSpecFromJob(job models.DiggerJob) (*spec.Spec, error) {
93136
RepoName: batch.RepoName,
94137
WorkflowFile: job.WorkflowFile,
95138
},
139+
Variables: variablesSpec,
96140
Policy: spec.PolicySpec{
97141
PolicyType: "http",
98142
},

‎cli/cmd/digger/default.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ var defaultCmd = &cobra.Command{
5454
lib_spec.BackendApiProvider{},
5555
lib_spec.BasicPolicyProvider{},
5656
lib_spec.PlanStorageProvider{},
57+
lib_spec.VariablesProvider{},
5758
comment_updater.CommentUpdaterProviderBasic{},
5859
)
5960
}

‎cli/cmd/digger/main_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -900,7 +900,7 @@ func TestGitHubNewPullRequestContext(t *testing.T) {
900900
}
901901

902902
event := context.Event.(github.PullRequestEvent)
903-
jobs, _, err := dggithub.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, diggerConfig)
903+
jobs, _, err := dggithub.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, diggerConfig, false)
904904
_, _, err = digger.RunJobs(jobs, &prManager, prManager, lock, reporter, &planStorage, policyChecker, comment_updater.NoopCommentUpdater{}, backendApi, "123", false, false, "1", "dir")
905905

906906
assert.NoError(t, err)
@@ -995,7 +995,7 @@ func TestGitHubNewPullRequestInMultiEnvProjectContext(t *testing.T) {
995995
impactedProjects, requestedProject, prNumber, err := dggithub.ProcessGitHubEvent(ghEvent, &diggerConfig, prManager)
996996
assert.NoError(t, err)
997997
event := context.Event.(github.PullRequestEvent)
998-
jobs, _, err := dggithub.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, diggerConfig)
998+
jobs, _, err := dggithub.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, diggerConfig, false)
999999
spew.Dump(lock.MapLock)
10001000
assert.Equal(t, pullRequestNumber, prNumber)
10011001
assert.Equal(t, 1, len(jobs))

‎cli/cmd/digger/run_spec.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ var runSpecCmd = &cobra.Command{
4040
lib_spec.BackendApiProvider{},
4141
lib_spec.BasicPolicyProvider{},
4242
lib_spec.PlanStorageProvider{},
43+
lib_spec.VariablesProvider{},
4344
comment_summary.CommentUpdaterProviderBasic{},
4445
)
4546
if err != nil {

‎cli/pkg/github/github.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh
135135
}
136136
workflow := diggerConfig.Workflows[projectConfig.Workflow]
137137

138-
stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars)
138+
stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars, true)
139139

140140
planStorage, err := storage.NewPlanStorage(ghToken, repoOwner, repositoryName, nil)
141141
if err != nil {
@@ -170,7 +170,7 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh
170170
}
171171
workflow := diggerConfig.Workflows[projectConfig.Workflow]
172172

173-
stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars)
173+
stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars, true)
174174

175175
StateEnvProvider, CommandEnvProvider := scheduler.GetStateAndCommandProviders(projectConfig)
176176

@@ -234,7 +234,7 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh
234234
coversAllImpactedProjects := false
235235
err = nil
236236
if prEvent, ok := ghEvent.(github.PullRequestEvent); ok {
237-
jobs, coversAllImpactedProjects, err = dg_github.ConvertGithubPullRequestEventToJobs(&prEvent, impactedProjects, requestedProject, *diggerConfig)
237+
jobs, coversAllImpactedProjects, err = dg_github.ConvertGithubPullRequestEventToJobs(&prEvent, impactedProjects, requestedProject, *diggerConfig, true)
238238
} else if commentEvent, ok := ghEvent.(github.IssueCommentEvent); ok {
239239
prBranchName, _, err := githubPrService.GetBranchName(*commentEvent.Issue.Number)
240240

‎cli/pkg/integration/integration_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -396,7 +396,7 @@ func TestHappyPath(t *testing.T) {
396396
impactedProjects, requestedProject, prNumber, err := dg_github.ProcessGitHubEvent(ghEvent, diggerConfig, &githubPrService)
397397
assert.NoError(t, err)
398398
event := ghEvent.(github.PullRequestEvent)
399-
jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, *diggerConfig)
399+
jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, *diggerConfig, false)
400400
assert.NoError(t, err)
401401
zipManager := storage.Zipper{}
402402
planStorage := &storage.GithubPlanStorage{
@@ -439,7 +439,7 @@ func TestHappyPath(t *testing.T) {
439439
impactedProjects, requestedProject, prNumber, err = dg_github.ProcessGitHubEvent(ghEvent, diggerConfig, &githubPrService)
440440
assert.NoError(t, err)
441441
prEvent := ghEvent.(github.PullRequestEvent)
442-
jobs, _, err = dg_github.ConvertGithubPullRequestEventToJobs(&prEvent, impactedProjects, requestedProject, *diggerConfig)
442+
jobs, _, err = dg_github.ConvertGithubPullRequestEventToJobs(&prEvent, impactedProjects, requestedProject, *diggerConfig, false)
443443
assert.NoError(t, err)
444444
_, _, err = digger.RunJobs(jobs, &githubPrService, &githubPrService, lock, reporter, planStorage, nil, comment_updater.NoopCommentUpdater{}, nil, "", false, false, "123", dir)
445445
assert.NoError(t, err)
@@ -551,7 +551,7 @@ func TestMultiEnvHappyPath(t *testing.T) {
551551
impactedProjects, requestedProject, prNumber, err := dg_github.ProcessGitHubEvent(ghEvent, diggerConfig, &githubPrService)
552552
assert.NoError(t, err)
553553
pEvent := ghEvent.(github.PullRequestEvent)
554-
jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&pEvent, impactedProjects, requestedProject, *diggerConfig)
554+
jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&pEvent, impactedProjects, requestedProject, *diggerConfig, false)
555555
assert.NoError(t, err)
556556

557557
zipManager := storage.Zipper{}
@@ -769,7 +769,7 @@ workflows:
769769
impactedProjects, requestedProject, prNumber, err := dg_github.ProcessGitHubEvent(ghEvent, diggerConfig, &githubPrService)
770770
assert.NoError(t, err)
771771
pEvent := ghEvent.(github.PullRequestEvent)
772-
jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&pEvent, impactedProjects, requestedProject, *diggerConfig)
772+
jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&pEvent, impactedProjects, requestedProject, *diggerConfig, false)
773773
assert.NoError(t, err)
774774

775775
zipManager := storage.Zipper{}

‎cli/pkg/spec/spec.go‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ func RunSpec(
3333
backedProvider spec.BackendApiProvider,
3434
policyProvider spec.SpecPolicyProvider,
3535
PlanStorageProvider spec.PlanStorageProvider,
36+
variablesProvider spec.VariablesProvider,
3637
commentUpdaterProvider comment_summary.CommentUpdaterProvider,
3738
) error {
3839

@@ -78,14 +79,6 @@ func RunSpec(
7879
reportError(spec, backendApi, message, err)
7980
}
8081

81-
// TODO: avoid calling GetChangedFilesHere, avoid loading digger config entirely
82-
// also see below TODO to leverage variables provider and avoid passing it to commentUpdaterProvider
83-
diggerConfig, _, _, err := digger_config.LoadDiggerConfig("./", false, []string{})
84-
if err != nil {
85-
usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("Failed to read Digger digger_config. %s", err), 4)
86-
}
87-
log.Printf("Digger digger_config read successfully\n")
88-
8982
// TODO: render mode being passable from the string
9083
commentUpdater, err := commentUpdaterProvider.Get(digger_config.CommentRenderModeBasic)
9184
if err != nil {
@@ -100,10 +93,17 @@ func RunSpec(
10093
}
10194

10295
// TODO: make this part purely based on variables providers
103-
workflow := diggerConfig.Workflows[job.ProjectWorkflow]
104-
stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars)
105-
job.StateEnvVars = lo.Assign(job.StateEnvVars, stateEnvVars)
106-
job.CommandEnvVars = lo.Assign(job.CommandEnvVars, commandEnvVars)
96+
97+
// get variables from the variables spec
98+
variablesMap, err := variablesProvider.GetVariables(spec.Variables)
99+
if err != nil {
100+
log.Printf("could not get variables from provider: %v", err)
101+
reporterError(spec, backendApi, err)
102+
usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get variables from provider: %v", err), 1)
103+
}
104+
job.StateEnvVars = lo.Assign(job.StateEnvVars, variablesMap)
105+
job.CommandEnvVars = lo.Assign(job.CommandEnvVars, variablesMap)
106+
job.RunEnvVars = lo.Assign(job.RunEnvVars, variablesMap)
107107

108108
jobs := []scheduler.Job{job}
109109

‎ee/cli/cmd/digger/default.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ var defaultCmd = &cobra.Command{
5353
lib_spec.BackendApiProvider{},
5454
policy.AdvancedPolicyProvider{},
5555
lib_spec.PlanStorageProvider{},
56+
lib_spec.VariablesProvider{},
5657
comment_summary.CommentUpdaterProviderBasic{},
5758
)
5859
}

‎ee/cli/cmd/digger/main_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -900,7 +900,7 @@ func TestGitHubNewPullRequestContext(t *testing.T) {
900900
}
901901

902902
event := context.Event.(github.PullRequestEvent)
903-
jobs, _, err := dggithub.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, diggerConfig)
903+
jobs, _, err := dggithub.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, diggerConfig, false)
904904
if err != nil {
905905
assert.NoError(t, err)
906906
log.Println(err)
@@ -1001,7 +1001,7 @@ func TestGitHubNewPullRequestInMultiEnvProjectContext(t *testing.T) {
10011001
impactedProjects, requestedProject, prNumber, err := dggithub.ProcessGitHubEvent(ghEvent, &diggerConfig, &prManager)
10021002
assert.NoError(t, err)
10031003
event := context.Event.(github.PullRequestEvent)
1004-
jobs, _, err := dggithub.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, diggerConfig)
1004+
jobs, _, err := dggithub.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, diggerConfig, false)
10051005

10061006
assert.Equal(t, pullRequestNumber, prNumber)
10071007
assert.Equal(t, 1, len(jobs))

0 commit comments

Comments
 (0)