Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

3.5.14-atlan-1.2.0 #25

Open
wants to merge 3 commits into
base: 3.5.14-atlan
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions pkg/apis/workflow/v1alpha1/validation_utils.go
Original file line number Diff line number Diff line change
@@ -62,10 +62,10 @@ func validateWorkflowFieldNames(names []string, isParamOrArtifact bool) error {
if len(errs) != 0 {
return fmt.Errorf("[%d].name: '%s' is invalid: %s", i, name, strings.Join(errs, ";"))
}
_, ok := nameSet[name]
if ok {
return fmt.Errorf("[%d].name '%s' is not unique", i, name)
}
// _, ok := nameSet[name]
// if ok {
// return fmt.Errorf("[%d].name '%s' is not unique", i, name)
// }
nameSet[name] = true
}
return nil
2 changes: 1 addition & 1 deletion version.go
Original file line number Diff line number Diff line change
@@ -46,7 +46,7 @@ func GetVersion() wfv1.Version {
versionStr += "+unknown"
}
}
versionStr = "v3.5.14-atlan-1.1.4"
versionStr = "v3.5.14-atlan-1.2.0"
return wfv1.Version{
Version: versionStr,
BuildDate: buildDate,
5 changes: 5 additions & 0 deletions workflow/artifacts/gcs/gcs.go
Original file line number Diff line number Diff line change
@@ -196,6 +196,11 @@ func listByPrefix(client *storage.Client, bucket, prefix, delim string) ([]strin
if err != nil {
return nil, err
}
// prefix is a file
if attrs.Name == prefix {
results = []string{attrs.Name}
return results, nil
}
// skip "folder" path like objects
// note that we still download content (including "subfolders")
// this is just a consequence of how objects are stored in GCS (no real hierarchy)
88 changes: 48 additions & 40 deletions workflow/validate/validate.go
Original file line number Diff line number Diff line change
@@ -478,14 +478,14 @@ func (ctx *templateValidationCtx) validateTemplate(tmpl *wfv1.Template, tmplCtx
}

}

templateScope := tmplCtx.GetTemplateScope()
tmplID := getTemplateID(tmpl)
_, ok := ctx.results[tmplID]
_, ok := ctx.results[templateScope+tmplID]
if ok {
// we can skip the rest since it has been validated.
return nil
}
ctx.results[tmplID] = true
ctx.results[templateScope+tmplID] = true

for globalVar, val := range ctx.globalParams {
scope[globalVar] = val
@@ -499,12 +499,13 @@ func (ctx *templateValidationCtx) validateTemplate(tmpl *wfv1.Template, tmplCtx
err = ctx.validateLeaf(scope, tmplCtx, newTmpl, workflowTemplateValidation)
}
if err != nil {
logrus.Error(err)
return err
}
err = validateOutputs(scope, ctx.globalParams, newTmpl, workflowTemplateValidation)
if err != nil {
return err
}
// err = validateOutputs(scope, ctx.globalParams, newTmpl, workflowTemplateValidation)
// if err != nil {
// return err
// }
if newTmpl.ArchiveLocation != nil {
errPrefix := fmt.Sprintf("templates.%s.archiveLocation", newTmpl.Name)
err = validateArtifactLocation(errPrefix, *newTmpl.ArchiveLocation)
@@ -560,6 +561,9 @@ func (ctx *templateValidationCtx) validateTemplateHolder(tmplHolder wfv1.Templat
logrus.Warnf("template reference need resolution: %v", err)
return nil, nil
}
if placeholderGenerator.IsPlaceholder(tmplRef.Template) {
return nil, nil
}
} else if tmplName != "" {
_, err := tmplCtx.GetTemplateByName(tmplName)
if err != nil {
@@ -640,13 +644,13 @@ func validateInputs(tmpl *wfv1.Template) (map[string]interface{}, error) {
return nil, errors.Errorf(errors.CodeBadRequest, "error in templates.%s.%s: %s", tmpl.Name, artRef, err.Error())
}
scope[fmt.Sprintf("inputs.artifacts.%s.path", art.Name)] = true
} else {
if art.Path != "" {
return nil, errors.Errorf(errors.CodeBadRequest, "templates.%s.%s.path only valid in container/script templates", tmpl.Name, artRef)
}
}
if art.From != "" {
return nil, errors.Errorf(errors.CodeBadRequest, "templates.%s.%s.from not valid in inputs", tmpl.Name, artRef)
// } else {
// if art.Path != "" {
// return nil, errors.Errorf(errors.CodeBadRequest, "templates.%s.%s.path only valid in container/script templates", tmpl.Name, artRef)
// }
// }
// if art.From != "" {
// return nil, errors.Errorf(errors.CodeBadRequest, "templates.%s.%s.from not valid in inputs", tmpl.Name, artRef)
}
errPrefix := fmt.Sprintf("templates.%s.%s", tmpl.Name, artRef)
err = validateArtifactLocation(errPrefix, art.ArtifactLocation)
@@ -691,18 +695,23 @@ func resolveAllVariables(scope map[string]interface{}, globalParams map[string]s
if (trimmedTag == "item" || strings.HasPrefix(trimmedTag, "item.")) && allowAllItemRefs {
// we are *probably* referencing a undetermined item using withParam
// NOTE: this is far from foolproof.
// Allow runtime resolution of workflow output parameter names
} else if strings.HasPrefix(trimmedTag, "workflow.outputs.parameters.") && allowAllWorkflowOutputParameterRefs {
// Allow runtime resolution of workflow output parameter names
} else if strings.HasPrefix(trimmedTag, "workflow.outputs.artifacts.") && allowAllWorkflowOutputArtifactRefs {
// Allow runtime resolution of workflow output artifact names
} else if strings.HasPrefix(trimmedTag, "outputs.") {
} else if strings.HasPrefix(trimmedTag, "inputs.") {
// We are self referencing for metric emission, allow it.
} else if strings.HasPrefix(trimmedTag, common.GlobalVarWorkflowCreationTimestamp) {
} else if strings.HasPrefix(trimmedTag, common.GlobalVarWorkflowCronScheduleTime) {
// Allow runtime resolution for "scheduledTime" which will pass from CronWorkflow
} else if strings.HasPrefix(trimmedTag, common.GlobalVarWorkflowDuration) {
} else if strings.HasPrefix(trimmedTag, "tasks.name") {
} else if strings.HasPrefix(trimmedTag, "steps.name") {
} else if strings.HasPrefix(trimmedTag, "steps.standardize") {
} else if strings.HasPrefix(trimmedTag, "workflow.labels") {
} else if strings.HasPrefix(trimmedTag, "pod.name") {
} else if strings.HasPrefix(trimmedTag, "node.name") {
} else if strings.HasPrefix(trimmedTag, "workflow.parameters") && workflowTemplateValidation {
// If we are simply validating a WorkflowTemplate in isolation, some of the parameters may come from the Workflow that uses it
@@ -909,16 +918,16 @@ func validateArgumentsValues(prefix string, arguments wfv1.Arguments, allowEmpty
}
return errors.Errorf(errors.CodeBadRequest, "%s%s.value is required", prefix, param.Name)
}
valueSpecifiedInEnumList := false
for _, enum := range param.Enum {
if enum == *param.Value {
valueSpecifiedInEnumList = true
break
}
}
if !valueSpecifiedInEnumList {
return errors.Errorf(errors.CodeBadRequest, "%s%s.value should be present in %s%s.enum list", prefix, param.Name, prefix, param.Name)
}
// valueSpecifiedInEnumList := false
// for _, enum := range param.Enum {
// if enum == *param.Value {
// valueSpecifiedInEnumList = true
// break
// }
// }
// if !valueSpecifiedInEnumList {
// return errors.Errorf(errors.CodeBadRequest, "%s%s.value should be present in %s%s.enum list", prefix, param.Name, prefix, param.Name)
// }
}
}
for _, art := range arguments.Artifacts {
@@ -944,10 +953,10 @@ func (ctx *templateValidationCtx) validateSteps(scope map[string]interface{}, tm
if step.Name == "" {
return errors.Errorf(errors.CodeBadRequest, "templates.%s.steps[%d].name is required", tmpl.Name, i)
}
_, ok := stepNames[step.Name]
if ok {
return errors.Errorf(errors.CodeBadRequest, "templates.%s.steps[%d].name '%s' is not unique", tmpl.Name, i, step.Name)
}
// _, ok := stepNames[step.Name]
// if ok {
// return errors.Errorf(errors.CodeBadRequest, "templates.%s.steps[%d].name '%s' is not unique", tmpl.Name, i, step.Name)
// }
if errs := isValidWorkflowFieldName(step.Name); len(errs) != 0 {
return errors.Errorf(errors.CodeBadRequest, "templates.%s.steps[%d].name '%s' is invalid: %s", tmpl.Name, i, step.Name, strings.Join(errs, ";"))
}
@@ -1263,10 +1272,10 @@ func validateWorkflowFieldNames(slice interface{}) error {
if len(errs) != 0 {
return errors.Errorf(errors.CodeBadRequest, "[%d].name: '%s' is invalid: %s", i, name, strings.Join(errs, ";"))
}
_, ok := names[name]
if ok {
return errors.Errorf(errors.CodeBadRequest, "[%d].name '%s' is not unique", i, name)
}
// _, ok := names[name]
// if ok {
// return errors.Errorf(errors.CodeBadRequest, "[%d].name '%s' is not unique", i, name)
// }
names[name] = true
}
return nil
@@ -1342,14 +1351,13 @@ func (ctx *templateValidationCtx) validateDAG(scope map[string]interface{}, tmpl

// Verify dependencies for all tasks can be resolved as well as template names
for _, task := range tmpl.DAG.Tasks {

if (usingDepends || len(task.Dependencies) > 0) && '0' <= task.Name[0] && task.Name[0] <= '9' {
return errors.Errorf(errors.CodeBadRequest, "templates.%s.tasks.%s name cannot begin with a digit when using either 'depends' or 'dependencies'", tmpl.Name, task.Name)
}

if usingDepends && len(task.Dependencies) > 0 {
return errors.Errorf(errors.CodeBadRequest, "templates.%s cannot use both 'depends' and 'dependencies' in the same DAG template", tmpl.Name)
}
// if usingDepends && len(task.Dependencies) > 0 {
// return errors.Errorf(errors.CodeBadRequest, "templates.%s cannot use both 'depends' and 'dependencies' in the same DAG template", tmpl.Name)
// }

if usingDepends && task.ContinueOn != nil {
return errors.Errorf(errors.CodeBadRequest, "templates.%s cannot use 'continueOn' when using 'depends'. Instead use 'dep-task.Failed'/'dep-task.Errored'", tmpl.Name)
@@ -1443,10 +1451,10 @@ func (ctx *templateValidationCtx) validateDAG(scope map[string]interface{}, tmpl
if err != nil {
return err
}
err = validateDAGTaskArgumentDependency(task.Arguments, ancestry)
if err != nil {
return errors.Errorf(errors.CodeBadRequest, "templates.%s.tasks.%s %s", tmpl.Name, task.Name, err.Error())
}
// err = validateDAGTaskArgumentDependency(task.Arguments, ancestry)
// if err != nil {
// return errors.Errorf(errors.CodeBadRequest, "templates.%s.tasks.%s %s", tmpl.Name, task.Name, err.Error())
// }
// Validate the template again with actual arguments.
_, err = ctx.validateTemplateHolder(&task, tmplCtx, &task.Arguments, workflowTemplateValidation)
if err != nil {