Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

3.5.14-atlan-1.2.0 #25

Open
wants to merge 3 commits into
base: 3.5.14-atlan
Choose a base branch
from
Open

3.5.14-atlan-1.2.0 #25

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f2624e1
fix: skip problematic validation
tczhao Oct 30, 2024
743f15c
fix: gcs fix list a file. Fixes #2841 (#14214)
tczhao Feb 23, 2025
07acf98
feat: version 3.5.14-atlan-1.2.0
tczhao Mar 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions pkg/apis/workflow/v1alpha1/validation_utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,10 @@ func validateWorkflowFieldNames(names []string, isParamOrArtifact bool) error {
if len(errs) != 0 {
return fmt.Errorf("[%d].name: '%s' is invalid: %s", i, name, strings.Join(errs, ";"))
}
_, ok := nameSet[name]
if ok {
return fmt.Errorf("[%d].name '%s' is not unique", i, name)
}
// _, ok := nameSet[name]
// if ok {
// return fmt.Errorf("[%d].name '%s' is not unique", i, name)
// }
nameSet[name] = true
}
return nil
Expand Down
2 changes: 1 addition & 1 deletion version.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ func GetVersion() wfv1.Version {
versionStr += "+unknown"
}
}
versionStr = "v3.5.14-atlan-1.1.4"
versionStr = "v3.5.14-atlan-1.2.0"
return wfv1.Version{
Version: versionStr,
BuildDate: buildDate,
Expand Down
5 changes: 5 additions & 0 deletions workflow/artifacts/gcs/gcs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,11 @@ func listByPrefix(client *storage.Client, bucket, prefix, delim string) ([]strin
if err != nil {
return nil, err
}
// prefix is a file
if attrs.Name == prefix {
results = []string{attrs.Name}
return results, nil
}
// skip "folder" path like objects
// note that we still download content (including "subfolders")
// this is just a consequence of how objects are stored in GCS (no real hierarchy)
Expand Down
88 changes: 48 additions & 40 deletions workflow/validate/validate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -478,14 +478,14 @@ func (ctx *templateValidationCtx) validateTemplate(tmpl *wfv1.Template, tmplCtx
}

}

templateScope := tmplCtx.GetTemplateScope()
tmplID := getTemplateID(tmpl)
_, ok := ctx.results[tmplID]
_, ok := ctx.results[templateScope+tmplID]
if ok {
// we can skip the rest since it has been validated.
return nil
}
ctx.results[tmplID] = true
ctx.results[templateScope+tmplID] = true

for globalVar, val := range ctx.globalParams {
scope[globalVar] = val
Expand All @@ -499,12 +499,13 @@ func (ctx *templateValidationCtx) validateTemplate(tmpl *wfv1.Template, tmplCtx
err = ctx.validateLeaf(scope, tmplCtx, newTmpl, workflowTemplateValidation)
}
if err != nil {
logrus.Error(err)
return err
}
err = validateOutputs(scope, ctx.globalParams, newTmpl, workflowTemplateValidation)
if err != nil {
return err
}
// err = validateOutputs(scope, ctx.globalParams, newTmpl, workflowTemplateValidation)
// if err != nil {
// return err
// }
if newTmpl.ArchiveLocation != nil {
errPrefix := fmt.Sprintf("templates.%s.archiveLocation", newTmpl.Name)
err = validateArtifactLocation(errPrefix, *newTmpl.ArchiveLocation)
Expand Down Expand Up @@ -560,6 +561,9 @@ func (ctx *templateValidationCtx) validateTemplateHolder(tmplHolder wfv1.Templat
logrus.Warnf("template reference need resolution: %v", err)
return nil, nil
}
if placeholderGenerator.IsPlaceholder(tmplRef.Template) {
return nil, nil
}
} else if tmplName != "" {
_, err := tmplCtx.GetTemplateByName(tmplName)
if err != nil {
Expand Down Expand Up @@ -640,13 +644,13 @@ func validateInputs(tmpl *wfv1.Template) (map[string]interface{}, error) {
return nil, errors.Errorf(errors.CodeBadRequest, "error in templates.%s.%s: %s", tmpl.Name, artRef, err.Error())
}
scope[fmt.Sprintf("inputs.artifacts.%s.path", art.Name)] = true
} else {
if art.Path != "" {
return nil, errors.Errorf(errors.CodeBadRequest, "templates.%s.%s.path only valid in container/script templates", tmpl.Name, artRef)
}
}
if art.From != "" {
return nil, errors.Errorf(errors.CodeBadRequest, "templates.%s.%s.from not valid in inputs", tmpl.Name, artRef)
// } else {
// if art.Path != "" {
// return nil, errors.Errorf(errors.CodeBadRequest, "templates.%s.%s.path only valid in container/script templates", tmpl.Name, artRef)
// }
// }
// if art.From != "" {
// return nil, errors.Errorf(errors.CodeBadRequest, "templates.%s.%s.from not valid in inputs", tmpl.Name, artRef)
}
errPrefix := fmt.Sprintf("templates.%s.%s", tmpl.Name, artRef)
err = validateArtifactLocation(errPrefix, art.ArtifactLocation)
Expand Down Expand Up @@ -691,18 +695,23 @@ func resolveAllVariables(scope map[string]interface{}, globalParams map[string]s
if (trimmedTag == "item" || strings.HasPrefix(trimmedTag, "item.")) && allowAllItemRefs {
// we are *probably* referencing a undetermined item using withParam
// NOTE: this is far from foolproof.
// Allow runtime resolution of workflow output parameter names
} else if strings.HasPrefix(trimmedTag, "workflow.outputs.parameters.") && allowAllWorkflowOutputParameterRefs {
// Allow runtime resolution of workflow output parameter names
} else if strings.HasPrefix(trimmedTag, "workflow.outputs.artifacts.") && allowAllWorkflowOutputArtifactRefs {
// Allow runtime resolution of workflow output artifact names
} else if strings.HasPrefix(trimmedTag, "outputs.") {
} else if strings.HasPrefix(trimmedTag, "inputs.") {
// We are self referencing for metric emission, allow it.
} else if strings.HasPrefix(trimmedTag, common.GlobalVarWorkflowCreationTimestamp) {
} else if strings.HasPrefix(trimmedTag, common.GlobalVarWorkflowCronScheduleTime) {
// Allow runtime resolution for "scheduledTime" which will pass from CronWorkflow
} else if strings.HasPrefix(trimmedTag, common.GlobalVarWorkflowDuration) {
} else if strings.HasPrefix(trimmedTag, "tasks.name") {
} else if strings.HasPrefix(trimmedTag, "steps.name") {
} else if strings.HasPrefix(trimmedTag, "steps.standardize") {
} else if strings.HasPrefix(trimmedTag, "workflow.labels") {
} else if strings.HasPrefix(trimmedTag, "pod.name") {
} else if strings.HasPrefix(trimmedTag, "node.name") {
} else if strings.HasPrefix(trimmedTag, "workflow.parameters") && workflowTemplateValidation {
// If we are simply validating a WorkflowTemplate in isolation, some of the parameters may come from the Workflow that uses it
Expand Down Expand Up @@ -909,16 +918,16 @@ func validateArgumentsValues(prefix string, arguments wfv1.Arguments, allowEmpty
}
return errors.Errorf(errors.CodeBadRequest, "%s%s.value is required", prefix, param.Name)
}
valueSpecifiedInEnumList := false
for _, enum := range param.Enum {
if enum == *param.Value {
valueSpecifiedInEnumList = true
break
}
}
if !valueSpecifiedInEnumList {
return errors.Errorf(errors.CodeBadRequest, "%s%s.value should be present in %s%s.enum list", prefix, param.Name, prefix, param.Name)
}
// valueSpecifiedInEnumList := false
// for _, enum := range param.Enum {
// if enum == *param.Value {
// valueSpecifiedInEnumList = true
// break
// }
// }
// if !valueSpecifiedInEnumList {
// return errors.Errorf(errors.CodeBadRequest, "%s%s.value should be present in %s%s.enum list", prefix, param.Name, prefix, param.Name)
// }
}
}
for _, art := range arguments.Artifacts {
Expand All @@ -944,10 +953,10 @@ func (ctx *templateValidationCtx) validateSteps(scope map[string]interface{}, tm
if step.Name == "" {
return errors.Errorf(errors.CodeBadRequest, "templates.%s.steps[%d].name is required", tmpl.Name, i)
}
_, ok := stepNames[step.Name]
if ok {
return errors.Errorf(errors.CodeBadRequest, "templates.%s.steps[%d].name '%s' is not unique", tmpl.Name, i, step.Name)
}
// _, ok := stepNames[step.Name]
// if ok {
// return errors.Errorf(errors.CodeBadRequest, "templates.%s.steps[%d].name '%s' is not unique", tmpl.Name, i, step.Name)
// }
if errs := isValidWorkflowFieldName(step.Name); len(errs) != 0 {
return errors.Errorf(errors.CodeBadRequest, "templates.%s.steps[%d].name '%s' is invalid: %s", tmpl.Name, i, step.Name, strings.Join(errs, ";"))
}
Expand Down Expand Up @@ -1263,10 +1272,10 @@ func validateWorkflowFieldNames(slice interface{}) error {
if len(errs) != 0 {
return errors.Errorf(errors.CodeBadRequest, "[%d].name: '%s' is invalid: %s", i, name, strings.Join(errs, ";"))
}
_, ok := names[name]
if ok {
return errors.Errorf(errors.CodeBadRequest, "[%d].name '%s' is not unique", i, name)
}
// _, ok := names[name]
// if ok {
// return errors.Errorf(errors.CodeBadRequest, "[%d].name '%s' is not unique", i, name)
// }
names[name] = true
}
return nil
Expand Down Expand Up @@ -1342,14 +1351,13 @@ func (ctx *templateValidationCtx) validateDAG(scope map[string]interface{}, tmpl

// Verify dependencies for all tasks can be resolved as well as template names
for _, task := range tmpl.DAG.Tasks {

if (usingDepends || len(task.Dependencies) > 0) && '0' <= task.Name[0] && task.Name[0] <= '9' {
return errors.Errorf(errors.CodeBadRequest, "templates.%s.tasks.%s name cannot begin with a digit when using either 'depends' or 'dependencies'", tmpl.Name, task.Name)
}

if usingDepends && len(task.Dependencies) > 0 {
return errors.Errorf(errors.CodeBadRequest, "templates.%s cannot use both 'depends' and 'dependencies' in the same DAG template", tmpl.Name)
}
// if usingDepends && len(task.Dependencies) > 0 {
// return errors.Errorf(errors.CodeBadRequest, "templates.%s cannot use both 'depends' and 'dependencies' in the same DAG template", tmpl.Name)
// }

if usingDepends && task.ContinueOn != nil {
return errors.Errorf(errors.CodeBadRequest, "templates.%s cannot use 'continueOn' when using 'depends'. Instead use 'dep-task.Failed'/'dep-task.Errored'", tmpl.Name)
Expand Down Expand Up @@ -1443,10 +1451,10 @@ func (ctx *templateValidationCtx) validateDAG(scope map[string]interface{}, tmpl
if err != nil {
return err
}
err = validateDAGTaskArgumentDependency(task.Arguments, ancestry)
if err != nil {
return errors.Errorf(errors.CodeBadRequest, "templates.%s.tasks.%s %s", tmpl.Name, task.Name, err.Error())
}
// err = validateDAGTaskArgumentDependency(task.Arguments, ancestry)
// if err != nil {
// return errors.Errorf(errors.CodeBadRequest, "templates.%s.tasks.%s %s", tmpl.Name, task.Name, err.Error())
// }
// Validate the template again with actual arguments.
_, err = ctx.validateTemplateHolder(&task, tmplCtx, &task.Arguments, workflowTemplateValidation)
if err != nil {
Expand Down
Loading