diff --git a/changelog/fragments/xunit-xml-output.yaml b/changelog/fragments/xunit-xml-output.yaml new file mode 100644 index 00000000000..6e29c393c2c --- /dev/null +++ b/changelog/fragments/xunit-xml-output.yaml @@ -0,0 +1,16 @@ +# entries is a list of entries to include in +# release notes and/or the migration guide +entries: + - description: > + Provide XML formatting option for scorecard users. Additionally transforms scorecard result types to xunit testsuite/testcase layout. + + # kind is one of: + # - addition + # - change + # - deprecation + # - removal + # - bugfix + kind: "addition" + + # Is this a breaking change? + breaking: false diff --git a/internal/cmd/operator-sdk/scorecard/cmd.go b/internal/cmd/operator-sdk/scorecard/cmd.go index cccb55f5dcf..541c2b20f36 100644 --- a/internal/cmd/operator-sdk/scorecard/cmd.go +++ b/internal/cmd/operator-sdk/scorecard/cmd.go @@ -17,10 +17,12 @@ package scorecard import ( "context" "encoding/json" + "encoding/xml" "errors" "fmt" "os" "path/filepath" + "strings" "time" "github.com/operator-framework/api/pkg/apis/scorecard/v1alpha3" @@ -30,6 +32,7 @@ import ( "k8s.io/apimachinery/pkg/labels" scorecardannotations "github.com/operator-framework/operator-sdk/internal/annotations/scorecard" + xunit "github.com/operator-framework/operator-sdk/internal/cmd/operator-sdk/scorecard/xunit" "github.com/operator-framework/operator-sdk/internal/flags" registryutil "github.com/operator-framework/operator-sdk/internal/registry" "github.com/operator-framework/operator-sdk/internal/scorecard" @@ -73,7 +76,7 @@ If the argument holds an image tag, it must be present remotely.`, scorecardCmd.Flags().StringVarP(&c.config, "config", "c", "", "path to scorecard config file") scorecardCmd.Flags().StringVarP(&c.namespace, "namespace", "n", "", "namespace to run the test images in") scorecardCmd.Flags().StringVarP(&c.outputFormat, "output", "o", "text", - "Output format for results. Valid values: text, json") + "Output format for results. Valid values: text, json, xunit") scorecardCmd.Flags().StringVarP(&c.serviceAccount, "service-account", "s", "default", "Service account to use for tests") scorecardCmd.Flags().BoolVarP(&c.list, "list", "L", false, @@ -102,12 +105,49 @@ func (c *scorecardCmd) printOutput(output v1alpha3.TestList) error { return fmt.Errorf("marshal json error: %v", err) } fmt.Printf("%s\n", string(bytes)) + case "xunit": + xunitOutput := c.convertXunit(output) + bytes, err := xml.MarshalIndent(xunitOutput, "", " ") + if err != nil { + return fmt.Errorf("marshal xml error: %v", err) + } + fmt.Printf("%s\n", string(bytes)) default: return fmt.Errorf("invalid output format selected") } return nil } +func (c *scorecardCmd) convertXunit(output v1alpha3.TestList) xunit.TestSuites { + var resultSuite xunit.TestSuites + resultSuite.Name = "scorecard" + + jsonTestItems := output.Items + for _, item := range jsonTestItems { + tempResults := item.Status.Results + for _, res := range tempResults { + var tCase xunit.TestCase + var tSuite xunit.TestSuite + tSuite.Name = res.Name + tCase.Name = res.Name + if res.State == v1alpha3.ErrorState { + tCase.Errors = append(tCase.Errors, xunit.XUnitComplexError{Type: "Error", Message: strings.Join(res.Errors, ",")}) + tSuite.Errors = strings.Join(res.Errors, ",") + } else if res.State == v1alpha3.FailState { + tCase.Failures = append(tCase.Failures, xunit.XUnitComplexFailure{Type: "Failure", Message: res.Log}) + tSuite.Failures = res.Log + } + tSuite.TestCases = append(tSuite.TestCases, tCase) + tSuite.URL = item.Spec.Image + //TODO: Add TestStuite ID when API updates version + //tSuite.ID = item.Spec.UniqueID + resultSuite.TestSuite = append(resultSuite.TestSuite, tSuite) + } + } + + return resultSuite +} + func (c *scorecardCmd) run() (err error) { // Extract bundle image contents if bundle is inferred to be an image. if _, err = os.Stat(c.bundle); err != nil && errors.Is(err, os.ErrNotExist) { diff --git a/internal/cmd/operator-sdk/scorecard/xunit/xunit.go b/internal/cmd/operator-sdk/scorecard/xunit/xunit.go new file mode 100644 index 00000000000..a06150c0841 --- /dev/null +++ b/internal/cmd/operator-sdk/scorecard/xunit/xunit.go @@ -0,0 +1,76 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package xunitapi + +// TestCase contain the core information from a test run, including its name and status +type TestCase struct { + // Name is the name of the test + Name string `json:"name,omitempty"` + Time string `json:"time,omitempty"` + Classname string `json:"classname,omitempty"` + Group string `json:"group,omitempty"` + Failures []XUnitComplexFailure `json:"failure,omitempty"` + Errors []XUnitComplexError `json:"error,omitempty"` + Skipped []XUnitComplexSkipped `json:"skipped,omitempty"` +} + +// TestSuite contains for details about a test beyond the final status +type TestSuite struct { + // Name is the name of the test + Name string `json:"name,omitempty"` + Tests string `json:"tests,omitempty"` + Failures string `json:"failures,omitempty"` + Errors string `json:"errors,omitempty"` + Group string `json:"group,omitempty"` + Skipped string `json:"skipped,omitempty"` + Timestamp string `json:"timestamp,omitempty"` + Hostname string `json:"hostnames,omitempty"` + ID string `json:"id,omitempty"` + Package string `json:"package,omitempty"` + File string `json:"file,omitempty"` + Log string `json:"log,omitempty"` + URL string `json:"url,omitempty"` + Version string `json:"version,omitempty"` + TestSuites []TestSuite `json:"testsuite,omitempty"` + TestCases []TestCase `json:"testcase,omitempty"` +} + +// TestSuites is the top level object for amassing Xunit test results +type TestSuites struct { + // Name is the name of the test + Name string `json:"name,omitempty"` + Tests string `json:"tests,omitempty"` + Failures string `json:"failures,omitempty"` + Errors string `json:"errors,omitempty"` + TestSuite []TestSuite `json:"testsuite,omitempty"` +} + +// XUnitComplexError contains a type header along with the error messages +type XUnitComplexError struct { + Type string `json:"type,omitempty"` + Message string `json:"message,omitempty"` +} + +// XUnitComplexFailure contains a type header along with the failure logs +type XUnitComplexFailure struct { + Type string `json:"type,omitempty"` + Message string `json:"message,omitempty"` +} + +// XUnitComplexSkipped contains a type header along with associated run logs +type XUnitComplexSkipped struct { + Type string `json:"type,omitempty"` + Message string `json:"message,omitempty"` +} diff --git a/internal/scorecard/scorecard.go b/internal/scorecard/scorecard.go index fdfa33640ea..3379501f89c 100644 --- a/internal/scorecard/scorecard.go +++ b/internal/scorecard/scorecard.go @@ -132,6 +132,8 @@ func (o Scorecard) runTest(ctx context.Context, test v1alpha3.TestConfiguration) } out := v1alpha3.NewTest() + //TODO: Add timestamp to result when API version updates + //out.Tstamp = time.Now().Format(time.RFC850) out.Spec = test out.Status = *result return out diff --git a/website/content/en/docs/cli/operator-sdk_scorecard.md b/website/content/en/docs/cli/operator-sdk_scorecard.md index e81739253c9..c0ed3efc8ab 100644 --- a/website/content/en/docs/cli/operator-sdk_scorecard.md +++ b/website/content/en/docs/cli/operator-sdk_scorecard.md @@ -23,7 +23,7 @@ operator-sdk scorecard [flags] --kubeconfig string kubeconfig path -L, --list Option to enable listing which tests are run -n, --namespace string namespace to run the test images in - -o, --output string Output format for results. Valid values: text, json (default "text") + -o, --output string Output format for results. Valid values: text, json, xunit (default "text") -l, --selector string label selector to determine which tests are run -s, --service-account string Service account to use for tests (default "default") -x, --skip-cleanup Disable resource cleanup after tests are run