Add XUnit Formatting Output to Scorecard (#5048)

Author: Ish Shah

changelog/fragments/xunit-xml-output.yaml

@@ -0,0 +1,16 @@
+# entries is a list of entries to include in
+# release notes and/or the migration guide
+entries:
+  - description: >
+      Provide XML formatting option for scorecard users. Additionally transforms scorecard result types to xunit testsuite/testcase layout.
+
+    # kind is one of:
+    # - addition
+    # - change
+    # - deprecation
+    # - removal
+    # - bugfix
+    kind: "addition"
+
+    # Is this a breaking change?
+    breaking: false

internal/cmd/operator-sdk/scorecard/cmd.go

@@ -17,10 +17,12 @@ package scorecard
 import (
 	"context"
 	"encoding/json"
+	"encoding/xml"
 	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/operator-framework/api/pkg/apis/scorecard/v1alpha3"
@@ -30,6 +32,7 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 
 	scorecardannotations "github.com/operator-framework/operator-sdk/internal/annotations/scorecard"
+	xunit "github.com/operator-framework/operator-sdk/internal/cmd/operator-sdk/scorecard/xunit"
 	"github.com/operator-framework/operator-sdk/internal/flags"
 	registryutil "github.com/operator-framework/operator-sdk/internal/registry"
 	"github.com/operator-framework/operator-sdk/internal/scorecard"
@@ -73,7 +76,7 @@ If the argument holds an image tag, it must be present remotely.`,
 	scorecardCmd.Flags().StringVarP(&c.config, "config", "c", "", "path to scorecard config file")
 	scorecardCmd.Flags().StringVarP(&c.namespace, "namespace", "n", "", "namespace to run the test images in")
 	scorecardCmd.Flags().StringVarP(&c.outputFormat, "output", "o", "text",
-		"Output format for results. Valid values: text, json")
+		"Output format for results. Valid values: text, json, xunit")
 	scorecardCmd.Flags().StringVarP(&c.serviceAccount, "service-account", "s", "default",
 		"Service account to use for tests")
 	scorecardCmd.Flags().BoolVarP(&c.list, "list", "L", false,
@@ -102,12 +105,49 @@ func (c *scorecardCmd) printOutput(output v1alpha3.TestList) error {
 			return fmt.Errorf("marshal json error: %v", err)
 		}
 		fmt.Printf("%s\n", string(bytes))
+	case "xunit":
+		xunitOutput := c.convertXunit(output)
+		bytes, err := xml.MarshalIndent(xunitOutput, "", "  ")
+		if err != nil {
+			return fmt.Errorf("marshal xml error: %v", err)
+		}
+		fmt.Printf("%s\n", string(bytes))
 	default:
 		return fmt.Errorf("invalid output format selected")
 	}
 	return nil
 }
 
+func (c *scorecardCmd) convertXunit(output v1alpha3.TestList) xunit.TestSuites {
+	var resultSuite xunit.TestSuites
+	resultSuite.Name = "scorecard"
+
+	jsonTestItems := output.Items
+	for _, item := range jsonTestItems {
+		tempResults := item.Status.Results
+		for _, res := range tempResults {
+			var tCase xunit.TestCase
+			var tSuite xunit.TestSuite
+			tSuite.Name = res.Name
+			tCase.Name = res.Name
+			if res.State == v1alpha3.ErrorState {
+				tCase.Errors = append(tCase.Errors, xunit.XUnitComplexError{Type: "Error", Message: strings.Join(res.Errors, ",")})
+				tSuite.Errors = strings.Join(res.Errors, ",")
+			} else if res.State == v1alpha3.FailState {
+				tCase.Failures = append(tCase.Failures, xunit.XUnitComplexFailure{Type: "Failure", Message: res.Log})
+				tSuite.Failures = res.Log
+			}
+			tSuite.TestCases = append(tSuite.TestCases, tCase)
+			tSuite.URL = item.Spec.Image
+			//TODO: Add TestStuite ID when API updates version
+			//tSuite.ID = item.Spec.UniqueID
+			resultSuite.TestSuite = append(resultSuite.TestSuite, tSuite)
+		}
+	}
+
+	return resultSuite
+}
+
 func (c *scorecardCmd) run() (err error) {
 	// Extract bundle image contents if bundle is inferred to be an image.
 	if _, err = os.Stat(c.bundle); err != nil && errors.Is(err, os.ErrNotExist) {

internal/cmd/operator-sdk/scorecard/xunit/xunit.go

@@ -0,0 +1,76 @@
+// Copyright 2020 The Operator-SDK Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package xunitapi
+
+// TestCase contain the core information from a test run, including its name and status
+type TestCase struct {
+	// Name is the name of the test
+	Name      string                `json:"name,omitempty"`
+	Time      string                `json:"time,omitempty"`
+	Classname string                `json:"classname,omitempty"`
+	Group     string                `json:"group,omitempty"`
+	Failures  []XUnitComplexFailure `json:"failure,omitempty"`
+	Errors    []XUnitComplexError   `json:"error,omitempty"`
+	Skipped   []XUnitComplexSkipped `json:"skipped,omitempty"`
+}
+
+// TestSuite contains for details about a test beyond the final status
+type TestSuite struct {
+	// Name is the name of the test
+	Name       string      `json:"name,omitempty"`
+	Tests      string      `json:"tests,omitempty"`
+	Failures   string      `json:"failures,omitempty"`
+	Errors     string      `json:"errors,omitempty"`
+	Group      string      `json:"group,omitempty"`
+	Skipped    string      `json:"skipped,omitempty"`
+	Timestamp  string      `json:"timestamp,omitempty"`
+	Hostname   string      `json:"hostnames,omitempty"`
+	ID         string      `json:"id,omitempty"`
+	Package    string      `json:"package,omitempty"`
+	File       string      `json:"file,omitempty"`
+	Log        string      `json:"log,omitempty"`
+	URL        string      `json:"url,omitempty"`
+	Version    string      `json:"version,omitempty"`
+	TestSuites []TestSuite `json:"testsuite,omitempty"`
+	TestCases  []TestCase  `json:"testcase,omitempty"`
+}
+
+// TestSuites is the top level object for amassing Xunit test results
+type TestSuites struct {
+	// Name is the name of the test
+	Name      string      `json:"name,omitempty"`
+	Tests     string      `json:"tests,omitempty"`
+	Failures  string      `json:"failures,omitempty"`
+	Errors    string      `json:"errors,omitempty"`
+	TestSuite []TestSuite `json:"testsuite,omitempty"`
+}
+
+// XUnitComplexError contains a type header along with the error messages
+type XUnitComplexError struct {
+	Type    string `json:"type,omitempty"`
+	Message string `json:"message,omitempty"`
+}
+
+// XUnitComplexFailure contains a type header along with the failure logs
+type XUnitComplexFailure struct {
+	Type    string `json:"type,omitempty"`
+	Message string `json:"message,omitempty"`
+}
+
+// XUnitComplexSkipped contains a type header along with associated run logs
+type XUnitComplexSkipped struct {
+	Type    string `json:"type,omitempty"`
+	Message string `json:"message,omitempty"`
+}

internal/scorecard/scorecard.go

@@ -132,6 +132,8 @@ func (o Scorecard) runTest(ctx context.Context, test v1alpha3.TestConfiguration)
 	}
 
 	out := v1alpha3.NewTest()
+	//TODO: Add timestamp to result when API version updates
+	//out.Tstamp = time.Now().Format(time.RFC850)
 	out.Spec = test
 	out.Status = *result
 	return out

website/content/en/docs/cli/operator-sdk_scorecard.md

@@ -23,7 +23,7 @@ operator-sdk scorecard [flags]
       --kubeconfig string        kubeconfig path
   -L, --list                     Option to enable listing which tests are run
   -n, --namespace string         namespace to run the test images in
-  -o, --output string            Output format for results. Valid values: text, json (default "text")
+  -o, --output string            Output format for results. Valid values: text, json, xunit (default "text")
   -l, --selector string          label selector to determine which tests are run
   -s, --service-account string   Service account to use for tests (default "default")
   -x, --skip-cleanup             Disable resource cleanup after tests are run