Add SupportBundle collection functionalities (#140)

This PR adds support bundle collection functionalities to Theia manager.

A new API endpoint apis/system.theia.antrea.io/v1alpha1/supportbundles is added to trigger
and retrieve log bundle collection. The following logs will be collected as part of the operation:
- Flow Aggregator logs (copied from /var/log/antrea/flow-aggregator/)
- Theia Manager logs (local log files)
- ClickHouse server logs (copied from /var/log/clickhouse-server/)
- Grafana logs (copied from /var/log/grafana/)
- Spark operator, driver and executor logs, if available during PR job (via streaming console logs)
- zookeeper (via streaming console logs)

Logging configuration of ClickHouse and Grafana are also added to enable better configuration
of log level and detention control.

To trigger log collection, or retrieve a log bundle job that finished earlier, run:
$ theia supportbundle

Theia CLI optional arg --since currently only supports filtering logs collected from Theia
manager and Flow aggregator. Support of other components will be added in a future change.

Signed-off-by: Shawn Wang <[email protected]>

Author: wsquan171

Diff for: build/charts/theia/README.md

@@ -28,6 +28,9 @@ Kubernetes: `>= 1.16.0-0`
 | clickhouse.cluster.zookeeperHosts | list | `[]` | To use a pre-installed ZooKeeper for ClickHouse data replication, please provide a list of your ZooKeeper hosts. To install a customized ZooKeeper, refer to <https://github.com/Altinity/clickhouse-operator/blob/master/docs/zookeeper_setup.md> |
 | clickhouse.connectionSecret | object | `{"password":"clickhouse_operator_password","username":"clickhouse_operator"}` | Credentials to connect to ClickHouse. They will be stored in a secret. |
 | clickhouse.image | object | `{"pullPolicy":"IfNotPresent","repository":"projects.registry.vmware.com/antrea/theia-clickhouse-server","tag":""}` | Container image used by ClickHouse. |
+| clickhouse.logger.count | int | `4` | The number of archived log files that ClickHouse stores. |
+| clickhouse.logger.level | string | `"information"` | Logging level. Acceptable values: trace, debug, information, warning, error. |
+| clickhouse.logger.size | string | `"100M"` | Size of log files. Applies to log and errorlog. Once the file reaches size, ClickHouse archives and renames it, and creates a new log file in its place. |
 | clickhouse.monitor.deletePercentage | float | `0.5` | The percentage of records in ClickHouse that will be deleted when the storage grows above threshold. Vary from 0 to 1. |
 | clickhouse.monitor.enable | bool | `true` | Determine whether to run a monitor to periodically check the ClickHouse memory usage and clean data. |
 | clickhouse.monitor.execInterval | string | `"1m"` | The time interval between two round of monitoring. Can be a plain integer using one of these unit suffixes ns, us (or µs), ms, s, m, h. |
@@ -50,6 +53,14 @@ Kubernetes: `>= 1.16.0-0`
 | grafana.homeDashboard | string | `"homepage.json"` | Default home dashboard. |
 | grafana.image | object | `{"pullPolicy":"IfNotPresent","repository":"projects.registry.vmware.com/antrea/theia-grafana","tag":"8.3.3"}` | Container image used by Grafana. |
 | grafana.installPlugins | list | `["https://downloads.antrea.io/artifacts/grafana-custom-plugins/theia-grafana-sankey-plugin-1.0.2.zip;theia-grafana-sankey-plugin","https://downloads.antrea.io/artifacts/grafana-custom-plugins/theia-grafana-chord-plugin-1.0.1.zip;theia-grafana-chord-plugin","grafana-clickhouse-datasource 1.0.1"]` | Grafana plugins to install. |
+| grafana.log | object | `{"daily_rotate":"true","level":"info","log_rotate":"true","max_days":"7","max_lines":"10.4.0-dev0","max_size_shift":"27","mode":"console file"}` | Grafana logging options. |
+| grafana.log.daily_rotate | string | `"true"` | Enable daily rotation of files, valid options are false or true. Default is true. Only applicable when “file” used in [log] mode. |
+| grafana.log.level | string | `"info"` | Logging level. Options are “debug”, “info”, “warn”, “error”, and “critical”. Default is info. |
+| grafana.log.log_rotate | string | `"true"` | Enable automated log rotation, valid options are false or true. Default is true. When enabled use the max_lines, max_size_shift, daily_rotate and max_days to configure the behavior of the log rotation. Only applicable when “file” used in [log] mode. |
+| grafana.log.max_days | string | `"7"` | Maximum number of days to keep log files. Default is "7". Only applicable when “file” used in [log] mode. |
+| grafana.log.max_lines | string | `"10.4.0-dev0"` | Maximum lines per file before rotating it. Default is "10.4.0-dev0". Only applicable when “file” used in [log] mode. |
+| grafana.log.max_size_shift | string | `"27"` | Maximum size of file before rotating it. Default is "27", which means 1 << 27, 128MB. Only applicable when “file” used in [log] mode. |
+| grafana.log.mode | string | `"console file"` | Logging mode. Options are “console”, “file”, and “syslog”. Default is “console” and “file”. Use spaces to separate multiple modes, e.g. console file |
 | grafana.loginSecret | object | `{"password":"admin","username":"admin"}` | Credentials to login to Grafana. They will be stored in a Secret. |
 | grafana.securityContext | object | `{"fsGroup":472,"supplementalGroups":[0]}` | Set securityContext. Use a specific uid, gid for grafana. |
 | grafana.service.tcpPort | int | `3000` | TCP port number for the Grafana service. |

Diff for: build/charts/theia/templates/clickhouse/clickhouseinstallation.yaml

@@ -42,6 +42,10 @@ spec:
       - host: {{ $host }}
       {{- end }}
       {{- end }}
+    settings:
+      logger/level: {{ .Values.clickhouse.logger.level }}
+      logger/size: {{ .Values.clickhouse.logger.size }}
+      logger/count: {{ .Values.clickhouse.logger.count }}
   defaults:
     templates:
       podTemplate: pod-template

Diff for: build/charts/theia/templates/grafana/deployment.yaml

@@ -61,6 +61,20 @@ spec:
                   key: admin-password
             - name: GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH
               value: /opt/grafana/dashboards/{{ .Values.grafana.homeDashboard }}
+            - name: GF_LOG_MODE
+              value: '{{ .Values.grafana.log.mode }}'
+            - name: GF_LOG_LEVEL
+              value: '{{ .Values.grafana.log.level }}'
+            - name: GF_LOG_FILE_LOG_ROTATE
+              value: '{{ .Values.grafana.log.log_rotate }}'
+            - name: GF_LOG_FILE_MAX_LINES
+              value: '{{ .Values.grafana.log.max_lines }}'
+            - name: GF_LOG_FILE_MAX_SIZE_SHIFT
+              value: '{{ .Values.grafana.log.max_size_shift }}'
+            - name: GF_LOG_FILE_DAILY_ROTATE
+              value: '{{ .Values.grafana.log.daily_rotate }}'
+            - name: GF_LOG_FILE_MAX_DAYS
+              value: '{{ .Values.grafana.log.max_days }}'
           ports:
             - containerPort: 3000
               name: http-grafana

Diff for: build/charts/theia/templates/theia-cli/clusterrole.yaml

@@ -21,4 +21,18 @@ rules:
       - clickhouse
     verbs:
       - get
+  - apiGroups:
+      - system.theia.antrea.io
+    resources:
+      - supportbundles
+    verbs:
+      - get
+      - create
+      - delete
+  - apiGroups:
+      - system.theia.antrea.io
+    resources:
+      - supportbundles/download
+    verbs:
+      - get
 {{- end }}

Diff for: build/charts/theia/templates/theia-manager/clusterrole.yaml

@@ -51,7 +51,13 @@ rules:
     verbs: ["update"]
   - apiGroups: [ "" ]
     resources: [ "pods" ]
-    verbs: ["list"]
+    verbs: ["get", "list"]
+  - apiGroups: [ "" ]
+    resources: [ "pods/exec" ]
+    verbs: ["get", "create"]
+  - apiGroups: [ "" ]
+    resources: [ "pods/log" ]
+    verbs: ["get"]
   - apiGroups: [ "" ]
     resources: [ "services", "secrets" ]
     verbs: ["get"]

Diff for: build/charts/theia/values.yaml

@@ -4,6 +4,14 @@ clickhouse:
     repository: "projects.registry.vmware.com/antrea/theia-clickhouse-server"
     pullPolicy: "IfNotPresent"
     tag: ""
+  logger:
+    # -- Logging level. Acceptable values: trace, debug, information, warning, error.
+    level: information
+    # --  Size of log files. Applies to log and errorlog. Once the file reaches size,
+    # ClickHouse archives and renames it, and creates a new log file in its place.
+    size: 100M
+    # -- The number of archived log files that ClickHouse stores.
+    count: 4
   monitor:
     # -- Determine whether to run a monitor to periodically check the ClickHouse
     # memory usage and clean data.
@@ -126,6 +134,29 @@ grafana:
   # -- Determine whether to install Grafana. It is used as a data visualization
   # and monitoring tool.  
   enable: true
+  # -- Grafana logging options.
+  log:
+    # -- Logging mode. Options are “console”, “file”, and “syslog”. Default is “console” and “file”.
+    # Use spaces to separate multiple modes, e.g. console file
+    mode: "console file"
+    # -- Logging level. Options are “debug”, “info”, “warn”, “error”, and “critical”. Default is info.
+    level: info
+    # -- Enable automated log rotation, valid options are false or true. Default is true.
+    # When enabled use the max_lines, max_size_shift, daily_rotate and max_days to configure
+    # the behavior of the log rotation. Only applicable when “file” used in [log] mode.
+    log_rotate: "true"
+    # -- Maximum lines per file before rotating it. Default is "1000000".
+    # Only applicable when “file” used in [log] mode.
+    max_lines: "1000000"
+    # -- Maximum size of file before rotating it. Default is "27", which means 1 << 27, 128MB.
+    # Only applicable when “file” used in [log] mode.
+    max_size_shift: "27"
+    # -- Enable daily rotation of files, valid options are false or true. Default is true.
+    # Only applicable when “file” used in [log] mode.
+    daily_rotate: "true"
+    # -- Maximum number of days to keep log files. Default is "7".
+    # Only applicable when “file” used in [log] mode.
+    max_days: "7"
   # -- Set securityContext.
   # Use a specific uid, gid for grafana.
   securityContext:

Diff for: build/yamls/flow-visibility.yml

@@ -6025,6 +6025,20 @@ spec:
               name: grafana-secret
         - name: GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH
           value: /opt/grafana/dashboards/homepage.json
+        - name: GF_LOG_MODE
+          value: console file
+        - name: GF_LOG_LEVEL
+          value: info
+        - name: GF_LOG_FILE_LOG_ROTATE
+          value: "true"
+        - name: GF_LOG_FILE_MAX_LINES
+          value: "1000000"
+        - name: GF_LOG_FILE_MAX_SIZE_SHIFT
+          value: "27"
+        - name: GF_LOG_FILE_DAILY_ROTATE
+          value: "true"
+        - name: GF_LOG_FILE_MAX_DAYS
+          value: "7"
         image: projects.registry.vmware.com/antrea/theia-grafana:8.3.3
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -6287,6 +6301,10 @@ spec:
         replicasCount: 1
         shardsCount: 1
       name: clickhouse
+    settings:
+      logger/count: 4
+      logger/level: information
+      logger/size: 100M
     users:
       clickhouse_operator/k8s_secret_password: flow-visibility/clickhouse-secret/password
       clickhouse_operator/networks/ip: ::/0

Diff for: ci/jenkins/test-vmc.sh

@@ -337,7 +337,7 @@ function deliver_antrea {
 
     control_plane_ip="$(kubectl get nodes -o wide --no-headers=true | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 ~ role {print $6}')"
 
-    ${GIT_CHECKOUT_DIR}/hack/generate-manifest.sh --ch-size 100Mi --ch-monitor-threshold 0.1 > ${GIT_CHECKOUT_DIR}/build/yamls/flow-visibility.yml
+    ${GIT_CHECKOUT_DIR}/hack/generate-manifest.sh --ch-size 100Mi --ch-monitor-threshold 0.1 --theia-manager > ${GIT_CHECKOUT_DIR}/build/yamls/flow-visibility.yml
     ${GIT_CHECKOUT_DIR}/hack/generate-manifest.sh --no-grafana --spark-operator --theia-manager > ${GIT_CHECKOUT_DIR}/build/yamls/flow-visibility-with-spark.yml
     ${GIT_CHECKOUT_DIR}/hack/generate-manifest.sh --no-grafana --theia-manager > ${GIT_CHECKOUT_DIR}/build/yamls/flow-visibility-ch-only.yml
 

Diff for: ci/kind/test-e2e-kind.sh

@@ -38,7 +38,7 @@ function print_usage {
 
 TESTBED_CMD=$(dirname $0)"/kind-setup.sh"
 YML_DIR=$(dirname $0)"/../../build/yamls"
-FLOW_VISIBILITY_CMD=$(dirname $0)"/../../hack/generate-manifest.sh --ch-size 100Mi --ch-monitor-threshold 0.1"
+FLOW_VISIBILITY_CMD=$(dirname $0)"/../../hack/generate-manifest.sh --ch-size 100Mi --ch-monitor-threshold 0.1 --theia-manager"
 FLOW_VISIBILITY_WITH_SPARK_CMD=$(dirname $0)"/../../hack/generate-manifest.sh --no-grafana --spark-operator --theia-manager"
 FLOW_VISIBILITY_CH_ONLY_CMD=$(dirname $0)"/../../hack/generate-manifest.sh --no-grafana --theia-manager"
 CH_OPERATOR_YML=$(dirname $0)"/../../build/charts/theia/crds/clickhouse-operator-install-bundle.yaml"

Diff for: cmd/theia-manager/theia-manager.go

@@ -47,6 +47,7 @@ const informerDefaultResync = 12 * time.Hour
 
 func createAPIServerConfig(
 	client kubernetes.Interface,
+	kubeConfig *rest.Config,
 	selfSignedCert bool,
 	bindPort int,
 	cipherSuites []uint16,
@@ -92,6 +93,7 @@ func createAPIServerConfig(
 	return apiserver.NewConfig(
 		serverConfig,
 		client,
+		kubeConfig,
 		caCertController,
 		nprq,
 		chq), nil
@@ -135,6 +137,7 @@ func run(o *Options) error {
 
 	apiServerConfig, err := createAPIServerConfig(
 		kubeClient,
+		kubeConfig,
 		*o.config.APIServer.SelfSignedCert,
 		o.config.APIServer.APIPort,
 		cipherSuites,

Diff for: go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/google/uuid v1.1.2
 	github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd
 	github.com/sirupsen/logrus v1.9.0
+	github.com/spf13/afero v1.9.2
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.7.1
@@ -21,12 +22,37 @@ require (
 	k8s.io/api v0.24.0
 	k8s.io/apimachinery v0.24.0
 	k8s.io/apiserver v0.24.0
+	k8s.io/cli-runtime v0.24.0
 	k8s.io/client-go v0.24.0
 	k8s.io/klog/v2 v2.60.1
 	k8s.io/kube-aggregator v0.24.0
+	k8s.io/kubectl v0.24.0
 	k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9
 )
 
+require (
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd // indirect
+	github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect
+	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
+	github.com/fatih/camelcase v1.0.0 // indirect
+	github.com/fvbommel/sortorder v1.0.1 // indirect
+	github.com/go-errors/errors v1.0.1 // indirect
+	github.com/google/btree v1.0.1 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
+	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
+	github.com/mitchellh/go-wordwrap v1.0.0 // indirect
+	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
+	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
+	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
+	github.com/russross/blackfriday v1.5.2 // indirect
+	github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect
+	go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
+	sigs.k8s.io/kustomize/api v0.11.4 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.13.6 // indirect
+)
+
 require (
 	antrea.io/libOpenflow v0.8.0 // indirect
 	antrea.io/ofnet v0.6.1 // indirect