CT Log end-to-end consistency test

.github/workflows/main.yml

@@ -67,6 +67,23 @@ jobs:
   e2e-tests:
     name: Run end-to-end tests
     runs-on: ubuntu-latest
+    services:
+      mysql:
+        # sudo systemctl enable mysql.service
+        # sudo systemctl start mysql
+        image: mysql
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: true
+          DATABASE_HOST: 127.0.0.1
+          # MYSQL_ROOT_PASSWORD: zaphod
+          MYSQL_ROOT_USER: root
+          MYSQL_DATABASE: test
+          # MYSQL_USER: root
+          # MYSQL_PASSWORD: zaphod
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+
 
     steps: 
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -88,6 +105,11 @@ jobs:
         with:
           go-version-file: './go.mod'
           check-latest: true
-      - name: run Rekor end-to-end test
-        run: ./pkg/test/rekor_e2e/rekor_monitor_e2e_test.sh
+      # - name: run Rekor end-to-end test
+      #   run: ./pkg/test/rekor_e2e/rekor_monitor_e2e_test.sh
+      # - name: Create mariadb
+      #   run: | 
+      #     sudo /etc/init.d/mysql start
+      - name: run CT end-to-end test
+        run: ./pkg/test/ct_e2e/ct_monitor_e2e_test_manual.sh
 

pkg/test/ct_e2e/ct_monitor_e2e_test.go

@@ -0,0 +1,70 @@
+// Copyright 2024 The Sigstore Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build ct_e2e
+// +build ct_e2e
+
+package ct_e2e
+
+import (
+	"os"
+	"testing"
+
+	"net/http"
+
+	ctclient "github.com/google/certificate-transparency-go/client"
+	"github.com/google/certificate-transparency-go/jsonclient"
+	"github.com/sigstore/rekor-monitor/pkg/ct"
+	"github.com/sigstore/rekor-monitor/pkg/identity"
+)
+
+const (
+	rekorURL          = "http://127.0.0.1:3000"
+	subject           = "[email protected]"
+	issuer            = "[email protected]"
+	extValueString    = "test cert value"
+	SubmissionCertB64 = "MIIEijCCA3KgAwIBAgICEk0wDQYJKoZIhvcNAQELBQAwKzEpMCcGA1UEAwwgY2Fja2xpbmcgY3J5cHRvZ3JhcGhlciBmYWtlIFJPT1QwHhcNMTUxMDIxMjAxMTUyWhcNMjAxMDE5MjAxMTUyWjAfMR0wGwYDVQQDExRoYXBweSBoYWNrZXIgZmFrZSBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMIKR3maBcUSsncXYzQT13D5Nr+Z3mLxMMh3TUdt6sACmqbJ0btRlgXfMtNLM2OU1I6a3Ju+tIZSdn2v21JBwvxUzpZQ4zy2cimIiMQDZCQHJwzC9GZn8HaW091iz9H0Go3A7WDXwYNmsdLNRi00o14UjoaVqaPsYrZWvRKaIRqaU0hHmS0AWwQSvN/93iMIXuyiwywmkwKbWnnxCQ/gsctKFUtcNrwEx9Wgj6KlhwDTyI1QWSBbxVYNyUgPFzKxrSmwMO0yNff7ho+QT9x5+Y/7XE59S4Mc4ZXxcXKew/gSlN9U5mvT+D2BhDtkCupdfsZNCQWp27A+b/DmrFI9NqsCAwEAAaOCAcIwggG+MBIGA1UdEwEB/wQIMAYBAf8CAQAwQwYDVR0eBDwwOqE4MAaCBC5taWwwCocIAAAAAAAAAAAwIocgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwDgYDVR0PAQH/BAQDAgGGMH8GCCsGAQUFBwEBBHMwcTAyBggrBgEFBQcwAYYmaHR0cDovL2lzcmcudHJ1c3RpZC5vY3NwLmlkZW50cnVzdC5jb20wOwYIKwYBBQUHMAKGL2h0dHA6Ly9hcHBzLmlkZW50cnVzdC5jb20vcm9vdHMvZHN0cm9vdGNheDMucDdjMB8GA1UdIwQYMBaAFOmkP+6epeby1dd5YDyTpi4kjpeqMFQGA1UdIARNMEswCAYGZ4EMAQIBMD8GCysGAQQBgt8TAQEBMDAwLgYIKwYBBQUHAgEWImh0dHA6Ly9jcHMucm9vdC14MS5sZXRzZW5jcnlwdC5vcmcwPAYDVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybC5pZGVudHJ1c3QuY29tL0RTVFJPT1RDQVgzQ1JMLmNybDAdBgNVHQ4EFgQU+3hPEvlgFYMsnxd/NBmzLjbqQYkwDQYJKoZIhvcNAQELBQADggEBAA0YAeLXOklx4hhCikUUl+BdnFfn1g0W5AiQLVNIOL6PnqXu0wjnhNyhqdwnfhYMnoy4idRh4lB6pz8Gf9pnlLd/DnWSV3gS+/I/mAl1dCkKby6H2V790e6IHmIK2KYm3jm+U++FIdGpBdsQTSdmiX/rAyuxMDM0adMkNBwTfQmZQCz6nGHw1QcSPZMvZpsC8SkvekzxsjF1otOrMUPNPQvtTWrVx8GlR2qfx/4xbQa1v2frNvFBCmO59goz+jnWvfTtj2NjwDZ7vlMBsPm16dbKYC840uvRoZjxqsdc3ChCZjqimFqlNG/xoPA8+dTicZzCXE9ijPIcvW6y1aa3bGw="
+)
+
+func TestCTConsistencyCheck(t *testing.T) {
+	fulcioClient, err := ctclient.New("http://127.0.0.1:8080/testlog", http.DefaultClient, jsonclient.Options{})
+	if err != nil {
+		t.Errorf("error instantiating ct client: %v", err)
+	}
+
+	tempDir := t.TempDir()
+	tempLogInfoFile, err := os.CreateTemp(tempDir, "")
+	if err != nil {
+		t.Errorf("failed to create temp log file: %v", err)
+	}
+	tempLogInfoFileName := tempLogInfoFile.Name()
+	defer os.Remove(tempLogInfoFileName)
+
+	_, _, err = ct.RunConsistencyCheck(fulcioClient, tempLogInfoFileName)
+	if err != nil {
+		t.Errorf("failed to successfully complete consistency check: %v", err)
+	}
+
+	_, err = ct.IdentitySearch(fulcioClient, 0, 1, identity.MonitoredValues{
+		CertificateIdentities: []identity.CertificateIdentity{
+			{
+				CertSubject: "test-cert-subject",
+				Issuers:     []string{},
+			},
+		},
+	})
+	if err != nil {
+		t.Errorf("failed to successfully complete identity search: %v", err)
+	}
+}

pkg/test/ct_e2e/ct_monitor_e2e_test_ctfe.sh

@@ -0,0 +1,127 @@
+#!/usr/bin/env bash
+#
+# Copyright 2024 The Sigstore Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+pushd $HOME
+
+echo "downloading service repos"
+for repo in certificate-transparency-go trillian; do
+    if [[ ! -d $repo ]]; then
+        git clone https://github.com/google/${repo}.git
+    fi
+done
+
+docker_compose="docker compose"
+
+pushd ./certificate-transparency-go/trillian/examples/deployment/docker/ctfe/
+docker compose up -d
+sleep 10
+popd
+
+
+die() {
+  echo "$*" > /dev/stderr
+  exit 1
+}
+
+collect_vars() {
+  # set unset environment variables to defaults
+  [ -z ${MYSQL_ROOT_USER+x} ] && MYSQL_ROOT_USER="root"
+  [ -z ${MYSQL_HOST+x} ] && MYSQL_HOST="127.0.0.1"
+  [ -z ${MYSQL_PORT+x} ] && MYSQL_PORT="3306"
+  [ -z ${MYSQL_DATABASE+x} ] && MYSQL_DATABASE="test"
+  [ -z ${MYSQL_USER+x} ] && MYSQL_USER="test"
+  [ -z ${MYSQL_PASSWORD+x} ] && MYSQL_PASSWORD="zaphod"
+  [ -z ${MYSQL_ROOT_PASSWORD+x} ] && MYSQL_ROOT_PASSWORD="zaphod"
+  [ -z ${MYSQL_USER_HOST+x} ] && MYSQL_USER_HOST="127.0.0.1"
+  FLAGS=()
+
+  # handle flags
+  FORCE=false
+  VERBOSE=false
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --force) FORCE=true ;;
+      --verbose) VERBOSE=true ;;
+      --help) usage; exit ;;
+      *) FLAGS+=("$1")
+    esac
+    shift 1
+  done
+
+  FLAGS+=(-u "${MYSQL_ROOT_USER}")
+  FLAGS+=(--host "${MYSQL_HOST}")
+  FLAGS+=(--port "${MYSQL_PORT}")
+
+  # Optionally print flags (before appending password)
+  [[ ${VERBOSE} = 'true' ]] && echo "- Using MySQL Flags: ${FLAGS[@]}"
+
+  # append password if supplied
+  [ -z ${MYSQL_ROOT_PASSWORD+x} ] || FLAGS+=(-p"${MYSQL_ROOT_PASSWORD}")
+}
+
+main() {
+  collect_vars "$@"
+
+  readonly TRILLIAN_PATH=$(go list -f '{{.Dir}}' github.com/google/trillian)
+
+  echo "Warning: about to destroy and reset database '${MYSQL_DATABASE}'"
+  echo "Resetting DB..."
+  mysql "${FLAGS[@]}" -e "DROP DATABASE IF EXISTS ${MYSQL_DATABASE};" || \
+  die "Error: Failed to drop database '${MYSQL_DATABASE}'."
+  mysql "${FLAGS[@]}" -e "CREATE DATABASE ${MYSQL_DATABASE};" || \
+  die "Error: Failed to create database '${MYSQL_DATABASE}'."
+  mysql "${FLAGS[@]}" -e "CREATE USER IF NOT EXISTS ${MYSQL_USER}@'${MYSQL_USER_HOST}' IDENTIFIED BY '${MYSQL_PASSWORD}';" || \
+  die "Error: Failed to create user '${MYSQL_USER}@${MYSQL_USER_HOST}'."
+  mysql "${FLAGS[@]}" -e "GRANT ALL ON ${MYSQL_DATABASE}.* TO ${MYSQL_USER}@'${MYSQL_USER_HOST}'" || \
+  die "Error: Failed to grant '${MYSQL_USER}' user all privileges on '${MYSQL_DATABASE}'."
+  # mysql "${FLAGS[@]}" -D ${MYSQL_DATABASE} < ./trillian/storage/mysql/schema/storage.sql || \
+  # die "Error: Failed to create tables in '${MYSQL_DATABASE}' database."
+  echo "Reset Complete"
+}
+
+main "$@"
+
+docker exec -i ctfe-db mariadb -pzaphod -Dtest < ./trillian/storage/mysql/schema/storage.sql
+docker exec -i ctfe-db mariadb -pzaphod -Dtest < ./certificate-transparency-go/trillian/ctfe/storage/mysql/schema.sql
+
+CTFE_CONF_DIR=/tmp/ctfedocker
+if [ ! -d $CTFE_CONF_DIR ]; then
+  mkdir ${CTFE_CONF_DIR}
+fi
+
+TREE_ID=$(go run github.com/google/trillian/cmd/createtree@master --admin_server=127.0.0.1:8090)
+sed "s/@TREE_ID@/${TREE_ID}/" ./certificate-transparency-go/trillian/examples/deployment/docker/ctfe/ct_server.cfg > ${CTFE_CONF_DIR}/ct_server.cfg
+cp ./certificate-transparency-go/trillian/testdata/fake-ca.cert ${CTFE_CONF_DIR}
+docker volume create --driver local --opt type=none --opt device=${CTFE_CONF_DIR} --opt o=bind ctfe_config
+
+pushd ./certificate-transparency-go/trillian/examples/deployment/docker/ctfe/
+docker compose down
+docker compose --profile frontend up -d
+sleep 30
+popd
+
+docker ps
+docker container logs ctfe-db
+
+popd
+go test -tags=ct_e2e -v -race ./pkg/test/ct_e2e/...
+
+pushd $HOME
+pushd ./certificate-transparency-go/trillian/examples/deployment/docker/ctfe/
+docker compose down
+popd

pkg/test/ct_e2e/ct_monitor_e2e_test_manual.sh

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+#
+# Copyright 2024 The Sigstore Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+pushd $HOME
+
+echo "downloading service repos"
+for repo in certificate-transparency-go trillian; do
+    if [[ ! -d $repo ]]; then
+        git clone https://github.com/google/${repo}.git
+    fi
+done
+
+die() {
+  echo "$*" > /dev/stderr
+  exit 1
+}
+
+collect_vars() {
+  # set unset environment variables to defaults
+#   [ -z ${MYSQL_ROOT_USER+x} ] && MYSQL_ROOT_USER="test"
+  [ -z ${MYSQL_HOST+x} ] && MYSQL_HOST="127.0.0.1"
+  [ -z ${MYSQL_PORT+x} ] && MYSQL_PORT="3306"
+  [ -z ${MYSQL_DATABASE+x} ] && MYSQL_DATABASE="test"
+#   [ -z ${MYSQL_USER+x} ] && MYSQL_USER="test"
+#   [ -z ${MYSQL_PASSWORD+x} ] && MYSQL_PASSWORD="zaphod"
+#   [ -z ${MYSQL_ROOT_PASSWORD+x} ] && MYSQL_ROOT_PASSWORD="zaphod"
+  [ -z ${MYSQL_USER_HOST+x} ] && MYSQL_USER_HOST="127.0.0.1"
+  FLAGS=()
+
+  # handle flags
+  FORCE=false
+  VERBOSE=false
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --force) FORCE=true ;;
+      --verbose) VERBOSE=true ;;
+      --help) usage; exit ;;
+      *) FLAGS+=("$1")
+    esac
+    shift 1
+  done
+
+  FLAGS+=(-u "${MYSQL_ROOT_USER}")
+  FLAGS+=(--host "${MYSQL_HOST}")
+  FLAGS+=(--port "${MYSQL_PORT}")
+
+  # Optionally print flags (before appending password)
+  [[ ${VERBOSE} = 'true' ]] && echo "- Using MySQL Flags: ${FLAGS[@]}"
+
+  # append password if supplied
+#   [ -z ${MYSQL_ROOT_PASSWORD+x} ] || FLAGS+=(-p"${MYSQL_ROOT_PASSWORD}")
+}
+
+main() {
+  collect_vars "$@"
+
+  readonly TRILLIAN_PATH=$(go list -f '{{.Dir}}' github.com/google/trillian)
+
+  echo "Warning: about to destroy and reset database '${MYSQL_DATABASE}'"
+  echo "Resetting DB..."
+  mysql "${FLAGS[@]}" -e "DROP DATABASE IF EXISTS ${MYSQL_DATABASE};" || \
+  die "Error: Failed to drop database '${MYSQL_DATABASE}'."
+  mysql "${FLAGS[@]}" -e "CREATE DATABASE ${MYSQL_DATABASE};" || \
+  die "Error: Failed to create database '${MYSQL_DATABASE}'."
+#   mysql "${FLAGS[@]}" -e "CREATE USER IF NOT EXISTS ${MYSQL_USER}@'${MYSQL_USER_HOST}' IDENTIFIED BY '${MYSQL_PASSWORD}';" || \
+#   die "Error: Failed to create user '${MYSQL_USER}@${MYSQL_USER_HOST}'."
+#   mysql "${FLAGS[@]}" -e "GRANT ALL ON ${MYSQL_DATABASE}.* TO ${MYSQL_USER}@'${MYSQL_USER_HOST}'" || \
+#   die "Error: Failed to grant '${MYSQL_USER}' user all privileges on '${MYSQL_DATABASE}'."
+  mysql "${FLAGS[@]}" -D ${MYSQL_DATABASE} < ./trillian/storage/mysql/schema/storage.sql || \
+  die "Error: Failed to create tables in '${MYSQL_DATABASE}' database."
+  mysql "${FLAGS[@]}" -D ${MYSQL_DATABASE} < ./certificate-transparency-go/trillian/ctfe/storage/mysql/schema.sql || \
+  die "Error: Failed to provision logs in '${MYSQL_DATABASE}' database."
+  echo "Reset Complete"
+}
+
+main "$@"
+
+pushd ./trillian
+go run github.com/google/trillian/cmd/trillian_log_server --mysql_uri="root@tcp(127.0.0.1:3306)/test" --rpc_endpoint=:8080 --http_endpoint=:8081 --logtostderr
+go run github.com/google/trillian/cmd/trillian_log_signer --mysql_uri="root@tcp(127.0.0.1:3306)/test" --force_master --rpc_endpoint=:8090 --http_endpoint=:8091 --logtostderr
+go run github.com/google/trillian/cmd/createtree --admin_server=:8080
+popd
+
+pushd ./certificate-transparency-go/trillian/examples/deployment/docker/ctfe/
+TRILLIAN_LOG_SERVER_RPC_ENDPOINT=127.0.0.1:8080
+go run github.com/google/certificate-transparency-go/trillian/ctfe/ct_server --log_config ./ct_server.cfg --http_endpoint=127.0.0.1:6966 --log_rpc_server ${TRILLIAN_LOG_SERVER_RPC_ENDPOINT} --logtostderr
+popd
+
+docker ps
+docker container logs ctfe-db
+
+popd
+go test -tags=ct_e2e -v -race ./pkg/test/ct_e2e/...