Refactor testfixtures

go.mod

@@ -61,7 +61,6 @@ require (
 	github.com/go-redsync/redsync/v4 v4.13.0
 	github.com/go-sql-driver/mysql v1.8.1
 	github.com/go-swagger/go-swagger v0.31.0
-	github.com/go-testfixtures/testfixtures/v3 v3.11.0
 	github.com/go-webauthn/webauthn v0.11.2
 	github.com/gobwas/glob v0.2.3
 	github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f
@@ -145,8 +144,6 @@ require (
 	filippo.io/edwards25519 v1.1.0 // indirect
 	git.sr.ht/~mariusor/go-xsd-duration v0.0.0-20220703122237-02e73435a078 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
-	github.com/ClickHouse/ch-go v0.63.1 // indirect
-	github.com/ClickHouse/clickhouse-go/v2 v2.24.0 // indirect
 	github.com/DataDog/zstd v1.5.6 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.3.0 // indirect
@@ -204,8 +201,6 @@ require (
 	github.com/go-ap/errors v0.0.0-20240910140019-1e9d33cc1568 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
 	github.com/go-enry/go-oniguruma v1.2.1 // indirect
-	github.com/go-faster/city v1.0.1 // indirect
-	github.com/go-faster/errors v0.7.1 // indirect
 	github.com/go-fed/httpsig v1.1.1-0.20201223112313-55836744818e // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-ini/ini v1.67.0 // indirect
@@ -270,7 +265,6 @@ require (
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/onsi/ginkgo v1.16.5 // indirect
-	github.com/paulmach/orb v0.11.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
@@ -285,7 +279,6 @@ require (
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sagikazarmark/locafero v0.6.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
-	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/shopspring/decimal v1.4.0 // indirect
 	github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
@@ -310,8 +303,6 @@ require (
 	github.com/zeebo/blake3 v0.2.4 // indirect
 	go.etcd.io/bbolt v1.3.11 // indirect
 	go.mongodb.org/mongo-driver v1.17.1 // indirect
-	go.opentelemetry.io/otel v1.31.0 // indirect
-	go.opentelemetry.io/otel/trace v1.31.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect

models/fixtures/action_run_job.yml

@@ -64,7 +64,7 @@
   name: job2
   attempt: 1
   job_id: job2
-  needs: [job1]
+  needs: '["job1"]'
   task_id: 51
   status: 5
   started: 1683636528

models/fixtures/protected_tag.yml

@@ -2,23 +2,23 @@
   id: 1
   repo_id: 4
   name_pattern: /v.+/
-  allowlist_user_i_ds: []
-  allowlist_team_i_ds: []
+  allowlist_user_i_ds: "[]"
+  allowlist_team_i_ds: "[]"
   created_unix: 1715596037
   updated_unix: 1715596037
 -
   id: 2
   repo_id: 1
   name_pattern: v-*
-  allowlist_user_i_ds: []
-  allowlist_team_i_ds: []
+  allowlist_user_i_ds: "[]"
+  allowlist_team_i_ds: "[]"
   created_unix: 1715596037
   updated_unix: 1715596037
 -
   id: 3
   repo_id: 1
   name_pattern: v-1.1
-  allowlist_user_i_ds: [2]
-  allowlist_team_i_ds: []
+  allowlist_user_i_ds: "[2]"
+  allowlist_team_i_ds: "[]"
   created_unix: 1715596037
   updated_unix: 1715596037

models/unittest/fixtures.go

@@ -6,19 +6,21 @@ package unittest
 
 import (
 	"fmt"
-	"os"
 	"time"
 
 	"code.gitea.io/gitea/models/db"
 	"code.gitea.io/gitea/modules/auth/password/hash"
 	"code.gitea.io/gitea/modules/setting"
 
-	"github.com/go-testfixtures/testfixtures/v3"
 	"xorm.io/xorm"
 	"xorm.io/xorm/schemas"
 )
 
-var fixturesLoader *testfixtures.Loader
+type FixturesLoader interface {
+	Load() error
+}
+
+var fixturesLoader FixturesLoader
 
 // GetXORMEngine gets the XORM engine
 func GetXORMEngine(engine ...*xorm.Engine) (x *xorm.Engine) {
@@ -31,38 +33,7 @@ func GetXORMEngine(engine ...*xorm.Engine) (x *xorm.Engine) {
 // InitFixtures initialize test fixtures for a test database
 func InitFixtures(opts FixturesOptions, engine ...*xorm.Engine) (err error) {
 	e := GetXORMEngine(engine...)
-	var fixtureOptionFiles func(*testfixtures.Loader) error
-	if opts.Dir != "" {
-		fixtureOptionFiles = testfixtures.Directory(opts.Dir)
-	} else {
-		fixtureOptionFiles = testfixtures.Files(opts.Files...)
-	}
-	dialect := "unknown"
-	switch e.Dialect().URI().DBType {
-	case schemas.POSTGRES:
-		dialect = "postgres"
-	case schemas.MYSQL:
-		dialect = "mysql"
-	case schemas.MSSQL:
-		dialect = "mssql"
-	case schemas.SQLITE:
-		dialect = "sqlite3"
-	default:
-		fmt.Println("Unsupported RDBMS for integration tests")
-		os.Exit(1)
-	}
-	loaderOptions := []func(loader *testfixtures.Loader) error{
-		testfixtures.Database(e.DB().DB),
-		testfixtures.Dialect(dialect),
-		testfixtures.DangerousSkipTestDatabaseCheck(),
-		fixtureOptionFiles,
-	}
-
-	if e.Dialect().URI().DBType == schemas.POSTGRES {
-		loaderOptions = append(loaderOptions, testfixtures.SkipResetSequences())
-	}
-
-	fixturesLoader, err = testfixtures.New(loaderOptions...)
+	fixturesLoader, err = NewFixturesLoader(e, opts)
 	if err != nil {
 		return err
 	}

models/unittest/fixtures_loader.go

@@ -0,0 +1,201 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package unittest
+
+import (
+	"database/sql"
+	"encoding/hex"
+	"fmt"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+	"xorm.io/xorm"
+	"xorm.io/xorm/schemas"
+)
+
+type fixtureItem struct {
+	tableName       string
+	tableNameQuoted string
+	sqlInserts      []string
+	sqlInsertArgs   [][]any
+
+	mssqlHasIdentityColumn bool
+}
+
+type fixturesLoaderInternal struct {
+	db               *sql.DB
+	dbType           schemas.DBType
+	files            []string
+	fixtures         map[string]*fixtureItem
+	quoteObject      func(string) string
+	paramPlaceholder func(idx int) string
+}
+
+func (f *fixturesLoaderInternal) mssqlTableHasIdentityColumn(db *sql.DB, tableName string) (bool, error) {
+	row := db.QueryRow(`SELECT COUNT(*) FROM sys.identity_columns WHERE OBJECT_ID = OBJECT_ID(?)`, tableName)
+	var count int
+	if err := row.Scan(&count); err != nil {
+		return false, err
+	}
+	return count > 0, nil
+}
+
+func (f *fixturesLoaderInternal) preprocessFixtureRow(row []map[string]any) (err error) {
+	for _, m := range row {
+		for k, v := range m {
+			if s, ok := v.(string); ok {
+				if strings.HasPrefix(s, "0x") {
+					if m[k], err = hex.DecodeString(s[2:]); err != nil {
+						return err
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func (f *fixturesLoaderInternal) prepareFixtureItem(file string) (_ *fixtureItem, err error) {
+	fixture := &fixtureItem{}
+	fixture.tableName, _, _ = strings.Cut(filepath.Base(file), ".")
+	fixture.tableNameQuoted = f.quoteObject(fixture.tableName)
+
+	if f.dbType == schemas.MSSQL {
+		fixture.mssqlHasIdentityColumn, err = f.mssqlTableHasIdentityColumn(f.db, fixture.tableName)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	data, err := os.ReadFile(file)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read file %q: %w", file, err)
+	}
+
+	var rows []map[string]any
+	if err = yaml.Unmarshal(data, &rows); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal yaml data from %q: %w", file, err)
+	}
+	if err = f.preprocessFixtureRow(rows); err != nil {
+		return nil, fmt.Errorf("failed to preprocess fixture rows from %q: %w", file, err)
+	}
+
+	var sqlBuf []byte
+	var sqlArguments []any
+	for _, row := range rows {
+		sqlBuf = append(sqlBuf, fmt.Sprintf("INSERT INTO %s (", fixture.tableNameQuoted)...)
+		for k, v := range row {
+			sqlBuf = append(sqlBuf, f.quoteObject(k)...)
+			sqlBuf = append(sqlBuf, ","...)
+			sqlArguments = append(sqlArguments, v)
+		}
+		sqlBuf = sqlBuf[:len(sqlBuf)-1]
+		sqlBuf = append(sqlBuf, ") VALUES ("...)
+		paramIdx := 1
+		for range row {
+			sqlBuf = append(sqlBuf, f.paramPlaceholder(paramIdx)...)
+			sqlBuf = append(sqlBuf, ',')
+			paramIdx++
+		}
+		sqlBuf[len(sqlBuf)-1] = ')'
+		fixture.sqlInserts = append(fixture.sqlInserts, string(sqlBuf))
+		fixture.sqlInsertArgs = append(fixture.sqlInsertArgs, slices.Clone(sqlArguments))
+		sqlBuf = sqlBuf[:0]
+		sqlArguments = sqlArguments[:0]
+	}
+	return fixture, nil
+}
+
+func (f *fixturesLoaderInternal) loadFixtures(tx *sql.Tx, file string) (err error) {
+	fixture := f.fixtures[file]
+	if fixture == nil {
+		if fixture, err = f.prepareFixtureItem(file); err != nil {
+			return err
+		}
+		f.fixtures[file] = fixture
+	}
+
+	_, err = tx.Exec(fmt.Sprintf("DELETE FROM %s", fixture.tableNameQuoted)) // sqlite3 doesn't support truncate
+	if err != nil {
+		return err
+	}
+
+	if fixture.mssqlHasIdentityColumn {
+		_, err = tx.Exec(fmt.Sprintf("SET IDENTITY_INSERT %s ON", fixture.tableNameQuoted))
+		if err != nil {
+			return err
+		}
+		defer func() { _, err = tx.Exec(fmt.Sprintf("SET IDENTITY_INSERT %s OFF", fixture.tableNameQuoted)) }()
+	}
+	for i := range fixture.sqlInserts {
+		_, err = tx.Exec(fixture.sqlInserts[i], fixture.sqlInsertArgs[i]...)
+	}
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (f *fixturesLoaderInternal) Load() error {
+	tx, err := f.db.Begin()
+	if err != nil {
+		return err
+	}
+	defer func() { _ = tx.Rollback() }()
+
+	for _, file := range f.files {
+		if err := f.loadFixtures(tx, file); err != nil {
+			return fmt.Errorf("failed to load fixtures from %s: %w", file, err)
+		}
+	}
+	return tx.Commit()
+}
+
+func FixturesFileFullPaths(dir string, files []string) ([]string, error) {
+	if files != nil && len(files) == 0 {
+		return nil, nil // load nothing
+	}
+	files = slices.Clone(files)
+	if len(files) == 0 {
+		entries, err := os.ReadDir(dir)
+		if err != nil {
+			return nil, err
+		}
+		for _, e := range entries {
+			files = append(files, e.Name())
+		}
+	}
+	for i, file := range files {
+		if !filepath.IsAbs(file) {
+			files[i] = filepath.Join(dir, file)
+		}
+	}
+	return files, nil
+}
+
+func NewFixturesLoader(x *xorm.Engine, opts FixturesOptions) (FixturesLoader, error) {
+	files, err := FixturesFileFullPaths(opts.Dir, opts.Files)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get fixtures files: %w", err)
+	}
+	f := &fixturesLoaderInternal{db: x.DB().DB, dbType: x.Dialect().URI().DBType, files: files, fixtures: map[string]*fixtureItem{}}
+	switch f.dbType {
+	case schemas.SQLITE:
+		f.quoteObject = func(s string) string { return fmt.Sprintf(`"%s"`, s) }
+		f.paramPlaceholder = func(idx int) string { return "?" }
+	case schemas.POSTGRES:
+		f.quoteObject = func(s string) string { return fmt.Sprintf(`"%s"`, s) }
+		f.paramPlaceholder = func(idx int) string { return fmt.Sprintf(`$%d`, idx) }
+	case schemas.MYSQL:
+		f.quoteObject = func(s string) string { return fmt.Sprintf("`%s`", s) }
+		f.paramPlaceholder = func(idx int) string { return "?" }
+	case schemas.MSSQL:
+		f.quoteObject = func(s string) string { return fmt.Sprintf("[%s]", s) }
+		f.paramPlaceholder = func(idx int) string { return "?" }
+	}
+	return f, nil
+}

modules/system/appstate_test.go

@@ -13,9 +13,7 @@ import (
 )
 
 func TestMain(m *testing.M) {
-	unittest.MainTest(m, &unittest.TestOptions{
-		FixtureFiles: []string{""}, // load nothing
-	})
+	unittest.MainTest(m, &unittest.TestOptions{FixtureFiles: []string{ /* load nothing */ }})
 }
 
 type testItem1 struct {
@@ -36,8 +34,6 @@ func (*testItem2) Name() string {
 }
 
 func TestAppStateDB(t *testing.T) {
-	assert.NoError(t, unittest.PrepareTestDatabase())
-
 	as := &DBStore{}
 
 	item1 := new(testItem1)