From f8bcfe8994f38ca33677b5a63784cec7aae6e780 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Fri, 1 Dec 2023 11:41:43 -0800 Subject: [PATCH 01/23] Remove defunct locking logic --- server/server.go | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/server/server.go b/server/server.go index bdf992d59e..19dd338dc0 100644 --- a/server/server.go +++ b/server/server.go @@ -69,7 +69,6 @@ const stdInFlag = "--stdin" const stdOutFlag = "--stdout" const stdErrFlag = "--stderr" const stdOutAndErrFlag = "--out-and-err" -const ignoreLocksFlag = "--ignore-lock-file" // RunOnDisk starts the server based on the given args, while also using the local disk as the backing store. // The returned WaitGroup may be used to wait for the server to close. @@ -108,7 +107,6 @@ func runServer(args []string, fs filesys.Filesys) (*int, *sync.WaitGroup) { nbs.TableIndexGCFinalizerWithStackTrace = false } - ignoreLockFile := false if len(args) > 0 { var doneDebugFlags bool for !doneDebugFlags && len(args) > 0 { @@ -162,11 +160,7 @@ func runServer(args []string, fs filesys.Filesys) (*int, *sync.WaitGroup) { color.NoColor = true args = args[2:] - - case ignoreLocksFlag: - ignoreLockFile = true - args = args[1:] - + default: doneDebugFlags = true } @@ -181,7 +175,6 @@ func runServer(args []string, fs filesys.Filesys) (*int, *sync.WaitGroup) { warnIfMaxFilesTooLow() dEnv := env.Load(ctx, env.GetCurrentUserHomeDir, fs, doltdb.LocalDirDoltDB, Version) - dEnv.IgnoreLockFile = ignoreLockFile globalConfig, ok := dEnv.Config.GetConfig(env.GlobalConfig) if !ok { @@ -267,7 +260,7 @@ func runServer(args []string, fs filesys.Filesys) (*int, *sync.WaitGroup) { } dEnv.FS = dataDirFS - mrEnv, err := env.MultiEnvForDirectory(ctx, dEnv.Config.WriteableConfig(), dataDirFS, dEnv.Version, dEnv.IgnoreLockFile, dEnv) + mrEnv, err := env.MultiEnvForDirectory(ctx, dEnv.Config.WriteableConfig(), dataDirFS, dEnv.Version, dEnv) if err != nil { cli.PrintErrln("failed to load database names") return intPointer(1), wg @@ -427,22 +420,7 @@ func buildLateBinder(ctx context.Context, cwdFS filesys.Filesys, rootEnv *env.Do if targetEnv == nil { targetEnv = rootEnv } - - isLocked, lock, err := targetEnv.GetLock() - if err != nil { - return nil, err - } - if isLocked { - if verbose { - cli.Println("verbose: starting remote mode") - } - - if !creds.Specified { - creds = &cli.UserPassword{Username: sqlserver.LocalConnectionUser, Password: lock.Secret, Specified: false} - } - return sqlserver.BuildConnectionStringQueryist(ctx, cwdFS, creds, apr, "localhost", lock.Port, false, useDb) - } - + if verbose { cli.Println("verbose: starting local mode") } From 28f37d4d64181726465176ca24a30a6e1370362a Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Tue, 5 Dec 2023 17:41:15 -0800 Subject: [PATCH 02/23] Refactored start up methods to return a controller --- main.go | 11 +++++- server/server.go | 98 +++++++++++++++++++++++------------------------- 2 files changed, 56 insertions(+), 53 deletions(-) diff --git a/main.go b/main.go index a036897ec2..dbe6e22a1a 100644 --- a/main.go +++ b/main.go @@ -15,13 +15,20 @@ package main import ( + "fmt" "os" "github.com/dolthub/doltgresql/server" ) func main() { - code, wg := server.RunOnDisk(os.Args[1:]) - wg.Wait() + code, controller := server.RunOnDisk(os.Args[1:]) + + err := controller.WaitForStop() + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + os.Exit(*code) } diff --git a/server/server.go b/server/server.go index 19dd338dc0..2ec15f493c 100644 --- a/server/server.go +++ b/server/server.go @@ -23,7 +23,6 @@ import ( _ "net/http/pprof" "os" "strings" - "sync" "github.com/dolthub/dolt/go/cmd/dolt/cli" "github.com/dolthub/dolt/go/cmd/dolt/commands" @@ -36,6 +35,7 @@ import ( "github.com/dolthub/dolt/go/libraries/utils/argparser" "github.com/dolthub/dolt/go/libraries/utils/config" "github.com/dolthub/dolt/go/libraries/utils/filesys" + "github.com/dolthub/dolt/go/libraries/utils/svcs" "github.com/dolthub/dolt/go/store/nbs" "github.com/dolthub/dolt/go/store/util/tempfiles" "github.com/dolthub/go-mysql-server/server" @@ -72,20 +72,19 @@ const stdOutAndErrFlag = "--out-and-err" // RunOnDisk starts the server based on the given args, while also using the local disk as the backing store. // The returned WaitGroup may be used to wait for the server to close. -func RunOnDisk(args []string) (*int, *sync.WaitGroup) { +func RunOnDisk(args []string) (*svcs.Controller, error) { return runServer(args, filesys.LocalFS) } // RunInMemory starts the server based on the given args, while also using RAM as the backing store. // The returned WaitGroup may be used to wait for the server to close. -func RunInMemory(args []string) (*int, *sync.WaitGroup) { +func RunInMemory(args []string) (*svcs.Controller, error) { return runServer(args, filesys.EmptyInMemFS("")) } // runServer starts the server based on the given args, using the provided file system as the backing store. // The returned WaitGroup may be used to wait for the server to close. -func runServer(args []string, fs filesys.Filesys) (*int, *sync.WaitGroup) { - wg := &sync.WaitGroup{} +func runServer(args []string, fs filesys.Filesys) (*svcs.Controller, error) { ctx := context.Background() // Inject the "sql-server" command if no other commands were given if len(args) == 0 || (len(args) > 0 && strings.HasPrefix(args[0], "-")) { @@ -129,8 +128,7 @@ func runServer(args []string, fs filesys.Filesys) (*int, *sync.WaitGroup) { f, err := os.Open(stdInFile) if err != nil { - cli.PrintErrln("Failed to open", stdInFile, err.Error()) - return intPointer(1), wg + return nil, fmt.Errorf("Failed to open %s: %w", stdInFile, err) } os.Stdin = f @@ -141,8 +139,7 @@ func runServer(args []string, fs filesys.Filesys) (*int, *sync.WaitGroup) { f, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, os.ModePerm) if err != nil { - cli.PrintErrln("Failed to open", filename, "for writing:", err.Error()) - return intPointer(1), wg + return nil, fmt.Errorf("Failed to open %s for writing: %w", filename, err) } switch args[0] { @@ -178,25 +175,22 @@ func runServer(args []string, fs filesys.Filesys) (*int, *sync.WaitGroup) { globalConfig, ok := dEnv.Config.GetConfig(env.GlobalConfig) if !ok { - cli.PrintErrln(color.RedString("Failed to get global config")) - return intPointer(1), wg + return nil, fmt.Errorf("Failed to get global config") } // The in-memory database is only used for testing/virtual environments, so the config may need modification - if _, ok = fs.(*filesys.InMemFS); ok && globalConfig.GetStringOrDefault(env.UserNameKey, "") == "" { + if _, ok = fs.(*filesys.InMemFS); ok && globalConfig.GetStringOrDefault(config.UserNameKey, "") == "" { globalConfig.SetStrings(map[string]string{ - env.UserNameKey: "postgres", - env.UserEmailKey: "postgres@somewhere.com", + config.UserNameKey: "postgres", + config.UserEmailKey: "postgres@somewhere.com", }) } apr, remainingArgs, subcommandName, err := parseGlobalArgsAndSubCommandName(globalConfig, args) if err == argparser.ErrHelp { //TODO: display some help message - doltCommand.PrintUsage("dolt") - return intPointer(0), wg + return nil, fmt.Errorf("help") } else if err != nil { - cli.PrintErrln(color.RedString("Failure to parse arguments: %v", err)) - return intPointer(1), wg + return nil, fmt.Errorf("failed to parse arguments: %w", err) } dataDir, hasDataDir := apr.GetValue(commands.DataDirFlag) @@ -204,30 +198,25 @@ func runServer(args []string, fs filesys.Filesys) (*int, *sync.WaitGroup) { // If a relative path was provided, this ensures we have an absolute path everywhere. dataDir, err = fs.Abs(dataDir) if err != nil { - cli.PrintErrln(color.RedString("Failed to get absolute path for %s: %v", dataDir, err)) - return intPointer(1), wg + return nil, fmt.Errorf("failed to get absolute path for %s: %w", dataDir, err) } if ok, dir := fs.Exists(dataDir); !ok || !dir { - cli.Println(color.RedString("Provided data directory does not exist: %s", dataDir)) - return intPointer(1), wg + return nil, fmt.Errorf("data directory %s does not exist", dataDir) } } if dEnv.CfgLoadErr != nil { - cli.PrintErrln(color.RedString("Failed to load the global config. %v", dEnv.CfgLoadErr)) - return intPointer(1), wg + return nil, fmt.Errorf("failed to load the global config: %w", dEnv.CfgLoadErr) } if dEnv.HasDoltDataDir() { - cli.PrintErrln(color.RedString("Cannot start a server within a directory containing a Dolt or Doltgres database." + - "To use the current directory as a database, start the server from the parent directory.")) - return intPointer(1), wg + return nil, fmt.Errorf("Cannot start a server within a directory containing a Dolt or Doltgres database." + + "To use the current directory as a database, start the server from the parent directory.") } err = reconfigIfTempFileMoveFails(dEnv) if err != nil { - cli.PrintErrln(color.RedString("Failed to setup the temporary directory. %v`", err)) - return intPointer(1), wg + return nil, fmt.Errorf("failed to set up the temporary directory: %w", err) } defer tempfiles.MovableTempFileProvider.Clean() @@ -255,15 +244,13 @@ func runServer(args []string, fs filesys.Filesys) (*int, *sync.WaitGroup) { cwdFS := dEnv.FS dataDirFS, err := dEnv.FS.WithWorkingDir(dataDir) if err != nil { - cli.PrintErrln(color.RedString("Failed to set the data directory. %v", err)) - return intPointer(1), wg + return nil, fmt.Errorf("failed to set the data directory: %w", err) } dEnv.FS = dataDirFS mrEnv, err := env.MultiEnvForDirectory(ctx, dEnv.Config.WriteableConfig(), dataDirFS, dEnv.Version, dEnv) if err != nil { - cli.PrintErrln("failed to load database names") - return intPointer(1), wg + return nil, fmt.Errorf("failed to load database names: %w", err) } _ = mrEnv.Iter(func(dbName string, dEnv *env.DoltEnv) (stop bool, err error) { dsess.DefineSystemVariablesForDB(dbName) @@ -272,29 +259,26 @@ func runServer(args []string, fs filesys.Filesys) (*int, *sync.WaitGroup) { err = dsess.InitPersistedSystemVars(dEnv) if err != nil { - cli.Printf("error: failed to load persisted global variables: %s\n", err.Error()) + return nil, fmt.Errorf("failed to load persisted system variables: %w", err) } // validate that --user and --password are set appropriately. aprAlt, creds, err := cli.BuildUserPasswordPrompt(apr) apr = aprAlt if err != nil { - cli.PrintErrln(color.RedString("Failed to parse credentials: %v", err)) - return intPointer(1), wg + return nil, fmt.Errorf("failed to parse credentials: %w", err) } lateBind, err := buildLateBinder(ctx, cwdFS, dEnv, mrEnv, creds, apr, subcommandName, false) if err != nil { - cli.PrintErrln(color.RedString("%v", err)) - return intPointer(1), wg + return nil, err } cliCtx, err := cli.NewCliContext(apr, dEnv.Config, lateBind) if err != nil { - cli.PrintErrln(color.RedString("Unexpected Error: %v", err)) - return intPointer(1), wg + return nil, err } - + ctx, stop := context.WithCancel(ctx) if serverMode { var serverConfig sqlserver.ServerConfig @@ -307,8 +291,7 @@ func runServer(args []string, fs filesys.Filesys) (*int, *sync.WaitGroup) { tlsConfig, err := sqlserver.LoadTLSConfig(serverConfig) if err != nil { stop() - cli.PrintErrln(err) - return intPointer(1), wg + return nil, err } if tlsConfig != nil && len(tlsConfig.Certificates) > 0 { certificate = tlsConfig.Certificates[0] @@ -323,8 +306,7 @@ func runServer(args []string, fs filesys.Filesys) (*int, *sync.WaitGroup) { subdirectoryFS, err := dEnv.FS.WithWorkingDir("doltgres") if err != nil { stop() - cli.PrintErrln(err) - return intPointer(1), wg + return nil, err } // We'll use a temporary environment to instantiate the subdirectory tempDEnv := env.Load(ctx, env.GetCurrentUserHomeDir, subdirectoryFS, doltdb.LocalDirDoltDB, Version) @@ -333,13 +315,26 @@ func runServer(args []string, fs filesys.Filesys) (*int, *sync.WaitGroup) { } } - // We're now running the server, so we can increment the WaitGroup. - wg.Add(1) + controller := svcs.NewController() + newCtx, cancelF := context.WithCancel(ctx) + go func() { + // Here we only forward along the SIGINT if the server starts + // up successfully. If the service does not start up + // successfully, or if WaitForStart() blocks indefinitely, then + // startServer() should have returned an error and we do not + // need to Stop the running server or deal with our canceled + // parent context. + if controller.WaitForStart() == nil { + <-ctx.Done() + controller.Stop() + cancelF() + } + }() + res := new(int) go func() { - defer wg.Done() - *res = doltCommand.Exec(ctx, "dolt", remainingArgs, dEnv, cliCtx) - stop() + err := sqlserver.StartServer(newCtx, "doltgreSQL-0.1", "dolt", remainingArgs, dEnv, controller) + // TODO: hande this if err = dbfactory.CloseAllLocalDatabases(); err != nil { cli.PrintErrln(err) @@ -348,7 +343,8 @@ func runServer(args []string, fs filesys.Filesys) (*int, *sync.WaitGroup) { } } }() - return res, wg + + return controller, nil } // buildLateBinder builds a LateBindQueryist for which is used to obtain the Queryist used for the length of the From 20576fe3208dc5517cebd125aad81d910ef92c31 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Wed, 6 Dec 2023 16:49:08 -0800 Subject: [PATCH 03/23] Refactored startup to return a controller --- main.go | 11 ++- server/server.go | 101 ++++++++++---------------- testing/go/framework_test.go | 21 +++--- testing/go/prepared_statement_test.go | 6 +- testing/go/ssl_test.go | 11 ++- 5 files changed, 72 insertions(+), 78 deletions(-) diff --git a/main.go b/main.go index dbe6e22a1a..fc7fa34b71 100644 --- a/main.go +++ b/main.go @@ -22,13 +22,18 @@ import ( ) func main() { - code, controller := server.RunOnDisk(os.Args[1:]) + controller, err := server.RunOnDisk(os.Args[1:]) + + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } - err := controller.WaitForStop() + err = controller.WaitForStop() if err != nil { fmt.Println(err.Error()) os.Exit(1) } - os.Exit(*code) + os.Exit(0) } diff --git a/server/server.go b/server/server.go index 2ec15f493c..00f4964e4d 100644 --- a/server/server.go +++ b/server/server.go @@ -22,12 +22,10 @@ import ( "math/rand" _ "net/http/pprof" "os" - "strings" "github.com/dolthub/dolt/go/cmd/dolt/cli" "github.com/dolthub/dolt/go/cmd/dolt/commands" "github.com/dolthub/dolt/go/cmd/dolt/commands/sqlserver" - "github.com/dolthub/dolt/go/libraries/doltcore/dbfactory" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dfunctions" @@ -86,19 +84,10 @@ func RunInMemory(args []string) (*svcs.Controller, error) { // The returned WaitGroup may be used to wait for the server to close. func runServer(args []string, fs filesys.Filesys) (*svcs.Controller, error) { ctx := context.Background() - // Inject the "sql-server" command if no other commands were given - if len(args) == 0 || (len(args) > 0 && strings.HasPrefix(args[0], "-")) { - args = append([]string{"sql-server"}, args...) - } - // The "sql-server" command will automatically initialize the repository - serverMode := false - if args[0] == "sql-server" { - serverMode = true - // Enforce a default port of 5432 - if serverArgs, err := (sqlserver.SqlServerCmd{}).ArgParser().Parse(args); err == nil { - if _, ok := serverArgs.GetValue("port"); !ok { - args = append(args, "--port=5432") - } + + if serverArgs, err := (sqlserver.SqlServerCmd{}).ArgParser().Parse(append([]string{"sql-server"}, args...)); err == nil { + if _, ok := serverArgs.GetValue("port"); !ok { + args = append(args, "--port=5432") } } @@ -185,7 +174,7 @@ func runServer(args []string, fs filesys.Filesys) (*svcs.Controller, error) { }) } - apr, remainingArgs, subcommandName, err := parseGlobalArgsAndSubCommandName(globalConfig, args) + apr, _, _, err := parseGlobalArgsAndSubCommandName(globalConfig, append([]string {"sql-server"}, args...)) if err == argparser.ErrHelp { //TODO: display some help message return nil, fmt.Errorf("help") @@ -269,7 +258,7 @@ func runServer(args []string, fs filesys.Filesys) (*svcs.Controller, error) { return nil, fmt.Errorf("failed to parse credentials: %w", err) } - lateBind, err := buildLateBinder(ctx, cwdFS, dEnv, mrEnv, creds, apr, subcommandName, false) + lateBind, err := buildLateBinder(ctx, cwdFS, dEnv, mrEnv, creds, apr, "sql-server", false) if err != nil { return nil, err } @@ -279,39 +268,37 @@ func runServer(args []string, fs filesys.Filesys) (*svcs.Controller, error) { return nil, err } - ctx, stop := context.WithCancel(ctx) - if serverMode { - var serverConfig sqlserver.ServerConfig - // Grab the config so that we can pull the TLS key & cert. If there's an error then we can ignore it here, as - // it'll be caught when the server actually tries to run. We'll also use the config to determine whether we need to - // create a doltgres directory. - if sqlServerApr, err := cli.ParseArgs(sqlserver.SqlServerCmd{}.ArgParser(), args, nil); err == nil { - if serverConfig, err = sqlserver.GetServerConfig(dEnv.FS, sqlServerApr); err == nil { - // We throw an error if there's an issue with the TLS cert though - tlsConfig, err := sqlserver.LoadTLSConfig(serverConfig) - if err != nil { - stop() - return nil, err - } - if tlsConfig != nil && len(tlsConfig.Certificates) > 0 { - certificate = tlsConfig.Certificates[0] - } + var serverConfig sqlserver.ServerConfig + // Grab the config so that we can pull the TLS key & cert. If there's an error then we can ignore it here, as + // it'll be caught when the server actually tries to run. We'll also use the config to determine whether we need to + // create a doltgres directory. + if sqlServerApr, err := cli.ParseArgs(sqlserver.SqlServerCmd{}.ArgParser(), args, nil); err == nil { + if serverConfig, err = sqlserver.GetServerConfig(dEnv.FS, sqlServerApr); err == nil { + // We throw an error if there's an issue with the TLS cert though + tlsConfig, err := sqlserver.LoadTLSConfig(serverConfig) + if err != nil { + return nil, err + } + if tlsConfig != nil && len(tlsConfig.Certificates) > 0 { + certificate = tlsConfig.Certificates[0] } } - // Server mode automatically initializes a doltgres database - if !dEnv.HasDoltDir() { - // Need to make sure that there isn't a doltgres subdirectory. If there is, we'll assume it's a db. - if exists, _ := dEnv.FS.Exists("doltgres"); !exists { - dEnv.FS.MkDirs("doltgres") - subdirectoryFS, err := dEnv.FS.WithWorkingDir("doltgres") - if err != nil { - stop() - return nil, err - } - // We'll use a temporary environment to instantiate the subdirectory - tempDEnv := env.Load(ctx, env.GetCurrentUserHomeDir, subdirectoryFS, doltdb.LocalDirDoltDB, Version) - _ = doltCommand.Exec(ctx, "dolt", []string{"init"}, tempDEnv, cliCtx) + } + // Server mode automatically initializes a doltgres database + if !dEnv.HasDoltDir() { + // Need to make sure that there isn't a doltgres subdirectory. If there is, we'll assume it's a db. + if exists, _ := dEnv.FS.Exists("doltgres"); !exists { + err := dEnv.FS.MkDirs("doltgres") + if err != nil { + return nil, err + } + subdirectoryFS, err := dEnv.FS.WithWorkingDir("doltgres") + if err != nil { + return nil, err } + // We'll use a temporary environment to instantiate the subdirectory + tempDEnv := env.Load(ctx, env.GetCurrentUserHomeDir, subdirectoryFS, doltdb.LocalDirDoltDB, Version) + _ = doltCommand.Exec(ctx, "dolt", []string{"init"}, tempDEnv, cliCtx) } } @@ -330,21 +317,13 @@ func runServer(args []string, fs filesys.Filesys) (*svcs.Controller, error) { cancelF() } }() - - res := new(int) - go func() { - err := sqlserver.StartServer(newCtx, "doltgreSQL-0.1", "dolt", remainingArgs, dEnv, controller) - // TODO: hande this - - if err = dbfactory.CloseAllLocalDatabases(); err != nil { - cli.PrintErrln(err) - if *res == 0 { - *res = 1 - } - } - }() - return controller, nil + // We need a username and password for many SQL commands, so set defaults if they don't exist + dEnv.Config.SetFailsafes(env.DefaultFailsafeConfig) + + sqlserver.ConfigureServices(serverConfig, controller, Version, dEnv) + go controller.Start(newCtx) + return controller, controller.WaitForStart() } // buildLateBinder builds a LateBindQueryist for which is used to obtain the Queryist used for the length of the diff --git a/testing/go/framework_test.go b/testing/go/framework_test.go index e92c79013d..8320873a28 100644 --- a/testing/go/framework_test.go +++ b/testing/go/framework_test.go @@ -22,10 +22,10 @@ import ( "math" "net" "os" - "sync" "testing" "time" + "github.com/dolthub/dolt/go/libraries/utils/svcs" "github.com/dolthub/go-mysql-server/server" "github.com/dolthub/go-mysql-server/sql" "github.com/jackc/pgx/v5" @@ -76,10 +76,13 @@ func RunScript(t *testing.T, script ScriptTest) { if len(scriptDatabase) == 0 { scriptDatabase = "postgres" } - ctx, conn, serverClosed := CreateServer(t, scriptDatabase) + + ctx, conn, controller := CreateServer(t, scriptDatabase) defer func() { conn.Close(ctx) - serverClosed.Wait() + controller.Stop() + err := controller.WaitForStop() + require.NoError(t, err) }() t.Run(script.Name, func(t *testing.T) { @@ -143,17 +146,17 @@ func RunScripts(t *testing.T, scripts []ScriptTest) { // CreateServer creates a server with the given database, returning a connection to the server. The server will close // when the connection is closed (or loses its connection to the server). The accompanying WaitGroup may be used to wait // until the server has closed. -func CreateServer(t *testing.T, database string) (context.Context, *pgx.Conn, *sync.WaitGroup) { +func CreateServer(t *testing.T, database string) (context.Context, *pgx.Conn, *svcs.Controller) { require.NotEmpty(t, database) port := GetUnusedPort(t) server.DefaultProtocolListenerFunc = dserver.NewLimitedListener - code, serverClosed := dserver.RunInMemory([]string{fmt.Sprintf("--port=%d", port), "--host=127.0.0.1"}) - require.Equal(t, 0, *code) - + controller, err := dserver.RunInMemory([]string{fmt.Sprintf("--port=%d", port), "--host=127.0.0.1"}) + require.NoError(t, err) + fmt.Printf("port is %d\n", port) ctx := context.Background() - err := func() error { + err = func() error { // The connection attempt may be made before the server has grabbed the port, so we'll retry the first // connection a few times. var conn *pgx.Conn @@ -178,7 +181,7 @@ func CreateServer(t *testing.T, database string) (context.Context, *pgx.Conn, *s conn, err := pgx.Connect(ctx, fmt.Sprintf("postgres://postgres:password@127.0.0.1:%d/%s", port, database)) require.NoError(t, err) - return ctx, conn, serverClosed + return ctx, conn, controller } // ReadRows reads all of the given rows into a slice, then closes the rows. This also normalizes all of the rows. diff --git a/testing/go/prepared_statement_test.go b/testing/go/prepared_statement_test.go index 534c34b015..f215a236f2 100755 --- a/testing/go/prepared_statement_test.go +++ b/testing/go/prepared_statement_test.go @@ -74,10 +74,12 @@ func RunScriptN(t *testing.T, script ScriptTest, n int) { if len(scriptDatabase) == 0 { scriptDatabase = "postgres" } - ctx, conn, serverClosed := CreateServer(t, scriptDatabase) + ctx, conn, controller := CreateServer(t, scriptDatabase) defer func() { conn.Close(ctx) - serverClosed.Wait() + controller.Stop() + err := controller.WaitForStop() + require.NoError(t, err) }() // Run the setup diff --git a/testing/go/ssl_test.go b/testing/go/ssl_test.go index c09752e308..ea13dbfb99 100644 --- a/testing/go/ssl_test.go +++ b/testing/go/ssl_test.go @@ -18,11 +18,16 @@ import ( func TestSSL(t *testing.T) { port := GetUnusedPort(t) server.DefaultProtocolListenerFunc = dserver.NewLimitedListener - code, _ := dserver.RunInMemory([]string{fmt.Sprintf("--port=%d", port), "--host=127.0.0.1"}) - require.Equal(t, 0, *code) + controller, err := dserver.RunInMemory([]string{fmt.Sprintf("--port=%d", port), "--host=127.0.0.1"}) + require.NoError(t, err) + defer func() { + controller.Stop() + require.NoError(t, controller.WaitForStop()) + }() + ctx := context.Background() - err := func() error { + err = func() error { // The connection attempt may be made before the server has grabbed the port, so we'll retry the first // connection a few times. var conn *pgx.Conn From d6bbf039cb9f09e793c7b49d120fbe3978eee7bb Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Thu, 7 Dec 2023 10:58:41 -0800 Subject: [PATCH 04/23] typo --- server/listener.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/listener.go b/server/listener.go index 921d5b5add..a692753753 100644 --- a/server/listener.go +++ b/server/listener.go @@ -96,7 +96,7 @@ func (l *Listener) HandleConnection(conn net.Conn) { var returnErr error defer func() { if r := recover(); r != nil { - fmt.Printf("Linener recovered panic: %v", r) + fmt.Printf("Listener recovered panic: %v", r) } if returnErr != nil { fmt.Println(returnErr.Error()) From 6cabd3595847470c1e6a427ba91b0bb812a33079 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Thu, 7 Dec 2023 11:40:08 -0800 Subject: [PATCH 05/23] Simplified config setup --- server/server.go | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/server/server.go b/server/server.go index 00f4964e4d..7a100501cc 100644 --- a/server/server.go +++ b/server/server.go @@ -268,23 +268,21 @@ func runServer(args []string, fs filesys.Filesys) (*svcs.Controller, error) { return nil, err } - var serverConfig sqlserver.ServerConfig - // Grab the config so that we can pull the TLS key & cert. If there's an error then we can ignore it here, as - // it'll be caught when the server actually tries to run. We'll also use the config to determine whether we need to - // create a doltgres directory. - if sqlServerApr, err := cli.ParseArgs(sqlserver.SqlServerCmd{}.ArgParser(), args, nil); err == nil { - if serverConfig, err = sqlserver.GetServerConfig(dEnv.FS, sqlServerApr); err == nil { - // We throw an error if there's an issue with the TLS cert though - tlsConfig, err := sqlserver.LoadTLSConfig(serverConfig) - if err != nil { - return nil, err - } - if tlsConfig != nil && len(tlsConfig.Certificates) > 0 { - certificate = tlsConfig.Certificates[0] - } - } + serverConfig, err := sqlserver.ServerConfigFromArgs("sql-server", args, dEnv) + if err != nil { + return nil, err } - // Server mode automatically initializes a doltgres database + + tlsConfig, err := sqlserver.LoadTLSConfig(serverConfig) + if err != nil { + return nil, err + } + + if tlsConfig != nil && len(tlsConfig.Certificates) > 0 { + certificate = tlsConfig.Certificates[0] + } + + // Automatically initializes a doltgres database if necessary if !dEnv.HasDoltDir() { // Need to make sure that there isn't a doltgres subdirectory. If there is, we'll assume it's a db. if exists, _ := dEnv.FS.Exists("doltgres"); !exists { From dd5a2d32d9816c71800e95b0037e9e0164a396df Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Thu, 7 Dec 2023 15:58:01 -0800 Subject: [PATCH 06/23] More cleanup, preparing to move non-server commands into main --- server/server.go | 205 ++++++++++++++++++++++++++--------------------- 1 file changed, 114 insertions(+), 91 deletions(-) diff --git a/server/server.go b/server/server.go index 7a100501cc..326dd3bfae 100644 --- a/server/server.go +++ b/server/server.go @@ -22,14 +22,18 @@ import ( "math/rand" _ "net/http/pprof" "os" + "strconv" + "strings" "github.com/dolthub/dolt/go/cmd/dolt/cli" "github.com/dolthub/dolt/go/cmd/dolt/commands" "github.com/dolthub/dolt/go/cmd/dolt/commands/sqlserver" + eventsapi "github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi/v1alpha1" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dfunctions" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" + "github.com/dolthub/dolt/go/libraries/events" "github.com/dolthub/dolt/go/libraries/utils/argparser" "github.com/dolthub/dolt/go/libraries/utils/config" "github.com/dolthub/dolt/go/libraries/utils/filesys" @@ -42,24 +46,83 @@ import ( "github.com/tidwall/gjson" ) -//TODO: cleanup this file - const ( Version = "0.1.0" ) -var doltCommand = cli.NewSubCommandHandler("doltgresql", "it's git for data", []cli.Command{ - commands.InitCmd{}, - commands.ConfigCmd{}, - commands.VersionCmd{VersionStr: Version}, - sqlserver.SqlServerCmd{VersionStr: Version}, -}) -var globalArgParser = cli.CreateGlobalArgParser("doltgresql") +var sqlServerDocs = cli.CommandDocumentationContent{ + ShortDesc: "Start a PostgreSQL-compatible server.", + LongDesc: "By default, starts a PostgreSQL-compatible server on the dolt database in the current directory. " + + "Databases are named after the directories they appear in" + + "Parameters can be specified using a yaml configuration file passed to the server via " + + "{{.EmphasisLeft}}--config {{.EmphasisRight}}, or by using the supported switches and flags to configure " + + "the server directly on the command line. If {{.EmphasisLeft}}--config {{.EmphasisRight}} is provided all" + + " other command line arguments are ignored.\n\nThis is an example yaml configuration file showing all supported" + + " items and their default values:\n\n" + + indentLines(sqlserver.ServerConfigAsYAMLConfig(sqlserver.DefaultServerConfig()).String()) + "\n\n" + ` +SUPPORTED CONFIG FILE FIELDS: + +{{.EmphasisLeft}}data_dir{{.EmphasisRight}}: A directory where the server will load dolt databases to serve, and create new ones. Defaults to the current directory. + +{{.EmphasisLeft}}cfg_dir{{.EmphasisRight}}: A directory where the server will load and store non-database configuration data, such as permission information. Defaults {{.EmphasisLeft}}$data_dir/.doltcfg{{.EmphasisRight}}. + +{{.EmphasisLeft}}log_level{{.EmphasisRight}}: Level of logging provided. Options are: {{.EmphasisLeft}}trace{{.EmphasisRight}}, {{.EmphasisLeft}}debug{{.EmphasisRight}}, {{.EmphasisLeft}}info{{.EmphasisRight}}, {{.EmphasisLeft}}warning{{.EmphasisRight}}, {{.EmphasisLeft}}error{{.EmphasisRight}}, and {{.EmphasisLeft}}fatal{{.EmphasisRight}}. + +{{.EmphasisLeft}}privilege_file{{.EmphasisRight}}: "Path to a file to load and store users and grants. Defaults to {{.EmphasisLeft}}$doltcfg-dir/privileges.db{{.EmphasisRight}}. Will be created as needed. + +{{.EmphasisLeft}}branch_control_file{{.EmphasisRight}}: Path to a file to load and store branch control permissions. Defaults to {{.EmphasisLeft}}$doltcfg-dir/branch_control.db{{.EmphasisRight}}. Will be created as needed. + +{{.EmphasisLeft}}max_logged_query_len{{.EmphasisRight}}: If greater than zero, truncates query strings in logging to the number of characters given. + +{{.EmphasisLeft}}behavior.read_only{{.EmphasisRight}}: If true database modification is disabled. Defaults to false. + +{{.EmphasisLeft}}behavior.autocommit{{.EmphasisRight}}: If true every statement is committed automatically. Defaults to true. @@autocommit can also be specified in each session. + +{{.EmphasisLeft}}behavior.dolt_transaction_commit{{.EmphasisRight}}: If true all SQL transaction commits will automatically create a Dolt commit, with a generated commit message. This is useful when a system working with Dolt wants to create versioned data, but doesn't want to directly use Dolt features such as dolt_commit(). + +{{.EmphasisLeft}}user.name{{.EmphasisRight}}: The username that connections should use for authentication + +{{.EmphasisLeft}}user.password{{.EmphasisRight}}: The password that connections should use for authentication. + +{{.EmphasisLeft}}listener.host{{.EmphasisRight}}: The host address that the server will run on. This may be {{.EmphasisLeft}}localhost{{.EmphasisRight}} or an IPv4 or IPv6 address + +{{.EmphasisLeft}}listener.port{{.EmphasisRight}}: The port that the server should listen on + +{{.EmphasisLeft}}listener.max_connections{{.EmphasisRight}}: The number of simultaneous connections that the server will accept + +{{.EmphasisLeft}}listener.read_timeout_millis{{.EmphasisRight}}: The number of milliseconds that the server will wait for a read operation + +{{.EmphasisLeft}}listener.write_timeout_millis{{.EmphasisRight}}: The number of milliseconds that the server will wait for a write operation + +{{.EmphasisLeft}}remotesapi.port{{.EmphasisRight}}: A port to listen for remote API operations on. If set to a positive integer, this server will accept connections from clients to clone, pull, etc. databases being served. + +{{.EmphasisLeft}}user_session_vars{{.EmphasisRight}}: A map of user name to a map of session variables to set on connection for each session. + +{{.EmphasisLeft}}cluster{{.EmphasisRight}}: Settings related to running this server in a replicated cluster. For information on setting these values, see https://docs.dolthub.com/sql-reference/server/replication + +If a config file is not provided many of these settings may be configured on the command line.`, + Synopsis: []string{ + "--config {{.LessThan}}file{{.GreaterThan}}", + "[-H {{.LessThan}}host{{.GreaterThan}}] [-P {{.LessThan}}port{{.GreaterThan}}] [-u {{.LessThan}}user{{.GreaterThan}}] [-p {{.LessThan}}password{{.GreaterThan}}] [-t {{.LessThan}}timeout{{.GreaterThan}}] [-l {{.LessThan}}loglevel{{.GreaterThan}}] [--data-dir {{.LessThan}}directory{{.GreaterThan}}] [-r]", + }, +} + +func indentLines(s string) string { + sb := strings.Builder{} + lines := strings.Split(s, "\n") + for _, line := range lines { + sb.WriteRune('\t') + sb.WriteString(line) + sb.WriteRune('\n') + } + return sb.String() +} func init() { server.DefaultProtocolListenerFunc = NewListener sqlserver.ExternalDisableUsers = true dfunctions.VersionString = Version + events.Application = eventsapi.AppID_APP_DOLTGRES } const chdirFlag = "--chdir" @@ -161,31 +224,15 @@ func runServer(args []string, fs filesys.Filesys) (*svcs.Controller, error) { warnIfMaxFilesTooLow() dEnv := env.Load(ctx, env.GetCurrentUserHomeDir, fs, doltdb.LocalDirDoltDB, Version) - - globalConfig, ok := dEnv.Config.GetConfig(env.GlobalConfig) - if !ok { - return nil, fmt.Errorf("Failed to get global config") - } - // The in-memory database is only used for testing/virtual environments, so the config may need modification - if _, ok = fs.(*filesys.InMemFS); ok && globalConfig.GetStringOrDefault(config.UserNameKey, "") == "" { - globalConfig.SetStrings(map[string]string{ - config.UserNameKey: "postgres", - config.UserEmailKey: "postgres@somewhere.com", - }) - } - - apr, _, _, err := parseGlobalArgsAndSubCommandName(globalConfig, append([]string {"sql-server"}, args...)) - if err == argparser.ErrHelp { - //TODO: display some help message - return nil, fmt.Errorf("help") - } else if err != nil { - return nil, fmt.Errorf("failed to parse arguments: %w", err) - } - + + ap := sqlserver.SqlServerCmd{}.ArgParser() + help, _ := cli.HelpAndUsagePrinters(cli.CommandDocsForCommandString("doltgres", sqlServerDocs, ap)) + apr := cli.ParseArgsOrDie(ap, args, help) + dataDir, hasDataDir := apr.GetValue(commands.DataDirFlag) if hasDataDir { // If a relative path was provided, this ensures we have an absolute path everywhere. - dataDir, err = fs.Abs(dataDir) + dataDir, err := fs.Abs(dataDir) if err != nil { return nil, fmt.Errorf("failed to get absolute path for %s: %w", dataDir, err) } @@ -203,7 +250,7 @@ func runServer(args []string, fs filesys.Filesys) (*svcs.Controller, error) { "To use the current directory as a database, start the server from the parent directory.") } - err = reconfigIfTempFileMoveFails(dEnv) + err := reconfigIfTempFileMoveFails(dEnv) if err != nil { return nil, fmt.Errorf("failed to set up the temporary directory: %w", err) } @@ -282,7 +329,7 @@ func runServer(args []string, fs filesys.Filesys) (*svcs.Controller, error) { certificate = tlsConfig.Certificates[0] } - // Automatically initializes a doltgres database if necessary + // Automatically initialize a doltgres database if necessary if !dEnv.HasDoltDir() { // Need to make sure that there isn't a doltgres subdirectory. If there is, we'll assume it's a db. if exists, _ := dEnv.FS.Exists("doltgres"); !exists { @@ -296,9 +343,18 @@ func runServer(args []string, fs filesys.Filesys) (*svcs.Controller, error) { } // We'll use a temporary environment to instantiate the subdirectory tempDEnv := env.Load(ctx, env.GetCurrentUserHomeDir, subdirectoryFS, doltdb.LocalDirDoltDB, Version) - _ = doltCommand.Exec(ctx, "dolt", []string{"init"}, tempDEnv, cliCtx) + res := commands.InitCmd{}.Exec(ctx, "init", []string{}, tempDEnv, cliCtx) + if res != 0 { + return nil, fmt.Errorf("failed to initialize doltgres database") + } } } + + // If we got this far, emit a usage event in the background while we launch the server + // Dolt is more permissive with events: it emits events even if the command fails in earliest phase. + // We'll also emit a heartbeat event every 24 hours the server is running. All events will be tagged with the + // doltgresql app id. + go emitUsageEvent(dEnv) controller := svcs.NewController() newCtx, cancelF := context.WithCancel(ctx) @@ -317,13 +373,36 @@ func runServer(args []string, fs filesys.Filesys) (*svcs.Controller, error) { }() // We need a username and password for many SQL commands, so set defaults if they don't exist - dEnv.Config.SetFailsafes(env.DefaultFailsafeConfig) + dEnv.Config.SetFailsafes(map[string]string{ + config.UserNameKey: "postgres", + config.UserEmailKey: "postgres@somewhere.com", + }) sqlserver.ConfigureServices(serverConfig, controller, Version, dEnv) go controller.Start(newCtx) return controller, controller.WaitForStart() } +func emitUsageEvent(dEnv *env.DoltEnv) { + metricsDisabled := dEnv.Config.GetStringOrDefault(config.MetricsDisabled, "false") + disabled, err := strconv.ParseBool(metricsDisabled) + if err != nil || disabled { + return + } + + evt := events.NewEvent(sqlserver.SqlServerCmd{}.EventType()) + evtCollector := events.NewCollector() + evtCollector.CloseEventAndAdd(evt) + clientEvents := evtCollector.Close() + + emitter, err := commands.GRPCEmitterForConfig(dEnv) + if err != nil { + return + } + + err = emitter.LogEvents(Version, clientEvents) +} + // buildLateBinder builds a LateBindQueryist for which is used to obtain the Queryist used for the length of the // command execution. func buildLateBinder(ctx context.Context, cwdFS filesys.Filesys, rootEnv *env.DoltEnv, mrEnv *env.MultiRepoEnv, creds *cli.UserPassword, apr *argparser.ArgParseResults, subcommandName string, verbose bool) (cli.LateBindQueryist, error) { @@ -409,62 +488,6 @@ func seedGlobalRand() { rand.Seed(int64(binary.LittleEndian.Uint64(bs))) } -// parseGlobalArgsAndSubCommandName parses the global arguments, including a profile if given or a default profile if exists. Also returns the subcommand name. -func parseGlobalArgsAndSubCommandName(globalConfig config.ReadWriteConfig, args []string) (apr *argparser.ArgParseResults, remaining []string, subcommandName string, err error) { - apr, remaining, err = globalArgParser.ParseGlobalArgs(args) - if err != nil { - return nil, nil, "", err - } - - subcommandName = remaining[0] - - useDefaultProfile := false - profileName, hasProfile := apr.GetValue(commands.ProfileFlag) - encodedProfiles, err := globalConfig.GetString(commands.GlobalCfgProfileKey) - if err != nil { - if err == config.ErrConfigParamNotFound { - if hasProfile { - return nil, nil, "", fmt.Errorf("no profiles found") - } else { - return apr, remaining, subcommandName, nil - } - } else { - return nil, nil, "", err - } - } - profiles, err := commands.DecodeProfile(encodedProfiles) - if err != nil { - return nil, nil, "", err - } - - if !hasProfile { - defaultProfile := gjson.Get(profiles, commands.DefaultProfileName) - if defaultProfile.Exists() { - args = append([]string{"--profile", commands.DefaultProfileName}, args...) - apr, remaining, err = globalArgParser.ParseGlobalArgs(args) - if err != nil { - return nil, nil, "", err - } - profileName, _ = apr.GetValue(commands.ProfileFlag) - useDefaultProfile = true - } - } - - if hasProfile || useDefaultProfile { - profileArgs, err := getProfile(apr, profileName, profiles) - if err != nil { - return nil, nil, "", err - } - args = append(profileArgs, args...) - apr, remaining, err = globalArgParser.ParseGlobalArgs(args) - if err != nil { - return nil, nil, "", err - } - } - - return -} - // getProfile retrieves the given profile from the provided list of profiles and returns the args (as flags) and values // for that profile in a []string. If the profile is not found, an error is returned. func getProfile(apr *argparser.ArgParseResults, profileName, profiles string) (result []string, err error) { From 186849e568b639f75be8bfec7af05c8c1e8cf087 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Thu, 7 Dec 2023 16:38:24 -0800 Subject: [PATCH 07/23] Moving more things out of server package into main --- server/fileno_check.go => fileno_check.go | 0 ..._check_darwin.go => fileno_check_darwin.go | 0 ...no_check_linux.go => fileno_check_linux.go | 0 server/server.go | 77 ++----------------- 4 files changed, 8 insertions(+), 69 deletions(-) rename server/fileno_check.go => fileno_check.go (100%) rename server/fileno_check_darwin.go => fileno_check_darwin.go (100%) rename server/fileno_check_linux.go => fileno_check_linux.go (100%) diff --git a/server/fileno_check.go b/fileno_check.go similarity index 100% rename from server/fileno_check.go rename to fileno_check.go diff --git a/server/fileno_check_darwin.go b/fileno_check_darwin.go similarity index 100% rename from server/fileno_check_darwin.go rename to fileno_check_darwin.go diff --git a/server/fileno_check_linux.go b/fileno_check_linux.go similarity index 100% rename from server/fileno_check_linux.go rename to fileno_check_linux.go diff --git a/server/server.go b/server/server.go index 326dd3bfae..a8c155f3f6 100644 --- a/server/server.go +++ b/server/server.go @@ -16,10 +16,7 @@ package server import ( "context" - crand "crypto/rand" - "encoding/binary" "fmt" - "math/rand" _ "net/http/pprof" "os" "strconv" @@ -43,7 +40,6 @@ import ( "github.com/dolthub/go-mysql-server/server" "github.com/dolthub/go-mysql-server/sql" "github.com/fatih/color" - "github.com/tidwall/gjson" ) const ( @@ -133,21 +129,22 @@ const stdOutAndErrFlag = "--out-and-err" // RunOnDisk starts the server based on the given args, while also using the local disk as the backing store. // The returned WaitGroup may be used to wait for the server to close. -func RunOnDisk(args []string) (*svcs.Controller, error) { - return runServer(args, filesys.LocalFS) +func RunOnDisk(ctx context.Context, args []string, dEnv *env.DoltEnv) (*svcs.Controller, error) { + return runServer(ctx, args, filesys.LocalFS, dEnv) } // RunInMemory starts the server based on the given args, while also using RAM as the backing store. // The returned WaitGroup may be used to wait for the server to close. func RunInMemory(args []string) (*svcs.Controller, error) { - return runServer(args, filesys.EmptyInMemFS("")) + ctx := context.Background() + fs := filesys.EmptyInMemFS("") + dEnv := env.Load(ctx, env.GetCurrentUserHomeDir, fs, doltdb.LocalDirDoltDB, Version) + return runServer(ctx, args, fs, dEnv) } // runServer starts the server based on the given args, using the provided file system as the backing store. // The returned WaitGroup may be used to wait for the server to close. -func runServer(args []string, fs filesys.Filesys) (*svcs.Controller, error) { - ctx := context.Background() - +func runServer(ctx context.Context, args []string, fs filesys.Filesys, dEnv *env.DoltEnv) (*svcs.Controller, error) { if serverArgs, err := (sqlserver.SqlServerCmd{}).ArgParser().Parse(append([]string{"sql-server"}, args...)); err == nil { if _, ok := serverArgs.GetValue("port"); !ok { args = append(args, "--port=5432") @@ -215,15 +212,6 @@ func runServer(args []string, fs filesys.Filesys) (*svcs.Controller, error) { } } } - - seedGlobalRand() - - restoreIO := cli.InitIO() - defer restoreIO() - - warnIfMaxFilesTooLow() - - dEnv := env.Load(ctx, env.GetCurrentUserHomeDir, fs, doltdb.LocalDirDoltDB, Version) ap := sqlserver.SqlServerCmd{}.ArgParser() help, _ := cli.HelpAndUsagePrinters(cli.CommandDocsForCommandString("doltgres", sqlServerDocs, ap)) @@ -477,53 +465,4 @@ func buildLateBinder(ctx context.Context, cwdFS filesys.Filesys, rootEnv *env.Do cli.Println("verbose: starting local mode") } return commands.BuildSqlEngineQueryist(ctx, cwdFS, mrEnv, creds, apr) -} - -func seedGlobalRand() { - bs := make([]byte, 8) - _, err := crand.Read(bs) - if err != nil { - panic("failed to initial rand " + err.Error()) - } - rand.Seed(int64(binary.LittleEndian.Uint64(bs))) -} - -// getProfile retrieves the given profile from the provided list of profiles and returns the args (as flags) and values -// for that profile in a []string. If the profile is not found, an error is returned. -func getProfile(apr *argparser.ArgParseResults, profileName, profiles string) (result []string, err error) { - prof := gjson.Get(profiles, profileName) - if prof.Exists() { - hasPassword := false - password := "" - for flag, value := range prof.Map() { - if !apr.Contains(flag) { - if flag == cli.PasswordFlag { - password = value.Str - } else if flag == "has-password" { - hasPassword = value.Bool() - } else if flag == cli.NoTLSFlag { - if value.Bool() { - result = append(result, "--"+flag) - continue - } - } else { - if value.Str != "" { - result = append(result, "--"+flag, value.Str) - } - } - } - } - if !apr.Contains(cli.PasswordFlag) && hasPassword { - result = append(result, "--"+cli.PasswordFlag, password) - } - return result, nil - } else { - return nil, fmt.Errorf("profile %s not found", profileName) - } -} - -func intPointer(val int) *int { - p := new(int) - *p = val - return p -} +} \ No newline at end of file From 3bd2773625288bf531bf5949ad3f34851459a8d9 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Fri, 8 Dec 2023 11:12:21 -0800 Subject: [PATCH 08/23] More refactoring --- fileno_check.go | 2 +- fileno_check_darwin.go | 2 +- fileno_check_linux.go | 2 +- go.mod | 26 ++++--- go.sum | 30 ++------ main.go | 164 ++++++++++++++++++++++++++++++++++++++++- 6 files changed, 187 insertions(+), 39 deletions(-) diff --git a/fileno_check.go b/fileno_check.go index 62dcc8d299..0b9d9f075b 100644 --- a/fileno_check.go +++ b/fileno_check.go @@ -15,7 +15,7 @@ //go:build !linux && !darwin // +build !linux,!darwin -package server +package main func warnIfMaxFilesTooLow() { } diff --git a/fileno_check_darwin.go b/fileno_check_darwin.go index bb93a6709f..78d9370c14 100644 --- a/fileno_check_darwin.go +++ b/fileno_check_darwin.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package server +package main import ( "github.com/fatih/color" diff --git a/fileno_check_linux.go b/fileno_check_linux.go index 5d584892d6..1c06e4e285 100644 --- a/fileno_check_linux.go +++ b/fileno_check_linux.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package server +package main import ( "golang.org/x/sys/unix" diff --git a/go.mod b/go.mod index 89ce9b8462..06d7de127f 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,13 @@ module github.com/dolthub/doltgresql +replace github.com/dolthub/dolt/go => ../dolt/go + +replace github.com/dolthub/go-mysql-server => ../go-mysql-server + +replace github.com/dolthub/vitess => ../vitess + +replace github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi => ../dolt/go/gen/proto/dolt/services/eventsapi + go 1.21 require ( @@ -7,23 +15,25 @@ require ( github.com/cockroachdb/apd/v2 v2.0.3-0.20200518165714-d020e156310a github.com/cockroachdb/errors v1.7.5 github.com/dolthub/dolt/go v0.40.5-0.20231130221002-0d5eeb513092 - github.com/dolthub/go-mysql-server v0.17.1-0.20231130181846-c0a0060c4fe7 + github.com/dolthub/go-mysql-server v0.17.1-0.20231205222834-2eb85072ed9d github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 - github.com/dolthub/vitess v0.0.0-20231127171856-2466012fb61f + github.com/dolthub/vitess v0.0.0-20231207010700-88fb35413580 github.com/fatih/color v1.13.0 github.com/gogo/protobuf v1.3.2 github.com/golang/geo v0.0.0-20200730024412-e86565bf3f35 - github.com/google/go-cmp v0.5.9 + github.com/google/go-cmp v0.6.0 github.com/grpc-ecosystem/grpc-gateway v1.16.0 github.com/jackc/pgx/v4 v4.18.1 github.com/jackc/pgx/v5 v5.4.3 github.com/lib/pq v1.10.2 github.com/madflojo/testcerts v1.1.1 github.com/pierrre/geohash v1.0.0 + github.com/sergi/go-diff v1.1.0 github.com/sirupsen/logrus v1.8.1 - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.8.3 github.com/tidwall/gjson v1.14.4 github.com/twpayne/go-geom v1.3.6 + golang.org/x/net v0.17.0 golang.org/x/sys v0.13.0 golang.org/x/text v0.13.0 ) @@ -44,7 +54,6 @@ require ( github.com/bcicen/jstream v1.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.1.3 // indirect - github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect github.com/cockroachdb/redact v1.0.6 // indirect @@ -59,7 +68,7 @@ require ( github.com/dolthub/jsonpath v0.0.2-0.20230525180605-8dc13778fd72 // indirect github.com/dolthub/maphash v0.0.0-20221220182448-74e1e1ea1577 // indirect github.com/dolthub/swiss v0.1.0 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect github.com/go-kit/kit v0.10.0 // indirect github.com/go-logr/logr v1.2.3 // indirect @@ -95,10 +104,9 @@ require ( github.com/kylelemons/godebug v1.1.0 // indirect github.com/lestrrat-go/strftime v1.0.4 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.16 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/mitchellh/go-ps v1.0.0 // indirect github.com/mitchellh/hashstructure v1.1.0 // indirect github.com/pierrec/lz4/v4 v4.1.6 // indirect github.com/pkg/errors v0.9.1 // indirect @@ -109,7 +117,6 @@ require ( github.com/prometheus/procfs v0.8.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect - github.com/sergi/go-diff v1.1.0 // indirect github.com/shopspring/decimal v1.2.0 // indirect github.com/silvasur/buzhash v0.0.0-20160816060738-9bdec3dec7c6 // indirect github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect @@ -132,7 +139,6 @@ require ( golang.org/x/crypto v0.14.0 // indirect golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect golang.org/x/sync v0.3.0 // indirect golang.org/x/term v0.13.0 // indirect diff --git a/go.sum b/go.sum index c98e74ae1c..80537449f3 100644 --- a/go.sum +++ b/go.sum @@ -75,8 +75,6 @@ github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0 github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= -github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= @@ -155,8 +153,6 @@ github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4r github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= @@ -216,18 +212,12 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dolthub/dolt/go v0.40.5-0.20231130221002-0d5eeb513092 h1:DvCGM7uxHtNe8b6C2vpX71oFhhHf1KoXeb+/y8wBkt8= -github.com/dolthub/dolt/go v0.40.5-0.20231130221002-0d5eeb513092/go.mod h1:PVPL+qOhr6+DPUW0QyHTvoTd0AbYBIRd7Y4ITrhi97s= -github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi v0.0.0-20231018220650-48f565111c6a h1:WYBHmrFuKPqPfLB8ab1WqL5MgR8PQICkfUs5RTjHZM8= -github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi v0.0.0-20231018220650-48f565111c6a/go.mod h1:Fi7KchJVfwMuPJkX4vJeAlNZkxCiVyhvVYfCgaSDlTU= github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 h1:u3PMzfF8RkKd3lB9pZ2bfn0qEG+1Gms9599cr0REMww= github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2/go.mod h1:mIEZOHnFx4ZMQeawhw9rhsj+0zwQj7adVsnBX7t+eKY= github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e h1:kPsT4a47cw1+y/N5SSCkma7FhAPw7KeGmD6c9PBZW9Y= github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168= -github.com/dolthub/go-mysql-server v0.17.1-0.20231130181846-c0a0060c4fe7 h1:aNrWLEmWLvD3vJR14XJ8PhRzRteOOg6/1coHqU5sOR8= -github.com/dolthub/go-mysql-server v0.17.1-0.20231130181846-c0a0060c4fe7/go.mod h1:vXlRKS39WHav9N51VsfYphKhmSA2t5FkhHmW3BtwH5I= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 h1:0HHu0GWJH0N6a6keStrHhUAK5/o9LVfkh44pvsV4514= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto= github.com/dolthub/jsonpath v0.0.2-0.20230525180605-8dc13778fd72 h1:NfWmngMi1CYUWU4Ix8wM+USEhjc+mhPlT9JUR/anvbQ= @@ -238,11 +228,10 @@ github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 h1:7/v8q9X github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81/go.mod h1:siLfyv2c92W1eN/R4QqG/+RjjX5W2+gCTRjZxBjI3TY= github.com/dolthub/swiss v0.1.0 h1:EaGQct3AqeP/MjASHLiH6i4TAmgbG/c4rA6a1bzCOPc= github.com/dolthub/swiss v0.1.0/go.mod h1:BeucyB08Vb1G9tumVN3Vp/pyY4AMUnr9p7Rz7wJ7kAQ= -github.com/dolthub/vitess v0.0.0-20231127171856-2466012fb61f h1:I480LKHhb4usnF3dYhp6J4ORKMrncNKaWYZvIZwlK+U= -github.com/dolthub/vitess v0.0.0-20231127171856-2466012fb61f/go.mod h1:IwjNXSQPymrja5pVqmfnYdcy7Uv7eNJNBPK/MEh9OOw= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= @@ -388,8 +377,8 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= @@ -626,8 +615,9 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= @@ -645,8 +635,6 @@ github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceT github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= -github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/hashstructure v1.1.0 h1:P6P1hdjqAAknpY/M1CGipelZgp+4y9ja9kmUZPXP+H0= @@ -805,8 +793,6 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= @@ -835,8 +821,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tealeg/xlsx v1.0.5 h1:+f8oFmvY8Gw1iUXzPk+kz+4GpbDZPK1FhPiQRd+ypgE= github.com/tealeg/xlsx v1.0.5/go.mod h1:btRS8dz54TDnvKNosuAqxrM1QgN1udgk9O34bDCnORM= github.com/tetratelabs/wazero v1.1.0 h1:EByoAhC+QcYpwSZJSs/aV0uokxPwBgKxfiokSUwAknQ= diff --git a/main.go b/main.go index fc7fa34b71..80c88752a7 100644 --- a/main.go +++ b/main.go @@ -15,25 +15,181 @@ package main import ( + "context" + crand "crypto/rand" + "encoding/binary" "fmt" + "math/rand" "os" + "github.com/dolthub/dolt/go/cmd/dolt/cli" + "github.com/dolthub/dolt/go/cmd/dolt/commands" + "github.com/dolthub/dolt/go/cmd/dolt/commands/sqlserver" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/env" + "github.com/dolthub/dolt/go/libraries/utils/argparser" + "github.com/dolthub/dolt/go/libraries/utils/config" + "github.com/dolthub/dolt/go/libraries/utils/filesys" "github.com/dolthub/doltgresql/server" + "github.com/tidwall/gjson" ) + +var doltgresCommands = cli.NewSubCommandHandler("doltgresql", "it's git for data", []cli.Command{ + commands.ConfigCmd{}, + commands.VersionCmd{VersionStr: server.Version}, + sqlserver.SqlServerCmd{VersionStr: server.Version}, +}) +var globalArgParser = cli.CreateGlobalArgParser("doltgresql") + func main() { - controller, err := server.RunOnDisk(os.Args[1:]) + ctx := context.Background() + seedGlobalRand() + + restoreIO := cli.InitIO() + defer restoreIO() + + warnIfMaxFilesTooLow() + + fs := filesys.LocalFS + dEnv := env.Load(ctx, env.GetCurrentUserHomeDir, fs, doltdb.LocalDirDoltDB, server.Version) + + globalConfig, _ := dEnv.Config.GetConfig(env.GlobalConfig) + apr, args, subCommandName, err := parseGlobalArgsAndSubCommandName(globalConfig, os.Args[1:]) + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + + if subCommandName == "" || subCommandName == "sql-server" { + runServer(ctx, dEnv) + } + + lateBind, err := buildLateBinder(ctx, cwdFS, dEnv, mrEnv, creds, apr, "sql-server", false) + if err != nil { + return nil, err + } + + cliCtx, err := cli.NewCliContext(apr, dEnv.Config, lateBind) + if err != nil { + return nil, err + } + + doltgresCommands.Exec(ctx, "doltgresql", args, dEnv) + + os.Exit(0) +} + +func runServer(ctx context.Context, dEnv *env.DoltEnv) { + controller, err := server.RunOnDisk(ctx, os.Args[1:], dEnv) if err != nil { fmt.Println(err.Error()) os.Exit(1) } - + err = controller.WaitForStop() if err != nil { fmt.Println(err.Error()) os.Exit(1) } - - os.Exit(0) } + +// parseGlobalArgsAndSubCommandName parses the global arguments, including a profile if given or a default profile if exists. Also returns the subcommand name. +func parseGlobalArgsAndSubCommandName(globalConfig config.ReadWriteConfig, args []string) (apr *argparser.ArgParseResults, remaining []string, subcommandName string, err error) { + apr, remaining, err = globalArgParser.ParseGlobalArgs(args) + if err != nil { + return nil, nil, "", err + } + + subcommandName = remaining[0] + + useDefaultProfile := false + profileName, hasProfile := apr.GetValue(commands.ProfileFlag) + encodedProfiles, err := globalConfig.GetString(commands.GlobalCfgProfileKey) + if err != nil { + if err == config.ErrConfigParamNotFound { + if hasProfile { + return nil, nil, "", fmt.Errorf("no profiles found") + } else { + return apr, remaining, subcommandName, nil + } + } else { + return nil, nil, "", err + } + } + profiles, err := commands.DecodeProfile(encodedProfiles) + if err != nil { + return nil, nil, "", err + } + + if !hasProfile { + defaultProfile := gjson.Get(profiles, commands.DefaultProfileName) + if defaultProfile.Exists() { + args = append([]string{"--profile", commands.DefaultProfileName}, args...) + apr, remaining, err = globalArgParser.ParseGlobalArgs(args) + if err != nil { + return nil, nil, "", err + } + profileName, _ = apr.GetValue(commands.ProfileFlag) + useDefaultProfile = true + } + } + + if hasProfile || useDefaultProfile { + profileArgs, err := getProfile(apr, profileName, profiles) + if err != nil { + return nil, nil, "", err + } + args = append(profileArgs, args...) + apr, remaining, err = globalArgParser.ParseGlobalArgs(args) + if err != nil { + return nil, nil, "", err + } + } + + return +} + +// getProfile retrieves the given profile from the provided list of profiles and returns the args (as flags) and values +// for that profile in a []string. If the profile is not found, an error is returned. +func getProfile(apr *argparser.ArgParseResults, profileName, profiles string) (result []string, err error) { + prof := gjson.Get(profiles, profileName) + if prof.Exists() { + hasPassword := false + password := "" + for flag, value := range prof.Map() { + if !apr.Contains(flag) { + if flag == cli.PasswordFlag { + password = value.Str + } else if flag == "has-password" { + hasPassword = value.Bool() + } else if flag == cli.NoTLSFlag { + if value.Bool() { + result = append(result, "--"+flag) + continue + } + } else { + if value.Str != "" { + result = append(result, "--"+flag, value.Str) + } + } + } + } + if !apr.Contains(cli.PasswordFlag) && hasPassword { + result = append(result, "--"+cli.PasswordFlag, password) + } + return result, nil + } else { + return nil, fmt.Errorf("profile %s not found", profileName) + } +} + +func seedGlobalRand() { + bs := make([]byte, 8) + _, err := crand.Read(bs) + if err != nil { + panic("failed to initial rand " + err.Error()) + } + rand.Seed(int64(binary.LittleEndian.Uint64(bs))) +} \ No newline at end of file From eeab3b50530bf099ac5438a27a7747b58f326e4a Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Fri, 8 Dec 2023 15:42:48 -0800 Subject: [PATCH 09/23] Moved a lot of things into main --- main.go | 215 ++++++++++++++++++++++++++++++++-- server/server.go | 291 ++++++++++++----------------------------------- 2 files changed, 279 insertions(+), 227 deletions(-) diff --git a/main.go b/main.go index 80c88752a7..a4be4ee399 100644 --- a/main.go +++ b/main.go @@ -27,10 +27,14 @@ import ( "github.com/dolthub/dolt/go/cmd/dolt/commands/sqlserver" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/env" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" "github.com/dolthub/dolt/go/libraries/utils/argparser" "github.com/dolthub/dolt/go/libraries/utils/config" + "github.com/dolthub/dolt/go/libraries/utils/file" "github.com/dolthub/dolt/go/libraries/utils/filesys" + "github.com/dolthub/dolt/go/store/util/tempfiles" "github.com/dolthub/doltgresql/server" + "github.com/dolthub/go-mysql-server/sql" "github.com/tidwall/gjson" ) @@ -61,27 +65,88 @@ func main() { os.Exit(1) } + cliCtx, err := configureCliCtx(apr, fs, dEnv, err, ctx) + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + + // the sql-server command has special cased logic since we have to wait for the server to stop if subCommandName == "" || subCommandName == "sql-server" { - runServer(ctx, dEnv) + runServer(ctx, dEnv, cliCtx) } - lateBind, err := buildLateBinder(ctx, cwdFS, dEnv, mrEnv, creds, apr, "sql-server", false) + exitCode := doltgresCommands.Exec(ctx, "doltgresql", args, dEnv, cliCtx) + os.Exit(exitCode) +} + +func configureCliCtx(apr *argparser.ArgParseResults, fs filesys.Filesys, dEnv *env.DoltEnv, err error, ctx context.Context) (cli.CliContext, error) { + dataDir, hasDataDir := apr.GetValue(commands.DataDirFlag) + if hasDataDir { + // If a relative path was provided, this ensures we have an absolute path everywhere. + dataDir, err := fs.Abs(dataDir) + if err != nil { + return nil, fmt.Errorf("failed to get absolute path for %s: %w", dataDir, err) + } + if ok, dir := fs.Exists(dataDir); !ok || !dir { + return nil, fmt.Errorf("data directory %s does not exist", dataDir) + } + } + + if dEnv.CfgLoadErr != nil { + return nil, fmt.Errorf("failed to load the global config: %w", dEnv.CfgLoadErr) + } + + if dEnv.HasDoltDataDir() { + return nil, fmt.Errorf("Cannot start a server within a directory containing a Dolt or Doltgres database." + + "To use the current directory as a database, start the server from the parent directory.") + } + + err = reconfigIfTempFileMoveFails(dEnv) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to set up the temporary directory: %w", err) } - cliCtx, err := cli.NewCliContext(apr, dEnv.Config, lateBind) + defer tempfiles.MovableTempFileProvider.Clean() + + cwdFS := dEnv.FS + dataDirFS, err := dEnv.FS.WithWorkingDir(dataDir) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to set the data directory: %w", err) } + dEnv.FS = dataDirFS - doltgresCommands.Exec(ctx, "doltgresql", args, dEnv) + mrEnv, err := env.MultiEnvForDirectory(ctx, dEnv.Config.WriteableConfig(), dataDirFS, dEnv.Version, dEnv) + if err != nil { + return nil, fmt.Errorf("failed to load database names: %w", err) + } + _ = mrEnv.Iter(func(dbName string, dEnv *env.DoltEnv) (stop bool, err error) { + dsess.DefineSystemVariablesForDB(dbName) + return false, nil + }) - os.Exit(0) + err = dsess.InitPersistedSystemVars(dEnv) + if err != nil { + return nil, fmt.Errorf("failed to load persisted system variables: %w", err) + } + + // validate that --user and --password are set appropriately. + aprAlt, creds, err := cli.BuildUserPasswordPrompt(apr) + apr = aprAlt + if err != nil { + return nil, fmt.Errorf("failed to parse credentials: %w", err) + } + + lateBind, err := buildLateBinder(ctx, cwdFS, dEnv, mrEnv, creds, apr, "sql-server", false) + if err != nil { + return nil, err + } + + return cli.NewCliContext(apr, dEnv.Config, lateBind) } -func runServer(ctx context.Context, dEnv *env.DoltEnv) { - controller, err := server.RunOnDisk(ctx, os.Args[1:], dEnv) +func runServer(ctx context.Context, dEnv *env.DoltEnv, cliCtx cli.CliContext) { + controller, err := server.RunOnDisk(ctx, os.Args[1:], dEnv, cliCtx) if err != nil { fmt.Println(err.Error()) @@ -192,4 +257,134 @@ func seedGlobalRand() { panic("failed to initial rand " + err.Error()) } rand.Seed(int64(binary.LittleEndian.Uint64(bs))) -} \ No newline at end of file +} + +// buildLateBinder builds a LateBindQueryist for which is used to obtain the Queryist used for the length of the +// command execution. +func buildLateBinder(ctx context.Context, cwdFS filesys.Filesys, rootEnv *env.DoltEnv, mrEnv *env.MultiRepoEnv, creds *cli.UserPassword, apr *argparser.ArgParseResults, subcommandName string, verbose bool) (cli.LateBindQueryist, error) { + + var targetEnv *env.DoltEnv = nil + + useDb, hasUseDb := apr.GetValue(commands.UseDbFlag) + useBranch, hasBranch := apr.GetValue(cli.BranchParam) + + if hasUseDb && hasBranch { + dbName, branchNameInDb := dsess.SplitRevisionDbName(useDb) + if len(branchNameInDb) != 0 { + return nil, fmt.Errorf("Ambiguous branch name: %s or %s", branchNameInDb, useBranch) + } + useDb = dbName + "/" + useBranch + } + // If the host flag is given, we are forced to use a remote connection to a server. + host, hasHost := apr.GetValue(cli.HostFlag) + if hasHost { + if !hasUseDb && subcommandName != "sql" { + return nil, fmt.Errorf("The --%s flag requires the additional --%s flag.", cli.HostFlag, commands.UseDbFlag) + } + + port, hasPort := apr.GetInt(cli.PortFlag) + if !hasPort { + port = 3306 + } + useTLS := !apr.Contains(cli.NoTLSFlag) + return sqlserver.BuildConnectionStringQueryist(ctx, cwdFS, creds, apr, host, port, useTLS, useDb) + } else { + _, hasPort := apr.GetInt(cli.PortFlag) + if hasPort { + return nil, fmt.Errorf("The --%s flag is only meaningful with the --%s flag.", cli.PortFlag, cli.HostFlag) + } + } + + if hasUseDb { + dbName, _ := dsess.SplitRevisionDbName(useDb) + targetEnv = mrEnv.GetEnv(dbName) + if targetEnv == nil { + return nil, fmt.Errorf("The provided --use-db %s does not exist.", dbName) + } + } else { + useDb = mrEnv.GetFirstDatabase() + if hasBranch { + useDb += "/" + useBranch + } + } + + if targetEnv == nil && useDb != "" { + targetEnv = mrEnv.GetEnv(useDb) + } + + // There is no target environment detected. This is allowed for a small number of commands. + // We don't expect that number to grow, so we list them here. + // It's also allowed when --help is passed. + // So we defer the error until the caller tries to use the cli.LateBindQueryist + isDoltEnvironmentRequired := subcommandName != "init" && subcommandName != "sql" && subcommandName != "sql-server" && subcommandName != "sql-client" + if targetEnv == nil && isDoltEnvironmentRequired { + return func(ctx context.Context) (cli.Queryist, *sql.Context, func(), error) { + return nil, nil, nil, fmt.Errorf("The current directory is not a valid dolt repository.") + }, nil + } + + // nil targetEnv will happen if the user ran a command in an empty directory or when there is a server running with + // no databases. CLI will try to connect to the server in this case. + if targetEnv == nil { + targetEnv = rootEnv + } + + if verbose { + cli.Println("verbose: starting local mode") + } + return commands.BuildSqlEngineQueryist(ctx, cwdFS, mrEnv, creds, apr) +} + +// If we cannot verify that we can move files for any reason, use a ./.dolt/tmp as the temp dir. +func reconfigIfTempFileMoveFails(dEnv *env.DoltEnv) error { + if !canMoveTempFile() { + tmpDir := "./.dolt/tmp" + + if !dEnv.HasDoltDir() { + tmpDir = "./.tmp" + } + + stat, err := os.Stat(tmpDir) + + if err != nil { + err := os.MkdirAll(tmpDir, os.ModePerm) + + if err != nil { + return fmt.Errorf("failed to create temp dir '%s': %s", tmpDir, err.Error()) + } + } else if !stat.IsDir() { + return fmt.Errorf("attempting to use '%s' as a temp directory, but there exists a file with that name", tmpDir) + } + + tempfiles.MovableTempFileProvider = tempfiles.NewTempFileProviderAt(tmpDir) + } + + return nil +} + +func canMoveTempFile() bool { + const testfile = "./testfile" + + f, err := os.CreateTemp("", "") + + if err != nil { + return false + } + + name := f.Name() + err = f.Close() + + if err != nil { + return false + } + + err = file.Rename(name, testfile) + + if err != nil { + _ = file.Remove(name) + return false + } + + _ = file.Remove(testfile) + return true +} diff --git a/server/server.go b/server/server.go index a8c155f3f6..c09adb51d1 100644 --- a/server/server.go +++ b/server/server.go @@ -31,14 +31,12 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dfunctions" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" "github.com/dolthub/dolt/go/libraries/events" - "github.com/dolthub/dolt/go/libraries/utils/argparser" "github.com/dolthub/dolt/go/libraries/utils/config" "github.com/dolthub/dolt/go/libraries/utils/filesys" "github.com/dolthub/dolt/go/libraries/utils/svcs" "github.com/dolthub/dolt/go/store/nbs" "github.com/dolthub/dolt/go/store/util/tempfiles" "github.com/dolthub/go-mysql-server/server" - "github.com/dolthub/go-mysql-server/sql" "github.com/fatih/color" ) @@ -129,8 +127,8 @@ const stdOutAndErrFlag = "--out-and-err" // RunOnDisk starts the server based on the given args, while also using the local disk as the backing store. // The returned WaitGroup may be used to wait for the server to close. -func RunOnDisk(ctx context.Context, args []string, dEnv *env.DoltEnv) (*svcs.Controller, error) { - return runServer(ctx, args, filesys.LocalFS, dEnv) +func RunOnDisk(ctx context.Context, args []string, dEnv *env.DoltEnv, cliCtx cli.CliContext) (*svcs.Controller, error) { + return runServer(ctx, args, filesys.LocalFS, dEnv, cliCtx) } // RunInMemory starts the server based on the given args, while also using RAM as the backing store. @@ -139,13 +137,14 @@ func RunInMemory(args []string) (*svcs.Controller, error) { ctx := context.Background() fs := filesys.EmptyInMemFS("") dEnv := env.Load(ctx, env.GetCurrentUserHomeDir, fs, doltdb.LocalDirDoltDB, Version) - return runServer(ctx, args, fs, dEnv) + return runServer(ctx, args, fs, dEnv, nil) } // runServer starts the server based on the given args, using the provided file system as the backing store. // The returned WaitGroup may be used to wait for the server to close. -func runServer(ctx context.Context, args []string, fs filesys.Filesys, dEnv *env.DoltEnv) (*svcs.Controller, error) { - if serverArgs, err := (sqlserver.SqlServerCmd{}).ArgParser().Parse(append([]string{"sql-server"}, args...)); err == nil { +func runServer(ctx context.Context, args []string, fs filesys.Filesys, dEnv *env.DoltEnv, cliCtx cli.CliContext) (*svcs.Controller, error) { + sqlServerCmd := sqlserver.SqlServerCmd{} + if serverArgs, err := sqlServerCmd.ArgParser().Parse(append([]string{"sql-server"}, args...)); err == nil { if _, ok := serverArgs.GetValue("port"); !ok { args = append(args, "--port=5432") } @@ -154,81 +153,12 @@ func runServer(ctx context.Context, args []string, fs filesys.Filesys, dEnv *env if os.Getenv("DOLT_VERBOSE_ASSERT_TABLE_FILES_CLOSED") == "" { nbs.TableIndexGCFinalizerWithStackTrace = false } - - if len(args) > 0 { - var doneDebugFlags bool - for !doneDebugFlags && len(args) > 0 { - switch args[0] { - // Currently goland doesn't support running with a different working directory when using go modules. - // This is a hack that allows a different working directory to be set after the application starts using - // chdir=. The syntax is not flexible and must match exactly this. - case chdirFlag: - err := os.Chdir(args[1]) - - if err != nil { - panic(err) - } - - args = args[2:] - - case stdInFlag: - stdInFile := args[1] - cli.Println("Using file contents as stdin:", stdInFile) - - f, err := os.Open(stdInFile) - if err != nil { - return nil, fmt.Errorf("Failed to open %s: %w", stdInFile, err) - } - - os.Stdin = f - args = args[2:] - - case stdOutFlag, stdErrFlag, stdOutAndErrFlag: - filename := args[1] - - f, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, os.ModePerm) - if err != nil { - return nil, fmt.Errorf("Failed to open %s for writing: %w", filename, err) - } - - switch args[0] { - case stdOutFlag: - cli.Println("Stdout being written to", filename) - cli.CliOut = f - case stdErrFlag: - cli.Println("Stderr being written to", filename) - cli.CliErr = f - case stdOutAndErrFlag: - cli.Println("Stdout and Stderr being written to", filename) - cli.CliOut = f - cli.CliErr = f - } - - color.NoColor = true - args = args[2:] - - default: - doneDebugFlags = true - } - } - } - - ap := sqlserver.SqlServerCmd{}.ArgParser() - help, _ := cli.HelpAndUsagePrinters(cli.CommandDocsForCommandString("doltgres", sqlServerDocs, ap)) - apr := cli.ParseArgsOrDie(ap, args, help) - dataDir, hasDataDir := apr.GetValue(commands.DataDirFlag) - if hasDataDir { - // If a relative path was provided, this ensures we have an absolute path everywhere. - dataDir, err := fs.Abs(dataDir) - if err != nil { - return nil, fmt.Errorf("failed to get absolute path for %s: %w", dataDir, err) - } - if ok, dir := fs.Exists(dataDir); !ok || !dir { - return nil, fmt.Errorf("data directory %s does not exist", dataDir) - } + args, err := redirectStdio(args) + if err != nil { + return nil, err } - + if dEnv.CfgLoadErr != nil { return nil, fmt.Errorf("failed to load the global config: %w", dEnv.CfgLoadErr) } @@ -238,71 +168,13 @@ func runServer(ctx context.Context, args []string, fs filesys.Filesys, dEnv *env "To use the current directory as a database, start the server from the parent directory.") } - err := reconfigIfTempFileMoveFails(dEnv) - if err != nil { - return nil, fmt.Errorf("failed to set up the temporary directory: %w", err) - } - defer tempfiles.MovableTempFileProvider.Clean() - // Find all database names and add global variables for them. This needs to - // occur before a call to dsess.InitPersistedSystemVars. Otherwise, database - // specific persisted system vars will fail to load. - // - // In general, there is a lot of work TODO in this area. System global - // variables are persisted to the Dolt local config if found and if not - // found the Dolt global config (typically ~/.dolt/config_global.json). - - // Depending on what directory a dolt sql-server is started in, users may - // see different variables values. For example, start a dolt sql-server in - // the dolt database folder and persist some system variable. - - // If dolt sql-server is started outside that folder, those system variables - // will be lost. This is particularly confusing for database specific system - // variables like `${db_name}_default_branch` (maybe these should not be - // part of Dolt config in the first place!). - - // Current working directory is preserved to ensure that user provided path arguments are always calculated - // relative to this directory. The root environment's FS will be updated to be the --data-dir path if the user - // specified one. - cwdFS := dEnv.FS - dataDirFS, err := dEnv.FS.WithWorkingDir(dataDir) - if err != nil { - return nil, fmt.Errorf("failed to set the data directory: %w", err) - } - dEnv.FS = dataDirFS - - mrEnv, err := env.MultiEnvForDirectory(ctx, dEnv.Config.WriteableConfig(), dataDirFS, dEnv.Version, dEnv) - if err != nil { - return nil, fmt.Errorf("failed to load database names: %w", err) - } - _ = mrEnv.Iter(func(dbName string, dEnv *env.DoltEnv) (stop bool, err error) { - dsess.DefineSystemVariablesForDB(dbName) - return false, nil - }) - err = dsess.InitPersistedSystemVars(dEnv) if err != nil { return nil, fmt.Errorf("failed to load persisted system variables: %w", err) } - // validate that --user and --password are set appropriately. - aprAlt, creds, err := cli.BuildUserPasswordPrompt(apr) - apr = aprAlt - if err != nil { - return nil, fmt.Errorf("failed to parse credentials: %w", err) - } - - lateBind, err := buildLateBinder(ctx, cwdFS, dEnv, mrEnv, creds, apr, "sql-server", false) - if err != nil { - return nil, err - } - - cliCtx, err := cli.NewCliContext(apr, dEnv.Config, lateBind) - if err != nil { - return nil, err - } - serverConfig, err := sqlserver.ServerConfigFromArgs("sql-server", args, dEnv) if err != nil { return nil, err @@ -312,7 +184,7 @@ func runServer(ctx context.Context, args []string, fs filesys.Filesys, dEnv *env if err != nil { return nil, err } - + if tlsConfig != nil && len(tlsConfig.Certificates) > 0 { certificate = tlsConfig.Certificates[0] } @@ -337,7 +209,7 @@ func runServer(ctx context.Context, args []string, fs filesys.Filesys, dEnv *env } } } - + // If we got this far, emit a usage event in the background while we launch the server // Dolt is more permissive with events: it emits events even if the command fails in earliest phase. // We'll also emit a heartbeat event every 24 hours the server is running. All events will be tagged with the @@ -365,12 +237,73 @@ func runServer(ctx context.Context, args []string, fs filesys.Filesys, dEnv *env config.UserNameKey: "postgres", config.UserEmailKey: "postgres@somewhere.com", }) - + sqlserver.ConfigureServices(serverConfig, controller, Version, dEnv) go controller.Start(newCtx) return controller, controller.WaitForStart() } +func redirectStdio(args []string) ([]string, error) { + if len(args) > 0 { + var doneDebugFlags bool + for !doneDebugFlags && len(args) > 0 { + switch args[0] { + // Currently goland doesn't support running with a different working directory when using go modules. + // This is a hack that allows a different working directory to be set after the application starts using + // chdir=. The syntax is not flexible and must match exactly this. + case chdirFlag: + err := os.Chdir(args[1]) + + if err != nil { + panic(err) + } + + args = args[2:] + + case stdInFlag: + stdInFile := args[1] + cli.Println("Using file contents as stdin:", stdInFile) + + f, err := os.Open(stdInFile) + if err != nil { + return nil, fmt.Errorf("Failed to open %s: %w", stdInFile, err) + } + + os.Stdin = f + args = args[2:] + + case stdOutFlag, stdErrFlag, stdOutAndErrFlag: + filename := args[1] + + f, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, os.ModePerm) + if err != nil { + return nil, fmt.Errorf("Failed to open %s for writing: %w", filename, err) + } + + switch args[0] { + case stdOutFlag: + cli.Println("Stdout being written to", filename) + cli.CliOut = f + case stdErrFlag: + cli.Println("Stderr being written to", filename) + cli.CliErr = f + case stdOutAndErrFlag: + cli.Println("Stdout and Stderr being written to", filename) + cli.CliOut = f + cli.CliErr = f + } + + color.NoColor = true + args = args[2:] + + default: + doneDebugFlags = true + } + } + } + return args, nil +} + func emitUsageEvent(dEnv *env.DoltEnv) { metricsDisabled := dEnv.Config.GetStringOrDefault(config.MetricsDisabled, "false") disabled, err := strconv.ParseBool(metricsDisabled) @@ -389,80 +322,4 @@ func emitUsageEvent(dEnv *env.DoltEnv) { } err = emitter.LogEvents(Version, clientEvents) -} - -// buildLateBinder builds a LateBindQueryist for which is used to obtain the Queryist used for the length of the -// command execution. -func buildLateBinder(ctx context.Context, cwdFS filesys.Filesys, rootEnv *env.DoltEnv, mrEnv *env.MultiRepoEnv, creds *cli.UserPassword, apr *argparser.ArgParseResults, subcommandName string, verbose bool) (cli.LateBindQueryist, error) { - - var targetEnv *env.DoltEnv = nil - - useDb, hasUseDb := apr.GetValue(commands.UseDbFlag) - useBranch, hasBranch := apr.GetValue(cli.BranchParam) - - if hasUseDb && hasBranch { - dbName, branchNameInDb := dsess.SplitRevisionDbName(useDb) - if len(branchNameInDb) != 0 { - return nil, fmt.Errorf("Ambiguous branch name: %s or %s", branchNameInDb, useBranch) - } - useDb = dbName + "/" + useBranch - } - // If the host flag is given, we are forced to use a remote connection to a server. - host, hasHost := apr.GetValue(cli.HostFlag) - if hasHost { - if !hasUseDb && subcommandName != "sql" { - return nil, fmt.Errorf("The --%s flag requires the additional --%s flag.", cli.HostFlag, commands.UseDbFlag) - } - - port, hasPort := apr.GetInt(cli.PortFlag) - if !hasPort { - port = 3306 - } - useTLS := !apr.Contains(cli.NoTLSFlag) - return sqlserver.BuildConnectionStringQueryist(ctx, cwdFS, creds, apr, host, port, useTLS, useDb) - } else { - _, hasPort := apr.GetInt(cli.PortFlag) - if hasPort { - return nil, fmt.Errorf("The --%s flag is only meaningful with the --%s flag.", cli.PortFlag, cli.HostFlag) - } - } - - if hasUseDb { - dbName, _ := dsess.SplitRevisionDbName(useDb) - targetEnv = mrEnv.GetEnv(dbName) - if targetEnv == nil { - return nil, fmt.Errorf("The provided --use-db %s does not exist.", dbName) - } - } else { - useDb = mrEnv.GetFirstDatabase() - if hasBranch { - useDb += "/" + useBranch - } - } - - if targetEnv == nil && useDb != "" { - targetEnv = mrEnv.GetEnv(useDb) - } - - // There is no target environment detected. This is allowed for a small number of commands. - // We don't expect that number to grow, so we list them here. - // It's also allowed when --help is passed. - // So we defer the error until the caller tries to use the cli.LateBindQueryist - isDoltEnvironmentRequired := subcommandName != "init" && subcommandName != "sql" && subcommandName != "sql-server" && subcommandName != "sql-client" - if targetEnv == nil && isDoltEnvironmentRequired { - return func(ctx context.Context) (cli.Queryist, *sql.Context, func(), error) { - return nil, nil, nil, fmt.Errorf("The current directory is not a valid dolt repository.") - }, nil - } - - // nil targetEnv will happen if the user ran a command in an empty directory or when there is a server running with - // no databases. CLI will try to connect to the server in this case. - if targetEnv == nil { - targetEnv = rootEnv - } - - if verbose { - cli.Println("verbose: starting local mode") - } - return commands.BuildSqlEngineQueryist(ctx, cwdFS, mrEnv, creds, apr) } \ No newline at end of file From 1523e6df4b53da34ff9aba37447a8326aada3223 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Fri, 8 Dec 2023 15:53:41 -0800 Subject: [PATCH 10/23] Got rid of cliCtx --- main.go | 14 +++++++------- server/server.go | 10 +++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/main.go b/main.go index a4be4ee399..5d3a63e982 100644 --- a/main.go +++ b/main.go @@ -64,6 +64,11 @@ func main() { fmt.Println(err.Error()) os.Exit(1) } + + // the sql-server command has special cased logic since we have to wait for the server to stop + if subCommandName == "" || subCommandName == "sql-server" { + runServer(ctx, dEnv) + } cliCtx, err := configureCliCtx(apr, fs, dEnv, err, ctx) if err != nil { @@ -71,11 +76,6 @@ func main() { os.Exit(1) } - // the sql-server command has special cased logic since we have to wait for the server to stop - if subCommandName == "" || subCommandName == "sql-server" { - runServer(ctx, dEnv, cliCtx) - } - exitCode := doltgresCommands.Exec(ctx, "doltgresql", args, dEnv, cliCtx) os.Exit(exitCode) } @@ -145,8 +145,8 @@ func configureCliCtx(apr *argparser.ArgParseResults, fs filesys.Filesys, dEnv *e return cli.NewCliContext(apr, dEnv.Config, lateBind) } -func runServer(ctx context.Context, dEnv *env.DoltEnv, cliCtx cli.CliContext) { - controller, err := server.RunOnDisk(ctx, os.Args[1:], dEnv, cliCtx) +func runServer(ctx context.Context, dEnv *env.DoltEnv) { + controller, err := server.RunOnDisk(ctx, os.Args[1:], dEnv) if err != nil { fmt.Println(err.Error()) diff --git a/server/server.go b/server/server.go index c09adb51d1..8526801fb3 100644 --- a/server/server.go +++ b/server/server.go @@ -127,8 +127,8 @@ const stdOutAndErrFlag = "--out-and-err" // RunOnDisk starts the server based on the given args, while also using the local disk as the backing store. // The returned WaitGroup may be used to wait for the server to close. -func RunOnDisk(ctx context.Context, args []string, dEnv *env.DoltEnv, cliCtx cli.CliContext) (*svcs.Controller, error) { - return runServer(ctx, args, filesys.LocalFS, dEnv, cliCtx) +func RunOnDisk(ctx context.Context, args []string, dEnv *env.DoltEnv) (*svcs.Controller, error) { + return runServer(ctx, args, dEnv) } // RunInMemory starts the server based on the given args, while also using RAM as the backing store. @@ -137,12 +137,12 @@ func RunInMemory(args []string) (*svcs.Controller, error) { ctx := context.Background() fs := filesys.EmptyInMemFS("") dEnv := env.Load(ctx, env.GetCurrentUserHomeDir, fs, doltdb.LocalDirDoltDB, Version) - return runServer(ctx, args, fs, dEnv, nil) + return runServer(ctx, args, dEnv) } // runServer starts the server based on the given args, using the provided file system as the backing store. // The returned WaitGroup may be used to wait for the server to close. -func runServer(ctx context.Context, args []string, fs filesys.Filesys, dEnv *env.DoltEnv, cliCtx cli.CliContext) (*svcs.Controller, error) { +func runServer(ctx context.Context, args []string, dEnv *env.DoltEnv) (*svcs.Controller, error) { sqlServerCmd := sqlserver.SqlServerCmd{} if serverArgs, err := sqlServerCmd.ArgParser().Parse(append([]string{"sql-server"}, args...)); err == nil { if _, ok := serverArgs.GetValue("port"); !ok { @@ -203,7 +203,7 @@ func runServer(ctx context.Context, args []string, fs filesys.Filesys, dEnv *env } // We'll use a temporary environment to instantiate the subdirectory tempDEnv := env.Load(ctx, env.GetCurrentUserHomeDir, subdirectoryFS, doltdb.LocalDirDoltDB, Version) - res := commands.InitCmd{}.Exec(ctx, "init", []string{}, tempDEnv, cliCtx) + res := commands.InitCmd{}.Exec(ctx, "init", []string{}, tempDEnv, nil) if res != 0 { return nil, fmt.Errorf("failed to initialize doltgres database") } From 28a245c50fa7e90287aeb21b2d4a9a9c54b2e248 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Fri, 8 Dec 2023 16:30:33 -0800 Subject: [PATCH 11/23] Bug fix --- main.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index 5d3a63e982..a5fdd04eb7 100644 --- a/main.go +++ b/main.go @@ -21,6 +21,7 @@ import ( "fmt" "math/rand" "os" + "strings" "github.com/dolthub/dolt/go/cmd/dolt/cli" "github.com/dolthub/dolt/go/cmd/dolt/commands" @@ -59,9 +60,16 @@ func main() { dEnv := env.Load(ctx, env.GetCurrentUserHomeDir, fs, doltdb.LocalDirDoltDB, server.Version) globalConfig, _ := dEnv.Config.GetConfig(env.GlobalConfig) - apr, args, subCommandName, err := parseGlobalArgsAndSubCommandName(globalConfig, os.Args[1:]) + + // Inject the "sql-server" command if no other commands were given + args := os.Args[1:] + if len(args) == 0 || (len(args) > 0 && strings.HasPrefix(args[0], "-")) { + args = append([]string{"sql-server"}, args...) + } + + apr, args, subCommandName, err := parseGlobalArgsAndSubCommandName(globalConfig, args) if err != nil { - fmt.Println(err.Error()) + cli.PrintErrln(err.Error()) os.Exit(1) } @@ -72,7 +80,7 @@ func main() { cliCtx, err := configureCliCtx(apr, fs, dEnv, err, ctx) if err != nil { - fmt.Println(err.Error()) + cli.PrintErrln(err.Error()) os.Exit(1) } From 6ca2e0797efa3306a0e75e7b4397b0e50f9c313f Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Fri, 8 Dec 2023 16:36:11 -0800 Subject: [PATCH 12/23] Moved failsafe earlier --- server/server.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/server/server.go b/server/server.go index 8526801fb3..89d5db10a1 100644 --- a/server/server.go +++ b/server/server.go @@ -189,6 +189,12 @@ func runServer(ctx context.Context, args []string, dEnv *env.DoltEnv) (*svcs.Con certificate = tlsConfig.Certificates[0] } + // We need a username and password for many SQL commands, so set defaults if they don't exist + dEnv.Config.SetFailsafes(map[string]string{ + config.UserNameKey: "postgres", + config.UserEmailKey: "postgres@somewhere.com", + }) + // Automatically initialize a doltgres database if necessary if !dEnv.HasDoltDir() { // Need to make sure that there isn't a doltgres subdirectory. If there is, we'll assume it's a db. @@ -231,13 +237,7 @@ func runServer(ctx context.Context, args []string, dEnv *env.DoltEnv) (*svcs.Con cancelF() } }() - - // We need a username and password for many SQL commands, so set defaults if they don't exist - dEnv.Config.SetFailsafes(map[string]string{ - config.UserNameKey: "postgres", - config.UserEmailKey: "postgres@somewhere.com", - }) - + sqlserver.ConfigureServices(serverConfig, controller, Version, dEnv) go controller.Start(newCtx) return controller, controller.WaitForStart() From f75a58d495a2b09efbcc3a8ca00af57c8ace8eb0 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Wed, 13 Dec 2023 13:37:22 -0800 Subject: [PATCH 13/23] Partial bug fix for running in mem --- server/server.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/server/server.go b/server/server.go index 89d5db10a1..c7d47bc74f 100644 --- a/server/server.go +++ b/server/server.go @@ -136,7 +136,15 @@ func RunOnDisk(ctx context.Context, args []string, dEnv *env.DoltEnv) (*svcs.Con func RunInMemory(args []string) (*svcs.Controller, error) { ctx := context.Background() fs := filesys.EmptyInMemFS("") - dEnv := env.Load(ctx, env.GetCurrentUserHomeDir, fs, doltdb.LocalDirDoltDB, Version) + dEnv := env.Load(ctx, env.GetCurrentUserHomeDir, fs, doltdb.InMemDoltDB, Version) + globalConfig, _ := dEnv.Config.GetConfig(env.GlobalConfig) + if globalConfig.GetStringOrDefault(config.UserNameKey, "") == "" { + globalConfig.SetStrings(map[string]string{ + config.UserNameKey: "postgres", + config.UserEmailKey: "postgres@somewhere.com", + }) + } + return runServer(ctx, args, dEnv) } From 5d05bfcf670c59f1794c9ab8ecda92ecc359e60c Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Wed, 13 Dec 2023 16:50:21 -0800 Subject: [PATCH 14/23] Moved more things into main --- main.go | 133 ++++++++++++++++++++++++++++++++++++++++++----- server/server.go | 122 ++++++++----------------------------------- 2 files changed, 141 insertions(+), 114 deletions(-) diff --git a/main.go b/main.go index a5fdd04eb7..be98077c26 100644 --- a/main.go +++ b/main.go @@ -21,6 +21,7 @@ import ( "fmt" "math/rand" "os" + "strconv" "strings" "github.com/dolthub/dolt/go/cmd/dolt/cli" @@ -29,6 +30,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" + "github.com/dolthub/dolt/go/libraries/events" "github.com/dolthub/dolt/go/libraries/utils/argparser" "github.com/dolthub/dolt/go/libraries/utils/config" "github.com/dolthub/dolt/go/libraries/utils/file" @@ -36,6 +38,7 @@ import ( "github.com/dolthub/dolt/go/store/util/tempfiles" "github.com/dolthub/doltgresql/server" "github.com/dolthub/go-mysql-server/sql" + "github.com/fatih/color" "github.com/tidwall/gjson" ) @@ -47,9 +50,23 @@ var doltgresCommands = cli.NewSubCommandHandler("doltgresql", "it's git for data }) var globalArgParser = cli.CreateGlobalArgParser("doltgresql") +const chdirFlag = "--chdir" +const stdInFlag = "--stdin" +const stdOutFlag = "--stdout" +const stdErrFlag = "--stderr" +const stdOutAndErrFlag = "--out-and-err" + func main() { ctx := context.Background() seedGlobalRand() + + args := os.Args[1:] + + args, err := redirectStdio(args) + if err != nil { + cli.PrintErrln(err.Error()) + os.Exit(1) + } restoreIO := cli.InitIO() defer restoreIO() @@ -62,7 +79,6 @@ func main() { globalConfig, _ := dEnv.Config.GetConfig(env.GlobalConfig) // Inject the "sql-server" command if no other commands were given - args := os.Args[1:] if len(args) == 0 || (len(args) > 0 && strings.HasPrefix(args[0], "-")) { args = append([]string{"sql-server"}, args...) } @@ -73,12 +89,19 @@ func main() { os.Exit(1) } - // the sql-server command has special cased logic since we have to wait for the server to stop + // The sql-server command has special cased logic since it doesn't invoke a Dolt command directly, but runs a server + // and waits for it to finish if subCommandName == "" || subCommandName == "sql-server" { - runServer(ctx, dEnv) + err = runServer(ctx, dEnv) + if err != nil { + cli.PrintErrln(err.Error()) + os.Exit(1) + } + os.Exit(0) } - cliCtx, err := configureCliCtx(apr, fs, dEnv, err, ctx) + // Otherwise, attempt to run the command indicated + cliCtx, err := configureCliCtx(subCommandName, apr, fs, dEnv, err, ctx) if err != nil { cli.PrintErrln(err.Error()) os.Exit(1) @@ -88,7 +111,7 @@ func main() { os.Exit(exitCode) } -func configureCliCtx(apr *argparser.ArgParseResults, fs filesys.Filesys, dEnv *env.DoltEnv, err error, ctx context.Context) (cli.CliContext, error) { +func configureCliCtx(subcommand string, apr *argparser.ArgParseResults, fs filesys.Filesys, dEnv *env.DoltEnv, err error, ctx context.Context) (cli.CliContext, error) { dataDir, hasDataDir := apr.GetValue(commands.DataDirFlag) if hasDataDir { // If a relative path was provided, this ensures we have an absolute path everywhere. @@ -145,7 +168,7 @@ func configureCliCtx(apr *argparser.ArgParseResults, fs filesys.Filesys, dEnv *e return nil, fmt.Errorf("failed to parse credentials: %w", err) } - lateBind, err := buildLateBinder(ctx, cwdFS, dEnv, mrEnv, creds, apr, "sql-server", false) + lateBind, err := buildLateBinder(ctx, cwdFS, dEnv, mrEnv, creds, apr, subcommand, false) if err != nil { return nil, err } @@ -153,19 +176,20 @@ func configureCliCtx(apr *argparser.ArgParseResults, fs filesys.Filesys, dEnv *e return cli.NewCliContext(apr, dEnv.Config, lateBind) } -func runServer(ctx context.Context, dEnv *env.DoltEnv) { +func runServer(ctx context.Context, dEnv *env.DoltEnv) error { + // Emit a usage event in the background while we launch the server + // Dolt is more permissive with events: it emits events even if the command fails in earliest phase. + // We'll also emit a heartbeat event every 24 hours the server is running. All events will be tagged with the + // doltgresql app id. + go emitUsageEvent(dEnv) + controller, err := server.RunOnDisk(ctx, os.Args[1:], dEnv) if err != nil { - fmt.Println(err.Error()) - os.Exit(1) + return err } - err = controller.WaitForStop() - if err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } + return controller.WaitForStop() } // parseGlobalArgsAndSubCommandName parses the global arguments, including a profile if given or a default profile if exists. Also returns the subcommand name. @@ -396,3 +420,84 @@ func canMoveTempFile() bool { _ = file.Remove(testfile) return true } + +func redirectStdio(args []string) ([]string, error) { + if len(args) > 0 { + var doneDebugFlags bool + for !doneDebugFlags && len(args) > 0 { + switch args[0] { + // Currently goland doesn't support running with a different working directory when using go modules. + // This is a hack that allows a different working directory to be set after the application starts using + // chdir=. The syntax is not flexible and must match exactly this. + case chdirFlag: + err := os.Chdir(args[1]) + + if err != nil { + panic(err) + } + + args = args[2:] + + case stdInFlag: + stdInFile := args[1] + cli.Println("Using file contents as stdin:", stdInFile) + + f, err := os.Open(stdInFile) + if err != nil { + return nil, fmt.Errorf("Failed to open %s: %w", stdInFile, err) + } + + os.Stdin = f + args = args[2:] + + case stdOutFlag, stdErrFlag, stdOutAndErrFlag: + filename := args[1] + + f, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, os.ModePerm) + if err != nil { + return nil, fmt.Errorf("Failed to open %s for writing: %w", filename, err) + } + + switch args[0] { + case stdOutFlag: + cli.Println("Stdout being written to", filename) + cli.CliOut = f + case stdErrFlag: + cli.Println("Stderr being written to", filename) + cli.CliErr = f + case stdOutAndErrFlag: + cli.Println("Stdout and Stderr being written to", filename) + cli.CliOut = f + cli.CliErr = f + } + + color.NoColor = true + args = args[2:] + + default: + doneDebugFlags = true + } + } + } + return args, nil +} + +func emitUsageEvent(dEnv *env.DoltEnv) { + metricsDisabled := dEnv.Config.GetStringOrDefault(config.MetricsDisabled, "false") + disabled, err := strconv.ParseBool(metricsDisabled) + if err != nil || disabled { + return + } + + evt := events.NewEvent(sqlserver.SqlServerCmd{}.EventType()) + evtCollector := events.NewCollector() + evtCollector.CloseEventAndAdd(evt) + clientEvents := evtCollector.Close() + + emitter, err := commands.GRPCEmitterForConfig(dEnv) + if err != nil { + return + } + + err = emitter.LogEvents(server.Version, clientEvents) +} \ No newline at end of file diff --git a/server/server.go b/server/server.go index c7d47bc74f..7e62d37f21 100644 --- a/server/server.go +++ b/server/server.go @@ -19,7 +19,6 @@ import ( "fmt" _ "net/http/pprof" "os" - "strconv" "strings" "github.com/dolthub/dolt/go/cmd/dolt/cli" @@ -31,13 +30,14 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dfunctions" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" "github.com/dolthub/dolt/go/libraries/events" + "github.com/dolthub/dolt/go/libraries/utils/argparser" "github.com/dolthub/dolt/go/libraries/utils/config" "github.com/dolthub/dolt/go/libraries/utils/filesys" "github.com/dolthub/dolt/go/libraries/utils/svcs" "github.com/dolthub/dolt/go/store/nbs" "github.com/dolthub/dolt/go/store/util/tempfiles" "github.com/dolthub/go-mysql-server/server" - "github.com/fatih/color" + "github.com/dolthub/go-mysql-server/sql" ) const ( @@ -119,12 +119,6 @@ func init() { events.Application = eventsapi.AppID_APP_DOLTGRES } -const chdirFlag = "--chdir" -const stdInFlag = "--stdin" -const stdOutFlag = "--stdout" -const stdErrFlag = "--stderr" -const stdOutAndErrFlag = "--out-and-err" - // RunOnDisk starts the server based on the given args, while also using the local disk as the backing store. // The returned WaitGroup may be used to wait for the server to close. func RunOnDisk(ctx context.Context, args []string, dEnv *env.DoltEnv) (*svcs.Controller, error) { @@ -162,11 +156,6 @@ func runServer(ctx context.Context, args []string, dEnv *env.DoltEnv) (*svcs.Con nbs.TableIndexGCFinalizerWithStackTrace = false } - args, err := redirectStdio(args) - if err != nil { - return nil, err - } - if dEnv.CfgLoadErr != nil { return nil, fmt.Errorf("failed to load the global config: %w", dEnv.CfgLoadErr) } @@ -178,7 +167,7 @@ func runServer(ctx context.Context, args []string, dEnv *env.DoltEnv) (*svcs.Con defer tempfiles.MovableTempFileProvider.Clean() - err = dsess.InitPersistedSystemVars(dEnv) + err := dsess.InitPersistedSystemVars(dEnv) if err != nil { return nil, fmt.Errorf("failed to load persisted system variables: %w", err) } @@ -215,21 +204,16 @@ func runServer(ctx context.Context, args []string, dEnv *env.DoltEnv) (*svcs.Con if err != nil { return nil, err } + // We'll use a temporary environment to instantiate the subdirectory - tempDEnv := env.Load(ctx, env.GetCurrentUserHomeDir, subdirectoryFS, doltdb.LocalDirDoltDB, Version) - res := commands.InitCmd{}.Exec(ctx, "init", []string{}, tempDEnv, nil) + tempDEnv := env.Load(ctx, env.GetCurrentUserHomeDir, subdirectoryFS, dEnv.UrlStr(), Version) + res := commands.InitCmd{}.Exec(ctx, "init", []string{}, tempDEnv, configCliContext{tempDEnv}) if res != 0 { return nil, fmt.Errorf("failed to initialize doltgres database") } } } - - // If we got this far, emit a usage event in the background while we launch the server - // Dolt is more permissive with events: it emits events even if the command fails in earliest phase. - // We'll also emit a heartbeat event every 24 hours the server is running. All events will be tagged with the - // doltgresql app id. - go emitUsageEvent(dEnv) - + controller := svcs.NewController() newCtx, cancelF := context.WithCancel(ctx) go func() { @@ -251,83 +235,21 @@ func runServer(ctx context.Context, args []string, dEnv *env.DoltEnv) (*svcs.Con return controller, controller.WaitForStart() } -func redirectStdio(args []string) ([]string, error) { - if len(args) > 0 { - var doneDebugFlags bool - for !doneDebugFlags && len(args) > 0 { - switch args[0] { - // Currently goland doesn't support running with a different working directory when using go modules. - // This is a hack that allows a different working directory to be set after the application starts using - // chdir=. The syntax is not flexible and must match exactly this. - case chdirFlag: - err := os.Chdir(args[1]) - - if err != nil { - panic(err) - } - - args = args[2:] - - case stdInFlag: - stdInFile := args[1] - cli.Println("Using file contents as stdin:", stdInFile) - - f, err := os.Open(stdInFile) - if err != nil { - return nil, fmt.Errorf("Failed to open %s: %w", stdInFile, err) - } - - os.Stdin = f - args = args[2:] - - case stdOutFlag, stdErrFlag, stdOutAndErrFlag: - filename := args[1] - - f, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, os.ModePerm) - if err != nil { - return nil, fmt.Errorf("Failed to open %s for writing: %w", filename, err) - } - - switch args[0] { - case stdOutFlag: - cli.Println("Stdout being written to", filename) - cli.CliOut = f - case stdErrFlag: - cli.Println("Stderr being written to", filename) - cli.CliErr = f - case stdOutAndErrFlag: - cli.Println("Stdout and Stderr being written to", filename) - cli.CliOut = f - cli.CliErr = f - } - - color.NoColor = true - args = args[2:] - - default: - doneDebugFlags = true - } - } - } - return args, nil +// configCliContext is a minimal implementation of CliContext that only supports Config() +type configCliContext struct { + dEnv *env.DoltEnv } -func emitUsageEvent(dEnv *env.DoltEnv) { - metricsDisabled := dEnv.Config.GetStringOrDefault(config.MetricsDisabled, "false") - disabled, err := strconv.ParseBool(metricsDisabled) - if err != nil || disabled { - return - } +func (c configCliContext) Config() *env.DoltCliConfig { + return c.dEnv.Config +} - evt := events.NewEvent(sqlserver.SqlServerCmd{}.EventType()) - evtCollector := events.NewCollector() - evtCollector.CloseEventAndAdd(evt) - clientEvents := evtCollector.Close() - - emitter, err := commands.GRPCEmitterForConfig(dEnv) - if err != nil { - return - } - - err = emitter.LogEvents(Version, clientEvents) -} \ No newline at end of file +func (c configCliContext) GlobalArgs() *argparser.ArgParseResults { + panic("ConfigCliContext does not support GlobalArgs()") +} + +func (c configCliContext) QueryEngine(ctx context.Context) (cli.Queryist, *sql.Context, func(), error) { + return nil, nil, nil, fmt.Errorf("ConfigCliContext does not support QueryEngine()") +} + +var _ cli.CliContext = configCliContext{} \ No newline at end of file From 93ec80476b4da8733ef54621bcab22462d665cc7 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Wed, 13 Dec 2023 16:59:27 -0800 Subject: [PATCH 15/23] Cleanup --- main.go | 19 ++++++++++++++----- server/server.go | 3 --- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/main.go b/main.go index be98077c26..fed0ad8804 100644 --- a/main.go +++ b/main.go @@ -27,6 +27,7 @@ import ( "github.com/dolthub/dolt/go/cmd/dolt/cli" "github.com/dolthub/dolt/go/cmd/dolt/commands" "github.com/dolthub/dolt/go/cmd/dolt/commands/sqlserver" + eventsapi "github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi/v1alpha1" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" @@ -42,7 +43,6 @@ import ( "github.com/tidwall/gjson" ) - var doltgresCommands = cli.NewSubCommandHandler("doltgresql", "it's git for data", []cli.Command{ commands.ConfigCmd{}, commands.VersionCmd{VersionStr: server.Version}, @@ -50,6 +50,10 @@ var doltgresCommands = cli.NewSubCommandHandler("doltgresql", "it's git for data }) var globalArgParser = cli.CreateGlobalArgParser("doltgresql") +func init() { + events.Application = eventsapi.AppID_APP_DOLTGRES +} + const chdirFlag = "--chdir" const stdInFlag = "--stdin" const stdOutFlag = "--stdout" @@ -176,11 +180,15 @@ func configureCliCtx(subcommand string, apr *argparser.ArgParseResults, fs files return cli.NewCliContext(apr, dEnv.Config, lateBind) } +// runServer launches a server on the env given and waits for it to finish func runServer(ctx context.Context, dEnv *env.DoltEnv) error { - // Emit a usage event in the background while we launch the server - // Dolt is more permissive with events: it emits events even if the command fails in earliest phase. - // We'll also emit a heartbeat event every 24 hours the server is running. All events will be tagged with the - // doltgresql app id. + // Emit a usage event in the background while we start the server. + // Dolt is more permissive with events: it emits events even if the command fails in the earliest possible phase, + // we emit an event only if we got far enough to attempt to launch a server (and we may not emit it if the server + // dies quickly enough). + // + // We also emit a heartbeat event every 24 hours the server is running. + // All events will be tagged with the doltgresql app id. go emitUsageEvent(dEnv) controller, err := server.RunOnDisk(ctx, os.Args[1:], dEnv) @@ -482,6 +490,7 @@ func redirectStdio(args []string) ([]string, error) { return args, nil } +// emitUsageEvent emits a usage event to the event server func emitUsageEvent(dEnv *env.DoltEnv) { metricsDisabled := dEnv.Config.GetStringOrDefault(config.MetricsDisabled, "false") disabled, err := strconv.ParseBool(metricsDisabled) diff --git a/server/server.go b/server/server.go index 7e62d37f21..c41e48f3f4 100644 --- a/server/server.go +++ b/server/server.go @@ -24,12 +24,10 @@ import ( "github.com/dolthub/dolt/go/cmd/dolt/cli" "github.com/dolthub/dolt/go/cmd/dolt/commands" "github.com/dolthub/dolt/go/cmd/dolt/commands/sqlserver" - eventsapi "github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi/v1alpha1" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dfunctions" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" - "github.com/dolthub/dolt/go/libraries/events" "github.com/dolthub/dolt/go/libraries/utils/argparser" "github.com/dolthub/dolt/go/libraries/utils/config" "github.com/dolthub/dolt/go/libraries/utils/filesys" @@ -116,7 +114,6 @@ func init() { server.DefaultProtocolListenerFunc = NewListener sqlserver.ExternalDisableUsers = true dfunctions.VersionString = Version - events.Application = eventsapi.AppID_APP_DOLTGRES } // RunOnDisk starts the server based on the given args, while also using the local disk as the backing store. From 7e2a325095f1dd33fb170d20ded00eab06177ecb Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Wed, 13 Dec 2023 17:29:44 -0800 Subject: [PATCH 16/23] Corrected help text --- server/server.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/server.go b/server/server.go index c41e48f3f4..c55b292bbb 100644 --- a/server/server.go +++ b/server/server.go @@ -169,7 +169,10 @@ func runServer(ctx context.Context, args []string, dEnv *env.DoltEnv) (*svcs.Con return nil, fmt.Errorf("failed to load persisted system variables: %w", err) } - serverConfig, err := sqlserver.ServerConfigFromArgs("sql-server", args, dEnv) + ap := sqlserver.SqlServerCmd{}.ArgParser() + help, _ := cli.HelpAndUsagePrinters(cli.CommandDocsForCommandString("sql-server", sqlServerDocs, ap)) + + serverConfig, err := sqlserver.ServerConfigFromArgs(ap, help, args, dEnv) if err != nil { return nil, err } From 3ac4acaa4920d8ae78175e037797b75ababe6206 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Wed, 13 Dec 2023 17:36:25 -0800 Subject: [PATCH 17/23] Fix merge error --- server/server.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/server/server.go b/server/server.go index ee3638eba3..dd3271404d 100644 --- a/server/server.go +++ b/server/server.go @@ -213,11 +213,9 @@ func runServer(ctx context.Context, args []string, dEnv *env.DoltEnv) (*svcs.Con } else if !isDirectory { // The else branch means that there's a Doltgres item, so we need to error if it's a file since we // enforce the creation of a Doltgres database/directory, which would create a name conflict with the file - stop() - cli.PrintErrln(fmt.Errorf("Attempted to create the default `doltgres` database, but a file with" + - " the same name was found. Please run the doltgres command in a directory that does not contain a" + - " file with the name doltgres")) - return intPointer(1), wg + return nil, fmt.Errorf("Attempted to create the default `doltgres` database, but a file with" + + " the same name was found. Please run the doltgres command in a directory that does not contain a" + + " file with the name doltgres") } } } From c3e6733c33658e83ba72c9b292483ca8ecaebcde Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Wed, 13 Dec 2023 17:39:17 -0800 Subject: [PATCH 18/23] Braces problem --- server/server.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/server.go b/server/server.go index dd3271404d..1159075f89 100644 --- a/server/server.go +++ b/server/server.go @@ -210,13 +210,13 @@ func runServer(ctx context.Context, args []string, dEnv *env.DoltEnv) (*svcs.Con res := commands.InitCmd{}.Exec(ctx, "init", []string{}, tempDEnv, configCliContext{tempDEnv}) if res != 0 { return nil, fmt.Errorf("failed to initialize doltgres database") - } else if !isDirectory { - // The else branch means that there's a Doltgres item, so we need to error if it's a file since we - // enforce the creation of a Doltgres database/directory, which would create a name conflict with the file - return nil, fmt.Errorf("Attempted to create the default `doltgres` database, but a file with" + - " the same name was found. Please run the doltgres command in a directory that does not contain a" + - " file with the name doltgres") } + } else if !isDirectory { + // The else branch means that there's a Doltgres item, so we need to error if it's a file since we + // enforce the creation of a Doltgres database/directory, which would create a name conflict with the file + return nil, fmt.Errorf("Attempted to create the default `doltgres` database, but a file with" + + " the same name was found. Please run the doltgres command in a directory that does not contain a" + + " file with the name doltgres") } } From c187a301150858f84da9f987f164d18011292bda Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Wed, 13 Dec 2023 17:41:22 -0800 Subject: [PATCH 19/23] Moved one more thing into main --- main.go | 5 +++++ server/server.go | 6 ------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/main.go b/main.go index fed0ad8804..a35bbc85b4 100644 --- a/main.go +++ b/main.go @@ -36,6 +36,7 @@ import ( "github.com/dolthub/dolt/go/libraries/utils/config" "github.com/dolthub/dolt/go/libraries/utils/file" "github.com/dolthub/dolt/go/libraries/utils/filesys" + "github.com/dolthub/dolt/go/store/nbs" "github.com/dolthub/dolt/go/store/util/tempfiles" "github.com/dolthub/doltgresql/server" "github.com/dolthub/go-mysql-server/sql" @@ -52,6 +53,10 @@ var globalArgParser = cli.CreateGlobalArgParser("doltgresql") func init() { events.Application = eventsapi.AppID_APP_DOLTGRES + + if os.Getenv("DOLT_VERBOSE_ASSERT_TABLE_FILES_CLOSED") == "" { + nbs.TableIndexGCFinalizerWithStackTrace = false + } } const chdirFlag = "--chdir" diff --git a/server/server.go b/server/server.go index 1159075f89..43083700dd 100644 --- a/server/server.go +++ b/server/server.go @@ -18,7 +18,6 @@ import ( "context" "fmt" _ "net/http/pprof" - "os" "strings" "github.com/dolthub/dolt/go/cmd/dolt/cli" @@ -32,7 +31,6 @@ import ( "github.com/dolthub/dolt/go/libraries/utils/config" "github.com/dolthub/dolt/go/libraries/utils/filesys" "github.com/dolthub/dolt/go/libraries/utils/svcs" - "github.com/dolthub/dolt/go/store/nbs" "github.com/dolthub/dolt/go/store/util/tempfiles" "github.com/dolthub/go-mysql-server/server" "github.com/dolthub/go-mysql-server/sql" @@ -148,10 +146,6 @@ func runServer(ctx context.Context, args []string, dEnv *env.DoltEnv) (*svcs.Con args = append(args, "--port=5432") } } - - if os.Getenv("DOLT_VERBOSE_ASSERT_TABLE_FILES_CLOSED") == "" { - nbs.TableIndexGCFinalizerWithStackTrace = false - } if dEnv.CfgLoadErr != nil { return nil, fmt.Errorf("failed to load the global config: %w", dEnv.CfgLoadErr) From 53a234e2d14801e570cbf327bf64bb4ee30e1eae Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Wed, 13 Dec 2023 17:44:27 -0800 Subject: [PATCH 20/23] Dependency upgrades --- go.mod | 16 +++++----------- go.sum | 13 +++++++++++++ 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 06d7de127f..7ad1751730 100644 --- a/go.mod +++ b/go.mod @@ -1,21 +1,14 @@ module github.com/dolthub/doltgresql -replace github.com/dolthub/dolt/go => ../dolt/go - -replace github.com/dolthub/go-mysql-server => ../go-mysql-server - -replace github.com/dolthub/vitess => ../vitess - -replace github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi => ../dolt/go/gen/proto/dolt/services/eventsapi - go 1.21 require ( github.com/PuerkitoBio/goquery v1.8.1 github.com/cockroachdb/apd/v2 v2.0.3-0.20200518165714-d020e156310a github.com/cockroachdb/errors v1.7.5 - github.com/dolthub/dolt/go v0.40.5-0.20231130221002-0d5eeb513092 - github.com/dolthub/go-mysql-server v0.17.1-0.20231205222834-2eb85072ed9d + github.com/dolthub/dolt/go v0.40.5-0.20231214014142-bf6f4efa51eb + github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi v0.0.0-20231213233028-64c353bf920f + github.com/dolthub/go-mysql-server v0.17.1-0.20231213201402-47a48c5f014b github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 github.com/dolthub/vitess v0.0.0-20231207010700-88fb35413580 github.com/fatih/color v1.13.0 @@ -60,7 +53,6 @@ require ( github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/denisbrodbeck/machineid v1.0.1 // indirect - github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi v0.0.0-20231018220650-48f565111c6a // indirect github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 // indirect github.com/dolthub/fslock v0.0.3 // indirect github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e // indirect @@ -79,6 +71,8 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/go-github/v57 v57.0.0 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/google/s2a-go v0.1.4 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.3.0 // indirect diff --git a/go.sum b/go.sum index 80537449f3..75efa468bc 100644 --- a/go.sum +++ b/go.sum @@ -212,12 +212,18 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dolthub/dolt/go v0.40.5-0.20231214014142-bf6f4efa51eb h1:d8+ucz07P14aEmwAwtTz2psj0DJYGknQPPGLxj0SiBo= +github.com/dolthub/dolt/go v0.40.5-0.20231214014142-bf6f4efa51eb/go.mod h1:EUxtWDbH5mNrd2BCPYU6vTU24e0ym9aNfHO+LyGSqms= +github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi v0.0.0-20231213233028-64c353bf920f h1:f250FTgZ/OaCql9G6WJt46l9VOIBF1mI81hW9cnmBNM= +github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi v0.0.0-20231213233028-64c353bf920f/go.mod h1:gHeHIDGU7em40EhFTliq62pExFcc1hxDTIZ9g5UqXYM= github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 h1:u3PMzfF8RkKd3lB9pZ2bfn0qEG+1Gms9599cr0REMww= github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2/go.mod h1:mIEZOHnFx4ZMQeawhw9rhsj+0zwQj7adVsnBX7t+eKY= github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e h1:kPsT4a47cw1+y/N5SSCkma7FhAPw7KeGmD6c9PBZW9Y= github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168= +github.com/dolthub/go-mysql-server v0.17.1-0.20231213201402-47a48c5f014b h1:FjnDirW0PRs5QfowiMq8Gp8ilyEWRSSoyErhUAGeqJQ= +github.com/dolthub/go-mysql-server v0.17.1-0.20231213201402-47a48c5f014b/go.mod h1:zJCyPiYe9VZ9xIQTv7S1OFKwyoVQoeGxZXNtkFxTcOI= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 h1:0HHu0GWJH0N6a6keStrHhUAK5/o9LVfkh44pvsV4514= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto= github.com/dolthub/jsonpath v0.0.2-0.20230525180605-8dc13778fd72 h1:NfWmngMi1CYUWU4Ix8wM+USEhjc+mhPlT9JUR/anvbQ= @@ -228,6 +234,8 @@ github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 h1:7/v8q9X github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81/go.mod h1:siLfyv2c92W1eN/R4QqG/+RjjX5W2+gCTRjZxBjI3TY= github.com/dolthub/swiss v0.1.0 h1:EaGQct3AqeP/MjASHLiH6i4TAmgbG/c4rA6a1bzCOPc= github.com/dolthub/swiss v0.1.0/go.mod h1:BeucyB08Vb1G9tumVN3Vp/pyY4AMUnr9p7Rz7wJ7kAQ= +github.com/dolthub/vitess v0.0.0-20231207010700-88fb35413580 h1:OSp1g3tRBMVIyxza4LN20rZ6yYEKqjf5hNNisVg/Lns= +github.com/dolthub/vitess v0.0.0-20231207010700-88fb35413580/go.mod h1:IwjNXSQPymrja5pVqmfnYdcy7Uv7eNJNBPK/MEh9OOw= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= @@ -372,6 +380,7 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -379,7 +388,11 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-github/v57 v57.0.0 h1:L+Y3UPTY8ALM8x+TV0lg+IEBI+upibemtBD8Q9u7zHs= +github.com/google/go-github/v57 v57.0.0/go.mod h1:s0omdnye0hvK/ecLvpsGfJMiRt85PimQh4oygmLIxHw= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= From aa64c8f2fb0127f2acdd0cf35f36424f647b521f Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Wed, 13 Dec 2023 17:50:45 -0800 Subject: [PATCH 21/23] Formatting --- main.go | 17 +++++++++-------- server/server.go | 30 +++++++++++++++--------------- testing/go/framework_test.go | 4 ++-- testing/go/ssl_test.go | 2 +- 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/main.go b/main.go index a35bbc85b4..4143a7424a 100644 --- a/main.go +++ b/main.go @@ -38,10 +38,11 @@ import ( "github.com/dolthub/dolt/go/libraries/utils/filesys" "github.com/dolthub/dolt/go/store/nbs" "github.com/dolthub/dolt/go/store/util/tempfiles" - "github.com/dolthub/doltgresql/server" "github.com/dolthub/go-mysql-server/sql" "github.com/fatih/color" "github.com/tidwall/gjson" + + "github.com/dolthub/doltgresql/server" ) var doltgresCommands = cli.NewSubCommandHandler("doltgresql", "it's git for data", []cli.Command{ @@ -68,9 +69,9 @@ const stdOutAndErrFlag = "--out-and-err" func main() { ctx := context.Background() seedGlobalRand() - + args := os.Args[1:] - + args, err := redirectStdio(args) if err != nil { cli.PrintErrln(err.Error()) @@ -97,7 +98,7 @@ func main() { cli.PrintErrln(err.Error()) os.Exit(1) } - + // The sql-server command has special cased logic since it doesn't invoke a Dolt command directly, but runs a server // and waits for it to finish if subCommandName == "" || subCommandName == "sql-server" { @@ -139,7 +140,7 @@ func configureCliCtx(subcommand string, apr *argparser.ArgParseResults, fs files if dEnv.HasDoltDataDir() { return nil, fmt.Errorf("Cannot start a server within a directory containing a Dolt or Doltgres database." + - "To use the current directory as a database, start the server from the parent directory.") + "To use the current directory as a database, start the server from the parent directory.") } err = reconfigIfTempFileMoveFails(dEnv) @@ -188,10 +189,10 @@ func configureCliCtx(subcommand string, apr *argparser.ArgParseResults, fs files // runServer launches a server on the env given and waits for it to finish func runServer(ctx context.Context, dEnv *env.DoltEnv) error { // Emit a usage event in the background while we start the server. - // Dolt is more permissive with events: it emits events even if the command fails in the earliest possible phase, + // Dolt is more permissive with events: it emits events even if the command fails in the earliest possible phase, // we emit an event only if we got far enough to attempt to launch a server (and we may not emit it if the server // dies quickly enough). - // + // // We also emit a heartbeat event every 24 hours the server is running. // All events will be tagged with the doltgresql app id. go emitUsageEvent(dEnv) @@ -514,4 +515,4 @@ func emitUsageEvent(dEnv *env.DoltEnv) { } err = emitter.LogEvents(server.Version, clientEvents) -} \ No newline at end of file +} diff --git a/server/server.go b/server/server.go index 43083700dd..62d8567c26 100644 --- a/server/server.go +++ b/server/server.go @@ -43,13 +43,13 @@ const ( var sqlServerDocs = cli.CommandDocumentationContent{ ShortDesc: "Start a PostgreSQL-compatible server.", LongDesc: "By default, starts a PostgreSQL-compatible server on the dolt database in the current directory. " + - "Databases are named after the directories they appear in" + - "Parameters can be specified using a yaml configuration file passed to the server via " + - "{{.EmphasisLeft}}--config {{.EmphasisRight}}, or by using the supported switches and flags to configure " + - "the server directly on the command line. If {{.EmphasisLeft}}--config {{.EmphasisRight}} is provided all" + - " other command line arguments are ignored.\n\nThis is an example yaml configuration file showing all supported" + - " items and their default values:\n\n" + - indentLines(sqlserver.ServerConfigAsYAMLConfig(sqlserver.DefaultServerConfig()).String()) + "\n\n" + ` + "Databases are named after the directories they appear in" + + "Parameters can be specified using a yaml configuration file passed to the server via " + + "{{.EmphasisLeft}}--config {{.EmphasisRight}}, or by using the supported switches and flags to configure " + + "the server directly on the command line. If {{.EmphasisLeft}}--config {{.EmphasisRight}} is provided all" + + " other command line arguments are ignored.\n\nThis is an example yaml configuration file showing all supported" + + " items and their default values:\n\n" + + indentLines(sqlserver.ServerConfigAsYAMLConfig(sqlserver.DefaultServerConfig()).String()) + "\n\n" + ` SUPPORTED CONFIG FILE FIELDS: {{.EmphasisLeft}}data_dir{{.EmphasisRight}}: A directory where the server will load dolt databases to serve, and create new ones. Defaults to the current directory. @@ -146,14 +146,14 @@ func runServer(ctx context.Context, args []string, dEnv *env.DoltEnv) (*svcs.Con args = append(args, "--port=5432") } } - + if dEnv.CfgLoadErr != nil { return nil, fmt.Errorf("failed to load the global config: %w", dEnv.CfgLoadErr) } if dEnv.HasDoltDataDir() { return nil, fmt.Errorf("Cannot start a server within a directory containing a Dolt or Doltgres database." + - "To use the current directory as a database, start the server from the parent directory.") + "To use the current directory as a database, start the server from the parent directory.") } defer tempfiles.MovableTempFileProvider.Clean() @@ -198,7 +198,7 @@ func runServer(ctx context.Context, args []string, dEnv *env.DoltEnv) (*svcs.Con if err != nil { return nil, err } - + // We'll use a temporary environment to instantiate the subdirectory tempDEnv := env.Load(ctx, env.GetCurrentUserHomeDir, subdirectoryFS, dEnv.UrlStr(), Version) res := commands.InitCmd{}.Exec(ctx, "init", []string{}, tempDEnv, configCliContext{tempDEnv}) @@ -209,11 +209,11 @@ func runServer(ctx context.Context, args []string, dEnv *env.DoltEnv) (*svcs.Con // The else branch means that there's a Doltgres item, so we need to error if it's a file since we // enforce the creation of a Doltgres database/directory, which would create a name conflict with the file return nil, fmt.Errorf("Attempted to create the default `doltgres` database, but a file with" + - " the same name was found. Please run the doltgres command in a directory that does not contain a" + - " file with the name doltgres") + " the same name was found. Please run the doltgres command in a directory that does not contain a" + + " file with the name doltgres") } } - + controller := svcs.NewController() newCtx, cancelF := context.WithCancel(ctx) go func() { @@ -229,7 +229,7 @@ func runServer(ctx context.Context, args []string, dEnv *env.DoltEnv) (*svcs.Con cancelF() } }() - + sqlserver.ConfigureServices(serverConfig, controller, Version, dEnv) go controller.Start(newCtx) return controller, controller.WaitForStart() @@ -252,4 +252,4 @@ func (c configCliContext) QueryEngine(ctx context.Context) (cli.Queryist, *sql.C return nil, nil, nil, fmt.Errorf("ConfigCliContext does not support QueryEngine()") } -var _ cli.CliContext = configCliContext{} \ No newline at end of file +var _ cli.CliContext = configCliContext{} diff --git a/testing/go/framework_test.go b/testing/go/framework_test.go index 8320873a28..ec02f874e0 100644 --- a/testing/go/framework_test.go +++ b/testing/go/framework_test.go @@ -76,7 +76,7 @@ func RunScript(t *testing.T, script ScriptTest) { if len(scriptDatabase) == 0 { scriptDatabase = "postgres" } - + ctx, conn, controller := CreateServer(t, scriptDatabase) defer func() { conn.Close(ctx) @@ -152,7 +152,7 @@ func CreateServer(t *testing.T, database string) (context.Context, *pgx.Conn, *s server.DefaultProtocolListenerFunc = dserver.NewLimitedListener controller, err := dserver.RunInMemory([]string{fmt.Sprintf("--port=%d", port), "--host=127.0.0.1"}) require.NoError(t, err) - + fmt.Printf("port is %d\n", port) ctx := context.Background() diff --git a/testing/go/ssl_test.go b/testing/go/ssl_test.go index ea13dbfb99..d0a20535c3 100644 --- a/testing/go/ssl_test.go +++ b/testing/go/ssl_test.go @@ -25,7 +25,7 @@ func TestSSL(t *testing.T) { controller.Stop() require.NoError(t, controller.WaitForStop()) }() - + ctx := context.Background() err = func() error { // The connection attempt may be made before the server has grabbed the port, so we'll retry the first From 2a4171194501bf7a3b148f0612dcf19d1730e3f7 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Thu, 14 Dec 2023 10:16:57 -0800 Subject: [PATCH 22/23] Get latest dolt, and add back init command --- go.mod | 2 +- go.sum | 4 ++-- main.go | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7ad1751730..32b6d87fc7 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/PuerkitoBio/goquery v1.8.1 github.com/cockroachdb/apd/v2 v2.0.3-0.20200518165714-d020e156310a github.com/cockroachdb/errors v1.7.5 - github.com/dolthub/dolt/go v0.40.5-0.20231214014142-bf6f4efa51eb + github.com/dolthub/dolt/go v0.40.5-0.20231214175736-2c32f8dc8f79 github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi v0.0.0-20231213233028-64c353bf920f github.com/dolthub/go-mysql-server v0.17.1-0.20231213201402-47a48c5f014b github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 diff --git a/go.sum b/go.sum index 75efa468bc..84ce1ae1b2 100644 --- a/go.sum +++ b/go.sum @@ -212,8 +212,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dolthub/dolt/go v0.40.5-0.20231214014142-bf6f4efa51eb h1:d8+ucz07P14aEmwAwtTz2psj0DJYGknQPPGLxj0SiBo= -github.com/dolthub/dolt/go v0.40.5-0.20231214014142-bf6f4efa51eb/go.mod h1:EUxtWDbH5mNrd2BCPYU6vTU24e0ym9aNfHO+LyGSqms= +github.com/dolthub/dolt/go v0.40.5-0.20231214175736-2c32f8dc8f79 h1:DXJJMtcu6mn0VBkwPL82ywI30OaArgKAIKvbGldshhI= +github.com/dolthub/dolt/go v0.40.5-0.20231214175736-2c32f8dc8f79/go.mod h1:BppH8WUk82ZDi43JnZsaSR1X7EQ3YRBUNkDupl6ne0g= github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi v0.0.0-20231213233028-64c353bf920f h1:f250FTgZ/OaCql9G6WJt46l9VOIBF1mI81hW9cnmBNM= github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi v0.0.0-20231213233028-64c353bf920f/go.mod h1:gHeHIDGU7em40EhFTliq62pExFcc1hxDTIZ9g5UqXYM= github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 h1:u3PMzfF8RkKd3lB9pZ2bfn0qEG+1Gms9599cr0REMww= diff --git a/main.go b/main.go index 4143a7424a..8373984293 100644 --- a/main.go +++ b/main.go @@ -46,6 +46,7 @@ import ( ) var doltgresCommands = cli.NewSubCommandHandler("doltgresql", "it's git for data", []cli.Command{ + commands.InitCmd{}, commands.ConfigCmd{}, commands.VersionCmd{VersionStr: server.Version}, sqlserver.SqlServerCmd{VersionStr: server.Version}, From 47dc11f6ee7e69c3791a6712ba1d80f8592713b8 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Thu, 14 Dec 2023 10:24:48 -0800 Subject: [PATCH 23/23] PR feedback --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 8373984293..739e2c04b6 100644 --- a/main.go +++ b/main.go @@ -102,7 +102,7 @@ func main() { // The sql-server command has special cased logic since it doesn't invoke a Dolt command directly, but runs a server // and waits for it to finish - if subCommandName == "" || subCommandName == "sql-server" { + if subCommandName == "sql-server" { err = runServer(ctx, dEnv) if err != nil { cli.PrintErrln(err.Error())