diff --git a/dev/sg/BUILD.bazel b/dev/sg/BUILD.bazel index ef87528e932a..559bab2204ec 100644 --- a/dev/sg/BUILD.bazel +++ b/dev/sg/BUILD.bazel @@ -111,7 +111,6 @@ go_library( "@com_github_opsgenie_opsgenie_go_sdk_v2//alert", "@com_github_opsgenie_opsgenie_go_sdk_v2//client", "@com_github_slack_go_slack//:slack", - "@com_github_sourcegraph_conc//pool", "@com_github_sourcegraph_log//:log", "@com_github_sourcegraph_run//:run", "@com_github_urfave_cli_v2//:cli", diff --git a/dev/sg/internal/run/bazel_command.go b/dev/sg/internal/run/bazel_command.go index c45fd3c421b5..99cde9b0541c 100644 --- a/dev/sg/internal/run/bazel_command.go +++ b/dev/sg/internal/run/bazel_command.go @@ -42,6 +42,14 @@ func (bc BazelCommand) GetConfig() SGConfigCommandOptions { return bc.Config } +func (bc BazelCommand) UpdateConfig(f func(*SGConfigCommandOptions)) SGConfigCommand { + f(&bc.Config) + return bc +} + +func (bc BazelCommand) GetBazelTarget() string { + return bc.Target +} func (bc BazelCommand) watchPaths() ([]string, error) { // If no target is defined, there is nothing to be built and watched if bc.Target == "" { diff --git a/dev/sg/internal/run/command.go b/dev/sg/internal/run/command.go index 77e0bf9d1b00..dafd83ad31fa 100644 --- a/dev/sg/internal/run/command.go +++ b/dev/sg/internal/run/command.go @@ -55,6 +55,11 @@ func (cmd Command) GetConfig() SGConfigCommandOptions { return cmd.Config } +func (cmd Command) UpdateConfig(f func(*SGConfigCommandOptions)) SGConfigCommand { + f(&cmd.Config) + return cmd +} + func (cmd Command) GetName() string { return cmd.Config.Name } @@ -66,6 +71,10 @@ func (cmd Command) GetBinaryLocation() (string, error) { return "", noBinaryError{name: cmd.Config.Name} } +func (cmd Command) GetBazelTarget() string { + return "" +} + func (cmd Command) GetExecCmd(ctx context.Context) (*exec.Cmd, error) { return exec.CommandContext(ctx, "bash", "-c", cmd.Cmd), nil } diff --git a/dev/sg/internal/run/docker_commmand.go b/dev/sg/internal/run/docker_commmand.go index 02aca99e8b79..f9016e9e1179 100644 --- a/dev/sg/internal/run/docker_commmand.go +++ b/dev/sg/internal/run/docker_commmand.go @@ -83,6 +83,15 @@ func (dc DockerCommand) GetBinaryLocation() (string, error) { return binaryLocation(dc.Target) } +func (dc DockerCommand) GetBazelTarget() string { + return dc.Target +} + +func (dc DockerCommand) UpdateConfig(f func(*SGConfigCommandOptions)) SGConfigCommand { + f(&dc.Config) + return dc +} + func (dc DockerCommand) StartWatch(ctx context.Context) (<-chan struct{}, error) { if watchPaths, err := dc.watchPaths(); err != nil { return nil, err diff --git a/dev/sg/internal/run/ibazel.go b/dev/sg/internal/run/ibazel.go index 316141849b95..4f92caeef8d3 100644 --- a/dev/sg/internal/run/ibazel.go +++ b/dev/sg/internal/run/ibazel.go @@ -34,13 +34,12 @@ type IBazel struct { events *iBazelEventHandler logsDir string logFile *os.File - dir string proc *startedCmd logs chan<- output.FancyLine } // returns a runner to interact with ibazel. -func NewIBazel(targets []string, dir string) (*IBazel, error) { +func NewIBazel(targets []string) (*IBazel, error) { logsDir, err := initLogsDir() if err != nil { return nil, err @@ -56,7 +55,6 @@ func NewIBazel(targets []string, dir string) (*IBazel, error) { events: newIBazelEventHandler(profileEventsPath(logsDir)), logsDir: logsDir, logFile: logFile, - dir: dir, }, nil } @@ -146,11 +144,15 @@ func (ib *IBazel) WaitForInitialBuild(ctx context.Context) error { return nil } -func (ib *IBazel) getCommandOptions(ctx context.Context) commandOptions { +func (ib *IBazel) getCommandOptions(ctx context.Context) (commandOptions, error) { + dir, err := root.RepositoryRoot() + if err != nil { + return commandOptions{}, err + } return commandOptions{ name: "iBazel", exec: ib.GetExecCmd(ctx), - dir: ib.dir, + dir: dir, // Don't output iBazel logs (which are all on stderr) until // initial build is complete as it will break the progress bar stderr: outputOptions{ @@ -159,13 +161,17 @@ func (ib *IBazel) getCommandOptions(ctx context.Context) commandOptions { ib.logFile, &patternMatcher{regex: watchErrorRegex, callback: ib.logWatchError}, }}, - } + }, nil } // Build starts an ibazel process to build the targets provided in the constructor // It runs perpetually, watching for file changes -func (ib *IBazel) build(ctx context.Context) (err error) { - ib.proc, err = startCmd(ctx, ib.getCommandOptions(ctx)) +func (ib *IBazel) build(ctx context.Context) error { + opts, err := ib.getCommandOptions(ctx) + if err != nil { + return err + } + ib.proc, err = startCmd(ctx, opts) return err } diff --git a/dev/sg/internal/run/installer.go b/dev/sg/internal/run/installer.go index ca86ea40206f..3f87bdce78a2 100644 --- a/dev/sg/internal/run/installer.go +++ b/dev/sg/internal/run/installer.go @@ -50,7 +50,7 @@ type InstallManager struct { stats *installAnalytics } -func Install(ctx context.Context, env map[string]string, verbose bool, cmds ...Installer) error { +func Install(ctx context.Context, env map[string]string, verbose bool, cmds []Installer) error { installer := newInstallManager(cmds, std.Out, env, verbose) installer.start(ctx) diff --git a/dev/sg/internal/run/run.go b/dev/sg/internal/run/run.go index 5eaede643972..597b37539266 100644 --- a/dev/sg/internal/run/run.go +++ b/dev/sg/internal/run/run.go @@ -24,7 +24,7 @@ type cmdRunner struct { verbose bool } -func Commands(ctx context.Context, parentEnv map[string]string, verbose bool, cmds ...SGConfigCommand) (err error) { +func Commands(ctx context.Context, parentEnv map[string]string, verbose bool, cmds []SGConfigCommand) (err error) { if len(cmds) == 0 { // Exit early if there are no commands to run. return nil diff --git a/dev/sg/internal/run/sgconfig_command.go b/dev/sg/internal/run/sgconfig_command.go index 33ca9b86a1e0..6d31052fd8ce 100644 --- a/dev/sg/internal/run/sgconfig_command.go +++ b/dev/sg/internal/run/sgconfig_command.go @@ -13,6 +13,10 @@ type SGConfigCommand interface { GetConfig() SGConfigCommandOptions GetBinaryLocation() (string, error) GetExecCmd(context.Context) (*exec.Cmd, error) + UpdateConfig(func(*SGConfigCommandOptions)) SGConfigCommand + + // Optionally returns a bazel target associated with this command + GetBazelTarget() string // Start a file watcher on the relevant filesystem sub-tree for this command StartWatch(context.Context) (<-chan struct{}, error) diff --git a/dev/sg/sg_run.go b/dev/sg/sg_run.go index 6aac3abce65d..23bc37a62d65 100644 --- a/dev/sg/sg_run.go +++ b/dev/sg/sg_run.go @@ -2,7 +2,6 @@ package main import ( "context" - "flag" "fmt" "sort" "strings" @@ -10,16 +9,14 @@ import ( "github.com/urfave/cli/v2" "gopkg.in/yaml.v3" - "github.com/sourcegraph/conc/pool" - "github.com/sourcegraph/sourcegraph/dev/sg/internal/category" - "github.com/sourcegraph/sourcegraph/dev/sg/internal/run" "github.com/sourcegraph/sourcegraph/dev/sg/internal/std" "github.com/sourcegraph/sourcegraph/dev/sg/interrupt" "github.com/sourcegraph/sourcegraph/lib/cliutil/completions" - "github.com/sourcegraph/sourcegraph/lib/output" ) +var deprecationNotice = "sg run is deprecated. Use 'sg start -cmd' instead.\n" + func init() { postInitHooks = append(postInitHooks, func(cmd *cli.Context) { @@ -39,9 +36,9 @@ func init() { var runCommand = &cli.Command{ Name: "run", - Usage: "Run the given commands", + Usage: deprecationNotice, ArgsUsage: "[command]", - UsageText: ` + UsageText: deprecationNotice + ` # Run specific commands sg run gitserver sg run frontend @@ -56,17 +53,13 @@ sg run gitserver frontend repo-updater sg run -describe jaeger `, Category: category.Dev, + Action: runExec, Flags: []cli.Flag{ &cli.BoolFlag{ Name: "describe", Usage: "Print details about selected run target", }, - &cli.BoolFlag{ - Name: "legacy", - Usage: "Force run to pick the non-bazel variant of the command", - }, }, - Action: runExec, BashComplete: completions.CompleteArgs(func() (options []string) { config, _ := getConfig() if config == nil { @@ -84,28 +77,13 @@ func runExec(ctx *cli.Context) error { if err != nil { return err } - legacy := ctx.Bool("legacy") - - args := ctx.Args().Slice() - if len(args) == 0 { - std.Out.WriteLine(output.Styled(output.StyleWarning, "No command specified")) - return flag.ErrHelp - } - - cmds := make([]run.SGConfigCommand, 0, len(args)) - for _, arg := range args { - if bazelCmd, ok := config.BazelCommands[arg]; ok && !legacy { - cmds = append(cmds, bazelCmd) - } else if cmd, ok := config.Commands[arg]; ok { - cmds = append(cmds, cmd) - } else { - std.Out.WriteLine(output.Styledf(output.StyleWarning, "ERROR: command %q not found :(", arg)) - return flag.ErrHelp - } + cmds, err := listToCommands(config, ctx.Args().Slice()) + if err != nil { + return err } if ctx.Bool("describe") { - for _, cmd := range cmds { + for _, cmd := range cmds.commands { out, err := yaml.Marshal(cmd) if err != nil { return err @@ -118,17 +96,14 @@ func runExec(ctx *cli.Context) error { return nil } - p := pool.New().WithContext(ctx.Context).WithCancelOnError() - p.Go(func(ctx context.Context) error { - return run.Commands(ctx, config.Env, verbose, cmds...) - }) - - return p.Wait() + return cmds.start(ctx.Context) } func constructRunCmdLongHelp() string { var out strings.Builder + fmt.Fprint(&out, deprecationNotice) + fmt.Fprintf(&out, "Runs the given command. If given a whitespace-separated list of commands it runs the set of commands.\n") config, err := getConfig() diff --git a/dev/sg/sg_start.go b/dev/sg/sg_start.go index 0bb5b2421524..4f81fbb3440b 100644 --- a/dev/sg/sg_start.go +++ b/dev/sg/sg_start.go @@ -8,6 +8,7 @@ import ( _ "net/http/pprof" "os" "path/filepath" + "slices" "sort" "strings" "time" @@ -79,6 +80,9 @@ sg start --debug=gitserver --error=enterprise-worker,enterprise-frontend enterpr # View configuration for a commandset sg start -describe single-program + +# Run a set of commands instead of a commandset +sg start --commands frontend gitserver `, Category: category.Dev, Flags: []cli.Flag{ @@ -94,7 +98,11 @@ sg start -describe single-program Name: "profile", Usage: "Starts up pprof on port 6060", }, - + &cli.BoolFlag{ + Name: "commands", + Aliases: []string{"cmd", "cmds"}, + Usage: "Signifies that you will be passing in individual commands to run, instead of a set of commands", + }, &cli.StringSliceFlag{ Name: "debug", Aliases: []string{"d"}, @@ -188,21 +196,6 @@ func startExec(ctx *cli.Context) error { return err } - args := ctx.Args().Slice() - if len(args) > 2 { - std.Out.WriteLine(output.Styled(output.StyleWarning, "ERROR: too many arguments")) - return flag.ErrHelp - } - - if len(args) != 1 { - if config.DefaultCommandset != "" { - args = append(args, config.DefaultCommandset) - } else { - std.Out.WriteLine(output.Styled(output.StyleWarning, "ERROR: No commandset specified and no 'defaultCommandset' specified in sg.config.yaml\n")) - return flag.ErrHelp - } - } - pid, exists, err := run.PidExistsWithArgs(os.Args[1:]) if err != nil { std.Out.WriteAlertf("Could not check if 'sg %s' is already running with the same arguments. Process: %d", strings.Join(os.Args[1:], " "), pid) @@ -219,10 +212,19 @@ func startExec(ctx *cli.Context) error { } } - commandset := args[0] - set, ok := config.Commandsets[commandset] - if !ok { - std.Out.WriteLine(output.Styledf(output.StyleWarning, "ERROR: commandset %q not found :(", commandset)) + // If the commands flag is passed, we just extract the command line arguments as + // a list of commands to run. + if ctx.Bool("commands") { + cmds, err := listToCommands(config, ctx.Args().Slice()) + if err != nil { + return err + } + return cmds.start(ctx.Context) + } + + set, err := getCommandSet(config, ctx.Args().Slice()) + if err != nil { + std.Out.WriteLine(output.Styledf(output.StyleWarning, "ERROR: extracting commandset failed %q :(", err)) return flag.ErrHelp } @@ -232,7 +234,7 @@ func startExec(ctx *cli.Context) error { return err } - return std.Out.WriteMarkdown(fmt.Sprintf("# %s\n\n```yaml\n%s\n```\n\n", commandset, string(out))) + return std.Out.WriteMarkdown(fmt.Sprintf("# %s\n\n```yaml\n%s\n```\n\n", set.Name, string(out))) } // If the commandset requires the dev-private repository to be cloned, we @@ -309,6 +311,15 @@ go tool pprof -help return startCommandSet(ctx.Context, set, config) } +func startCommandSet(ctx context.Context, set *sgconf.Commandset, conf *sgconf.Config) error { + commands, err := commandSetToCommands(conf, set) + if err != nil { + return err + } + + return commands.start(ctx) +} + func shouldUpdateDevPrivate(ctx context.Context, path, branch string) (bool, error) { // git fetch so that we check whether there are any remote changes if err := sgrun.Bash(ctx, fmt.Sprintf("git fetch origin %s", branch)).Dir(path).Run().Wait(); err != nil { @@ -323,91 +334,157 @@ func shouldUpdateDevPrivate(ctx context.Context, path, branch string) (bool, err } -func startCommandSet(ctx context.Context, set *sgconf.Commandset, conf *sgconf.Config) error { - if err := runChecksWithName(ctx, set.Checks); err != nil { - return err - } +func getCommandSet(config *sgconf.Config, args []string) (*sgconf.Commandset, error) { + switch length := len(args); { + case length > 1: + std.Out.WriteLine(output.Styled(output.StyleWarning, "ERROR: too many arguments")) + return nil, flag.ErrHelp + case length == 1: + if set, ok := config.Commandsets[args[0]]; ok { + return set, nil + } else { + std.Out.WriteLine(output.Styledf(output.StyleWarning, "ERROR: commandset %q not found :(", args[0])) + return nil, flag.ErrHelp + } - repoRoot, err := root.RepositoryRoot() - if err != nil { - return err + default: + if set, ok := config.Commandsets[config.DefaultCommandset]; ok { + return set, nil + } else { + std.Out.WriteLine(output.Styled(output.StyleWarning, "ERROR: No commandset specified and no 'defaultCommandset' specified in sg.config.yaml\n")) + return nil, flag.ErrHelp + } } +} - cmds, err := getCommands(set.Commands, set, conf.Commands) - if err != nil { - return err +type Commands struct { + checks []string + commands []run.SGConfigCommand + env map[string]string + ibazel *run.IBazel +} + +func (cmds *Commands) add(cmd ...run.SGConfigCommand) { + cmds.commands = append(cmds.commands, cmd...) +} + +func (cmds *Commands) getBazelTargets() (targets []string) { + for _, cmd := range cmds.commands { + target := cmd.GetBazelTarget() + if target != "" && !slices.Contains(targets, target) { + targets = append(targets, target) + } } - bcmds, err := getCommands(set.BazelCommands, set, conf.BazelCommands) - if err != nil { - return err + return targets +} + +func (cmds *Commands) getInstallers() (installers []run.Installer, err error) { + for _, cmd := range cmds.commands { + if installer, ok := cmd.(run.Installer); ok { + installers = append(installers, installer) + } } + targets := cmds.getBazelTargets() + if len(targets) > 0 { + if cmds.ibazel, err = run.NewIBazel(targets); err != nil { + return nil, err + } + installers = append(installers, cmds.ibazel) + } + return +} - dcmds, err := getCommands(set.DockerCommands, set, conf.DockerCommands) - if err != nil { +func (cmds *Commands) start(ctx context.Context) error { + if err := runChecksWithName(ctx, cmds.checks); err != nil { return err } - if len(cmds)+len(bcmds)+len(dcmds) == 0 { + if len(cmds.commands) == 0 { std.Out.WriteLine(output.Styled(output.StyleWarning, "WARNING: no commands to run")) return nil } - env := conf.Env - for k, v := range set.Env { - env[k] = v + installers, err := cmds.getInstallers() + if err != nil { + return err } - installers := make([]run.Installer, 0, len(cmds)+1) - for _, cmd := range cmds { - installers = append(installers, cmd) + if err := run.Install(ctx, cmds.env, verbose, installers); err != nil { + return err } - var ibazel *run.IBazel - if len(bcmds)+len(dcmds) > 0 { - var targets []string - for _, cmd := range bcmds { - targets = append(targets, cmd.Target) - } - for _, cmd := range dcmds { - targets = append(targets, cmd.Target) - } + if cmds.ibazel != nil { + cmds.ibazel.StartOutput() + defer cmds.ibazel.Close() + } - ibazel, err = run.NewIBazel(targets, repoRoot) - if err != nil { - return err + return run.Commands(ctx, cmds.env, verbose, cmds.commands) +} + +func listToCommands(config *sgconf.Config, names []string) (*Commands, error) { + var cmds Commands + for _, arg := range names { + if cmd, ok := getCommand(config, arg); ok { + cmds.add(cmd) + } else { + std.Out.WriteLine(output.Styledf(output.StyleWarning, "ERROR: command %q not found :(", arg)) + return nil, flag.ErrHelp } - defer ibazel.Close() - installers = append(installers, ibazel) } - if err := run.Install(ctx, env, verbose, installers...); err != nil { - return err + cmds.env = config.Env + + return &cmds, nil +} + +func commandSetToCommands(config *sgconf.Config, set *sgconf.Commandset) (*Commands, error) { + cmds := Commands{} + if ccmds, err := getCommands(set.Commands, set, config.Commands); err != nil { + return nil, err + } else { + cmds.add(ccmds...) } - if ibazel != nil { - ibazel.StartOutput() + if bcmds, err := getCommands(set.BazelCommands, set, config.BazelCommands); err != nil { + return nil, err + } else { + cmds.add(bcmds...) } - levelOverrides := logLevelOverrides() - configCmds := make([]run.SGConfigCommand, 0, len(bcmds)+len(cmds)) - for _, cmd := range bcmds { - enrichWithLogLevels(&cmd.Config, levelOverrides) - configCmds = append(configCmds, cmd) + if dcmds, err := getCommands(set.DockerCommands, set, config.DockerCommands); err != nil { + return nil, err + } else { + cmds.add(dcmds...) } - for _, cmd := range cmds { - enrichWithLogLevels(&cmd.Config, levelOverrides) - configCmds = append(configCmds, cmd) + cmds.env = config.Env + for k, v := range set.Env { + cmds.env[k] = v } - for _, cmd := range dcmds { - enrichWithLogLevels(&cmd.Config, levelOverrides) - configCmds = append(configCmds, cmd) + + addLogLevel := createLogLevelAdder(logLevelOverrides()) + for i, cmd := range cmds.commands { + cmds.commands[i] = cmd.UpdateConfig(addLogLevel) } - return run.Commands(ctx, env, verbose, configCmds...) + return &cmds, nil + } -func getCommands[T run.SGConfigCommand](commands []string, set *sgconf.Commandset, conf map[string]T) ([]T, error) { +func getCommand(config *sgconf.Config, name string) (run.SGConfigCommand, bool) { + if cmd, ok := config.BazelCommands[name]; ok { + return cmd, ok + } + if cmd, ok := config.DockerCommands[name]; ok { + return cmd, ok + } + if cmd, ok := config.Commands[name]; ok { + return cmd, ok + } + return nil, false +} + +func getCommands[T run.SGConfigCommand](commands []string, set *sgconf.Commandset, conf map[string]T) ([]run.SGConfigCommand, error) { exceptList := exceptServices.Value() exceptSet := make(map[string]interface{}, len(exceptList)) for _, svc := range exceptList { @@ -420,7 +497,7 @@ func getCommands[T run.SGConfigCommand](commands []string, set *sgconf.Commandse onlySet[svc] = struct{}{} } - cmds := make([]T, 0, len(commands)) + cmds := make([]run.SGConfigCommand, 0, len(commands)) for _, name := range commands { cmd, ok := conf[name] if !ok { @@ -467,16 +544,18 @@ func logLevelOverrides() map[string]string { } // enrichWithLogLevels will add any logger level overrides to a given command if they have been specified. -func enrichWithLogLevels(config *run.SGConfigCommandOptions, overrides map[string]string) { - logLevelVariable := "SRC_LOG_LEVEL" +func createLogLevelAdder(overrides map[string]string) func(*run.SGConfigCommandOptions) { + return func(config *run.SGConfigCommandOptions) { + logLevelVariable := "SRC_LOG_LEVEL" + + if level, ok := overrides[config.Name]; ok { + std.Out.WriteLine(output.Styledf(output.StylePending, "Setting log level: %s for command %s.", level, config.Name)) + if config.Env == nil { + config.Env = make(map[string]string, 1) + } - if level, ok := overrides[config.Name]; ok { - std.Out.WriteLine(output.Styledf(output.StylePending, "Setting log level: %s for command %s.", level, config.Name)) - if config.Env == nil { - config.Env = make(map[string]string, 1) config.Env[logLevelVariable] = level } - config.Env[logLevelVariable] = level } }