From 5392938eca8e54071e5a498dca0ebcf9cba73e47 Mon Sep 17 00:00:00 2001 From: justinsb Date: Wed, 19 Feb 2025 06:51:04 -0500 Subject: [PATCH] metal: split host creation from enrollment This is needeed for bootstrapping the control plane, because it's a CRD so can't be registered until the control plane is running. It's also quite nice because we might want to review the contents of the host CRD, e.g. to verify the key out-of-band. --- cmd/kops/toolbox_enroll.go | 2 + docs/cli/kops_toolbox_enroll.md | 1 + pkg/commands/toolbox_enroll.go | 141 +++++++++++-------- tests/e2e/scenarios/bare-metal/scenario-ipv6 | 10 +- upup/pkg/fi/loader/options_loader.go | 2 +- 5 files changed, 90 insertions(+), 66 deletions(-) diff --git a/cmd/kops/toolbox_enroll.go b/cmd/kops/toolbox_enroll.go index e5eb7067aeb4d..4fc7f648214e4 100644 --- a/cmd/kops/toolbox_enroll.go +++ b/cmd/kops/toolbox_enroll.go @@ -51,5 +51,7 @@ func NewCmdToolboxEnroll(f commandutils.Factory, out io.Writer) *cobra.Command { cmd.Flags().StringVar(&options.SSHUser, "ssh-user", options.SSHUser, "user for ssh") cmd.Flags().IntVar(&options.SSHPort, "ssh-port", options.SSHPort, "port for ssh") + cmd.Flags().BoolVar(&options.BuildHost, "build-host", options.BuildHost, "only build the host resource, don't apply it or enroll the node") + return cmd } diff --git a/docs/cli/kops_toolbox_enroll.md b/docs/cli/kops_toolbox_enroll.md index 1d40473486fbf..796f4686f5cbc 100644 --- a/docs/cli/kops_toolbox_enroll.md +++ b/docs/cli/kops_toolbox_enroll.md @@ -22,6 +22,7 @@ kops toolbox enroll [CLUSTER] [flags] ### Options ``` + --build-host only build the host resource, don't apply it or enroll the node --cluster string Name of cluster to join -h, --help help for enroll --host string IP/hostname for machine to add diff --git a/pkg/commands/toolbox_enroll.go b/pkg/commands/toolbox_enroll.go index 013e66a88ab3d..a0382b4d54f8c 100644 --- a/pkg/commands/toolbox_enroll.go +++ b/pkg/commands/toolbox_enroll.go @@ -66,6 +66,9 @@ type ToolboxEnrollOptions struct { SSHUser string SSHPort int + // BuildHost is a flag to only build the host resource, don't apply it or enroll the node + BuildHost bool + // PodCIDRs is the list of IP Address ranges to use for pods that run on this node PodCIDRs []string } @@ -100,14 +103,6 @@ func RunToolboxEnroll(ctx context.Context, f commandutils.Factory, out io.Writer if err != nil { return err } - fullInstanceGroup, err := configBuilder.GetFullInstanceGroup(ctx) - if err != nil { - return err - } - bootstrapData, err := configBuilder.GetBootstrapData(ctx) - if err != nil { - return err - } // Enroll the node over SSH. if options.Host != "" { @@ -116,72 +111,109 @@ func RunToolboxEnroll(ctx context.Context, f commandutils.Factory, out io.Writer return err } - if err := enrollHost(ctx, fullInstanceGroup, options, bootstrapData, restConfig); err != nil { + sudo := true + if options.SSHUser == "root" { + sudo = false + } + + target, err := NewSSHHost(ctx, options.Host, options.SSHPort, options.SSHUser, sudo) + if err != nil { return err } - } + defer target.Close() - return nil -} + hostData, err := buildHostData(ctx, target, options) + if err != nil { + return err + } -func enrollHost(ctx context.Context, ig *kops.InstanceGroup, options *ToolboxEnrollOptions, bootstrapData *BootstrapData, restConfig *rest.Config) error { - scheme := runtime.NewScheme() - if err := v1alpha2.AddToScheme(scheme); err != nil { - return fmt.Errorf("building kubernetes scheme: %w", err) - } - kubeClient, err := client.New(restConfig, client.Options{ - Scheme: scheme, - }) - if err != nil { - return fmt.Errorf("building kubernetes client: %w", err) - } + if options.BuildHost { + klog.Infof("building host data for %+v", hostData) + b, err := yaml.Marshal(hostData) + if err != nil { + return fmt.Errorf("error marshalling host data: %w", err) + } + fmt.Fprintf(out, "%s\n", string(b)) + } else { + fullInstanceGroup, err := configBuilder.GetFullInstanceGroup(ctx) + if err != nil { + return err + } + bootstrapData, err := configBuilder.GetBootstrapData(ctx) + if err != nil { + return err + } - sudo := true - if options.SSHUser == "root" { - sudo = false + if err := enrollHost(ctx, fullInstanceGroup, options, bootstrapData, restConfig, hostData, target); err != nil { + return err + } + } } - sshTarget, err := NewSSHHost(ctx, options.Host, options.SSHPort, options.SSHUser, sudo) - if err != nil { - return err - } - defer sshTarget.Close() + return nil +} +// buildHostData builds an instance of the Host CRD, based on information in the options and by SSHing to the target host. +func buildHostData(ctx context.Context, target *SSHHost, options *ToolboxEnrollOptions) (*v1alpha2.Host, error) { publicKeyPath := "/etc/kubernetes/kops/pki/machine/public.pem" - publicKeyBytes, err := sshTarget.readFile(ctx, publicKeyPath) + publicKeyBytes, err := target.readFile(ctx, publicKeyPath) if err != nil { if errors.Is(err, fs.ErrNotExist) { publicKeyBytes = nil } else { - return fmt.Errorf("error reading public key %q: %w", publicKeyPath, err) + return nil, fmt.Errorf("error reading public key %q: %w", publicKeyPath, err) } } + // Create the key if it doesn't exist publicKeyBytes = bytes.TrimSpace(publicKeyBytes) if len(publicKeyBytes) == 0 { - if _, err := sshTarget.runScript(ctx, scriptCreateKey, ExecOptions{Sudo: sudo, Echo: true}); err != nil { - return err + if _, err := target.runScript(ctx, scriptCreateKey, ExecOptions{Echo: true}); err != nil { + return nil, err } - b, err := sshTarget.readFile(ctx, publicKeyPath) + b, err := target.readFile(ctx, publicKeyPath) if err != nil { - return fmt.Errorf("error reading public key %q (after creation): %w", publicKeyPath, err) + return nil, fmt.Errorf("error reading public key %q (after creation): %w", publicKeyPath, err) } publicKeyBytes = b } klog.Infof("public key is %s", string(publicKeyBytes)) - hostname, err := sshTarget.getHostname(ctx) + hostname, err := target.getHostname(ctx) if err != nil { - return err + return nil, err + } + + host := &v1alpha2.Host{} + host.SetGroupVersionKind(v1alpha2.SchemeGroupVersion.WithKind("Host")) + host.Namespace = "kops-system" + host.Name = hostname + host.Spec.InstanceGroup = options.InstanceGroup + host.Spec.PublicKey = string(publicKeyBytes) + host.Spec.PodCIDRs = options.PodCIDRs + + return host, nil +} + +func enrollHost(ctx context.Context, ig *kops.InstanceGroup, options *ToolboxEnrollOptions, bootstrapData *BootstrapData, restConfig *rest.Config, hostData *v1alpha2.Host, sshTarget *SSHHost) error { + scheme := runtime.NewScheme() + if err := v1alpha2.AddToScheme(scheme); err != nil { + return fmt.Errorf("building kubernetes scheme: %w", err) + } + kubeClient, err := client.New(restConfig, client.Options{ + Scheme: scheme, + }) + if err != nil { + return fmt.Errorf("building kubernetes client: %w", err) } // We can't create the host resource in the API server for control-plane nodes, // because the API server (likely) isn't running yet. if !ig.IsControlPlane() { - if err := createHostResourceInAPIServer(ctx, options, hostname, publicKeyBytes, kubeClient); err != nil { - return err + if err := kubeClient.Create(ctx, hostData); err != nil { + return fmt.Errorf("failed to create host %s/%s: %w", hostData.Namespace, hostData.Name, err) } } @@ -192,28 +224,13 @@ func enrollHost(ctx context.Context, ig *kops.InstanceGroup, options *ToolboxEnr } if len(bootstrapData.NodeupScript) != 0 { - if _, err := sshTarget.runScript(ctx, string(bootstrapData.NodeupScript), ExecOptions{Sudo: sudo, Echo: true}); err != nil { + if _, err := sshTarget.runScript(ctx, string(bootstrapData.NodeupScript), ExecOptions{Echo: true}); err != nil { return err } } return nil } -func createHostResourceInAPIServer(ctx context.Context, options *ToolboxEnrollOptions, nodeName string, publicKey []byte, client client.Client) error { - host := &v1alpha2.Host{} - host.Namespace = "kops-system" - host.Name = nodeName - host.Spec.InstanceGroup = options.InstanceGroup - host.Spec.PublicKey = string(publicKey) - host.Spec.PodCIDRs = options.PodCIDRs - - if err := client.Create(ctx, host); err != nil { - return fmt.Errorf("failed to create host %s/%s: %w", host.Namespace, host.Name, err) - } - - return nil -} - const scriptCreateKey = ` #!/bin/bash set -o errexit @@ -314,7 +331,7 @@ func (s *SSHHost) runScript(ctx context.Context, script string, options ExecOpti p := vfs.NewSSHPath(s.sshClient, s.hostname, scriptPath, s.sudo) defer func() { - if _, err := s.runCommand(ctx, "rm -rf "+tempDir, ExecOptions{Sudo: s.sudo, Echo: false}); err != nil { + if _, err := s.runCommand(ctx, "rm -rf "+tempDir, ExecOptions{Echo: false}); err != nil { klog.Warningf("error cleaning up temp directory %q: %v", tempDir, err) } }() @@ -335,7 +352,6 @@ type CommandOutput struct { // ExecOptions holds options for running a command remotely. type ExecOptions struct { - Sudo bool Echo bool } @@ -352,10 +368,11 @@ func (s *SSHHost) runCommand(ctx context.Context, command string, options ExecOp session.Stderr = &output.Stderr if options.Echo { - session.Stdout = io.MultiWriter(os.Stdout, session.Stdout) + // We send both to stderr, so we don't "corrupt" stdout + session.Stdout = io.MultiWriter(os.Stderr, session.Stdout) session.Stderr = io.MultiWriter(os.Stderr, session.Stderr) } - if options.Sudo { + if s.sudo { command = "sudo " + command } if err := session.Run(command); err != nil { @@ -367,7 +384,7 @@ func (s *SSHHost) runCommand(ctx context.Context, command string, options ExecOp // getHostname gets the hostname of the SSH target. // This is used as the node name when registering the node. func (s *SSHHost) getHostname(ctx context.Context) (string, error) { - output, err := s.runCommand(ctx, "hostname", ExecOptions{Sudo: false, Echo: true}) + output, err := s.runCommand(ctx, "hostname", ExecOptions{Echo: true}) if err != nil { return "", fmt.Errorf("failed to get hostname: %w", err) } diff --git a/tests/e2e/scenarios/bare-metal/scenario-ipv6 b/tests/e2e/scenarios/bare-metal/scenario-ipv6 index c31c97debf1d4..e6b5f35c8081d 100755 --- a/tests/e2e/scenarios/bare-metal/scenario-ipv6 +++ b/tests/e2e/scenarios/bare-metal/scenario-ipv6 @@ -237,6 +237,13 @@ for i in {1..60}; do sleep 10 done +# Create CRD and namespace for host records +kubectl create ns kops-system +kubectl apply -f ${REPO_ROOT}/k8s/crds/kops.k8s.io_hosts.yaml + +# Create the host record (we can't auto create for control plane nodes) +${KOPS} toolbox enroll --cluster ${CLUSTER_NAME} --instance-group control-plane-main --host ${VM0_IP} --pod-cidr ${VM0_POD_CIDR} --v=2 --build-host | kubectl apply -f - + kubectl get nodes kubectl get pods -A @@ -247,9 +254,6 @@ sleep 10 kubectl get nodes kubectl get pods -A -# For host records -kubectl create ns kops-system -kubectl apply -f ${REPO_ROOT}/k8s/crds/kops.k8s.io_hosts.yaml # kops-controller extra permissions kubectl apply --server-side -f - <