metal: split host creation from enrollment

This is needeed for bootstrapping the control plane,
because it's a CRD so can't be registered until the control plane is running.

It's also quite nice because we might want to review the contents of the
host CRD, e.g. to verify the key out-of-band.

cmd/kops/toolbox_enroll.go

@@ -51,5 +51,7 @@ func NewCmdToolboxEnroll(f commandutils.Factory, out io.Writer) *cobra.Command {
 	cmd.Flags().StringVar(&options.SSHUser, "ssh-user", options.SSHUser, "user for ssh")
 	cmd.Flags().IntVar(&options.SSHPort, "ssh-port", options.SSHPort, "port for ssh")
 
+	cmd.Flags().BoolVar(&options.BuildHost, "build-host", options.BuildHost, "only build the host resource, don't apply it or enroll the node")
+
 	return cmd
 }

pkg/commands/toolbox_enroll.go

@@ -66,6 +66,9 @@ type ToolboxEnrollOptions struct {
 	SSHUser string
 	SSHPort int
 
+	// BuildHost is a flag to only build the host resource, don't apply it or enroll the node
+	BuildHost bool
+
 	// PodCIDRs is the list of IP Address ranges to use for pods that run on this node
 	PodCIDRs []string
 }
@@ -100,14 +103,6 @@ func RunToolboxEnroll(ctx context.Context, f commandutils.Factory, out io.Writer
 	if err != nil {
 		return err
 	}
-	fullInstanceGroup, err := configBuilder.GetFullInstanceGroup(ctx)
-	if err != nil {
-		return err
-	}
-	bootstrapData, err := configBuilder.GetBootstrapData(ctx)
-	if err != nil {
-		return err
-	}
 
 	// Enroll the node over SSH.
 	if options.Host != "" {
@@ -116,72 +111,109 @@ func RunToolboxEnroll(ctx context.Context, f commandutils.Factory, out io.Writer
 			return err
 		}
 
-		if err := enrollHost(ctx, fullInstanceGroup, options, bootstrapData, restConfig); err != nil {
+		sudo := true
+		if options.SSHUser == "root" {
+			sudo = false
+		}
+
+		sshTarget, err := NewSSHHost(ctx, options.Host, options.SSHPort, options.SSHUser, sudo)
+		if err != nil {
 			return err
 		}
-	}
+		defer sshTarget.Close()
 
-	return nil
-}
+		hostData, err := buildHostData(ctx, sshTarget, options)
+		if err != nil {
+			return err
+		}
 
-func enrollHost(ctx context.Context, ig *kops.InstanceGroup, options *ToolboxEnrollOptions, bootstrapData *BootstrapData, restConfig *rest.Config) error {
-	scheme := runtime.NewScheme()
-	if err := v1alpha2.AddToScheme(scheme); err != nil {
-		return fmt.Errorf("building kubernetes scheme: %w", err)
-	}
-	kubeClient, err := client.New(restConfig, client.Options{
-		Scheme: scheme,
-	})
-	if err != nil {
-		return fmt.Errorf("building kubernetes client: %w", err)
-	}
+		if options.BuildHost {
+			klog.Infof("building host data for %+v", hostData)
+			b, err := yaml.Marshal(hostData)
+			if err != nil {
+				return fmt.Errorf("error marshalling host data: %w", err)
+			}
+			fmt.Fprintf(out, "%s\n", string(b))
+		} else {
+			fullInstanceGroup, err := configBuilder.GetFullInstanceGroup(ctx)
+			if err != nil {
+				return err
+			}
+			bootstrapData, err := configBuilder.GetBootstrapData(ctx)
+			if err != nil {
+				return err
+			}
 
-	sudo := true
-	if options.SSHUser == "root" {
-		sudo = false
+			if err := enrollHost(ctx, fullInstanceGroup, options, bootstrapData, restConfig, hostData, sshTarget); err != nil {
+				return err
+			}
+		}
 	}
 
-	sshTarget, err := NewSSHHost(ctx, options.Host, options.SSHPort, options.SSHUser, sudo)
-	if err != nil {
-		return err
-	}
-	defer sshTarget.Close()
+	return nil
+}
 
+// buildHostData builds an instance of the Host CRD, based on information in the options and by SSHing to the target host.
+func buildHostData(ctx context.Context, sshTarget *SSHHost, options *ToolboxEnrollOptions) (*v1alpha2.Host, error) {
 	publicKeyPath := "/etc/kubernetes/kops/pki/machine/public.pem"
 
 	publicKeyBytes, err := sshTarget.readFile(ctx, publicKeyPath)
 	if err != nil {
 		if errors.Is(err, fs.ErrNotExist) {
 			publicKeyBytes = nil
 		} else {
-			return fmt.Errorf("error reading public key %q: %w", publicKeyPath, err)
+			return nil, fmt.Errorf("error reading public key %q: %w", publicKeyPath, err)
 		}
 	}
 
+	// Create the key if it doesn't exist
 	publicKeyBytes = bytes.TrimSpace(publicKeyBytes)
 	if len(publicKeyBytes) == 0 {
-		if _, err := sshTarget.runScript(ctx, scriptCreateKey, ExecOptions{Sudo: sudo, Echo: true}); err != nil {
-			return err
+		if _, err := sshTarget.runScript(ctx, scriptCreateKey, ExecOptions{Echo: true}); err != nil {
+			return nil, err
 		}
 
 		b, err := sshTarget.readFile(ctx, publicKeyPath)
 		if err != nil {
-			return fmt.Errorf("error reading public key %q (after creation): %w", publicKeyPath, err)
+			return nil, fmt.Errorf("error reading public key %q (after creation): %w", publicKeyPath, err)
 		}
 		publicKeyBytes = b
 	}
 	klog.Infof("public key is %s", string(publicKeyBytes))
 
 	hostname, err := sshTarget.getHostname(ctx)
 	if err != nil {
-		return err
+		return nil, err
+	}
+
+	host := &v1alpha2.Host{}
+	host.SetGroupVersionKind(v1alpha2.SchemeGroupVersion.WithKind("Host"))
+	host.Namespace = "kops-system"
+	host.Name = hostname
+	host.Spec.InstanceGroup = options.InstanceGroup
+	host.Spec.PublicKey = string(publicKeyBytes)
+	host.Spec.PodCIDRs = options.PodCIDRs
+
+	return host, nil
+}
+
+func enrollHost(ctx context.Context, ig *kops.InstanceGroup, options *ToolboxEnrollOptions, bootstrapData *BootstrapData, restConfig *rest.Config, hostData *v1alpha2.Host, sshTarget *SSHHost) error {
+	scheme := runtime.NewScheme()
+	if err := v1alpha2.AddToScheme(scheme); err != nil {
+		return fmt.Errorf("building kubernetes scheme: %w", err)
+	}
+	kubeClient, err := client.New(restConfig, client.Options{
+		Scheme: scheme,
+	})
+	if err != nil {
+		return fmt.Errorf("building kubernetes client: %w", err)
 	}
 
 	// We can't create the host resource in the API server for control-plane nodes,
 	// because the API server (likely) isn't running yet.
 	if !ig.IsControlPlane() {
-		if err := createHostResourceInAPIServer(ctx, options, hostname, publicKeyBytes, kubeClient); err != nil {
-			return err
+		if err := kubeClient.Create(ctx, hostData); err != nil {
+			return fmt.Errorf("failed to create host %s/%s: %w", hostData.Namespace, hostData.Name, err)
 		}
 	}
 
@@ -192,28 +224,13 @@ func enrollHost(ctx context.Context, ig *kops.InstanceGroup, options *ToolboxEnr
 	}
 
 	if len(bootstrapData.NodeupScript) != 0 {
-		if _, err := sshTarget.runScript(ctx, string(bootstrapData.NodeupScript), ExecOptions{Sudo: sudo, Echo: true}); err != nil {
+		if _, err := sshTarget.runScript(ctx, string(bootstrapData.NodeupScript), ExecOptions{Echo: true}); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func createHostResourceInAPIServer(ctx context.Context, options *ToolboxEnrollOptions, nodeName string, publicKey []byte, client client.Client) error {
-	host := &v1alpha2.Host{}
-	host.Namespace = "kops-system"
-	host.Name = nodeName
-	host.Spec.InstanceGroup = options.InstanceGroup
-	host.Spec.PublicKey = string(publicKey)
-	host.Spec.PodCIDRs = options.PodCIDRs
-
-	if err := client.Create(ctx, host); err != nil {
-		return fmt.Errorf("failed to create host %s/%s: %w", host.Namespace, host.Name, err)
-	}
-
-	return nil
-}
-
 const scriptCreateKey = `
 #!/bin/bash
 set -o errexit
@@ -314,7 +331,7 @@ func (s *SSHHost) runScript(ctx context.Context, script string, options ExecOpti
 	p := vfs.NewSSHPath(s.sshClient, s.hostname, scriptPath, s.sudo)
 
 	defer func() {
-		if _, err := s.runCommand(ctx, "rm -rf "+tempDir, ExecOptions{Sudo: s.sudo, Echo: false}); err != nil {
+		if _, err := s.runCommand(ctx, "rm -rf "+tempDir, ExecOptions{Echo: false}); err != nil {
 			klog.Warningf("error cleaning up temp directory %q: %v", tempDir, err)
 		}
 	}()
@@ -335,7 +352,6 @@ type CommandOutput struct {
 
 // ExecOptions holds options for running a command remotely.
 type ExecOptions struct {
-	Sudo bool
 	Echo bool
 }
 
@@ -352,10 +368,11 @@ func (s *SSHHost) runCommand(ctx context.Context, command string, options ExecOp
 	session.Stderr = &output.Stderr
 
 	if options.Echo {
-		session.Stdout = io.MultiWriter(os.Stdout, session.Stdout)
+		// We send both to stderr, so we don't "corrupt" stdout
+		session.Stdout = io.MultiWriter(os.Stderr, session.Stdout)
 		session.Stderr = io.MultiWriter(os.Stderr, session.Stderr)
 	}
-	if options.Sudo {
+	if s.sudo {
 		command = "sudo " + command
 	}
 	if err := session.Run(command); err != nil {
@@ -367,7 +384,7 @@ func (s *SSHHost) runCommand(ctx context.Context, command string, options ExecOp
 // getHostname gets the hostname of the SSH target.
 // This is used as the node name when registering the node.
 func (s *SSHHost) getHostname(ctx context.Context) (string, error) {
-	output, err := s.runCommand(ctx, "hostname", ExecOptions{Sudo: false, Echo: true})
+	output, err := s.runCommand(ctx, "hostname", ExecOptions{Echo: true})
 	if err != nil {
 		return "", fmt.Errorf("failed to get hostname: %w", err)
 	}

tests/e2e/scenarios/bare-metal/scenario-ipv6

@@ -237,6 +237,13 @@ for i in {1..60}; do
   sleep 10
 done
 
+# Create CRD and namespace for host records
+kubectl create ns kops-system
+kubectl apply -f ${REPO_ROOT}/k8s/crds/kops.k8s.io_hosts.yaml
+
+# Create the host record (we can't auto create for control plane nodes)
+${KOPS} toolbox enroll --cluster ${CLUSTER_NAME} --instance-group control-plane-main --host ${VM0_IP} --pod-cidr ${VM0_POD_CIDR} --v=2 --build-host | kubectl apply -f -
+
 kubectl get nodes
 kubectl get pods -A
 
@@ -247,9 +254,6 @@ sleep 10
 kubectl get nodes
 kubectl get pods -A
 
-# For host records
-kubectl create ns kops-system
-kubectl apply -f ${REPO_ROOT}/k8s/crds/kops.k8s.io_hosts.yaml
 
 # kops-controller extra permissions
 kubectl apply --server-side -f - <<EOF

upup/pkg/fi/loader/options_loader.go

@@ -55,7 +55,7 @@ func (l *OptionsLoader[T]) iterate(userConfig T, current T) (T, error) {
 	reflectutils.JSONMergeStruct(next, current)
 
 	for _, t := range l.Builders {
-		klog.V(2).Infof("executing builder %T", t)
+		klog.V(4).Infof("executing builder %T", t)
 
 		err := t.BuildOptions(next)
 		if err != nil {