diff --git a/cmd/kops/toolbox_enroll.go b/cmd/kops/toolbox_enroll.go index e5eb7067aeb4d..4fc7f648214e4 100644 --- a/cmd/kops/toolbox_enroll.go +++ b/cmd/kops/toolbox_enroll.go @@ -51,5 +51,7 @@ func NewCmdToolboxEnroll(f commandutils.Factory, out io.Writer) *cobra.Command { cmd.Flags().StringVar(&options.SSHUser, "ssh-user", options.SSHUser, "user for ssh") cmd.Flags().IntVar(&options.SSHPort, "ssh-port", options.SSHPort, "port for ssh") + cmd.Flags().BoolVar(&options.BuildHost, "build-host", options.BuildHost, "only build the host resource, don't apply it or enroll the node") + return cmd } diff --git a/docs/cli/kops_toolbox_enroll.md b/docs/cli/kops_toolbox_enroll.md index 1d40473486fbf..796f4686f5cbc 100644 --- a/docs/cli/kops_toolbox_enroll.md +++ b/docs/cli/kops_toolbox_enroll.md @@ -22,6 +22,7 @@ kops toolbox enroll [CLUSTER] [flags] ### Options ``` + --build-host only build the host resource, don't apply it or enroll the node --cluster string Name of cluster to join -h, --help help for enroll --host string IP/hostname for machine to add diff --git a/pkg/commands/toolbox_enroll.go b/pkg/commands/toolbox_enroll.go index 013e66a88ab3d..cf22d3d795f65 100644 --- a/pkg/commands/toolbox_enroll.go +++ b/pkg/commands/toolbox_enroll.go @@ -66,6 +66,9 @@ type ToolboxEnrollOptions struct { SSHUser string SSHPort int + // BuildHost is a flag to only build the host resource, don't apply it or enroll the node + BuildHost bool + // PodCIDRs is the list of IP Address ranges to use for pods that run on this node PodCIDRs []string } @@ -100,14 +103,6 @@ func RunToolboxEnroll(ctx context.Context, f commandutils.Factory, out io.Writer if err != nil { return err } - fullInstanceGroup, err := configBuilder.GetFullInstanceGroup(ctx) - if err != nil { - return err - } - bootstrapData, err := configBuilder.GetBootstrapData(ctx) - if err != nil { - return err - } // Enroll the node over SSH. if options.Host != "" { @@ -116,37 +111,50 @@ func RunToolboxEnroll(ctx context.Context, f commandutils.Factory, out io.Writer return err } - if err := enrollHost(ctx, fullInstanceGroup, options, bootstrapData, restConfig); err != nil { + sudo := true + if options.SSHUser == "root" { + sudo = false + } + + sshTarget, err := NewSSHHost(ctx, options.Host, options.SSHPort, options.SSHUser, sudo) + if err != nil { return err } - } + defer sshTarget.Close() - return nil -} + hostData, err := buildHostData(ctx, sshTarget, options) + if err != nil { + return err + } -func enrollHost(ctx context.Context, ig *kops.InstanceGroup, options *ToolboxEnrollOptions, bootstrapData *BootstrapData, restConfig *rest.Config) error { - scheme := runtime.NewScheme() - if err := v1alpha2.AddToScheme(scheme); err != nil { - return fmt.Errorf("building kubernetes scheme: %w", err) - } - kubeClient, err := client.New(restConfig, client.Options{ - Scheme: scheme, - }) - if err != nil { - return fmt.Errorf("building kubernetes client: %w", err) - } + if options.BuildHost { + klog.Infof("building host data for %+v", hostData) + b, err := yaml.Marshal(hostData) + if err != nil { + return fmt.Errorf("error marshalling host data: %w", err) + } + fmt.Fprintf(out, "%s\n", string(b)) + } else { + fullInstanceGroup, err := configBuilder.GetFullInstanceGroup(ctx) + if err != nil { + return err + } + bootstrapData, err := configBuilder.GetBootstrapData(ctx) + if err != nil { + return err + } - sudo := true - if options.SSHUser == "root" { - sudo = false + if err := enrollHost(ctx, fullInstanceGroup, options, bootstrapData, restConfig, hostData, sshTarget); err != nil { + return err + } + } } - sshTarget, err := NewSSHHost(ctx, options.Host, options.SSHPort, options.SSHUser, sudo) - if err != nil { - return err - } - defer sshTarget.Close() + return nil +} +// buildHostData builds an instance of the Host CRD, based on information in the options and by SSHing to the target host. +func buildHostData(ctx context.Context, sshTarget *SSHHost, options *ToolboxEnrollOptions) (*v1alpha2.Host, error) { publicKeyPath := "/etc/kubernetes/kops/pki/machine/public.pem" publicKeyBytes, err := sshTarget.readFile(ctx, publicKeyPath) @@ -154,19 +162,20 @@ func enrollHost(ctx context.Context, ig *kops.InstanceGroup, options *ToolboxEnr if errors.Is(err, fs.ErrNotExist) { publicKeyBytes = nil } else { - return fmt.Errorf("error reading public key %q: %w", publicKeyPath, err) + return nil, fmt.Errorf("error reading public key %q: %w", publicKeyPath, err) } } + // Create the key if it doesn't exist publicKeyBytes = bytes.TrimSpace(publicKeyBytes) if len(publicKeyBytes) == 0 { - if _, err := sshTarget.runScript(ctx, scriptCreateKey, ExecOptions{Sudo: sudo, Echo: true}); err != nil { - return err + if _, err := sshTarget.runScript(ctx, scriptCreateKey, ExecOptions{Echo: true}); err != nil { + return nil, err } b, err := sshTarget.readFile(ctx, publicKeyPath) if err != nil { - return fmt.Errorf("error reading public key %q (after creation): %w", publicKeyPath, err) + return nil, fmt.Errorf("error reading public key %q (after creation): %w", publicKeyPath, err) } publicKeyBytes = b } @@ -174,14 +183,37 @@ func enrollHost(ctx context.Context, ig *kops.InstanceGroup, options *ToolboxEnr hostname, err := sshTarget.getHostname(ctx) if err != nil { - return err + return nil, err + } + + host := &v1alpha2.Host{} + host.SetGroupVersionKind(v1alpha2.SchemeGroupVersion.WithKind("Host")) + host.Namespace = "kops-system" + host.Name = hostname + host.Spec.InstanceGroup = options.InstanceGroup + host.Spec.PublicKey = string(publicKeyBytes) + host.Spec.PodCIDRs = options.PodCIDRs + + return host, nil +} + +func enrollHost(ctx context.Context, ig *kops.InstanceGroup, options *ToolboxEnrollOptions, bootstrapData *BootstrapData, restConfig *rest.Config, hostData *v1alpha2.Host, sshTarget *SSHHost) error { + scheme := runtime.NewScheme() + if err := v1alpha2.AddToScheme(scheme); err != nil { + return fmt.Errorf("building kubernetes scheme: %w", err) + } + kubeClient, err := client.New(restConfig, client.Options{ + Scheme: scheme, + }) + if err != nil { + return fmt.Errorf("building kubernetes client: %w", err) } // We can't create the host resource in the API server for control-plane nodes, // because the API server (likely) isn't running yet. if !ig.IsControlPlane() { - if err := createHostResourceInAPIServer(ctx, options, hostname, publicKeyBytes, kubeClient); err != nil { - return err + if err := kubeClient.Create(ctx, hostData); err != nil { + return fmt.Errorf("failed to create host %s/%s: %w", hostData.Namespace, hostData.Name, err) } } @@ -192,28 +224,13 @@ func enrollHost(ctx context.Context, ig *kops.InstanceGroup, options *ToolboxEnr } if len(bootstrapData.NodeupScript) != 0 { - if _, err := sshTarget.runScript(ctx, string(bootstrapData.NodeupScript), ExecOptions{Sudo: sudo, Echo: true}); err != nil { + if _, err := sshTarget.runScript(ctx, string(bootstrapData.NodeupScript), ExecOptions{Echo: true}); err != nil { return err } } return nil } -func createHostResourceInAPIServer(ctx context.Context, options *ToolboxEnrollOptions, nodeName string, publicKey []byte, client client.Client) error { - host := &v1alpha2.Host{} - host.Namespace = "kops-system" - host.Name = nodeName - host.Spec.InstanceGroup = options.InstanceGroup - host.Spec.PublicKey = string(publicKey) - host.Spec.PodCIDRs = options.PodCIDRs - - if err := client.Create(ctx, host); err != nil { - return fmt.Errorf("failed to create host %s/%s: %w", host.Namespace, host.Name, err) - } - - return nil -} - const scriptCreateKey = ` #!/bin/bash set -o errexit @@ -314,7 +331,7 @@ func (s *SSHHost) runScript(ctx context.Context, script string, options ExecOpti p := vfs.NewSSHPath(s.sshClient, s.hostname, scriptPath, s.sudo) defer func() { - if _, err := s.runCommand(ctx, "rm -rf "+tempDir, ExecOptions{Sudo: s.sudo, Echo: false}); err != nil { + if _, err := s.runCommand(ctx, "rm -rf "+tempDir, ExecOptions{Echo: false}); err != nil { klog.Warningf("error cleaning up temp directory %q: %v", tempDir, err) } }() @@ -335,7 +352,6 @@ type CommandOutput struct { // ExecOptions holds options for running a command remotely. type ExecOptions struct { - Sudo bool Echo bool } @@ -352,10 +368,11 @@ func (s *SSHHost) runCommand(ctx context.Context, command string, options ExecOp session.Stderr = &output.Stderr if options.Echo { - session.Stdout = io.MultiWriter(os.Stdout, session.Stdout) + // We send both to stderr, so we don't "corrupt" stdout + session.Stdout = io.MultiWriter(os.Stderr, session.Stdout) session.Stderr = io.MultiWriter(os.Stderr, session.Stderr) } - if options.Sudo { + if s.sudo { command = "sudo " + command } if err := session.Run(command); err != nil { @@ -367,7 +384,7 @@ func (s *SSHHost) runCommand(ctx context.Context, command string, options ExecOp // getHostname gets the hostname of the SSH target. // This is used as the node name when registering the node. func (s *SSHHost) getHostname(ctx context.Context) (string, error) { - output, err := s.runCommand(ctx, "hostname", ExecOptions{Sudo: false, Echo: true}) + output, err := s.runCommand(ctx, "hostname", ExecOptions{Echo: true}) if err != nil { return "", fmt.Errorf("failed to get hostname: %w", err) } diff --git a/tests/e2e/scenarios/bare-metal/scenario-ipv6 b/tests/e2e/scenarios/bare-metal/scenario-ipv6 index c31c97debf1d4..e6b5f35c8081d 100755 --- a/tests/e2e/scenarios/bare-metal/scenario-ipv6 +++ b/tests/e2e/scenarios/bare-metal/scenario-ipv6 @@ -237,6 +237,13 @@ for i in {1..60}; do sleep 10 done +# Create CRD and namespace for host records +kubectl create ns kops-system +kubectl apply -f ${REPO_ROOT}/k8s/crds/kops.k8s.io_hosts.yaml + +# Create the host record (we can't auto create for control plane nodes) +${KOPS} toolbox enroll --cluster ${CLUSTER_NAME} --instance-group control-plane-main --host ${VM0_IP} --pod-cidr ${VM0_POD_CIDR} --v=2 --build-host | kubectl apply -f - + kubectl get nodes kubectl get pods -A @@ -247,9 +254,6 @@ sleep 10 kubectl get nodes kubectl get pods -A -# For host records -kubectl create ns kops-system -kubectl apply -f ${REPO_ROOT}/k8s/crds/kops.k8s.io_hosts.yaml # kops-controller extra permissions kubectl apply --server-side -f - <