From d60a53139f769a628fa0f73211c3890641cd8471 Mon Sep 17 00:00:00 2001 From: blublinsky Date: Thu, 14 Dec 2023 15:04:55 +0000 Subject: [PATCH 1/5] added support for secure API server build --- apiserver/Makefile | 18 +++++++++++++----- apiserver/deploy/base/secure/apiserver.yaml | 4 ++-- .../deploy/base/secure/kustomization.yaml | 3 +++ .../deploy/local/secure/kustomization.yaml | 3 +++ 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/apiserver/Makefile b/apiserver/Makefile index 53c113cb75c..b8bc19754cc 100644 --- a/apiserver/Makefile +++ b/apiserver/Makefile @@ -147,7 +147,6 @@ ifneq (${OPERATOR_IMAGE_TAG}, latest) $(ENGINE) pull kuberay/security-proxy:$(OPERATOR_IMAGE_TAG) endif $(KIND) load docker-image kuberay/security-proxy:$(OPERATOR_IMAGE_TAG) -n $(KIND_CLUSTER_NAME) - .PHONY: load-ray-test-image load-ray-test-image: ## Load the ray test images $(ENGINE) pull $(E2E_API_SERVER_RAY_IMAGE) @@ -164,13 +163,22 @@ docker-push: ## Push image for the api server. ##@ Deployment .PHONY: install -install: kustomize ## Install the kuberay api server to the K8s cluster specified in ~/.kube/config. - cd deploy/local && $(KUSTOMIZE) edit set image kuberay/apiserver=$(IMG) - $(KUSTOMIZE) build deploy/local | kubectl create -f - +install: kustomize docker-image load-image ## Install the kuberay api server to the K8s cluster specified in ~/.kube/config. + cd deploy/local/insecure && $(KUSTOMIZE) edit set image kuberay/apiserver=$(IMG) + $(KUSTOMIZE) build deploy/local/insecure | kubectl create -f - .PHONY: uninstall uninstall: ## Install the kuberay api server to the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build deploy/local | kubectl delete -f - + $(KUSTOMIZE) build deploy/local/insecure | kubectl delete -f - + +.PHONY: install-secure +install-secure: kustomize docker-image security-proxy-image load-image load-security-proxy-image ## Install the kuberay api server to the K8s cluster specified in ~/.kube/config. + cd deploy/local/secure && $(KUSTOMIZE) edit set image kuberay/apiserver=$(IMG) && $(KUSTOMIZE) edit set image kuberay/security-proxy=kuberay/security-proxy:latest + $(KUSTOMIZE) build deploy/local/secure | kubectl create -f - + +.PHONY: uninstall-secure +uninstall-secure: ## Install the kuberay api server to the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build deploy/local/secure | kubectl delete -f - .PHONY: deploy deploy: ## Deploy via helm the kuberay api server to the K8s cluster specified in ~/.kube/config. diff --git a/apiserver/deploy/base/secure/apiserver.yaml b/apiserver/deploy/base/secure/apiserver.yaml index dd66f03f32f..237c9fbf7e7 100644 --- a/apiserver/deploy/base/secure/apiserver.yaml +++ b/apiserver/deploy/base/secure/apiserver.yaml @@ -41,8 +41,8 @@ spec: httpGet: path: /healthz port: http - - name: security-proxy-container - image: quay.io/ibmdpdev/api-server-proxy:secure + - name: kuberay-security-proxy + image: kuberay/security-proxy:nightly imagePullPolicy: IfNotPresent ports: - name: http diff --git a/apiserver/deploy/base/secure/kustomization.yaml b/apiserver/deploy/base/secure/kustomization.yaml index e9dff844f71..416034988f4 100644 --- a/apiserver/deploy/base/secure/kustomization.yaml +++ b/apiserver/deploy/base/secure/kustomization.yaml @@ -13,3 +13,6 @@ images: - name: kuberay/apiserver newName: kuberay/apiserver newTag: nightly +- name: kuberay/security-proxy + newName: kuberay/security-proxy + newTag: nightly diff --git a/apiserver/deploy/local/secure/kustomization.yaml b/apiserver/deploy/local/secure/kustomization.yaml index 22f8ae8d708..6c8b85d5443 100644 --- a/apiserver/deploy/local/secure/kustomization.yaml +++ b/apiserver/deploy/local/secure/kustomization.yaml @@ -19,3 +19,6 @@ images: - name: kuberay/apiserver newName: kuberay/apiserver newTag: latest +- name: kuberay/security-proxy + newName: kuberay/security-proxy + newTag: latest From c3ebea473d6b0da49ec3ed786ea01fd44a680fcf Mon Sep 17 00:00:00 2001 From: blublinsky Date: Thu, 14 Dec 2023 15:30:48 +0000 Subject: [PATCH 2/5] added support for secure API server build --- apiserver/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apiserver/Makefile b/apiserver/Makefile index b8bc19754cc..9984102bb50 100644 --- a/apiserver/Makefile +++ b/apiserver/Makefile @@ -97,7 +97,7 @@ e2e-test: ## Run end to end tests using a pre-exiting cluster. go test ./test/e2e/... $(GO_TEST_FLAGS) -timeout 60m -race -count=1 -parallel 4 .PHONY: local-e2e-test ## Run end to end tests on newly created cluster. -local-e2e-test: docker-image operator-image cluster load-image load-operator-image deploy-operator deploy load-ray-test-image e2e-test clean-cluster ## Run end to end tests, create a fresh kind cluster will all components deployed. +local-e2e-test: operator-image cluster load-operator-image deploy-operator install load-ray-test-image e2e-test clean-cluster ## Run end to end tests, create a fresh kind cluster will all components deployed. ##@ Testing Setup KIND_CONFIG ?= hack/kind-cluster-config.yaml @@ -147,6 +147,7 @@ ifneq (${OPERATOR_IMAGE_TAG}, latest) $(ENGINE) pull kuberay/security-proxy:$(OPERATOR_IMAGE_TAG) endif $(KIND) load docker-image kuberay/security-proxy:$(OPERATOR_IMAGE_TAG) -n $(KIND_CLUSTER_NAME) + .PHONY: load-ray-test-image load-ray-test-image: ## Load the ray test images $(ENGINE) pull $(E2E_API_SERVER_RAY_IMAGE) From a12f60817cdff054e46f3c9887c37af8b57da533 Mon Sep 17 00:00:00 2001 From: blublinsky Date: Fri, 15 Dec 2023 11:59:24 +0000 Subject: [PATCH 3/5] added support for secure API server build --- apiserver/Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apiserver/Makefile b/apiserver/Makefile index 9984102bb50..2a9999a2a40 100644 --- a/apiserver/Makefile +++ b/apiserver/Makefile @@ -103,6 +103,7 @@ local-e2e-test: operator-image cluster load-operator-image deploy-operator insta KIND_CONFIG ?= hack/kind-cluster-config.yaml KIND_CLUSTER_NAME ?= ray-api-server-cluster OPERATOR_IMAGE_TAG ?= latest +SECURITY_IMAGE_TAG ?= latest .PHONY: cluster cluster: kind ## Start kind development cluster. $(KIND) create cluster -n $(KIND_CLUSTER_NAME) --config $(KIND_CONFIG) @@ -143,10 +144,10 @@ endif .PHONY: load-security-proxy-image load-security-proxy-image: ## Load the security proxy image to the kind cluster created with make cluster. -ifneq (${OPERATOR_IMAGE_TAG}, latest) - $(ENGINE) pull kuberay/security-proxy:$(OPERATOR_IMAGE_TAG) +ifneq (${SECURITY_IMAGE_TAG}, latest) + $(ENGINE) pull kuberay/security-proxy:$(SECURITY_IMAGE_TAG) endif - $(KIND) load docker-image kuberay/security-proxy:$(OPERATOR_IMAGE_TAG) -n $(KIND_CLUSTER_NAME) + $(KIND) load docker-image kuberay/security-proxy:$(SECURITY_IMAGE_TAG) -n $(KIND_CLUSTER_NAME) .PHONY: load-ray-test-image load-ray-test-image: ## Load the ray test images From 96d15de37c204511ae2c04d5e1cc66ad6000368f Mon Sep 17 00:00:00 2001 From: blublinsky Date: Fri, 15 Dec 2023 12:47:49 +0000 Subject: [PATCH 4/5] added support for secure API server build --- apiserver/DEVELOPMENT.md | 18 ++++++++++++++---- apiserver/Makefile | 10 +++++----- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/apiserver/DEVELOPMENT.md b/apiserver/DEVELOPMENT.md index 735248fe834..103bb7fac57 100644 --- a/apiserver/DEVELOPMENT.md +++ b/apiserver/DEVELOPMENT.md @@ -159,7 +159,9 @@ make docker-image #### Start Kubernetes Deployment -Note that you should make your KubeRay API server image available by either pushing it to an image registry, such as DockerHub or Quay, or by loading the image into the Kubernetes cluster. If you are using a Kind cluster for development, you can run `make load-image` to load the newly built API server image into the Kind cluster. The operator image will also be needed to be loaded on your cluster. You can use `make operator-image` to build a fresh image from sources, and, if you are using a Kind cluster for development, you can run `make load-operator-image`. +Note that you should make your KubeRay API server image available by either pushing it to an image registry, such as DockerHub or Quay, or by loading the image into the Kubernetes cluster. If you are using a Kind cluster for development, you can run `make load-image` to load the newly built API server image into the Kind cluster. The operator image will also be needed to be loaded on your cluster. If you want run secure API server, you can build security proxy using `make security-proxy-image` and load it to the cluster using `make load-security-proxy-image` + +You can use `make operator-image` to build a fresh image from sources, and, if you are using a Kind cluster for development, you can run `make load-operator-image`. ```bash #Optionally, to load the api server image into the local kind cluster created with make cluster @@ -168,8 +170,11 @@ make load-image #To use the helm charts make deploy -#To use the configuration from deploy/base +#To use the configuration from deploy/base for insecure API server make install + +#To use the configuration from deploy/base for secure API server +make install-secure ``` #### Stop Kubernetes Deployment @@ -178,8 +183,11 @@ make install #To use the helm charts make undeploy -#To use the configuration +#To use the configuration insecure make uninstall + +#To use the configuration secure +make uninstall-secure ``` #### Local Kind Cluster Deployment @@ -190,7 +198,9 @@ As a convenience for local development the following `make` targets are provided * `make clean-cluster` -- deletes the local kind cluster created with `make cluster` * `load-image` -- loads the docker image defined by the `IMG` make variable into the kind cluster. The default value for variable is: `kuberay/apiserver:latest`. The name of the image can be changed by using `make load-image -e IMG=` * `operator-image` -- Build the operator image to be loaded in your kind cluster. The operator image build is `kuberay/operator:latest`. The image tag can be overridden from the command line: ( example: `make operator-image -e OPERATOR_IMAGE_TAG=foo`) -* `load-operator-image` -- Load the operator image to the kind cluster created with `make cluster`. It should be used in conjunction with the `deploy-operator targe` +* `load-operator-image` -- Load the operator image to the kind cluster created with `make cluster`. It should be used in conjunction with the `deploy-operator target` +* `security-proxy-image` -- Build the security proxy image to be loaded in your kind cluster. The security proxy image build is `kuberay/security-proxy:latest`. The image tag can be overridden from the command line: ( example: `make security-proxy-image -e SECURITY_IMAGE_TAG=foo`) +* `load-security-proxy-image` -- Load the security proxy image to the kind cluster created with `make cluster`. It should be used in conjunction with the `install-secure` * `deploy-operator` -- Deploy operator into your cluster. The tag for the operator image is `kuberay/operator:latest`. * `undeploy-operator` -- Undeploy operator from your cluster * `load-ray-test-image` -- Load the ray test images into the cluster. diff --git a/apiserver/Makefile b/apiserver/Makefile index 2a9999a2a40..f99207368f2 100644 --- a/apiserver/Makefile +++ b/apiserver/Makefile @@ -122,7 +122,7 @@ operator-image: ## Build the operator image to be loaded in your kind cluster. .PHONY: security-proxy-image security-proxy-image: ## Build the security proxy image to be loaded in your kind cluster. - cd ../experimental && $(MAKE) docker-image -e IMG=kuberay/security-proxy:$(OPERATOR_IMAGE_TAG) + cd ../experimental && $(MAKE) docker-image -e IMG=kuberay/security-proxy:$(SECURITY_IMAGE_TAG) .PHONY: deploy-operator deploy-operator: ## Deploy operator via helm into the K8s cluster specified in ~/.kube/config. @@ -165,21 +165,21 @@ docker-push: ## Push image for the api server. ##@ Deployment .PHONY: install -install: kustomize docker-image load-image ## Install the kuberay api server to the K8s cluster specified in ~/.kube/config. +install: kustomize docker-image load-image ## Install the kuberay api server without security to the K8s cluster specified in ~/.kube/config. cd deploy/local/insecure && $(KUSTOMIZE) edit set image kuberay/apiserver=$(IMG) $(KUSTOMIZE) build deploy/local/insecure | kubectl create -f - .PHONY: uninstall -uninstall: ## Install the kuberay api server to the K8s cluster specified in ~/.kube/config. +uninstall: ## Remove the kuberay api without security server from the K8s cluster specified in ~/.kube/config. $(KUSTOMIZE) build deploy/local/insecure | kubectl delete -f - .PHONY: install-secure -install-secure: kustomize docker-image security-proxy-image load-image load-security-proxy-image ## Install the kuberay api server to the K8s cluster specified in ~/.kube/config. +install-secure: kustomize docker-image security-proxy-image load-image load-security-proxy-image ## Install the kuberay api server with security to the K8s cluster specified in ~/.kube/config. cd deploy/local/secure && $(KUSTOMIZE) edit set image kuberay/apiserver=$(IMG) && $(KUSTOMIZE) edit set image kuberay/security-proxy=kuberay/security-proxy:latest $(KUSTOMIZE) build deploy/local/secure | kubectl create -f - .PHONY: uninstall-secure -uninstall-secure: ## Install the kuberay api server to the K8s cluster specified in ~/.kube/config. +uninstall-secure: ## Remove the kuberay api with security server from the K8s cluster specified in ~/.kube/config. $(KUSTOMIZE) build deploy/local/secure | kubectl delete -f - .PHONY: deploy From 1cc12333e6d0032bf1ba00d1cd003fbee13c9513 Mon Sep 17 00:00:00 2001 From: blublinsky Date: Fri, 15 Dec 2023 12:49:24 +0000 Subject: [PATCH 5/5] added support for secure API server build --- apiserver/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apiserver/Makefile b/apiserver/Makefile index f99207368f2..a2bcbf28c42 100644 --- a/apiserver/Makefile +++ b/apiserver/Makefile @@ -170,7 +170,7 @@ install: kustomize docker-image load-image ## Install the kuberay api server wi $(KUSTOMIZE) build deploy/local/insecure | kubectl create -f - .PHONY: uninstall -uninstall: ## Remove the kuberay api without security server from the K8s cluster specified in ~/.kube/config. +uninstall: ## Remove the kuberay api server without security server from the K8s cluster specified in ~/.kube/config. $(KUSTOMIZE) build deploy/local/insecure | kubectl delete -f - .PHONY: install-secure @@ -179,7 +179,7 @@ install-secure: kustomize docker-image security-proxy-image load-image load-secu $(KUSTOMIZE) build deploy/local/secure | kubectl create -f - .PHONY: uninstall-secure -uninstall-secure: ## Remove the kuberay api with security server from the K8s cluster specified in ~/.kube/config. +uninstall-secure: ## Remove the kuberay api server with security server from the K8s cluster specified in ~/.kube/config. $(KUSTOMIZE) build deploy/local/secure | kubectl delete -f - .PHONY: deploy