diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..9f401f87e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,48 @@ +--- +name: CI/CD Workflow + +on: + push: + branches: + - main + paths: + - '**.yaml' + - '**/Dockerfile' + - '**/charts/**' + tags: + - 'v*' + +env: + IMAGE_NGINX_CACHE: nginx-cache + REGISTRY: ghcr.io/${{ github.repository_owner }} + PUSH: 1 + LOAD: 1 + NGINX_CACHE_TAG: v0.1.0 + TAG: v0.3.1 + PLATFORM_ARCH: linux/amd64 + +jobs: + build-and-push: + name: Build Cozystack + runs-on: ubuntu-latest + services: + registry: + image: registry:2 + ports: + - 5000:5000 + steps: + - name: Set up Docker Registry + run: | + if [ "$GITHUB_ACTIONS" = "true" ]; then + echo "REGISTRY=ghcr.io/${{ github.repository_owner }}" >> $GITHUB_ENV + else + echo "REGISTRY=localhost:5000/cozystack_local" >> $GITHUB_ENV + fi + + - uses: actions/checkout@v3 + - name: Build usig make + run: | + make + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..a79650b46 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,48 @@ +name: Lint + +on: + push: + branches: [ main ] # Lint only on pushes to the main branch + pull_request: + branches: [ main ] # Lint on PRs targeting the main branch + +permissions: + contents: read + +jobs: + lint: + name: Super-Linter + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Run Super-Linter + uses: github/super-linter@v4 + env: + # To report GitHub Actions status checks + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VALIDATE_ALL_CODEBASE: false # Lint only changed files + VALIDATE_TERRAFORM: false # Disable Terraform linting (remove if you need it) + DEFAULT_BRANCH: main # Set your default branch + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Enable only the linters you need for your project + VALIDATE_JAVASCRIPT_ES: true + VALIDATE_PYTHON_BLACK: true + VALIDATE_HTML: false + VALIDATE_GO: false + VALIDATE_XML: false + VALIDATE_JAVA: false + VALIDATE_DOCKERFILE: false + # turn off JSCPD copy/paste detection, which results in lots of results for examples and devops repos + VALIDATE_JSCPD: false + # turn off shfmt shell formatter as we already have shellcheck + VALIDATE_SHELL_SHFMT: false + VALIDATE_EDITORCONFIG: false + # prevent Kubernetes CRD API's from causing kubeval to fail + # also change schema location to an up-to-date list + # https://github.com/yannh/kubernetes-json-schema/#kubeval + KUBERNETES_KUBEVAL_OPTIONS: --ignore-missing-schemas --schema-location https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/ diff --git a/.github/workflows/linters/.markdown-lint.yml b/.github/workflows/linters/.markdown-lint.yml new file mode 100644 index 000000000..335724b49 --- /dev/null +++ b/.github/workflows/linters/.markdown-lint.yml @@ -0,0 +1,7 @@ +--- + # MD013/line-length - Line length + MD013: + # Number of characters, default is 80 + line_length: 9999 + # check code blocks? + code_blocks: false diff --git a/.github/workflows/linters/.yaml-lint.yml b/.github/workflows/linters/.yaml-lint.yml new file mode 100644 index 000000000..95afab763 --- /dev/null +++ b/.github/workflows/linters/.yaml-lint.yml @@ -0,0 +1,55 @@ + +yaml-files: +- '*.yaml' +- '*.yml' +- '.yamllint' + +rules: + braces: + level: warning + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 1 + max-spaces-inside-empty: 5 + brackets: + level: warning + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 1 + max-spaces-inside-empty: 5 + colons: + level: warning + max-spaces-before: 0 + max-spaces-after: 1 + commas: + level: warning + max-spaces-before: 0 + min-spaces-after: 1 + max-spaces-after: 1 + comments: disable + comments-indentation: disable + document-end: disable + document-start: disable + empty-lines: + level: warning + max: 2 + max-start: 0 + max-end: 0 + hyphens: + level: warning + max-spaces-after: 1 + indentation: + level: warning + spaces: consistent + indent-sequences: true + check-multi-line-strings: false + key-duplicates: enable + line-length: disable + new-line-at-end-of-file: disable + new-lines: + type: unix + trailing-spaces: disable + line-length: + max: 130 + allow-non-breakable-words: true + allow-non-breakable-inline-mappings: false diff --git a/packages/apps/Makefile b/packages/apps/Makefile index 506c87505..de61e3fa4 100644 --- a/packages/apps/Makefile +++ b/packages/apps/Makefile @@ -11,7 +11,7 @@ repo: rm -rf "$(TMP)" fix-chartnames: - find . -maxdepth 2 -name Chart.yaml | awk -F/ '{print $$2}' | while read i; do sed -i "s/^name: .*/name: $$i/" "$$i/Chart.yaml"; done + find . -maxdepth 2 -name Chart.yaml | awk -F/ '{print $$2}' | while read i; do sed "s/^name: .*/name: $$i/" "$$i/Chart.yaml"; done gen-versions-map: fix-chartnames ../../hack/gen_versions_map.sh diff --git a/packages/apps/ferretdb/ferretdb.svg b/packages/apps/ferretdb/ferretdb.svg new file mode 100644 index 000000000..196871e1b --- /dev/null +++ b/packages/apps/ferretdb/ferretdb.svg @@ -0,0 +1,54 @@ + + + + + + + + + + + + diff --git a/packages/core/platform/bundles/paas-proxmox.yaml b/packages/core/platform/bundles/paas-proxmox.yaml new file mode 100644 index 000000000..dd82de126 --- /dev/null +++ b/packages/core/platform/bundles/paas-proxmox.yaml @@ -0,0 +1,138 @@ +{{- $cozyConfig := lookup "v1" "ConfigMap" "cozy-system" "cozystack" }} + +releases: +- name: cilium + releaseName: cilium + chart: cozy-cilium + namespace: cozy-cilium + privileged: true + dependsOn: [] + +- name: kubeovn + releaseName: kubeovn + chart: cozy-kubeovn + namespace: cozy-kubeovn + privileged: true + dependsOn: [cilium] + values: + cozystack: + nodesHash: {{ include "cozystack.master-node-ips" . | sha256sum }} + kube-ovn: + ipv4: + POD_CIDR: "{{ index $cozyConfig.data "ipv4-pod-cidr" }}" + POD_GATEWAY: "{{ index $cozyConfig.data "ipv4-pod-gateway" }}" + SVC_CIDR: "{{ index $cozyConfig.data "ipv4-svc-cidr" }}" + JOIN_CIDR: "{{ index $cozyConfig.data "ipv4-join-cidr" }}" + +- name: cert-manager + releaseName: cert-manager + chart: cozy-cert-manager + namespace: cozy-cert-manager + dependsOn: [cilium,kubeovn] + +- name: cert-manager-issuers + releaseName: cert-manager-issuers + chart: cozy-cert-manager-issuers + namespace: cozy-cert-manager + dependsOn: [cilium,kubeovn,cert-manager] + +- name: victoria-metrics-operator + releaseName: victoria-metrics-operator + chart: cozy-victoria-metrics-operator + namespace: cozy-victoria-metrics-operator + dependsOn: [cilium,kubeovn,cert-manager] + +- name: monitoring + releaseName: monitoring + chart: cozy-monitoring + namespace: cozy-monitoring + privileged: true + dependsOn: [cilium,kubeovn,victoria-metrics-operator] + +- name: grafana-operator + releaseName: grafana-operator + chart: cozy-grafana-operator + namespace: cozy-grafana-operator + dependsOn: [cilium,kubeovn] + +- name: mariadb-operator + releaseName: mariadb-operator + chart: cozy-mariadb-operator + namespace: cozy-mariadb-operator + dependsOn: [cilium,kubeovn,cert-manager,victoria-metrics-operator] + +- name: postgres-operator + releaseName: postgres-operator + chart: cozy-postgres-operator + namespace: cozy-postgres-operator + dependsOn: [cilium,kubeovn,cert-manager] + +- name: rabbitmq-operator + releaseName: rabbitmq-operator + chart: cozy-rabbitmq-operator + namespace: cozy-rabbitmq-operator + dependsOn: [cilium,kubeovn] + +- name: redis-operator + releaseName: redis-operator + chart: cozy-redis-operator + namespace: cozy-redis-operator + dependsOn: [cilium,kubeovn] + +- name: proxmox-csi-operator + releaseName: proxmox-csi-operator + chart: cozy-proxmox-csi-operator + namespace: cozy-proxmox + dependsOn: [cilium,kubeovn,cert-manager] + +- name: proxmox-csi-operator + releaseName: proxmox-csi-operator + chart: cozy-proxmox-csi-operator + namespace: cozy-proxmox + dependsOn: [cilium,kubeovn,cert-manager] + +- name: telepresence + releaseName: traffic-manager + chart: cozy-telepresence + namespace: cozy-telepresence + dependsOn: [cilium,kubeovn] + +- name: dashboard + releaseName: dashboard + chart: cozy-dashboard + namespace: cozy-dashboard + dependsOn: [cilium,kubeovn] + {{- if .Capabilities.APIVersions.Has "source.toolkit.fluxcd.io/v1beta2" }} + {{- with (lookup "source.toolkit.fluxcd.io/v1beta2" "HelmRepository" "cozy-public" "").items }} + values: + kubeapps: + redis: + master: + podAnnotations: + {{- range $index, $repo := . }} + {{- with (($repo.status).artifact).revision }} + repository.cozystack.io/{{ $repo.metadata.name }}: {{ quote . }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + +- name: kamaji + releaseName: kamaji + chart: cozy-kamaji + namespace: cozy-kamaji + dependsOn: [cilium,kubeovn,cert-manager] + +- name: capi-operator + releaseName: capi-operator + chart: cozy-capi-operator + namespace: cozy-cluster-api + privileged: true + dependsOn: [cilium,kubeovn,cert-manager] + +- name: capi-providers + releaseName: capi-providers + chart: cozy-capi-providers + namespace: cozy-cluster-api + privileged: true + dependsOn: [cilium,kubeovn,capi-operator] \ No newline at end of file diff --git a/packages/extra/Makefile b/packages/extra/Makefile index 5586c59e1..7212987d3 100644 --- a/packages/extra/Makefile +++ b/packages/extra/Makefile @@ -11,7 +11,7 @@ repo: rm -rf "$(TMP)" fix-chartnames: - find . -maxdepth 2 -name Chart.yaml | awk -F/ '{print $$2}' | while read i; do sed -i "s/^name: .*/name: $$i/" "$$i/Chart.yaml"; done + find . -maxdepth 2 -name Chart.yaml | awk -F/ '{print $$2}' | while read i; do sed "s/^name: .*/name: $$i/" "$$i/Chart.yaml"; done gen-versions-map: fix-chartnames ../../hack/gen_versions_map.sh diff --git a/packages/extra/etcd/templates/datastore.yaml b/packages/extra/etcd/templates/datastore.yaml index 591f65192..40c7bf6df 100644 --- a/packages/extra/etcd/templates/datastore.yaml +++ b/packages/extra/etcd/templates/datastore.yaml @@ -47,4 +47,4 @@ metadata: name: etcd-client-tls annotations: helm.sh/hook: pre-install - helm.sh/resource-policy: keep \ No newline at end of file + helm.sh/resource-policy: keep diff --git a/packages/system/capi-providers/templates/providers.yaml b/packages/system/capi-providers/templates/providers.yaml index 0c2f1b2a1..514029828 100644 --- a/packages/system/capi-providers/templates/providers.yaml +++ b/packages/system/capi-providers/templates/providers.yaml @@ -37,3 +37,25 @@ metadata: spec: # https://github.com/kubernetes-sigs/cluster-api-provider-kubevirt version: v0.1.9 + +{{- if .Values.providers.proxmox }} +--- +apiVersion: operator.cluster.x-k8s.io/v1alpha2 +kind: InfrastructureProvider +metadata: + name: proxmox +spec: + # https://github.com/ionos-cloud/cluster-api-provider-proxmox + version: v0.6.2 +{{- end }} + +{{- if .Values.providers.hetzner }} +--- +apiVersion: operator.cluster.x-k8s.io/v1alpha2 +kind: InfrastructureProvider +metadata: + name: hetzner +spec: + # https://github.com/syself/cluster-api-provider-hetzner + version: v1.0.1 +{{- end }} diff --git a/packages/system/capi-providers/values.yaml b/packages/system/capi-providers/values.yaml new file mode 100644 index 000000000..63c6a7b2c --- /dev/null +++ b/packages/system/capi-providers/values.yaml @@ -0,0 +1,4 @@ +providers: + kubevirt: true + proxmox: true + hetzner: false \ No newline at end of file diff --git a/packages/system/proxmox-csi-node/Chart.yaml b/packages/system/proxmox-csi-node/Chart.yaml new file mode 100644 index 000000000..2a63eb08f --- /dev/null +++ b/packages/system/proxmox-csi-node/Chart.yaml @@ -0,0 +1,2 @@ +name: cozy-proxmox-csi-node +version: 0.1.0 diff --git a/packages/system/proxmox-csi-node/templates/deploy.yaml b/packages/system/proxmox-csi-node/templates/deploy.yaml new file mode 100644 index 000000000..52b8e1300 --- /dev/null +++ b/packages/system/proxmox-csi-node/templates/deploy.yaml @@ -0,0 +1,278 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: csi.kubevirt.io +spec: + attachRequired: true + podInfoOnMount: true + fsGroupPolicy: ReadWriteOnceWithFSType +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: proxmox-csi-controller-sa + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: proxmox-csi-controller-cr +rules: + - apiGroups: [''] + resources: ['persistentvolumes'] + verbs: ['create', 'delete', 'get', 'list', 'watch', 'update', 'patch'] + - apiGroups: [''] + resources: ['secrets'] + verbs: ['get', 'list'] + - apiGroups: [''] + resources: ['persistentvolumeclaims'] + verbs: ['get', 'list', 'watch', 'update'] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: [''] + resources: ['nodes'] + verbs: ['get', 'list', 'watch'] + - apiGroups: ['storage.k8s.io'] + resources: ['volumeattachments'] + verbs: ['get', 'list', 'watch', 'update', 'patch'] + - apiGroups: ['storage.k8s.io'] + resources: ['storageclasses'] + verbs: ['get', 'list', 'watch'] + - apiGroups: ['csi.storage.k8s.io'] + resources: ['csidrivers'] + verbs: ['get', 'list', 'watch', 'update', 'create'] + - apiGroups: [''] + resources: ['events'] + verbs: ['list', 'watch', 'create', 'update', 'patch'] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "volumeattachments/status" ] + verbs: [ "get", "list", "watch", "update", "patch" ] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + verbs: ["use"] + resourceNames: ["privileged"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: proxmox-csi-controller-binding +subjects: + - kind: ServiceAccount + name: proxmox-csi-controller-sa + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: proxmox-csi-controller-cr + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: proxmox-csi-node-sa + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: proxmox-csi-node-cr +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + verbs: ["use"] + resourceNames: ["privileged"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: proxmox-csi-node-binding +subjects: + - kind: ServiceAccount + name: proxmox-csi-node-sa + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: proxmox-csi-node-cr + apiGroup: rbac.authorization.k8s.io +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: proxmox-csi-node + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: + app: {{ .Release.Namespace }} + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: {{ .Release.Namespace }} + spec: + serviceAccount: proxmox-csi-node-sa + priorityClassName: system-node-critical + tolerations: + - operator: Exists + containers: + - name: csi-driver + securityContext: + privileged: true + allowPrivilegeEscalation: true + imagePullPolicy: Always + image: ghcr.io/kvaps/test:proxmox-csi-driver + args: + - "--endpoint=unix:/csi/csi.sock" + - "--node-name=$(KUBE_NODE_NAME)" + - "--v=5" + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + - name: device-dir + mountPath: /dev + - name: udev + mountPath: /run/udev + ports: + - name: healthz + containerPort: 10300 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + resources: + requests: + memory: 50Mi + cpu: 10m + - name: csi-node-driver-registrar + image: quay.io/openshift/origin-csi-node-driver-registrar:latest + args: + - "--csi-address=$(ADDRESS)" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + - "--v=5" + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/csi.kubevirt.io-reg.sock /csi/csi.sock"] + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/csi.proxmox.io/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: + requests: + memory: 20Mi + cpu: 5m + - name: csi-liveness-probe + image: quay.io/openshift/origin-csi-livenessprobe:latest + args: + - "--csi-address=/csi/csi.sock" + - "--probe-timeout=3s" + - "--health-port=10300" + volumeMounts: + - name: plugin-dir + mountPath: /csi + resources: + requests: + memory: 20Mi + cpu: 5m + volumes: + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi.proxmox.io/ + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: device-dir + hostPath: + path: /dev + type: Directory + - name: udev + hostPath: + path: /run/udev +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: driver-config + namespace: {{ .Release.Namespace }} +data: + infraClusterNamespace: + infraClusterLabels: csi-driver/cluster=tenant +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: proxmox + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: csi.proxmox.io +parameters: + infraStorageClassName: replicated + bus: scsi diff --git a/packages/system/proxmox-csi/Chart.yaml b/packages/system/proxmox-csi/Chart.yaml new file mode 100644 index 000000000..0aa338005 --- /dev/null +++ b/packages/system/proxmox-csi/Chart.yaml @@ -0,0 +1,2 @@ +name: app +version: 0.0.0 diff --git a/packages/system/proxmox-csi/Makefile b/packages/system/proxmox-csi/Makefile new file mode 100644 index 000000000..357a739e1 --- /dev/null +++ b/packages/system/proxmox-csi/Makefile @@ -0,0 +1,13 @@ +include ../../hack/app-helm.mk + +update: + rm -rf charts + tag=$$(git ls-remote --tags --sort="v:refname" https://github.com/sergelogvinov/proxmox-cloud-controller-manager | awk -F'[/^]' 'END{print $$3}') && \ + curl -sSL https://github.com/sergelogvinov/proxmox-cloud-controller-manager/archive/refs/tags/$${tag}.tar.gz | \ + tar xzvf - --strip 1 proxmox-cloud-controller-manager-$${tag#*v}/charts + sed -i 's/^ namespace: .*/ namespace: kube-system/' charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml + tag=$$(git ls-remote --tags --sort="v:refname" https://github.com/sergelogvinov/proxmox-csi-plugin | awk -F'[/^]' 'END{print $$3}') && \ + curl -sSL https://github.com/sergelogvinov/proxmox-csi-plugin/archive/refs/tags/$${tag}.tar.gz | \ + tar xzvf - --strip 1 proxmox-csi-plugin-$${tag#*v}/charts + rm -f charts/proxmox-csi-plugin/templates/namespace.yaml + patch -p 3 < patches/namespace.patch diff --git a/packages/system/proxmox-csi/README.md b/packages/system/proxmox-csi/README.md new file mode 100644 index 000000000..d95c5324f --- /dev/null +++ b/packages/system/proxmox-csi/README.md @@ -0,0 +1,6 @@ +# Proxmox CSI Plugin + +Plugin that provides CSI interface for Proxmox + +- GitHub: https://github.com/sergelogvinov/proxmox-csi-plugin +- Telegram: https://t.me/ru_talos diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/.helmignore b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/Chart.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/Chart.yaml new file mode 100644 index 000000000..4213926b8 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: proxmox-cloud-controller-manager +description: Cloud Controller Manager plugin for Proxmox +type: application +home: https://github.com/sergelogvinov/proxmox-cloud-controller-manager +icon: https://raw.githubusercontent.com/sergelogvinov/proxmox-cloud-controller-manager/main/charts/proxmox-cloud-controller-manager/icon.png +sources: + - https://github.com/sergelogvinov/proxmox-cloud-controller-manager +keywords: + - ccm + - proxmox + - kubernetes +maintainers: + - name: sergelogvinov + url: https://github.com/sergelogvinov +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.2.8 +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: v0.5.1 diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/README.md b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/README.md new file mode 100644 index 000000000..64e1ef6e5 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/README.md @@ -0,0 +1,109 @@ +# proxmox-cloud-controller-manager + +![Version: 0.2.8](https://img.shields.io/badge/Version-0.2.8-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.5.1](https://img.shields.io/badge/AppVersion-v0.5.1-informational?style=flat-square) + +Cloud Controller Manager plugin for Proxmox + +The Cloud Controller Manager (CCM) is responsible for managing node resources in cloud-based Kubernetes environments. + +Key functions of the Cloud Controller Manager: +- `Node Management`: It manages nodes by initializing new nodes when they join the cluster (e.g., during scaling up) and removing nodes when they are no longer needed (e.g., during scaling down). +- `Cloud-Specific Operations`: The CCM ensures that the cloud provider's API is integrated into the Kubernetes cluster to control and automate tasks like load balancing, storage provisioning, and node lifecycle management. + +**Homepage:** + +## Maintainers + +| Name | Email | Url | +| ---- | ------ | --- | +| sergelogvinov | | | + +## Source Code + +* + +## Requirements + +You need to set `--cloud-provider=external` in the kubelet argument for all nodes in the cluster. + +## Proxmox permissions + +```shell +# Create role CCM +pveum role add CCM -privs "VM.Audit" +# Create user and grant permissions +pveum user add kubernetes@pve +pveum aclmod / -user kubernetes@pve -role CCM +pveum user token add kubernetes@pve ccm -privsep 0 +``` + +## Helm values example + +```yaml +# proxmox-ccm.yaml + +config: + clusters: + - url: https://cluster-api-1.exmple.com:8006/api2/json + insecure: false + token_id: "kubernetes@pve!csi" + token_secret: "key" + region: cluster-1 + +enabledControllers: + # Remove `cloud-node` if you use it with Talos CCM + - cloud-node + - cloud-node-lifecycle + +# Deploy CCM only on control-plane nodes +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists +tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule +``` + +Deploy chart: + +```shell +helm upgrade -i --namespace=kube-system -f proxmox-ccm.yaml \ + proxmox-cloud-controller-manager oci://ghcr.io/sergelogvinov/charts/proxmox-cloud-controller-manager +``` + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| replicaCount | int | `1` | | +| image.repository | string | `"ghcr.io/sergelogvinov/proxmox-cloud-controller-manager"` | Proxmox CCM image. | +| image.pullPolicy | string | `"IfNotPresent"` | Always or IfNotPresent | +| image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion. | +| imagePullSecrets | list | `[]` | | +| nameOverride | string | `""` | | +| fullnameOverride | string | `""` | | +| extraArgs | list | `[]` | Any extra arguments for talos-cloud-controller-manager | +| enabledControllers | list | `["cloud-node","cloud-node-lifecycle"]` | List of controllers should be enabled. Use '*' to enable all controllers. Support only `cloud-node,cloud-node-lifecycle` controllers. | +| logVerbosityLevel | int | `2` | Log verbosity level. See https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md for description of individual verbosity levels. | +| existingConfigSecret | string | `nil` | Proxmox cluster config stored in secrets. | +| existingConfigSecretKey | string | `"config.yaml"` | Proxmox cluster config stored in secrets key. | +| config | object | `{"clusters":[]}` | Proxmox cluster config. | +| serviceAccount | object | `{"annotations":{},"create":true,"name":""}` | Pods Service Account. ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ | +| priorityClassName | string | `"system-cluster-critical"` | CCM pods' priorityClassName. | +| initContainers | list | `[]` | Add additional init containers to the CCM pods. ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ | +| hostAliases | list | `[]` | hostAliases Deployment pod host aliases ref: https://kubernetes.io/docs/tasks/network/customize-hosts-file-for-pods/ | +| podAnnotations | object | `{}` | Annotations for data pods. ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ | +| podSecurityContext | object | `{"fsGroup":10258,"fsGroupChangePolicy":"OnRootMismatch","runAsGroup":10258,"runAsNonRoot":true,"runAsUser":10258}` | Pods Security Context. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod | +| securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"seccompProfile":{"type":"RuntimeDefault"}}` | Container Security Context. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod | +| resources | object | `{"requests":{"cpu":"10m","memory":"32Mi"}}` | Resource requests and limits. ref: https://kubernetes.io/docs/user-guide/compute-resources/ | +| useDaemonSet | bool | `false` | Deploy CCM in Daemonset mode. CCM will use hostNetwork. It allows to use CCM without CNI plugins. | +| updateStrategy | object | `{"rollingUpdate":{"maxUnavailable":1},"type":"RollingUpdate"}` | Deployment update strategy type. ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment | +| nodeSelector | object | `{}` | Node labels for data pods assignment. ref: https://kubernetes.io/docs/user-guide/node-selection/ | +| tolerations | list | `[{"effect":"NoSchedule","key":"node-role.kubernetes.io/control-plane","operator":"Exists"},{"effect":"NoSchedule","key":"node.cloudprovider.kubernetes.io/uninitialized","operator":"Exists"}]` | Tolerations for data pods assignment. ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ | +| affinity | object | `{}` | Affinity for data pods assignment. ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity | +| extraVolumes | list | `[]` | Additional volumes for Pods | +| extraVolumeMounts | list | `[]` | Additional volume mounts for Pods | diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/README.md.gotmpl b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/README.md.gotmpl new file mode 100644 index 000000000..0971b9b6b --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/README.md.gotmpl @@ -0,0 +1,76 @@ +{{ template "chart.header" . }} + +{{ template "chart.deprecationWarning" . }} + +{{ template "chart.badgesSection" . }} + +{{ template "chart.description" . }} + +The Cloud Controller Manager (CCM) is responsible for managing node resources in cloud-based Kubernetes environments. + +Key functions of the Cloud Controller Manager: +- `Node Management`: It manages nodes by initializing new nodes when they join the cluster (e.g., during scaling up) and removing nodes when they are no longer needed (e.g., during scaling down). +- `Cloud-Specific Operations`: The CCM ensures that the cloud provider's API is integrated into the Kubernetes cluster to control and automate tasks like load balancing, storage provisioning, and node lifecycle management. + +{{ template "chart.homepageLine" . }} + +{{ template "chart.maintainersSection" . }} + +{{ template "chart.sourcesSection" . }} + +{{ template "chart.requirementsSection" . }} + +## Requirements + +You need to set `--cloud-provider=external` in the kubelet argument for all nodes in the cluster. + +## Proxmox permissions + +```shell +# Create role CCM +pveum role add CCM -privs "VM.Audit" +# Create user and grant permissions +pveum user add kubernetes@pve +pveum aclmod / -user kubernetes@pve -role CCM +pveum user token add kubernetes@pve ccm -privsep 0 +``` + +## Helm values example + +```yaml +# proxmox-ccm.yaml + +config: + clusters: + - url: https://cluster-api-1.exmple.com:8006/api2/json + insecure: false + token_id: "kubernetes@pve!csi" + token_secret: "key" + region: cluster-1 + +enabledControllers: + # Remove `cloud-node` if you use it with Talos CCM + - cloud-node + - cloud-node-lifecycle + +# Deploy CCM only on control-plane nodes +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists +tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule +``` + +Deploy chart: + +```shell +helm upgrade -i --namespace=kube-system -f proxmox-ccm.yaml \ + proxmox-cloud-controller-manager oci://ghcr.io/sergelogvinov/charts/proxmox-cloud-controller-manager +``` + +{{ template "chart.valuesSection" . }} diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/ci/values.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/ci/values.yaml new file mode 100644 index 000000000..2d59a0343 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/ci/values.yaml @@ -0,0 +1,31 @@ +image: + repository: ghcr.io/sergelogvinov/proxmox-cloud-controller-manager + pullPolicy: Always + tag: edge + +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + +logVerbosityLevel: 4 + +enabledControllers: + - cloud-node + - cloud-node-lifecycle + +config: + clusters: + - url: https://cluster-api-1.exmple.com:8006/api2/json + insecure: false + token_id: "user!token-id" + token_secret: "secret" + region: cluster-1 + - url: https://cluster-api-2.exmple.com:8006/api2/json + insecure: false + token_id: "user!token-id" + token_secret: "secret" + region: cluster-2 diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/icon.png b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/icon.png new file mode 100644 index 000000000..36a30d2c7 Binary files /dev/null and b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/icon.png differ diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/NOTES.txt b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/NOTES.txt new file mode 100644 index 000000000..e69de29bb diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/_helpers.tpl b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/_helpers.tpl new file mode 100644 index 000000000..ee962923e --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/_helpers.tpl @@ -0,0 +1,69 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "proxmox-cloud-controller-manager.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "proxmox-cloud-controller-manager.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "proxmox-cloud-controller-manager.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "proxmox-cloud-controller-manager.labels" -}} +helm.sh/chart: {{ include "proxmox-cloud-controller-manager.chart" . }} +{{ include "proxmox-cloud-controller-manager.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "proxmox-cloud-controller-manager.selectorLabels" -}} +app.kubernetes.io/name: {{ include "proxmox-cloud-controller-manager.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "proxmox-cloud-controller-manager.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "proxmox-cloud-controller-manager.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Generate string of enabled controllers. Might have a trailing comma (,) which needs to be trimmed. +*/}} +{{- define "proxmox-cloud-controller-manager.enabledControllers" }} +{{- range .Values.enabledControllers -}}{{ . }},{{- end -}} +{{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/deployment.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/deployment.yaml new file mode 100644 index 000000000..d7defe7b7 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/deployment.yaml @@ -0,0 +1,147 @@ +apiVersion: apps/v1 +{{- if .Values.useDaemonSet }} +kind: DaemonSet +{{- else }} +kind: Deployment +{{- end }} +metadata: + name: {{ include "proxmox-cloud-controller-manager.fullname" . }} + labels: + {{- include "proxmox-cloud-controller-manager.labels" . | nindent 4 }} + namespace: {{ .Release.Namespace }} +spec: + {{- if not .Values.useDaemonSet }} + replicas: {{ .Values.replicaCount }} + strategy: + type: {{ .Values.updateStrategy.type }} + {{- else }} + updateStrategy: + type: {{ .Values.updateStrategy.type }} + {{- end }} + selector: + matchLabels: + {{- include "proxmox-cloud-controller-manager.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- if .Values.config }} + checksum/config: {{ toJson .Values.config | sha256sum }} + {{- end }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "proxmox-cloud-controller-manager.selectorLabels" . | nindent 8 }} + spec: + enableServiceLinks: false + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "proxmox-cloud-controller-manager.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- if .Values.useDaemonSet }} + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + {{- end }} + {{- with .Values.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + initContainers: {{- toYaml .Values.initContainers | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --v={{ .Values.logVerbosityLevel }} + - --cloud-provider=proxmox + - --cloud-config=/etc/proxmox/config.yaml + - --controllers={{- trimAll "," (include "proxmox-cloud-controller-manager.enabledControllers" . ) }} + - --leader-elect-resource-name=cloud-controller-manager-proxmox + - --use-service-account-credentials + - --secure-port=10258 + - --authorization-always-allow-paths=/healthz,/livez,/readyz,/metrics + {{- with .Values.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: metrics + containerPort: 10258 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: metrics + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 30 + timeoutSeconds: 5 + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: cloud-config + mountPath: /etc/proxmox + readOnly: true + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + affinity: + {{- with .Values.affinity }} + {{- toYaml . | nindent 8 }} + {{- else }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + {{- include "proxmox-cloud-controller-manager.selectorLabels" . | nindent 20 }} + topologyKey: topology.kubernetes.io/zone + weight: 1 + {{- end }} + tolerations: + {{- with .Values.tolerations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.useDaemonSet }} + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + {{- end }} + {{- if not .Values.useDaemonSet }} + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + {{- include "proxmox-cloud-controller-manager.selectorLabels" . | nindent 14 }} + {{- end }} + volumes: + {{- if .Values.existingConfigSecret }} + - name: cloud-config + secret: + secretName: {{ .Values.existingConfigSecret }} + items: + - key: {{ .Values.existingConfigSecretKey }} + path: config.yaml + defaultMode: 416 + {{- else }} + - name: cloud-config + secret: + secretName: {{ include "proxmox-cloud-controller-manager.fullname" . }} + defaultMode: 416 + {{- end }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/role.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/role.yaml new file mode 100644 index 000000000..b35bf2b7c --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/role.yaml @@ -0,0 +1,53 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:{{ include "proxmox-cloud-controller-manager.fullname" . }} + labels: + {{- include "proxmox-cloud-controller-manager.labels" . | nindent 4 }} +rules: +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get +- apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml new file mode 100644 index 000000000..32b065ec5 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml @@ -0,0 +1,26 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:{{ include "proxmox-cloud-controller-manager.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:{{ include "proxmox-cloud-controller-manager.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ include "proxmox-cloud-controller-manager.fullname" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: system:{{ include "proxmox-cloud-controller-manager.fullname" . }}:extension-apiserver-authentication-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: {{ include "proxmox-cloud-controller-manager.fullname" . }} + namespace: {{ .Release.Namespace }} diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/secrets.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/secrets.yaml new file mode 100644 index 000000000..ce60f6220 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/secrets.yaml @@ -0,0 +1,11 @@ +{{- if ne (len .Values.config.clusters) 0 }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "proxmox-cloud-controller-manager.fullname" . }} + labels: + {{- include "proxmox-cloud-controller-manager.labels" . | nindent 4 }} + namespace: {{ .Release.Namespace }} +data: + config.yaml: {{ toYaml .Values.config | b64enc | quote }} +{{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/serviceaccount.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/serviceaccount.yaml new file mode 100644 index 000000000..b9ecfcc25 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "proxmox-cloud-controller-manager.serviceAccountName" . }} + labels: + {{- include "proxmox-cloud-controller-manager.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.edge.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.edge.yaml new file mode 100644 index 000000000..36d35f709 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.edge.yaml @@ -0,0 +1,17 @@ +image: + pullPolicy: Always + tag: edge + +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + +logVerbosityLevel: 4 + +enabledControllers: + - cloud-node + - cloud-node-lifecycle diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.talos.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.talos.yaml new file mode 100644 index 000000000..38b2e84de --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.talos.yaml @@ -0,0 +1,12 @@ +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + +logVerbosityLevel: 4 + +enabledControllers: + - cloud-node-lifecycle diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.yaml new file mode 100644 index 000000000..37d38de97 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.yaml @@ -0,0 +1,177 @@ +# Default values for proxmox-cloud-controller-manager. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + # -- Proxmox CCM image. + repository: ghcr.io/sergelogvinov/proxmox-cloud-controller-manager + # -- Always or IfNotPresent + pullPolicy: IfNotPresent + # -- Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +# -- Any extra arguments for talos-cloud-controller-manager +extraArgs: [] + # - --cluster-name=kubernetes + +# -- List of controllers should be enabled. +# Use '*' to enable all controllers. +# Support only `cloud-node,cloud-node-lifecycle` controllers. +enabledControllers: + - cloud-node + - cloud-node-lifecycle + # - route + # - service + +# -- Log verbosity level. See https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md +# for description of individual verbosity levels. +logVerbosityLevel: 2 + +# -- Proxmox cluster config stored in secrets. +existingConfigSecret: ~ +# -- Proxmox cluster config stored in secrets key. +existingConfigSecretKey: config.yaml + +# -- Proxmox cluster config. +config: + clusters: [] + # - url: https://cluster-api-1.exmple.com:8006/api2/json + # insecure: false + # token_id: "login!name" + # token_secret: "secret" + # region: cluster-1 + +# -- Pods Service Account. +# ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +# -- CCM pods' priorityClassName. +priorityClassName: system-cluster-critical + +# -- Add additional init containers to the CCM pods. +# ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ +initContainers: [] + # - name: loadbalancer + # restartPolicy: Always + # image: ghcr.io/sergelogvinov/haproxy:2.8.3-alpine3.18 + # imagePullPolicy: IfNotPresent + # env: + # - name: SVC + # value: "proxmox.domain.com" + # - name: PORT + # value: "8006" + # securityContext: + # runAsUser: 99 + # runAsGroup: 99 + # resources: + # limits: + # cpu: 50m + # memory: 64Mi + # requests: + # cpu: 50m + # memory: 32Mi + +# -- hostAliases Deployment pod host aliases +# ref: https://kubernetes.io/docs/tasks/network/customize-hosts-file-for-pods/ +hostAliases: [] + # - ip: 127.0.0.1 + # hostnames: + # - proxmox.domain.com + +# -- Annotations for data pods. +# ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +podAnnotations: {} + +# -- Pods Security Context. +# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod +podSecurityContext: + runAsNonRoot: true + runAsUser: 10258 + runAsGroup: 10258 + fsGroup: 10258 + fsGroupChangePolicy: "OnRootMismatch" + +# -- Container Security Context. +# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + +# -- Resource requests and limits. +# ref: https://kubernetes.io/docs/user-guide/compute-resources/ +resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + requests: + cpu: 10m + memory: 32Mi + +# -- Deploy CCM in Daemonset mode. +# CCM will use hostNetwork. +# It allows to use CCM without CNI plugins. +useDaemonSet: false + +# -- Deployment update strategy type. +# ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + +# -- Node labels for data pods assignment. +# ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + # node-role.kubernetes.io/control-plane: "" + +# -- Tolerations for data pods assignment. +# ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + +# -- Affinity for data pods assignment. +# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +affinity: {} +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchExpressions: +# - key: node-role.kubernetes.io/control-plane +# operator: Exists + +# -- Additional volumes for Pods +extraVolumes: [] +# - name: ca +# secret: +# secretName: my-ca +# -- Additional volume mounts for Pods +extraVolumeMounts: [] +# - mountPath: /etc/ssl/certs/ca-certificates.crt +# name: ca +# subPath: ca.crt diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/.helmignore b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/Chart.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/Chart.yaml new file mode 100644 index 000000000..49aa71607 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/Chart.yaml @@ -0,0 +1,27 @@ +# yaml-language-server: $schema=https://json.schemastore.org/chart.json +apiVersion: v2 +name: proxmox-csi-plugin +description: Container Storage Interface plugin for Proxmox +type: application +home: https://github.com/sergelogvinov/proxmox-csi-plugin +icon: https://raw.githubusercontent.com/sergelogvinov/proxmox-csi-plugin/main/charts/proxmox-csi-plugin/icon.png +sources: + - https://github.com/sergelogvinov/proxmox-csi-plugin +keywords: + - csi + - storage + - block-storage + - volume + - proxmox +maintainers: + - name: sergelogvinov + url: https://github.com/sergelogvinov +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.2.13 +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: v0.8.2 diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/README.md b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/README.md new file mode 100644 index 000000000..fd67aab78 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/README.md @@ -0,0 +1,149 @@ +# proxmox-csi-plugin + +![Version: 0.2.13](https://img.shields.io/badge/Version-0.2.13-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.8.2](https://img.shields.io/badge/AppVersion-v0.8.2-informational?style=flat-square) + +Container Storage Interface plugin for Proxmox + +The Container Storage Interface (CSI) plugin is a specification designed to standardize the way container orchestration systems like Kubernetes, interact with different storage systems. The CSI plugin abstracts the underlying storage, enabling the seamless integration of different storage solutions (such as local block devices, file systems, or cloud-based storage) with containerized applications. + +This plugin allows Kubernetes to use `Proxmox VE` storage as a persistent storage solution for stateful applications. +Supported storage types: +- Directory +- LVM +- LVM-thin +- ZFS +- NFS +- Ceph + +**Homepage:** + +## Maintainers + +| Name | Email | Url | +| ---- | ------ | --- | +| sergelogvinov | | | + +## Source Code + +* + +## Proxmox permissions + +```shell +# Create role CSI +pveum role add CSI -privs "VM.Audit VM.Config.Disk Datastore.Allocate Datastore.AllocateSpace Datastore.Audit" +# Create user and grant permissions +pveum user add kubernetes-csi@pve +pveum aclmod / -user kubernetes-csi@pve -role CSI +pveum user token add kubernetes-csi@pve csi -privsep 0 +``` + +## Helm values example + +```yaml +# proxmox-csi.yaml + +config: + clusters: + - url: https://cluster-api-1.exmple.com:8006/api2/json + insecure: false + token_id: "kubernetes-csi@pve!csi" + token_secret: "key" + region: cluster-1 + +# Deploy Node CSI driver only on proxmox nodes +node: + nodeSelector: + # It will work only with Talos CCM, remove it overwise + node.cloudprovider.kubernetes.io/platform: nocloud + tolerations: + - operator: Exists + +# Deploy CSI controller only on control-plane nodes +nodeSelector: + node-role.kubernetes.io/control-plane: "" +tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + +# Define storage classes +# See https://pve.proxmox.com/wiki/Storage +storageClass: + - name: proxmox-data-xfs + storage: data + reclaimPolicy: Delete + fstype: xfs + - name: proxmox-data + storage: data + reclaimPolicy: Delete + fstype: ext4 + cache: writethrough +``` + +## Deploy + +```shell +# Prepare namespace +kubectl create ns csi-proxmox +kubectl label ns csi-proxmox pod-security.kubernetes.io/enforce=privileged +# Install Proxmox CSI plugin +helm upgrade -i --namespace=csi-proxmox -f proxmox-csi.yaml \ + proxmox-csi-plugin oci://ghcr.io/sergelogvinov/charts/proxmox-csi-plugin +``` + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| replicaCount | int | `1` | | +| imagePullSecrets | list | `[]` | | +| nameOverride | string | `""` | | +| fullnameOverride | string | `""` | | +| createNamespace | bool | `false` | Create namespace. Very useful when using helm template. | +| priorityClassName | string | `"system-cluster-critical"` | Controller pods priorityClassName. | +| serviceAccount | object | `{"annotations":{},"create":true,"name":""}` | Pods Service Account. ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ | +| provisionerName | string | `"csi.proxmox.sinextra.dev"` | CSI Driver provisioner name. Currently, cannot be customized. | +| clusterID | string | `"kubernetes"` | Cluster name. Currently, cannot be customized. | +| logVerbosityLevel | int | `5` | Log verbosity level. See https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md for description of individual verbosity levels. | +| timeout | string | `"3m"` | Connection timeout between sidecars. | +| existingConfigSecret | string | `nil` | Proxmox cluster config stored in secrets. | +| existingConfigSecretKey | string | `"config.yaml"` | Proxmox cluster config stored in secrets key. | +| configFile | string | `"/etc/proxmox/config.yaml"` | Proxmox cluster config path. | +| config | object | `{"clusters":[]}` | Proxmox cluster config. | +| storageClass | list | `[]` | Storage class definition. | +| controller.podAnnotations | object | `{}` | Annotations for controller pod. ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ | +| controller.plugin.image | object | `{"pullPolicy":"IfNotPresent","repository":"ghcr.io/sergelogvinov/proxmox-csi-controller","tag":""}` | Controller CSI Driver. | +| controller.plugin.resources | object | `{"requests":{"cpu":"10m","memory":"16Mi"}}` | Controller resource requests and limits. ref: https://kubernetes.io/docs/user-guide/compute-resources/ | +| controller.attacher.image | object | `{"pullPolicy":"IfNotPresent","repository":"registry.k8s.io/sig-storage/csi-attacher","tag":"v4.4.4"}` | CSI Attacher. | +| controller.attacher.resources | object | `{"requests":{"cpu":"10m","memory":"16Mi"}}` | Attacher resource requests and limits. ref: https://kubernetes.io/docs/user-guide/compute-resources/ | +| controller.provisioner.image | object | `{"pullPolicy":"IfNotPresent","repository":"registry.k8s.io/sig-storage/csi-provisioner","tag":"v3.6.4"}` | CSI Provisioner. | +| controller.provisioner.resources | object | `{"requests":{"cpu":"10m","memory":"16Mi"}}` | Provisioner resource requests and limits. ref: https://kubernetes.io/docs/user-guide/compute-resources/ | +| controller.resizer.image | object | `{"pullPolicy":"IfNotPresent","repository":"registry.k8s.io/sig-storage/csi-resizer","tag":"v1.9.4"}` | CSI Resizer. | +| controller.resizer.resources | object | `{"requests":{"cpu":"10m","memory":"16Mi"}}` | Resizer resource requests and limits. ref: https://kubernetes.io/docs/user-guide/compute-resources/ | +| node.plugin.image | object | `{"pullPolicy":"IfNotPresent","repository":"ghcr.io/sergelogvinov/proxmox-csi-node","tag":""}` | Node CSI Driver. | +| node.plugin.resources | object | `{}` | Node CSI Driver resource requests and limits. ref: https://kubernetes.io/docs/user-guide/compute-resources/ | +| node.driverRegistrar.image | object | `{"pullPolicy":"IfNotPresent","repository":"registry.k8s.io/sig-storage/csi-node-driver-registrar","tag":"v2.9.4"}` | Node CSI driver registrar. | +| node.driverRegistrar.resources | object | `{"requests":{"cpu":"10m","memory":"16Mi"}}` | Node registrar resource requests and limits. ref: https://kubernetes.io/docs/user-guide/compute-resources/ | +| node.kubeletDir | string | `"/var/lib/kubelet"` | Location of the /var/lib/kubelet directory as some k8s distribution differ from the standard. | +| node.nodeSelector | object | `{}` | Node labels for node-plugin assignment. ref: https://kubernetes.io/docs/user-guide/node-selection/ | +| node.tolerations | list | `[{"effect":"NoSchedule","key":"node.kubernetes.io/unschedulable","operator":"Exists"},{"effect":"NoSchedule","key":"node.kubernetes.io/disk-pressure","operator":"Exists"}]` | Tolerations for node-plugin assignment. ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ | +| livenessprobe.image | object | `{"pullPolicy":"IfNotPresent","repository":"registry.k8s.io/sig-storage/livenessprobe","tag":"v2.11.0"}` | Common livenessprobe sidecar. | +| livenessprobe.failureThreshold | int | `5` | Failure threshold for livenessProbe | +| livenessprobe.initialDelaySeconds | int | `10` | Initial delay seconds for livenessProbe | +| livenessprobe.timeoutSeconds | int | `10` | Timeout seconds for livenessProbe | +| livenessprobe.periodSeconds | int | `60` | Period seconds for livenessProbe | +| livenessprobe.resources | object | `{"requests":{"cpu":"10m","memory":"16Mi"}}` | Liveness probe resource requests and limits. ref: https://kubernetes.io/docs/user-guide/compute-resources/ | +| initContainers | list | `[]` | Add additional init containers for the CSI controller pods. ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ | +| hostAliases | list | `[]` | hostAliases Deployment pod host aliases ref: https://kubernetes.io/docs/tasks/network/customize-hosts-file-for-pods/ | +| podAnnotations | object | `{}` | Annotations for controller pod. ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ | +| podSecurityContext | object | `{"fsGroup":65532,"fsGroupChangePolicy":"OnRootMismatch","runAsGroup":65532,"runAsNonRoot":true,"runAsUser":65532}` | Controller Security Context. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod | +| securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"seccompProfile":{"type":"RuntimeDefault"}}` | Controller Container Security Context. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod | +| updateStrategy | object | `{"rollingUpdate":{"maxUnavailable":1},"type":"RollingUpdate"}` | Controller deployment update strategy type. ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment | +| metrics | object | `{"enabled":false,"port":8080,"type":"annotation"}` | Prometheus metrics | +| metrics.enabled | bool | `false` | Enable Prometheus metrics. | +| metrics.port | int | `8080` | Prometheus metrics port. | +| nodeSelector | object | `{}` | Node labels for controller assignment. ref: https://kubernetes.io/docs/user-guide/node-selection/ | +| tolerations | list | `[]` | Tolerations for controller assignment. ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ | +| affinity | object | `{}` | Affinity for controller assignment. ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity | +| extraVolumes | list | `[]` | Additional volumes for Pods | +| extraVolumeMounts | list | `[]` | | diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/README.md.gotmpl b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/README.md.gotmpl new file mode 100644 index 000000000..5c3e60c08 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/README.md.gotmpl @@ -0,0 +1,92 @@ +{{ template "chart.header" . }} + +{{ template "chart.deprecationWarning" . }} + +{{ template "chart.badgesSection" . }} + +{{ template "chart.description" . }} + +The Container Storage Interface (CSI) plugin is a specification designed to standardize the way container orchestration systems like Kubernetes, interact with different storage systems. The CSI plugin abstracts the underlying storage, enabling the seamless integration of different storage solutions (such as local block devices, file systems, or cloud-based storage) with containerized applications. + +This plugin allows Kubernetes to use `Proxmox VE` storage as a persistent storage solution for stateful applications. +Supported storage types: +- Directory +- LVM +- LVM-thin +- ZFS +- NFS +- Ceph + +{{ template "chart.homepageLine" . }} + +{{ template "chart.maintainersSection" . }} + +{{ template "chart.sourcesSection" . }} + +{{ template "chart.requirementsSection" . }} + +## Proxmox permissions + +```shell +# Create role CSI +pveum role add CSI -privs "VM.Audit VM.Config.Disk Datastore.Allocate Datastore.AllocateSpace Datastore.Audit" +# Create user and grant permissions +pveum user add kubernetes-csi@pve +pveum aclmod / -user kubernetes-csi@pve -role CSI +pveum user token add kubernetes-csi@pve csi -privsep 0 +``` + +## Helm values example + +```yaml +# proxmox-csi.yaml + +config: + clusters: + - url: https://cluster-api-1.exmple.com:8006/api2/json + insecure: false + token_id: "kubernetes-csi@pve!csi" + token_secret: "key" + region: cluster-1 + +# Deploy Node CSI driver only on proxmox nodes +node: + nodeSelector: + # It will work only with Talos CCM, remove it overwise + node.cloudprovider.kubernetes.io/platform: nocloud + tolerations: + - operator: Exists + +# Deploy CSI controller only on control-plane nodes +nodeSelector: + node-role.kubernetes.io/control-plane: "" +tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + +# Define storage classes +# See https://pve.proxmox.com/wiki/Storage +storageClass: + - name: proxmox-data-xfs + storage: data + reclaimPolicy: Delete + fstype: xfs + - name: proxmox-data + storage: data + reclaimPolicy: Delete + fstype: ext4 + cache: writethrough +``` + +## Deploy + +```shell +# Prepare namespace +kubectl create ns csi-proxmox +kubectl label ns csi-proxmox pod-security.kubernetes.io/enforce=privileged +# Install Proxmox CSI plugin +helm upgrade -i --namespace=csi-proxmox -f proxmox-csi.yaml \ + proxmox-csi-plugin oci://ghcr.io/sergelogvinov/charts/proxmox-csi-plugin +``` + +{{ template "chart.valuesSection" . }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/ci/values.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/ci/values.yaml new file mode 100644 index 000000000..7c8341402 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/ci/values.yaml @@ -0,0 +1,22 @@ + +node: + nodeSelector: + node.cloudprovider.kubernetes.io/platform: nocloud + tolerations: + - operator: Exists + +nodeSelector: + node-role.kubernetes.io/control-plane: "" +tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + +storageClass: + - name: proxmox-data-xfs + storage: data + reclaimPolicy: Delete + fstype: xfs + - name: proxmox-data + storage: data + reclaimPolicy: Delete + ssd: true diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/icon.png b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/icon.png new file mode 100644 index 000000000..abe9fdc67 Binary files /dev/null and b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/icon.png differ diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/NOTES.txt b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/NOTES.txt new file mode 100644 index 000000000..e69de29bb diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/_helpers.tpl b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/_helpers.tpl new file mode 100644 index 000000000..2a7f80273 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/_helpers.tpl @@ -0,0 +1,71 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "proxmox-csi-plugin.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "proxmox-csi-plugin.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "proxmox-csi-plugin.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "proxmox-csi-plugin.labels" -}} +helm.sh/chart: {{ include "proxmox-csi-plugin.chart" . }} +app.kubernetes.io/name: {{ include "proxmox-csi-plugin.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "proxmox-csi-plugin.selectorLabels" -}} +app.kubernetes.io/name: {{ include "proxmox-csi-plugin.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: controller +{{- end }} + +{{- define "proxmox-csi-plugin-node.selectorLabels" -}} +app.kubernetes.io/name: {{ include "proxmox-csi-plugin.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: node +{{- end }} + + +{{/* +Create the name of the service account to use +*/}} +{{- define "proxmox-csi-plugin.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "proxmox-csi-plugin.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/_storage.tpl b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/_storage.tpl new file mode 100644 index 000000000..ad7906c4b --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/_storage.tpl @@ -0,0 +1,13 @@ +{{/* +storageClass parameters uses to merge the default parameters with the user provided parameters. +*/}} +{{- define "storageClass.parameters" -}} +csi.storage.k8s.io/fstype: {{ default "ext4" .fstype }} +storage: {{ .storage | required "Proxmox Storage name must be provided." }} +{{- with .cache }} +cache: {{ . }} +{{- end }} +{{- if .ssd }} +ssd: "true" +{{- end }} +{{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-clusterrole.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-clusterrole.yaml new file mode 100644 index 000000000..6737e23c6 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-clusterrole.yaml @@ -0,0 +1,37 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }}-controller + namespace: {{ .Release.Namespace }} + labels: + {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "patch", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get","list", "watch", "create", "update", "patch"] + + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-deployment.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-deployment.yaml new file mode 100644 index 000000000..b17116f40 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-deployment.yaml @@ -0,0 +1,178 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }}-controller + namespace: {{ .Release.Namespace }} + labels: + {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + strategy: + type: {{ .Values.updateStrategy.type }} + rollingUpdate: + {{- toYaml .Values.updateStrategy.rollingUpdate | nindent 6 }} + selector: + matchLabels: + {{- include "proxmox-csi-plugin.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ toJson .Values.config | sha256sum }} + {{- with default .Values.podAnnotations .Values.controller.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if and .Values.metrics.enabled (eq .Values.metrics.type "annotation") }} + prometheus.io/scrape: "true" + prometheus.io/port: {{ .Values.metrics.port | quote }} + {{- end }} + labels: + {{- include "proxmox-csi-plugin.selectorLabels" . | nindent 8 }} + spec: + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + enableServiceLinks: false + serviceAccountName: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-controller + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + hostAliases: {{- toYaml .Values.hostAliases | nindent 8 }} + initContainers: {{- toYaml .Values.initContainers | nindent 8 }} + containers: + - name: {{ include "proxmox-csi-plugin.fullname" . }}-controller + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.controller.plugin.image.repository }}:{{ .Values.controller.plugin.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.controller.plugin.image.pullPolicy }} + args: + - "-v={{ .Values.logVerbosityLevel }}" + - "--csi-address=unix:///csi/csi.sock" + - "--cloud-config={{ .Values.configFile }}" + {{- if .Values.metrics.enabled }} + - "--metrics-address=:{{ .Values.metrics.port }}" + {{- end }} + ports: + {{- if .Values.metrics.enabled }} + - name: metrics + containerPort: {{ .Values.metrics.port }} + protocol: TCP + {{- end }} + resources: + {{- toYaml .Values.controller.plugin.resources | nindent 12 }} + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: cloud-config + mountPath: /etc/proxmox/ + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + - name: csi-attacher + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.controller.attacher.image.repository }}:{{ .Values.controller.attacher.image.tag }}" + imagePullPolicy: {{ .Values.controller.attacher.image.pullPolicy }} + args: + - "-v={{ .Values.logVerbosityLevel }}" + - "--csi-address=unix:///csi/csi.sock" + - "--timeout={{ .Values.timeout }}" + - "--leader-election" + - "--default-fstype=ext4" + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: {{ toYaml .Values.controller.attacher.resources | nindent 12 }} + - name: csi-provisioner + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.controller.provisioner.image.repository }}:{{ .Values.controller.provisioner.image.tag }}" + imagePullPolicy: {{ .Values.controller.provisioner.image.pullPolicy }} + args: + - "-v={{ .Values.logVerbosityLevel }}" + - "--csi-address=unix:///csi/csi.sock" + - "--timeout={{ .Values.timeout }}" + - "--leader-election" + - "--default-fstype=ext4" + - "--feature-gates=Topology=True" + - "--enable-capacity" + - "--capacity-ownerref-level=2" + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: {{ toYaml .Values.controller.provisioner.resources | nindent 12 }} + - name: csi-resizer + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.controller.resizer.image.repository }}:{{ .Values.controller.resizer.image.tag }}" + imagePullPolicy: {{ .Values.controller.resizer.image.pullPolicy }} + args: + - "-v={{ .Values.logVerbosityLevel }}" + - "--csi-address=unix:///csi/csi.sock" + - "--timeout={{ .Values.timeout }}" + - "--handle-volume-inuse-error=false" + - "--leader-election" + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: {{ toYaml .Values.controller.resizer.resources | nindent 12 }} + - name: liveness-probe + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.livenessprobe.image.repository }}:{{ .Values.livenessprobe.image.tag }}" + imagePullPolicy: {{ .Values.livenessprobe.image.pullPolicy }} + args: + - "-v={{ .Values.logVerbosityLevel }}" + - "--csi-address=unix:///csi/csi.sock" + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: {{ toYaml .Values.livenessprobe.resources | nindent 12 }} + volumes: + - name: socket-dir + emptyDir: {} + {{- if .Values.existingConfigSecret }} + - name: cloud-config + secret: + secretName: {{ .Values.existingConfigSecret }} + items: + - key: {{ .Values.existingConfigSecretKey }} + path: config.yaml + {{- else }} + - name: cloud-config + secret: + secretName: {{ include "proxmox-csi-plugin.fullname" . }} + {{- end }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + {{- include "proxmox-csi-plugin.selectorLabels" . | nindent 14 }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-role.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-role.yaml new file mode 100644 index 000000000..b69d223f9 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-role.yaml @@ -0,0 +1,21 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }}-controller + namespace: {{ .Release.Namespace }} + labels: + {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + + - apiGroups: ["storage.k8s.io"] + resources: ["csistoragecapacities"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get"] + - apiGroups: ["apps"] + resources: ["replicasets"] + verbs: ["get"] diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-rolebinding.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-rolebinding.yaml new file mode 100644 index 000000000..478e0f6f1 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-rolebinding.yaml @@ -0,0 +1,26 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }}-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "proxmox-csi-plugin.fullname" . }}-controller +subjects: + - kind: ServiceAccount + name: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-controller + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }}-controller + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "proxmox-csi-plugin.fullname" . }}-controller +subjects: + - kind: ServiceAccount + name: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-controller + namespace: {{ .Release.Namespace }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/csidriver.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/csidriver.yaml new file mode 100644 index 000000000..5d0652e86 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/csidriver.yaml @@ -0,0 +1,10 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: {{ .Values.provisionerName }} +spec: + attachRequired: true + podInfoOnMount: true + storageCapacity: true + volumeLifecycleModes: + - Persistent diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/namespace.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/namespace.yaml new file mode 100644 index 000000000..8c4ba9c4a --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/namespace.yaml @@ -0,0 +1,10 @@ +{{- if and .Values.createNamespace (ne .Release.Namespace "kube-system") }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Release.Namespace }} + labels: + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: baseline + pod-security.kubernetes.io/warn: baseline +{{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-clusterrole.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-clusterrole.yaml new file mode 100644 index 000000000..02af491f5 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-clusterrole.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }}-node + namespace: {{ .Release.Namespace }} + labels: + {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-deployment.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-deployment.yaml new file mode 100644 index 000000000..0219a78f4 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-deployment.yaml @@ -0,0 +1,135 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }}-node + namespace: {{ .Release.Namespace }} + labels: + {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} +spec: + updateStrategy: + type: {{ .Values.updateStrategy.type }} + selector: + matchLabels: + {{- include "proxmox-csi-plugin-node.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "proxmox-csi-plugin-node.selectorLabels" . | nindent 8 }} + spec: + priorityClassName: system-node-critical + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + enableServiceLinks: false + serviceAccountName: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-node + securityContext: + runAsUser: 0 + runAsGroup: 0 + containers: + - name: {{ include "proxmox-csi-plugin.fullname" . }}-node + securityContext: + privileged: true + capabilities: + drop: + - ALL + add: + - SYS_ADMIN + - CHOWN + - DAC_OVERRIDE + seccompProfile: + type: RuntimeDefault + image: "{{ .Values.node.plugin.image.repository }}:{{ .Values.node.plugin.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.node.plugin.image.pullPolicy }} + args: + - "-v={{ .Values.logVerbosityLevel }}" + - "--csi-address=unix:///csi/csi.sock" + - "--node-id=$(NODE_NAME)" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: {{- toYaml .Values.node.plugin.resources | nindent 12 }} + volumeMounts: + - name: socket + mountPath: /csi + - name: kubelet + mountPath: {{ .Values.node.kubeletDir }} + mountPropagation: Bidirectional + - name: dev + mountPath: /dev + - name: sys + mountPath: /sys + - name: csi-node-driver-registrar + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + image: "{{ .Values.node.driverRegistrar.image.repository }}:{{ .Values.node.driverRegistrar.image.tag }}" + imagePullPolicy: {{ .Values.node.driverRegistrar.image.pullPolicy }} + args: + - "-v={{ .Values.logVerbosityLevel }}" + - "--csi-address=unix:///csi/csi.sock" + - "--kubelet-registration-path={{ .Values.node.kubeletDir }}/plugins/{{ .Values.provisionerName }}/csi.sock" + volumeMounts: + - name: socket + mountPath: /csi + - name: registration + mountPath: /registration + resources: {{- toYaml .Values.node.driverRegistrar.resources | nindent 12 }} + - name: liveness-probe + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + image: "{{ .Values.livenessprobe.image.repository }}:{{ .Values.livenessprobe.image.tag }}" + imagePullPolicy: {{ .Values.livenessprobe.image.pullPolicy }} + args: + - "-v={{ .Values.logVerbosityLevel }}" + - "--csi-address=unix:///csi/csi.sock" + volumeMounts: + - name: socket + mountPath: /csi + resources: {{- toYaml .Values.livenessprobe.resources | nindent 12 }} + volumes: + - name: socket + hostPath: + path: {{ .Values.node.kubeletDir }}/plugins/{{ .Values.provisionerName }}/ + type: DirectoryOrCreate + - name: registration + hostPath: + path: {{ .Values.node.kubeletDir }}/plugins_registry/ + type: Directory + - name: kubelet + hostPath: + path: {{ .Values.node.kubeletDir }} + type: Directory + - name: dev + hostPath: + path: /dev + type: Directory + - name: sys + hostPath: + path: /sys + type: Directory + {{- with .Values.node.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.node.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-rolebinding.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-rolebinding.yaml new file mode 100644 index 000000000..4585dd66a --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-rolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }}-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "proxmox-csi-plugin.fullname" . }}-node +subjects: + - kind: ServiceAccount + name: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-node + namespace: {{ .Release.Namespace }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/secrets.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/secrets.yaml new file mode 100644 index 000000000..f96f5d574 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/secrets.yaml @@ -0,0 +1,12 @@ +{{- if ne (len .Values.config.clusters) 0 }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} +type: Opaque +data: + config.yaml: {{ toYaml .Values.config | b64enc | quote }} +{{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/serviceaccount.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/serviceaccount.yaml new file mode 100644 index 000000000..d6a455484 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/serviceaccount.yaml @@ -0,0 +1,25 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-controller + namespace: {{ .Release.Namespace }} + labels: + {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-node + namespace: {{ .Release.Namespace }} + labels: + {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/storageclass.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/storageclass.yaml new file mode 100644 index 000000000..702c4118a --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/storageclass.yaml @@ -0,0 +1,29 @@ +{{- range $storage := .Values.storageClass }} +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ $storage.name | required "StorageClass name must be provided." }} + {{- with $storage.labels }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $storage.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +provisioner: {{ $.Values.provisionerName }} +allowVolumeExpansion: true +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: {{ default "Delete" $storage.reclaimPolicy }} +parameters: + {{- mustMergeOverwrite (default (dict) $storage.extraParameters) (include "storageClass.parameters" . | fromYaml) | toYaml | nindent 2 -}} +{{- with $storage.mountOptions }} +mountOptions: + {{- . | toYaml | nindent 2 }} +{{- end }} +{{- with $storage.allowedTopologies }} +allowedTopologies: + {{- . | toYaml | nindent 2 }} +{{- end }} +--- +{{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.edge.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.edge.yaml new file mode 100644 index 000000000..974c87e9d --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.edge.yaml @@ -0,0 +1,29 @@ + +createNamespace: true + +controller: + plugin: + image: + pullPolicy: Always + tag: edge + +node: + plugin: + image: + pullPolicy: Always + tag: edge + +nodeSelector: + node-role.kubernetes.io/control-plane: "" +tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + +storageClass: + - name: proxmox-data-xfs + storage: data + reclaimPolicy: Delete + fstype: xfs + - name: proxmox-data + storage: data + ssd: true diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.talos.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.talos.yaml new file mode 100644 index 000000000..d507b31e4 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.talos.yaml @@ -0,0 +1,23 @@ + +createNamespace: true + +node: + nodeSelector: + node.cloudprovider.kubernetes.io/platform: nocloud + tolerations: + - operator: Exists + +nodeSelector: + node-role.kubernetes.io/control-plane: "" +tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + +storageClass: + - name: proxmox-data-xfs + storage: data + reclaimPolicy: Delete + fstype: xfs + - name: proxmox-data + storage: data + reclaimPolicy: Delete diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.yaml new file mode 100644 index 000000000..99c9fac1d --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.yaml @@ -0,0 +1,308 @@ +# Default values for proxmox-csi-plugin. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +# -- Create namespace. +# Very useful when using helm template. +createNamespace: false + +# -- Controller pods priorityClassName. +priorityClassName: system-cluster-critical + +# -- Pods Service Account. +# ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +# -- CSI Driver provisioner name. +# Currently, cannot be customized. +provisionerName: csi.proxmox.sinextra.dev + +# -- Cluster name. +# Currently, cannot be customized. +clusterID: kubernetes + +# -- Log verbosity level. See https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md +# for description of individual verbosity levels. +logVerbosityLevel: 5 + +# -- Connection timeout between sidecars. +timeout: 3m + +# -- Proxmox cluster config stored in secrets. +existingConfigSecret: ~ +# -- Proxmox cluster config stored in secrets key. +existingConfigSecretKey: config.yaml + +# -- Proxmox cluster config path. +configFile: /etc/proxmox/config.yaml + +# -- Proxmox cluster config. +config: + clusters: [] + # - url: https://cluster-api-1.exmple.com:8006/api2/json + # insecure: false + # token_id: "login!name" + # token_secret: "secret" + # region: cluster-1 + +# -- Storage class definition. +storageClass: [] + # - name: proxmox-data-xfs + # storage: data + # reclaimPolicy: Delete + # fstype: ext4|xfs + # + # # https://pve.proxmox.com/wiki/Performance_Tweaks + # cache: directsync|none|writeback|writethrough + # ssd: true + # extraParameters: + # # https://kubernetes-csi.github.io/docs/secrets-and-credentials-storage-class.html + # csi.storage.k8s.io/node-stage-secret-name: "proxmox-csi-secret" + # csi.storage.k8s.io/node-stage-secret-namespace: "kube-system" + # csi.storage.k8s.io/node-expand-secret-name: "proxmox-csi-secret" + # csi.storage.k8s.io/node-expand-secret-namespace: "kube-system" + # + # mountOptions: + # - discard + # + # # I am not recommending to use this option, but it is possible. + # # You should use region and zone topology keys together. + # allowedTopologies: + # - matchLabelExpressions: + # - key: topology.kubernetes.io/region + # values: + # - Region-1 + # - key: topology.kubernetes.io/zone + # values: + # - pve-1 + # - pve-3 + +controller: + # -- Annotations for controller pod. + # ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ + podAnnotations: {} + # prometheus.io/scrape: "true" + # prometheus.io/port: "8080" + + plugin: + # -- Controller CSI Driver. + image: + repository: ghcr.io/sergelogvinov/proxmox-csi-controller + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + # -- Controller resource requests and limits. + # ref: https://kubernetes.io/docs/user-guide/compute-resources/ + resources: + requests: + cpu: 10m + memory: 16Mi + attacher: + # -- CSI Attacher. + image: + repository: registry.k8s.io/sig-storage/csi-attacher + pullPolicy: IfNotPresent + tag: v4.4.4 + # -- Attacher resource requests and limits. + # ref: https://kubernetes.io/docs/user-guide/compute-resources/ + resources: + requests: + cpu: 10m + memory: 16Mi + provisioner: + # -- CSI Provisioner. + image: + repository: registry.k8s.io/sig-storage/csi-provisioner + pullPolicy: IfNotPresent + tag: v3.6.4 + # -- Provisioner resource requests and limits. + # ref: https://kubernetes.io/docs/user-guide/compute-resources/ + resources: + requests: + cpu: 10m + memory: 16Mi + resizer: + # -- CSI Resizer. + image: + repository: registry.k8s.io/sig-storage/csi-resizer + pullPolicy: IfNotPresent + tag: v1.9.4 + # -- Resizer resource requests and limits. + # ref: https://kubernetes.io/docs/user-guide/compute-resources/ + resources: + requests: + cpu: 10m + memory: 16Mi + +node: + plugin: + # -- Node CSI Driver. + image: + repository: ghcr.io/sergelogvinov/proxmox-csi-node + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + # -- Node CSI Driver resource requests and limits. + # ref: https://kubernetes.io/docs/user-guide/compute-resources/ + resources: {} + driverRegistrar: + # -- Node CSI driver registrar. + image: + repository: registry.k8s.io/sig-storage/csi-node-driver-registrar + pullPolicy: IfNotPresent + tag: v2.9.4 + # -- Node registrar resource requests and limits. + # ref: https://kubernetes.io/docs/user-guide/compute-resources/ + resources: + requests: + cpu: 10m + memory: 16Mi + + # -- Location of the /var/lib/kubelet directory as some k8s distribution differ from the standard. + kubeletDir: /var/lib/kubelet + + # -- Node labels for node-plugin assignment. + # ref: https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for node-plugin assignment. + # ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: + - key: node.kubernetes.io/unschedulable + operator: Exists + effect: NoSchedule + - key: node.kubernetes.io/disk-pressure + operator: Exists + effect: NoSchedule + +livenessprobe: + # -- Common livenessprobe sidecar. + image: + repository: registry.k8s.io/sig-storage/livenessprobe + pullPolicy: IfNotPresent + tag: v2.11.0 + # -- Failure threshold for livenessProbe + failureThreshold: 5 + # -- Initial delay seconds for livenessProbe + initialDelaySeconds: 10 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 10 + # -- Period seconds for livenessProbe + periodSeconds: 60 + # -- Liveness probe resource requests and limits. + # ref: https://kubernetes.io/docs/user-guide/compute-resources/ + resources: + requests: + cpu: 10m + memory: 16Mi + +# -- Add additional init containers for the CSI controller pods. +# ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ +initContainers: [] + # - name: loadbalancer + # restartPolicy: Always + # image: ghcr.io/sergelogvinov/haproxy:2.8.3-alpine3.18 + # imagePullPolicy: IfNotPresent + # env: + # - name: SVC + # value: "proxmox.domain.com" + # - name: PORT + # value: "8006" + # securityContext: + # runAsUser: 99 + # runAsGroup: 99 + # resources: + # limits: + # cpu: 50m + # memory: 64Mi + # requests: + # cpu: 50m + # memory: 32Mi + +# -- hostAliases Deployment pod host aliases +# ref: https://kubernetes.io/docs/tasks/network/customize-hosts-file-for-pods/ +hostAliases: [] + # - ip: 127.0.0.1 + # hostnames: + # - proxmox.domain.com + +# -- Annotations for controller pod. +# ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +podAnnotations: {} + +# -- Controller Security Context. +# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod +podSecurityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + fsGroupChangePolicy: OnRootMismatch + +# -- Controller Container Security Context. +# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + readOnlyRootFilesystem: true + +# -- Controller deployment update strategy type. +# ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + +# -- Prometheus metrics +metrics: + # -- Enable Prometheus metrics. + enabled: false + # -- Prometheus metrics port. + port: 8080 + + type: annotation + + +# -- Node labels for controller assignment. +# ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + # node-role.kubernetes.io/control-plane: "" + +# -- Tolerations for controller assignment. +# ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: [] + # - key: node-role.kubernetes.io/control-plane + # effect: NoSchedule + +# -- Affinity for controller assignment. +# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +affinity: {} + +# -- Additional volumes for Pods +extraVolumes: [] +# - name: ca +# secret: +# secretName: my-ca +# -- Additional volume mounts for Pods + +extraVolumeMounts: [] +# - mountPath: /etc/ssl/certs/ca-certificates.crt +# name: ca +# subPath: ca.crt diff --git a/packages/system/proxmox-csi/patches/namespace.patch b/packages/system/proxmox-csi/patches/namespace.patch new file mode 100644 index 000000000..c06485768 --- /dev/null +++ b/packages/system/proxmox-csi/patches/namespace.patch @@ -0,0 +1,13 @@ +diff --git a/apps/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml b/apps/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml +index 0ed037f..32b065e 100644 +--- a/apps/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml ++++ b/apps/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml +@@ -9,7 +9,7 @@ roleRef: + subjects: + - kind: ServiceAccount + name: {{ include "proxmox-cloud-controller-manager.fullname" . }} +- namespace: kube-system ++ namespace: {{ .Release.Namespace }} + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding diff --git a/packages/system/proxmox-csi/tests/1.yaml b/packages/system/proxmox-csi/tests/1.yaml new file mode 100644 index 000000000..e8a19fc92 --- /dev/null +++ b/packages/system/proxmox-csi/tests/1.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: task-pv-claim +spec: + storageClassName: proxmox-lvm + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 3Gi +--- +apiVersion: v1 +kind: Pod +metadata: + name: task-pv-pod +spec: + volumes: + - name: task-pv-storage + persistentVolumeClaim: + claimName: task-pv-claim + containers: + - name: task-pv-container + image: nginx + ports: + - containerPort: 80 + name: "http-server" + volumeMounts: + - mountPath: "/usr/share/nginx/html" + name: task-pv-storage diff --git a/packages/system/proxmox-csi/values.yaml b/packages/system/proxmox-csi/values.yaml new file mode 100644 index 000000000..19a5f9f19 --- /dev/null +++ b/packages/system/proxmox-csi/values.yaml @@ -0,0 +1,22 @@ +proxmox-cloud-controller-manager: + fullnameOverride: proxmox-cloud-controller-manager + + enabledControllers: + - cloud-node + - cloud-node-lifecycle + + # Deploy CCM only on control-plane nodes + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + +proxmox-csi-plugin: + fullnameOverride: proxmox-csi-plugin + + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule diff --git a/scripts/common-envs.mk b/scripts/common-envs.mk index 668266c5a..a5f80103e 100644 --- a/scripts/common-envs.mk +++ b/scripts/common-envs.mk @@ -1,4 +1,5 @@ REGISTRY := ghcr.io/aenix-io/cozystack + PUSH := 1 LOAD := 0 VERSION = $(patsubst v%,%,$(shell git describe --tags --abbrev=0)) diff --git a/scripts/installer.sh b/scripts/installer.sh index 5130fc3ed..4ec4e5074 100755 --- a/scripts/installer.sh +++ b/scripts/installer.sh @@ -50,10 +50,10 @@ wait_for_crds() { } install_basic_charts() { - if [ "$BUNDLE" = "paas-full" ] || [ "$BUNDLE" = "distro-full" ]; then - make -C packages/system/cilium apply resume + if [ "$BUNDLE" = "paas-full" ] || [ "$BUNDLE" = "distro-full" || "$BUNDLE" = "paas-proxmox" ]; then + make -C packages/system/cilium apply resume fi - if [ "$BUNDLE" = "paas-full" ]; then + if [ "$BUNDLE" = "paas-full" || "$BUNDLE" = "paas-proxmox" ]; then make -C packages/system/kubeovn apply resume fi }