diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..9f401f87e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,48 @@ +--- +name: CI/CD Workflow + +on: + push: + branches: + - main + paths: + - '**.yaml' + - '**/Dockerfile' + - '**/charts/**' + tags: + - 'v*' + +env: + IMAGE_NGINX_CACHE: nginx-cache + REGISTRY: ghcr.io/${{ github.repository_owner }} + PUSH: 1 + LOAD: 1 + NGINX_CACHE_TAG: v0.1.0 + TAG: v0.3.1 + PLATFORM_ARCH: linux/amd64 + +jobs: + build-and-push: + name: Build Cozystack + runs-on: ubuntu-latest + services: + registry: + image: registry:2 + ports: + - 5000:5000 + steps: + - name: Set up Docker Registry + run: | + if [ "$GITHUB_ACTIONS" = "true" ]; then + echo "REGISTRY=ghcr.io/${{ github.repository_owner }}" >> $GITHUB_ENV + else + echo "REGISTRY=localhost:5000/cozystack_local" >> $GITHUB_ENV + fi + + - uses: actions/checkout@v3 + - name: Build usig make + run: | + make + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..a79650b46 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,48 @@ +name: Lint + +on: + push: + branches: [ main ] # Lint only on pushes to the main branch + pull_request: + branches: [ main ] # Lint on PRs targeting the main branch + +permissions: + contents: read + +jobs: + lint: + name: Super-Linter + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Run Super-Linter + uses: github/super-linter@v4 + env: + # To report GitHub Actions status checks + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VALIDATE_ALL_CODEBASE: false # Lint only changed files + VALIDATE_TERRAFORM: false # Disable Terraform linting (remove if you need it) + DEFAULT_BRANCH: main # Set your default branch + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Enable only the linters you need for your project + VALIDATE_JAVASCRIPT_ES: true + VALIDATE_PYTHON_BLACK: true + VALIDATE_HTML: false + VALIDATE_GO: false + VALIDATE_XML: false + VALIDATE_JAVA: false + VALIDATE_DOCKERFILE: false + # turn off JSCPD copy/paste detection, which results in lots of results for examples and devops repos + VALIDATE_JSCPD: false + # turn off shfmt shell formatter as we already have shellcheck + VALIDATE_SHELL_SHFMT: false + VALIDATE_EDITORCONFIG: false + # prevent Kubernetes CRD API's from causing kubeval to fail + # also change schema location to an up-to-date list + # https://github.com/yannh/kubernetes-json-schema/#kubeval + KUBERNETES_KUBEVAL_OPTIONS: --ignore-missing-schemas --schema-location https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/ diff --git a/.github/workflows/linters/.markdown-lint.yml b/.github/workflows/linters/.markdown-lint.yml new file mode 100644 index 000000000..335724b49 --- /dev/null +++ b/.github/workflows/linters/.markdown-lint.yml @@ -0,0 +1,7 @@ +--- + # MD013/line-length - Line length + MD013: + # Number of characters, default is 80 + line_length: 9999 + # check code blocks? + code_blocks: false diff --git a/.github/workflows/linters/.yaml-lint.yml b/.github/workflows/linters/.yaml-lint.yml new file mode 100644 index 000000000..95afab763 --- /dev/null +++ b/.github/workflows/linters/.yaml-lint.yml @@ -0,0 +1,55 @@ + +yaml-files: +- '*.yaml' +- '*.yml' +- '.yamllint' + +rules: + braces: + level: warning + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 1 + max-spaces-inside-empty: 5 + brackets: + level: warning + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 1 + max-spaces-inside-empty: 5 + colons: + level: warning + max-spaces-before: 0 + max-spaces-after: 1 + commas: + level: warning + max-spaces-before: 0 + min-spaces-after: 1 + max-spaces-after: 1 + comments: disable + comments-indentation: disable + document-end: disable + document-start: disable + empty-lines: + level: warning + max: 2 + max-start: 0 + max-end: 0 + hyphens: + level: warning + max-spaces-after: 1 + indentation: + level: warning + spaces: consistent + indent-sequences: true + check-multi-line-strings: false + key-duplicates: enable + line-length: disable + new-line-at-end-of-file: disable + new-lines: + type: unix + trailing-spaces: disable + line-length: + max: 130 + allow-non-breakable-words: true + allow-non-breakable-inline-mappings: false diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..082b19437 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "makefile.configureOnOpen": false +} \ No newline at end of file diff --git a/packages/apps/Makefile b/packages/apps/Makefile index 8751a0484..c22b2eece 100644 --- a/packages/apps/Makefile +++ b/packages/apps/Makefile @@ -5,7 +5,7 @@ repo: cd .. && ../hack/package_chart.sh apps $(OUT) $(TMP) library fix-chartnames: - find . -maxdepth 2 -name Chart.yaml | awk -F/ '{print $$2}' | while read i; do sed -i "s/^name: .*/name: $$i/" "$$i/Chart.yaml"; done + find . -maxdepth 2 -name Chart.yaml | awk -F/ '{print $$2}' | while read i; do sed "s/^name: .*/name: $$i/" "$$i/Chart.yaml"; done gen-versions-map: fix-chartnames ../../hack/gen_versions_map.sh diff --git a/packages/apps/ferretdb/ferretdb.svg b/packages/apps/ferretdb/ferretdb.svg new file mode 100644 index 000000000..196871e1b --- /dev/null +++ b/packages/apps/ferretdb/ferretdb.svg @@ -0,0 +1,54 @@ + + + + + + + + + + + + diff --git a/packages/core/platform/bundles/paas-proxmox.yaml b/packages/core/platform/bundles/paas-proxmox.yaml new file mode 100644 index 000000000..dd82de126 --- /dev/null +++ b/packages/core/platform/bundles/paas-proxmox.yaml @@ -0,0 +1,138 @@ +{{- $cozyConfig := lookup "v1" "ConfigMap" "cozy-system" "cozystack" }} + +releases: +- name: cilium + releaseName: cilium + chart: cozy-cilium + namespace: cozy-cilium + privileged: true + dependsOn: [] + +- name: kubeovn + releaseName: kubeovn + chart: cozy-kubeovn + namespace: cozy-kubeovn + privileged: true + dependsOn: [cilium] + values: + cozystack: + nodesHash: {{ include "cozystack.master-node-ips" . | sha256sum }} + kube-ovn: + ipv4: + POD_CIDR: "{{ index $cozyConfig.data "ipv4-pod-cidr" }}" + POD_GATEWAY: "{{ index $cozyConfig.data "ipv4-pod-gateway" }}" + SVC_CIDR: "{{ index $cozyConfig.data "ipv4-svc-cidr" }}" + JOIN_CIDR: "{{ index $cozyConfig.data "ipv4-join-cidr" }}" + +- name: cert-manager + releaseName: cert-manager + chart: cozy-cert-manager + namespace: cozy-cert-manager + dependsOn: [cilium,kubeovn] + +- name: cert-manager-issuers + releaseName: cert-manager-issuers + chart: cozy-cert-manager-issuers + namespace: cozy-cert-manager + dependsOn: [cilium,kubeovn,cert-manager] + +- name: victoria-metrics-operator + releaseName: victoria-metrics-operator + chart: cozy-victoria-metrics-operator + namespace: cozy-victoria-metrics-operator + dependsOn: [cilium,kubeovn,cert-manager] + +- name: monitoring + releaseName: monitoring + chart: cozy-monitoring + namespace: cozy-monitoring + privileged: true + dependsOn: [cilium,kubeovn,victoria-metrics-operator] + +- name: grafana-operator + releaseName: grafana-operator + chart: cozy-grafana-operator + namespace: cozy-grafana-operator + dependsOn: [cilium,kubeovn] + +- name: mariadb-operator + releaseName: mariadb-operator + chart: cozy-mariadb-operator + namespace: cozy-mariadb-operator + dependsOn: [cilium,kubeovn,cert-manager,victoria-metrics-operator] + +- name: postgres-operator + releaseName: postgres-operator + chart: cozy-postgres-operator + namespace: cozy-postgres-operator + dependsOn: [cilium,kubeovn,cert-manager] + +- name: rabbitmq-operator + releaseName: rabbitmq-operator + chart: cozy-rabbitmq-operator + namespace: cozy-rabbitmq-operator + dependsOn: [cilium,kubeovn] + +- name: redis-operator + releaseName: redis-operator + chart: cozy-redis-operator + namespace: cozy-redis-operator + dependsOn: [cilium,kubeovn] + +- name: proxmox-csi-operator + releaseName: proxmox-csi-operator + chart: cozy-proxmox-csi-operator + namespace: cozy-proxmox + dependsOn: [cilium,kubeovn,cert-manager] + +- name: proxmox-csi-operator + releaseName: proxmox-csi-operator + chart: cozy-proxmox-csi-operator + namespace: cozy-proxmox + dependsOn: [cilium,kubeovn,cert-manager] + +- name: telepresence + releaseName: traffic-manager + chart: cozy-telepresence + namespace: cozy-telepresence + dependsOn: [cilium,kubeovn] + +- name: dashboard + releaseName: dashboard + chart: cozy-dashboard + namespace: cozy-dashboard + dependsOn: [cilium,kubeovn] + {{- if .Capabilities.APIVersions.Has "source.toolkit.fluxcd.io/v1beta2" }} + {{- with (lookup "source.toolkit.fluxcd.io/v1beta2" "HelmRepository" "cozy-public" "").items }} + values: + kubeapps: + redis: + master: + podAnnotations: + {{- range $index, $repo := . }} + {{- with (($repo.status).artifact).revision }} + repository.cozystack.io/{{ $repo.metadata.name }}: {{ quote . }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + +- name: kamaji + releaseName: kamaji + chart: cozy-kamaji + namespace: cozy-kamaji + dependsOn: [cilium,kubeovn,cert-manager] + +- name: capi-operator + releaseName: capi-operator + chart: cozy-capi-operator + namespace: cozy-cluster-api + privileged: true + dependsOn: [cilium,kubeovn,cert-manager] + +- name: capi-providers + releaseName: capi-providers + chart: cozy-capi-providers + namespace: cozy-cluster-api + privileged: true + dependsOn: [cilium,kubeovn,capi-operator] \ No newline at end of file diff --git a/packages/extra/Makefile b/packages/extra/Makefile index e05e4985c..f2071677d 100644 --- a/packages/extra/Makefile +++ b/packages/extra/Makefile @@ -5,7 +5,7 @@ repo: cd .. && ../hack/package_chart.sh extra $(OUT) $(TMP) library fix-chartnames: - find . -maxdepth 2 -name Chart.yaml | awk -F/ '{print $$2}' | while read i; do sed -i "s/^name: .*/name: $$i/" "$$i/Chart.yaml"; done + find . -maxdepth 2 -name Chart.yaml | awk -F/ '{print $$2}' | while read i; do sed "s/^name: .*/name: $$i/" "$$i/Chart.yaml"; done gen-versions-map: fix-chartnames ../../hack/gen_versions_map.sh diff --git a/packages/system/capi-providers/README.md b/packages/system/capi-providers/README.md new file mode 100644 index 000000000..a841a04a1 --- /dev/null +++ b/packages/system/capi-providers/README.md @@ -0,0 +1,166 @@ +# Cluster API Providers + +This directory contains configurations for various Cluster API providers. + +## Proxmox Integration + +### Requirements +- Kubernetes cluster +- Proxmox VE server +- Access to Proxmox API +- Installed Cluster API + +### Configuration + +1. Enable Proxmox provider in your configuration: + +```yaml +providers: + proxmox: true +``` + +2. Ensure you have the necessary secrets for Proxmox access: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: proxmox-credentials + namespace: default +type: Opaque +stringData: + username: your-proxmox-username + password: your-proxmox-password + url: https://your-proxmox-server:8006/api2/json +``` + +### Usage + +1. Create a cluster: + +```yaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: my-proxmox-cluster +spec: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: ProxmoxCluster + name: my-proxmox-cluster +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: ProxmoxCluster +metadata: + name: my-proxmox-cluster +spec: + server: your-proxmox-server + insecure: false + controlPlaneEndpoint: + host: your-load-balancer-host + port: 6443 +``` + +2. Create a machine: + +```yaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Machine +metadata: + name: my-proxmox-machine +spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfig + name: my-proxmox-machine + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: ProxmoxMachine + name: my-proxmox-machine +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: ProxmoxMachine +metadata: + name: my-proxmox-machine +spec: + nodeName: your-proxmox-node + template: ubuntu-2004-template + cores: 2 + memory: 4096 + diskSize: 20 +``` + +### Debugging + +The project includes two scripts for debugging Proxmox integration: + +1. `create-proxmox-cluster.sh`: + - Creates a cluster with built-in debugging capabilities + - Checks provider status + - Monitors cluster creation progress + - Provides detailed logs on failure + +2. `debug-proxmox-cluster.sh`: + - Interactive debugging menu + - Color-coded output + - Comprehensive checks for: + - Cluster API providers status + - Cluster resources + - Provider logs + - Machine logs + - Proxmox connection + - Cluster events + +#### Debugging Commands + +1. Check provider status: +```bash +kubectl get pods -n capi-proxmox-system +``` + +2. Check provider logs: +```bash +kubectl logs -n capi-proxmox-system -l control-plane=controller-manager +``` + +3. Check machine status: +```bash +kubectl get machines -A +``` + +4. Check events: +```bash +kubectl get events --field-selector involvedObject.kind=ProxmoxMachine +``` + +5. Check Proxmox connection: +```bash +kubectl get secret proxmox-credentials +``` + +#### Common Issues and Solutions + +1. Provider Pod Issues: + - Check if the pod is running: `kubectl get pods -n capi-proxmox-system` + - Check pod logs: `kubectl logs -n capi-proxmox-system ` + - Verify Proxmox credentials in the secret + +2. Machine Creation Issues: + - Check machine status: `kubectl get machines -A` + - Check Proxmox machine status: `kubectl get proxmoxmachines -A` + - Verify VM template exists in Proxmox + +3. Connection Issues: + - Verify Proxmox URL is accessible + - Check credentials in the secret + - Ensure Proxmox API is enabled and accessible + +### Known Limitations +- Only Linux systems are supported +- A pre-created VM template is required +- Only qemu/kvm virtual machines are supported + +### Additional Information +- [Official cluster-api-provider-proxmox documentation](https://github.com/ionos-cloud/cluster-api-provider-proxmox) +- [Cluster API documentation](https://cluster-api.sigs.k8s.io/) \ No newline at end of file diff --git a/packages/system/capi-providers/docs/talos-proxmox-setup.md b/packages/system/capi-providers/docs/talos-proxmox-setup.md new file mode 100644 index 000000000..5e6c3abc6 --- /dev/null +++ b/packages/system/capi-providers/docs/talos-proxmox-setup.md @@ -0,0 +1,203 @@ +# Setting up Talos Linux on Proxmox for CozyStack + +This guide describes how to set up Talos Linux on Proxmox and run CozyStack. + +## Prerequisites + +- Proxmox VE server +- Access to Proxmox API +- `talosctl` CLI tool installed +- `kubectl` CLI tool installed +- `clusterctl` CLI tool installed + +## Step 1: Download Talos Linux Image + +1. Download the latest Talos Linux image for Proxmox: +```bash +curl -LO https://github.com/siderolabs/talos/releases/latest/download/metal-amd64.raw.xz +``` + +2. Decompress the image: +```bash +xz -d metal-amd64.raw.xz +``` + +## Step 2: Create VM Template in Proxmox + +1. Create a new VM in Proxmox: + - Name: `talos-template` + - Memory: 4GB minimum + - CPU: 2 cores minimum + - Disk: 20GB minimum + - Network: Bridge mode + +2. Import the Talos image: +```bash +qm importdisk 100 metal-amd64.raw local-lvm +``` + +3. Attach the disk to the VM: + - Go to VM hardware + - Add the imported disk + - Set as boot disk + +4. Configure VM settings: + - Enable UEFI boot + - Set machine type to `q35` + - Add TPM device + - Enable nested virtualization + +5. Create a template from the VM: + - Right-click on the VM + - Select "Convert to template" + +## Step 3: Configure Talos + +1. Create Talos configuration: +```yaml +# talos-config.yaml +version: v1alpha1 +machine: + type: controlplane + certSANs: + - 192.168.1.10 # Replace with your VM IP + kubelet: + extraArgs: + cloud-provider: external + network: + hostname: talos-control-plane + interfaces: + - interface: eth0 + dhcp: true +cluster: + controlPlane: + endpoint: https://192.168.1.10:6443 # Replace with your VM IP + network: + dnsDomain: cluster.local + podSubnets: + - 10.244.0.0/16 + serviceSubnets: + - 10.96.0.0/12 +``` + +2. Generate Talos configuration: +```bash +talosctl gen config \ + --with-secrets secrets.yaml \ + --with-docs=false \ + --with-examples=false \ + --config-patch-control-plane talos-config.yaml \ + talos-cluster https://192.168.1.10:6443 +``` + +## Step 4: Create VM from Template + +1. Clone the template to create a new VM: +```bash +qm clone 100 101 --name talos-control-plane +``` + +2. Start the VM: +```bash +qm start 101 +``` + +3. Wait for Talos to boot and get its IP address: +```bash +qm status 101 +``` + +## Step 5: Apply Talos Configuration + +1. Apply the configuration: +```bash +talosctl apply-config \ + --insecure \ + --nodes 192.168.1.10 \ + --file controlplane.yaml +``` + +2. Wait for Talos to be ready: +```bash +talosctl health --nodes 192.168.1.10 +``` + +3. Get the kubeconfig: +```bash +talosctl kubeconfig --nodes 192.168.1.10 +``` + +## Step 6: Deploy CozyStack + +1. Set up environment variables: +```bash +export KUBECONFIG=./kubeconfig +export COZYSTACK_VERSION=latest +``` + +2. Deploy CozyStack: +```bash +# Clone CozyStack repository +git clone https://github.com/your-org/cozystack.git +cd cozystack + +# Deploy using Helm +helm install cozystack ./packages/system/charts/cozystack \ + --namespace cozystack \ + --create-namespace \ + --version ${COZYSTACK_VERSION} +``` + +3. Verify deployment: +```bash +kubectl get pods -n cozystack +``` + +## Troubleshooting + +### Common Issues + +1. VM won't boot: + - Check UEFI boot settings + - Verify disk attachment + - Check network configuration + +2. Talos configuration fails: + - Verify IP addresses + - Check network connectivity + - Review Talos logs: `talosctl logs` + +3. CozyStack deployment issues: + - Check Kubernetes cluster status + - Verify resource requirements + - Review pod logs + +### Useful Commands + +1. Talos management: +```bash +# View Talos logs +talosctl logs + +# Check Talos health +talosctl health + +# Get Talos version +talosctl version +``` + +2. Kubernetes management: +```bash +# Check cluster status +kubectl get nodes +kubectl get pods -A + +# View CozyStack logs +kubectl logs -n cozystack -l app.kubernetes.io/name=cozystack +``` + +## Additional Resources + +- [Talos Linux Documentation](https://www.talos.dev/docs/latest/) +- [Proxmox VE Documentation](https://pve.proxmox.com/wiki/Main_Page) +- [CozyStack Documentation](https://github.com/your-org/cozystack/docs) \ No newline at end of file diff --git a/packages/system/capi-providers/scripts/create-proxmox-cluster.sh b/packages/system/capi-providers/scripts/create-proxmox-cluster.sh new file mode 100755 index 000000000..99b0210fb --- /dev/null +++ b/packages/system/capi-providers/scripts/create-proxmox-cluster.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +# Debug functions +debug_log() { + echo "[DEBUG] $(date '+%Y-%m-%d %H:%M:%S') - $1" +} + +check_provider_status() { + debug_log "Checking Cluster API provider status..." + kubectl get pods -n capi-system + kubectl get pods -n capi-kubeadm-bootstrap-system + kubectl get pods -n capi-kubeadm-control-plane-system + kubectl get pods -n capi-proxmox-system +} + +check_cluster_status() { + debug_log "Checking cluster status..." + kubectl get clusters -A + kubectl get machines -A + kubectl get proxmoxclusters -A + kubectl get proxmoxmachines -A +} + +check_provider_logs() { + debug_log "Checking provider logs..." + for namespace in capi-system capi-kubeadm-bootstrap-system capi-kubeadm-control-plane-system capi-proxmox-system; do + echo "=== Logs from $namespace ===" + kubectl logs -n $namespace -l control-plane=controller-manager --tail=100 + done +} + +check_machine_logs() { + debug_log "Checking machine logs..." + kubectl get machines -A -o wide + for machine in $(kubectl get machines -A -o jsonpath='{.items[*].metadata.name}'); do + echo "=== Logs for machine $machine ===" + kubectl logs -n default -l cluster.x-k8s.io/machine-name=$machine --tail=100 + done +} + +# Check if required environment variables are set +required_vars=( + "PROXMOX_USERNAME" + "PROXMOX_PASSWORD" + "PROXMOX_URL" + "PROXMOX_SERVER" + "PROXMOX_NODE" + "VM_TEMPLATE" + "KUBERNETES_VERSION" + "LOAD_BALANCER_HOST" +) + +for var in "${required_vars[@]}"; do + if [ -z "${!var}" ]; then + echo "Error: Required environment variable $var is not set" + exit 1 + fi +done + +# Create a temporary directory for processed manifests +TEMP_DIR=$(mktemp -d) +trap 'rm -rf "$TEMP_DIR"' EXIT + +# Process the manifests with environment variables +debug_log "Processing manifests..." +envsubst < templates/proxmox-examples.yaml > "$TEMP_DIR/processed-manifests.yaml" + +# Apply the manifests +debug_log "Applying Cluster API manifests..." +kubectl apply -f "$TEMP_DIR/processed-manifests.yaml" + +# Initial status check +debug_log "Performing initial status check..." +check_provider_status +check_cluster_status + +echo "Waiting for cluster to be ready..." +kubectl wait --for=condition=ready cluster/proxmox-cluster --timeout=300s || { + debug_log "Cluster failed to become ready. Checking logs..." + check_provider_logs + check_machine_logs + check_cluster_status + exit 1 +} + +debug_log "Cluster is ready. Final status check..." +check_provider_status +check_cluster_status + +echo "Cluster creation completed. You can monitor the progress with:" +echo "kubectl get clusters" +echo "kubectl get machines" +echo "kubectl get proxmoxclusters" +echo "kubectl get proxmoxmachines" + +# Add debug commands +echo -e "\nDebug commands:" +echo "1. Check provider logs:" +echo " kubectl logs -n capi-proxmox-system -l control-plane=controller-manager" +echo "2. Check machine status:" +echo " kubectl get machines -A -o wide" +echo "3. Check cluster status:" +echo " kubectl get clusters -A" +echo "4. Check Proxmox provider status:" +echo " kubectl get proxmoxclusters -A" +echo "5. Check Proxmox machines:" +echo " kubectl get proxmoxmachines -A" \ No newline at end of file diff --git a/packages/system/capi-providers/scripts/debug-proxmox-cluster.sh b/packages/system/capi-providers/scripts/debug-proxmox-cluster.sh new file mode 100755 index 000000000..02c3ec0b8 --- /dev/null +++ b/packages/system/capi-providers/scripts/debug-proxmox-cluster.sh @@ -0,0 +1,128 @@ +#!/bin/bash + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +# Debug functions +debug_log() { + echo -e "${YELLOW}[DEBUG] $(date '+%Y-%m-%d %H:%M:%S') - $1${NC}" +} + +error_log() { + echo -e "${RED}[ERROR] $(date '+%Y-%m-%d %H:%M:%S') - $1${NC}" +} + +success_log() { + echo -e "${GREEN}[SUCCESS] $(date '+%Y-%m-%d %H:%M:%S') - $1${NC}" +} + +# Check Cluster API providers +check_providers() { + debug_log "Checking Cluster API providers..." + echo "=== Core Provider ===" + kubectl get pods -n capi-system -l control-plane=controller-manager + + echo -e "\n=== Bootstrap Provider ===" + kubectl get pods -n capi-kubeadm-bootstrap-system -l control-plane=controller-manager + + echo -e "\n=== Control Plane Provider ===" + kubectl get pods -n capi-kubeadm-control-plane-system -l control-plane=controller-manager + + echo -e "\n=== Proxmox Provider ===" + kubectl get pods -n capi-proxmox-system -l control-plane=controller-manager +} + +# Check cluster resources +check_resources() { + debug_log "Checking cluster resources..." + echo "=== Clusters ===" + kubectl get clusters -A -o wide + + echo -e "\n=== Machines ===" + kubectl get machines -A -o wide + + echo -e "\n=== Proxmox Clusters ===" + kubectl get proxmoxclusters -A -o wide + + echo -e "\n=== Proxmox Machines ===" + kubectl get proxmoxmachines -A -o wide +} + +# Check provider logs +check_provider_logs() { + debug_log "Checking provider logs..." + for namespace in capi-system capi-kubeadm-bootstrap-system capi-kubeadm-control-plane-system capi-proxmox-system; do + echo "=== Logs from $namespace ===" + kubectl logs -n $namespace -l control-plane=controller-manager --tail=100 + done +} + +# Check machine logs +check_machine_logs() { + debug_log "Checking machine logs..." + for machine in $(kubectl get machines -A -o jsonpath='{.items[*].metadata.name}'); do + echo "=== Logs for machine $machine ===" + kubectl logs -n default -l cluster.x-k8s.io/machine-name=$machine --tail=100 + done +} + +# Check Proxmox connection +check_proxmox_connection() { + debug_log "Checking Proxmox connection..." + kubectl get secret proxmox-credentials -o jsonpath='{.data.url}' | base64 -d + echo -e "\nChecking Proxmox provider pods..." + kubectl get pods -n capi-proxmox-system -o wide +} + +# Check events +check_events() { + debug_log "Checking events..." + echo "=== Cluster Events ===" + kubectl get events --field-selector involvedObject.kind=Cluster + + echo -e "\n=== Machine Events ===" + kubectl get events --field-selector involvedObject.kind=Machine + + echo -e "\n=== ProxmoxCluster Events ===" + kubectl get events --field-selector involvedObject.kind=ProxmoxCluster + + echo -e "\n=== ProxmoxMachine Events ===" + kubectl get events --field-selector involvedObject.kind=ProxmoxMachine +} + +# Main menu +while true; do + echo -e "\n${YELLOW}Proxmox Cluster API Debug Menu${NC}" + echo "1. Check Cluster API providers" + echo "2. Check cluster resources" + echo "3. Check provider logs" + echo "4. Check machine logs" + echo "5. Check Proxmox connection" + echo "6. Check events" + echo "7. Run all checks" + echo "8. Exit" + + read -p "Select an option (1-8): " option + + case $option in + 1) check_providers ;; + 2) check_resources ;; + 3) check_provider_logs ;; + 4) check_machine_logs ;; + 5) check_proxmox_connection ;; + 6) check_events ;; + 7) + check_providers + check_resources + check_provider_logs + check_machine_logs + check_proxmox_connection + check_events + ;; + 8) exit 0 ;; + *) echo "Invalid option" ;; + esac +done \ No newline at end of file diff --git a/packages/system/capi-providers/templates/proxmox-examples.yaml b/packages/system/capi-providers/templates/proxmox-examples.yaml new file mode 100644 index 000000000..b206b5b7f --- /dev/null +++ b/packages/system/capi-providers/templates/proxmox-examples.yaml @@ -0,0 +1,121 @@ +--- +# 1. Create a secret for Proxmox credentials +apiVersion: v1 +kind: Secret +metadata: + name: proxmox-credentials + namespace: default +type: Opaque +stringData: + username: ${PROXMOX_USERNAME} + password: ${PROXMOX_PASSWORD} + url: ${PROXMOX_URL} +--- +# 2. Create a cluster +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: proxmox-cluster +spec: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: ProxmoxCluster + name: proxmox-cluster +--- +# 3. Define Proxmox cluster infrastructure +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: ProxmoxCluster +metadata: + name: proxmox-cluster +spec: + server: ${PROXMOX_SERVER} + insecure: false + controlPlaneEndpoint: + host: ${LOAD_BALANCER_HOST} + port: 6443 +--- +# 4. Create a control plane machine +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Machine +metadata: + name: proxmox-control-plane + labels: + cluster.x-k8s.io/control-plane: "true" +spec: + version: ${KUBERNETES_VERSION} + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfig + name: proxmox-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: ProxmoxMachine + name: proxmox-control-plane +--- +# 5. Define Proxmox machine infrastructure for control plane +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: ProxmoxMachine +metadata: + name: proxmox-control-plane +spec: + nodeName: ${PROXMOX_NODE} + template: ${VM_TEMPLATE} + cores: 2 + memory: 4096 + diskSize: 20 +--- +# 6. Create a worker machine +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Machine +metadata: + name: proxmox-worker +spec: + version: ${KUBERNETES_VERSION} + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfig + name: proxmox-worker + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: ProxmoxMachine + name: proxmox-worker +--- +# 7. Define Proxmox machine infrastructure for worker +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: ProxmoxMachine +metadata: + name: proxmox-worker +spec: + nodeName: ${PROXMOX_NODE} + template: ${VM_TEMPLATE} + cores: 2 + memory: 4096 + diskSize: 20 +--- +# 8. Define Kubeadm configuration for control plane +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfig +metadata: + name: proxmox-control-plane +spec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: external + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external +--- +# 9. Define Kubeadm configuration for worker +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfig +metadata: + name: proxmox-worker +spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external \ No newline at end of file diff --git a/packages/system/capi-providers/values.yaml b/packages/system/capi-providers/values.yaml new file mode 100644 index 000000000..57dc83a11 --- /dev/null +++ b/packages/system/capi-providers/values.yaml @@ -0,0 +1,4 @@ +providers: + kubevirt: false + proxmox: true + hetzner: false \ No newline at end of file diff --git a/packages/system/proxmox-csi-node/Chart.yaml b/packages/system/proxmox-csi-node/Chart.yaml new file mode 100644 index 000000000..2a63eb08f --- /dev/null +++ b/packages/system/proxmox-csi-node/Chart.yaml @@ -0,0 +1,2 @@ +name: cozy-proxmox-csi-node +version: 0.1.0 diff --git a/packages/system/proxmox-csi-node/templates/deploy.yaml b/packages/system/proxmox-csi-node/templates/deploy.yaml new file mode 100644 index 000000000..52b8e1300 --- /dev/null +++ b/packages/system/proxmox-csi-node/templates/deploy.yaml @@ -0,0 +1,278 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: csi.kubevirt.io +spec: + attachRequired: true + podInfoOnMount: true + fsGroupPolicy: ReadWriteOnceWithFSType +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: proxmox-csi-controller-sa + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: proxmox-csi-controller-cr +rules: + - apiGroups: [''] + resources: ['persistentvolumes'] + verbs: ['create', 'delete', 'get', 'list', 'watch', 'update', 'patch'] + - apiGroups: [''] + resources: ['secrets'] + verbs: ['get', 'list'] + - apiGroups: [''] + resources: ['persistentvolumeclaims'] + verbs: ['get', 'list', 'watch', 'update'] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: [''] + resources: ['nodes'] + verbs: ['get', 'list', 'watch'] + - apiGroups: ['storage.k8s.io'] + resources: ['volumeattachments'] + verbs: ['get', 'list', 'watch', 'update', 'patch'] + - apiGroups: ['storage.k8s.io'] + resources: ['storageclasses'] + verbs: ['get', 'list', 'watch'] + - apiGroups: ['csi.storage.k8s.io'] + resources: ['csidrivers'] + verbs: ['get', 'list', 'watch', 'update', 'create'] + - apiGroups: [''] + resources: ['events'] + verbs: ['list', 'watch', 'create', 'update', 'patch'] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "volumeattachments/status" ] + verbs: [ "get", "list", "watch", "update", "patch" ] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + verbs: ["use"] + resourceNames: ["privileged"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: proxmox-csi-controller-binding +subjects: + - kind: ServiceAccount + name: proxmox-csi-controller-sa + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: proxmox-csi-controller-cr + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: proxmox-csi-node-sa + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: proxmox-csi-node-cr +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + verbs: ["use"] + resourceNames: ["privileged"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: proxmox-csi-node-binding +subjects: + - kind: ServiceAccount + name: proxmox-csi-node-sa + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: proxmox-csi-node-cr + apiGroup: rbac.authorization.k8s.io +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: proxmox-csi-node + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: + app: {{ .Release.Namespace }} + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: {{ .Release.Namespace }} + spec: + serviceAccount: proxmox-csi-node-sa + priorityClassName: system-node-critical + tolerations: + - operator: Exists + containers: + - name: csi-driver + securityContext: + privileged: true + allowPrivilegeEscalation: true + imagePullPolicy: Always + image: ghcr.io/kvaps/test:proxmox-csi-driver + args: + - "--endpoint=unix:/csi/csi.sock" + - "--node-name=$(KUBE_NODE_NAME)" + - "--v=5" + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + - name: device-dir + mountPath: /dev + - name: udev + mountPath: /run/udev + ports: + - name: healthz + containerPort: 10300 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + resources: + requests: + memory: 50Mi + cpu: 10m + - name: csi-node-driver-registrar + image: quay.io/openshift/origin-csi-node-driver-registrar:latest + args: + - "--csi-address=$(ADDRESS)" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + - "--v=5" + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/csi.kubevirt.io-reg.sock /csi/csi.sock"] + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/csi.proxmox.io/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: + requests: + memory: 20Mi + cpu: 5m + - name: csi-liveness-probe + image: quay.io/openshift/origin-csi-livenessprobe:latest + args: + - "--csi-address=/csi/csi.sock" + - "--probe-timeout=3s" + - "--health-port=10300" + volumeMounts: + - name: plugin-dir + mountPath: /csi + resources: + requests: + memory: 20Mi + cpu: 5m + volumes: + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi.proxmox.io/ + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: device-dir + hostPath: + path: /dev + type: Directory + - name: udev + hostPath: + path: /run/udev +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: driver-config + namespace: {{ .Release.Namespace }} +data: + infraClusterNamespace: + infraClusterLabels: csi-driver/cluster=tenant +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: proxmox + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: csi.proxmox.io +parameters: + infraStorageClassName: replicated + bus: scsi diff --git a/packages/system/proxmox-csi/Chart.yaml b/packages/system/proxmox-csi/Chart.yaml new file mode 100644 index 000000000..0aa338005 --- /dev/null +++ b/packages/system/proxmox-csi/Chart.yaml @@ -0,0 +1,2 @@ +name: app +version: 0.0.0 diff --git a/packages/system/proxmox-csi/Makefile b/packages/system/proxmox-csi/Makefile new file mode 100644 index 000000000..357a739e1 --- /dev/null +++ b/packages/system/proxmox-csi/Makefile @@ -0,0 +1,13 @@ +include ../../hack/app-helm.mk + +update: + rm -rf charts + tag=$$(git ls-remote --tags --sort="v:refname" https://github.com/sergelogvinov/proxmox-cloud-controller-manager | awk -F'[/^]' 'END{print $$3}') && \ + curl -sSL https://github.com/sergelogvinov/proxmox-cloud-controller-manager/archive/refs/tags/$${tag}.tar.gz | \ + tar xzvf - --strip 1 proxmox-cloud-controller-manager-$${tag#*v}/charts + sed -i 's/^ namespace: .*/ namespace: kube-system/' charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml + tag=$$(git ls-remote --tags --sort="v:refname" https://github.com/sergelogvinov/proxmox-csi-plugin | awk -F'[/^]' 'END{print $$3}') && \ + curl -sSL https://github.com/sergelogvinov/proxmox-csi-plugin/archive/refs/tags/$${tag}.tar.gz | \ + tar xzvf - --strip 1 proxmox-csi-plugin-$${tag#*v}/charts + rm -f charts/proxmox-csi-plugin/templates/namespace.yaml + patch -p 3 < patches/namespace.patch diff --git a/packages/system/proxmox-csi/README.md b/packages/system/proxmox-csi/README.md new file mode 100644 index 000000000..d95c5324f --- /dev/null +++ b/packages/system/proxmox-csi/README.md @@ -0,0 +1,6 @@ +# Proxmox CSI Plugin + +Plugin that provides CSI interface for Proxmox + +- GitHub: https://github.com/sergelogvinov/proxmox-csi-plugin +- Telegram: https://t.me/ru_talos diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/.helmignore b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/Chart.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/Chart.yaml new file mode 100644 index 000000000..cb9ac997c --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: proxmox-cloud-controller-manager +description: Cloud Controller Manager plugin for Proxmox +type: application +home: https://github.com/sergelogvinov/proxmox-cloud-controller-manager +icon: https://raw.githubusercontent.com/sergelogvinov/proxmox-cloud-controller-manager/main/charts/proxmox-cloud-controller-manager/icon.png +sources: + - https://github.com/sergelogvinov/proxmox-cloud-controller-manager +keywords: + - ccm + - proxmox + - kubernetes +maintainers: + - name: sergelogvinov + url: https://github.com/sergelogvinov +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.3.0 +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: v0.7.0 diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/README.md b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/README.md new file mode 100644 index 000000000..64e1ef6e5 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/README.md @@ -0,0 +1,109 @@ +# proxmox-cloud-controller-manager + +![Version: 0.2.8](https://img.shields.io/badge/Version-0.2.8-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.5.1](https://img.shields.io/badge/AppVersion-v0.5.1-informational?style=flat-square) + +Cloud Controller Manager plugin for Proxmox + +The Cloud Controller Manager (CCM) is responsible for managing node resources in cloud-based Kubernetes environments. + +Key functions of the Cloud Controller Manager: +- `Node Management`: It manages nodes by initializing new nodes when they join the cluster (e.g., during scaling up) and removing nodes when they are no longer needed (e.g., during scaling down). +- `Cloud-Specific Operations`: The CCM ensures that the cloud provider's API is integrated into the Kubernetes cluster to control and automate tasks like load balancing, storage provisioning, and node lifecycle management. + +**Homepage:** + +## Maintainers + +| Name | Email | Url | +| ---- | ------ | --- | +| sergelogvinov | | | + +## Source Code + +* + +## Requirements + +You need to set `--cloud-provider=external` in the kubelet argument for all nodes in the cluster. + +## Proxmox permissions + +```shell +# Create role CCM +pveum role add CCM -privs "VM.Audit" +# Create user and grant permissions +pveum user add kubernetes@pve +pveum aclmod / -user kubernetes@pve -role CCM +pveum user token add kubernetes@pve ccm -privsep 0 +``` + +## Helm values example + +```yaml +# proxmox-ccm.yaml + +config: + clusters: + - url: https://cluster-api-1.exmple.com:8006/api2/json + insecure: false + token_id: "kubernetes@pve!csi" + token_secret: "key" + region: cluster-1 + +enabledControllers: + # Remove `cloud-node` if you use it with Talos CCM + - cloud-node + - cloud-node-lifecycle + +# Deploy CCM only on control-plane nodes +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists +tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule +``` + +Deploy chart: + +```shell +helm upgrade -i --namespace=kube-system -f proxmox-ccm.yaml \ + proxmox-cloud-controller-manager oci://ghcr.io/sergelogvinov/charts/proxmox-cloud-controller-manager +``` + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| replicaCount | int | `1` | | +| image.repository | string | `"ghcr.io/sergelogvinov/proxmox-cloud-controller-manager"` | Proxmox CCM image. | +| image.pullPolicy | string | `"IfNotPresent"` | Always or IfNotPresent | +| image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion. | +| imagePullSecrets | list | `[]` | | +| nameOverride | string | `""` | | +| fullnameOverride | string | `""` | | +| extraArgs | list | `[]` | Any extra arguments for talos-cloud-controller-manager | +| enabledControllers | list | `["cloud-node","cloud-node-lifecycle"]` | List of controllers should be enabled. Use '*' to enable all controllers. Support only `cloud-node,cloud-node-lifecycle` controllers. | +| logVerbosityLevel | int | `2` | Log verbosity level. See https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md for description of individual verbosity levels. | +| existingConfigSecret | string | `nil` | Proxmox cluster config stored in secrets. | +| existingConfigSecretKey | string | `"config.yaml"` | Proxmox cluster config stored in secrets key. | +| config | object | `{"clusters":[]}` | Proxmox cluster config. | +| serviceAccount | object | `{"annotations":{},"create":true,"name":""}` | Pods Service Account. ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ | +| priorityClassName | string | `"system-cluster-critical"` | CCM pods' priorityClassName. | +| initContainers | list | `[]` | Add additional init containers to the CCM pods. ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ | +| hostAliases | list | `[]` | hostAliases Deployment pod host aliases ref: https://kubernetes.io/docs/tasks/network/customize-hosts-file-for-pods/ | +| podAnnotations | object | `{}` | Annotations for data pods. ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ | +| podSecurityContext | object | `{"fsGroup":10258,"fsGroupChangePolicy":"OnRootMismatch","runAsGroup":10258,"runAsNonRoot":true,"runAsUser":10258}` | Pods Security Context. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod | +| securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"seccompProfile":{"type":"RuntimeDefault"}}` | Container Security Context. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod | +| resources | object | `{"requests":{"cpu":"10m","memory":"32Mi"}}` | Resource requests and limits. ref: https://kubernetes.io/docs/user-guide/compute-resources/ | +| useDaemonSet | bool | `false` | Deploy CCM in Daemonset mode. CCM will use hostNetwork. It allows to use CCM without CNI plugins. | +| updateStrategy | object | `{"rollingUpdate":{"maxUnavailable":1},"type":"RollingUpdate"}` | Deployment update strategy type. ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment | +| nodeSelector | object | `{}` | Node labels for data pods assignment. ref: https://kubernetes.io/docs/user-guide/node-selection/ | +| tolerations | list | `[{"effect":"NoSchedule","key":"node-role.kubernetes.io/control-plane","operator":"Exists"},{"effect":"NoSchedule","key":"node.cloudprovider.kubernetes.io/uninitialized","operator":"Exists"}]` | Tolerations for data pods assignment. ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ | +| affinity | object | `{}` | Affinity for data pods assignment. ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity | +| extraVolumes | list | `[]` | Additional volumes for Pods | +| extraVolumeMounts | list | `[]` | Additional volume mounts for Pods | diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/README.md.gotmpl b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/README.md.gotmpl new file mode 100644 index 000000000..0971b9b6b --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/README.md.gotmpl @@ -0,0 +1,76 @@ +{{ template "chart.header" . }} + +{{ template "chart.deprecationWarning" . }} + +{{ template "chart.badgesSection" . }} + +{{ template "chart.description" . }} + +The Cloud Controller Manager (CCM) is responsible for managing node resources in cloud-based Kubernetes environments. + +Key functions of the Cloud Controller Manager: +- `Node Management`: It manages nodes by initializing new nodes when they join the cluster (e.g., during scaling up) and removing nodes when they are no longer needed (e.g., during scaling down). +- `Cloud-Specific Operations`: The CCM ensures that the cloud provider's API is integrated into the Kubernetes cluster to control and automate tasks like load balancing, storage provisioning, and node lifecycle management. + +{{ template "chart.homepageLine" . }} + +{{ template "chart.maintainersSection" . }} + +{{ template "chart.sourcesSection" . }} + +{{ template "chart.requirementsSection" . }} + +## Requirements + +You need to set `--cloud-provider=external` in the kubelet argument for all nodes in the cluster. + +## Proxmox permissions + +```shell +# Create role CCM +pveum role add CCM -privs "VM.Audit" +# Create user and grant permissions +pveum user add kubernetes@pve +pveum aclmod / -user kubernetes@pve -role CCM +pveum user token add kubernetes@pve ccm -privsep 0 +``` + +## Helm values example + +```yaml +# proxmox-ccm.yaml + +config: + clusters: + - url: https://cluster-api-1.exmple.com:8006/api2/json + insecure: false + token_id: "kubernetes@pve!csi" + token_secret: "key" + region: cluster-1 + +enabledControllers: + # Remove `cloud-node` if you use it with Talos CCM + - cloud-node + - cloud-node-lifecycle + +# Deploy CCM only on control-plane nodes +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists +tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule +``` + +Deploy chart: + +```shell +helm upgrade -i --namespace=kube-system -f proxmox-ccm.yaml \ + proxmox-cloud-controller-manager oci://ghcr.io/sergelogvinov/charts/proxmox-cloud-controller-manager +``` + +{{ template "chart.valuesSection" . }} diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/ci/values.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/ci/values.yaml new file mode 100644 index 000000000..2d59a0343 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/ci/values.yaml @@ -0,0 +1,31 @@ +image: + repository: ghcr.io/sergelogvinov/proxmox-cloud-controller-manager + pullPolicy: Always + tag: edge + +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + +logVerbosityLevel: 4 + +enabledControllers: + - cloud-node + - cloud-node-lifecycle + +config: + clusters: + - url: https://cluster-api-1.exmple.com:8006/api2/json + insecure: false + token_id: "user!token-id" + token_secret: "secret" + region: cluster-1 + - url: https://cluster-api-2.exmple.com:8006/api2/json + insecure: false + token_id: "user!token-id" + token_secret: "secret" + region: cluster-2 diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/icon.png b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/icon.png new file mode 100644 index 000000000..36a30d2c7 Binary files /dev/null and b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/icon.png differ diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/NOTES.txt b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/NOTES.txt new file mode 100644 index 000000000..e69de29bb diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/_helpers.tpl b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/_helpers.tpl new file mode 100644 index 000000000..ee962923e --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/_helpers.tpl @@ -0,0 +1,69 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "proxmox-cloud-controller-manager.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "proxmox-cloud-controller-manager.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "proxmox-cloud-controller-manager.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "proxmox-cloud-controller-manager.labels" -}} +helm.sh/chart: {{ include "proxmox-cloud-controller-manager.chart" . }} +{{ include "proxmox-cloud-controller-manager.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "proxmox-cloud-controller-manager.selectorLabels" -}} +app.kubernetes.io/name: {{ include "proxmox-cloud-controller-manager.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "proxmox-cloud-controller-manager.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "proxmox-cloud-controller-manager.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Generate string of enabled controllers. Might have a trailing comma (,) which needs to be trimmed. +*/}} +{{- define "proxmox-cloud-controller-manager.enabledControllers" }} +{{- range .Values.enabledControllers -}}{{ . }},{{- end -}} +{{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/deployment.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/deployment.yaml new file mode 100644 index 000000000..d7defe7b7 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/deployment.yaml @@ -0,0 +1,147 @@ +apiVersion: apps/v1 +{{- if .Values.useDaemonSet }} +kind: DaemonSet +{{- else }} +kind: Deployment +{{- end }} +metadata: + name: {{ include "proxmox-cloud-controller-manager.fullname" . }} + labels: + {{- include "proxmox-cloud-controller-manager.labels" . | nindent 4 }} + namespace: {{ .Release.Namespace }} +spec: + {{- if not .Values.useDaemonSet }} + replicas: {{ .Values.replicaCount }} + strategy: + type: {{ .Values.updateStrategy.type }} + {{- else }} + updateStrategy: + type: {{ .Values.updateStrategy.type }} + {{- end }} + selector: + matchLabels: + {{- include "proxmox-cloud-controller-manager.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- if .Values.config }} + checksum/config: {{ toJson .Values.config | sha256sum }} + {{- end }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "proxmox-cloud-controller-manager.selectorLabels" . | nindent 8 }} + spec: + enableServiceLinks: false + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "proxmox-cloud-controller-manager.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- if .Values.useDaemonSet }} + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + {{- end }} + {{- with .Values.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + initContainers: {{- toYaml .Values.initContainers | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --v={{ .Values.logVerbosityLevel }} + - --cloud-provider=proxmox + - --cloud-config=/etc/proxmox/config.yaml + - --controllers={{- trimAll "," (include "proxmox-cloud-controller-manager.enabledControllers" . ) }} + - --leader-elect-resource-name=cloud-controller-manager-proxmox + - --use-service-account-credentials + - --secure-port=10258 + - --authorization-always-allow-paths=/healthz,/livez,/readyz,/metrics + {{- with .Values.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: metrics + containerPort: 10258 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: metrics + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 30 + timeoutSeconds: 5 + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: cloud-config + mountPath: /etc/proxmox + readOnly: true + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + affinity: + {{- with .Values.affinity }} + {{- toYaml . | nindent 8 }} + {{- else }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + {{- include "proxmox-cloud-controller-manager.selectorLabels" . | nindent 20 }} + topologyKey: topology.kubernetes.io/zone + weight: 1 + {{- end }} + tolerations: + {{- with .Values.tolerations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.useDaemonSet }} + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + {{- end }} + {{- if not .Values.useDaemonSet }} + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + {{- include "proxmox-cloud-controller-manager.selectorLabels" . | nindent 14 }} + {{- end }} + volumes: + {{- if .Values.existingConfigSecret }} + - name: cloud-config + secret: + secretName: {{ .Values.existingConfigSecret }} + items: + - key: {{ .Values.existingConfigSecretKey }} + path: config.yaml + defaultMode: 416 + {{- else }} + - name: cloud-config + secret: + secretName: {{ include "proxmox-cloud-controller-manager.fullname" . }} + defaultMode: 416 + {{- end }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/role.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/role.yaml new file mode 100644 index 000000000..b35bf2b7c --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/role.yaml @@ -0,0 +1,53 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:{{ include "proxmox-cloud-controller-manager.fullname" . }} + labels: + {{- include "proxmox-cloud-controller-manager.labels" . | nindent 4 }} +rules: +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get +- apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml new file mode 100644 index 000000000..32b065ec5 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml @@ -0,0 +1,26 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:{{ include "proxmox-cloud-controller-manager.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:{{ include "proxmox-cloud-controller-manager.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ include "proxmox-cloud-controller-manager.fullname" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: system:{{ include "proxmox-cloud-controller-manager.fullname" . }}:extension-apiserver-authentication-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: {{ include "proxmox-cloud-controller-manager.fullname" . }} + namespace: {{ .Release.Namespace }} diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/secrets.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/secrets.yaml new file mode 100644 index 000000000..ce60f6220 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/secrets.yaml @@ -0,0 +1,11 @@ +{{- if ne (len .Values.config.clusters) 0 }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "proxmox-cloud-controller-manager.fullname" . }} + labels: + {{- include "proxmox-cloud-controller-manager.labels" . | nindent 4 }} + namespace: {{ .Release.Namespace }} +data: + config.yaml: {{ toYaml .Values.config | b64enc | quote }} +{{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/serviceaccount.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/serviceaccount.yaml new file mode 100644 index 000000000..b9ecfcc25 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "proxmox-cloud-controller-manager.serviceAccountName" . }} + labels: + {{- include "proxmox-cloud-controller-manager.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.edge.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.edge.yaml new file mode 100644 index 000000000..36d35f709 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.edge.yaml @@ -0,0 +1,17 @@ +image: + pullPolicy: Always + tag: edge + +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + +logVerbosityLevel: 4 + +enabledControllers: + - cloud-node + - cloud-node-lifecycle diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.talos.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.talos.yaml new file mode 100644 index 000000000..38b2e84de --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.talos.yaml @@ -0,0 +1,12 @@ +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + +logVerbosityLevel: 4 + +enabledControllers: + - cloud-node-lifecycle diff --git a/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.yaml b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.yaml new file mode 100644 index 000000000..37d38de97 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-cloud-controller-manager/values.yaml @@ -0,0 +1,177 @@ +# Default values for proxmox-cloud-controller-manager. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + # -- Proxmox CCM image. + repository: ghcr.io/sergelogvinov/proxmox-cloud-controller-manager + # -- Always or IfNotPresent + pullPolicy: IfNotPresent + # -- Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +# -- Any extra arguments for talos-cloud-controller-manager +extraArgs: [] + # - --cluster-name=kubernetes + +# -- List of controllers should be enabled. +# Use '*' to enable all controllers. +# Support only `cloud-node,cloud-node-lifecycle` controllers. +enabledControllers: + - cloud-node + - cloud-node-lifecycle + # - route + # - service + +# -- Log verbosity level. See https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md +# for description of individual verbosity levels. +logVerbosityLevel: 2 + +# -- Proxmox cluster config stored in secrets. +existingConfigSecret: ~ +# -- Proxmox cluster config stored in secrets key. +existingConfigSecretKey: config.yaml + +# -- Proxmox cluster config. +config: + clusters: [] + # - url: https://cluster-api-1.exmple.com:8006/api2/json + # insecure: false + # token_id: "login!name" + # token_secret: "secret" + # region: cluster-1 + +# -- Pods Service Account. +# ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +# -- CCM pods' priorityClassName. +priorityClassName: system-cluster-critical + +# -- Add additional init containers to the CCM pods. +# ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ +initContainers: [] + # - name: loadbalancer + # restartPolicy: Always + # image: ghcr.io/sergelogvinov/haproxy:2.8.3-alpine3.18 + # imagePullPolicy: IfNotPresent + # env: + # - name: SVC + # value: "proxmox.domain.com" + # - name: PORT + # value: "8006" + # securityContext: + # runAsUser: 99 + # runAsGroup: 99 + # resources: + # limits: + # cpu: 50m + # memory: 64Mi + # requests: + # cpu: 50m + # memory: 32Mi + +# -- hostAliases Deployment pod host aliases +# ref: https://kubernetes.io/docs/tasks/network/customize-hosts-file-for-pods/ +hostAliases: [] + # - ip: 127.0.0.1 + # hostnames: + # - proxmox.domain.com + +# -- Annotations for data pods. +# ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +podAnnotations: {} + +# -- Pods Security Context. +# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod +podSecurityContext: + runAsNonRoot: true + runAsUser: 10258 + runAsGroup: 10258 + fsGroup: 10258 + fsGroupChangePolicy: "OnRootMismatch" + +# -- Container Security Context. +# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + +# -- Resource requests and limits. +# ref: https://kubernetes.io/docs/user-guide/compute-resources/ +resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + requests: + cpu: 10m + memory: 32Mi + +# -- Deploy CCM in Daemonset mode. +# CCM will use hostNetwork. +# It allows to use CCM without CNI plugins. +useDaemonSet: false + +# -- Deployment update strategy type. +# ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + +# -- Node labels for data pods assignment. +# ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + # node-role.kubernetes.io/control-plane: "" + +# -- Tolerations for data pods assignment. +# ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + +# -- Affinity for data pods assignment. +# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +affinity: {} +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchExpressions: +# - key: node-role.kubernetes.io/control-plane +# operator: Exists + +# -- Additional volumes for Pods +extraVolumes: [] +# - name: ca +# secret: +# secretName: my-ca +# -- Additional volume mounts for Pods +extraVolumeMounts: [] +# - mountPath: /etc/ssl/certs/ca-certificates.crt +# name: ca +# subPath: ca.crt diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/.helmignore b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/Chart.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/Chart.yaml new file mode 100644 index 000000000..d21db42c0 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/Chart.yaml @@ -0,0 +1,27 @@ +# yaml-language-server: $schema=https://json.schemastore.org/chart.json +apiVersion: v2 +name: proxmox-csi-plugin +description: Container Storage Interface plugin for Proxmox +type: application +home: https://github.com/sergelogvinov/proxmox-csi-plugin +icon: https://raw.githubusercontent.com/sergelogvinov/proxmox-csi-plugin/main/charts/proxmox-csi-plugin/icon.png +sources: + - https://github.com/sergelogvinov/proxmox-csi-plugin +keywords: + - csi + - storage + - block-storage + - volume + - proxmox +maintainers: + - name: sergelogvinov + url: https://github.com/sergelogvinov +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.3.0 +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: v0.11.0 diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/README.md b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/README.md new file mode 100644 index 000000000..fd67aab78 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/README.md @@ -0,0 +1,149 @@ +# proxmox-csi-plugin + +![Version: 0.2.13](https://img.shields.io/badge/Version-0.2.13-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.8.2](https://img.shields.io/badge/AppVersion-v0.8.2-informational?style=flat-square) + +Container Storage Interface plugin for Proxmox + +The Container Storage Interface (CSI) plugin is a specification designed to standardize the way container orchestration systems like Kubernetes, interact with different storage systems. The CSI plugin abstracts the underlying storage, enabling the seamless integration of different storage solutions (such as local block devices, file systems, or cloud-based storage) with containerized applications. + +This plugin allows Kubernetes to use `Proxmox VE` storage as a persistent storage solution for stateful applications. +Supported storage types: +- Directory +- LVM +- LVM-thin +- ZFS +- NFS +- Ceph + +**Homepage:** + +## Maintainers + +| Name | Email | Url | +| ---- | ------ | --- | +| sergelogvinov | | | + +## Source Code + +* + +## Proxmox permissions + +```shell +# Create role CSI +pveum role add CSI -privs "VM.Audit VM.Config.Disk Datastore.Allocate Datastore.AllocateSpace Datastore.Audit" +# Create user and grant permissions +pveum user add kubernetes-csi@pve +pveum aclmod / -user kubernetes-csi@pve -role CSI +pveum user token add kubernetes-csi@pve csi -privsep 0 +``` + +## Helm values example + +```yaml +# proxmox-csi.yaml + +config: + clusters: + - url: https://cluster-api-1.exmple.com:8006/api2/json + insecure: false + token_id: "kubernetes-csi@pve!csi" + token_secret: "key" + region: cluster-1 + +# Deploy Node CSI driver only on proxmox nodes +node: + nodeSelector: + # It will work only with Talos CCM, remove it overwise + node.cloudprovider.kubernetes.io/platform: nocloud + tolerations: + - operator: Exists + +# Deploy CSI controller only on control-plane nodes +nodeSelector: + node-role.kubernetes.io/control-plane: "" +tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + +# Define storage classes +# See https://pve.proxmox.com/wiki/Storage +storageClass: + - name: proxmox-data-xfs + storage: data + reclaimPolicy: Delete + fstype: xfs + - name: proxmox-data + storage: data + reclaimPolicy: Delete + fstype: ext4 + cache: writethrough +``` + +## Deploy + +```shell +# Prepare namespace +kubectl create ns csi-proxmox +kubectl label ns csi-proxmox pod-security.kubernetes.io/enforce=privileged +# Install Proxmox CSI plugin +helm upgrade -i --namespace=csi-proxmox -f proxmox-csi.yaml \ + proxmox-csi-plugin oci://ghcr.io/sergelogvinov/charts/proxmox-csi-plugin +``` + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| replicaCount | int | `1` | | +| imagePullSecrets | list | `[]` | | +| nameOverride | string | `""` | | +| fullnameOverride | string | `""` | | +| createNamespace | bool | `false` | Create namespace. Very useful when using helm template. | +| priorityClassName | string | `"system-cluster-critical"` | Controller pods priorityClassName. | +| serviceAccount | object | `{"annotations":{},"create":true,"name":""}` | Pods Service Account. ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ | +| provisionerName | string | `"csi.proxmox.sinextra.dev"` | CSI Driver provisioner name. Currently, cannot be customized. | +| clusterID | string | `"kubernetes"` | Cluster name. Currently, cannot be customized. | +| logVerbosityLevel | int | `5` | Log verbosity level. See https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md for description of individual verbosity levels. | +| timeout | string | `"3m"` | Connection timeout between sidecars. | +| existingConfigSecret | string | `nil` | Proxmox cluster config stored in secrets. | +| existingConfigSecretKey | string | `"config.yaml"` | Proxmox cluster config stored in secrets key. | +| configFile | string | `"/etc/proxmox/config.yaml"` | Proxmox cluster config path. | +| config | object | `{"clusters":[]}` | Proxmox cluster config. | +| storageClass | list | `[]` | Storage class definition. | +| controller.podAnnotations | object | `{}` | Annotations for controller pod. ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ | +| controller.plugin.image | object | `{"pullPolicy":"IfNotPresent","repository":"ghcr.io/sergelogvinov/proxmox-csi-controller","tag":""}` | Controller CSI Driver. | +| controller.plugin.resources | object | `{"requests":{"cpu":"10m","memory":"16Mi"}}` | Controller resource requests and limits. ref: https://kubernetes.io/docs/user-guide/compute-resources/ | +| controller.attacher.image | object | `{"pullPolicy":"IfNotPresent","repository":"registry.k8s.io/sig-storage/csi-attacher","tag":"v4.4.4"}` | CSI Attacher. | +| controller.attacher.resources | object | `{"requests":{"cpu":"10m","memory":"16Mi"}}` | Attacher resource requests and limits. ref: https://kubernetes.io/docs/user-guide/compute-resources/ | +| controller.provisioner.image | object | `{"pullPolicy":"IfNotPresent","repository":"registry.k8s.io/sig-storage/csi-provisioner","tag":"v3.6.4"}` | CSI Provisioner. | +| controller.provisioner.resources | object | `{"requests":{"cpu":"10m","memory":"16Mi"}}` | Provisioner resource requests and limits. ref: https://kubernetes.io/docs/user-guide/compute-resources/ | +| controller.resizer.image | object | `{"pullPolicy":"IfNotPresent","repository":"registry.k8s.io/sig-storage/csi-resizer","tag":"v1.9.4"}` | CSI Resizer. | +| controller.resizer.resources | object | `{"requests":{"cpu":"10m","memory":"16Mi"}}` | Resizer resource requests and limits. ref: https://kubernetes.io/docs/user-guide/compute-resources/ | +| node.plugin.image | object | `{"pullPolicy":"IfNotPresent","repository":"ghcr.io/sergelogvinov/proxmox-csi-node","tag":""}` | Node CSI Driver. | +| node.plugin.resources | object | `{}` | Node CSI Driver resource requests and limits. ref: https://kubernetes.io/docs/user-guide/compute-resources/ | +| node.driverRegistrar.image | object | `{"pullPolicy":"IfNotPresent","repository":"registry.k8s.io/sig-storage/csi-node-driver-registrar","tag":"v2.9.4"}` | Node CSI driver registrar. | +| node.driverRegistrar.resources | object | `{"requests":{"cpu":"10m","memory":"16Mi"}}` | Node registrar resource requests and limits. ref: https://kubernetes.io/docs/user-guide/compute-resources/ | +| node.kubeletDir | string | `"/var/lib/kubelet"` | Location of the /var/lib/kubelet directory as some k8s distribution differ from the standard. | +| node.nodeSelector | object | `{}` | Node labels for node-plugin assignment. ref: https://kubernetes.io/docs/user-guide/node-selection/ | +| node.tolerations | list | `[{"effect":"NoSchedule","key":"node.kubernetes.io/unschedulable","operator":"Exists"},{"effect":"NoSchedule","key":"node.kubernetes.io/disk-pressure","operator":"Exists"}]` | Tolerations for node-plugin assignment. ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ | +| livenessprobe.image | object | `{"pullPolicy":"IfNotPresent","repository":"registry.k8s.io/sig-storage/livenessprobe","tag":"v2.11.0"}` | Common livenessprobe sidecar. | +| livenessprobe.failureThreshold | int | `5` | Failure threshold for livenessProbe | +| livenessprobe.initialDelaySeconds | int | `10` | Initial delay seconds for livenessProbe | +| livenessprobe.timeoutSeconds | int | `10` | Timeout seconds for livenessProbe | +| livenessprobe.periodSeconds | int | `60` | Period seconds for livenessProbe | +| livenessprobe.resources | object | `{"requests":{"cpu":"10m","memory":"16Mi"}}` | Liveness probe resource requests and limits. ref: https://kubernetes.io/docs/user-guide/compute-resources/ | +| initContainers | list | `[]` | Add additional init containers for the CSI controller pods. ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ | +| hostAliases | list | `[]` | hostAliases Deployment pod host aliases ref: https://kubernetes.io/docs/tasks/network/customize-hosts-file-for-pods/ | +| podAnnotations | object | `{}` | Annotations for controller pod. ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ | +| podSecurityContext | object | `{"fsGroup":65532,"fsGroupChangePolicy":"OnRootMismatch","runAsGroup":65532,"runAsNonRoot":true,"runAsUser":65532}` | Controller Security Context. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod | +| securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"seccompProfile":{"type":"RuntimeDefault"}}` | Controller Container Security Context. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod | +| updateStrategy | object | `{"rollingUpdate":{"maxUnavailable":1},"type":"RollingUpdate"}` | Controller deployment update strategy type. ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment | +| metrics | object | `{"enabled":false,"port":8080,"type":"annotation"}` | Prometheus metrics | +| metrics.enabled | bool | `false` | Enable Prometheus metrics. | +| metrics.port | int | `8080` | Prometheus metrics port. | +| nodeSelector | object | `{}` | Node labels for controller assignment. ref: https://kubernetes.io/docs/user-guide/node-selection/ | +| tolerations | list | `[]` | Tolerations for controller assignment. ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ | +| affinity | object | `{}` | Affinity for controller assignment. ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity | +| extraVolumes | list | `[]` | Additional volumes for Pods | +| extraVolumeMounts | list | `[]` | | diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/README.md.gotmpl b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/README.md.gotmpl new file mode 100644 index 000000000..5c3e60c08 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/README.md.gotmpl @@ -0,0 +1,92 @@ +{{ template "chart.header" . }} + +{{ template "chart.deprecationWarning" . }} + +{{ template "chart.badgesSection" . }} + +{{ template "chart.description" . }} + +The Container Storage Interface (CSI) plugin is a specification designed to standardize the way container orchestration systems like Kubernetes, interact with different storage systems. The CSI plugin abstracts the underlying storage, enabling the seamless integration of different storage solutions (such as local block devices, file systems, or cloud-based storage) with containerized applications. + +This plugin allows Kubernetes to use `Proxmox VE` storage as a persistent storage solution for stateful applications. +Supported storage types: +- Directory +- LVM +- LVM-thin +- ZFS +- NFS +- Ceph + +{{ template "chart.homepageLine" . }} + +{{ template "chart.maintainersSection" . }} + +{{ template "chart.sourcesSection" . }} + +{{ template "chart.requirementsSection" . }} + +## Proxmox permissions + +```shell +# Create role CSI +pveum role add CSI -privs "VM.Audit VM.Config.Disk Datastore.Allocate Datastore.AllocateSpace Datastore.Audit" +# Create user and grant permissions +pveum user add kubernetes-csi@pve +pveum aclmod / -user kubernetes-csi@pve -role CSI +pveum user token add kubernetes-csi@pve csi -privsep 0 +``` + +## Helm values example + +```yaml +# proxmox-csi.yaml + +config: + clusters: + - url: https://cluster-api-1.exmple.com:8006/api2/json + insecure: false + token_id: "kubernetes-csi@pve!csi" + token_secret: "key" + region: cluster-1 + +# Deploy Node CSI driver only on proxmox nodes +node: + nodeSelector: + # It will work only with Talos CCM, remove it overwise + node.cloudprovider.kubernetes.io/platform: nocloud + tolerations: + - operator: Exists + +# Deploy CSI controller only on control-plane nodes +nodeSelector: + node-role.kubernetes.io/control-plane: "" +tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + +# Define storage classes +# See https://pve.proxmox.com/wiki/Storage +storageClass: + - name: proxmox-data-xfs + storage: data + reclaimPolicy: Delete + fstype: xfs + - name: proxmox-data + storage: data + reclaimPolicy: Delete + fstype: ext4 + cache: writethrough +``` + +## Deploy + +```shell +# Prepare namespace +kubectl create ns csi-proxmox +kubectl label ns csi-proxmox pod-security.kubernetes.io/enforce=privileged +# Install Proxmox CSI plugin +helm upgrade -i --namespace=csi-proxmox -f proxmox-csi.yaml \ + proxmox-csi-plugin oci://ghcr.io/sergelogvinov/charts/proxmox-csi-plugin +``` + +{{ template "chart.valuesSection" . }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/ci/values.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/ci/values.yaml new file mode 100644 index 000000000..7c8341402 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/ci/values.yaml @@ -0,0 +1,22 @@ + +node: + nodeSelector: + node.cloudprovider.kubernetes.io/platform: nocloud + tolerations: + - operator: Exists + +nodeSelector: + node-role.kubernetes.io/control-plane: "" +tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + +storageClass: + - name: proxmox-data-xfs + storage: data + reclaimPolicy: Delete + fstype: xfs + - name: proxmox-data + storage: data + reclaimPolicy: Delete + ssd: true diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/icon.png b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/icon.png new file mode 100644 index 000000000..abe9fdc67 Binary files /dev/null and b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/icon.png differ diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/NOTES.txt b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/NOTES.txt new file mode 100644 index 000000000..e69de29bb diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/_helpers.tpl b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/_helpers.tpl new file mode 100644 index 000000000..2a7f80273 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/_helpers.tpl @@ -0,0 +1,71 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "proxmox-csi-plugin.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "proxmox-csi-plugin.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "proxmox-csi-plugin.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "proxmox-csi-plugin.labels" -}} +helm.sh/chart: {{ include "proxmox-csi-plugin.chart" . }} +app.kubernetes.io/name: {{ include "proxmox-csi-plugin.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "proxmox-csi-plugin.selectorLabels" -}} +app.kubernetes.io/name: {{ include "proxmox-csi-plugin.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: controller +{{- end }} + +{{- define "proxmox-csi-plugin-node.selectorLabels" -}} +app.kubernetes.io/name: {{ include "proxmox-csi-plugin.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: node +{{- end }} + + +{{/* +Create the name of the service account to use +*/}} +{{- define "proxmox-csi-plugin.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "proxmox-csi-plugin.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/_storage.tpl b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/_storage.tpl new file mode 100644 index 000000000..ad7906c4b --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/_storage.tpl @@ -0,0 +1,13 @@ +{{/* +storageClass parameters uses to merge the default parameters with the user provided parameters. +*/}} +{{- define "storageClass.parameters" -}} +csi.storage.k8s.io/fstype: {{ default "ext4" .fstype }} +storage: {{ .storage | required "Proxmox Storage name must be provided." }} +{{- with .cache }} +cache: {{ . }} +{{- end }} +{{- if .ssd }} +ssd: "true" +{{- end }} +{{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-clusterrole.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-clusterrole.yaml new file mode 100644 index 000000000..6737e23c6 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-clusterrole.yaml @@ -0,0 +1,37 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }}-controller + namespace: {{ .Release.Namespace }} + labels: + {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "patch", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get","list", "watch", "create", "update", "patch"] + + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-deployment.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-deployment.yaml new file mode 100644 index 000000000..b17116f40 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-deployment.yaml @@ -0,0 +1,178 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }}-controller + namespace: {{ .Release.Namespace }} + labels: + {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + strategy: + type: {{ .Values.updateStrategy.type }} + rollingUpdate: + {{- toYaml .Values.updateStrategy.rollingUpdate | nindent 6 }} + selector: + matchLabels: + {{- include "proxmox-csi-plugin.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ toJson .Values.config | sha256sum }} + {{- with default .Values.podAnnotations .Values.controller.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if and .Values.metrics.enabled (eq .Values.metrics.type "annotation") }} + prometheus.io/scrape: "true" + prometheus.io/port: {{ .Values.metrics.port | quote }} + {{- end }} + labels: + {{- include "proxmox-csi-plugin.selectorLabels" . | nindent 8 }} + spec: + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + enableServiceLinks: false + serviceAccountName: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-controller + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + hostAliases: {{- toYaml .Values.hostAliases | nindent 8 }} + initContainers: {{- toYaml .Values.initContainers | nindent 8 }} + containers: + - name: {{ include "proxmox-csi-plugin.fullname" . }}-controller + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.controller.plugin.image.repository }}:{{ .Values.controller.plugin.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.controller.plugin.image.pullPolicy }} + args: + - "-v={{ .Values.logVerbosityLevel }}" + - "--csi-address=unix:///csi/csi.sock" + - "--cloud-config={{ .Values.configFile }}" + {{- if .Values.metrics.enabled }} + - "--metrics-address=:{{ .Values.metrics.port }}" + {{- end }} + ports: + {{- if .Values.metrics.enabled }} + - name: metrics + containerPort: {{ .Values.metrics.port }} + protocol: TCP + {{- end }} + resources: + {{- toYaml .Values.controller.plugin.resources | nindent 12 }} + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: cloud-config + mountPath: /etc/proxmox/ + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + - name: csi-attacher + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.controller.attacher.image.repository }}:{{ .Values.controller.attacher.image.tag }}" + imagePullPolicy: {{ .Values.controller.attacher.image.pullPolicy }} + args: + - "-v={{ .Values.logVerbosityLevel }}" + - "--csi-address=unix:///csi/csi.sock" + - "--timeout={{ .Values.timeout }}" + - "--leader-election" + - "--default-fstype=ext4" + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: {{ toYaml .Values.controller.attacher.resources | nindent 12 }} + - name: csi-provisioner + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.controller.provisioner.image.repository }}:{{ .Values.controller.provisioner.image.tag }}" + imagePullPolicy: {{ .Values.controller.provisioner.image.pullPolicy }} + args: + - "-v={{ .Values.logVerbosityLevel }}" + - "--csi-address=unix:///csi/csi.sock" + - "--timeout={{ .Values.timeout }}" + - "--leader-election" + - "--default-fstype=ext4" + - "--feature-gates=Topology=True" + - "--enable-capacity" + - "--capacity-ownerref-level=2" + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: {{ toYaml .Values.controller.provisioner.resources | nindent 12 }} + - name: csi-resizer + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.controller.resizer.image.repository }}:{{ .Values.controller.resizer.image.tag }}" + imagePullPolicy: {{ .Values.controller.resizer.image.pullPolicy }} + args: + - "-v={{ .Values.logVerbosityLevel }}" + - "--csi-address=unix:///csi/csi.sock" + - "--timeout={{ .Values.timeout }}" + - "--handle-volume-inuse-error=false" + - "--leader-election" + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: {{ toYaml .Values.controller.resizer.resources | nindent 12 }} + - name: liveness-probe + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.livenessprobe.image.repository }}:{{ .Values.livenessprobe.image.tag }}" + imagePullPolicy: {{ .Values.livenessprobe.image.pullPolicy }} + args: + - "-v={{ .Values.logVerbosityLevel }}" + - "--csi-address=unix:///csi/csi.sock" + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: {{ toYaml .Values.livenessprobe.resources | nindent 12 }} + volumes: + - name: socket-dir + emptyDir: {} + {{- if .Values.existingConfigSecret }} + - name: cloud-config + secret: + secretName: {{ .Values.existingConfigSecret }} + items: + - key: {{ .Values.existingConfigSecretKey }} + path: config.yaml + {{- else }} + - name: cloud-config + secret: + secretName: {{ include "proxmox-csi-plugin.fullname" . }} + {{- end }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + {{- include "proxmox-csi-plugin.selectorLabels" . | nindent 14 }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-role.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-role.yaml new file mode 100644 index 000000000..b69d223f9 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-role.yaml @@ -0,0 +1,21 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }}-controller + namespace: {{ .Release.Namespace }} + labels: + {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + + - apiGroups: ["storage.k8s.io"] + resources: ["csistoragecapacities"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get"] + - apiGroups: ["apps"] + resources: ["replicasets"] + verbs: ["get"] diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-rolebinding.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-rolebinding.yaml new file mode 100644 index 000000000..478e0f6f1 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/controller-rolebinding.yaml @@ -0,0 +1,26 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }}-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "proxmox-csi-plugin.fullname" . }}-controller +subjects: + - kind: ServiceAccount + name: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-controller + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }}-controller + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "proxmox-csi-plugin.fullname" . }}-controller +subjects: + - kind: ServiceAccount + name: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-controller + namespace: {{ .Release.Namespace }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/csidriver.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/csidriver.yaml new file mode 100644 index 000000000..5d0652e86 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/csidriver.yaml @@ -0,0 +1,10 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: {{ .Values.provisionerName }} +spec: + attachRequired: true + podInfoOnMount: true + storageCapacity: true + volumeLifecycleModes: + - Persistent diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/namespace.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/namespace.yaml new file mode 100644 index 000000000..8c4ba9c4a --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/namespace.yaml @@ -0,0 +1,10 @@ +{{- if and .Values.createNamespace (ne .Release.Namespace "kube-system") }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Release.Namespace }} + labels: + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: baseline + pod-security.kubernetes.io/warn: baseline +{{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-clusterrole.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-clusterrole.yaml new file mode 100644 index 000000000..02af491f5 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-clusterrole.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }}-node + namespace: {{ .Release.Namespace }} + labels: + {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-deployment.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-deployment.yaml new file mode 100644 index 000000000..0219a78f4 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-deployment.yaml @@ -0,0 +1,135 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }}-node + namespace: {{ .Release.Namespace }} + labels: + {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} +spec: + updateStrategy: + type: {{ .Values.updateStrategy.type }} + selector: + matchLabels: + {{- include "proxmox-csi-plugin-node.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "proxmox-csi-plugin-node.selectorLabels" . | nindent 8 }} + spec: + priorityClassName: system-node-critical + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + enableServiceLinks: false + serviceAccountName: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-node + securityContext: + runAsUser: 0 + runAsGroup: 0 + containers: + - name: {{ include "proxmox-csi-plugin.fullname" . }}-node + securityContext: + privileged: true + capabilities: + drop: + - ALL + add: + - SYS_ADMIN + - CHOWN + - DAC_OVERRIDE + seccompProfile: + type: RuntimeDefault + image: "{{ .Values.node.plugin.image.repository }}:{{ .Values.node.plugin.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.node.plugin.image.pullPolicy }} + args: + - "-v={{ .Values.logVerbosityLevel }}" + - "--csi-address=unix:///csi/csi.sock" + - "--node-id=$(NODE_NAME)" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: {{- toYaml .Values.node.plugin.resources | nindent 12 }} + volumeMounts: + - name: socket + mountPath: /csi + - name: kubelet + mountPath: {{ .Values.node.kubeletDir }} + mountPropagation: Bidirectional + - name: dev + mountPath: /dev + - name: sys + mountPath: /sys + - name: csi-node-driver-registrar + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + image: "{{ .Values.node.driverRegistrar.image.repository }}:{{ .Values.node.driverRegistrar.image.tag }}" + imagePullPolicy: {{ .Values.node.driverRegistrar.image.pullPolicy }} + args: + - "-v={{ .Values.logVerbosityLevel }}" + - "--csi-address=unix:///csi/csi.sock" + - "--kubelet-registration-path={{ .Values.node.kubeletDir }}/plugins/{{ .Values.provisionerName }}/csi.sock" + volumeMounts: + - name: socket + mountPath: /csi + - name: registration + mountPath: /registration + resources: {{- toYaml .Values.node.driverRegistrar.resources | nindent 12 }} + - name: liveness-probe + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + image: "{{ .Values.livenessprobe.image.repository }}:{{ .Values.livenessprobe.image.tag }}" + imagePullPolicy: {{ .Values.livenessprobe.image.pullPolicy }} + args: + - "-v={{ .Values.logVerbosityLevel }}" + - "--csi-address=unix:///csi/csi.sock" + volumeMounts: + - name: socket + mountPath: /csi + resources: {{- toYaml .Values.livenessprobe.resources | nindent 12 }} + volumes: + - name: socket + hostPath: + path: {{ .Values.node.kubeletDir }}/plugins/{{ .Values.provisionerName }}/ + type: DirectoryOrCreate + - name: registration + hostPath: + path: {{ .Values.node.kubeletDir }}/plugins_registry/ + type: Directory + - name: kubelet + hostPath: + path: {{ .Values.node.kubeletDir }} + type: Directory + - name: dev + hostPath: + path: /dev + type: Directory + - name: sys + hostPath: + path: /sys + type: Directory + {{- with .Values.node.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.node.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-rolebinding.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-rolebinding.yaml new file mode 100644 index 000000000..4585dd66a --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/node-rolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }}-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "proxmox-csi-plugin.fullname" . }}-node +subjects: + - kind: ServiceAccount + name: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-node + namespace: {{ .Release.Namespace }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/secrets.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/secrets.yaml new file mode 100644 index 000000000..f96f5d574 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/secrets.yaml @@ -0,0 +1,12 @@ +{{- if ne (len .Values.config.clusters) 0 }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "proxmox-csi-plugin.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} +type: Opaque +data: + config.yaml: {{ toYaml .Values.config | b64enc | quote }} +{{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/serviceaccount.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/serviceaccount.yaml new file mode 100644 index 000000000..d6a455484 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/serviceaccount.yaml @@ -0,0 +1,25 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-controller + namespace: {{ .Release.Namespace }} + labels: + {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "proxmox-csi-plugin.serviceAccountName" . }}-node + namespace: {{ .Release.Namespace }} + labels: + {{- include "proxmox-csi-plugin.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/storageclass.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/storageclass.yaml new file mode 100644 index 000000000..702c4118a --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/templates/storageclass.yaml @@ -0,0 +1,29 @@ +{{- range $storage := .Values.storageClass }} +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ $storage.name | required "StorageClass name must be provided." }} + {{- with $storage.labels }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $storage.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +provisioner: {{ $.Values.provisionerName }} +allowVolumeExpansion: true +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: {{ default "Delete" $storage.reclaimPolicy }} +parameters: + {{- mustMergeOverwrite (default (dict) $storage.extraParameters) (include "storageClass.parameters" . | fromYaml) | toYaml | nindent 2 -}} +{{- with $storage.mountOptions }} +mountOptions: + {{- . | toYaml | nindent 2 }} +{{- end }} +{{- with $storage.allowedTopologies }} +allowedTopologies: + {{- . | toYaml | nindent 2 }} +{{- end }} +--- +{{- end }} diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.edge.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.edge.yaml new file mode 100644 index 000000000..974c87e9d --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.edge.yaml @@ -0,0 +1,29 @@ + +createNamespace: true + +controller: + plugin: + image: + pullPolicy: Always + tag: edge + +node: + plugin: + image: + pullPolicy: Always + tag: edge + +nodeSelector: + node-role.kubernetes.io/control-plane: "" +tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + +storageClass: + - name: proxmox-data-xfs + storage: data + reclaimPolicy: Delete + fstype: xfs + - name: proxmox-data + storage: data + ssd: true diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.talos.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.talos.yaml new file mode 100644 index 000000000..d507b31e4 --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.talos.yaml @@ -0,0 +1,23 @@ + +createNamespace: true + +node: + nodeSelector: + node.cloudprovider.kubernetes.io/platform: nocloud + tolerations: + - operator: Exists + +nodeSelector: + node-role.kubernetes.io/control-plane: "" +tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + +storageClass: + - name: proxmox-data-xfs + storage: data + reclaimPolicy: Delete + fstype: xfs + - name: proxmox-data + storage: data + reclaimPolicy: Delete diff --git a/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.yaml b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.yaml new file mode 100644 index 000000000..99c9fac1d --- /dev/null +++ b/packages/system/proxmox-csi/charts/proxmox-csi-plugin/values.yaml @@ -0,0 +1,308 @@ +# Default values for proxmox-csi-plugin. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +# -- Create namespace. +# Very useful when using helm template. +createNamespace: false + +# -- Controller pods priorityClassName. +priorityClassName: system-cluster-critical + +# -- Pods Service Account. +# ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +# -- CSI Driver provisioner name. +# Currently, cannot be customized. +provisionerName: csi.proxmox.sinextra.dev + +# -- Cluster name. +# Currently, cannot be customized. +clusterID: kubernetes + +# -- Log verbosity level. See https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md +# for description of individual verbosity levels. +logVerbosityLevel: 5 + +# -- Connection timeout between sidecars. +timeout: 3m + +# -- Proxmox cluster config stored in secrets. +existingConfigSecret: ~ +# -- Proxmox cluster config stored in secrets key. +existingConfigSecretKey: config.yaml + +# -- Proxmox cluster config path. +configFile: /etc/proxmox/config.yaml + +# -- Proxmox cluster config. +config: + clusters: [] + # - url: https://cluster-api-1.exmple.com:8006/api2/json + # insecure: false + # token_id: "login!name" + # token_secret: "secret" + # region: cluster-1 + +# -- Storage class definition. +storageClass: [] + # - name: proxmox-data-xfs + # storage: data + # reclaimPolicy: Delete + # fstype: ext4|xfs + # + # # https://pve.proxmox.com/wiki/Performance_Tweaks + # cache: directsync|none|writeback|writethrough + # ssd: true + # extraParameters: + # # https://kubernetes-csi.github.io/docs/secrets-and-credentials-storage-class.html + # csi.storage.k8s.io/node-stage-secret-name: "proxmox-csi-secret" + # csi.storage.k8s.io/node-stage-secret-namespace: "kube-system" + # csi.storage.k8s.io/node-expand-secret-name: "proxmox-csi-secret" + # csi.storage.k8s.io/node-expand-secret-namespace: "kube-system" + # + # mountOptions: + # - discard + # + # # I am not recommending to use this option, but it is possible. + # # You should use region and zone topology keys together. + # allowedTopologies: + # - matchLabelExpressions: + # - key: topology.kubernetes.io/region + # values: + # - Region-1 + # - key: topology.kubernetes.io/zone + # values: + # - pve-1 + # - pve-3 + +controller: + # -- Annotations for controller pod. + # ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ + podAnnotations: {} + # prometheus.io/scrape: "true" + # prometheus.io/port: "8080" + + plugin: + # -- Controller CSI Driver. + image: + repository: ghcr.io/sergelogvinov/proxmox-csi-controller + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + # -- Controller resource requests and limits. + # ref: https://kubernetes.io/docs/user-guide/compute-resources/ + resources: + requests: + cpu: 10m + memory: 16Mi + attacher: + # -- CSI Attacher. + image: + repository: registry.k8s.io/sig-storage/csi-attacher + pullPolicy: IfNotPresent + tag: v4.4.4 + # -- Attacher resource requests and limits. + # ref: https://kubernetes.io/docs/user-guide/compute-resources/ + resources: + requests: + cpu: 10m + memory: 16Mi + provisioner: + # -- CSI Provisioner. + image: + repository: registry.k8s.io/sig-storage/csi-provisioner + pullPolicy: IfNotPresent + tag: v3.6.4 + # -- Provisioner resource requests and limits. + # ref: https://kubernetes.io/docs/user-guide/compute-resources/ + resources: + requests: + cpu: 10m + memory: 16Mi + resizer: + # -- CSI Resizer. + image: + repository: registry.k8s.io/sig-storage/csi-resizer + pullPolicy: IfNotPresent + tag: v1.9.4 + # -- Resizer resource requests and limits. + # ref: https://kubernetes.io/docs/user-guide/compute-resources/ + resources: + requests: + cpu: 10m + memory: 16Mi + +node: + plugin: + # -- Node CSI Driver. + image: + repository: ghcr.io/sergelogvinov/proxmox-csi-node + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + # -- Node CSI Driver resource requests and limits. + # ref: https://kubernetes.io/docs/user-guide/compute-resources/ + resources: {} + driverRegistrar: + # -- Node CSI driver registrar. + image: + repository: registry.k8s.io/sig-storage/csi-node-driver-registrar + pullPolicy: IfNotPresent + tag: v2.9.4 + # -- Node registrar resource requests and limits. + # ref: https://kubernetes.io/docs/user-guide/compute-resources/ + resources: + requests: + cpu: 10m + memory: 16Mi + + # -- Location of the /var/lib/kubelet directory as some k8s distribution differ from the standard. + kubeletDir: /var/lib/kubelet + + # -- Node labels for node-plugin assignment. + # ref: https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for node-plugin assignment. + # ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: + - key: node.kubernetes.io/unschedulable + operator: Exists + effect: NoSchedule + - key: node.kubernetes.io/disk-pressure + operator: Exists + effect: NoSchedule + +livenessprobe: + # -- Common livenessprobe sidecar. + image: + repository: registry.k8s.io/sig-storage/livenessprobe + pullPolicy: IfNotPresent + tag: v2.11.0 + # -- Failure threshold for livenessProbe + failureThreshold: 5 + # -- Initial delay seconds for livenessProbe + initialDelaySeconds: 10 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 10 + # -- Period seconds for livenessProbe + periodSeconds: 60 + # -- Liveness probe resource requests and limits. + # ref: https://kubernetes.io/docs/user-guide/compute-resources/ + resources: + requests: + cpu: 10m + memory: 16Mi + +# -- Add additional init containers for the CSI controller pods. +# ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ +initContainers: [] + # - name: loadbalancer + # restartPolicy: Always + # image: ghcr.io/sergelogvinov/haproxy:2.8.3-alpine3.18 + # imagePullPolicy: IfNotPresent + # env: + # - name: SVC + # value: "proxmox.domain.com" + # - name: PORT + # value: "8006" + # securityContext: + # runAsUser: 99 + # runAsGroup: 99 + # resources: + # limits: + # cpu: 50m + # memory: 64Mi + # requests: + # cpu: 50m + # memory: 32Mi + +# -- hostAliases Deployment pod host aliases +# ref: https://kubernetes.io/docs/tasks/network/customize-hosts-file-for-pods/ +hostAliases: [] + # - ip: 127.0.0.1 + # hostnames: + # - proxmox.domain.com + +# -- Annotations for controller pod. +# ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +podAnnotations: {} + +# -- Controller Security Context. +# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod +podSecurityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + fsGroupChangePolicy: OnRootMismatch + +# -- Controller Container Security Context. +# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + readOnlyRootFilesystem: true + +# -- Controller deployment update strategy type. +# ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + +# -- Prometheus metrics +metrics: + # -- Enable Prometheus metrics. + enabled: false + # -- Prometheus metrics port. + port: 8080 + + type: annotation + + +# -- Node labels for controller assignment. +# ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + # node-role.kubernetes.io/control-plane: "" + +# -- Tolerations for controller assignment. +# ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: [] + # - key: node-role.kubernetes.io/control-plane + # effect: NoSchedule + +# -- Affinity for controller assignment. +# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +affinity: {} + +# -- Additional volumes for Pods +extraVolumes: [] +# - name: ca +# secret: +# secretName: my-ca +# -- Additional volume mounts for Pods + +extraVolumeMounts: [] +# - mountPath: /etc/ssl/certs/ca-certificates.crt +# name: ca +# subPath: ca.crt diff --git a/packages/system/proxmox-csi/patches/namespace.patch b/packages/system/proxmox-csi/patches/namespace.patch new file mode 100644 index 000000000..c06485768 --- /dev/null +++ b/packages/system/proxmox-csi/patches/namespace.patch @@ -0,0 +1,13 @@ +diff --git a/apps/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml b/apps/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml +index 0ed037f..32b065e 100644 +--- a/apps/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml ++++ b/apps/proxmox-csi/charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml +@@ -9,7 +9,7 @@ roleRef: + subjects: + - kind: ServiceAccount + name: {{ include "proxmox-cloud-controller-manager.fullname" . }} +- namespace: kube-system ++ namespace: {{ .Release.Namespace }} + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding diff --git a/packages/system/proxmox-csi/tests/1.yaml b/packages/system/proxmox-csi/tests/1.yaml new file mode 100644 index 000000000..e8a19fc92 --- /dev/null +++ b/packages/system/proxmox-csi/tests/1.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: task-pv-claim +spec: + storageClassName: proxmox-lvm + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 3Gi +--- +apiVersion: v1 +kind: Pod +metadata: + name: task-pv-pod +spec: + volumes: + - name: task-pv-storage + persistentVolumeClaim: + claimName: task-pv-claim + containers: + - name: task-pv-container + image: nginx + ports: + - containerPort: 80 + name: "http-server" + volumeMounts: + - mountPath: "/usr/share/nginx/html" + name: task-pv-storage diff --git a/packages/system/proxmox-csi/values.yaml b/packages/system/proxmox-csi/values.yaml new file mode 100644 index 000000000..19a5f9f19 --- /dev/null +++ b/packages/system/proxmox-csi/values.yaml @@ -0,0 +1,22 @@ +proxmox-cloud-controller-manager: + fullnameOverride: proxmox-cloud-controller-manager + + enabledControllers: + - cloud-node + - cloud-node-lifecycle + + # Deploy CCM only on control-plane nodes + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + +proxmox-csi-plugin: + fullnameOverride: proxmox-csi-plugin + + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule diff --git a/scripts/installer.sh b/scripts/installer.sh index 2ffeba7b6..4b013b40f 100755 --- a/scripts/installer.sh +++ b/scripts/installer.sh @@ -50,10 +50,10 @@ wait_for_crds() { } install_basic_charts() { - if [ "$BUNDLE" = "paas-full" ] || [ "$BUNDLE" = "distro-full" ]; then - make -C packages/system/cilium apply resume + if [ "$BUNDLE" = "paas-full" ] || [ "$BUNDLE" = "distro-full" || "$BUNDLE" = "paas-proxmox" ]; then + make -C packages/system/cilium apply resume fi - if [ "$BUNDLE" = "paas-full" ]; then + if [ "$BUNDLE" = "paas-full" || "$BUNDLE" = "paas-proxmox" ]; then make -C packages/system/kubeovn apply resume fi }