feat(deployment): control plane connects to conf server (#7392)

Signed-off-by: spacewander <[email protected]>

Author: spacewander

apisix/cli/schema.lua

@@ -305,7 +305,20 @@ local deployment_schema = {
                     },
                 },
                 required = {"config_provider", "conf_server"}
-            }
+            },
+            certs = {
+                properties = {
+                    cert = { type = "string" },
+                    cert_key = { type = "string" },
+                    trusted_ca_cert = { type = "string" },
+                },
+                dependencies = {
+                    cert = {
+                        required = {"cert_key"},
+                    },
+                },
+                default = {},
+            },
         },
         required = {"etcd", "role_control_plane"}
     }

apisix/cli/snippet.lua

@@ -63,6 +63,13 @@ function _M.generate_conf_server(env, conf)
         end
     end
 
+    local trusted_ca_cert
+    if conf.deployment.certs then
+        if conf.deployment.certs.trusted_ca_cert then
+            trusted_ca_cert = pl_path.abspath(conf.deployment.certs.trusted_ca_cert)
+        end
+    end
+
     local conf_render = template.compile([[
     upstream apisix_conf_backend {
         server 0.0.0.0:80;
@@ -71,6 +78,11 @@ function _M.generate_conf_server(env, conf)
             conf_server.balancer()
         }
     }
+
+    {% if trusted_ca_cert then %}
+    lua_ssl_trusted_certificate {* trusted_ca_cert *};
+    {% end %}
+
     server {
         {% if control_plane then %}
         listen {* control_plane.listen *} ssl;
@@ -144,6 +156,7 @@ function _M.generate_conf_server(env, conf)
         enable_https = enable_https,
         client_cert = client_cert,
         client_cert_key = client_cert_key,
+        trusted_ca_cert = trusted_ca_cert,
     })
 end
 

apisix/core/config_etcd.lua

@@ -812,18 +812,13 @@ function _M.init()
         return true
     end
 
-    local etcd_cli, err = get_etcd()
+    -- don't go through proxy during start because the proxy is not available
+    local etcd_cli, prefix, err = etcd_apisix.new_without_proxy()
     if not etcd_cli then
         return nil, "failed to start a etcd instance: " .. err
     end
 
-    -- don't go through proxy during start because the proxy is not available
-    local proxy = etcd_cli.unix_socket_proxy
-    etcd_cli.unix_socket_proxy = nil
-    local etcd_conf = local_conf.etcd
-    local prefix = etcd_conf.prefix
     local res, err = readdir(etcd_cli, prefix, create_formatter(prefix))
-    etcd_cli.unix_socket_proxy = proxy
     if not res then
         return nil, err
     end

apisix/core/etcd.lua

@@ -29,13 +29,53 @@ local setmetatable      = setmetatable
 local string            = string
 local tonumber          = tonumber
 local ngx_config_prefix = ngx.config.prefix()
+local ngx_socket_tcp    = ngx.socket.tcp
 
 
 local is_http = ngx.config.subsystem == "http"
 local _M = {}
 
 
--- this function create the etcd client instance used in the Admin API
+local function has_mtls_support()
+    local s = ngx_socket_tcp()
+    return s.tlshandshake ~= nil
+end
+
+
+local function _new(etcd_conf)
+    local prefix = etcd_conf.prefix
+    etcd_conf.http_host = etcd_conf.host
+    etcd_conf.host = nil
+    etcd_conf.prefix = nil
+    etcd_conf.protocol = "v3"
+    etcd_conf.api_prefix = "/v3"
+
+    -- default to verify etcd cluster certificate
+    etcd_conf.ssl_verify = true
+    if etcd_conf.tls then
+        if etcd_conf.tls.verify == false then
+            etcd_conf.ssl_verify = false
+        end
+
+        if etcd_conf.tls.cert then
+            etcd_conf.ssl_cert_path = etcd_conf.tls.cert
+            etcd_conf.ssl_key_path = etcd_conf.tls.key
+        end
+
+        if etcd_conf.tls.sni then
+            etcd_conf.sni = etcd_conf.tls.sni
+        end
+    end
+
+    local etcd_cli, err = etcd.new(etcd_conf)
+    if not etcd_cli then
+        return nil, nil, err
+    end
+
+    return etcd_cli, prefix
+end
+
+
 local function new()
     local local_conf, err = fetch_local_conf()
     if not local_conf then
@@ -60,32 +100,20 @@ local function new()
             proxy_by_conf_server = true
 
         elseif local_conf.deployment.role == "control_plane" then
-            -- TODO: add the proxy conf in control_plane
-            proxy_by_conf_server = true
-        end
-    end
-
-    local prefix = etcd_conf.prefix
-    etcd_conf.http_host = etcd_conf.host
-    etcd_conf.host = nil
-    etcd_conf.prefix = nil
-    etcd_conf.protocol = "v3"
-    etcd_conf.api_prefix = "/v3"
-
-    -- default to verify etcd cluster certificate
-    etcd_conf.ssl_verify = true
-    if etcd_conf.tls then
-        if etcd_conf.tls.verify == false then
-            etcd_conf.ssl_verify = false
-        end
+            local addr = local_conf.deployment.role_control_plane.conf_server.listen
+            etcd_conf.host = {"https://" .. addr}
+            etcd_conf.tls = {
+                verify = false,
+            }
 
-        if etcd_conf.tls.cert then
-            etcd_conf.ssl_cert_path = etcd_conf.tls.cert
-            etcd_conf.ssl_key_path = etcd_conf.tls.key
-        end
+            if has_mtls_support() and local_conf.deployment.certs.cert then
+                local cert = local_conf.deployment.certs.cert
+                local cert_key = local_conf.deployment.certs.cert_key
+                etcd_conf.tls.cert = cert
+                etcd_conf.tls.key = cert_key
+            end
 
-        if etcd_conf.tls.sni then
-            etcd_conf.sni = etcd_conf.tls.sni
+            proxy_by_conf_server = true
         end
     end
 
@@ -102,15 +130,28 @@ local function new()
         })
     end
 
-    local etcd_cli
-    etcd_cli, err = etcd.new(etcd_conf)
-    if not etcd_cli then
+    return _new(etcd_conf)
+end
+_M.new = new
+
+
+---
+-- Create an etcd client which will connect to etcd without being proxyed by conf server.
+-- This method is used in init_worker phase when the conf server is not ready.
+--
+-- @function core.etcd.new_without_proxy
+-- @treturn table|nil the etcd client, or nil if failed.
+-- @treturn string|nil the configured prefix of etcd keys, or nil if failed.
+-- @treturn nil|string the error message.
+function _M.new_without_proxy()
+    local local_conf, err = fetch_local_conf()
+    if not local_conf then
         return nil, nil, err
     end
 
-    return etcd_cli, prefix
+    local etcd_conf = clone_tab(local_conf.etcd)
+    return _new(etcd_conf)
 end
-_M.new = new
 
 
 -- convert ETCD v3 entry to v2 one

docs/en/latest/architecture-design/deployment-role.md

@@ -123,6 +123,29 @@ deployment:
         trusted_ca_cert: /path/to/ca-cert
 ```
 
+As OpenResty <= 1.21.4 doesn't support sending mTLS request, if you need to accept the connections from APISIX running on these OpenResty versions,
+you need to disable client certificate verification in the CP instance.
+
+Here is the example of configuration:
+
+```yaml title="conf/config.yaml"
+deployment:
+    role: control_plane
+    role_control_plan:
+        config_provider: etcd
+        conf_server:
+            listen: 0.0.0.0:9280
+            cert: /path/to/ca-cert
+            cert_key: /path/to/ca-cert
+    etcd:
+       host:
+           - https://xxxx
+       prefix: /apisix
+       timeout: 30
+    certs:
+        trusted_ca_cert: /path/to/ca-cert
+```
+
 ### Standalone
 
 In this mode, APISIX is deployed as DP and reads configurations from yaml file in the local file system.

t/cli/test_deployment_control_plane.sh

@@ -40,6 +40,7 @@ fi
 
 echo "passed: should check deployment schema during init"
 
+# The 'admin.apisix.dev' is injected by ci/common.sh@set_coredns
 echo '
 apisix:
     enable_admin: false
@@ -48,14 +49,15 @@ deployment:
     role_control_plane:
         config_provider: etcd
         conf_server:
-            listen: 0.0.0.0:12345
+            listen: admin.apisix.dev:12345
             cert: t/certs/mtls_server.crt
             cert_key: t/certs/mtls_server.key
-            client_ca_cert: t/certs/mtls_ca.crt
     etcd:
         prefix: "/apisix"
         host:
             - http://127.0.0.1:2379
+    certs:
+        trusted_ca_cert: t/certs/mtls_ca.crt
 ' > conf/config.yaml
 
 make run
@@ -71,20 +73,25 @@ fi
 
 echo "passed: control_plane should enable Admin API"
 
+# use https
+# The 'admin.apisix.dev' is injected by ci/common.sh@set_coredns
 echo '
 deployment:
     role: control_plane
     role_control_plane:
         config_provider: etcd
         conf_server:
-            listen: 0.0.0.0:12345
+            listen: admin.apisix.dev:12345
             cert: t/certs/mtls_server.crt
             cert_key: t/certs/mtls_server.key
-            client_ca_cert: t/certs/mtls_ca.crt
     etcd:
         prefix: "/apisix"
         host:
             - http://127.0.0.1:2379
+    certs:
+        cert: t/certs/mtls_client.crt
+        cert_key: t/certs/mtls_client.key
+        trusted_ca_cert: t/certs/mtls_ca.crt
 ' > conf/config.yaml
 
 make run

t/cli/test_deployment_control_plane_customed_nginx.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+. ./t/cli/common.sh
+
+exit_if_not_customed_nginx
+
+# use mTLS
+# The 'admin.apisix.dev' is injected by ci/common.sh@set_coredns
+echo '
+deployment:
+    role: control_plane
+    role_control_plane:
+        config_provider: etcd
+        conf_server:
+            listen: admin.apisix.dev:12345
+            cert: t/certs/mtls_server.crt
+            cert_key: t/certs/mtls_server.key
+            client_ca_cert: t/certs/mtls_ca.crt
+    etcd:
+        prefix: "/apisix"
+        host:
+            - http://127.0.0.1:2379
+    certs:
+        cert: t/certs/mtls_client.crt
+        cert_key: t/certs/mtls_client.key
+        trusted_ca_cert: t/certs/mtls_ca.crt
+' > conf/config.yaml
+
+make run
+sleep 1
+
+code=$(curl -o /dev/null -s -w %{http_code} http://127.0.0.1:9080/apisix/admin/routes -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1')
+make stop
+
+if [ ! $code -eq 200 ]; then
+    echo "failed: could not work with etcd"
+    exit 1
+fi
+
+echo "passed: work well with etcd in control plane"