change(jwt-auth): unify apisix/core/vault.lua and apisix/secret/vault.lua (#8660)

Fixes #8424

Author: shreemaan-abhishek

README.md

@@ -140,7 +140,7 @@ A/B testing, canary release, blue-green deployment, limit rate, defense against
   - [Elasticsearch](docs/en/latest/plugins/elasticsearch-logger.md): push logs to Elasticsearch.
   - [Datadog](docs/en/latest/plugins/datadog.md): push custom metrics to the DogStatsD server, comes bundled with [Datadog agent](https://docs.datadoghq.com/agent/), over the UDP protocol. DogStatsD basically is an implementation of StatsD protocol which collects the custom metrics for Apache APISIX agent, aggregates it into a single data point and sends it to the configured Datadog server.
   - [Helm charts](https://github.com/apache/apisix-helm-chart)
-  - [HashiCorp Vault](https://www.vaultproject.io/): Support secret management solution for accessing secrets from Vault secure storage backed in a low trust environment. Currently, RS256 keys (public-private key pairs) or secret keys can be linked from vault in [jwt-auth](docs/en/latest/plugins/jwt-auth.md#enable-jwt-auth-with-vault-compatibility) authentication plugin.
+  - [HashiCorp Vault](https://www.vaultproject.io/): Support secret management solution for accessing secrets from Vault secure storage backed in a low trust environment. Currently, RS256 keys (public-private key pairs) or secret keys can be linked from vault in jwt-auth authentication plugin using [APISIX Secret](docs/en/latest/terminology/secret.md) resource.
 
 - **Highly scalable**
   - [Custom plugins](docs/en/latest/plugin-develop.md): Allows hooking of common phases, such as `rewrite`, `access`, `header filter`, `body filter` and `log`, also allows to hook the `balancer` stage.

apisix/core/vault.lua

@@ -18,7 +18,6 @@ local core     = require("apisix.core")
 local jwt      = require("resty.jwt")
 local consumer_mod = require("apisix.consumer")
 local resty_random = require("resty.random")
-local vault        = require("apisix.core.vault")
 local new_tab = require ("table.new")
 
 local ngx_encode_base64 = ngx.encode_base64
@@ -71,10 +70,6 @@ local consumer_schema = {
             type = "boolean",
             default = false
         },
-        vault = {
-            type = "object",
-            properties = {}
-        },
         lifetime_grace_period = {
             type = "integer",
             minimum = 0,
@@ -102,19 +97,6 @@ local consumer_schema = {
                     },
                     required = {"public_key", "private_key"},
                 },
-                {
-                    properties = {
-                        vault = {
-                            type = "object",
-                            properties = {}
-                        },
-                        algorithm = {
-                            enum = {"RS256", "ES256"},
-                        },
-                    },
-                    required = {"vault"},
-                },
-
             }
         }
     },
@@ -147,11 +129,6 @@ function _M.check_schema(conf, schema_type)
         return false, err
     end
 
-    if conf.vault then
-        core.log.info("skipping jwt-auth schema validation with vault")
-        return true
-    end
-
     if conf.algorithm ~= "RS256" and conf.algorithm ~= "ES256" and not conf.secret then
         conf.secret = ngx_encode_base64(resty_random.bytes(32, true))
     elseif conf.base64_secret then
@@ -161,8 +138,8 @@ function _M.check_schema(conf, schema_type)
     end
 
     if conf.algorithm == "RS256" or conf.algorithm == "ES256" then
-        -- Possible options are a) both are in vault, b) both in schema
-        -- c) one in schema, another in vault.
+        -- Possible options are a) public key is missing
+        -- b) private key is missing
         if not conf.public_key then
             return false, "missing valid public key"
         end
@@ -243,25 +220,8 @@ local function fetch_jwt_token(conf, ctx)
     return val
 end
 
-
-local function get_vault_path(username)
-    return "consumer/".. username .. "/jwt-auth"
-end
-
-
 local function get_secret(conf, consumer_name)
     local secret = conf.secret
-    if conf.vault then
-        local res, err = vault.get(get_vault_path(consumer_name))
-        if not res then
-            return nil, err
-        end
-
-        if not res.data or not res.data.secret then
-            return nil, "secret could not found in vault: " .. core.json.encode(res)
-        end
-        secret = res.data.secret
-    end
 
     if conf.base64_secret then
         return ngx_decode_base64(secret)
@@ -274,32 +234,16 @@ end
 local function get_rsa_or_ecdsa_keypair(conf, consumer_name)
     local public_key = conf.public_key
     local private_key = conf.private_key
-    -- if keys are present in conf, no need to query vault (fallback)
+
     if public_key and private_key then
         return public_key, private_key
+    elseif public_key and not private_key then
+        return nil, nil, "missing private key"
+    elseif not public_key and private_key then
+        return nil, nil, "missing public key"
+    else
+        return nil, nil, "public and private keys are missing"
     end
-
-    local vout = {}
-    if conf.vault then
-        local res, err = vault.get(get_vault_path(consumer_name))
-        if not res then
-            return nil, nil, err
-        end
-
-        if not res.data then
-            return nil, nil, "key pairs could not found in vault: " .. core.json.encode(res)
-        end
-        vout = res.data
-    end
-
-    if not public_key and not vout.public_key then
-        return nil, nil, "missing public key, not found in config/vault"
-    end
-    if not private_key and not vout.private_key then
-        return nil, nil, "missing private key, not found in config/vault"
-    end
-
-    return public_key or vout.public_key, private_key or vout.private_key
 end
 
 

apisix/plugins/jwt-auth.lua

@@ -45,12 +45,6 @@ local _M = {
     schema = schema
 }
 
--- This code is copied from apisix/core/vault.lua.
--- The functions in apisix/core/vault.lua are currently only used in the jwt-auth plugin,
--- and it is not suitable to be placed in the core module of APISIX.
---
--- When KMS is fully functional, we will remove apisix/core/vault.lua.
---
 local function make_request_to_vault(conf, method, key, data)
     local httpc = http.new()
     -- config timeout or default to 5000 ms

apisix/secret/vault.lua

@@ -250,19 +250,6 @@ nginx_config:                     # config for render the template to generate n
       tars: 1m
       cas-auth: 10m
 
-# HashiCorp Vault storage backend for sensitive data retrieval. The config shows an example of what APISIX expects if you
-# wish to integrate Vault for secret (sensetive string, public private keys etc.) retrieval. APISIX communicates with Vault
-# server HTTP APIs. By default, APISIX doesn't need this configuration.
-# vault:
-#   host: "http://0.0.0.0:8200"   # The host address where the vault server is running.
-#   timeout: 10                   # request timeout 30 seconds
-#   token: root                   # Authentication token to access Vault HTTP APIs
-#   prefix: kv/apisix             # APISIX supports vault kv engine v1, where sensitive data are being stored
-                                  # and retrieved through vault HTTP APIs. enabling a prefix allows you to better enforcement of
-                                  # policies, generate limited scoped tokens and tightly control the data that can be accessed
-                                  # from APISIX.
-
-
 #discovery:                       # service discovery center
 #  dns:
 #    servers:

conf/config-default.yaml

@@ -50,7 +50,7 @@ APISIX facilitates interface traffic handling for websites, mobile and IoT appli
 - Multi-platform support: APISIX can run from bare-metal machines to Kubernetes providing a vendor neutral, multi-platform solution. It also provides integration to cloud services like AWS Lambda, Azure Function, Lua functions and Apache OpenWhisk.
 - Fully dynamic: APISIX supports hot-reloading, meaning you don't need to restart the service to reflect changes in the configuration.
 - Fine-grained routing: APISIX supports using all [built-in NGINX variables](http://nginx.org/en/docs/varindex.html) for routing. You can define custom matching functions to filter requests and match Route.
-- Ops-friendly: APISIX is renowned for its ops-friendliness by DevOps teams. It integrates with tools and platforms like [HashiCorp Vault](./plugins/jwt-auth.md#usage-with-hashicorp-vault), [Zipkin](./plugins/zipkin.md), [Apache SkyWalking](./plugins/skywalking.md), [Consul](./discovery/consul_kv.md), [Nacos](./discovery/nacos.md) and [Eureka](./discovery.md). With [APISIX Dashboard](https://github.com/apache/apisix-dashboard), operators can configure APISIX through an easy-to-use and intuitive UI.
+- Ops-friendly: APISIX is renowned for its ops-friendliness by DevOps teams. It integrates with tools and platforms like [HashiCorp Vault](./terminology/secret.md#use-vault-to-manage-secrets), [Zipkin](./plugins/zipkin.md), [Apache SkyWalking](./plugins/skywalking.md), [Consul](./discovery/consul_kv.md), [Nacos](./discovery/nacos.md) and [Eureka](./discovery.md). With [APISIX Dashboard](https://github.com/apache/apisix-dashboard), operators can configure APISIX through an easy-to-use and intuitive UI.
 - Multi-language Plugin support: APISIX supports multiple programming languages for Plugin development. Developers can choose a language-specific SDK to write custom Plugins.
 
 ## Key concepts