|
| 1 | +-- |
| 2 | +-- Licensed to the Apache Software Foundation (ASF) under one or more |
| 3 | +-- contributor license agreements. See the NOTICE file distributed with |
| 4 | +-- this work for additional information regarding copyright ownership. |
| 5 | +-- The ASF licenses this file to You under the Apache License, Version 2.0 |
| 6 | +-- (the "License"); you may not use this file except in compliance with |
| 7 | +-- the License. You may obtain a copy of the License at |
| 8 | +-- |
| 9 | +-- http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +-- |
| 11 | +-- Unless required by applicable law or agreed to in writing, software |
| 12 | +-- distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | +-- See the License for the specific language governing permissions and |
| 15 | +-- limitations under the License. |
| 16 | +-- |
| 17 | +local require = require |
| 18 | +local setmetatable = setmetatable |
| 19 | +local ipairs = ipairs |
| 20 | +local type = type |
| 21 | +local core = require("apisix.core") |
| 22 | +local limit_count = require("apisix.plugins.limit-count.init") |
| 23 | + |
| 24 | +local plugin_name = "ai-rate-limiting" |
| 25 | + |
| 26 | +local instance_limit_schema = { |
| 27 | + type = "object", |
| 28 | + properties = { |
| 29 | + name = {type = "string"}, |
| 30 | + limit = {type = "integer", minimum = 1}, |
| 31 | + time_window = {type = "integer", minimum = 1} |
| 32 | + }, |
| 33 | + required = {"name", "limit", "time_window"} |
| 34 | +} |
| 35 | + |
| 36 | +local schema = { |
| 37 | + type = "object", |
| 38 | + properties = { |
| 39 | + limit = {type = "integer", exclusiveMinimum = 0}, |
| 40 | + time_window = {type = "integer", exclusiveMinimum = 0}, |
| 41 | + show_limit_quota_header = {type = "boolean", default = true}, |
| 42 | + limit_strategy = { |
| 43 | + type = "string", |
| 44 | + enum = {"total_tokens", "prompt_tokens", "completion_tokens"}, |
| 45 | + default = "total_tokens", |
| 46 | + description = "The strategy to limit the tokens" |
| 47 | + }, |
| 48 | + instances = { |
| 49 | + type = "array", |
| 50 | + items = instance_limit_schema |
| 51 | + }, |
| 52 | + rejected_code = { |
| 53 | + type = "integer", minimum = 200, maximum = 599, default = 503 |
| 54 | + }, |
| 55 | + rejected_msg = { |
| 56 | + type = "string", minLength = 1 |
| 57 | + }, |
| 58 | + }, |
| 59 | + required = {"limit", "time_window"}, |
| 60 | +} |
| 61 | + |
| 62 | +local _M = { |
| 63 | + version = 0.1, |
| 64 | + priority = 1030, |
| 65 | + name = plugin_name, |
| 66 | + schema = schema |
| 67 | +} |
| 68 | + |
| 69 | +local limit_conf_cache = core.lrucache.new({ |
| 70 | + ttl = 300, count = 512 |
| 71 | +}) |
| 72 | + |
| 73 | + |
| 74 | +function _M.check_schema(conf) |
| 75 | + return core.schema.check(schema, conf) |
| 76 | +end |
| 77 | + |
| 78 | + |
| 79 | +local function transform_limit_conf(plugin_conf, instance_conf, instance_name) |
| 80 | + local key = plugin_name .. "#global" |
| 81 | + local limit = plugin_conf.limit |
| 82 | + local time_window = plugin_conf.time_window |
| 83 | + local name = instance_name or "" |
| 84 | + if instance_conf then |
| 85 | + name = instance_conf.name |
| 86 | + key = instance_conf.name |
| 87 | + limit = instance_conf.limit |
| 88 | + time_window = instance_conf.time_window |
| 89 | + end |
| 90 | + return { |
| 91 | + _vid = key, |
| 92 | + |
| 93 | + key = key, |
| 94 | + count = limit, |
| 95 | + time_window = time_window, |
| 96 | + rejected_code = plugin_conf.rejected_code, |
| 97 | + rejected_msg = plugin_conf.rejected_msg, |
| 98 | + show_limit_quota_header = plugin_conf.show_limit_quota_header, |
| 99 | + -- limit-count need these fields |
| 100 | + policy = "local", |
| 101 | + key_type = "constant", |
| 102 | + allow_degradation = false, |
| 103 | + sync_interval = -1, |
| 104 | + |
| 105 | + limit_header = "X-AI-RateLimit-Limit-" .. name, |
| 106 | + remaining_header = "X-AI-RateLimit-Remaining-" .. name, |
| 107 | + reset_header = "X-AI-RateLimit-Reset-" .. name, |
| 108 | + } |
| 109 | +end |
| 110 | + |
| 111 | + |
| 112 | +local function fetch_limit_conf_kvs(conf) |
| 113 | + local mt = { |
| 114 | + __index = function(t, k) |
| 115 | + local limit_conf = transform_limit_conf(conf, nil, k) |
| 116 | + t[k] = limit_conf |
| 117 | + return limit_conf |
| 118 | + end |
| 119 | + } |
| 120 | + local limit_conf_kvs = setmetatable({}, mt) |
| 121 | + local conf_instances = conf.instances or {} |
| 122 | + for _, limit_conf in ipairs(conf_instances) do |
| 123 | + limit_conf_kvs[limit_conf.name] = transform_limit_conf(conf, limit_conf) |
| 124 | + end |
| 125 | + return limit_conf_kvs |
| 126 | +end |
| 127 | + |
| 128 | + |
| 129 | +function _M.access(conf, ctx) |
| 130 | + local ai_instance_name = ctx.picked_ai_instance_name |
| 131 | + if not ai_instance_name then |
| 132 | + return |
| 133 | + end |
| 134 | + |
| 135 | + local limit_conf_kvs = limit_conf_cache(conf, nil, fetch_limit_conf_kvs, conf) |
| 136 | + local limit_conf = limit_conf_kvs[ai_instance_name] |
| 137 | + local code, msg = limit_count.rate_limit(limit_conf, ctx, plugin_name, 1, true) |
| 138 | + ctx.ai_rate_limiting = code and true or false |
| 139 | + return code, msg |
| 140 | +end |
| 141 | + |
| 142 | + |
| 143 | +function _M.check_instance_status(conf, ctx, instance_name) |
| 144 | + if conf == nil then |
| 145 | + local plugins = ctx.plugins |
| 146 | + for _, plugin in ipairs(plugins) do |
| 147 | + if plugin.name == plugin_name then |
| 148 | + conf = plugin |
| 149 | + end |
| 150 | + end |
| 151 | + end |
| 152 | + if not conf then |
| 153 | + return true |
| 154 | + end |
| 155 | + |
| 156 | + instance_name = instance_name or ctx.picked_ai_instance_name |
| 157 | + if not instance_name then |
| 158 | + return nil, "missing instance_name" |
| 159 | + end |
| 160 | + |
| 161 | + if type(instance_name) ~= "string" then |
| 162 | + return nil, "invalid instance_name" |
| 163 | + end |
| 164 | + |
| 165 | + local limit_conf_kvs = limit_conf_cache(conf, nil, fetch_limit_conf_kvs, conf) |
| 166 | + local limit_conf = limit_conf_kvs[instance_name] |
| 167 | + local code, _ = limit_count.rate_limit(limit_conf, ctx, plugin_name, 1, true) |
| 168 | + if code then |
| 169 | + core.log.info("rate limit for instance: ", instance_name, " code: ", code) |
| 170 | + return false |
| 171 | + end |
| 172 | + return true |
| 173 | +end |
| 174 | + |
| 175 | + |
| 176 | +local function get_token_usage(conf, ctx) |
| 177 | + local usage = ctx.ai_token_usage |
| 178 | + if not usage then |
| 179 | + return |
| 180 | + end |
| 181 | + return usage[conf.limit_strategy] |
| 182 | +end |
| 183 | + |
| 184 | + |
| 185 | +function _M.log(conf, ctx) |
| 186 | + local instance_name = ctx.picked_ai_instance_name |
| 187 | + if not instance_name then |
| 188 | + return |
| 189 | + end |
| 190 | + |
| 191 | + if ctx.ai_rate_limiting then |
| 192 | + return |
| 193 | + end |
| 194 | + |
| 195 | + local used_tokens = get_token_usage(conf, ctx) |
| 196 | + if not used_tokens then |
| 197 | + core.log.error("failed to get token usage for llm service") |
| 198 | + return |
| 199 | + end |
| 200 | + |
| 201 | + core.log.info("instance name: ", instance_name, " used tokens: ", used_tokens) |
| 202 | + |
| 203 | + local limit_conf_kvs = limit_conf_cache(conf, nil, fetch_limit_conf_kvs, conf) |
| 204 | + local limit_conf = limit_conf_kvs[instance_name] |
| 205 | + limit_count.rate_limit(limit_conf, ctx, plugin_name, used_tokens) |
| 206 | +end |
| 207 | + |
| 208 | + |
| 209 | +return _M |
0 commit comments