feat: increment route update for radixtree host uri, radixtree uri and radi…

apisix/core/config_etcd.lua

@@ -499,7 +499,7 @@ local function load_full_data(self, dir_res, headers)
             item.clean_handlers = {}
 
             if self.filter then
-                self.filter(item)
+                self.filter(item, 1, self)
             end
         end
 
@@ -551,7 +551,7 @@ local function load_full_data(self, dir_res, headers)
                 item.clean_handlers = {}
 
                 if self.filter then
-                    self.filter(item)
+                    self.filter(item, #values, self)
                 end
             end
 
@@ -695,9 +695,10 @@ local function sync_data(self)
             return false
         end
 
+        local pre_val
         local pre_index = self.values_hash[key]
         if pre_index then
-            local pre_val = self.values[pre_index]
+            pre_val = self.values[pre_index]
             if pre_val then
                 config_util.fire_all_clean_handlers(pre_val)
             end
@@ -755,7 +756,7 @@ local function sync_data(self)
         -- /plugins' filter need to known self.values when it is called
         -- so the filter should be called after self.values set.
         if self.filter then
-            self.filter(res)
+            self.filter(res, pre_val, self)
         end
 
         self.conf_version = self.conf_version + 1

apisix/http/route.lua

@@ -73,6 +73,7 @@ function _M.create_radixtree_uri_router(routes, uri_routes, with_parameter)
             core.log.info("insert uri route: ",
                           core.json.delay_encode(route.value, true))
             core.table.insert(uri_routes, {
+                id = route.value.id,
                 paths = route.value.uris or route.value.uri,
                 methods = route.value.methods,
                 priority = route.value.priority,
@@ -92,7 +93,11 @@ function _M.create_radixtree_uri_router(routes, uri_routes, with_parameter)
         end
     end
 
-    event.push(event.CONST.BUILD_ROUTER, routes)
+    local router_module = require("apisix.router")
+    if router_module.router_http then
+        event.push(event.CONST.BUILD_ROUTER, routes)
+    end
+
     core.log.info("route items: ", core.json.delay_encode(uri_routes, true))
 
     if with_parameter then

apisix/http/router/radixtree_host_uri.lua

@@ -20,6 +20,10 @@ local core = require("apisix.core")
 local event = require("apisix.core.event")
 local get_services = require("apisix.http.service").services
 local service_fetch = require("apisix.http.service").get
+local ar = require("apisix.router")
+local table = require("apisix.core.table")
+local json = require("apisix.core.json")
+local rdx = require("resty.radixtree")
 local ipairs = ipairs
 local type = type
 local tab_insert = table.insert
@@ -35,7 +39,7 @@ local _M = {version = 0.1}
 
 
 local function push_host_router(route, host_routes, only_uri_routes)
-    if type(route) ~= "table" then
+    if route == nil or (type(route) ~= "table" or  route.value == nil) then
         return
     end
 
@@ -70,6 +74,7 @@ local function push_host_router(route, host_routes, only_uri_routes)
     end
 
     local radixtree_route = {
+        id = route.value.id,
         paths = route.value.uris or route.value.uri,
         methods = route.value.methods,
         priority = route.value.priority,
@@ -90,14 +95,15 @@ local function push_host_router(route, host_routes, only_uri_routes)
         return
     end
 
-    for i, host in ipairs(hosts) do
-        local host_rev = host:reverse()
-        if not host_routes[host_rev] then
-            host_routes[host_rev] = {radixtree_route}
-        else
-            tab_insert(host_routes[host_rev], radixtree_route)
-        end
+    local rev = {}
+    for i, h in ipairs(hosts) do
+        tab_insert(rev, h:reverse())
     end
+
+    core.table.insert(host_routes, {
+        rev = rev,
+        route = radixtree_route
+    })
 end
 
 
@@ -115,13 +121,13 @@ local function create_radixtree_router(routes)
         end
     end
 
-    -- create router: host_router
     local host_router_routes = {}
-    for host_rev, routes in pairs(host_routes) do
-        local sub_router = router.new(routes)
+    for i, hr in ipairs(host_routes) do
+        local sub_router = router.new(hr["route"])
 
         core.table.insert(host_router_routes, {
-            paths = host_rev,
+            id = hr["route"]["id"],
+            paths = hr["rev"],
             filter_fun = function(vars, opts, ...)
                 return sub_router:dispatch(vars.uri, opts, ...)
             end,
@@ -143,14 +149,96 @@ local function create_radixtree_router(routes)
 end
 
 
+local function incremental_operate_radixtree(routes)
+    if ar.need_create_radixtree then
+        core.log.notice("create object of radixtree host uri after load_full_data or init.", #routes)
+        create_radixtree_router(routes)
+        ar.need_create_radixtree = false
+        table.clear(ar.sync_tb)
+        return
+    end
+
+    local op, route, last_route
+    local router_opts = {
+        no_param_match = true
+    }
+
+    event.push(event.CONST.BUILD_ROUTER, routes)
+    for k, _ in pairs(ar.sync_tb) do
+        op = ar.sync_tb[k]["op"]
+        route = ar.sync_tb[k]["outer_routeoute"]
+        last_route = ar.sync_tb[k]["last_route"]
+        local host_routes = {}
+        local only_uri_routes = {}
+        local last_host_routes = {}
+        local last_only_uri_routes = {}
+        local outer_route, last_outer_route = {paths={}}, {paths={}}
+        local inner_route, last_inner_route = {}, {}
+        local sub_router
+
+        push_host_router(route, host_routes, only_uri_routes)
+        push_host_router(last_route, last_host_routes, last_only_uri_routes)
+
+        if #host_routes > 0 or #last_host_routes > 0 then
+            if route then
+                if #host_routes > 0 then
+                    inner_route = host_routes[1]["route"]
+                end
+
+                if #last_host_routes > 0 then
+                    last_inner_route = last_host_routes[1]["route"]
+                    last_outer_route = {
+                        id = last_route.value.id,
+                        paths = last_host_routes[1]["rev"],
+                    }
+                end
+
+                sub_router = router.new(inner_route)
+
+                outer_route =  {
+                    id = route.value.id,
+                    paths = host_routes[1]["rev"],
+                    filter_fun = function(vars, opts, ...)
+                        return sub_router:dispatch(vars.uri, opts, ...)
+                    end,
+                    handler = function (api_ctx, match_opts)
+                        api_ctx.real_curr_req_matched_host = match_opts.matched._path
+                    end
+                }
+            end
+
+            core.log.notice("update routes watched from etcd into radixtree.", json.encode(route))
+            local err = host_router:update_route(last_outer_route, outer_route, router_opts)
+            if err ~= nil then
+                core.log.error("update a route into radixtree failed.", json.encode(route), err)
+                return
+            end
+        else
+            local cur_tmp = only_uri_routes[1] or {paths = {}}
+            local last_tmp = last_only_uri_routes[1] or {paths = {}}
+
+            core.log.notice("update routes watched from etcd into radixtree.", json.encode(route))
+            local err = only_uri_router:update_route(last_tmp, cur_tmp, router_opts)
+            if err ~= nil then
+                core.log.error("update a route into radixtree failed.", json.encode(route), err)
+                return
+            end
+        end
+
+        ar.sync_tb[k] = nil
+    end
+end
+
+
     local match_opts = {}
 function _M.match(api_ctx)
     local user_routes = _M.user_routes
     local _, service_version = get_services()
     if not cached_router_version or cached_router_version ~= user_routes.conf_version
         or not cached_service_version or cached_service_version ~= service_version
     then
-        create_radixtree_router(user_routes.values)
+        --create_radixtree_router(user_routes.values)
+        incremental_operate_radixtree(user_routes.values)
         cached_router_version = user_routes.conf_version
         cached_service_version = service_version
     end

apisix/http/router/radixtree_uri.lua

@@ -18,24 +18,128 @@ local require = require
 local core = require("apisix.core")
 local base_router = require("apisix.http.route")
 local get_services = require("apisix.http.service").services
+local ar = require("apisix.router")
+local json = require("apisix.core.json")
+local table = require("apisix.core.table")
+local event = require("apisix.core.event")
 local cached_router_version
 local cached_service_version
+local uri_routes = {}
+local uri_router
+local match_opts = {}
 
 
 local _M = {version = 0.2}
 
 
-    local uri_routes = {}
-    local uri_router
-    local match_opts = {}
+function _M.incremental_operate_radixtree(routes, no_param)
+    local sync_tb = ar.sync_tb
+    if ar.need_create_radixtree then
+        uri_router = base_router.create_radixtree_uri_router(routes, uri_routes, false)
+        ar.need_create_radixtree = false
+        table.clear(ar.sync_tb)
+        return
+    end
+
+    local op, route, last_route, err
+    local cur_tmp, last_tmp
+    local router_opts = {
+        no_param_match = no_param
+    }
+
+    event.push(event.CONST.BUILD_ROUTER, routes)
+    for k, v in pairs(ar.sync_tb) do
+        op = ar.sync_tb[k]["op"]
+        route = ar.sync_tb[k]["cur_route"]
+        last_route = ar.sync_tb[k]["last_route"]
+        cur_tmp = {}
+        last_tmp = {}
+
+        if route and route.value then
+            local status = table.try_read_attr(route, "value", "status")
+            if status and status == 0 then
+                goto CONTINUE
+            end
+
+            local filter_fun, err
+            if route.value.filter_func then
+                filter_fun, err = loadstring(
+                    "return " .. route.value.filter_func,
+                    "router#" .. route.value.id
+                )
+                if not filter_fun then
+                    core.log.error("failed to load filter function: ", err, " route id", route.value.id)
+                    return
+                end
+
+                filter_fun = filter_fun()
+            end
+
+            cur_tmp = {
+                id = route.value.id,
+                paths = route.value.uris or route.value.uri,
+                methods = route.value.methods,
+                priority = route.value.priority,
+                hosts = route.value.hosts or route.value.host,
+                remote_addrs = route.value.remote_addrs or route.value.remote_addr,
+                vars = route.value.vars,
+                filter_fun = filter_fun,
+                handler = function(api_ctx, match_opts)
+                    api_ctx.matched_params = nil
+                    api_ctx.matched_route = route
+                    api_ctx.curr_req_matched = match_opts.matched
+                end
+            }
+        end
+
+        if last_route and last_route.value then
+            last_tmp = {
+                id = last_route.value.id,
+                paths = last_route.value.uris or last_route.value.uri,
+                methods = last_route.value.methods,
+                priority = last_route.value.priority,
+                hosts = last_route.value.hosts or last_route.value.host,
+                remote_addrs = last_route.value.remote_addrs or last_route.value.remote_addr,
+                vars = last_route.value.vars
+            }
+        end
+
+        if op == "update" then
+            core.log.notice("update routes watched from etcd into radixtree.", json.encode(route))
+            err = uri_router:update_route(last_tmp, cur_tmp, router_opts)
+            if err ~= nil then
+                core.log.error("update a route into radixtree failed.", json.encode(route), err)
+                return
+            end
+        elseif op == "create" then
+            core.log.notice("create routes watched from etcd into radixtree.", json.encode(route))
+            err = uri_router:add_route(cur_tmp, router_opts)
+            if err ~= nil then
+                core.log.error("add routes into radixtree failed.", json.encode(route), err)
+                return
+            end
+        elseif op == "delete" then
+            core.log.notice("delete routes watched from etcd into radixtree.", json.encode(last_route))
+            err = uri_router:delete_route(last_tmp, router_opts)
+            if err ~= nil then
+                core.log.error("delete a route into radixtree failed.", json.encode(last_route), err)
+                return
+            end
+        end
+
+        ar.sync_tb[k] = nil
+        ::CONTINUE::
+    end
+end
+
+
 function _M.match(api_ctx)
     local user_routes = _M.user_routes
     local _, service_version = get_services()
     if not cached_router_version or cached_router_version ~= user_routes.conf_version
         or not cached_service_version or cached_service_version ~= service_version
     then
-        uri_router = base_router.create_radixtree_uri_router(user_routes.values,
-                                                             uri_routes, false)
+        _M.incremental_operate_radixtree(user_routes.values,true)
         cached_router_version = user_routes.conf_version
         cached_service_version = service_version
     end

apisix/http/router/radixtree_uri_with_parameter.lua

@@ -18,24 +18,30 @@ local require = require
 local core = require("apisix.core")
 local base_router = require("apisix.http.route")
 local get_services = require("apisix.http.service").services
+local apisix_router = require("apisix.router")
+local json = require("apisix.core.json")
+local table = require("apisix.core.table")
+local event = require("apisix.core.event")
+local radixtree_uri = requre("apisix.http.router.radixtree_uri")
 local cached_router_version
 local cached_service_version
+local uri_routes = {}
+local uri_router
+local match_opts = {}
 
 
 local _M = {}
 
 
-    local uri_routes = {}
-    local uri_router
-    local match_opts = {}
+
+
 function _M.match(api_ctx)
     local user_routes = _M.user_routes
     local _, service_version = get_services()
     if not cached_router_version or cached_router_version ~= user_routes.conf_version
         or not cached_service_version or cached_service_version ~= service_version
     then
-        uri_router = base_router.create_radixtree_uri_router(user_routes.values,
-                                                             uri_routes, true)
+        radixtree_uri.incremental_operate_radixtree(user_routes.values)
         cached_router_version = user_routes.conf_version
         cached_service_version = service_version
     end