diff --git a/README.markdown b/README.markdown index 21cafe689d..491c79c082 100644 --- a/README.markdown +++ b/README.markdown @@ -3041,6 +3041,11 @@ Nginx API for Lua * [ngx.shared.DICT.replace](#ngxshareddictreplace) * [ngx.shared.DICT.delete](#ngxshareddictdelete) * [ngx.shared.DICT.incr](#ngxshareddictincr) +* [ngx.shared.DICT.lpush](#ngxshareddictlpush) +* [ngx.shared.DICT.rpush](#ngxshareddictrpush) +* [ngx.shared.DICT.lpop](#ngxshareddictlpop) +* [ngx.shared.DICT.rpop](#ngxshareddictrpop) +* [ngx.shared.DICT.llen](#ngxshareddictllen) * [ngx.shared.DICT.flush_all](#ngxshareddictflush_all) * [ngx.shared.DICT.flush_expired](#ngxshareddictflush_expired) * [ngx.shared.DICT.get_keys](#ngxshareddictget_keys) @@ -6035,6 +6040,11 @@ The resulting object `dict` has the following methods: * [replace](#ngxshareddictreplace) * [delete](#ngxshareddictdelete) * [incr](#ngxshareddictincr) +* [lpush](#ngxshareddictlpush) +* [rpush](#ngxshareddictrpush) +* [lpop](#ngxshareddictlpop) +* [rpop](#ngxshareddictrpop) +* [llen](#ngxshareddictllen) * [flush_all](#ngxshareddictflush_all) * [flush_expired](#ngxshareddictflush_expired) * [get_keys](#ngxshareddictget_keys) @@ -6093,7 +6103,7 @@ ngx.shared.DICT.get **context:** *set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua** -Retrieving the value in the dictionary [ngx.shared.DICT](#ngxshareddict) for the key `key`. If the key does not exist or has been expired, then `nil` will be returned. +Retrieving the value in the dictionary [ngx.shared.DICT](#ngxshareddict) for the key `key`. If the key does not exist or has expired, then `nil` will be returned. In case of errors, `nil` and a string describing the error will be returned. @@ -6298,6 +6308,86 @@ See also [ngx.shared.DICT](#ngxshareddict). [Back to TOC](#nginx-api-for-lua) +ngx.shared.DICT.lpush +--------------------- +**syntax:** *length, err = ngx.shared.DICT:lpush(key, value)* + +**context:** *init_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.** + +Inserts the specified (numerical or string) `value` at the head of the list named `key` in the shm-based dictionary [ngx.shared.DICT](#ngxshareddict). Returns the number of elements in the list after the push operation. + +If `key` does not exist, it is created as an empty list before performing the push operations. When the `key` already takes a value that is not a list, it will return `nil` and `"value not a list"`. + +It never overrides the (least recently used) unexpired items in the store when running out of storage in the shared memory zone. In this case, it will immediately return `nil` and the string "no memory". + +This feature was first introduced in the `v0.*.*` release. + +See also [ngx.shared.DICT](#ngxshareddict). + +[Back to TOC](#nginx-api-for-lua) + +ngx.shared.DICT.rpush +--------------------- +**syntax:** *length, err = ngx.shared.DICT:rpush(key, value)* + +**context:** *init_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.** + +Similar to the [lpush](#ngxshareddictlpush) method, but inserts the specified (numerical or string) `value` at the tail of the list named `key`. + +This feature was first introduced in the `v0.*.*` release. + +See also [ngx.shared.DICT](#ngxshareddict). + +[Back to TOC](#nginx-api-for-lua) + +ngx.shared.DICT.lpop +-------------------- +**syntax:** *val, err = ngx.shared.DICT:lpop(key)* + +**context:** *init_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.** + +Removes and returns the first element of the list named `key` in the shm-based dictionary [ngx.shared.DICT](#ngxshareddict). + +If `key` does not exist, it will return `nil`. When the `key` already takes a value that is not a list, it will return `nil` and `"value not a list"`. + +This feature was first introduced in the `v0.*.*` release. + +See also [ngx.shared.DICT](#ngxshareddict). + +[Back to TOC](#nginx-api-for-lua) + +ngx.shared.DICT.rpop +-------------------- +**syntax:** *val, err = ngx.shared.DICT:rpop(key)* + +**context:** *init_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.** + +Removes and returns the last element of the list named `key` in the shm-based dictionary [ngx.shared.DICT](#ngxshareddict). + +If `key` does not exist, it will return `nil`. When the `key` already takes a value that is not a list, it will return `nil` and `"value not a list"`. + +This feature was first introduced in the `v0.*.*` release. + +See also [ngx.shared.DICT](#ngxshareddict). + +[Back to TOC](#nginx-api-for-lua) + +ngx.shared.DICT.llen +-------------------- +**syntax:** *len, err = ngx.shared.DICT:llen(key)* + +**context:** *init_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.** + +Returns the length of the list named `key` in the shm-based dictionary [ngx.shared.DICT](#ngxshareddict). + +If key does not exist, it is interpreted as an empty list and 0 is returned. When the `key` already takes a value that is not a list, it will return `nil` and `"value not a list"`. + +This feature was first introduced in the `v0.*.*` release. + +See also [ngx.shared.DICT](#ngxshareddict). + +[Back to TOC](#nginx-api-for-lua) + ngx.shared.DICT.flush_all ------------------------- **syntax:** *ngx.shared.DICT:flush_all()* diff --git a/doc/HttpLuaModule.wiki b/doc/HttpLuaModule.wiki index b6438eca3a..180d38fb4e 100644 --- a/doc/HttpLuaModule.wiki +++ b/doc/HttpLuaModule.wiki @@ -5053,6 +5053,11 @@ The resulting object dict has the following methods: * [[#ngx.shared.DICT.replace|replace]] * [[#ngx.shared.DICT.delete|delete]] * [[#ngx.shared.DICT.incr|incr]] +* [[#ngx.shared.DICT.lpush|lpush]] +* [[#ngx.shared.DICT.rpush|rpush]] +* [[#ngx.shared.DICT.lpop|lpop]] +* [[#ngx.shared.DICT.rpop|rpop]] +* [[#ngx.shared.DICT.llen|llen]] * [[#ngx.shared.DICT.flush_all|flush_all]] * [[#ngx.shared.DICT.flush_expired|flush_expired]] * [[#ngx.shared.DICT.get_keys|get_keys]] @@ -5106,7 +5111,7 @@ This feature was first introduced in the v0.3.1rc22 release. '''context:''' ''set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua*'' -Retrieving the value in the dictionary [[#ngx.shared.DICT|ngx.shared.DICT]] for the key key. If the key does not exist or has been expired, then nil will be returned. +Retrieving the value in the dictionary [[#ngx.shared.DICT|ngx.shared.DICT]] for the key key. If the key does not exist or has expired, then nil will be returned. In case of errors, nil and a string describing the error will be returned. @@ -5281,6 +5286,71 @@ The optional `init` parameter was first added in the v0.10.6 releas See also [[#ngx.shared.DICT|ngx.shared.DICT]]. +== ngx.shared.DICT.lpush == +'''syntax:''' ''length, err = ngx.shared.DICT:lpush(key, value)'' + +'''context:''' ''init_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*'' + +Inserts the specified (numerical or string) value at the head of the list named key in the shm-based dictionary [[#ngx.shared.DICT|ngx.shared.DICT]]. Returns the number of elements in the list after the push operation. + +If key does not exist, it is created as an empty list before performing the push operations. When the key already takes a value that is not a list, it will return nil and "value not a list". + +It never overrides the (least recently used) unexpired items in the store when running out of storage in the shared memory zone. In this case, it will immediately return nil and the string "no memory". + +This feature was first introduced in the v0.*.* release. + +See also [[#ngx.shared.DICT|ngx.shared.DICT]]. + +== ngx.shared.DICT.rpush == +'''syntax:''' ''length, err = ngx.shared.DICT:rpush(key, value)'' + +'''context:''' ''init_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*'' + +Similar to the [[#ngx.shared.DICT.lpush|lpush]] method, but inserts the specified (numerical or string) value at the tail of the list named key. + +This feature was first introduced in the v0.*.* release. + +See also [[#ngx.shared.DICT|ngx.shared.DICT]]. + +== ngx.shared.DICT.lpop == +'''syntax:''' ''val, err = ngx.shared.DICT:lpop(key)'' + +'''context:''' ''init_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*'' + +Removes and returns the first element of the list named key in the shm-based dictionary [[#ngx.shared.DICT|ngx.shared.DICT]]. + +If key does not exist, it will return nil. When the key already takes a value that is not a list, it will return nil and "value not a list". + +This feature was first introduced in the v0.*.* release. + +See also [[#ngx.shared.DICT|ngx.shared.DICT]]. + +== ngx.shared.DICT.rpop == +'''syntax:''' ''val, err = ngx.shared.DICT:rpop(key)'' + +'''context:''' ''init_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*'' + +Removes and returns the last element of the list named key in the shm-based dictionary [[#ngx.shared.DICT|ngx.shared.DICT]]. + +If key does not exist, it will return nil. When the key already takes a value that is not a list, it will return nil and "value not a list". + +This feature was first introduced in the v0.*.* release. + +See also [[#ngx.shared.DICT|ngx.shared.DICT]]. + +== ngx.shared.DICT.llen == +'''syntax:''' ''len, err = ngx.shared.DICT:llen(key)'' + +'''context:''' ''init_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*'' + +Returns the length of the list named key in the shm-based dictionary [[#ngx.shared.DICT|ngx.shared.DICT]]. + +If key does not exist, it is interpreted as an empty list and 0 is returned. When the key already takes a value that is not a list, it will return nil and "value not a list". + +This feature was first introduced in the v0.*.* release. + +See also [[#ngx.shared.DICT|ngx.shared.DICT]]. + == ngx.shared.DICT.flush_all == '''syntax:''' ''ngx.shared.DICT:flush_all()'' diff --git a/src/ngx_http_lua_shdict.c b/src/ngx_http_lua_shdict.c index e9974b4a49..b987f7b8d0 100644 --- a/src/ngx_http_lua_shdict.c +++ b/src/ngx_http_lua_shdict.c @@ -34,6 +34,13 @@ static int ngx_http_lua_shdict_delete(lua_State *L); static int ngx_http_lua_shdict_flush_all(lua_State *L); static int ngx_http_lua_shdict_flush_expired(lua_State *L); static int ngx_http_lua_shdict_get_keys(lua_State *L); +static int ngx_http_lua_shdict_lpush(lua_State *L); +static int ngx_http_lua_shdict_rpush(lua_State *L); +static int ngx_http_lua_shdict_push_helper(lua_State *L, int flags); +static int ngx_http_lua_shdict_lpop(lua_State *L); +static int ngx_http_lua_shdict_rpop(lua_State *L); +static int ngx_http_lua_shdict_pop_helper(lua_State *L, int flags); +static int ngx_http_lua_shdict_llen(lua_State *L); static ngx_inline ngx_shm_zone_t *ngx_http_lua_shdict_get_zone(lua_State *L, @@ -45,11 +52,32 @@ static ngx_inline ngx_shm_zone_t *ngx_http_lua_shdict_get_zone(lua_State *L, #define NGX_HTTP_LUA_SHDICT_SAFE_STORE 0x0004 +#define NGX_HTTP_LUA_SHDICT_LEFT 0x0001 +#define NGX_HTTP_LUA_SHDICT_RIGHT 0x0002 + + enum { SHDICT_USERDATA_INDEX = 1, }; +enum { + SHDICT_TNIL = 0, /* same as LUA_TNIL */ + SHDICT_TBOOLEAN = 1, /* same as LUA_TBOOLEAN */ + SHDICT_TNUMBER = 3, /* same as LUA_TNUMBER */ + SHDICT_TSTRING = 4, /* same as LUA_TSTRING */ + SHDICT_TLIST = 5, +}; + + +static ngx_inline ngx_queue_t * +ngx_http_lua_shdict_get_list_head(ngx_http_lua_shdict_node_t *sd, u_short len) +{ + return (ngx_queue_t *) ngx_align_ptr(((u_char *) &sd->data + len), + NGX_ALIGNMENT); +} + + ngx_int_t ngx_http_lua_shdict_init_zone(ngx_shm_zone_t *shm_zone, void *data) { @@ -90,7 +118,7 @@ ngx_http_lua_shdict_init_zone(ngx_shm_zone_t *shm_zone, void *data) ngx_rbtree_init(&ctx->sh->rbtree, &ctx->sh->sentinel, ngx_http_lua_shdict_rbtree_insert_value); - ngx_queue_init(&ctx->sh->queue); + ngx_queue_init(&ctx->sh->lru_queue); len = sizeof(" in lua_shared_dict zone \"\"") + shm_zone->shm.name.len; @@ -214,7 +242,7 @@ ngx_http_lua_shdict_lookup(ngx_shm_zone_t *shm_zone, ngx_uint_t hash, if (rc == 0) { ngx_queue_remove(&sd->queue); - ngx_queue_insert_head(&ctx->sh->queue, &sd->queue); + ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue); *sdp = sd; @@ -249,13 +277,14 @@ ngx_http_lua_shdict_lookup(ngx_shm_zone_t *shm_zone, ngx_uint_t hash, static int ngx_http_lua_shdict_expire(ngx_http_lua_shdict_ctx_t *ctx, ngx_uint_t n) { - ngx_time_t *tp; - uint64_t now; - ngx_queue_t *q; - int64_t ms; - ngx_rbtree_node_t *node; - ngx_http_lua_shdict_node_t *sd; - int freed = 0; + ngx_time_t *tp; + uint64_t now; + ngx_queue_t *q, *list_queue, *lq; + int64_t ms; + ngx_rbtree_node_t *node; + ngx_http_lua_shdict_node_t *sd; + int freed = 0; + ngx_http_lua_shdict_list_node_t *lnode; tp = ngx_timeofday(); @@ -269,11 +298,11 @@ ngx_http_lua_shdict_expire(ngx_http_lua_shdict_ctx_t *ctx, ngx_uint_t n) while (n < 3) { - if (ngx_queue_empty(&ctx->sh->queue)) { + if (ngx_queue_empty(&ctx->sh->lru_queue)) { return freed; } - q = ngx_queue_last(&ctx->sh->queue); + q = ngx_queue_last(&ctx->sh->lru_queue); sd = ngx_queue_data(q, ngx_http_lua_shdict_node_t, queue); @@ -289,6 +318,20 @@ ngx_http_lua_shdict_expire(ngx_http_lua_shdict_ctx_t *ctx, ngx_uint_t n) } } + if (sd->value_type == SHDICT_TLIST) { + list_queue = ngx_http_lua_shdict_get_list_head(sd, sd->key_len); + + for (lq = ngx_queue_head(list_queue); + lq != ngx_queue_sentinel(list_queue); + lq = ngx_queue_next(lq)) + { + lnode = ngx_queue_data(lq, ngx_http_lua_shdict_list_node_t, + queue); + + ngx_slab_free_locked(ctx->shpool, lnode); + } + } + ngx_queue_remove(q); node = (ngx_rbtree_node_t *) @@ -316,7 +359,7 @@ ngx_http_lua_inject_shdict_api(ngx_http_lua_main_conf_t *lmcf, lua_State *L) lua_createtable(L, 0, lmcf->shm_zones->nelts /* nrec */); /* ngx.shared */ - lua_createtable(L, 0 /* narr */, 13 /* nrec */); /* shared mt */ + lua_createtable(L, 0 /* narr */, 18 /* nrec */); /* shared mt */ lua_pushcfunction(L, ngx_http_lua_shdict_get); lua_setfield(L, -2, "get"); @@ -345,6 +388,21 @@ ngx_http_lua_inject_shdict_api(ngx_http_lua_main_conf_t *lmcf, lua_State *L) lua_pushcfunction(L, ngx_http_lua_shdict_delete); lua_setfield(L, -2, "delete"); + lua_pushcfunction(L, ngx_http_lua_shdict_lpush); + lua_setfield(L, -2, "lpush"); + + lua_pushcfunction(L, ngx_http_lua_shdict_rpush); + lua_setfield(L, -2, "rpush"); + + lua_pushcfunction(L, ngx_http_lua_shdict_lpop); + lua_setfield(L, -2, "lpop"); + + lua_pushcfunction(L, ngx_http_lua_shdict_rpop); + lua_setfield(L, -2, "rpop"); + + lua_pushcfunction(L, ngx_http_lua_shdict_llen); + lua_setfield(L, -2, "llen"); + lua_pushcfunction(L, ngx_http_lua_shdict_flush_all); lua_setfield(L, -2, "flush_all"); @@ -503,12 +561,13 @@ ngx_http_lua_shdict_get_helper(lua_State *L, int get_stale) value.len = (size_t) sd->value_len; switch (value_type) { - case LUA_TSTRING: + + case SHDICT_TSTRING: lua_pushlstring(L, (char *) value.data, value.len); break; - case LUA_TNUMBER: + case SHDICT_TNUMBER: if (value.len != sizeof(double)) { @@ -524,7 +583,7 @@ ngx_http_lua_shdict_get_helper(lua_State *L, int get_stale) lua_pushnumber(L, num); break; - case LUA_TBOOLEAN: + case SHDICT_TBOOLEAN: if (value.len != sizeof(u_char)) { @@ -540,6 +599,14 @@ ngx_http_lua_shdict_get_helper(lua_State *L, int get_stale) lua_pushboolean(L, c ? 1 : 0); break; + case SHDICT_TLIST: + + ngx_shmtx_unlock(&ctx->shpool->mutex); + + lua_pushnil(L); + lua_pushliteral(L, "value is a list"); + return 2; + default: ngx_shmtx_unlock(&ctx->shpool->mutex); @@ -621,8 +688,8 @@ ngx_http_lua_shdict_flush_all(lua_State *L) ngx_shmtx_lock(&ctx->shpool->mutex); - for (q = ngx_queue_head(&ctx->sh->queue); - q != ngx_queue_sentinel(&ctx->sh->queue); + for (q = ngx_queue_head(&ctx->sh->lru_queue); + q != ngx_queue_sentinel(&ctx->sh->lru_queue); q = ngx_queue_next(q)) { sd = ngx_queue_data(q, ngx_http_lua_shdict_node_t, queue); @@ -640,16 +707,17 @@ ngx_http_lua_shdict_flush_all(lua_State *L) static int ngx_http_lua_shdict_flush_expired(lua_State *L) { - ngx_queue_t *q, *prev; - ngx_http_lua_shdict_node_t *sd; - ngx_http_lua_shdict_ctx_t *ctx; - ngx_shm_zone_t *zone; - ngx_time_t *tp; - int freed = 0; - int attempts = 0; - ngx_rbtree_node_t *node; - uint64_t now; - int n; + ngx_queue_t *q, *prev, *list_queue, *lq; + ngx_http_lua_shdict_node_t *sd; + ngx_http_lua_shdict_ctx_t *ctx; + ngx_shm_zone_t *zone; + ngx_time_t *tp; + int freed = 0; + int attempts = 0; + ngx_rbtree_node_t *node; + uint64_t now; + int n; + ngx_http_lua_shdict_list_node_t *lnode; n = lua_gettop(L); @@ -672,7 +740,7 @@ ngx_http_lua_shdict_flush_expired(lua_State *L) ngx_shmtx_lock(&ctx->shpool->mutex); - if (ngx_queue_empty(&ctx->sh->queue)) { + if (ngx_queue_empty(&ctx->sh->lru_queue)) { ngx_shmtx_unlock(&ctx->shpool->mutex); lua_pushnumber(L, 0); return 1; @@ -682,14 +750,29 @@ ngx_http_lua_shdict_flush_expired(lua_State *L) now = (uint64_t) tp->sec * 1000 + tp->msec; - q = ngx_queue_last(&ctx->sh->queue); + q = ngx_queue_last(&ctx->sh->lru_queue); - while (q != ngx_queue_sentinel(&ctx->sh->queue)) { + while (q != ngx_queue_sentinel(&ctx->sh->lru_queue)) { prev = ngx_queue_prev(q); sd = ngx_queue_data(q, ngx_http_lua_shdict_node_t, queue); if (sd->expires != 0 && sd->expires <= now) { + + if (sd->value_type == SHDICT_TLIST) { + list_queue = ngx_http_lua_shdict_get_list_head(sd, sd->key_len); + + for (lq = ngx_queue_head(list_queue); + lq != ngx_queue_sentinel(list_queue); + lq = ngx_queue_next(lq)) + { + lnode = ngx_queue_data(lq, ngx_http_lua_shdict_list_node_t, + queue); + + ngx_slab_free_locked(ctx->shpool, lnode); + } + } + ngx_queue_remove(q); node = (ngx_rbtree_node_t *) @@ -753,7 +836,7 @@ ngx_http_lua_shdict_get_keys(lua_State *L) ngx_shmtx_lock(&ctx->shpool->mutex); - if (ngx_queue_empty(&ctx->sh->queue)) { + if (ngx_queue_empty(&ctx->sh->lru_queue)) { ngx_shmtx_unlock(&ctx->shpool->mutex); lua_createtable(L, 0, 0); return 1; @@ -765,9 +848,9 @@ ngx_http_lua_shdict_get_keys(lua_State *L) /* first run through: get total number of elements we need to allocate */ - q = ngx_queue_last(&ctx->sh->queue); + q = ngx_queue_last(&ctx->sh->lru_queue); - while (q != ngx_queue_sentinel(&ctx->sh->queue)) { + while (q != ngx_queue_sentinel(&ctx->sh->lru_queue)) { prev = ngx_queue_prev(q); sd = ngx_queue_data(q, ngx_http_lua_shdict_node_t, queue); @@ -787,9 +870,9 @@ ngx_http_lua_shdict_get_keys(lua_State *L) /* second run through: add keys to table */ total = 0; - q = ngx_queue_last(&ctx->sh->queue); + q = ngx_queue_last(&ctx->sh->lru_queue); - while (q != ngx_queue_sentinel(&ctx->sh->queue)) { + while (q != ngx_queue_sentinel(&ctx->sh->lru_queue)) { prev = ngx_queue_prev(q); sd = ngx_queue_data(q, ngx_http_lua_shdict_node_t, queue); @@ -870,6 +953,7 @@ ngx_http_lua_shdict_set_helper(lua_State *L, int flags) /* indicates whether to foricibly override other * valid entries */ int32_t user_flags = 0; + ngx_queue_t *queue, *q; n = lua_gettop(L); @@ -914,17 +998,18 @@ ngx_http_lua_shdict_set_helper(lua_State *L, int flags) value_type = lua_type(L, 3); switch (value_type) { - case LUA_TSTRING: + + case SHDICT_TSTRING: value.data = (u_char *) lua_tolstring(L, 3, &value.len); break; - case LUA_TNUMBER: + case SHDICT_TNUMBER: value.len = sizeof(double); num = lua_tonumber(L, 3); value.data = (u_char *) # break; - case LUA_TBOOLEAN: + case SHDICT_TBOOLEAN: value.len = sizeof(u_char); c = lua_toboolean(L, 3) ? 1 : 0; value.data = &c; @@ -1015,14 +1100,17 @@ ngx_http_lua_shdict_set_helper(lua_State *L, int flags) replace: - if (value.data && value.len == (size_t) sd->value_len) { + if (value.data + && value.len == (size_t) sd->value_len + && sd->value_type != SHDICT_TLIST) + { ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0, "lua shared dict set: found old entry and value " "size matched, reusing it"); ngx_queue_remove(&sd->queue); - ngx_queue_insert_head(&ctx->sh->queue, &sd->queue); + ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue); sd->key_len = (u_short) key.len; @@ -1060,6 +1148,21 @@ ngx_http_lua_shdict_set_helper(lua_State *L, int flags) remove: + if (sd->value_type == SHDICT_TLIST) { + queue = ngx_http_lua_shdict_get_list_head(sd, key.len); + + for (q = ngx_queue_head(queue); + q != ngx_queue_sentinel(queue); + q = ngx_queue_next(q)) + { + p = (u_char *) ngx_queue_data(q, + ngx_http_lua_shdict_list_node_t, + queue); + + ngx_slab_free_locked(ctx->shpool, p); + } + } + ngx_queue_remove(&sd->queue); node = (ngx_rbtree_node_t *) @@ -1158,7 +1261,7 @@ ngx_http_lua_shdict_set_helper(lua_State *L, int flags) ngx_rbtree_insert(&ctx->sh->rbtree, node); - ngx_queue_insert_head(&ctx->sh->queue, &sd->queue); + ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue); ngx_shmtx_unlock(&ctx->shpool->mutex); @@ -1269,7 +1372,7 @@ ngx_http_lua_shdict_incr(lua_State *L) "value size matched, reusing it"); ngx_queue_remove(&sd->queue); - ngx_queue_insert_head(&ctx->sh->queue, &sd->queue); + ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue); dd("go to setvalue"); goto setvalue; @@ -1285,7 +1388,7 @@ ngx_http_lua_shdict_incr(lua_State *L) /* rc == NGX_OK */ - if (sd->value_type != LUA_TNUMBER || sd->value_len != sizeof(double)) { + if (sd->value_type != SHDICT_TNUMBER || sd->value_len != sizeof(double)) { ngx_shmtx_unlock(&ctx->shpool->mutex); lua_pushnil(L); @@ -1294,7 +1397,7 @@ ngx_http_lua_shdict_incr(lua_State *L) } ngx_queue_remove(&sd->queue); - ngx_queue_insert_head(&ctx->sh->queue, &sd->queue); + ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue); dd("setting value type to %d", (int) sd->value_type); @@ -1377,7 +1480,7 @@ ngx_http_lua_shdict_incr(lua_State *L) ngx_rbtree_insert(&ctx->sh->rbtree, node); - ngx_queue_insert_head(&ctx->sh->queue, &sd->queue); + ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue); setvalue: @@ -1442,7 +1545,8 @@ ngx_http_lua_shared_dict_get(ngx_shm_zone_t *zone, u_char *key_data, len = (size_t) sd->value_len; switch (value->type) { - case LUA_TSTRING: + + case SHDICT_TSTRING: if (value->value.s.data == NULL || value->value.s.len == 0) { ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0, "no string buffer " @@ -1461,7 +1565,7 @@ ngx_http_lua_shared_dict_get(ngx_shm_zone_t *zone, u_char *key_data, ngx_memcpy(value->value.s.data, data, len); break; - case LUA_TNUMBER: + case SHDICT_TNUMBER: if (len != sizeof(double)) { ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0, "bad lua number " @@ -1475,7 +1579,7 @@ ngx_http_lua_shared_dict_get(ngx_shm_zone_t *zone, u_char *key_data, ngx_memcpy(&value->value.b, data, len); break; - case LUA_TBOOLEAN: + case SHDICT_TBOOLEAN: if (len != sizeof(u_char)) { ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0, "bad lua boolean " @@ -1503,6 +1607,574 @@ ngx_http_lua_shared_dict_get(ngx_shm_zone_t *zone, u_char *key_data, } +static int +ngx_http_lua_shdict_lpush(lua_State *L) +{ + return ngx_http_lua_shdict_push_helper(L, NGX_HTTP_LUA_SHDICT_LEFT); +} + + +static int +ngx_http_lua_shdict_rpush(lua_State *L) +{ + return ngx_http_lua_shdict_push_helper(L, NGX_HTTP_LUA_SHDICT_RIGHT); +} + + +static int +ngx_http_lua_shdict_push_helper(lua_State *L, int flags) +{ + int n; + ngx_str_t key; + uint32_t hash; + ngx_int_t rc; + ngx_http_lua_shdict_ctx_t *ctx; + ngx_http_lua_shdict_node_t *sd; + ngx_str_t value; + int value_type; + double num; + ngx_rbtree_node_t *node; + ngx_shm_zone_t *zone; + ngx_queue_t *queue, *q; + ngx_http_lua_shdict_list_node_t *lnode; + + n = lua_gettop(L); + + if (n != 3) { + return luaL_error(L, "expecting 3 arguments, " + "but only seen %d", n); + } + + if (lua_type(L, 1) != LUA_TTABLE) { + return luaL_error(L, "bad \"zone\" argument"); + } + + zone = ngx_http_lua_shdict_get_zone(L, 1); + if (zone == NULL) { + return luaL_error(L, "bad \"zone\" argument"); + } + + ctx = zone->data; + + if (lua_isnil(L, 2)) { + lua_pushnil(L); + lua_pushliteral(L, "nil key"); + return 2; + } + + key.data = (u_char *) luaL_checklstring(L, 2, &key.len); + + if (key.len == 0) { + lua_pushnil(L); + lua_pushliteral(L, "empty key"); + return 2; + } + + if (key.len > 65535) { + lua_pushnil(L); + lua_pushliteral(L, "key too long"); + return 2; + } + + hash = ngx_crc32_short(key.data, key.len); + + value_type = lua_type(L, 3); + + switch (value_type) { + + case SHDICT_TSTRING: + value.data = (u_char *) lua_tolstring(L, 3, &value.len); + break; + + case SHDICT_TNUMBER: + value.len = sizeof(double); + num = lua_tonumber(L, 3); + value.data = (u_char *) # + break; + + default: + lua_pushnil(L); + lua_pushliteral(L, "bad value type"); + return 2; + } + + ngx_shmtx_lock(&ctx->shpool->mutex); + +#if 1 + ngx_http_lua_shdict_expire(ctx, 1); +#endif + + rc = ngx_http_lua_shdict_lookup(zone, hash, key.data, key.len, &sd); + + dd("shdict lookup returned %d", (int) rc); + + /* exists but expired */ + + if (rc == NGX_DONE) { + + if (sd->value_type != SHDICT_TLIST) { + /* TODO: reuse when length matched */ + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0, + "lua shared dict push: found old entry and value " + "type not matched, remove it first"); + + ngx_queue_remove(&sd->queue); + + node = (ngx_rbtree_node_t *) + ((u_char *) sd - offsetof(ngx_rbtree_node_t, color)); + + ngx_rbtree_delete(&ctx->sh->rbtree, node); + + ngx_slab_free_locked(ctx->shpool, node); + + dd("go to init_list"); + goto init_list; + } + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0, + "lua shared dict push: found old entry and value " + "type matched, reusing it"); + + sd->expires = 0; + + /* free list nodes */ + + queue = ngx_http_lua_shdict_get_list_head(sd, key.len); + + for (q = ngx_queue_head(queue); + q != ngx_queue_sentinel(queue); + q = ngx_queue_next(q)) + { + /* TODO: reuse matched size list node */ + lnode = ngx_queue_data(q, ngx_http_lua_shdict_list_node_t, queue); + ngx_slab_free_locked(ctx->shpool, lnode); + } + + ngx_queue_init(queue); + + ngx_queue_remove(&sd->queue); + ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue); + + dd("go to push_node"); + goto push_node; + } + + /* exists and not expired */ + + if (rc == NGX_OK) { + + if (sd->value_type != SHDICT_TLIST) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + + lua_pushnil(L); + lua_pushliteral(L, "value not a list"); + return 2; + } + + queue = ngx_http_lua_shdict_get_list_head(sd, key.len); + + ngx_queue_remove(&sd->queue); + ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue); + + dd("go to push_node"); + goto push_node; + } + + /* rc == NGX_DECLINED, not found */ + +init_list: + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0, + "lua shared dict list: creating a new entry"); + + /* NOTICE: we assume the begin point aligned in slab, be careful */ + n = offsetof(ngx_rbtree_node_t, color) + + offsetof(ngx_http_lua_shdict_node_t, data) + + key.len + + sizeof(ngx_queue_t); + + dd("length before aligned: %d", n); + + n = (int) (uintptr_t) ngx_align_ptr(n, NGX_ALIGNMENT); + + dd("length after aligned: %d", n); + + node = ngx_slab_alloc_locked(ctx->shpool, n); + + if (node == NULL) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + + lua_pushboolean(L, 0); + lua_pushliteral(L, "no memory"); + return 2; + } + + sd = (ngx_http_lua_shdict_node_t *) &node->color; + + queue = ngx_http_lua_shdict_get_list_head(sd, key.len); + + node->key = hash; + sd->key_len = (u_short) key.len; + + sd->expires = 0; + + sd->value_len = 0; + + dd("setting value type to %d", (int) SHDICT_TLIST); + + sd->value_type = (uint8_t) SHDICT_TLIST; + + ngx_memcpy(sd->data, key.data, key.len); + + ngx_queue_init(queue); + + ngx_rbtree_insert(&ctx->sh->rbtree, node); + + ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue); + +push_node: + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0, + "lua shared dict list: creating a new list node"); + + n = offsetof(ngx_http_lua_shdict_list_node_t, data) + + value.len; + + dd("list node length: %d", n); + + lnode = ngx_slab_alloc_locked(ctx->shpool, n); + + if (lnode == NULL) { + + if (sd->value_len == 0) { + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0, + "lua shared dict list: no memory for create" + " list node and list empty, remove it"); + + ngx_queue_remove(&sd->queue); + + node = (ngx_rbtree_node_t *) + ((u_char *) sd - offsetof(ngx_rbtree_node_t, color)); + + ngx_rbtree_delete(&ctx->sh->rbtree, node); + + ngx_slab_free_locked(ctx->shpool, node); + } + + ngx_shmtx_unlock(&ctx->shpool->mutex); + + lua_pushboolean(L, 0); + lua_pushliteral(L, "no memory"); + return 2; + } + + dd("setting list length to %d", sd->value_len + 1); + + sd->value_len = sd->value_len + 1; + + dd("setting list node value length to %d", (int) value.len); + + lnode->value_len = (uint32_t) value.len; + + dd("setting list node value type to %d", value_type); + + lnode->value_type = (uint8_t) value_type; + + ngx_memcpy(lnode->data, value.data, value.len); + + if (flags == NGX_HTTP_LUA_SHDICT_LEFT) { + ngx_queue_insert_head(queue, &lnode->queue); + + } else { + ngx_queue_insert_tail(queue, &lnode->queue); + } + + ngx_shmtx_unlock(&ctx->shpool->mutex); + + lua_pushnumber(L, sd->value_len); + return 1; +} + + +static int +ngx_http_lua_shdict_lpop(lua_State *L) +{ + return ngx_http_lua_shdict_pop_helper(L, NGX_HTTP_LUA_SHDICT_LEFT); +} + + +static int +ngx_http_lua_shdict_rpop(lua_State *L) +{ + return ngx_http_lua_shdict_pop_helper(L, NGX_HTTP_LUA_SHDICT_RIGHT); +} + + +static int +ngx_http_lua_shdict_pop_helper(lua_State *L, int flags) +{ + int n; + ngx_str_t name; + ngx_str_t key; + uint32_t hash; + ngx_int_t rc; + ngx_http_lua_shdict_ctx_t *ctx; + ngx_http_lua_shdict_node_t *sd; + ngx_str_t value; + int value_type; + double num; + ngx_rbtree_node_t *node; + ngx_shm_zone_t *zone; + ngx_queue_t *queue; + ngx_http_lua_shdict_list_node_t *lnode; + + n = lua_gettop(L); + + if (n != 2) { + return luaL_error(L, "expecting 2 arguments, " + "but only seen %d", n); + } + + if (lua_type(L, 1) != LUA_TTABLE) { + return luaL_error(L, "bad \"zone\" argument"); + } + + zone = ngx_http_lua_shdict_get_zone(L, 1); + if (zone == NULL) { + return luaL_error(L, "bad \"zone\" argument"); + } + + ctx = zone->data; + name = ctx->name; + + if (lua_isnil(L, 2)) { + lua_pushnil(L); + lua_pushliteral(L, "nil key"); + return 2; + } + + key.data = (u_char *) luaL_checklstring(L, 2, &key.len); + + if (key.len == 0) { + lua_pushnil(L); + lua_pushliteral(L, "empty key"); + return 2; + } + + if (key.len > 65535) { + lua_pushnil(L); + lua_pushliteral(L, "key too long"); + return 2; + } + + hash = ngx_crc32_short(key.data, key.len); + + ngx_shmtx_lock(&ctx->shpool->mutex); + +#if 1 + ngx_http_lua_shdict_expire(ctx, 1); +#endif + + rc = ngx_http_lua_shdict_lookup(zone, hash, key.data, key.len, &sd); + + dd("shdict lookup returned %d", (int) rc); + + if (rc == NGX_DECLINED || rc == NGX_DONE) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + lua_pushnil(L); + return 1; + } + + /* rc == NGX_OK */ + + if (sd->value_type != SHDICT_TLIST) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + + lua_pushnil(L); + lua_pushliteral(L, "value not a list"); + return 2; + } + + if (sd->value_len <= 0) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + + return luaL_error(L, "bad lua list length found for key %s " + "in shared_dict %s: %lu", key.data, name.data, + (unsigned long) sd->value_len); + } + + queue = ngx_http_lua_shdict_get_list_head(sd, key.len); + + if (flags == NGX_HTTP_LUA_SHDICT_LEFT) { + queue = ngx_queue_head(queue); + + } else { + queue = ngx_queue_last(queue); + } + + lnode = ngx_queue_data(queue, ngx_http_lua_shdict_list_node_t, queue); + + value_type = lnode->value_type; + + dd("data: %p", lnode->data); + dd("value len: %d", (int) sd->value_len); + + value.data = lnode->data; + value.len = (size_t) lnode->value_len; + + switch (value_type) { + + case SHDICT_TSTRING: + + lua_pushlstring(L, (char *) value.data, value.len); + break; + + case SHDICT_TNUMBER: + + if (value.len != sizeof(double)) { + + ngx_shmtx_unlock(&ctx->shpool->mutex); + + return luaL_error(L, "bad lua list node number value size found " + "for key %s in shared_dict %s: %lu", key.data, + name.data, (unsigned long) value.len); + } + + ngx_memcpy(&num, value.data, sizeof(double)); + + lua_pushnumber(L, num); + break; + + default: + + ngx_shmtx_unlock(&ctx->shpool->mutex); + + return luaL_error(L, "bad list node value type found for key %s in " + "shared_dict %s: %d", key.data, name.data, + value_type); + } + + ngx_queue_remove(queue); + + ngx_slab_free_locked(ctx->shpool, lnode); + + if (sd->value_len == 1) { + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0, + "lua shared dict list: empty node after pop, " + "remove it"); + + ngx_queue_remove(&sd->queue); + + node = (ngx_rbtree_node_t *) + ((u_char *) sd - offsetof(ngx_rbtree_node_t, color)); + + ngx_rbtree_delete(&ctx->sh->rbtree, node); + + ngx_slab_free_locked(ctx->shpool, node); + + } else { + sd->value_len = sd->value_len - 1; + + ngx_queue_remove(&sd->queue); + ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue); + } + + ngx_shmtx_unlock(&ctx->shpool->mutex); + + return 1; +} + + +static int +ngx_http_lua_shdict_llen(lua_State *L) +{ + int n; + ngx_str_t key; + uint32_t hash; + ngx_int_t rc; + ngx_http_lua_shdict_ctx_t *ctx; + ngx_http_lua_shdict_node_t *sd; + ngx_shm_zone_t *zone; + + n = lua_gettop(L); + + if (n != 2) { + return luaL_error(L, "expecting 2 arguments, " + "but only seen %d", n); + } + + if (lua_type(L, 1) != LUA_TTABLE) { + return luaL_error(L, "bad \"zone\" argument"); + } + + zone = ngx_http_lua_shdict_get_zone(L, 1); + if (zone == NULL) { + return luaL_error(L, "bad \"zone\" argument"); + } + + ctx = zone->data; + + if (lua_isnil(L, 2)) { + lua_pushnil(L); + lua_pushliteral(L, "nil key"); + return 2; + } + + key.data = (u_char *) luaL_checklstring(L, 2, &key.len); + + if (key.len == 0) { + lua_pushnil(L); + lua_pushliteral(L, "empty key"); + return 2; + } + + if (key.len > 65535) { + lua_pushnil(L); + lua_pushliteral(L, "key too long"); + return 2; + } + + hash = ngx_crc32_short(key.data, key.len); + + ngx_shmtx_lock(&ctx->shpool->mutex); + +#if 1 + ngx_http_lua_shdict_expire(ctx, 1); +#endif + + rc = ngx_http_lua_shdict_lookup(zone, hash, key.data, key.len, &sd); + + dd("shdict lookup returned %d", (int) rc); + + if (rc == NGX_OK) { + + if (sd->value_type != SHDICT_TLIST) { + ngx_shmtx_unlock(&ctx->shpool->mutex); + + lua_pushnil(L); + lua_pushliteral(L, "value not a list"); + return 2; + } + + ngx_queue_remove(&sd->queue); + ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue); + + ngx_shmtx_unlock(&ctx->shpool->mutex); + + lua_pushnumber(L, (lua_Number) sd->value_len); + return 1; + } + + ngx_shmtx_unlock(&ctx->shpool->mutex); + + lua_pushnumber(L, 0); + return 1; +} + + ngx_shm_zone_t * ngx_http_lua_find_zone(u_char *name_data, size_t name_len) { @@ -1554,6 +2226,7 @@ ngx_http_lua_ffi_shdict_store(ngx_shm_zone_t *zone, int op, u_char *key, uint32_t hash; ngx_int_t rc; ngx_time_t *tp; + ngx_queue_t *queue, *q; ngx_rbtree_node_t *node; ngx_http_lua_shdict_ctx_t *ctx; ngx_http_lua_shdict_node_t *sd; @@ -1571,17 +2244,18 @@ ngx_http_lua_ffi_shdict_store(ngx_shm_zone_t *zone, int op, u_char *key, hash = ngx_crc32_short(key, key_len); switch (value_type) { - case LUA_TSTRING: + + case SHDICT_TSTRING: /* do nothing */ break; - case LUA_TNUMBER: + case SHDICT_TNUMBER: dd("num value: %lf", num_value); str_value_buf = (u_char *) &num_value; str_value_len = sizeof(double); break; - case LUA_TBOOLEAN: + case SHDICT_TBOOLEAN: c = num_value ? 1 : 0; str_value_buf = &c; str_value_len = sizeof(u_char); @@ -1619,6 +2293,7 @@ ngx_http_lua_ffi_shdict_store(ngx_shm_zone_t *zone, int op, u_char *key, *errmsg = "not found"; return NGX_DECLINED; } + /* rc == NGX_OK */ goto replace; @@ -1653,14 +2328,17 @@ ngx_http_lua_ffi_shdict_store(ngx_shm_zone_t *zone, int op, u_char *key, replace: - if (str_value_buf && str_value_len == (size_t) sd->value_len) { + if (str_value_buf + && str_value_len == (size_t) sd->value_len + && sd->value_type != SHDICT_TLIST) + { ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0, "lua shared dict set: found old entry and value " "size matched, reusing it"); ngx_queue_remove(&sd->queue); - ngx_queue_insert_head(&ctx->sh->queue, &sd->queue); + ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue); sd->key_len = (u_short) key_len; @@ -1695,6 +2373,21 @@ ngx_http_lua_ffi_shdict_store(ngx_shm_zone_t *zone, int op, u_char *key, remove: + if (sd->value_type == SHDICT_TLIST) { + queue = ngx_http_lua_shdict_get_list_head(sd, key_len); + + for (q = ngx_queue_head(queue); + q != ngx_queue_sentinel(queue); + q = ngx_queue_next(q)) + { + p = (u_char *) ngx_queue_data(q, + ngx_http_lua_shdict_list_node_t, + queue); + + ngx_slab_free_locked(ctx->shpool, p); + } + } + ngx_queue_remove(&sd->queue); node = (ngx_rbtree_node_t *) @@ -1783,7 +2476,7 @@ ngx_http_lua_ffi_shdict_store(ngx_shm_zone_t *zone, int op, u_char *key, ngx_memcpy(p, str_value_buf, str_value_len); ngx_rbtree_insert(&ctx->sh->rbtree, node); - ngx_queue_insert_head(&ctx->sh->queue, &sd->queue); + ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue); ngx_shmtx_unlock(&ctx->shpool->mutex); return NGX_OK; @@ -1847,12 +2540,12 @@ ngx_http_lua_ffi_shdict_get(ngx_shm_zone_t *zone, u_char *key, value.len = (size_t) sd->value_len; if (*str_value_len < (size_t) value.len) { - if (*value_type == LUA_TBOOLEAN) { + if (*value_type == SHDICT_TBOOLEAN) { ngx_shmtx_unlock(&ctx->shpool->mutex); return NGX_ERROR; } - if (*value_type == LUA_TSTRING) { + if (*value_type == SHDICT_TSTRING) { *str_value_buf = malloc(value.len); if (*str_value_buf == NULL) { ngx_shmtx_unlock(&ctx->shpool->mutex); @@ -1862,12 +2555,13 @@ ngx_http_lua_ffi_shdict_get(ngx_shm_zone_t *zone, u_char *key, } switch (*value_type) { - case LUA_TSTRING: + + case SHDICT_TSTRING: *str_value_len = value.len; ngx_memcpy(*str_value_buf, value.data, value.len); break; - case LUA_TNUMBER: + case SHDICT_TNUMBER: if (value.len != sizeof(double)) { ngx_shmtx_unlock(&ctx->shpool->mutex); @@ -1882,7 +2576,7 @@ ngx_http_lua_ffi_shdict_get(ngx_shm_zone_t *zone, u_char *key, ngx_memcpy(num_value, value.data, sizeof(double)); break; - case LUA_TBOOLEAN: + case SHDICT_TBOOLEAN: if (value.len != sizeof(u_char)) { ngx_shmtx_unlock(&ctx->shpool->mutex); @@ -1978,7 +2672,7 @@ ngx_http_lua_ffi_shdict_incr(ngx_shm_zone_t *zone, u_char *key, "value size matched, reusing it"); ngx_queue_remove(&sd->queue); - ngx_queue_insert_head(&ctx->sh->queue, &sd->queue); + ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue); dd("go to setvalue"); goto setvalue; @@ -1994,14 +2688,14 @@ ngx_http_lua_ffi_shdict_incr(ngx_shm_zone_t *zone, u_char *key, /* rc == NGX_OK */ - if (sd->value_type != LUA_TNUMBER || sd->value_len != sizeof(double)) { + if (sd->value_type != SHDICT_TNUMBER || sd->value_len != sizeof(double)) { ngx_shmtx_unlock(&ctx->shpool->mutex); *err = "not a number"; return NGX_ERROR; } ngx_queue_remove(&sd->queue); - ngx_queue_insert_head(&ctx->sh->queue, &sd->queue); + ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue); dd("setting value type to %d", (int) sd->value_type); @@ -2082,7 +2776,7 @@ ngx_http_lua_ffi_shdict_incr(ngx_shm_zone_t *zone, u_char *key, ngx_rbtree_insert(&ctx->sh->rbtree, node); - ngx_queue_insert_head(&ctx->sh->queue, &sd->queue); + ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue); setvalue: @@ -2115,8 +2809,8 @@ ngx_http_lua_ffi_shdict_flush_all(ngx_shm_zone_t *zone) ngx_shmtx_lock(&ctx->shpool->mutex); - for (q = ngx_queue_head(&ctx->sh->queue); - q != ngx_queue_sentinel(&ctx->sh->queue); + for (q = ngx_queue_head(&ctx->sh->lru_queue); + q != ngx_queue_sentinel(&ctx->sh->lru_queue); q = ngx_queue_next(q)) { sd = ngx_queue_data(q, ngx_http_lua_shdict_node_t, queue); diff --git a/src/ngx_http_lua_shdict.h b/src/ngx_http_lua_shdict.h index aa1bb5800a..5c4ff36e6b 100644 --- a/src/ngx_http_lua_shdict.h +++ b/src/ngx_http_lua_shdict.h @@ -24,10 +24,18 @@ typedef struct { } ngx_http_lua_shdict_node_t; +typedef struct { + ngx_queue_t queue; + uint32_t value_len; + uint8_t value_type; + u_char data[1]; +} ngx_http_lua_shdict_list_node_t; + + typedef struct { ngx_rbtree_t rbtree; ngx_rbtree_node_t sentinel; - ngx_queue_t queue; + ngx_queue_t lru_queue; } ngx_http_lua_shdict_shctx_t; diff --git a/t/062-count.t b/t/062-count.t index a24611cfb6..e16291b734 100644 --- a/t/062-count.t +++ b/t/062-count.t @@ -283,7 +283,7 @@ n = 4 --- request GET /test --- response_body -n = 13 +n = 18 --- no_error_log [error] diff --git a/t/133-shdict-list.t b/t/133-shdict-list.t new file mode 100644 index 0000000000..ade228fed9 --- /dev/null +++ b/t/133-shdict-list.t @@ -0,0 +1,697 @@ +# vim:set ft= ts=4 sw=4 et fdm=marker: +use Test::Nginx::Socket::Lua; + +#worker_connections(1014); +#master_process_enabled(1); +#log_level('warn'); + +#repeat_each(2); + +plan tests => repeat_each() * (blocks() * 3 + 0); + +#no_diff(); +no_long_string(); +#master_on(); +#workers(2); + +run_tests(); + +__DATA__ + +=== TEST 1: lpush & lpop +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local dogs = ngx.shared.dogs + + local len, err = dogs:lpush("foo", "bar") + if len then + ngx.say("push success") + else + ngx.say("push err: ", err) + end + + local val, err = dogs:llen("foo") + ngx.say(val, " ", err) + + local val, err = dogs:lpop("foo") + ngx.say(val, " ", err) + + local val, err = dogs:llen("foo") + ngx.say(val, " ", err) + + local val, err = dogs:lpop("foo") + ngx.say(val, " ", err) + } + } +--- request +GET /test +--- response_body +push success +1 nil +bar nil +0 nil +nil nil +--- no_error_log +[error] + + + +=== TEST 2: get operation on list type +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local dogs = ngx.shared.dogs + + local len, err = dogs:lpush("foo", "bar") + if len then + ngx.say("push success") + else + ngx.say("push err: ", err) + end + + local val, err = dogs:get("foo") + ngx.say(val, " ", err) + } + } +--- request +GET /test +--- response_body +push success +nil value is a list +--- no_error_log +[error] + + + +=== TEST 3: set operation on list type +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local dogs = ngx.shared.dogs + + local len, err = dogs:lpush("foo", "bar") + if len then + ngx.say("push success") + else + ngx.say("push err: ", err) + end + + local ok, err = dogs:set("foo", "bar") + ngx.say(ok, " ", err) + + local val, err = dogs:get("foo") + ngx.say(val, " ", err) + } + } +--- request +GET /test +--- response_body +push success +true nil +bar nil +--- no_error_log +[error] + + + +=== TEST 4: replace operation on list type +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local dogs = ngx.shared.dogs + + local len, err = dogs:lpush("foo", "bar") + if len then + ngx.say("push success") + else + ngx.say("push err: ", err) + end + + local ok, err = dogs:replace("foo", "bar") + ngx.say(ok, " ", err) + + local val, err = dogs:get("foo") + ngx.say(val, " ", err) + } + } +--- request +GET /test +--- response_body +push success +true nil +bar nil +--- no_error_log +[error] + + + +=== TEST 5: add operation on list type +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local dogs = ngx.shared.dogs + + local len, err = dogs:lpush("foo", "bar") + if len then + ngx.say("push success") + else + ngx.say("push err: ", err) + end + + local ok, err = dogs:add("foo", "bar") + ngx.say(ok, " ", err) + + local val, err = dogs:get("foo") + ngx.say(val, " ", err) + } + } +--- request +GET /test +--- response_body +push success +false exists +nil value is a list +--- no_error_log +[error] + + + +=== TEST 6: delete operation on list type +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local dogs = ngx.shared.dogs + + local len, err = dogs:lpush("foo", "bar") + if len then + ngx.say("push success") + else + ngx.say("push err: ", err) + end + + local ok, err = dogs:delete("foo") + ngx.say(ok, " ", err) + + local val, err = dogs:get("foo") + ngx.say(val, " ", err) + } + } +--- request +GET /test +--- response_body +push success +true nil +nil nil +--- no_error_log +[error] + + + +=== TEST 7: incr operation on list type +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local dogs = ngx.shared.dogs + + local len, err = dogs:lpush("foo", "bar") + if len then + ngx.say("push success") + else + ngx.say("push err: ", err) + end + + local ok, err = dogs:incr("foo", 1) + ngx.say(ok, " ", err) + + local val, err = dogs:get("foo") + ngx.say(val, " ", err) + } + } +--- request +GET /test +--- response_body +push success +nil not a number +nil value is a list +--- no_error_log +[error] + + + +=== TEST 8: get_keys operation on list type +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local dogs = ngx.shared.dogs + + local len, err = dogs:lpush("foo", "bar") + if len then + ngx.say("push success") + else + ngx.say("push err: ", err) + end + + local keys, err = dogs:get_keys() + ngx.say("key: ", keys[1]) + } + } +--- request +GET /test +--- response_body +push success +key: foo +--- no_error_log +[error] + + + +=== TEST 9: push operation on key-value type +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local dogs = ngx.shared.dogs + + local ok, err = dogs:set("foo", "bar") + if ok then + ngx.say("set success") + else + ngx.say("set err: ", err) + end + + local len, err = dogs:lpush("foo", "bar") + ngx.say(len, " ", err) + + local val, err = dogs:get("foo") + ngx.say(val, " ", err) + } + } +--- request +GET /test +--- response_body +set success +nil value not a list +bar nil +--- no_error_log +[error] + + + +=== TEST 10: pop operation on key-value type +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local dogs = ngx.shared.dogs + + local ok, err = dogs:set("foo", "bar") + if ok then + ngx.say("set success") + else + ngx.say("set err: ", err) + end + + local val, err = dogs:lpop("foo") + ngx.say(val, " ", err) + + local val, err = dogs:get("foo") + ngx.say(val, " ", err) + } + } +--- request +GET /test +--- response_body +set success +nil value not a list +bar nil +--- no_error_log +[error] + + + +=== TEST 11: llen operation on key-value type +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local dogs = ngx.shared.dogs + + local ok, err = dogs:set("foo", "bar") + if ok then + ngx.say("set success") + else + ngx.say("set err: ", err) + end + + local val, err = dogs:llen("foo") + ngx.say(val, " ", err) + + local val, err = dogs:get("foo") + ngx.say(val, " ", err) + } + } +--- request +GET /test +--- response_body +set success +nil value not a list +bar nil +--- no_error_log +[error] + + + +=== TEST 12: lpush and lpop +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local dogs = ngx.shared.dogs + + for i = 1, 3 do + local len, err = dogs:lpush("foo", i) + if len ~= i then + ngx.say("push err: ", err) + break + end + end + + for i = 1, 3 do + local val, err = dogs:lpop("foo") + if not val then + ngx.say("pop err: ", err) + break + else + ngx.say(val) + end + end + } + } +--- request +GET /test +--- response_body +3 +2 +1 +--- no_error_log +[error] + + + +=== TEST 13: lpush and rpop +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local dogs = ngx.shared.dogs + + for i = 1, 3 do + local len, err = dogs:lpush("foo", i) + if len ~= i then + ngx.say("push err: ", err) + break + end + end + + for i = 1, 3 do + local val, err = dogs:rpop("foo") + if not val then + ngx.say("pop err: ", err) + break + else + ngx.say(val) + end + end + } + } +--- request +GET /test +--- response_body +1 +2 +3 +--- no_error_log +[error] + + + +=== TEST 14: rpush and lpop +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local dogs = ngx.shared.dogs + + for i = 1, 3 do + local len, err = dogs:rpush("foo", i) + if len ~= i then + ngx.say("push err: ", err) + break + end + end + + for i = 1, 3 do + local val, err = dogs:lpop("foo") + if not val then + ngx.say("pop err: ", err) + break + else + ngx.say(val) + end + end + } + } +--- request +GET /test +--- response_body +1 +2 +3 +--- no_error_log +[error] + + + +=== TEST 15: list removed: expired +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local dogs = ngx.shared.dogs + + local max + for i = 1, 10000 do + local key = string.format("%05d", i) + + local len , err = dogs:lpush(key, i) + if not len then + max = i + break + end + end + + local keys = dogs:get_keys(0) + + ngx.say("max-1 matched keys length: ", max-1 == #keys) + + dogs:flush_all() + + local keys = dogs:get_keys(0) + + ngx.say("keys all expired, left number: ", #keys) + + for i = 10000, 1, -1 do + local key = string.format("%05d", i) + + local len, err = dogs:lpush(key, i) + if not len then + ngx.say("loop again, max matched: ", 10001-i == max) + break + end + end + + dogs:flush_all() + + dogs:flush_expired() + + for i = 1, 10000 do + local key = string.format("%05d", i) + + local len, err = dogs:lpush(key, i) + if not len then + ngx.say("loop again, max matched: ", i == max) + break + end + end + } + } +--- request +GET /test +--- response_body +max-1 matched keys length: true +keys all expired, left number: 0 +loop again, max matched: true +loop again, max matched: true +--- no_error_log +[error] + + + +=== TEST 16: list removed: forcibly +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local dogs = ngx.shared.dogs + + local max + for i = 1, 20000 do + local ok, err, forcible = dogs:set(i, i) + if not ok or forcible then + max = i + break + end + end + + local two = dogs:get(2) + + ngx.say("two == number 2: ", two == 2) + + dogs:flush_all() + dogs:flush_expired() + + local keys = dogs:get_keys(0) + + ngx.say("no one left: ", #keys) + + for i = 1, 20000 do + local key = string.format("%05d", i) + + local len, err = dogs:lpush(key, i) + if not len then + break + end + end + + for i = 1, max do + local ok, err = dogs:set(i, i) + if not ok then + ngx.say("set err: ", err) + break + end + end + + local two = dogs:get(2) + + ngx.say("two == number 2: ", two == 2) + } + } +--- request +GET /test +--- response_body +two == number 2: true +no one left: 0 +two == number 2: true +--- no_error_log +[error] + + + +=== TEST 16: expire on all types +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local dogs = ngx.shared.dogs + + local len, err = dogs:lpush("list", "foo") + if not len then + ngx.say("push err: ", err) + end + + local ok, err = dogs:set("key", "bar") + if not ok then + ngx.say("set err: ", err) + end + + local keys = dogs:get_keys(0) + + ngx.say("keys number: ", #keys) + + dogs:flush_all() + + local keys = dogs:get_keys(0) + + ngx.say("keys number: ", #keys) + } + } +--- request +GET /test +--- response_body +keys number: 2 +keys number: 0 +--- no_error_log +[error] + + + +=== TEST 17: long list node +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local dogs = ngx.shared.dogs + + local long_str = string.rep("foo", 10) + + for i = 1, 3 do + local len, err = dogs:lpush("list", long_str) + if not len then + ngx.say("push err: ", err) + end + end + + for i = 1, 3 do + local val, err = dogs:lpop("list") + if val then + ngx.say(val) + end + end + } + } +--- request +GET /test +--- response_body +foofoofoofoofoofoofoofoofoofoo +foofoofoofoofoofoofoofoofoofoo +foofoofoofoofoofoofoofoofoofoo +--- no_error_log +[error] +