From dd7128fbd81553f90e99f730b667d216d759e5d1 Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Tue, 31 May 2016 10:23:03 +0200 Subject: [PATCH 01/42] WIP: Add jit_stack_size option --- src/ngx_http_lua_common.h | 1 + src/ngx_http_lua_module.c | 1 + src/ngx_http_lua_regex.c | 18 ++++++++++++++++++ 3 files changed, 20 insertions(+) diff --git a/src/ngx_http_lua_common.h b/src/ngx_http_lua_common.h index f37d776a9e..3e58eff947 100644 --- a/src/ngx_http_lua_common.h +++ b/src/ngx_http_lua_common.h @@ -168,6 +168,7 @@ struct ngx_http_lua_main_conf_s { ngx_int_t regex_cache_entries; ngx_int_t regex_cache_max_entries; ngx_int_t regex_match_limit; + pcre_jit_stack jit_stack; #endif ngx_array_t *shm_zones; /* of ngx_shm_zone_t* */ diff --git a/src/ngx_http_lua_module.c b/src/ngx_http_lua_module.c index 875f9334f7..500a737151 100644 --- a/src/ngx_http_lua_module.c +++ b/src/ngx_http_lua_module.c @@ -835,6 +835,7 @@ ngx_http_lua_create_main_conf(ngx_conf_t *cf) #if (NGX_PCRE) lmcf->regex_cache_max_entries = NGX_CONF_UNSET; lmcf->regex_match_limit = NGX_CONF_UNSET; + lmcf->jit_stack = pcre_jit_stack_alloc(32*1024, 32*1024); #endif lmcf->postponed_to_rewrite_phase_end = NGX_CONF_UNSET; lmcf->postponed_to_access_phase_end = NGX_CONF_UNSET; diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index d882061033..3c529162b2 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -363,6 +363,7 @@ ngx_http_lua_ngx_re_match_helper(lua_State *L, int wantcaps) old_pool = ngx_http_lua_pcre_malloc_init(pool); sd = pcre_study(re_comp.regex, PCRE_STUDY_JIT_COMPILE, &msg); + pcre_assign_jit_stack(sd, NULL, lmcf->jit_stack); ngx_http_lua_pcre_malloc_done(old_pool); @@ -825,6 +826,7 @@ ngx_http_lua_ngx_re_gmatch(lua_State *L) old_pool = ngx_http_lua_pcre_malloc_init(pool); sd = pcre_study(re_comp.regex, PCRE_STUDY_JIT_COMPILE, &msg); + pcre_assign_jit_stack(sd, NULL, lmcf->jit_stack); ngx_http_lua_pcre_malloc_done(old_pool); @@ -2478,6 +2480,22 @@ ngx_http_lua_ffi_max_regex_cache_size(void) } return (uint32_t) lmcf->regex_cache_max_entries; } + +void +ngx_http_lua_ffi_set_jit_stack_size(int size) +{ + int min_size; + ngx_http_lua_main_conf_t *lmcf; + + min_size = MIN(32 * 1024, size); + lmcf = ngx_http_cycle_get_module_main_conf(ngx_cycle, + ngx_http_lua_module); + if (lmcf == NULL) { + return; + } + pcre_jit_stack_free(lmcf->jit_stack); + lmcf->jit_stack = pcre_jit_stack_alloc(min_size, size); +} #endif /* NGX_LUA_NO_FFI_API */ From 2a4758215d6f3aafcdd0041695e7c8413ba5d21f Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Thu, 2 Jun 2016 12:17:12 +0200 Subject: [PATCH 02/42] Fix build, add guards and extract magic numbers --- src/ngx_http_lua_common.h | 19 ++++++++++++++++++- src/ngx_http_lua_module.c | 19 ++++++++++++++++++- src/ngx_http_lua_regex.c | 15 +++++++++------ 3 files changed, 45 insertions(+), 8 deletions(-) diff --git a/src/ngx_http_lua_common.h b/src/ngx_http_lua_common.h index 3e58eff947..3b0a417963 100644 --- a/src/ngx_http_lua_common.h +++ b/src/ngx_http_lua_common.h @@ -22,6 +22,19 @@ #include +#if (NGX_PCRE) + +#include + +#if (PCRE_MAJOR > 8) || (PCRE_MAJOR == 8 && PCRE_MINOR >= 21) +# define LUA_HAVE_PCRE_JIT 1 +#else +# define LUA_HAVE_PCRE_JIT 0 +#endif + +#endif + + #if !defined(nginx_version) || (nginx_version < 1006000) #error at least nginx 1.6.0 is required but found an older version #endif @@ -168,7 +181,11 @@ struct ngx_http_lua_main_conf_s { ngx_int_t regex_cache_entries; ngx_int_t regex_cache_max_entries; ngx_int_t regex_match_limit; - pcre_jit_stack jit_stack; + +#if (LUA_HAVE_PCRE_JIT) + pcre_jit_stack *jit_stack; +#endif + #endif ngx_array_t *shm_zones; /* of ngx_shm_zone_t* */ diff --git a/src/ngx_http_lua_module.c b/src/ngx_http_lua_module.c index 500a737151..6b310c91fe 100644 --- a/src/ngx_http_lua_module.c +++ b/src/ngx_http_lua_module.c @@ -31,6 +31,18 @@ #include "ngx_http_lua_headers.h" +#if (NGX_PCRE) + +#if (PCRE_MAJOR > 8) || (PCRE_MAJOR == 8 && PCRE_MINOR >= 21) +# define LUA_HAVE_PCRE_JIT 1 +# define NGX_LUA_RE_MIN_JIT_STACK_SIZE 32 * 1024 +#else +# define LUA_HAVE_PCRE_JIT 0 +#endif + +#endif + + static void *ngx_http_lua_create_main_conf(ngx_conf_t *cf); static char *ngx_http_lua_init_main_conf(ngx_conf_t *cf, void *conf); static void *ngx_http_lua_create_srv_conf(ngx_conf_t *cf); @@ -835,7 +847,12 @@ ngx_http_lua_create_main_conf(ngx_conf_t *cf) #if (NGX_PCRE) lmcf->regex_cache_max_entries = NGX_CONF_UNSET; lmcf->regex_match_limit = NGX_CONF_UNSET; - lmcf->jit_stack = pcre_jit_stack_alloc(32*1024, 32*1024); + +#if (LUA_HAVE_PCRE_JIT) + lmcf->jit_stack = pcre_jit_stack_alloc(NGX_LUA_RE_MIN_JIT_STACK_SIZE, + NGX_LUA_RE_MIN_JIT_STACK_SIZE); +#endif + #endif lmcf->postponed_to_rewrite_phase_end = NGX_CONF_UNSET; lmcf->postponed_to_access_phase_end = NGX_CONF_UNSET; diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index 3c529162b2..30c805de1c 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -42,6 +42,8 @@ #define NGX_LUA_RE_DFA_MODE_WORKSPACE_COUNT (100) +#define NGX_LUA_RE_MIN_JIT_STACK_SIZE 32 * 1024 + typedef struct { #ifndef NGX_LUA_NO_FFI_API @@ -2481,20 +2483,21 @@ ngx_http_lua_ffi_max_regex_cache_size(void) return (uint32_t) lmcf->regex_cache_max_entries; } + void ngx_http_lua_ffi_set_jit_stack_size(int size) { - int min_size; ngx_http_lua_main_conf_t *lmcf; - min_size = MIN(32 * 1024, size); - lmcf = ngx_http_cycle_get_module_main_conf(ngx_cycle, - ngx_http_lua_module); - if (lmcf == NULL) { + if (size < NGX_LUA_RE_MIN_JIT_STACK_SIZE) { return; } + + lmcf = ngx_http_cycle_get_module_main_conf(ngx_cycle, + ngx_http_lua_module); pcre_jit_stack_free(lmcf->jit_stack); - lmcf->jit_stack = pcre_jit_stack_alloc(min_size, size); + lmcf->jit_stack = pcre_jit_stack_alloc(NGX_LUA_RE_MIN_JIT_STACK_SIZE, + size); } #endif /* NGX_LUA_NO_FFI_API */ From b74c736b756abc5d2664c0a076d6236e6dc849f8 Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Sat, 11 Jun 2016 10:29:08 +0200 Subject: [PATCH 03/42] Add ffi implementation --- src/ngx_http_lua_regex.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index 30c805de1c..c142c5c5f1 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -2214,6 +2214,10 @@ ngx_http_lua_ffi_compile_regex(const unsigned char *pat, size_t pat_len, lmcf = ngx_http_cycle_get_module_main_conf(ngx_cycle, ngx_http_lua_module); + if (sd) { + pcre_assign_jit_stack(sd, NULL, lmcf->jit_stack); + } + if (sd && lmcf && lmcf->regex_match_limit > 0) { sd->flags |= PCRE_EXTRA_MATCH_LIMIT; sd->match_limit = lmcf->regex_match_limit; @@ -2490,7 +2494,7 @@ ngx_http_lua_ffi_set_jit_stack_size(int size) ngx_http_lua_main_conf_t *lmcf; if (size < NGX_LUA_RE_MIN_JIT_STACK_SIZE) { - return; + size = NGX_LUA_RE_MIN_JIT_STACK_SIZE; } lmcf = ngx_http_cycle_get_module_main_conf(ngx_cycle, From 3fe572b75727623613c5f2601b209a4c95e0e288 Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Sat, 11 Jun 2016 14:13:53 +0200 Subject: [PATCH 04/42] Add non-ffi implementation --- src/ngx_http_lua_regex.c | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index c142c5c5f1..e884734dae 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -1926,12 +1926,35 @@ ngx_http_lua_ngx_re_sub_helper(lua_State *L, unsigned global) } +static int +ngx_http_lua_ngx_re_opt(lua_State *L) +{ + ngx_str_t option; + int nargs; + int value = 0; + + nargs = lua_gettop(L); + if (nargs != 2) { + return luaL_error(L, "expecting two arguments, but got %d", nargs); + } + + option.data = (u_char *) luaL_checklstring(L, 1, &option.len); + value = luaL_checkint(L, 2); + + if (strcmp(option.data, "jit_stack_size") == 0) { + ngx_http_lua_set_jit_stack_size(value); + } + + return 0; +} + + void ngx_http_lua_inject_regex_api(lua_State *L) { /* ngx.re */ - lua_createtable(L, 0, 5 /* nrec */); /* .re */ + lua_createtable(L, 0, 6 /* nrec */); /* .re */ lua_pushcfunction(L, ngx_http_lua_ngx_re_find); lua_setfield(L, -2, "find"); @@ -1948,6 +1971,9 @@ ngx_http_lua_inject_regex_api(lua_State *L) lua_pushcfunction(L, ngx_http_lua_ngx_re_gsub); lua_setfield(L, -2, "gsub"); + lua_pushcfunction(L, ngx_http_lua_ngx_re_opt); + lua_setfield(L, -2, "opt"); + lua_setfield(L, -2, "re"); } @@ -2486,10 +2512,11 @@ ngx_http_lua_ffi_max_regex_cache_size(void) } return (uint32_t) lmcf->regex_cache_max_entries; } +#endif /* NGX_LUA_NO_FFI_API */ void -ngx_http_lua_ffi_set_jit_stack_size(int size) +ngx_http_lua_set_jit_stack_size(int size) { ngx_http_lua_main_conf_t *lmcf; @@ -2503,7 +2530,6 @@ ngx_http_lua_ffi_set_jit_stack_size(int size) lmcf->jit_stack = pcre_jit_stack_alloc(NGX_LUA_RE_MIN_JIT_STACK_SIZE, size); } -#endif /* NGX_LUA_NO_FFI_API */ #endif /* NGX_PCRE */ From ff911396b9c96ee484b4f1942bb48c120ec1e2e9 Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Sat, 11 Jun 2016 14:24:37 +0200 Subject: [PATCH 05/42] Add tests --- t/120-re-find.t | 62 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/t/120-re-find.t b/t/120-re-find.t index 73e6134fd7..46f3d66c25 100644 --- a/t/120-re-find.t +++ b/t/120-re-find.t @@ -69,7 +69,7 @@ matched: 1234 --- response_body from: 1 to: 0 -matched: +matched: --- no_error_log [error] @@ -917,3 +917,63 @@ to: 4 pos: 5 --- no_error_log [error] + + + +=== TEST 32: default jit_stack_size too small +--- config + location /re { + content_by_lua ' + -- regex is taken from https://github.com/JuliaLang/julia/issues/8278 + local s = [[71.163.72.113 - - [30/Jul/2014:16:40:55 -0700] "GET emptymind.org/thevacantwall/wp-content/uploads/2013/02/DSC_006421.jpg HTTP/1.1" 200 492513 "http://images.search.yahoo.com/images/view;_ylt=AwrB8py9gdlTGEwADcSjzbkF;_ylu=X3oDMTI2cGZrZTA5BHNlYwNmcC1leHAEc2xrA2V4cARvaWQDNTA3NTRiMzYzY2E5OTEwNjBiMjc2YWJhMjkxMTEzY2MEZ3BvcwM0BGl0A2Jpbmc-?back=http%3A%2F%2Fus.yhs4.search.yahoo.com%2Fyhs%2Fsearch%3Fei%3DUTF-8%26p%3Dapartheid%2Bwall%2Bin%2Bpalestine%26type%3Dgrvydef%26param1%3D1%26param2%3Dsid%253Db01676f9c26355f014f8a9db87545d61%2526b%253DChrome%2526ip%253D71.163.72.113%2526p%253Dgroovorio%2526x%253DAC811262A746D3CD%2526dt%253DS940%2526f%253D7%2526a%253Dgrv_tuto1_14_30%26hsimp%3Dyhs-fullyhosted_003%26hspart%3Dironsource&w=588&h=387&imgurl=occupiedpalestine.files.wordpress.com%2F2012%2F08%2F5-peeking-through-the-wall.jpg%3Fw%3D588%26h%3D387&rurl=http%3A%2F%2Fwww.stopdebezetting.com%2Fwereldpers%2Fcompare-the-berlin-wall-vs-israel-s-apartheid-wall-in-palestine.html&size=49.0KB&name=...+%3Cb%3EApartheid+wall+in+Palestine%3C%2Fb%3E...+%7C+Or+you+go+peeking+through+the+%3Cb%3Ewall%3C%2Fb%3E&p=apartheid+wall+in+palestine&oid=50754b363ca991060b276aba291113cc&fr2=&fr=&tt=...+%3Cb%3EApartheid+wall+in+Palestine%3C%2Fb%3E...+%7C+Or+you+go+peeking+through+the+%3Cb%3Ewall%3C%2Fb%3E&b=0&ni=21&no=4&ts=&tab=organic&sigr=13evdtqdq&sigb=19k7nsjvb&sigi=12o2la1db&sigt=12lia2m0j&sign=12lia2m0j&.crumb=.yUtKgFI6DE&hsimp=yhs-fullyhosted_003&hspart=ironsource" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36]] + local regex = [[([\d\.]+) ([\w.-]+) ([\w.-]+) (\[.+\]) "([^"\r\n]*|[^"\r\n\[]*\[.+\][^"]+|[^"\r\n]+.[^"]+)" (\d{3}) (\d+|-) ("(?:[^"]|\")+)"? ("(?:[^"]|\")+)"?]] + local from, to, err = ngx.re.find(s, regex, "jo") + if from or to then + ngx.say("from: ", from) + ngx.say("to: ", to) + else + if err then + ngx.say("error: ", err) + return + end + ngx.say("not matched!") + end + '; + } +--- request + GET /re +--- response_body +error: pcre_exec() failed: -27 +--- no_error_log +[error] + + + +=== TEST 33: increase jit_stack_size +--- config + location /re { + content_by_lua ' + -- regex is taken from https://github.com/JuliaLang/julia/issues/8278 + ngx.re.opt("jit_stack_size", 128 * 1024) + local s = [[71.163.72.113 - - [30/Jul/2014:16:40:55 -0700] "GET emptymind.org/thevacantwall/wp-content/uploads/2013/02/DSC_006421.jpg HTTP/1.1" 200 492513 "http://images.search.yahoo.com/images/view;_ylt=AwrB8py9gdlTGEwADcSjzbkF;_ylu=X3oDMTI2cGZrZTA5BHNlYwNmcC1leHAEc2xrA2V4cARvaWQDNTA3NTRiMzYzY2E5OTEwNjBiMjc2YWJhMjkxMTEzY2MEZ3BvcwM0BGl0A2Jpbmc-?back=http%3A%2F%2Fus.yhs4.search.yahoo.com%2Fyhs%2Fsearch%3Fei%3DUTF-8%26p%3Dapartheid%2Bwall%2Bin%2Bpalestine%26type%3Dgrvydef%26param1%3D1%26param2%3Dsid%253Db01676f9c26355f014f8a9db87545d61%2526b%253DChrome%2526ip%253D71.163.72.113%2526p%253Dgroovorio%2526x%253DAC811262A746D3CD%2526dt%253DS940%2526f%253D7%2526a%253Dgrv_tuto1_14_30%26hsimp%3Dyhs-fullyhosted_003%26hspart%3Dironsource&w=588&h=387&imgurl=occupiedpalestine.files.wordpress.com%2F2012%2F08%2F5-peeking-through-the-wall.jpg%3Fw%3D588%26h%3D387&rurl=http%3A%2F%2Fwww.stopdebezetting.com%2Fwereldpers%2Fcompare-the-berlin-wall-vs-israel-s-apartheid-wall-in-palestine.html&size=49.0KB&name=...+%3Cb%3EApartheid+wall+in+Palestine%3C%2Fb%3E...+%7C+Or+you+go+peeking+through+the+%3Cb%3Ewall%3C%2Fb%3E&p=apartheid+wall+in+palestine&oid=50754b363ca991060b276aba291113cc&fr2=&fr=&tt=...+%3Cb%3EApartheid+wall+in+Palestine%3C%2Fb%3E...+%7C+Or+you+go+peeking+through+the+%3Cb%3Ewall%3C%2Fb%3E&b=0&ni=21&no=4&ts=&tab=organic&sigr=13evdtqdq&sigb=19k7nsjvb&sigi=12o2la1db&sigt=12lia2m0j&sign=12lia2m0j&.crumb=.yUtKgFI6DE&hsimp=yhs-fullyhosted_003&hspart=ironsource" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36]] + local regex = [[([\d\.]+) ([\w.-]+) ([\w.-]+) (\[.+\]) "([^"\r\n]*|[^"\r\n\[]*\[.+\][^"]+|[^"\r\n]+.[^"]+)" (\d{3}) (\d+|-) ("(?:[^"]|\")+)"? ("(?:[^"]|\")+)"?]] + local from, to, err = ngx.re.find(s, regex, "jo") + if from or to then + ngx.say("from: ", from) + ngx.say("to: ", to) + else + if err then + ngx.say("error: ", err) + return + end + ngx.say("not matched!") + end + '; + } +--- request + GET /re +--- response_body +from: 1 +to: 1563 +--- no_error_log +[error] From 188935fe9338f08b0648f55206e41ad2a284a6e4 Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Sat, 11 Jun 2016 14:25:50 +0200 Subject: [PATCH 06/42] Fix segfault by introducing a memory leak Please remove this as soon as the issue is better understood --- src/ngx_http_lua_regex.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index e884734dae..0ae386eaaa 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -2526,7 +2526,7 @@ ngx_http_lua_set_jit_stack_size(int size) lmcf = ngx_http_cycle_get_module_main_conf(ngx_cycle, ngx_http_lua_module); - pcre_jit_stack_free(lmcf->jit_stack); + // pcre_jit_stack_free(lmcf->jit_stack); // Y U NO WORK? lmcf->jit_stack = pcre_jit_stack_alloc(NGX_LUA_RE_MIN_JIT_STACK_SIZE, size); } From 5f5b6caa4e9919716333e5394bab017512e5a6b8 Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Sat, 11 Jun 2016 15:07:43 +0200 Subject: [PATCH 07/42] Fix compiler warnings --- src/ngx_http_lua_regex.c | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index 0ae386eaaa..42f5a129ea 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -1926,10 +1926,27 @@ ngx_http_lua_ngx_re_sub_helper(lua_State *L, unsigned global) } +void +ngx_http_lua_set_jit_stack_size(int size) +{ + ngx_http_lua_main_conf_t *lmcf; + + if (size < NGX_LUA_RE_MIN_JIT_STACK_SIZE) { + size = NGX_LUA_RE_MIN_JIT_STACK_SIZE; + } + + lmcf = ngx_http_cycle_get_module_main_conf(ngx_cycle, + ngx_http_lua_module); + // pcre_jit_stack_free(lmcf->jit_stack); // Y U NO WORK? + lmcf->jit_stack = pcre_jit_stack_alloc(NGX_LUA_RE_MIN_JIT_STACK_SIZE, + size); +} + + static int ngx_http_lua_ngx_re_opt(lua_State *L) { - ngx_str_t option; + const char *option; int nargs; int value = 0; @@ -1938,10 +1955,10 @@ ngx_http_lua_ngx_re_opt(lua_State *L) return luaL_error(L, "expecting two arguments, but got %d", nargs); } - option.data = (u_char *) luaL_checklstring(L, 1, &option.len); + option = luaL_checklstring(L, 1, NULL); value = luaL_checkint(L, 2); - if (strcmp(option.data, "jit_stack_size") == 0) { + if (strcmp(option, "jit_stack_size") == 0) { ngx_http_lua_set_jit_stack_size(value); } @@ -2515,23 +2532,6 @@ ngx_http_lua_ffi_max_regex_cache_size(void) #endif /* NGX_LUA_NO_FFI_API */ -void -ngx_http_lua_set_jit_stack_size(int size) -{ - ngx_http_lua_main_conf_t *lmcf; - - if (size < NGX_LUA_RE_MIN_JIT_STACK_SIZE) { - size = NGX_LUA_RE_MIN_JIT_STACK_SIZE; - } - - lmcf = ngx_http_cycle_get_module_main_conf(ngx_cycle, - ngx_http_lua_module); - // pcre_jit_stack_free(lmcf->jit_stack); // Y U NO WORK? - lmcf->jit_stack = pcre_jit_stack_alloc(NGX_LUA_RE_MIN_JIT_STACK_SIZE, - size); -} - - #endif /* NGX_PCRE */ /* vi:set ft=c ts=4 sw=4 et fdm=marker: */ From b7c71f5b601c9704ca6d9e6ba77f431fd583f2ca Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Sat, 11 Jun 2016 15:36:41 +0200 Subject: [PATCH 08/42] Update 062-count test to reflect that ngx.re now has 6 methods --- t/062-count.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/062-count.t b/t/062-count.t index c3973adc2f..bdca99c30d 100644 --- a/t/062-count.t +++ b/t/062-count.t @@ -363,7 +363,7 @@ n = 6 --- request GET /test --- response_body -n = 5 +n = 6 --- no_error_log [error] From 5ff6a75afd06df7a56ebc781dceb4f5ba6b9a6d0 Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Sat, 11 Jun 2016 15:44:24 +0200 Subject: [PATCH 09/42] Add back empty space that was removed by editor settings --- t/120-re-find.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/120-re-find.t b/t/120-re-find.t index 46f3d66c25..1ef8a0fd81 100644 --- a/t/120-re-find.t +++ b/t/120-re-find.t @@ -69,7 +69,7 @@ matched: 1234 --- response_body from: 1 to: 0 -matched: +matched: --- no_error_log [error] From 1529aaa1207441b654eb4b0db486f665869e14ce Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Wed, 15 Jun 2016 22:57:02 +0200 Subject: [PATCH 10/42] Default lmcf->jit_stack to NULL, fix memory leak and add error messages --- src/ngx_http_lua_module.c | 18 ------------------ src/ngx_http_lua_regex.c | 39 +++++++++++++++++++++++++++++++-------- 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/src/ngx_http_lua_module.c b/src/ngx_http_lua_module.c index 6b310c91fe..875f9334f7 100644 --- a/src/ngx_http_lua_module.c +++ b/src/ngx_http_lua_module.c @@ -31,18 +31,6 @@ #include "ngx_http_lua_headers.h" -#if (NGX_PCRE) - -#if (PCRE_MAJOR > 8) || (PCRE_MAJOR == 8 && PCRE_MINOR >= 21) -# define LUA_HAVE_PCRE_JIT 1 -# define NGX_LUA_RE_MIN_JIT_STACK_SIZE 32 * 1024 -#else -# define LUA_HAVE_PCRE_JIT 0 -#endif - -#endif - - static void *ngx_http_lua_create_main_conf(ngx_conf_t *cf); static char *ngx_http_lua_init_main_conf(ngx_conf_t *cf, void *conf); static void *ngx_http_lua_create_srv_conf(ngx_conf_t *cf); @@ -847,12 +835,6 @@ ngx_http_lua_create_main_conf(ngx_conf_t *cf) #if (NGX_PCRE) lmcf->regex_cache_max_entries = NGX_CONF_UNSET; lmcf->regex_match_limit = NGX_CONF_UNSET; - -#if (LUA_HAVE_PCRE_JIT) - lmcf->jit_stack = pcre_jit_stack_alloc(NGX_LUA_RE_MIN_JIT_STACK_SIZE, - NGX_LUA_RE_MIN_JIT_STACK_SIZE); -#endif - #endif lmcf->postponed_to_rewrite_phase_end = NGX_CONF_UNSET; lmcf->postponed_to_access_phase_end = NGX_CONF_UNSET; diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index 42f5a129ea..b6d456919a 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -365,7 +365,10 @@ ngx_http_lua_ngx_re_match_helper(lua_State *L, int wantcaps) old_pool = ngx_http_lua_pcre_malloc_init(pool); sd = pcre_study(re_comp.regex, PCRE_STUDY_JIT_COMPILE, &msg); - pcre_assign_jit_stack(sd, NULL, lmcf->jit_stack); + + if (lmcf->jit_stack) { + pcre_assign_jit_stack(sd, NULL, lmcf->jit_stack); + } ngx_http_lua_pcre_malloc_done(old_pool); @@ -828,7 +831,10 @@ ngx_http_lua_ngx_re_gmatch(lua_State *L) old_pool = ngx_http_lua_pcre_malloc_init(pool); sd = pcre_study(re_comp.regex, PCRE_STUDY_JIT_COMPILE, &msg); - pcre_assign_jit_stack(sd, NULL, lmcf->jit_stack); + + if (lmcf->jit_stack) { + pcre_assign_jit_stack(sd, NULL, lmcf->jit_stack); + } ngx_http_lua_pcre_malloc_done(old_pool); @@ -1926,7 +1932,7 @@ ngx_http_lua_ngx_re_sub_helper(lua_State *L, unsigned global) } -void +static ngx_int_t ngx_http_lua_set_jit_stack_size(int size) { ngx_http_lua_main_conf_t *lmcf; @@ -1937,9 +1943,19 @@ ngx_http_lua_set_jit_stack_size(int size) lmcf = ngx_http_cycle_get_module_main_conf(ngx_cycle, ngx_http_lua_module); - // pcre_jit_stack_free(lmcf->jit_stack); // Y U NO WORK? + + if (lmcf->jit_stack) { + pcre_jit_stack_free(lmcf->jit_stack); + } + lmcf->jit_stack = pcre_jit_stack_alloc(NGX_LUA_RE_MIN_JIT_STACK_SIZE, size); + + if (!lmcf->jit_stack) { + return NGX_ERROR; + } + + return NGX_OK; } @@ -1949,6 +1965,7 @@ ngx_http_lua_ngx_re_opt(lua_State *L) const char *option; int nargs; int value = 0; + int result; nargs = lua_gettop(L); if (nargs != 2) { @@ -1958,11 +1975,17 @@ ngx_http_lua_ngx_re_opt(lua_State *L) option = luaL_checklstring(L, 1, NULL); value = luaL_checkint(L, 2); - if (strcmp(option, "jit_stack_size") == 0) { - ngx_http_lua_set_jit_stack_size(value); + if (ngx_strcmp(option, "jit_stack_size") == 0) { + result = ngx_http_lua_set_jit_stack_size(value); + + if (result != NGX_OK) { + return luaL_error(L, "PCRE jit stack allocation failed"); + } + + return 0; } - return 0; + return luaL_error(L, "unrecognized option name"); } @@ -2257,7 +2280,7 @@ ngx_http_lua_ffi_compile_regex(const unsigned char *pat, size_t pat_len, lmcf = ngx_http_cycle_get_module_main_conf(ngx_cycle, ngx_http_lua_module); - if (sd) { + if (sd && lmcf->jit_stack) { pcre_assign_jit_stack(sd, NULL, lmcf->jit_stack); } From 87838da1cf7c75e0186a66591f626168194ff180 Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Wed, 15 Jun 2016 23:33:31 +0200 Subject: [PATCH 11/42] Expose ngx_http_lua_set_jit_stack_size so that ffi can call it --- src/ngx_http_lua_regex.c | 2 +- src/ngx_http_lua_regex.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index b6d456919a..7394e6ae9e 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -1932,7 +1932,7 @@ ngx_http_lua_ngx_re_sub_helper(lua_State *L, unsigned global) } -static ngx_int_t +ngx_int_t ngx_http_lua_set_jit_stack_size(int size) { ngx_http_lua_main_conf_t *lmcf; diff --git a/src/ngx_http_lua_regex.h b/src/ngx_http_lua_regex.h index f5f8e2f875..fe0935af8c 100644 --- a/src/ngx_http_lua_regex.h +++ b/src/ngx_http_lua_regex.h @@ -13,7 +13,8 @@ #if (NGX_PCRE) -void ngx_http_lua_inject_regex_api(lua_State *L); +void ngx_http_lua_inject_regex_api(lua_State *L); +ngx_int_t ngx_http_lua_set_jit_stack_size(int size); #endif From db057fb5afcced8ad9e7e88e92d3e6dbc05bfb74 Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Wed, 15 Jun 2016 23:42:23 +0200 Subject: [PATCH 12/42] Remove duplicate # blocks --- src/ngx_http_lua_regex.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index 7394e6ae9e..8e8864a339 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -17,14 +17,6 @@ #include "ngx_http_lua_script.h" #include "ngx_http_lua_pcrefix.h" #include "ngx_http_lua_util.h" -#include - - -#if (PCRE_MAJOR > 8) || (PCRE_MAJOR == 8 && PCRE_MINOR >= 21) -# define LUA_HAVE_PCRE_JIT 1 -#else -# define LUA_HAVE_PCRE_JIT 0 -#endif #if (PCRE_MAJOR >= 6) From 7c3f7d81e0dac95541704a4584241a631334cfed Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Thu, 30 Jun 2016 20:02:58 +0200 Subject: [PATCH 13/42] Disallow the changing of jit_stack_size once regex cache is populated --- src/ngx_http_lua_regex.c | 19 +++++++++++++------ t/120-re-find.t | 38 ++++++++++++++++++++++++++++++-------- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index 8e8864a339..5432f1213d 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -1929,13 +1929,17 @@ ngx_http_lua_set_jit_stack_size(int size) { ngx_http_lua_main_conf_t *lmcf; + lmcf = ngx_http_cycle_get_module_main_conf(ngx_cycle, + ngx_http_lua_module); + + if (lmcf->regex_cache_entries > 0) { + return -1; + } + if (size < NGX_LUA_RE_MIN_JIT_STACK_SIZE) { size = NGX_LUA_RE_MIN_JIT_STACK_SIZE; } - lmcf = ngx_http_cycle_get_module_main_conf(ngx_cycle, - ngx_http_lua_module); - if (lmcf->jit_stack) { pcre_jit_stack_free(lmcf->jit_stack); } @@ -1944,10 +1948,10 @@ ngx_http_lua_set_jit_stack_size(int size) size); if (!lmcf->jit_stack) { - return NGX_ERROR; + return -2; } - return NGX_OK; + return 0; } @@ -1970,7 +1974,10 @@ ngx_http_lua_ngx_re_opt(lua_State *L) if (ngx_strcmp(option, "jit_stack_size") == 0) { result = ngx_http_lua_set_jit_stack_size(value); - if (result != NGX_OK) { + if (result == -1) { + return luaL_error(L, "Changing jit stack size is not allowed when " + "regexs have already been compiled and cached"); + } else if (result == -2) { return luaL_error(L, "PCRE jit stack allocation failed"); } diff --git a/t/120-re-find.t b/t/120-re-find.t index 1ef8a0fd81..f8e295d340 100644 --- a/t/120-re-find.t +++ b/t/120-re-find.t @@ -69,7 +69,7 @@ matched: 1234 --- response_body from: 1 to: 0 -matched: +matched: --- no_error_log [error] @@ -925,9 +925,9 @@ pos: 5 location /re { content_by_lua ' -- regex is taken from https://github.com/JuliaLang/julia/issues/8278 - local s = [[71.163.72.113 - - [30/Jul/2014:16:40:55 -0700] "GET emptymind.org/thevacantwall/wp-content/uploads/2013/02/DSC_006421.jpg HTTP/1.1" 200 492513 "http://images.search.yahoo.com/images/view;_ylt=AwrB8py9gdlTGEwADcSjzbkF;_ylu=X3oDMTI2cGZrZTA5BHNlYwNmcC1leHAEc2xrA2V4cARvaWQDNTA3NTRiMzYzY2E5OTEwNjBiMjc2YWJhMjkxMTEzY2MEZ3BvcwM0BGl0A2Jpbmc-?back=http%3A%2F%2Fus.yhs4.search.yahoo.com%2Fyhs%2Fsearch%3Fei%3DUTF-8%26p%3Dapartheid%2Bwall%2Bin%2Bpalestine%26type%3Dgrvydef%26param1%3D1%26param2%3Dsid%253Db01676f9c26355f014f8a9db87545d61%2526b%253DChrome%2526ip%253D71.163.72.113%2526p%253Dgroovorio%2526x%253DAC811262A746D3CD%2526dt%253DS940%2526f%253D7%2526a%253Dgrv_tuto1_14_30%26hsimp%3Dyhs-fullyhosted_003%26hspart%3Dironsource&w=588&h=387&imgurl=occupiedpalestine.files.wordpress.com%2F2012%2F08%2F5-peeking-through-the-wall.jpg%3Fw%3D588%26h%3D387&rurl=http%3A%2F%2Fwww.stopdebezetting.com%2Fwereldpers%2Fcompare-the-berlin-wall-vs-israel-s-apartheid-wall-in-palestine.html&size=49.0KB&name=...+%3Cb%3EApartheid+wall+in+Palestine%3C%2Fb%3E...+%7C+Or+you+go+peeking+through+the+%3Cb%3Ewall%3C%2Fb%3E&p=apartheid+wall+in+palestine&oid=50754b363ca991060b276aba291113cc&fr2=&fr=&tt=...+%3Cb%3EApartheid+wall+in+Palestine%3C%2Fb%3E...+%7C+Or+you+go+peeking+through+the+%3Cb%3Ewall%3C%2Fb%3E&b=0&ni=21&no=4&ts=&tab=organic&sigr=13evdtqdq&sigb=19k7nsjvb&sigi=12o2la1db&sigt=12lia2m0j&sign=12lia2m0j&.crumb=.yUtKgFI6DE&hsimp=yhs-fullyhosted_003&hspart=ironsource" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36]] - local regex = [[([\d\.]+) ([\w.-]+) ([\w.-]+) (\[.+\]) "([^"\r\n]*|[^"\r\n\[]*\[.+\][^"]+|[^"\r\n]+.[^"]+)" (\d{3}) (\d+|-) ("(?:[^"]|\")+)"? ("(?:[^"]|\")+)"?]] - local from, to, err = ngx.re.find(s, regex, "jo") + local very_long_string = [[71.163.72.113 - - [30/Jul/2014:16:40:55 -0700] "GET emptymind.org/thevacantwall/wp-content/uploads/2013/02/DSC_006421.jpg HTTP/1.1" 200 492513 "http://images.search.yahoo.com/images/view;_ylt=AwrB8py9gdlTGEwADcSjzbkF;_ylu=X3oDMTI2cGZrZTA5BHNlYwNmcC1leHAEc2xrA2V4cARvaWQDNTA3NTRiMzYzY2E5OTEwNjBiMjc2YWJhMjkxMTEzY2MEZ3BvcwM0BGl0A2Jpbmc-?back=http%3A%2F%2Fus.yhs4.search.yahoo.com%2Fyhs%2Fsearch%3Fei%3DUTF-8%26p%3Dapartheid%2Bwall%2Bin%2Bpalestine%26type%3Dgrvydef%26param1%3D1%26param2%3Dsid%253Db01676f9c26355f014f8a9db87545d61%2526b%253DChrome%2526ip%253D71.163.72.113%2526p%253Dgroovorio%2526x%253DAC811262A746D3CD%2526dt%253DS940%2526f%253D7%2526a%253Dgrv_tuto1_14_30%26hsimp%3Dyhs-fullyhosted_003%26hspart%3Dironsource&w=588&h=387&imgurl=occupiedpalestine.files.wordpress.com%2F2012%2F08%2F5-peeking-through-the-wall.jpg%3Fw%3D588%26h%3D387&rurl=http%3A%2F%2Fwww.stopdebezetting.com%2Fwereldpers%2Fcompare-the-berlin-wall-vs-israel-s-apartheid-wall-in-palestine.html&size=49.0KB&name=...+%3Cb%3EApartheid+wall+in+Palestine%3C%2Fb%3E...+%7C+Or+you+go+peeking+through+the+%3Cb%3Ewall%3C%2Fb%3E&p=apartheid+wall+in+palestine&oid=50754b363ca991060b276aba291113cc&fr2=&fr=&tt=...+%3Cb%3EApartheid+wall+in+Palestine%3C%2Fb%3E...+%7C+Or+you+go+peeking+through+the+%3Cb%3Ewall%3C%2Fb%3E&b=0&ni=21&no=4&ts=&tab=organic&sigr=13evdtqdq&sigb=19k7nsjvb&sigi=12o2la1db&sigt=12lia2m0j&sign=12lia2m0j&.crumb=.yUtKgFI6DE&hsimp=yhs-fullyhosted_003&hspart=ironsource" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36]] + local very_complicated_regex = [[([\d\.]+) ([\w.-]+) ([\w.-]+) (\[.+\]) "([^"\r\n]*|[^"\r\n\[]*\[.+\][^"]+|[^"\r\n]+.[^"]+)" (\d{3}) (\d+|-) ("(?:[^"]|\")+)"? ("(?:[^"]|\")+)"?]] + local from, to, err = ngx.re.find(very_long_string, very_complicated_regex, "jo") if from or to then ngx.say("from: ", from) ngx.say("to: ", to) @@ -950,14 +950,17 @@ error: pcre_exec() failed: -27 === TEST 33: increase jit_stack_size +--- http_config + init_by_lua_block { + ngx.re.opt("jit_stack_size", 128 * 1024) + } --- config location /re { content_by_lua ' -- regex is taken from https://github.com/JuliaLang/julia/issues/8278 - ngx.re.opt("jit_stack_size", 128 * 1024) - local s = [[71.163.72.113 - - [30/Jul/2014:16:40:55 -0700] "GET emptymind.org/thevacantwall/wp-content/uploads/2013/02/DSC_006421.jpg HTTP/1.1" 200 492513 "http://images.search.yahoo.com/images/view;_ylt=AwrB8py9gdlTGEwADcSjzbkF;_ylu=X3oDMTI2cGZrZTA5BHNlYwNmcC1leHAEc2xrA2V4cARvaWQDNTA3NTRiMzYzY2E5OTEwNjBiMjc2YWJhMjkxMTEzY2MEZ3BvcwM0BGl0A2Jpbmc-?back=http%3A%2F%2Fus.yhs4.search.yahoo.com%2Fyhs%2Fsearch%3Fei%3DUTF-8%26p%3Dapartheid%2Bwall%2Bin%2Bpalestine%26type%3Dgrvydef%26param1%3D1%26param2%3Dsid%253Db01676f9c26355f014f8a9db87545d61%2526b%253DChrome%2526ip%253D71.163.72.113%2526p%253Dgroovorio%2526x%253DAC811262A746D3CD%2526dt%253DS940%2526f%253D7%2526a%253Dgrv_tuto1_14_30%26hsimp%3Dyhs-fullyhosted_003%26hspart%3Dironsource&w=588&h=387&imgurl=occupiedpalestine.files.wordpress.com%2F2012%2F08%2F5-peeking-through-the-wall.jpg%3Fw%3D588%26h%3D387&rurl=http%3A%2F%2Fwww.stopdebezetting.com%2Fwereldpers%2Fcompare-the-berlin-wall-vs-israel-s-apartheid-wall-in-palestine.html&size=49.0KB&name=...+%3Cb%3EApartheid+wall+in+Palestine%3C%2Fb%3E...+%7C+Or+you+go+peeking+through+the+%3Cb%3Ewall%3C%2Fb%3E&p=apartheid+wall+in+palestine&oid=50754b363ca991060b276aba291113cc&fr2=&fr=&tt=...+%3Cb%3EApartheid+wall+in+Palestine%3C%2Fb%3E...+%7C+Or+you+go+peeking+through+the+%3Cb%3Ewall%3C%2Fb%3E&b=0&ni=21&no=4&ts=&tab=organic&sigr=13evdtqdq&sigb=19k7nsjvb&sigi=12o2la1db&sigt=12lia2m0j&sign=12lia2m0j&.crumb=.yUtKgFI6DE&hsimp=yhs-fullyhosted_003&hspart=ironsource" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36]] - local regex = [[([\d\.]+) ([\w.-]+) ([\w.-]+) (\[.+\]) "([^"\r\n]*|[^"\r\n\[]*\[.+\][^"]+|[^"\r\n]+.[^"]+)" (\d{3}) (\d+|-) ("(?:[^"]|\")+)"? ("(?:[^"]|\")+)"?]] - local from, to, err = ngx.re.find(s, regex, "jo") + local very_long_string = [[71.163.72.113 - - [30/Jul/2014:16:40:55 -0700] "GET emptymind.org/thevacantwall/wp-content/uploads/2013/02/DSC_006421.jpg HTTP/1.1" 200 492513 "http://images.search.yahoo.com/images/view;_ylt=AwrB8py9gdlTGEwADcSjzbkF;_ylu=X3oDMTI2cGZrZTA5BHNlYwNmcC1leHAEc2xrA2V4cARvaWQDNTA3NTRiMzYzY2E5OTEwNjBiMjc2YWJhMjkxMTEzY2MEZ3BvcwM0BGl0A2Jpbmc-?back=http%3A%2F%2Fus.yhs4.search.yahoo.com%2Fyhs%2Fsearch%3Fei%3DUTF-8%26p%3Dapartheid%2Bwall%2Bin%2Bpalestine%26type%3Dgrvydef%26param1%3D1%26param2%3Dsid%253Db01676f9c26355f014f8a9db87545d61%2526b%253DChrome%2526ip%253D71.163.72.113%2526p%253Dgroovorio%2526x%253DAC811262A746D3CD%2526dt%253DS940%2526f%253D7%2526a%253Dgrv_tuto1_14_30%26hsimp%3Dyhs-fullyhosted_003%26hspart%3Dironsource&w=588&h=387&imgurl=occupiedpalestine.files.wordpress.com%2F2012%2F08%2F5-peeking-through-the-wall.jpg%3Fw%3D588%26h%3D387&rurl=http%3A%2F%2Fwww.stopdebezetting.com%2Fwereldpers%2Fcompare-the-berlin-wall-vs-israel-s-apartheid-wall-in-palestine.html&size=49.0KB&name=...+%3Cb%3EApartheid+wall+in+Palestine%3C%2Fb%3E...+%7C+Or+you+go+peeking+through+the+%3Cb%3Ewall%3C%2Fb%3E&p=apartheid+wall+in+palestine&oid=50754b363ca991060b276aba291113cc&fr2=&fr=&tt=...+%3Cb%3EApartheid+wall+in+Palestine%3C%2Fb%3E...+%7C+Or+you+go+peeking+through+the+%3Cb%3Ewall%3C%2Fb%3E&b=0&ni=21&no=4&ts=&tab=organic&sigr=13evdtqdq&sigb=19k7nsjvb&sigi=12o2la1db&sigt=12lia2m0j&sign=12lia2m0j&.crumb=.yUtKgFI6DE&hsimp=yhs-fullyhosted_003&hspart=ironsource" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36]] + local very_complicated_regex = [[([\d\.]+) ([\w.-]+) ([\w.-]+) (\[.+\]) "([^"\r\n]*|[^"\r\n\[]*\[.+\][^"]+|[^"\r\n]+.[^"]+)" (\d{3}) (\d+|-) ("(?:[^"]|\")+)"? ("(?:[^"]|\")+)"?]] + local from, to, err = ngx.re.find(very_long_string, very_complicated_regex, "jo") if from or to then ngx.say("from: ", from) ngx.say("to: ", to) @@ -977,3 +980,22 @@ from: 1 to: 1563 --- no_error_log [error] + + + +=== TEST 34: jit_stack_size change disallowed once regex cache is populated +--- config + location /re { + content_by_lua ' + local s = "hello, 1234" + ngx.re.find(s, "(hello world)|([0-9])", "jo") + local status, err = pcall(function() ngx.re.opt("jit_stack_size", 128 * 1024) end) + ngx.say(err) + '; + } +--- request + GET /re +--- response_body +content_by_lua(nginx.conf:45):4: Changing jit stack size is not allowed when regexs have already been compiled and cached +--- no_error_log +[error] From a9b52cb702d75af557e2ebe7be91083053a5abc0 Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Thu, 7 Jul 2016 14:47:30 +0200 Subject: [PATCH 14/42] Add test "passing unknown options to ngx.re.opt throws an error" --- src/ngx_http_lua_regex.c | 2 +- t/120-re-find.t | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index 5432f1213d..570879ff5b 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -1984,7 +1984,7 @@ ngx_http_lua_ngx_re_opt(lua_State *L) return 0; } - return luaL_error(L, "unrecognized option name"); + return luaL_error(L, "unrecognized option name for ngx.re.opt"); } diff --git a/t/120-re-find.t b/t/120-re-find.t index f8e295d340..203bab32ae 100644 --- a/t/120-re-find.t +++ b/t/120-re-find.t @@ -999,3 +999,20 @@ to: 1563 content_by_lua(nginx.conf:45):4: Changing jit stack size is not allowed when regexs have already been compiled and cached --- no_error_log [error] + + + +=== TEST 35: passing unknown options to ngx.re.opt throws an error +--- config + location /re { + content_by_lua ' + local status, err = pcall(function() ngx.re.opt("foo", 123) end) + ngx.say(err) + '; + } +--- request + GET /re +--- response_body +content_by_lua(nginx.conf:43):2: unrecognized option name for ngx.re.opt +--- no_error_log +[error] From c10f4d5150a4ae7e74a72c424074f0cdcebbce1c Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Sun, 13 Nov 2016 13:05:41 +0100 Subject: [PATCH 15/42] Style: Align strings and keep line length under 80 chars --- src/ngx_http_lua_regex.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index 570879ff5b..aa8efe8a23 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -1976,7 +1976,9 @@ ngx_http_lua_ngx_re_opt(lua_State *L) if (result == -1) { return luaL_error(L, "Changing jit stack size is not allowed when " - "regexs have already been compiled and cached"); + "some regexs have already been compiled and " + "cached"); + } else if (result == -2) { return luaL_error(L, "PCRE jit stack allocation failed"); } From ef13eb2670d54d77448f2ea99ea39ea5f53f4d59 Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Sun, 13 Nov 2016 13:06:07 +0100 Subject: [PATCH 16/42] Use *_by_lua_block in new tests --- t/120-re-find.t | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/t/120-re-find.t b/t/120-re-find.t index 203bab32ae..3efa92f60c 100644 --- a/t/120-re-find.t +++ b/t/120-re-find.t @@ -923,7 +923,7 @@ pos: 5 === TEST 32: default jit_stack_size too small --- config location /re { - content_by_lua ' + content_by_lua_block { -- regex is taken from https://github.com/JuliaLang/julia/issues/8278 local very_long_string = [[71.163.72.113 - - [30/Jul/2014:16:40:55 -0700] "GET emptymind.org/thevacantwall/wp-content/uploads/2013/02/DSC_006421.jpg HTTP/1.1" 200 492513 "http://images.search.yahoo.com/images/view;_ylt=AwrB8py9gdlTGEwADcSjzbkF;_ylu=X3oDMTI2cGZrZTA5BHNlYwNmcC1leHAEc2xrA2V4cARvaWQDNTA3NTRiMzYzY2E5OTEwNjBiMjc2YWJhMjkxMTEzY2MEZ3BvcwM0BGl0A2Jpbmc-?back=http%3A%2F%2Fus.yhs4.search.yahoo.com%2Fyhs%2Fsearch%3Fei%3DUTF-8%26p%3Dapartheid%2Bwall%2Bin%2Bpalestine%26type%3Dgrvydef%26param1%3D1%26param2%3Dsid%253Db01676f9c26355f014f8a9db87545d61%2526b%253DChrome%2526ip%253D71.163.72.113%2526p%253Dgroovorio%2526x%253DAC811262A746D3CD%2526dt%253DS940%2526f%253D7%2526a%253Dgrv_tuto1_14_30%26hsimp%3Dyhs-fullyhosted_003%26hspart%3Dironsource&w=588&h=387&imgurl=occupiedpalestine.files.wordpress.com%2F2012%2F08%2F5-peeking-through-the-wall.jpg%3Fw%3D588%26h%3D387&rurl=http%3A%2F%2Fwww.stopdebezetting.com%2Fwereldpers%2Fcompare-the-berlin-wall-vs-israel-s-apartheid-wall-in-palestine.html&size=49.0KB&name=...+%3Cb%3EApartheid+wall+in+Palestine%3C%2Fb%3E...+%7C+Or+you+go+peeking+through+the+%3Cb%3Ewall%3C%2Fb%3E&p=apartheid+wall+in+palestine&oid=50754b363ca991060b276aba291113cc&fr2=&fr=&tt=...+%3Cb%3EApartheid+wall+in+Palestine%3C%2Fb%3E...+%7C+Or+you+go+peeking+through+the+%3Cb%3Ewall%3C%2Fb%3E&b=0&ni=21&no=4&ts=&tab=organic&sigr=13evdtqdq&sigb=19k7nsjvb&sigi=12o2la1db&sigt=12lia2m0j&sign=12lia2m0j&.crumb=.yUtKgFI6DE&hsimp=yhs-fullyhosted_003&hspart=ironsource" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36]] local very_complicated_regex = [[([\d\.]+) ([\w.-]+) ([\w.-]+) (\[.+\]) "([^"\r\n]*|[^"\r\n\[]*\[.+\][^"]+|[^"\r\n]+.[^"]+)" (\d{3}) (\d+|-) ("(?:[^"]|\")+)"? ("(?:[^"]|\")+)"?]] @@ -938,7 +938,7 @@ pos: 5 end ngx.say("not matched!") end - '; + } } --- request GET /re @@ -956,7 +956,7 @@ error: pcre_exec() failed: -27 } --- config location /re { - content_by_lua ' + content_by_lua_block { -- regex is taken from https://github.com/JuliaLang/julia/issues/8278 local very_long_string = [[71.163.72.113 - - [30/Jul/2014:16:40:55 -0700] "GET emptymind.org/thevacantwall/wp-content/uploads/2013/02/DSC_006421.jpg HTTP/1.1" 200 492513 "http://images.search.yahoo.com/images/view;_ylt=AwrB8py9gdlTGEwADcSjzbkF;_ylu=X3oDMTI2cGZrZTA5BHNlYwNmcC1leHAEc2xrA2V4cARvaWQDNTA3NTRiMzYzY2E5OTEwNjBiMjc2YWJhMjkxMTEzY2MEZ3BvcwM0BGl0A2Jpbmc-?back=http%3A%2F%2Fus.yhs4.search.yahoo.com%2Fyhs%2Fsearch%3Fei%3DUTF-8%26p%3Dapartheid%2Bwall%2Bin%2Bpalestine%26type%3Dgrvydef%26param1%3D1%26param2%3Dsid%253Db01676f9c26355f014f8a9db87545d61%2526b%253DChrome%2526ip%253D71.163.72.113%2526p%253Dgroovorio%2526x%253DAC811262A746D3CD%2526dt%253DS940%2526f%253D7%2526a%253Dgrv_tuto1_14_30%26hsimp%3Dyhs-fullyhosted_003%26hspart%3Dironsource&w=588&h=387&imgurl=occupiedpalestine.files.wordpress.com%2F2012%2F08%2F5-peeking-through-the-wall.jpg%3Fw%3D588%26h%3D387&rurl=http%3A%2F%2Fwww.stopdebezetting.com%2Fwereldpers%2Fcompare-the-berlin-wall-vs-israel-s-apartheid-wall-in-palestine.html&size=49.0KB&name=...+%3Cb%3EApartheid+wall+in+Palestine%3C%2Fb%3E...+%7C+Or+you+go+peeking+through+the+%3Cb%3Ewall%3C%2Fb%3E&p=apartheid+wall+in+palestine&oid=50754b363ca991060b276aba291113cc&fr2=&fr=&tt=...+%3Cb%3EApartheid+wall+in+Palestine%3C%2Fb%3E...+%7C+Or+you+go+peeking+through+the+%3Cb%3Ewall%3C%2Fb%3E&b=0&ni=21&no=4&ts=&tab=organic&sigr=13evdtqdq&sigb=19k7nsjvb&sigi=12o2la1db&sigt=12lia2m0j&sign=12lia2m0j&.crumb=.yUtKgFI6DE&hsimp=yhs-fullyhosted_003&hspart=ironsource" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36]] local very_complicated_regex = [[([\d\.]+) ([\w.-]+) ([\w.-]+) (\[.+\]) "([^"\r\n]*|[^"\r\n\[]*\[.+\][^"]+|[^"\r\n]+.[^"]+)" (\d{3}) (\d+|-) ("(?:[^"]|\")+)"? ("(?:[^"]|\")+)"?]] @@ -971,7 +971,7 @@ error: pcre_exec() failed: -27 end ngx.say("not matched!") end - '; + } } --- request GET /re @@ -986,17 +986,17 @@ to: 1563 === TEST 34: jit_stack_size change disallowed once regex cache is populated --- config location /re { - content_by_lua ' + content_by_lua_block { local s = "hello, 1234" ngx.re.find(s, "(hello world)|([0-9])", "jo") local status, err = pcall(function() ngx.re.opt("jit_stack_size", 128 * 1024) end) ngx.say(err) - '; + } } --- request GET /re --- response_body -content_by_lua(nginx.conf:45):4: Changing jit stack size is not allowed when regexs have already been compiled and cached +content_by_lua(nginx.conf:45):4: Changing jit stack size is not allowed when some regexs have already been compiled and cached --- no_error_log [error] @@ -1005,10 +1005,10 @@ content_by_lua(nginx.conf:45):4: Changing jit stack size is not allowed when reg === TEST 35: passing unknown options to ngx.re.opt throws an error --- config location /re { - content_by_lua ' + content_by_lua_block { local status, err = pcall(function() ngx.re.opt("foo", 123) end) ngx.say(err) - '; + } } --- request GET /re From 2c261c965252ccdd4ac58b01a87376796b19d92d Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Sun, 13 Nov 2016 17:19:58 +0100 Subject: [PATCH 17/42] Incorporate style feedback --- src/ngx_http_lua_regex.c | 22 ++++++++++++---------- src/ngx_http_lua_regex.h | 2 +- t/120-re-find.t | 2 +- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index aa8efe8a23..288428f9bc 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -1947,7 +1947,7 @@ ngx_http_lua_set_jit_stack_size(int size) lmcf->jit_stack = pcre_jit_stack_alloc(NGX_LUA_RE_MIN_JIT_STACK_SIZE, size); - if (!lmcf->jit_stack) { + if (lmcf->jit_stack == NULL) { return -2; } @@ -1960,8 +1960,8 @@ ngx_http_lua_ngx_re_opt(lua_State *L) { const char *option; int nargs; - int value = 0; - int result; + int value; + int rc; nargs = lua_gettop(L); if (nargs != 2) { @@ -1971,22 +1971,24 @@ ngx_http_lua_ngx_re_opt(lua_State *L) option = luaL_checklstring(L, 1, NULL); value = luaL_checkint(L, 2); - if (ngx_strcmp(option, "jit_stack_size") == 0) { - result = ngx_http_lua_set_jit_stack_size(value); + if (ngx_strncmp(option, "jit_stack_size") == 0) { + rc = ngx_http_lua_set_jit_stack_size(value); - if (result == -1) { + if (rc == -1) { return luaL_error(L, "Changing jit stack size is not allowed when " - "some regexs have already been compiled and " - "cached"); + "some regexs have already been compiled and " + "cached"); - } else if (result == -2) { + } + + if (rc == -2) { return luaL_error(L, "PCRE jit stack allocation failed"); } return 0; } - return luaL_error(L, "unrecognized option name for ngx.re.opt"); + return luaL_error(L, "unrecognized option name"); } diff --git a/src/ngx_http_lua_regex.h b/src/ngx_http_lua_regex.h index fe0935af8c..9636912910 100644 --- a/src/ngx_http_lua_regex.h +++ b/src/ngx_http_lua_regex.h @@ -13,7 +13,7 @@ #if (NGX_PCRE) -void ngx_http_lua_inject_regex_api(lua_State *L); +void ngx_http_lua_inject_regex_api(lua_State *L); ngx_int_t ngx_http_lua_set_jit_stack_size(int size); #endif diff --git a/t/120-re-find.t b/t/120-re-find.t index 3efa92f60c..4882692e3d 100644 --- a/t/120-re-find.t +++ b/t/120-re-find.t @@ -1013,6 +1013,6 @@ content_by_lua(nginx.conf:45):4: Changing jit stack size is not allowed when som --- request GET /re --- response_body -content_by_lua(nginx.conf:43):2: unrecognized option name for ngx.re.opt +content_by_lua(nginx.conf:43):2: unrecognized option name --- no_error_log [error] From 2337c8c751931505bac54cd802d44692eafbbc6f Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Sun, 13 Nov 2016 17:21:03 +0100 Subject: [PATCH 18/42] Add more pcre jit guards --- src/ngx_http_lua_regex.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index 288428f9bc..5663dc9be8 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -1924,6 +1924,8 @@ ngx_http_lua_ngx_re_sub_helper(lua_State *L, unsigned global) } +#if LUA_HAVE_PCRE_JIT + ngx_int_t ngx_http_lua_set_jit_stack_size(int size) { @@ -1954,6 +1956,8 @@ ngx_http_lua_set_jit_stack_size(int size) return 0; } +#endif /* LUA_HAVE_PCRE_JIT */ + static int ngx_http_lua_ngx_re_opt(lua_State *L) @@ -1971,6 +1975,8 @@ ngx_http_lua_ngx_re_opt(lua_State *L) option = luaL_checklstring(L, 1, NULL); value = luaL_checkint(L, 2); +#if LUA_HAVE_PCRE_JIT + if (ngx_strncmp(option, "jit_stack_size") == 0) { rc = ngx_http_lua_set_jit_stack_size(value); @@ -1988,6 +1994,8 @@ ngx_http_lua_ngx_re_opt(lua_State *L) return 0; } +#endif /* LUA_HAVE_PCRE_JIT */ + return luaL_error(L, "unrecognized option name"); } From 15a7a0d2c7a34fda3ed500b7b552268ee48eb196 Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Sun, 13 Nov 2016 17:21:18 +0100 Subject: [PATCH 19/42] Avoid magic numbers --- src/ngx_http_lua_regex.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index 5663dc9be8..5a93a29caf 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -1935,7 +1935,7 @@ ngx_http_lua_set_jit_stack_size(int size) ngx_http_lua_module); if (lmcf->regex_cache_entries > 0) { - return -1; + return NGX_DECLINED; } if (size < NGX_LUA_RE_MIN_JIT_STACK_SIZE) { @@ -1950,7 +1950,7 @@ ngx_http_lua_set_jit_stack_size(int size) size); if (lmcf->jit_stack == NULL) { - return -2; + return NGX_ERROR; } return 0; @@ -1980,14 +1980,14 @@ ngx_http_lua_ngx_re_opt(lua_State *L) if (ngx_strncmp(option, "jit_stack_size") == 0) { rc = ngx_http_lua_set_jit_stack_size(value); - if (rc == -1) { + if (rc == NGX_DECLINED) { return luaL_error(L, "Changing jit stack size is not allowed when " "some regexs have already been compiled and " "cached"); } - if (rc == -2) { + if (rc == NGX_ERROR) { return luaL_error(L, "PCRE jit stack allocation failed"); } From aafc6e10e2ec7b4c5ebeb89466ee234d6cfb100f Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Sun, 13 Nov 2016 17:42:03 +0100 Subject: [PATCH 20/42] Pass a length to ngx_strncmp --- src/ngx_http_lua_regex.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index 5a93a29caf..9589831acd 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -1963,6 +1963,7 @@ static int ngx_http_lua_ngx_re_opt(lua_State *L) { const char *option; + size_t option_len; int nargs; int value; int rc; @@ -1972,12 +1973,12 @@ ngx_http_lua_ngx_re_opt(lua_State *L) return luaL_error(L, "expecting two arguments, but got %d", nargs); } - option = luaL_checklstring(L, 1, NULL); + option = luaL_checklstring(L, 1, &option_len); value = luaL_checkint(L, 2); #if LUA_HAVE_PCRE_JIT - if (ngx_strncmp(option, "jit_stack_size") == 0) { + if (ngx_strncmp(option, "jit_stack_size", option_len) == 0) { rc = ngx_http_lua_set_jit_stack_size(value); if (rc == NGX_DECLINED) { From 87a8b6165e87dbfb23049fe9fb115680995238e2 Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Sun, 13 Nov 2016 18:25:03 +0100 Subject: [PATCH 21/42] travis --- .reload | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 .reload diff --git a/.reload b/.reload new file mode 100644 index 0000000000..e69de29bb2 From 9498bf41d834d4425a515dd329b21fe71edf0cb8 Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Sun, 13 Nov 2016 18:50:23 +0100 Subject: [PATCH 22/42] Add documentation for ngx.re.opt --- README.markdown | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/README.markdown b/README.markdown index 946f002c61..c1ba4123a4 100644 --- a/README.markdown +++ b/README.markdown @@ -3122,6 +3122,7 @@ Nginx API for Lua * [ngx.re.gmatch](#ngxregmatch) * [ngx.re.sub](#ngxresub) * [ngx.re.gsub](#ngxregsub) +* [ngx.re.opt](#ngxreopt) * [ngx.shared.DICT](#ngxshareddict) * [ngx.shared.DICT.get](#ngxshareddictget) * [ngx.shared.DICT.get_stale](#ngxshareddictget_stale) @@ -6115,6 +6116,40 @@ This feature was first introduced in the `v0.2.1rc15` release. [Back to TOC](#nginx-api-for-lua) +ngx.re.opt +----------- +**syntax:** *ngx.re.opt(option, value)* + +**context:** *init_by_lua*, init_worker_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua** + +Allows changing of regex settings. Currently, it can only change the `jit_stack_size` of the PCRE engine. For example, + +```nginx + + init_by_lua_block { ngx.re.opt("jit_stack_size", 128 * 1024) } + + server { + location /re { + content_by_lua_block { + -- full regex and string are taken from https://github.com/JuliaLang/julia/issues/8278 + local very_long_string = [[71.163.72.113 - - [30/Jul/2014:16:40:55 -0700] ...]] + local very_complicated_regex = [[([\d\.]+) ([\w.-]+) ([\w.-]+) (\[.+\]) ...]] + local from, to, err = ngx.re.find(very_long_string, very_complicated_regex, "jo") + + -- with the regular jit_stack_size, we would the error 'pcre_exec() failed: -27' + -- instead, we get a match + ngx.print(from .. "-" .. to) -- prints '1-1563' + } + } + } +``` + +This method requires the PCRE library enabled in Nginx. ([Known Issue With Special Escaping Sequences](#special-escaping-sequences)). + +This feature was first introduced in the `XXX` release. + +[Back to TOC](#nginx-api-for-lua) + ngx.shared.DICT --------------- **syntax:** *dict = ngx.shared.DICT* From 1d2e80ce5a8cb1dd3c430bada29c185ecb7c5418 Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Mon, 14 Nov 2016 09:03:26 +0100 Subject: [PATCH 23/42] Avoid more magic numbers --- src/ngx_http_lua_regex.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index 9589831acd..0ef230601d 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -1953,7 +1953,7 @@ ngx_http_lua_set_jit_stack_size(int size) return NGX_ERROR; } - return 0; + return NGX_OK; } #endif /* LUA_HAVE_PCRE_JIT */ @@ -1992,7 +1992,7 @@ ngx_http_lua_ngx_re_opt(lua_State *L) return luaL_error(L, "PCRE jit stack allocation failed"); } - return 0; + return NGX_OK; } #endif /* LUA_HAVE_PCRE_JIT */ From a56355a5e7c66251b741d52f635970fa115efeb2 Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Mon, 14 Nov 2016 09:13:05 +0100 Subject: [PATCH 24/42] Add documentation to wiki file --- README.markdown | 4 ++-- doc/HttpLuaModule.wiki | 30 ++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/README.markdown b/README.markdown index c1ba4123a4..601cefe5a6 100644 --- a/README.markdown +++ b/README.markdown @@ -6117,7 +6117,7 @@ This feature was first introduced in the `v0.2.1rc15` release. [Back to TOC](#nginx-api-for-lua) ngx.re.opt ------------ +---------- **syntax:** *ngx.re.opt(option, value)* **context:** *init_by_lua*, init_worker_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua** @@ -6136,7 +6136,7 @@ Allows changing of regex settings. Currently, it can only change the `jit_stack_ local very_complicated_regex = [[([\d\.]+) ([\w.-]+) ([\w.-]+) (\[.+\]) ...]] local from, to, err = ngx.re.find(very_long_string, very_complicated_regex, "jo") - -- with the regular jit_stack_size, we would the error 'pcre_exec() failed: -27' + -- with the regular jit_stack_size, we would get the error 'pcre_exec() failed: -27' -- instead, we get a match ngx.print(from .. "-" .. to) -- prints '1-1563' } diff --git a/doc/HttpLuaModule.wiki b/doc/HttpLuaModule.wiki index bbf2a2ecd7..69689d0df3 100644 --- a/doc/HttpLuaModule.wiki +++ b/doc/HttpLuaModule.wiki @@ -5120,6 +5120,36 @@ This method requires the PCRE library enabled in Nginx. ([[#Special Escaping Se This feature was first introduced in the v0.2.1rc15 release. +== ngx.re.opt == +'''syntax:''' ''ngx.re.opt(option, value)'' + +'''context:''' ''init_by_lua*, init_worker_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua*'' + +Allows changing of regex settings. Currently, it can only change the jit_stack_size of the PCRE engine. For example, + + +init_by_lua_block { ngx.re.opt("jit_stack_size", 128 * 1024) } + +server { + location /re { + content_by_lua_block { + -- full regex and string are taken from https://github.com/JuliaLang/julia/issues/8278 + local very_long_string = [[71.163.72.113 - - [30/Jul/2014:16:40:55 -0700] ...]] + local very_complicated_regex = [[([\d\.]+) ([\w.-]+) ([\w.-]+) (\[.+\]) ...]] + local from, to, err = ngx.re.find(very_long_string, very_complicated_regex, "jo") + + -- with the regular jit_stack_size, we would get the error 'pcre_exec() failed: -27' + -- instead, we get a match + ngx.print(from .. "-" .. to) -- prints '1-1563' + } + } +} + + +This method requires the PCRE library enabled in Nginx. ([[#Special Escaping Sequences|Known Issue With Special Escaping Sequences]]). + +This feature was first introduced in the XXX release. + == ngx.shared.DICT == '''syntax:''' ''dict = ngx.shared.DICT'' From 7e858bd70c6873946e2d19a1c478d7aee9d4a43c Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Mon, 14 Nov 2016 09:25:54 +0100 Subject: [PATCH 25/42] Update tests to check ngx.re.opt in content_by_lua --- t/120-re-find.t | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/t/120-re-find.t b/t/120-re-find.t index 4882692e3d..099e183995 100644 --- a/t/120-re-find.t +++ b/t/120-re-find.t @@ -988,15 +988,37 @@ to: 1563 location /re { content_by_lua_block { local s = "hello, 1234" - ngx.re.find(s, "(hello world)|([0-9])", "jo") + local from, to = ngx.re.find(s, "(hello world)|([0-9])") local status, err = pcall(function() ngx.re.opt("jit_stack_size", 128 * 1024) end) + ngx.say("from: ", from) + ngx.say("to: ", to) + ngx.say(err) + + local from, to = ngx.re.find(s, "(hello world)|([0-9])", "j") + local status, err = pcall(function() ngx.re.opt("jit_stack_size", 128 * 1024) end) + ngx.say("from: ", from) + ngx.say("to: ", to) + ngx.say(err) + + local from, to = ngx.re.find(s, "(hello world)|([0-9])", "jo") + local status, err = pcall(function() ngx.re.opt("jit_stack_size", 128 * 1024) end) + ngx.say("from: ", from) + ngx.say("to: ", to) ngx.say(err) } } --- request GET /re --- response_body -content_by_lua(nginx.conf:45):4: Changing jit stack size is not allowed when some regexs have already been compiled and cached +from: 8 +to: 8 +nil +from: 8 +to: 8 +nil +from: 8 +to: 8 +content_by_lua(nginx.conf:59):16: Changing jit stack size is not allowed when some regexs have already been compiled and cached --- no_error_log [error] From d6498095b483238cf442bc208c60715edc79ce01 Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Mon, 14 Nov 2016 10:38:05 +0100 Subject: [PATCH 26/42] Fix return value of ngx_http_lua_ngx_re_opt --- src/ngx_http_lua_regex.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index 0ef230601d..eca93ef5c4 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -1992,7 +1992,7 @@ ngx_http_lua_ngx_re_opt(lua_State *L) return luaL_error(L, "PCRE jit stack allocation failed"); } - return NGX_OK; + return 0; } #endif /* LUA_HAVE_PCRE_JIT */ From d947745573366f37346672e586f7c94617f37b25 Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Tue, 15 Nov 2016 09:55:16 +0100 Subject: [PATCH 27/42] Call ngx_http_lua_pcre_malloc_{init,done} around pcre_{malloc,free} --- src/ngx_http_lua_regex.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index eca93ef5c4..bb9a5fa499 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -1930,6 +1930,7 @@ ngx_int_t ngx_http_lua_set_jit_stack_size(int size) { ngx_http_lua_main_conf_t *lmcf; + ngx_pool_t *pool, *old_pool; lmcf = ngx_http_cycle_get_module_main_conf(ngx_cycle, ngx_http_lua_module); @@ -1942,13 +1943,25 @@ ngx_http_lua_set_jit_stack_size(int size) size = NGX_LUA_RE_MIN_JIT_STACK_SIZE; } + pool = lmcf->pool; + + dd("server pool %p", lmcf->pool); + if (lmcf->jit_stack) { + old_pool = ngx_http_lua_pcre_malloc_init(pool); + pcre_jit_stack_free(lmcf->jit_stack); + + ngx_http_lua_pcre_malloc_done(old_pool); } + old_pool = ngx_http_lua_pcre_malloc_init(pool); + lmcf->jit_stack = pcre_jit_stack_alloc(NGX_LUA_RE_MIN_JIT_STACK_SIZE, size); + ngx_http_lua_pcre_malloc_done(old_pool); + if (lmcf->jit_stack == NULL) { return NGX_ERROR; } From d8c8968cad009df21186fdcf3c9bc3da82aaedcb Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Tue, 15 Nov 2016 10:07:18 +0100 Subject: [PATCH 28/42] Mark ngx_http_lua_set_jit_stack_size as ffi --- src/ngx_http_lua_regex.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index bb9a5fa499..d01fbae3d6 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -1927,7 +1927,7 @@ ngx_http_lua_ngx_re_sub_helper(lua_State *L, unsigned global) #if LUA_HAVE_PCRE_JIT ngx_int_t -ngx_http_lua_set_jit_stack_size(int size) +ngx_http_lua_ffi_set_jit_stack_size(int size) { ngx_http_lua_main_conf_t *lmcf; ngx_pool_t *pool, *old_pool; @@ -1992,7 +1992,7 @@ ngx_http_lua_ngx_re_opt(lua_State *L) #if LUA_HAVE_PCRE_JIT if (ngx_strncmp(option, "jit_stack_size", option_len) == 0) { - rc = ngx_http_lua_set_jit_stack_size(value); + rc = ngx_http_lua_ffi_set_jit_stack_size(value); if (rc == NGX_DECLINED) { return luaL_error(L, "Changing jit stack size is not allowed when " From cad4ee3e1e4359456d50c494427e509c01dc8cad Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Tue, 22 Nov 2016 13:41:02 +0100 Subject: [PATCH 29/42] Test 33: Use grep_error_log to pass the second test iteration --- t/120-re-find.t | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/t/120-re-find.t b/t/120-re-find.t index 099e183995..3f18958186 100644 --- a/t/120-re-find.t +++ b/t/120-re-find.t @@ -989,22 +989,21 @@ to: 1563 content_by_lua_block { local s = "hello, 1234" local from, to = ngx.re.find(s, "(hello world)|([0-9])") - local status, err = pcall(function() ngx.re.opt("jit_stack_size", 128 * 1024) end) ngx.say("from: ", from) ngx.say("to: ", to) - ngx.say(err) + if err then ngx.log(ngx.ERR, err) end - local from, to = ngx.re.find(s, "(hello world)|([0-9])", "j") local status, err = pcall(function() ngx.re.opt("jit_stack_size", 128 * 1024) end) + local from, to = ngx.re.find(s, "(hello world)|([0-9])", "j") ngx.say("from: ", from) ngx.say("to: ", to) - ngx.say(err) + if err then ngx.log(ngx.ERR, err) end - local from, to = ngx.re.find(s, "(hello world)|([0-9])", "jo") local status, err = pcall(function() ngx.re.opt("jit_stack_size", 128 * 1024) end) + local from, to = ngx.re.find(s, "(hello world)|([0-9])", "jo") ngx.say("from: ", from) ngx.say("to: ", to) - ngx.say(err) + if err then ngx.log(ngx.ERR, err) end } } --- request @@ -1012,15 +1011,17 @@ to: 1563 --- response_body from: 8 to: 8 -nil from: 8 to: 8 -nil from: 8 to: 8 -content_by_lua(nginx.conf:59):16: Changing jit stack size is not allowed when some regexs have already been compiled and cached ---- no_error_log -[error] + +--- grep_error_log eval +Changing jit stack size is not allowed when some regexs have already been compiled and cached + +--- grep_error_log_out eval +["", "Changing jit stack size is not allowed when some regexs have already been compiled and cached\nChanging jit stack size is not allowed when some regexs have already been compiled and cached\n"] + From becfeda948094b3c85f43573e9c6d03bffa39566 Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Tue, 22 Nov 2016 14:15:00 +0100 Subject: [PATCH 30/42] Simplify test 33 --- t/120-re-find.t | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/t/120-re-find.t b/t/120-re-find.t index 3f18958186..7aba40775a 100644 --- a/t/120-re-find.t +++ b/t/120-re-find.t @@ -991,19 +991,7 @@ to: 1563 local from, to = ngx.re.find(s, "(hello world)|([0-9])") ngx.say("from: ", from) ngx.say("to: ", to) - if err then ngx.log(ngx.ERR, err) end - - local status, err = pcall(function() ngx.re.opt("jit_stack_size", 128 * 1024) end) - local from, to = ngx.re.find(s, "(hello world)|([0-9])", "j") - ngx.say("from: ", from) - ngx.say("to: ", to) - if err then ngx.log(ngx.ERR, err) end - - local status, err = pcall(function() ngx.re.opt("jit_stack_size", 128 * 1024) end) - local from, to = ngx.re.find(s, "(hello world)|([0-9])", "jo") - ngx.say("from: ", from) - ngx.say("to: ", to) - if err then ngx.log(ngx.ERR, err) end + ngx.re.opt("jit_stack_size", 128 * 1024) } } --- request @@ -1011,16 +999,12 @@ to: 1563 --- response_body from: 8 to: 8 -from: 8 -to: 8 -from: 8 -to: 8 --- grep_error_log eval Changing jit stack size is not allowed when some regexs have already been compiled and cached --- grep_error_log_out eval -["", "Changing jit stack size is not allowed when some regexs have already been compiled and cached\nChanging jit stack size is not allowed when some regexs have already been compiled and cached\n"] +["", "Changing jit stack size is not allowed when some regexs have already been compiled and cached\n"] From f1001a9aecaaff6fe39ae081ee7bcbbacf90b47c Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Tue, 22 Nov 2016 14:32:13 +0100 Subject: [PATCH 31/42] Fix test 33 --- t/120-re-find.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/120-re-find.t b/t/120-re-find.t index 7aba40775a..d73253f6ca 100644 --- a/t/120-re-find.t +++ b/t/120-re-find.t @@ -988,7 +988,7 @@ to: 1563 location /re { content_by_lua_block { local s = "hello, 1234" - local from, to = ngx.re.find(s, "(hello world)|([0-9])") + local from, to = ngx.re.find(s, "(hello world)|([0-9])", "jo") ngx.say("from: ", from) ngx.say("to: ", to) ngx.re.opt("jit_stack_size", 128 * 1024) From cd9bb7016cb342f62855641feecefaf044f0eddf Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Fri, 16 Dec 2016 16:41:00 +0100 Subject: [PATCH 32/42] Repair test 33 --- t/120-re-find.t | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/t/120-re-find.t b/t/120-re-find.t index d73253f6ca..b91d806461 100644 --- a/t/120-re-find.t +++ b/t/120-re-find.t @@ -987,11 +987,12 @@ to: 1563 --- config location /re { content_by_lua_block { + local status, err = pcall(ngx.re.opt, "jit_stack_size", 128 * 1024) + if err then ngx.log(ngx.INFO, err) end local s = "hello, 1234" local from, to = ngx.re.find(s, "(hello world)|([0-9])", "jo") ngx.say("from: ", from) ngx.say("to: ", to) - ngx.re.opt("jit_stack_size", 128 * 1024) } } --- request @@ -1001,7 +1002,7 @@ from: 8 to: 8 --- grep_error_log eval -Changing jit stack size is not allowed when some regexs have already been compiled and cached +qr/Changing jit stack size is not allowed when some regexs have already been compiled and cached/ --- grep_error_log_out eval ["", "Changing jit stack size is not allowed when some regexs have already been compiled and cached\n"] @@ -1013,7 +1014,7 @@ Changing jit stack size is not allowed when some regexs have already been compil --- config location /re { content_by_lua_block { - local status, err = pcall(function() ngx.re.opt("foo", 123) end) + local status, err = pcall(ngx.re.opt, "foo", 123) ngx.say(err) } } From e024b8f6aa0f7a10a7b7e99ae5bbbaa2c6859437 Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Fri, 16 Dec 2016 17:32:05 +0100 Subject: [PATCH 33/42] Repair rebase artifacts --- t/120-re-find.t | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/t/120-re-find.t b/t/120-re-find.t index b91d806461..a2fd02695a 100644 --- a/t/120-re-find.t +++ b/t/120-re-find.t @@ -69,7 +69,7 @@ matched: 1234 --- response_body from: 1 to: 0 -matched: +matched: --- no_error_log [error] @@ -1021,6 +1021,6 @@ qr/Changing jit stack size is not allowed when some regexs have already been com --- request GET /re --- response_body -content_by_lua(nginx.conf:43):2: unrecognized option name +unrecognized option name --- no_error_log [error] From 4c735a5d9d83ef053b82b2773549571b3e0663c9 Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Fri, 16 Dec 2016 22:06:11 +0100 Subject: [PATCH 34/42] Remove Lua-5.1 api --- README.markdown | 35 ------------- doc/HttpLuaModule.wiki | 30 ----------- src/ngx_http_lua_regex.c | 47 +---------------- src/ngx_http_lua_regex.h | 2 +- t/062-count.t | 2 +- t/120-re-find.t | 107 --------------------------------------- 6 files changed, 3 insertions(+), 220 deletions(-) diff --git a/README.markdown b/README.markdown index 601cefe5a6..946f002c61 100644 --- a/README.markdown +++ b/README.markdown @@ -3122,7 +3122,6 @@ Nginx API for Lua * [ngx.re.gmatch](#ngxregmatch) * [ngx.re.sub](#ngxresub) * [ngx.re.gsub](#ngxregsub) -* [ngx.re.opt](#ngxreopt) * [ngx.shared.DICT](#ngxshareddict) * [ngx.shared.DICT.get](#ngxshareddictget) * [ngx.shared.DICT.get_stale](#ngxshareddictget_stale) @@ -6116,40 +6115,6 @@ This feature was first introduced in the `v0.2.1rc15` release. [Back to TOC](#nginx-api-for-lua) -ngx.re.opt ----------- -**syntax:** *ngx.re.opt(option, value)* - -**context:** *init_by_lua*, init_worker_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua** - -Allows changing of regex settings. Currently, it can only change the `jit_stack_size` of the PCRE engine. For example, - -```nginx - - init_by_lua_block { ngx.re.opt("jit_stack_size", 128 * 1024) } - - server { - location /re { - content_by_lua_block { - -- full regex and string are taken from https://github.com/JuliaLang/julia/issues/8278 - local very_long_string = [[71.163.72.113 - - [30/Jul/2014:16:40:55 -0700] ...]] - local very_complicated_regex = [[([\d\.]+) ([\w.-]+) ([\w.-]+) (\[.+\]) ...]] - local from, to, err = ngx.re.find(very_long_string, very_complicated_regex, "jo") - - -- with the regular jit_stack_size, we would get the error 'pcre_exec() failed: -27' - -- instead, we get a match - ngx.print(from .. "-" .. to) -- prints '1-1563' - } - } - } -``` - -This method requires the PCRE library enabled in Nginx. ([Known Issue With Special Escaping Sequences](#special-escaping-sequences)). - -This feature was first introduced in the `XXX` release. - -[Back to TOC](#nginx-api-for-lua) - ngx.shared.DICT --------------- **syntax:** *dict = ngx.shared.DICT* diff --git a/doc/HttpLuaModule.wiki b/doc/HttpLuaModule.wiki index 69689d0df3..bbf2a2ecd7 100644 --- a/doc/HttpLuaModule.wiki +++ b/doc/HttpLuaModule.wiki @@ -5120,36 +5120,6 @@ This method requires the PCRE library enabled in Nginx. ([[#Special Escaping Se This feature was first introduced in the v0.2.1rc15 release. -== ngx.re.opt == -'''syntax:''' ''ngx.re.opt(option, value)'' - -'''context:''' ''init_by_lua*, init_worker_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua*'' - -Allows changing of regex settings. Currently, it can only change the jit_stack_size of the PCRE engine. For example, - - -init_by_lua_block { ngx.re.opt("jit_stack_size", 128 * 1024) } - -server { - location /re { - content_by_lua_block { - -- full regex and string are taken from https://github.com/JuliaLang/julia/issues/8278 - local very_long_string = [[71.163.72.113 - - [30/Jul/2014:16:40:55 -0700] ...]] - local very_complicated_regex = [[([\d\.]+) ([\w.-]+) ([\w.-]+) (\[.+\]) ...]] - local from, to, err = ngx.re.find(very_long_string, very_complicated_regex, "jo") - - -- with the regular jit_stack_size, we would get the error 'pcre_exec() failed: -27' - -- instead, we get a match - ngx.print(from .. "-" .. to) -- prints '1-1563' - } - } -} - - -This method requires the PCRE library enabled in Nginx. ([[#Special Escaping Sequences|Known Issue With Special Escaping Sequences]]). - -This feature was first introduced in the XXX release. - == ngx.shared.DICT == '''syntax:''' ''dict = ngx.shared.DICT'' diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index d01fbae3d6..1e2156a3db 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -1972,54 +1972,12 @@ ngx_http_lua_ffi_set_jit_stack_size(int size) #endif /* LUA_HAVE_PCRE_JIT */ -static int -ngx_http_lua_ngx_re_opt(lua_State *L) -{ - const char *option; - size_t option_len; - int nargs; - int value; - int rc; - - nargs = lua_gettop(L); - if (nargs != 2) { - return luaL_error(L, "expecting two arguments, but got %d", nargs); - } - - option = luaL_checklstring(L, 1, &option_len); - value = luaL_checkint(L, 2); - -#if LUA_HAVE_PCRE_JIT - - if (ngx_strncmp(option, "jit_stack_size", option_len) == 0) { - rc = ngx_http_lua_ffi_set_jit_stack_size(value); - - if (rc == NGX_DECLINED) { - return luaL_error(L, "Changing jit stack size is not allowed when " - "some regexs have already been compiled and " - "cached"); - - } - - if (rc == NGX_ERROR) { - return luaL_error(L, "PCRE jit stack allocation failed"); - } - - return 0; - } - -#endif /* LUA_HAVE_PCRE_JIT */ - - return luaL_error(L, "unrecognized option name"); -} - - void ngx_http_lua_inject_regex_api(lua_State *L) { /* ngx.re */ - lua_createtable(L, 0, 6 /* nrec */); /* .re */ + lua_createtable(L, 0, 5 /* nrec */); /* .re */ lua_pushcfunction(L, ngx_http_lua_ngx_re_find); lua_setfield(L, -2, "find"); @@ -2036,9 +1994,6 @@ ngx_http_lua_inject_regex_api(lua_State *L) lua_pushcfunction(L, ngx_http_lua_ngx_re_gsub); lua_setfield(L, -2, "gsub"); - lua_pushcfunction(L, ngx_http_lua_ngx_re_opt); - lua_setfield(L, -2, "opt"); - lua_setfield(L, -2, "re"); } diff --git a/src/ngx_http_lua_regex.h b/src/ngx_http_lua_regex.h index 9636912910..40f8969fa6 100644 --- a/src/ngx_http_lua_regex.h +++ b/src/ngx_http_lua_regex.h @@ -14,7 +14,7 @@ #if (NGX_PCRE) void ngx_http_lua_inject_regex_api(lua_State *L); -ngx_int_t ngx_http_lua_set_jit_stack_size(int size); +ngx_int_t ngx_http_lua_ffi_set_jit_stack_size(int size); #endif diff --git a/t/062-count.t b/t/062-count.t index bdca99c30d..c3973adc2f 100644 --- a/t/062-count.t +++ b/t/062-count.t @@ -363,7 +363,7 @@ n = 6 --- request GET /test --- response_body -n = 6 +n = 5 --- no_error_log [error] diff --git a/t/120-re-find.t b/t/120-re-find.t index a2fd02695a..73e6134fd7 100644 --- a/t/120-re-find.t +++ b/t/120-re-find.t @@ -917,110 +917,3 @@ to: 4 pos: 5 --- no_error_log [error] - - - -=== TEST 32: default jit_stack_size too small ---- config - location /re { - content_by_lua_block { - -- regex is taken from https://github.com/JuliaLang/julia/issues/8278 - local very_long_string = [[71.163.72.113 - - [30/Jul/2014:16:40:55 -0700] "GET emptymind.org/thevacantwall/wp-content/uploads/2013/02/DSC_006421.jpg HTTP/1.1" 200 492513 "http://images.search.yahoo.com/images/view;_ylt=AwrB8py9gdlTGEwADcSjzbkF;_ylu=X3oDMTI2cGZrZTA5BHNlYwNmcC1leHAEc2xrA2V4cARvaWQDNTA3NTRiMzYzY2E5OTEwNjBiMjc2YWJhMjkxMTEzY2MEZ3BvcwM0BGl0A2Jpbmc-?back=http%3A%2F%2Fus.yhs4.search.yahoo.com%2Fyhs%2Fsearch%3Fei%3DUTF-8%26p%3Dapartheid%2Bwall%2Bin%2Bpalestine%26type%3Dgrvydef%26param1%3D1%26param2%3Dsid%253Db01676f9c26355f014f8a9db87545d61%2526b%253DChrome%2526ip%253D71.163.72.113%2526p%253Dgroovorio%2526x%253DAC811262A746D3CD%2526dt%253DS940%2526f%253D7%2526a%253Dgrv_tuto1_14_30%26hsimp%3Dyhs-fullyhosted_003%26hspart%3Dironsource&w=588&h=387&imgurl=occupiedpalestine.files.wordpress.com%2F2012%2F08%2F5-peeking-through-the-wall.jpg%3Fw%3D588%26h%3D387&rurl=http%3A%2F%2Fwww.stopdebezetting.com%2Fwereldpers%2Fcompare-the-berlin-wall-vs-israel-s-apartheid-wall-in-palestine.html&size=49.0KB&name=...+%3Cb%3EApartheid+wall+in+Palestine%3C%2Fb%3E...+%7C+Or+you+go+peeking+through+the+%3Cb%3Ewall%3C%2Fb%3E&p=apartheid+wall+in+palestine&oid=50754b363ca991060b276aba291113cc&fr2=&fr=&tt=...+%3Cb%3EApartheid+wall+in+Palestine%3C%2Fb%3E...+%7C+Or+you+go+peeking+through+the+%3Cb%3Ewall%3C%2Fb%3E&b=0&ni=21&no=4&ts=&tab=organic&sigr=13evdtqdq&sigb=19k7nsjvb&sigi=12o2la1db&sigt=12lia2m0j&sign=12lia2m0j&.crumb=.yUtKgFI6DE&hsimp=yhs-fullyhosted_003&hspart=ironsource" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36]] - local very_complicated_regex = [[([\d\.]+) ([\w.-]+) ([\w.-]+) (\[.+\]) "([^"\r\n]*|[^"\r\n\[]*\[.+\][^"]+|[^"\r\n]+.[^"]+)" (\d{3}) (\d+|-) ("(?:[^"]|\")+)"? ("(?:[^"]|\")+)"?]] - local from, to, err = ngx.re.find(very_long_string, very_complicated_regex, "jo") - if from or to then - ngx.say("from: ", from) - ngx.say("to: ", to) - else - if err then - ngx.say("error: ", err) - return - end - ngx.say("not matched!") - end - } - } ---- request - GET /re ---- response_body -error: pcre_exec() failed: -27 ---- no_error_log -[error] - - - -=== TEST 33: increase jit_stack_size ---- http_config - init_by_lua_block { - ngx.re.opt("jit_stack_size", 128 * 1024) - } ---- config - location /re { - content_by_lua_block { - -- regex is taken from https://github.com/JuliaLang/julia/issues/8278 - local very_long_string = [[71.163.72.113 - - [30/Jul/2014:16:40:55 -0700] "GET emptymind.org/thevacantwall/wp-content/uploads/2013/02/DSC_006421.jpg HTTP/1.1" 200 492513 "http://images.search.yahoo.com/images/view;_ylt=AwrB8py9gdlTGEwADcSjzbkF;_ylu=X3oDMTI2cGZrZTA5BHNlYwNmcC1leHAEc2xrA2V4cARvaWQDNTA3NTRiMzYzY2E5OTEwNjBiMjc2YWJhMjkxMTEzY2MEZ3BvcwM0BGl0A2Jpbmc-?back=http%3A%2F%2Fus.yhs4.search.yahoo.com%2Fyhs%2Fsearch%3Fei%3DUTF-8%26p%3Dapartheid%2Bwall%2Bin%2Bpalestine%26type%3Dgrvydef%26param1%3D1%26param2%3Dsid%253Db01676f9c26355f014f8a9db87545d61%2526b%253DChrome%2526ip%253D71.163.72.113%2526p%253Dgroovorio%2526x%253DAC811262A746D3CD%2526dt%253DS940%2526f%253D7%2526a%253Dgrv_tuto1_14_30%26hsimp%3Dyhs-fullyhosted_003%26hspart%3Dironsource&w=588&h=387&imgurl=occupiedpalestine.files.wordpress.com%2F2012%2F08%2F5-peeking-through-the-wall.jpg%3Fw%3D588%26h%3D387&rurl=http%3A%2F%2Fwww.stopdebezetting.com%2Fwereldpers%2Fcompare-the-berlin-wall-vs-israel-s-apartheid-wall-in-palestine.html&size=49.0KB&name=...+%3Cb%3EApartheid+wall+in+Palestine%3C%2Fb%3E...+%7C+Or+you+go+peeking+through+the+%3Cb%3Ewall%3C%2Fb%3E&p=apartheid+wall+in+palestine&oid=50754b363ca991060b276aba291113cc&fr2=&fr=&tt=...+%3Cb%3EApartheid+wall+in+Palestine%3C%2Fb%3E...+%7C+Or+you+go+peeking+through+the+%3Cb%3Ewall%3C%2Fb%3E&b=0&ni=21&no=4&ts=&tab=organic&sigr=13evdtqdq&sigb=19k7nsjvb&sigi=12o2la1db&sigt=12lia2m0j&sign=12lia2m0j&.crumb=.yUtKgFI6DE&hsimp=yhs-fullyhosted_003&hspart=ironsource" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36]] - local very_complicated_regex = [[([\d\.]+) ([\w.-]+) ([\w.-]+) (\[.+\]) "([^"\r\n]*|[^"\r\n\[]*\[.+\][^"]+|[^"\r\n]+.[^"]+)" (\d{3}) (\d+|-) ("(?:[^"]|\")+)"? ("(?:[^"]|\")+)"?]] - local from, to, err = ngx.re.find(very_long_string, very_complicated_regex, "jo") - if from or to then - ngx.say("from: ", from) - ngx.say("to: ", to) - else - if err then - ngx.say("error: ", err) - return - end - ngx.say("not matched!") - end - } - } ---- request - GET /re ---- response_body -from: 1 -to: 1563 ---- no_error_log -[error] - - - -=== TEST 34: jit_stack_size change disallowed once regex cache is populated ---- config - location /re { - content_by_lua_block { - local status, err = pcall(ngx.re.opt, "jit_stack_size", 128 * 1024) - if err then ngx.log(ngx.INFO, err) end - local s = "hello, 1234" - local from, to = ngx.re.find(s, "(hello world)|([0-9])", "jo") - ngx.say("from: ", from) - ngx.say("to: ", to) - } - } ---- request - GET /re ---- response_body -from: 8 -to: 8 - ---- grep_error_log eval -qr/Changing jit stack size is not allowed when some regexs have already been compiled and cached/ - ---- grep_error_log_out eval -["", "Changing jit stack size is not allowed when some regexs have already been compiled and cached\n"] - - - - -=== TEST 35: passing unknown options to ngx.re.opt throws an error ---- config - location /re { - content_by_lua_block { - local status, err = pcall(ngx.re.opt, "foo", 123) - ngx.say(err) - } - } ---- request - GET /re ---- response_body -unrecognized option name ---- no_error_log -[error] From c4c74de120d9d4213c25af11dc50c9b14d9e0726 Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Fri, 16 Dec 2016 22:06:50 +0100 Subject: [PATCH 35/42] Remove temporary file --- .reload | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .reload diff --git a/.reload b/.reload deleted file mode 100644 index e69de29bb2..0000000000 From 3b2356c222eda98881da21f0d3495c758dd28edc Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Mon, 19 Dec 2016 10:09:22 +0100 Subject: [PATCH 36/42] Remove check that no regexs have been compiled This is now handled in resty.core's ngx.re --- src/ngx_http_lua_regex.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index 1e2156a3db..bf44604d1e 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -1935,10 +1935,6 @@ ngx_http_lua_ffi_set_jit_stack_size(int size) lmcf = ngx_http_cycle_get_module_main_conf(ngx_cycle, ngx_http_lua_module); - if (lmcf->regex_cache_entries > 0) { - return NGX_DECLINED; - } - if (size < NGX_LUA_RE_MIN_JIT_STACK_SIZE) { size = NGX_LUA_RE_MIN_JIT_STACK_SIZE; } From 89318e57539c1e685566880a6c3dc06b3efdaf9a Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Mon, 9 Jan 2017 12:33:09 +0100 Subject: [PATCH 37/42] Guard against 'sd' being NULL --- src/ngx_http_lua_regex.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index bf44604d1e..4f09c7ebf1 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -358,7 +358,7 @@ ngx_http_lua_ngx_re_match_helper(lua_State *L, int wantcaps) sd = pcre_study(re_comp.regex, PCRE_STUDY_JIT_COMPILE, &msg); - if (lmcf->jit_stack) { + if (sd && lmcf->jit_stack) { pcre_assign_jit_stack(sd, NULL, lmcf->jit_stack); } @@ -824,7 +824,7 @@ ngx_http_lua_ngx_re_gmatch(lua_State *L) sd = pcre_study(re_comp.regex, PCRE_STUDY_JIT_COMPILE, &msg); - if (lmcf->jit_stack) { + if (sd && lmcf->jit_stack) { pcre_assign_jit_stack(sd, NULL, lmcf->jit_stack); } @@ -2216,6 +2216,9 @@ ngx_http_lua_ffi_compile_regex(const unsigned char *pat, size_t pat_len, goto error; } + lmcf = ngx_http_cycle_get_module_main_conf(ngx_cycle, + ngx_http_lua_module); + #if (LUA_HAVE_PCRE_JIT) if (flags & NGX_LUA_RE_MODE_JIT) { @@ -2251,15 +2254,12 @@ ngx_http_lua_ffi_compile_regex(const unsigned char *pat, size_t pat_len, ngx_http_lua_pcre_malloc_done(old_pool); } -#endif /* LUA_HAVE_PCRE_JIT */ - - lmcf = ngx_http_cycle_get_module_main_conf(ngx_cycle, - ngx_http_lua_module); - if (sd && lmcf->jit_stack) { pcre_assign_jit_stack(sd, NULL, lmcf->jit_stack); } +#endif /* LUA_HAVE_PCRE_JIT */ + if (sd && lmcf && lmcf->regex_match_limit > 0) { sd->flags |= PCRE_EXTRA_MATCH_LIMIT; sd->match_limit = lmcf->regex_match_limit; From 9a003866fc4381f1ecbc674f09cf29e5652f410e Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Mon, 9 Jan 2017 12:35:10 +0100 Subject: [PATCH 38/42] Return error in ngx_http_lua_ffi_set_jit_stack_size if no JIT is available --- src/ngx_http_lua_regex.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index 4f09c7ebf1..6cbd5be0b7 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -1924,11 +1924,11 @@ ngx_http_lua_ngx_re_sub_helper(lua_State *L, unsigned global) } -#if LUA_HAVE_PCRE_JIT - ngx_int_t ngx_http_lua_ffi_set_jit_stack_size(int size) { +#if LUA_HAVE_PCRE_JIT + ngx_http_lua_main_conf_t *lmcf; ngx_pool_t *pool, *old_pool; @@ -1963,9 +1963,14 @@ ngx_http_lua_ffi_set_jit_stack_size(int size) } return NGX_OK; -} + +#else /* LUA_HAVE_PCRE_JIT */ + + return NGX_ERROR; #endif /* LUA_HAVE_PCRE_JIT */ +} + void From bbcc991d3f8701ee587ed96b84b299b1cfb5e482 Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Thu, 12 Jan 2017 10:01:59 +0100 Subject: [PATCH 39/42] Spacing --- src/ngx_http_lua_regex.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index 6cbd5be0b7..4a8b9710c4 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -1972,7 +1972,6 @@ ngx_http_lua_ffi_set_jit_stack_size(int size) } - void ngx_http_lua_inject_regex_api(lua_State *L) { From cc6564b4355477acdec4543fd8431c8e944e87d4 Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Thu, 12 Jan 2017 10:03:40 +0100 Subject: [PATCH 40/42] Return NGX_DECLINED when pcre jit is not found --- src/ngx_http_lua_regex.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index 4a8b9710c4..0686bf0212 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -1966,7 +1966,7 @@ ngx_http_lua_ffi_set_jit_stack_size(int size) #else /* LUA_HAVE_PCRE_JIT */ - return NGX_ERROR; + return NGX_DECLINED; #endif /* LUA_HAVE_PCRE_JIT */ } From 5a6cd878cc5e544d7cc8d8a702b496801e73fb84 Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Fri, 14 Apr 2017 19:03:08 +0200 Subject: [PATCH 41/42] Pass error messages from C to Lua --- src/ngx_http_lua_regex.c | 10 ++++++++-- src/ngx_http_lua_regex.h | 3 ++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index 0686bf0212..2882c1920a 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -1925,7 +1925,8 @@ ngx_http_lua_ngx_re_sub_helper(lua_State *L, unsigned global) ngx_int_t -ngx_http_lua_ffi_set_jit_stack_size(int size) +ngx_http_lua_ffi_set_jit_stack_size(int size, u_char *errstr, + size_t errstr_size) { #if LUA_HAVE_PCRE_JIT @@ -1959,6 +1960,9 @@ ngx_http_lua_ffi_set_jit_stack_size(int size) ngx_http_lua_pcre_malloc_done(old_pool); if (lmcf->jit_stack == NULL) { + *errstr_size = ngx_snprintf(errstr, *errstr_size, + "pcre jit stack allocation failed") + - errstr; return NGX_ERROR; } @@ -1966,7 +1970,9 @@ ngx_http_lua_ffi_set_jit_stack_size(int size) #else /* LUA_HAVE_PCRE_JIT */ - return NGX_DECLINED; + *errstr_size = ngx_snprintf(errstr, *errstr_size, + "no pcre jit support found") - errstr; + return NGX_ERROR; #endif /* LUA_HAVE_PCRE_JIT */ } diff --git a/src/ngx_http_lua_regex.h b/src/ngx_http_lua_regex.h index 40f8969fa6..116779577f 100644 --- a/src/ngx_http_lua_regex.h +++ b/src/ngx_http_lua_regex.h @@ -14,7 +14,8 @@ #if (NGX_PCRE) void ngx_http_lua_inject_regex_api(lua_State *L); -ngx_int_t ngx_http_lua_ffi_set_jit_stack_size(int size); +ngx_int_t ngx_http_lua_ffi_set_jit_stack_size(int size, u_char *errstr, + size_t errstr_size); #endif From e40dd9a07d080c4b77c31518c2d34d89a9d28526 Mon Sep 17 00:00:00 2001 From: Andreas Lubbe Date: Fri, 14 Apr 2017 19:16:04 +0200 Subject: [PATCH 42/42] Add missing pointer declaration --- src/ngx_http_lua_regex.c | 2 +- src/ngx_http_lua_regex.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ngx_http_lua_regex.c b/src/ngx_http_lua_regex.c index 2882c1920a..20a45cdcb1 100644 --- a/src/ngx_http_lua_regex.c +++ b/src/ngx_http_lua_regex.c @@ -1926,7 +1926,7 @@ ngx_http_lua_ngx_re_sub_helper(lua_State *L, unsigned global) ngx_int_t ngx_http_lua_ffi_set_jit_stack_size(int size, u_char *errstr, - size_t errstr_size) + size_t *errstr_size) { #if LUA_HAVE_PCRE_JIT diff --git a/src/ngx_http_lua_regex.h b/src/ngx_http_lua_regex.h index 116779577f..9752827dcf 100644 --- a/src/ngx_http_lua_regex.h +++ b/src/ngx_http_lua_regex.h @@ -15,7 +15,7 @@ #if (NGX_PCRE) void ngx_http_lua_inject_regex_api(lua_State *L); ngx_int_t ngx_http_lua_ffi_set_jit_stack_size(int size, u_char *errstr, - size_t errstr_size); + size_t *errstr_size); #endif