Merge pull request #11 from thibaultcha/tests/jit

feat(uuid) JIT v3 and v4 generation + JIT tests

.ci/setup_openresty.sh

 set -e
 
 OPENRESTY_DOWNLOAD=$DOWNLOAD_CACHE/openresty-$OPENRESTY
+LUAROCKS_DOWNLOAD=$DOWNLOAD_CACHE/luarocks-$LUAROCKS
 
 mkdir -p $OPENRESTY_DOWNLOAD
 
@@ -11,7 +12,13 @@ if [ ! "$(ls -A $OPENRESTY_DOWNLOAD)" ]; then
   popd
 fi
 
+if [ ! "$(ls -A $LUAROCKS_DOWNLOAD)" ]; then
+  git clone https://github.com/keplerproject/luarocks.git $LUAROCKS_DOWNLOAD
+fi
+
+
 OPENRESTY_INSTALL=$INSTALL_CACHE/openresty-$OPENRESTY
+LUAROCKS_INSTALL=$INSTALL_CACHE/luarocks-$LUAROCKS
 
 mkdir -p $OPENRESTY_INSTALL
 
@@ -25,7 +32,22 @@ if [ ! "$(ls -A $OPENRESTY_INSTALL)" ]; then
   popd
 fi
 
-export PATH="$PATH:$OPENRESTY_INSTALL/nginx/sbin"
+if [ ! "$(ls -A $LUAROCKS_INSTALL)" ]; then
+  pushd $LUAROCKS_DOWNLOAD
+    git checkout v$LUAROCKS
+    ./configure \
+      --prefix=$LUAROCKS_INSTALL \
+      --lua-suffix=jit \
+      --with-lua=$OPENRESTY_INSTALL/luajit \
+      --with-lua-include=$OPENRESTY_INSTALL/luajit/include/luajit-2.1
+    make build
+    make install
+  popd
+fi
+
+export PATH="$PATH:$OPENRESTY_INSTALL/nginx/sbin:$LUAROCKS_INSTALL/bin"
+
+eval `luarocks path`
 
 git clone git://github.com/travis-perl/helpers travis-perl-helpers
 pushd travis-perl-helpers

.travis.yml

+# vim:set sts=2 ts=2 sw=2 et:
+
 language: perl
 
 perl: 5.18
@@ -15,18 +17,16 @@ addons:
 
 env:
   global:
-    - OPENRESTY=1.11.2.1
+    - OPENRESTY=1.11.2.2
+    - LUAROCKS=2.4.2
     - DOWNLOAD_CACHE=$HOME/download-cache
     - INSTALL_CACHE=$HOME/install-cache
 
 before_install:
   - source .ci/setup_openresty.sh
-  - pip install --user hererocks
-  - hererocks lua_install -r^ -l 5.1
-  - export PATH=$PATH:$PWD/lua_install/bin
-  - eval `luarocks path`
   - luarocks install luacheck
   - luarocks install luacov-coveralls
+  - luarocks install Lua-cURL
 
 install:
   - luarocks make
@@ -34,7 +34,7 @@ install:
 script:
   - make lint
   - make coverage
-  - luacov-coveralls -i lib/resty
+  - luacov-coveralls -v -i lib/resty
 
 cache:
   cpan: true

Makefile

@@ -6,6 +6,7 @@ test:
 
 lint:
 	@luacheck -q lib --std 'luajit+ngx_lua' \
+	  --no-unused-args \
 	  --no-redefined
 
 bench:

bench.lua

@@ -2,20 +2,28 @@ if not ngx or not jit then
   error('must run in resty-cli with LuaJIT')
 end
 
+--[[
+local v = require "jit.v"
+v.on("/tmp/dump.log")
+--]]
+
 -------------
 -- Settings
 -------------
 local n_uuids = 1e6
 local p_valid_uuids = 70
 
-package.path = 'lib/?.lua;'..package.path
+package.path = 'lib/?.lua;' .. package.path
 
-local cuuid = require 'lua_uuid'
-local lua_uuid = require 'uuid'
-local ffi_uuid = require 'resty.uuid'
+local cuuid       = require 'lua_uuid'
+local lua_uuid    = require 'uuid'
+local ffi_uuid    = require 'resty.uuid'
 local luajit_uuid = require 'resty.jit-uuid'
+
 package.loaded['resty.jit-uuid'] = nil
+
 ngx.config.nginx_configure = function() return '' end
+
 local luajit_uuid_no_pcre = require 'resty.jit-uuid'
 
 math.randomseed(os.time())
@@ -23,6 +31,8 @@ math.randomseed(os.time())
 ---------------------
 -- UUID v4 generation
 ---------------------
+
+
 local tests = {
   ['C binding     '] = cuuid,
   ['Pure Lua      '] = lua_uuid.new,
@@ -30,8 +40,11 @@ local tests = {
   ['FFI binding   '] = ffi_uuid.generate_random
 }
 
+
 local v4_results = {}
 local time_reference
+
+
 for k, uuid in pairs(tests) do
   collectgarbage()
 
@@ -47,37 +60,47 @@ for k, uuid in pairs(tests) do
   end
 end
 
+
 for _, res in ipairs(v4_results) do
   res.diff = ((res.time - time_reference)/time_reference)*100
 end
 
+
 table.sort(v4_results, function(a, b) return a.time < b.time end)
 
+
 print(string.format('%s with %g UUIDs', jit.version, n_uuids))
 print('UUID v4 (random) generation')
 for i, result in ipairs(v4_results) do
-  print(string.format('%d. %s\ttook:\t%fs\t%+d%%', i, result.module, result.time, result.diff))
+  print(string.format('%d. %s\ttook:\t%fs\t%+d%%', i, result.module,
+                      result.time, result.diff))
 end
 
+
 ---------------------
 -- UUID v3 generation
 ---------------------
 
+
 -- unique names, unique namespaces: no strings interned
-tests = {
+local tests = {
   ['resty-jit-uuid'] = assert(luajit_uuid.factory_v3('cc7da0b0-0743-11e6-968a-bfd4d8c62f62'))
 }
+
+
 local names = {}
 for i = 1, n_uuids do
   names[i] = ffi_uuid.generate_random()
 end
 
+
 local v3_results = {}
+
+
 for k, factory in pairs(tests) do
   collectgarbage()
 
   local tstart = os.clock()
-  local check = {}
   for i = 1, n_uuids do
     factory(names[i])
   end
@@ -86,32 +109,40 @@ for k, factory in pairs(tests) do
   v3_results[#v3_results+1] = {module = k, time = time}
 end
 
+
 table.sort(v3_results, function(a, b) return a.time < b.time end)
 
+
 print('\nUUID v3 (name-based and MD5) generation if supported')
 for i, result in ipairs(v3_results) do
   print(string.format('%d. %s\ttook:\t%fs', i, result.module, result.time))
 end
 
+
 ---------------------
 -- UUID v5 generation
 ---------------------
 
+
 -- unique names, unique namespaces: no strings interned
-tests = {
+local tests = {
   ['resty-jit-uuid'] = assert(luajit_uuid.factory_v5('1b985f4a-06be-11e6-aff4-ff8d14e25128'))
 }
+
+
 local names = {}
 for i = 1, n_uuids do
   names[i] = ffi_uuid.generate_random()
 end
 
+
 local v5_results = {}
+
+
 for k, factory in pairs(tests) do
   collectgarbage()
 
   local tstart = os.clock()
-  local check = {}
   for i = 1, n_uuids do
     factory(names[i])
   end
@@ -120,56 +151,81 @@ for k, factory in pairs(tests) do
   v5_results[#v5_results+1] = {module = k, time = time}
 end
 
+
 table.sort(v5_results, function(a, b) return a.time < b.time end)
 
+
 print('\nUUID v5 (name-based and SHA-1) generation if supported')
 for i, result in ipairs(v5_results) do
   print(string.format('%d. %s\ttook:\t%fs', i, result.module, result.time))
 end
 
+
 -------------
 -- Validation
 -------------
-tests = {
+
+
+local tests = {
   ['FFI binding                      '] = ffi_uuid.is_valid,
   ['resty-jit-uuid (JIT PCRE enabled)'] = luajit_uuid.is_valid,
   ['resty-jit-uuid (Lua patterns)    '] = luajit_uuid_no_pcre.is_valid
 }
 
+
 local uuids = {}
 local p_invalid_uuids = p_valid_uuids + (100 - p_valid_uuids) / 2
+
+
 for i = 1, n_uuids do
   local r = math.random(0, 100)
+
   if r <= p_valid_uuids then
     uuids[i] = ffi_uuid.generate_random()
+
   elseif r <= p_invalid_uuids then
     uuids[i] = '03111af4-f2ee-11e5-ba5e-43ddcc7efcdZ' -- invalid UUID
+
   else
     uuids[i] = '03111af4-f2ee-11e5-ba5e-43ddcc7efcd' -- invalid length
   end
 end
 
+
 local valid_results = {}
+
+
 for k, validate in pairs(tests) do
   collectgarbage()
-  local tstart = os.clock()
+
   local check = {}
+  local tstart = os.clock()
+
   for i = 1, n_uuids do
     local ok = validate(uuids[i])
     check[ok] = true
   end
+
   -- make sure there is no false positives here
   if not check[true] or not check[false] then
     error('all validations have the same result for '..k)
   end
-  valid_results[#valid_results+1] = {module = k, time = os.clock() - tstart}
+
+  valid_results[#valid_results+1] = {
+    module = k,
+    time = os.clock() - tstart
+  }
 end
 
+
 table.sort(valid_results, function(a, b) return a.time < b.time end)
 
+
 print(string.format('\nUUID validation if supported (set of %d%% valid, %d%% invalid)',
   p_valid_uuids,
   100 - p_valid_uuids))
+
 for i, result in ipairs(valid_results) do
-  print(string.format('%d. %s\ttook:\t%fs', i, result.module, result.time))
+  print(string.format('%d. %s\ttook:\t%fs', i,
+        result.module, result.time))
 end

lib/resty/jit-uuid.lua

--- vim:set ts=4 sw=4 et:
+-- vim:set ts=4 sts=4 sw=4 et:
 
 --- jit-uuid
 -- Fast and dependency-free UUID library for LuaJIT/ngx_lua.
@@ -149,7 +149,7 @@ do
     -- local u1 = uuid()             ---> __call metamethod
     -- local u2 = uuid.generate_v4()
     function _M.generate_v4()
-        return fmt('%s%s%s%s-%s%s-%s%s-%s%s-%s%s%s%s%s%s',
+        return (fmt('%s%s%s%s-%s%s-%s%s-%s%s-%s%s%s%s%s%s',
                     tohex(random(0, 255), 2),
                     tohex(random(0, 255), 2),
                     tohex(random(0, 255), 2),
@@ -169,7 +169,7 @@ do
                     tohex(random(0, 255), 2),
                     tohex(random(0, 255), 2),
                     tohex(random(0, 255), 2),
-                    tohex(random(0, 255), 2))
+                    tohex(random(0, 255), 2)))
     end
 end
 
@@ -181,15 +181,71 @@ end
 
 do
     if ngx then
+        local ffi = require 'ffi'
+
+
         local tonumber = tonumber
         local assert   = assert
+        local error    = error
         local concat   = table.concat
-        local gmatch   = string.gmatch
         local type     = type
         local char     = string.char
         local fmt      = string.format
         local sub      = string.sub
-        local buf      = {}
+        local gmatch   = ngx.re.gmatch
+        local sha1_bin = ngx.sha1_bin
+        local md5      = ngx.md5
+        local C        = ffi.C
+        local ffi_new  = ffi.new
+        local ffi_str  = ffi.string
+        local ffi_cast = ffi.cast
+        local new_tab
+        do
+            local ok
+            ok, new_tab = pcall(require, 'table.new')
+            if not ok then
+                new_tab = function(narr, nrec) return {} end
+            end
+        end
+
+
+        ffi.cdef [[
+            typedef unsigned char u_char;
+            typedef intptr_t ngx_int_t;
+
+            u_char * ngx_hex_dump(u_char *dst, const u_char *src, size_t len);
+            ngx_int_t ngx_hextoi(u_char *line, size_t n);
+        ]]
+
+
+        local str_type    = ffi.typeof('uint8_t[?]')
+        local u_char_type = ffi.typeof('u_char *')
+
+
+        local function bin_tohex(s)
+            local slen = #s
+            local blen = slen * 2
+            local buf = ffi_new(str_type, blen)
+
+            C.ngx_hex_dump(buf, s, slen)
+
+            return ffi_str(buf, blen)
+        end
+
+
+        local function hex_to_i(s)
+            local buf = ffi_cast(u_char_type, s)
+
+            local n = tonumber(C.ngx_hextoi(buf, #s))
+            if n == -1 then
+                error("could not convert hex to number")
+            end
+
+            return n
+        end
+
+
+        local buf = new_tab(16, 0)
 
 
         local function factory(namespace, hash_fn)
@@ -198,16 +254,23 @@ do
             end
 
             local i = 0
-            local iter = gmatch(namespace, '([%a%d][%a%d])') -- pattern faster than PCRE without resty.core
+            local iter, err = gmatch(namespace, [[([\da-f][\da-f])]])
+            if not iter then
+                return nil, 'could not create iter: ' .. err
+            end
 
             while true do
-                local m = iter()
+                local m, err = iter()
+                if err then
+                    return nil, err
+                end
+
                 if not m then
                     break
                 end
 
                 i = i + 1
-                buf[i] = char(tonumber(m, 16))
+                buf[i] = char(tonumber(m[0], 16))
             end
 
             assert(i == 16, "invalid binary namespace buffer length")
@@ -219,17 +282,35 @@ do
 
                 local hash, ver, var = hash_fn(concat(buf, ''), name)
 
-                return fmt('%s-%s-%s%s-%s%s-%s', sub(hash, 1, 8),
+                return (fmt('%s-%s-%s%s-%s%s-%s', sub(hash, 1, 8),
                                                 sub(hash, 9, 12),
                                                 ver,
                                                 sub(hash, 15, 16),
                                                 var,
                                                 sub(hash, 19, 20),
-                                                sub(hash, 21, 32))
+                                                sub(hash, 21, 32)))
             end
         end
 
 
+        local function v3_hash(binary, name)
+            local hash = md5(binary .. name)
+
+            return hash,
+            tohex(bor(band(hex_to_i(sub(hash, 13, 14)), 0x0F), 0x30), 2),
+            tohex(bor(band(hex_to_i(sub(hash, 17, 18)), 0x3F), 0x80), 2)
+        end
+
+
+        local function v5_hash(binary, name)
+            local hash = bin_tohex(sha1_bin(binary .. name))
+
+            return hash,
+            tohex(bor(band(hex_to_i(sub(hash, 13, 14)), 0x0F), 0x50), 2),
+            tohex(bor(band(hex_to_i(sub(hash, 17, 18)), 0x3F), 0x80), 2)
+        end
+
+
         --- Instanciate a v3 UUID factory.
         -- @function factory_v3
         -- Creates a closure generating namespaced v3 UUIDs.
@@ -246,21 +327,8 @@ do
         --
         -- local u2 = fact('foobar')
         -- ---> e8d3eeba-7723-3b72-bbc5-8f598afa6773
-        do
-            local md5 = ngx.md5
-
-
-            local function v3_hash(binary, name)
-                local hash = md5(binary .. name)
-                return hash,
-                        tohex(bor(band(tonumber(sub(hash, 13, 14), 16), 0x0F), 0x30), 2),
-                        tohex(bor(band(tonumber(sub(hash, 17, 18), 16), 0x3F), 0x80), 2)
-            end
-
-
-            function _M.factory_v3(namespace)
-                return factory(namespace, v3_hash)
-            end
+        function _M.factory_v3(namespace)
+            return factory(namespace, v3_hash)
         end
 
 
@@ -280,42 +348,8 @@ do
         --
         -- local u2 = fact('foobar')
         -- ---> c9be99fc-326b-5066-bdba-dcd31a6d01ab
-        do
-            local ffi = require 'ffi'
-
-
-            local sha1_bin = ngx.sha1_bin
-            local C        = ffi.C
-            local ffi_new  = ffi.new
-            local ffi_str  = ffi.string
-            local str_type = ffi.typeof('uint8_t[?]')
-
-            ffi.cdef [[
-                typedef unsigned char u_char;
-                u_char * ngx_hex_dump(u_char *dst, const u_char *src, size_t len);
-            ]]
-
-
-            local function bin_tohex(s)
-                local len = #s * 2
-                local buf = ffi_new(str_type, len)
-                C.ngx_hex_dump(buf, s, #s)
-
-                return ffi_str(buf, len)
-            end
-
-
-            local function v5_hash(binary, name)
-                local hash = bin_tohex(sha1_bin(binary .. name))
-                return hash,
-                        tohex(bor(band(tonumber(sub(hash, 13, 14), 16), 0x0F), 0x50), 2),
-                        tohex(bor(band(tonumber(sub(hash, 17, 18), 16), 0x3F), 0x80), 2)
-            end
-
-
-            function _M.factory_v5(namespace)
-                return factory(namespace, v5_hash)
-            end
+        function _M.factory_v5(namespace)
+            return factory(namespace, v5_hash)
         end
 
 

t/01-sanity.t

-# vi:ts=4 sw=4 et fdm=marker:
+# vi: sts=4 ts=4 sw=4 et fdm=marker:
 use Test::Nginx::Socket::Lua;
 use t::Util;
 

t/02-is_valid.t

-# vim:set ts=4 sw=4 et fdm=marker:
+# vim:set sts=4 ts=4 sw=4 et fdm=marker:
 use Test::Nginx::Socket::Lua;
 use t::Util;
 
@@ -6,7 +6,7 @@ our $HttpConfig = $t::Util::HttpConfig;
 
 master_on();
 
-plan tests => repeat_each() * blocks() * 3;
+plan tests => repeat_each() * blocks() * 3 + 4;
 
 run_tests();
 
@@ -124,3 +124,50 @@ false
 --- no_error_log
 [error]
 
+
+
+=== TEST 3: is_valid() JIT compiles with invalid
+--- http_config eval: $t::Util::HttpConfigJit
+--- config
+    location /t {
+        content_by_lua_block {
+            local uuid = require 'resty.jit-uuid'
+
+            for _ = 1, 100 do
+                uuid.is_valid("")
+            end
+        }
+    }
+--- request
+GET /t
+--- response_body
+
+--- error_log eval
+qr/\[TRACE   \d+ content_by_lua\(nginx\.conf:\d+\):4 loop\]/
+--- no_error_log
+[error]
+-- NYI:
+
+
+
+=== TEST 4: is_valid() JIT compiles with valid
+--- http_config eval: $t::Util::HttpConfigJit
+--- config
+    location /t {
+        content_by_lua_block {
+            local uuid = require 'resty.jit-uuid'
+
+            for _ = 1, 100 do
+                uuid.is_valid("cbb297c0-a956-486d-ad1d-f9b42df9465a")
+            end
+        }
+    }
+--- request
+GET /t
+--- response_body
+
+--- error_log eval
+qr/\[TRACE   \d+ content_by_lua\(nginx\.conf:\d+\):4 loop\]/
+--- no_error_log
+[error]
+-- NYI:

t/03-v4.t

-# vim:set ts=4 sw=4 et fdm=marker:
+# vim:set sts=4 ts=4 sw=4 et fdm=marker:
 use Test::Nginx::Socket::Lua;
 use t::Util;
 
@@ -6,7 +6,7 @@ our $HttpConfig = $t::Util::HttpConfig;
 
 master_on();
 
-plan tests => repeat_each() * blocks() * 3;
+plan tests => repeat_each() * blocks() * 3 + 2;
 
 run_tests();
 
@@ -30,3 +30,27 @@ GET /t
 ^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$
 --- no_error_log
 [error]
+
+
+
+=== TEST 2: generate_v4() JIT compiles
+--- http_config eval: $t::Util::HttpConfigJit
+--- config
+    location /t {
+        content_by_lua_block {
+            local uuid = require 'resty.jit-uuid'
+
+            for _ = 1, 100 do
+                uuid()
+            end
+        }
+    }
+--- request
+GET /t
+--- response_body
+
+--- error_log eval
+qr/\[TRACE   \d+ content_by_lua\(nginx\.conf:\d+\):4 loop\]/
+--- no_error_log
+[error]
+-- NYI:

t/04-v3.t

-# vim:set ts=4 sw=4 et fdm=marker:
+# vim:set sts=4 ts=4 sw=4 et fdm=marker:
 use Test::Nginx::Socket::Lua;
 use t::Util;
 
@@ -6,7 +6,7 @@ our $HttpConfig = $t::Util::HttpConfig;
 
 master_on();
 
-plan tests => repeat_each() * blocks() * 3 - 2;
+plan tests => repeat_each() * blocks() * 3 - 2 + 2;
 
 run_tests();
 
@@ -173,3 +173,28 @@ name must be a string
 --- no_error_log
 [error]
 
+
+
+=== TEST 8: generate_v3() JIT compiles with resty.core
+--- http_config eval: $t::Util::HttpConfigJit
+--- config
+    location /t {
+        content_by_lua_block {
+            local uuid = require 'resty.jit-uuid'
+
+            local factory = uuid.factory_v3('cbb297c0-a956-486d-ad1d-f9b42df9465a')
+
+            for _ = 1, 100 do
+                factory('hello')
+            end
+        }
+    }
+--- request
+GET /t
+--- response_body
+
+--- error_log eval
+qr/\[TRACE   \d+ content_by_lua\(nginx\.conf:\d+\):6 loop\]/
+--- no_error_log
+[error]
+-- NYI:

t/05-v5.t

-# vim:set ts=4 sw=4 et fdm=marker:
+# vim:set sts=4 ts=4 sw=4 et fdm=marker:
 use Test::Nginx::Socket::Lua;
 use t::Util;
 
@@ -6,7 +6,7 @@ our $HttpConfig = $t::Util::HttpConfig;
 
 master_on();
 
-plan tests => repeat_each() * blocks() * 3 - 2;
+plan tests => repeat_each() * blocks() * 3 - 2 + 2;
 
 run_tests();
 
@@ -173,3 +173,28 @@ name must be a string
 --- no_error_log
 [error]
 
+
+
+=== TEST 8: generate_v5() JIT compiles with resty.core
+--- http_config eval: $t::Util::HttpConfigJit
+--- config
+    location /t {
+        content_by_lua_block {
+            local uuid = require 'resty.jit-uuid'
+
+            local factory = uuid.factory_v5('cbb297c0-a956-486d-ad1d-f9b42df9465a')
+
+            for _ = 1, 100 do
+                factory('hello')
+            end
+        }
+    }
+--- request
+GET /t
+--- response_body
+
+--- error_log eval
+qr/\[TRACE   \d+ content_by_lua\(nginx\.conf:\d+\):6 loop\]/
+--- no_error_log
+[error]
+-- NYI:

t/Util.pm

@@ -17,11 +17,30 @@ _EOC_
 }
 
 our $HttpConfig = <<_EOC_;
-    lua_package_path \'./lib/?.lua;./lib/?/init.lua;;\';
+    lua_package_path './lib/?.lua;./lib/?/init.lua;;';
 
     init_by_lua_block {
       $LuaCovRunner
     }
 _EOC_
 
+our $HttpConfigJit = <<_EOC_;
+    lua_package_path './lib/?.lua;./lib/?/init.lua;;';
+
+    init_by_lua_block {
+        local verbose = false
+
+        if verbose then
+            local dump = require "jit.dump"
+            dump.on(nil, "$Test::Nginx::Util::ErrLogFile")
+
+        else
+            local v = require "jit.v"
+            v.on("$Test::Nginx::Util::ErrLogFile")
+        end
+
+        require "resty.core"
+    }
+_EOC_
+
 1;