Module:Testharness

-- -- Scribunto unit test framework. -- @module             th -- @version             2.0.5 -- @usage -- @see                Global Lua Modules/Testharness

-- Module package. local th = {}

-- Module variables. local i18n = require('Dev:I18n').loadMessages('Testharness') local bool = { ['1']   = true, ['true'] = true } local frame

-- Unit testing UI and reporting logic. -- @section            u8t_ui local u8t_ui = {}

-- Unit testing report. u8t_ui.report = mw.html.create:wikitext(table.concat({ ,   ,    '' }))

-- UI color map. u8t_ui.colors = { ['pass'] = '#32cd32', ['fail'] = '#990000', ['warn'] = '#ffba01', ['nil'] = '#aaaaaa' }

-- Badge generator - for emphasised text. -- @param              {table} o configuration -- @param              {string} o.t badge wikitext -- @param              {string} o.s status of color -- @returns            {string} HTML badge function u8t_ui.badge(o) return tostring(mw.html.create('span')       :css({ ['background'] = u8t_ui.colors[tostring(o.s)], ['border'] = 'none', ['cursor'] = 'inherit', ['font-weight'] = 'bold' })       :attr('class', 'wds-button')        :wikitext(o.t)    ) end

-- Bubble generator - for result indicators. -- @param              {table} o configuration -- @param              {string} t bubble wikitext -- @param              {string} s status of color -- @returns            {string} HTML bubble function u8t_ui.bubble(o) return tostring(mw.html.create('span')       :css({ ['background'] = u8t_ui.colors[tostring(o.s)], ['border'] = 'none', ['border-radius'] = '0.5em', ['padding'] = '0 0.4em', ['vertical-align'] = 'middle' })       :attr('class', 'wds-button wds-font-size-xxs')        :wikitext(o.t)    ) end

-- Unit testing utilites (all private). -- @section            u8t_utils local u8t_utils = {}

-- Scribunto preprocessing turns '^#' into '^\n#'. -- Temporarily remove this until (if and when) this bug is fixed. -- @param              {string} t Raw wikitext. -- @returns            {string} Preprocessed text. function u8t_utils.pp(t) return (mw.ustring.gsub(frame:preprocess(t), '^\n#', '#')) end

-- String comparator. -- @param              {string} s1 first string -- @param              {string} s2 second string -- @param              {string|nil} k key of member -- @returns            {string} differing 1-index/key or empy string function u8t_utils.diff_str(s1, s2, k)   if s1 == nil and s2 == nil then return '' end if type(s1) ~= type(s2) then return 'type' end if type(s1) == type(s2) and type(s1) ~= 'string' then return s1 == s2 and '' or ' ' end s1, s2 = tostring(s1), tostring(s2) if s1 == s2 then return '' else local max = math.min(#s1, #s2) for i = 1, max do           if s1:sub(i,i) ~= s2:sub(i,i) then return k and tostring(k) or tostring(i) end end return k and tostring(k) or tostring(max + 1) end end

-- Table comparator. -- @param              {table} t1 First table. -- @param              {table} t2 Second table. -- @param              {string|nil} k Key of parent member. -- @returns            {string} First different key or empty string. function u8t_utils.diff_tbl(t1, t2, k)   -- Type comparision. local ty1 = type(t1) local ty2 = type(t2) if ty1 ~= ty2 then return k and ('"' .. k .. '"') or 'type' end local oty = { ['string'] = 1, ['number'] = 1, ['nil']   = 1 }   -- String comparision. if oty[ty1] and oty[ty2] then return u8t_utils.diff_str(t1, t2, k)   end -- Strip table methods. u8t_utils.filter_method(t1) u8t_utils.filter_method(t2) -- Table comparision. for k1, v1 in pairs(t1) do       local v2 = t2[k1] local n1 = u8t_utils.diff_tbl(v1, v2, ((k and (k .. '.') or '') .. k1)) if n1 ~= '' then return '"' .. n1 .. '"' elseif v2 == nil then return '"' .. k1 .. '"' end end for k2, v2 in pairs(t2) do       local v1 = t1[k2] local n2 = u8t_utils.diff_tbl(v1, v2, ((k and (k .. '.') or '') .. k2)) if n2 ~= '' then return '"' .. n2 .. '"' elseif v1 == nil then return '"' .. k2 .. '"' end end -- Return empty string if equal. return '' end

-- Utility to remove methods from a table. -- @param              {table} t Table. function u8t_utils.filter_method(t) for k, v in pairs(t) do       if type(v) == 'function' then t[k] = nil end end end

-- Value-string conversion. -- @param              {string|table|number} v Value. -- @returns            {string} String representation. function u8t_utils.val_to_str(v) if type(v) == 'string' then v = mw.ustring.gsub(v, '\n', '\\n') if mw.ustring.match(mw.ustring.gsub(v, '[^\'"]', ''), '^"+$') then return "'" .. v .. "'"       end return '"' .. mw.ustring.gsub(v, '"', '\\"' ) .. '"' else return type(v) == 'table' and u8t_utils.tbl_to_str(v) or tostring(v) end end

-- Table key-string conversion. -- @param              {string|table|number} k Key. -- @returns            {string} String representation. function u8t_utils.tbl_key_to_str(k) if type(k) == 'string' and mw.ustring.match(k, '^[_%a][_%a%d]*$') then return k   else return '[' .. u8t_utils.val_to_str(k) .. ']'   end end

-- Table-string conversion. -- @param              {table} tbl Table. -- @returns            {string} String representation. function u8t_utils.tbl_to_str(tbl) local ret, ikeys = {}, {} for k, v in ipairs(tbl) do       table.insert(ret, u8t_utils.val_to_str(v)) ikeys[k] = true end for k, v in pairs(tbl) do       if not ikeys[k] then table.insert(ret, u8t_utils.tbl_key_to_str(k) .. ' = ' .. u8t_utils.val_to_str(v)) end end return '{' .. table.concat(ret, ', ') .. '}' end

-- Wikitext representations of Lua code. u8t_utils.wikitext = {}

-- Lua method test case in wikitext form. -- @param              {string} n Module name. -- @param              {string} n Member name. -- @param              {string|table|number} c Test case. -- @param              {table} o Test options. -- @returns            {string} String representation. function u8t_utils.wikitext.method(n, m, c, o)   local t = type(c) == 'table' and u8t_utils.tbl_to_str(c) return tostring(n) .. (o.self == true and ':' or '.') .. tostring(m) .. '(' .. (type(c) == 'table' and (o.unpk == true           and mw.ustring.match(t, '^{([%s%S]+)}$')            or  t)        or  type(c) == 'string' and '"' .. tostring(c) .. '"' or (c ~= nil and tostring(c) or '') ) .. ')' end

-- Lua table test case in wikitext form. -- @param              {string} n Module name. -- @param              {string} n Member name. -- @param              {string|number} c Test case. -- @param              {table} o Test options. -- @returns            {string} String representation. function u8t_utils.wikitext.table(n, m, c, o)   return tostring(n) .. '.' .. tostring(m) .. '[' .. ((type(c) == 'string')       and '"' .. tostring(c) .. '"'        or  ''    ) .. ']' end

-- Lua invocation in wikitext form. -- @param              {string} n Module name. -- @param              {string} n Member name. -- @param              {string} c Test case. -- @param              {table} o Test options. -- @returns            {string} String representation. function u8t_utils.wikitext.invocation(n, m, c, o)   return '' end

-- Unit tester class. -- @type               U8T -- @section            u8t local U8T = {}

-- Unit tester for package methods. -- @name               U8T:test_method -- @param              {function} m Test member. -- @param              {string|table|number|nil} c Test case. -- @param              {table} o Test options. -- @returns            {table} Test case result data. function U8T:test_method(m, c, o, p)   local t, e, s = os.clock if o.unpk == true then if o.self == true then e, s = pcall(m, p, unpack(c)) else e, s = pcall(m, unpack(c)) end else if o.self == true then e, s = pcall(m, p, c)       else e, s = pcall(m, c)       end end t = 1000 * (os.clock - t)   return { e, s, t } end

-- Unit tester for invocation methods. -- @name               U8T:test_invocation -- @param              {function} m Test member. -- @param              {string|nil} c test case. -- @param              {table} o test options. -- @returns            {table} Test case result data. function U8T:test_invocation(m, c, o)   -- Function variables. local n = {} local a = {} -- Test case argument parsing. if type(c) == 'string' then -- Parser defense. for s in mw.text.gsplit(c, '|') do           local ob = (#mw.text.split((n[#n] or ), '{') + #mw.text.split((n[#n] or ), '%[') - 2) or 0 -- opening braces local cb = (#mw.text.split((n[#n] or ), '}') + #mw.text.split((n[#n] or ), '%]') - 2) or 0 -- closing braces if (ob - cb) == 0 then n[#n+1] = s           else n[#n] = n[#n] .. '|' .. s           end end -- Argument and whitespace parsing. local p_i = 0 for i, s in ipairs(n) do           -- Named parameters. if mw.ustring.find(s, '^%s*[^=%s{}]+%s*=') then local p, v = mw.ustring.match(s, '^([^=]+)=([%s%S]*)$') p, v = mw.text.trim(p), mw.text.trim(v) n[i] = p .. '=' .. v           -- Anonymous parameters. else p_i = p_i + 1 n[i] = tostring(p_i) .. '=' .. s           end end -- Argument extraction to table. for i, s in ipairs(n) do           local p, v = mw.ustring.match(s, '([^=]+)=([%s%S]*)') p = (tonumber(p) == nil) and p or tonumber(p) a[p] = v       end -- Argument preprocessing. for p, s in pairs(a) do           a[p] = u8t_utils.pp(s) end end -- Invoke method using synthetic frame. frame.args = a   local t = os.clock local out = { pcall(m, frame) } out[2] = out[2] == nil and '' or tostring(out[2]) out[3] = (1000 * (os.clock - t)) -- Return invocation output. return out end

-- Unit tester for package methods. -- @name               U8T:test_table -- @param              {table} m Test member. -- @param              {string|number} c Test case. -- @param              {table} o Test options. -- @returns            {table} Test case result data. function U8T:test_table(m, c, o)   return { true, m[c], 0 } end

-- Unit test execution engine. -- @name               U8T:run_tests -- @param              {table} test_module Module package. -- @param              {string|number} c test case -- @param              {table} o test options -- @returns            {table} Test case result data. function U8T:run_tests(test_module, config) -- Generate test data. local test_results, time_exec = {}, 0 for test_member, test_data in pairs(self) do       test_results[test_member] = { ['options'] = test_data.options }       for test_index, test_case in ipairs(test_data.tests) do            -- Execute member test. local test_res = self['test_' .. test_data.options.mode](               self, test_module[test_member], test_case[1], test_data.options, test_module            ) time_exec = time_exec + test_res[3] local pp = test_data.options.preprocess or (test_case[3] or {}).pp           -- Generate test case result. table.insert(test_results[test_member], {               -- Test case data.                ['case']   = test_case[1],                ['actual'] = test_res[2],                ['expect'] = pp                    and u8t_utils.pp(test_case[2])                    or  test_case[2],                -- Test case analysis.                ['err']    = {                    ((test_case[3] or {}).err == true),                    (test_res[1] == false)                }            }) -- Test case error preprocessing. local l = #test_results[test_member] local r = test_results[test_member][l].err if r[2] == true then test_results[test_member][l].actual = mw.ustring.match(test_results[test_member][l].actual, '%d:%s([%s%S]+)') end -- Test case comparision. local a = test_results[test_member][l].actual local e = test_results[test_member][l].expect test_results[test_member][l].diff_i = (r[1] == r[2]) and (test_data.options.deep == true                   and u8t_utils.diff_tbl(e, a)                    or  u8t_utils.diff_str(e, a)                ) or 'error' end end -- Derived data required for report. local member_fails, failed_members, total_tests, failed_tests = {}, {}, 0, 0 for test_member, test_output in pairs(test_results) do       for test_index, test_result in ipairs(test_output) do            -- Result count and status. total_tests = total_tests + 1 member_fails[test_member] = member_fails[test_member] or 0 test_result.status = (               (test_result.err[1] == test_result.err[2]) and                #test_result.diff_i == 0            ) -- Test logic. if test_result.status == false then member_fails[test_member] = member_fails[test_member] + 1 failed_tests = failed_tests + 1 end end end for test_member, fail_count in pairs(member_fails) do       if fail_count ~= 0 then table.insert(failed_members, test_member) end end -- Generate report. local suite_status = (failed_tests == 0) u8t_ui.table = u8t_ui.report --- Report infobox. :tag('table') :attr({               ['class'] = 'infobox WikiaTable',                ['style'] = 'margin-right: 0; margin-top: 0'            }) :tag('tr') :tag('th') :wikitext(i18n:msg('suite-status')) :done :tag('td') :wikitext(u8t_ui.bubble {                       ['t'] = suite_status and i18n:msg('passed') or i18n:msg('failed'),                        ['s'] = suite_status and 'pass' or 'fail'                    }) :done :done :tag('tr') :tag('th') :wikitext(i18n:msg('test-cases')) :done :tag('td') :wikitext(u8t_ui.bubble {                       ['t'] =                            tostring(total_tests - failed_tests) ..                            '/' .. tostring(total_tests),                        ['s'] = suite_status and 'pass' or 'fail'                    }) :done :done :tag('tr') :tag('th') :wikitext(i18n:msg('code-cov')) :done :tag('td') :wikitext(u8t_ui.bubble {                       ['t'] = config.cov.res,                        ['s'] = (function(c) local r = mw.text.split(c.res, '/') if #c.pos == 0 then return 'nil' elseif r[1] ~= r[2] then return 'warn' else return 'pass' end end)(config.cov)                   }) :done :done :done --- Report lede. :tag('b') :wikitext(i18n:msg('report-lede', config.name)) :done --- Report results. :tag('ul') :tag('li') :wikitext(i18n:msg('report-missing') .. ': ' .. (#config.cov.neg ~= 0 and ' •  ' or i18n:msg('report-none') )           ):done :tag('li') :wikitext(i18n:msg('report-failing') .. ': ' .. (#failed_members ~= 0 and ' •  ' .. ''                       or  i18n:msg('report-none') )               )            :done :tag('li') :wikitext(i18n:msg('report-time') .. ': ' .. tostring(math.floor(time_exec * 10 + 0.5) / 10) .. 'ms') :done :done :tag('h2') :wikitext(i18n:msg('test-cases')) :done :tag('table'):attr({           ['class'] = 'WikiaTable',            ['style'] = 'width: 100%'        }) -- Report table. for test_member, test_output in pairs(test_results) do       local member_nowiki = test_output.options.nowiki == true and function(...) return mw.text.nowiki(tostring(...)) end or tostring u8t_ui.table :tag('tr') :tag('th') :attr('colspan', (config.diff == true) and 5 or 4) :tag('h3') :attr('id', test_member) :css('display', 'inline') :wikitext(' ') :done :wikitext(' ' .. u8t_ui.bubble {                       ['t'] = (#test_output - (member_fails[test_member] or 0)) .. '/' .. #test_output,                        ['s'] = ((member_fails[test_member] or 0) == 0) and 'pass' or 'fail'                    }) :done :done :tag('tr') :tag('th') :wikitext(i18n:msg('header-status')) :done :tag('th') :wikitext(i18n:msg('header-code')) :done :tag('th') :wikitext(i18n:msg('header-expect')) :done :tag('th') :wikitext(i18n:msg('header-actual')) :done :node(config.diff == true                   and mw.html.create('th'):wikitext(i18n:msg('header-diff'))                    or  nil                ) :done for test_index, test_result in ipairs(test_output) do           u8t_ui.table:tag('tr') :tag('td') :wikitext(u8t_ui.badge {                       ['t'] = i18n:msg((test_result.status == true) and 'passing' or 'failing'),                        ['s'] = (test_result.status == true) and 'pass' or 'fail'                    }) :done :tag('td') :tag('pre') :css({                           ['max-width']   = '200px',                            ['white-space'] = 'pre-wrap',                            ['word-wrap']   = 'break-word'                        }) :wikitext(mw.text.nowiki(u8t_utils.wikitext[test_output.options.mode](                           test_output.options.mode == 'invocation'                                and (mw.ustring.gsub((mw.ustring.gsub(config.name, 'Module:', '', 1)), '^%u', mw.ustring.lower))                                or  config.pkg,                            test_member,                            test_result.case,                            test_output.options                        ))) :done :done :tag('td') :css({                       ['white-space'] = 'pre-wrap',                        ['word-break']  = 'break-all'                    }) :wikitext(                       (test_result.err[1] == true and u8t_ui.bubble { ['t'] = i18n:msg('error'), ['s'] = 'warn' } .. ' '                           or  '' ) ..                       member_nowiki(type(test_result.expect) == 'table' and u8t_utils.tbl_to_str(test_result.expect) or test_result.expect )                   )                :done :tag('td') :css({                       ['white-space'] = 'pre-wrap',                        ['word-break']  = 'break-all'                    }) :wikitext(                       (test_result.err[2] == true and u8t_ui.bubble { ['t'] = i18n:msg('error'), ['s'] = 'warn' } .. ' '                           or  '' ) ..                       member_nowiki(type(test_result.actual) == 'table' and u8t_utils.tbl_to_str(test_result.actual) or test_result.actual )                   )                :done :node(config.diff == true                   and mw.html.create('td')                        :wikitext(test_result.diff_i)                    or  nil                ) :done :done end end return tostring(u8t_ui.report) end

-- Unit test suite constructor. -- @param              {table} t Test data. -- @returns            {table} U8T instance. function U8T:new(t) setmetatable(t, self) self.__index = self return t end

-- Test harness logic. -- @section            th

-- Code coverage analyser. -- @param              {string} module_name Module name. -- @param              {string} test_module Test module package. -- @param              {string} test_cases Test case data. -- @returns            {table} Code coverage data. function th.code_cov(module_name, test_module, test_cases) -- Function variables. local m = {} local r = mw.title.new(module_name):getContent local rn = r:match('\nreturn (%S+)\n?') local rfn = {} local c = {} local c_old = {} local o = {} local n = {} -- Direct coverage. for k, v in pairs(test_module) do       table.insert(m, k)        rfn[k] = r:match(rn .. '[:.]' .. k .. '(.-\n%S+)\n') if test_cases[k] then c[k] = true end end -- Source code coverage. while (c_old ~= c) do       c_old = c        for k, v in pairs(c) do            for i, p in ipairs({ 'self', rn }) do                for mbr in rfn[k]:gmatch(p .. '[[.:"\']+([^\'%s([]+)') do                    if test_module[mbr] then c[mbr] = true end                end            end        end    end    -- Package members with positive coverage.    for k, v in pairs(c) do        table.insert(o, k);    end    table.sort(o)    -- Package members with negative coverage.    for k, v in pairs(test_module) do        if c[k] == nil then             table.insert(n, k);        end    end    table.sort(n)    -- Output result.    return {        ['pkg'] = rn,        ['neg'] = n,        ['pos'] = o,        ['res'] = tostring(#o) .. '/' .. tostring(#m)    } end

-- Test harness function. -- @param              {table} f Frame object. -- @returns            {string} Test report. function th.run_tests(f) -- Frame arguments. frame = f   local module_name = (frame.args or {}).modulename or '' local test_data  = (frame.args or {}).testdata or '' local differs_at = (frame.args or {}).differs_at or 'true' -- Argument validation and defaults. local module_ns = mw.site.namespaces[828].canonicalName module_name = #module_name ~= 0 and frame.args.modulename or mw.title.getCurrentTitle.baseText module_name = module_ns .. ':' .. module_name test_data = #test_data ~= 0 and module_ns .. ':' .. test_data or module_name .. '/testcases' differs_at = (bool[differs_at] == true or #differs_at == 0) and true or false -- Load module package. local test_module = require(module_name) -- Unit test configuration. local config = { ['diff'] = differs_at, ['name'] = module_name }   -- Load test cases. local _, test_cases = pcall(require, test_data) test_cases = test_cases or {} -- Remove string members. for member_key, val in pairs(test_module) do       if type(val) == 'string' then test_module[member_key] = nil end end -- Check execution validity. for test_member, test_config in pairs(test_cases) do       if            not test_module[test_member] or      -- Nonexistent member. type(test_config) ~= 'table' or     -- Old UnitTests format. not (test_config.options or {}).mode -- Unconfigured tests. then test_cases[test_member] = nil end end -- Instantiate test cases. local u8t = U8T:new(test_cases or {}) -- Code coverage. config.cov = (function(c)       config.pkg = c.pkg        c.pkg = nil        return c    end)(th.code_cov(module_name, test_module, test_cases)) -- Execute tests & generate report. return u8t:run_tests(test_module, config)

end

return th --