Module:Testharness/consolefriendly

-- -- Scribunto unit test framework. -- @module             th -- @version             2.0.7 -- @usage -- @see                Global Lua Modules/Testharness

-- Module package. local th = {}

-- Module variables. local wdsButton = require('Dev:WDS Button') local yesno = require('Dev:Yesno') local i18n = require('Dev:I18n').loadMessages('Testharness') local frame

-- Unit testing utilites (all private). -- @section            u8t_utils local u8t_utils = {}

-- Scribunto preprocessing turns '^(#|*)' into '^\n$1'. -- Remove the trailing newlines for ease of testing. -- @param              {string} str Raw wikitext. -- @returns            {string} Preprocessed text. function u8t_utils.pp(str) return mw.text.trim(frame.preprocess and frame:preprocess(str) or str) end

-- String comparator. -- @param              {string} str1 first string -- @param              {string} str2 second string -- @param              {string|nil} key member key -- @returns            {string} differing 1-index/key or empy string function u8t_utils.diff_str(str1, str2, key) if str1 == nil and str2 == nil then return '' end local typ1 = type(str1) local typ2 = type(str2) if typ1 ~= typ2 then return 'type' end if typ1 == typ2 and typ1 ~= 'string' then return str1 == str2 and '' or ' ' end str1 = tostring(str1) str2 = tostring(str2) if str1 == str2 then return '' end local max = math.min(#str1, #str2) for i = 1, max do       if str1:sub(i, i) ~= str2:sub(i, i) then return key and tostring(key) or tostring(i) end end return key and tostring(key) or tostring(max + 1) end

-- Table comparator. -- @param              {table} tbl1 First table. -- @param              {table} tbl2 Second table. -- @param              {string|nil} key Key of parent member. -- @returns            {string} First different key or empty string. function u8t_utils.diff_tbl(tbl1, tbl2, key) -- Type comparision. local ty1 = type(tbl1) local ty2 = type(tbl2) if ty1 ~= ty2 then return key and ('"' .. key .. '"') or 'type' end local OTHER_TYPES = { ['string'] = 1, ['number'] = 1, ['nil']   = 1 }   -- String comparision. if OTHER_TYPES[ty1] and OTHER_TYPES[ty2] then return u8t_utils.diff_str(tbl1, tbl2, key) end -- Strip table methods. u8t_utils.filter_method(tbl1) u8t_utils.filter_method(tbl2) -- Table comparision. for key1, val1 in pairs(tbl1) do       local val2 = tbl2[key1] local nest1 = u8t_utils.diff_tbl(val1, val2, ((key1 and (key1 .. '.') or '') .. key1)) if nest1 ~= '' then return '"' .. nest1 .. '"' elseif val2 == nil then return '"' .. key1 .. '"' end end for key2, val2 in pairs(tbl2) do       local val1 = tbl1[key2] local nest2 = u8t_utils.diff_tbl(val1, val2, ((key2 and (key2 .. '.') or '') .. key2)) if nest2 ~= '' then return '"' .. nest2 .. '"' elseif val1 == nil then return '"' .. key2 .. '"' end end -- Return empty string if equal. return '' end

-- Utility to remove methods from a table. -- @param              {table} tbl Table. function u8t_utils.filter_method(tbl) for key, val in pairs(tbl) do       if type(val) == 'function' then tbl[key] = nil end end end

-- Escapes a string. -- @param              {string} str String to escape. -- @returns            {string} Escaped string. function u8t_utils.escape_str(str) local test = str:find('"') and not str:find('\'')   local quote = test and "'" or '"' local pattern = test and '\n' or '[\n"]'   local repl = {        ['\n'] = '\\n',        ['"'] = '\\"'    }    return quote .. str:gsub(pattern, repl) .. quote end

-- Value-string conversion. -- @param              {string|table|number} val Value. -- @returns            {string} String representation. function u8t_utils.val_to_str(val) local typ = type(val) local fn   if typ == 'string' then fn = u8t_utils.escape_str elseif typ == 'table' then fn = u8t_utils.tbl_to_str else fn = tostring end return fn(val) end

-- Table key-string conversion. -- @param              {string|table|number} key Key. -- @returns            {string} String representation. function u8t_utils.tbl_key_to_str(key) if type(key) == 'string' and mw.ustring.match(key, '^[_%a][_%a%d]*$') then return key else return '[' .. u8t_utils.val_to_str(key) .. ']'   end end

-- Table-string conversion. -- @param              {table} tbl Table. -- @returns            {string} String representation. function u8t_utils.tbl_to_str(tbl) local ret = {} local ikeys = {} for key, val in ipairs(tbl) do       table.insert(ret, u8t_utils.val_to_str(val)) ikeys[key] = true end for key, val in pairs(tbl) do       if not ikeys[key] then table.insert(ret, u8t_utils.tbl_key_to_str(key) .. ' = ' .. u8t_utils.val_to_str(val)) end end return '{' .. table.concat(ret, ', ') .. '}' end

-- Wikitext representations of Lua code. u8t_utils.wikitext = {}

-- Lua method test case in wikitext form. -- @param              {string} n Module name. -- @param              {string} m Member name. -- @param              {string|table|number} c Test case. -- @param              {table} o Test options. -- @returns            {string} String representation. function u8t_utils.wikitext.method(n, m, c, o)   local tmp if type(c) == 'table' then local t = u8t_utils.tbl_to_str(c) if o.unpk == true then tmp = t:gsub('^{', ):gsub('}$', ) else tmp = t       end elseif type(c) == 'string' then tmp = '"' .. c .. '"' elseif c ~= nil then tmp = tostring(c) else tmp = '' end return tostring(n) .. (o.self == true and ':' or '.') .. tostring(m) .. '(' .. tmp .. ')' end

-- Lua table test case in wikitext form. -- @param              {string} n Module name. -- @param              {string} m Member name. -- @param              {string|number} c Test case. -- @param              {table} o Test options. -- @returns            {string} String representation. function u8t_utils.wikitext.table(n, m, c, o)   local quote = type(c) == 'string' and '"' or ''    return tostring(n) .. '.' .. tostring(m) .. '[' .. quote .. tostring(c) .. quote .. ']' end

-- Lua invocation in wikitext form. -- @param              {string} n Module name. -- @param              {string} m Member name. -- @param              {string} c Test case. -- @param              {table} o Test options. -- @returns            {string} String representation. function u8t_utils.wikitext.invocation(n, m, c, o)   return '' end

-- Unit tester class. -- @type               U8T -- @section            u8t local U8T = {}

-- Unit tester for package methods. -- @name               U8T.test_method -- @param              {function} m Test member. -- @param              {string|table|number|nil} c Test case. -- @param              {table} o Test options. -- @param              {table} p Module package. -- @returns            {table} Test case result data. function U8T.test_method(m, c, o, p)   local t = os.clock local e, s   if o.unpk == true then if o.self == true then e, s = pcall(m, p, unpack(c)) else e, s = pcall(m, unpack(c)) end else if o.self == true then e, s = pcall(m, p, c)       else e, s = pcall(m, c)       end end return { e, s, 1000 * (os.clock - t) } end

-- Unit tester for invocation methods. -- @name               U8T.test_invocation -- @param              {function} m Test member. -- @param              {string|nil} c test case. -- @param              {table} o test options. -- @returns            {table} Test case result data. function U8T.test_invocation(m, c, o)   -- Function variables. local n = {} local a = {} -- Test case argument parsing. if type(c) == 'string' then -- Parser defense. for s in mw.text.gsplit(c, '|') do           local _, ob = (n[#n] or ):gsub('[{%[]', ) -- opening braces local _, cb = (n[#n] or ):gsub('[}%]]', ) -- closing braces if (ob - cb) == 0 then n[#n+1] = s           else n[#n] = n[#n] .. '|' .. s           end end -- Argument and whitespace parsing. local p_i = 0 for i, s in ipairs(n) do           -- Named parameters. if mw.ustring.find(s, '^%s*[^=%s{}]+%s*=') then local p, v = mw.ustring.match(s, '^([^=]+)=([%s%S]*)$') n[i] = mw.text.trim(p) .. '=' .. mw.text.trim(v) -- Anonymous parameters. else p_i = p_i + 1 n[i] = tostring(p_i) .. '=' .. s           end end -- Argument extraction to table. for i, s in ipairs(n) do           local p, v = mw.ustring.match(s, '([^=]+)=([%s%S]*)') a[tonumber(p) or p] = v       end -- Argument preprocessing. for p, s in pairs(a) do           a[p] = frame.preprocess and frame:preprocess(s) or s        end end -- Invoke method using synthetic frame. frame.args = a   local t = os.clock local out = { pcall(m, frame) } out[2] = out[2] == nil and '' or tostring(out[2]) out[3] = (1000 * (os.clock - t)) -- Return invocation output. return out end

-- Unit tester for package methods. -- @name               U8T.test_table -- @param              {table} m Test member. -- @param              {string|number} c Test case. -- @param              {table} o Test options. -- @returns            {table} Test case result data. function U8T.test_table(m, c, o)   return { true, m[c], 0 } end

-- @todo Document this function. -- @todo Tidy parameters. local function render_report(suite_status, total_tests, failed_tests, config, failed_members, time_exec, test_results, member_fails) -- Unit testing report. local report = mw.html.create:wikitext(       ,        ,        ''    ) -- Report infobox. report:tag('table') :addClass('infobox WikiaTable') :css({           ['margin-right'] = '0',            ['margin-top']   = '0'        }) :tag('tr') :tag('th') :wikitext(i18n:msg('suite-status')) :done :tag('td') :wikitext(wdsButton._bubble( suite_status and i18n:msg('passed') or i18n:msg('failed'), suite_status and 'pass' or 'fail' ))               :done :done :tag('tr') :tag('th') :wikitext(i18n:msg('test-cases')) :done :tag('td') :wikitext(wdsButton._bubble( tostring(total_tests - failed_tests) .. '/' .. tostring(total_tests), suite_status and 'pass' or 'fail' ))               :done :done :tag('tr') :tag('th') :wikitext(i18n:msg('code-cov')) :done :tag('td') :wikitext(wdsButton._bubble( config.res, (function(c)                       local r = mw.text.split(c.res, '/')                        if #c.pos == 0 then                            return 'nil'                        elseif r[1] ~= r[2] then                            return 'warn'                        else                            return 'pass'                        end                    end)(config) ))               :done :done -- Report lede. report:tag('b') :wikitext(i18n:msg('report-lede', config.name)) -- Report summary. report:tag('ul') :tag('li') :wikitext(i18n:msg('report-missing') .. ': ' .. (#config.neg ~= 0 and ' •  ' or i18n:msg('report-none') ))           :done :tag('li') :wikitext(i18n:msg('report-failing') .. ': ' .. (#failed_members ~= 0 and ' •  ' .. ''                   or  i18n:msg('report-none') )           )            :done :tag('li') :wikitext(i18n:msg('report-time') .. ': ' .. tostring(math.floor(time_exec * 10 + 0.5) / 10) .. 'ms') :done -- Report results header. report:tag('h2') :wikitext(i18n:msg('test-cases')) -- Report results. for test_member, test_output in pairs(test_results) do       local member_nowiki = test_output.options.nowiki == true and function(...) return mw.text.nowiki(tostring(...)) end or tostring local result_table = report:tag('table') :addClass('WikiaTable mw-collapsible') :css('width', '100%') -- Member header. result_table:tag('tr') :tag('th') :attr('colspan', (config.diff == true) and 5 or 4) :tag('h3') :attr('id', test_member) :css('display', 'inline') :wikitext(' ') :done :wikitext(' ' .. wdsButton._bubble( (#test_output - (member_fails[test_member] or 0)) .. '/' .. #test_output, ((member_fails[test_member] or 0) == 0) and 'pass' or 'fail' ))               :done -- Member column headers. result_table:tag('tr') :tag('th') :wikitext(i18n:msg('header-status')) :done :tag('th') :wikitext(i18n:msg('header-code')) :done :tag('th') :wikitext(i18n:msg('header-expect')) :done :tag('th') :wikitext(i18n:msg('header-actual')) :done :node(config.diff == true               and mw.html.create('th'):wikitext(i18n:msg('header-diff'))                or  nil            ) -- Member column data. for test_index, test_result in ipairs(test_output) do           result_table:tag('tr') :tag('td') :wikitext(wdsButton._badge( i18n:msg((test_result.status == true) and 'passing' or 'failing'), (test_result.status == true) and 'pass' or 'fail' ))                   :done :tag('td') :tag('pre') :css({                           ['max-width']   = '200px',                            ['white-space'] = 'pre-wrap',                            ['word-wrap']   = 'break-word'                        }) :wikitext(mw.text.nowiki(u8t_utils.wikitext[test_output.options.mode](                           test_output.options.mode == 'invocation'                                and (mw.ustring.gsub((config.name:gsub('Module:', '', 1)), '^%u', mw.ustring.lower))                                or  config.pkg,                            test_member,                            test_result.case,                            test_output.options                        ))) :done :done :tag('td') :css({                       ['white-space'] = 'pre-wrap',                        ['word-break']  = 'break-all'                    }) :wikitext(                       (test_result.err[1] == true and wdsButton._bubble(                                   i18n:msg('error'),                                    'warn'                                ) .. ' '                           or  '' ) ..                       member_nowiki(type(test_result.expect) == 'table' and u8t_utils.tbl_to_str(test_result.expect) or test_result.expect )                   )                    :done :tag('td') :css({                       ['white-space'] = 'pre-wrap',                        ['word-break']  = 'break-all'                    }) :wikitext(                       (test_result.err[2] == true and wdsButton._bubble(                                   i18n:msg('error'),                                    'warn'                                ) .. ' '                           or  '' ) ..                       member_nowiki(type(test_result.actual) == 'table' and u8t_utils.tbl_to_str(test_result.actual) or test_result.actual )                   )                    :done :node(config.diff == true                   and mw.html.create('td')                        :wikitext(test_result.diff_i)                    or  nil                ) end end return tostring(report) end

-- Unit test execution engine. -- @name               run_tests -- @param              {table} test_suite Test case suite. -- @param              {table} test_module Module package. -- @param              {table} config Test configuration. -- @returns            {table} Test case result data. local function run_tests(test_suite, test_module, config) -- Generate test data. local test_results = {} local time_exec = 0 for test_member, test_data in pairs(test_suite) do       test_results[test_member] = { ['options'] = test_data.options }       for test_index, test_case in ipairs(test_data.tests) do            -- Execute member test. local test_res = U8T['test_' .. test_data.options.mode](               test_module[test_member], test_case[1], test_data.options, test_module            ) time_exec = time_exec + test_res[3] local pp = test_data.options.preprocess or (test_case[3] or {}).pp           -- Generate test case result. table.insert(test_results[test_member], {               -- Test case data.                ['case']   = test_case[1],                ['actual'] = test_res[2],                ['expect'] = pp                    and u8t_utils.pp(test_case[2])                    or  test_case[2],                -- Test case analysis.                ['err']    = {                    ((test_case[3] or {}).err == true),                    (test_res[1] == false)                }            }) -- Test case error preprocessing. local l = #test_results[test_member] local r = test_results[test_member][l].err if r[2] == true then test_results[test_member][l].actual = mw.ustring.match(test_results[test_member][l].actual, '%d:%s([%s%S]+)') end -- Test case comparision. local a = test_results[test_member][l].actual local e = test_results[test_member][l].expect test_results[test_member][l].diff_i = (r[1] == r[2]) and (test_data.options.deep == true                   and u8t_utils.diff_tbl(e, a)                    or  u8t_utils.diff_str(e, a)                ) or 'error' end end -- Derived data required for report. local member_fails = {} local failed_members = {} local total_tests = 0 local failed_tests = 0 for test_member, test_output in pairs(test_results) do       for test_index, test_result in ipairs(test_output) do            -- Result count and status. total_tests = total_tests + 1 member_fails[test_member] = member_fails[test_member] or 0 test_result.status = (               (test_result.err[1] == test_result.err[2]) and                #test_result.diff_i == 0            ) -- Test logic. if test_result.status == false then member_fails[test_member] = member_fails[test_member] + 1 failed_tests = failed_tests + 1 end end end for test_member, fail_count in pairs(member_fails) do       if fail_count ~= 0 then table.insert(failed_members, test_member) end end -- Generate report. local suite_status = (failed_tests == 0) return render_report(suite_status, total_tests, failed_tests, config, failed_members, time_exec, test_results, member_fails) end

-- Test harness logic. -- @section            th

-- Code coverage analyser. -- @param              {string} module_name Module name. -- @param              {string} test_module Test module package. -- @param              {string} test_cases Test case data. -- @returns            {table} Code coverage data. function th.code_cov(module_name, test_module, test_suite) -- Function variables. local members = {} local raw_text = mw.title.new(module_name):getContent local package_name = raw_text:match('\nreturn (%S+)\n?') or 'p'   local raw_functions = {} local coverage = {} local coverage_old = {} local positive = {} local negative = {} -- Direct coverage. for member_key, val in pairs(test_module) do       table.insert(members, member_key) raw_functions[member_key] = raw_text:match(package_name .. '[:.]' .. member_key .. '(.-\n%S+)\n') if test_suite[member_key] then coverage[member_key] = true end end -- Source code coverage. while (coverage_old ~= coverage) do       coverage_old = coverage for member_key, _ in pairs(coverage) do           for _, p in ipairs({ 'self', package_name }) do                -- Test for code access. for member_key in raw_functions[member_key]:gmatch(p .. '[[.:"\']+([^\'%s([]+)') do                   if test_module[member_key] then                        coverage[member_key] = true                    end                end            end        end    end    -- Package members with positive coverage.    for member_key, _ in pairs(coverage) do        table.insert(positive, member_key)    end    table.sort(positive)    -- Package members with negative coverage.    for member_key, _ in pairs(test_module) do        if coverage[member_key] == nil then            table.insert(negative, member_key);        end    end    table.sort(negative)    -- Output result.    return {        ['pkg'] = package_name,        ['neg'] = negative,        ['pos'] = positive,        ['res'] = tostring(#positive) .. '/' .. tostring(#members)    } end

-- Test harness function. -- @param              {table} f Frame object. -- @returns            {string} Test report. function th.run_tests(f) -- Frame arguments. frame = f   local module_name = (frame.args or {}).modulename or '' local test_data  = (frame.args or {}).testdata or '' local differs_at = (frame.args or {}).differs_at or 'true' -- Argument validation and defaults. local module_ns = mw.site.namespaces[828].canonicalName module_name = #module_name ~= 0 and frame.args.modulename or mw.title.getCurrentTitle.baseText module_name = module_ns .. ':' .. module_name test_data = #test_data ~= 0 and module_ns .. ':' .. test_data or module_name .. '/testcases' differs_at = (yesno(differs_at) == true or #differs_at == 0) and true or false -- Load module package. local test_module = require(module_name) -- Load test cases. local _, test_suite = pcall(require, test_data) test_suite = test_suite or {} -- Remove string members. for member_key, val in pairs(test_module) do       if type(val) == 'string' then test_module[member_key] = nil end end -- Check execution validity. for test_member, test_data in pairs(test_suite) do       if            not test_module[test_member] or      -- Nonexistent member. type(test_data) ~= 'table' or       -- Old UnitTests format. not (test_data.options or {}).mode  -- Unconfigured tests. then test_suite[test_member] = nil end end -- Unit test configuration. local config = th.code_cov(module_name, test_module, test_suite) config['diff'] = differs_at config['name'] = module_name -- Execute tests & generate report. return run_tests(test_suite or {}, test_module, config) end

return th --