mNo edit summary |
m (lesser amount of debug) |
||
| Line 618: | Line 618: | ||
-- Generate report. |
-- Generate report. |
||
local suite_status = (failed_tests == 0) |
local suite_status = (failed_tests == 0) |
||
| − | -- debug |
||
| − | -- if this shows 2 fns, then we in trouble |
||
| − | -- it shows 2 fns |
||
| − | -- todo: find where the original i18n loaded from |
||
| − | mw.log('run_tests handleargs', handleArgs, handleArgs1) |
||
local ersult, result = pcall(render_report, suite_status, total_tests, failed_tests, config, failed_members, time_exec, test_results, member_fails) |
local ersult, result = pcall(render_report, suite_status, total_tests, failed_tests, config, failed_members, time_exec, test_results, member_fails) |
||
if ersult then |
if ersult then |
||
Latest revision as of 16:56, 15 February 2020
Module documentation[create]
The documentation for this module is missing. Click here to create it.
-- <nowiki>
-- Scribunto unit test framework.
-- @module th
-- @version 2.0.7
-- @usage {{#invoke:testharness|modulename=Module}}
-- @see [[Global Lua Modules/Testharness]]
-- Module package.
local th = {}
-- Module variables.
local wdsButton = require('Dev:WDS Button')
local yesno = require('Dev:Yesno')
local i18n = require('Dev:I18n/consolefriendly').loadMessages('Testharness')
local frame
-- Unit testing utilites (all private).
-- @section u8t_utils
local u8t_utils = {}
-- converts things to human-frienldy strings
-- https://stackoverflow.com/a/27028488
-- @param {object} o thing to dump
-- @returns {string} dumped thing
function dump(o)
if type(o) == 'table' then
local s = '{ '
for k,v in pairs(o) do
if type(k) ~= 'number' then k = '"'..k..'"' end
s = s .. '['..k..'] = ' .. dump(v) .. ','
end
return s .. '} '
else
return tostring(o)
end
end-- dump
-- Scribunto preprocessing turns '^(#|*)' into '^\n$1'.
-- Remove the trailing newlines for ease of testing.
-- @param {string} str Raw wikitext.
-- @returns {string} Preprocessed text.
function u8t_utils.pp(str)
return mw.text.trim(frame.preprocess and frame:preprocess(str) or str)
end
-- String comparator.
-- @param {string} str1 first string
-- @param {string} str2 second string
-- @param {string|nil} key member key
-- @returns {string} differing 1-index/key or empy string
function u8t_utils.diff_str(str1, str2, key)
if str1 == nil and str2 == nil then
return ''
end
local typ1 = type(str1)
local typ2 = type(str2)
if typ1 ~= typ2 then
return 'type'
end
if typ1 == typ2 and typ1 ~= 'string' then
return str1 == str2 and '' or ' '
end
str1 = tostring(str1)
str2 = tostring(str2)
if str1 == str2 then
return ''
end
local max = math.min(#str1, #str2)
for i = 1, max do
if str1:sub(i, i) ~= str2:sub(i, i) then
return key and tostring(key) or tostring(i)
end
end
return key and tostring(key) or tostring(max + 1)
end
-- Table comparator.
-- @param {table} tbl1 First table.
-- @param {table} tbl2 Second table.
-- @param {string|nil} key Key of parent member.
-- @returns {string} First different key or empty string.
function u8t_utils.diff_tbl(tbl1, tbl2, key)
-- Type comparision.
local ty1 = type(tbl1)
local ty2 = type(tbl2)
if ty1 ~= ty2 then
return key and ('"' .. key .. '"') or 'type'
end
local OTHER_TYPES = {
['string'] = 1,
['number'] = 1,
['nil'] = 1
}
-- String comparision.
if OTHER_TYPES[ty1] and OTHER_TYPES[ty2] then
return u8t_utils.diff_str(tbl1, tbl2, key)
end
-- Strip table methods.
u8t_utils.filter_method(tbl1)
u8t_utils.filter_method(tbl2)
-- Table comparision.
for key1, val1 in pairs(tbl1) do
local val2 = tbl2[key1]
local nest1 = u8t_utils.diff_tbl(val1, val2, ((key1 and (key1 .. '.') or '') .. key1))
if nest1 ~= '' then
return '"' .. nest1 .. '"'
elseif val2 == nil then
return '"' .. key1 .. '"'
end
end
for key2, val2 in pairs(tbl2) do
local val1 = tbl1[key2]
local nest2 = u8t_utils.diff_tbl(val1, val2, ((key2 and (key2 .. '.') or '') .. key2))
if nest2 ~= '' then
return '"' .. nest2 .. '"'
elseif val1 == nil then
return '"' .. key2 .. '"'
end
end
-- Return empty string if equal.
return ''
end
-- Utility to remove methods from a table.
-- @param {table} tbl Table.
function u8t_utils.filter_method(tbl)
for key, val in pairs(tbl) do
if type(val) == 'function' then
tbl[key] = nil
end
end
end
-- Escapes a string.
-- @param {string} str String to escape.
-- @returns {string} Escaped string.
function u8t_utils.escape_str(str)
local test = str:find('"') and not str:find('\'')
local quote = test and "'" or '"'
local pattern = test and '\n' or '[\n"]'
local repl = {
['\n'] = '\\n',
['"'] = '\\"'
}
return quote .. str:gsub(pattern, repl) .. quote
end
-- Value-string conversion.
-- @param {string|table|number} val Value.
-- @returns {string} String representation.
function u8t_utils.val_to_str(val)
local typ = type(val)
local fn
if typ == 'string' then
fn = u8t_utils.escape_str
elseif typ == 'table' then
fn = u8t_utils.tbl_to_str
else
fn = tostring
end
return fn(val)
end
-- Table key-string conversion.
-- @param {string|table|number} key Key.
-- @returns {string} String representation.
function u8t_utils.tbl_key_to_str(key)
if type(key) == 'string' and mw.ustring.match(key, '^[_%a][_%a%d]*$') then
return key
else
return '[' .. u8t_utils.val_to_str(key) .. ']'
end
end
-- Table-string conversion.
-- @param {table} tbl Table.
-- @returns {string} String representation.
function u8t_utils.tbl_to_str(tbl)
local ret = {}
local ikeys = {}
for key, val in ipairs(tbl) do
table.insert(ret, u8t_utils.val_to_str(val))
ikeys[key] = true
end
for key, val in pairs(tbl) do
if not ikeys[key] then
table.insert(ret, u8t_utils.tbl_key_to_str(key) .. ' = ' .. u8t_utils.val_to_str(val))
end
end
return '{' .. table.concat(ret, ', ') .. '}'
end
-- Wikitext representations of Lua code.
u8t_utils.wikitext = {}
-- Lua method test case in wikitext form.
-- @param {string} n Module name.
-- @param {string} m Member name.
-- @param {string|table|number} c Test case.
-- @param {table} o Test options.
-- @returns {string} String representation.
function u8t_utils.wikitext.method(n, m, c, o)
local tmp
if type(c) == 'table' then
local t = u8t_utils.tbl_to_str(c)
if o.unpk == true then
tmp = t:gsub('^{', ''):gsub('}$', '')
else
tmp = t
end
elseif type(c) == 'string' then
tmp = '"' .. c .. '"'
elseif c ~= nil then
tmp = tostring(c)
else
tmp = ''
end
return tostring(n) .. (o.self == true and ':' or '.') .. tostring(m) .. '(' .. tmp .. ')'
end
-- Lua table test case in wikitext form.
-- @param {string} n Module name.
-- @param {string} m Member name.
-- @param {string|number} c Test case.
-- @param {table} o Test options.
-- @returns {string} String representation.
function u8t_utils.wikitext.table(n, m, c, o)
local quote = type(c) == 'string' and '"' or ''
return tostring(n) .. '.' .. tostring(m) .. '[' .. quote .. tostring(c) .. quote .. ']'
end
-- Lua invocation in wikitext form.
-- @param {string} n Module name.
-- @param {string} m Member name.
-- @param {string} c Test case.
-- @param {table} o Test options.
-- @returns {string} String representation.
function u8t_utils.wikitext.invocation(n, m, c, o)
return '{{#invoke:' .. tostring(n) .. '|' .. tostring(m) .. (c and ('|' .. c) or '') .. '}}'
end
-- Unit tester class.
-- @type U8T
-- @section u8t
local U8T = {}
-- Unit tester for package methods.
-- @name U8T.test_method
-- @param {function} m Test member.
-- @param {string|table|number|nil} c Test case.
-- @param {table} o Test options.
-- @param {table} p Module package.
-- @returns {table} Test case result data.
function U8T.test_method(m, c, o, p)
local t = os.clock()
local e, s
if o.unpk == true then
if o.self == true then
e, s = pcall(m, p, unpack(c))
else
e, s = pcall(m, unpack(c))
end
else
if o.self == true then
e, s = pcall(m, p, c)
else
e, s = pcall(m, c)
end
end
return { e, s, 1000 * (os.clock() - t) }
end
-- Unit tester for invocation methods.
-- @name U8T.test_invocation
-- @param {function} m Test member.
-- @param {string|nil} c test case.
-- @param {table} o test options.
-- @returns {table} Test case result data.
function U8T.test_invocation(m, c, o)
-- Function variables.
local n = {}
local a = {}
-- Test case argument parsing.
if type(c) == 'string' then
-- Parser defense.
for s in mw.text.gsplit(c, '|') do
local _, ob = (n[#n] or ''):gsub('[{%[]', '') -- opening braces
local _, cb = (n[#n] or ''):gsub('[}%]]', '') -- closing braces
if (ob - cb) == 0 then
n[#n+1] = s
else
n[#n] = n[#n] .. '|' .. s
end
end
-- Argument and whitespace parsing.
local p_i = 0
for i, s in ipairs(n) do
-- Named parameters.
if mw.ustring.find(s, '^%s*[^=%s{}]+%s*=') then
local p, v = mw.ustring.match(s, '^([^=]+)=([%s%S]*)$')
n[i] = mw.text.trim(p) .. '=' .. mw.text.trim(v)
-- Anonymous parameters.
else
p_i = p_i + 1
n[i] = tostring(p_i) .. '=' .. s
end
end
-- Argument extraction to table.
for i, s in ipairs(n) do
local p, v = mw.ustring.match(s, '([^=]+)=([%s%S]*)')
a[tonumber(p) or p] = v
end
-- Argument preprocessing.
for p, s in pairs(a) do
a[p] = frame.preprocess and frame:preprocess(s) or s
end
end
-- Invoke method using synthetic frame.
frame.args = a
local t = os.clock()
local out = { pcall(m, frame) }
out[2] = out[2] == nil and '' or tostring(out[2])
out[3] = (1000 * (os.clock() - t))
-- Return invocation output.
return out
end
-- Unit tester for package methods.
-- @name U8T.test_table
-- @param {table} m Test member.
-- @param {string|number} c Test case.
-- @param {table} o Test options.
-- @returns {table} Test case result data.
function U8T.test_table(m, c, o)
return { true, m[c], 0 }
end
-- @todo Document this function.
-- @todo Tidy parameters.
local function render_report(suite_status, total_tests, failed_tests, config, failed_members, time_exec, test_results, member_fails)
-- Unit testing report.
local report = mw.html.create():wikitext(
'__FORCETOC__',
'__NOEDITSECTION__',
'[[Category:Lua test suites]]'
)
-- Report infobox.
report:tag('table')
:addClass('infobox WikiaTable')
:css({
['margin-right'] = '0',
['margin-top'] = '0'
})
:tag('tr')
:tag('th')
:wikitext(i18n:msg('suite-status'))
:done()
:tag('td')
:wikitext(wdsButton._bubble(
suite_status and i18n:msg('passed') or i18n:msg('failed'),
suite_status and 'pass' or 'fail'
))
:done()
:done()
:tag('tr')
:tag('th')
:wikitext(i18n:msg('test-cases'))
:done()
:tag('td')
:wikitext(wdsButton._bubble(
tostring(total_tests - failed_tests) ..
'/' .. tostring(total_tests),
suite_status and 'pass' or 'fail'
))
:done()
:done()
:tag('tr')
:tag('th')
:wikitext(i18n:msg('code-cov'))
:done()
:tag('td')
:wikitext(wdsButton._bubble(
config.res,
(function(c)
local r = mw.text.split(c.res, '/')
if #c.pos == 0 then
return 'nil'
elseif r[1] ~= r[2] then
return 'warn'
else
return 'pass'
end
end)(config)
))
:done()
:done()
-- Report lede.
-- debug
mw.log('rr config', config.name, dump(config))-- what are we trying to i?
report:tag('b')
:wikitext(i18n:msg('report-lede', config.name))
-- close b?
-- Report summary.
report:tag('ul')
:tag('li')
:wikitext(i18n:msg('report-missing') .. ': ' .. (#config.neg ~= 0
and '<code>' .. table.concat(config.neg, '</code> • <code>') .. '</code>'
or i18n:msg('report-none')
))
:done()
:tag('li')
:wikitext(i18n:msg('report-failing') .. ': ' .. (#failed_members ~= 0
and '<code>' .. table.concat(failed_members, '</code> • <code>') .. '</code>'
.. '[[Category:Failing Lua test suites]]'
or i18n:msg('report-none')
)
)
:done()
:tag('li')
:wikitext(i18n:msg('report-time') .. ': ' .. tostring(math.floor(time_exec * 10 + 0.5) / 10) .. 'ms')
:done()
-- Report results header.
report:tag('h2')
:wikitext(i18n:msg('test-cases'))
-- Report results.
for test_member, test_output in pairs(test_results) do
local member_nowiki = test_output.options.nowiki == true
and function(...) return mw.text.nowiki(tostring(...)) end
or tostring
local result_table = report:tag('table')
:addClass('WikiaTable mw-collapsible')
:css('width', '100%')
-- Member header.
result_table:tag('tr')
:tag('th')
:attr('colspan', (config.diff == true) and 5 or 4)
:tag('h3')
:attr('id', test_member)
:css('display', 'inline')
:wikitext('<code>' .. config.pkg .. (test_output.options.self and ':' or '.') .. test_member .. '</code>')
:done()
:wikitext(' ' .. wdsButton._bubble(
(#test_output - (member_fails[test_member] or 0)) .. '/' .. #test_output,
((member_fails[test_member] or 0) == 0) and 'pass' or 'fail'
))
:done()
-- Member column headers.
result_table:tag('tr')
:tag('th')
:wikitext(i18n:msg('header-status'))
:done()
:tag('th')
:wikitext(i18n:msg('header-code'))
:done()
:tag('th')
:wikitext(i18n:msg('header-expect'))
:done()
:tag('th')
:wikitext(i18n:msg('header-actual'))
:done()
:node(config.diff == true
and mw.html.create('th'):wikitext(i18n:msg('header-diff'))
or nil
)
-- Member column data.
for test_index, test_result in ipairs(test_output) do
result_table:tag('tr')
:tag('td')
:wikitext(wdsButton._badge(
i18n:msg((test_result.status == true) and 'passing' or 'failing'),
(test_result.status == true) and 'pass' or 'fail'
))
:done()
:tag('td')
:tag('pre')
:css({
['max-width'] = '200px',
['white-space'] = 'pre-wrap',
['word-wrap'] = 'break-word'
})
:wikitext(mw.text.nowiki(u8t_utils.wikitext[test_output.options.mode](
test_output.options.mode == 'invocation'
and (mw.ustring.gsub((config.name:gsub('Module:', '', 1)), '^%u', mw.ustring.lower))
or config.pkg,
test_member,
test_result.case,
test_output.options
)))
:done()
:done()
:tag('td')
:css({
['white-space'] = 'pre-wrap',
['word-break'] = 'break-all'
})
:wikitext(
(test_result.err[1] == true
and wdsButton._bubble(
i18n:msg('error'),
'warn'
) .. '<br />'
or ''
) ..
member_nowiki(type(test_result.expect) == 'table'
and u8t_utils.tbl_to_str(test_result.expect)
or test_result.expect
)
)
:done()
:tag('td')
:css({
['white-space'] = 'pre-wrap',
['word-break'] = 'break-all'
})
:wikitext(
(test_result.err[2] == true
and wdsButton._bubble(
i18n:msg('error'),
'warn'
) .. '<br />'
or ''
) ..
member_nowiki(type(test_result.actual) == 'table'
and u8t_utils.tbl_to_str(test_result.actual)
or test_result.actual
)
)
:done()
:node(config.diff == true
and mw.html.create('td')
:wikitext(test_result.diff_i)
or nil
)
end
end
return tostring(report)
end
-- Unit test execution engine.
-- @name run_tests
-- @param {table} test_suite Test case suite.
-- @param {table} test_module Module package.
-- @param {table} config Test configuration.
-- @returns {table} Test case result data.
local function run_tests(test_suite, test_module, config)
-- Generate test data.
local test_results = {}
local time_exec = 0
for test_member, test_data in pairs(test_suite) do
test_results[test_member] = {
['options'] = test_data.options
}
for test_index, test_case in ipairs(test_data.tests) do
-- Execute member test.
local test_res = U8T['test_' .. test_data.options.mode](
test_module[test_member], test_case[1], test_data.options, test_module
)
time_exec = time_exec + test_res[3]
local pp = test_data.options.preprocess or (test_case[3] or {}).pp
-- Generate test case result.
table.insert(test_results[test_member], {
-- Test case data.
['case'] = test_case[1],
['actual'] = test_res[2],
['expect'] = pp
and u8t_utils.pp(test_case[2])
or test_case[2],
-- Test case analysis.
['err'] = {
((test_case[3] or {}).err == true),
(test_res[1] == false)
}
})
-- Test case error preprocessing.
local l = #test_results[test_member]
local r = test_results[test_member][l].err
if r[2] == true then
test_results[test_member][l].actual =
mw.ustring.match(test_results[test_member][l].actual, '%d:%s([%s%S]+)')
end
-- Test case comparision.
local a = test_results[test_member][l].actual
local e = test_results[test_member][l].expect
test_results[test_member][l].diff_i = (r[1] == r[2])
and (test_data.options.deep == true
and u8t_utils.diff_tbl(e, a)
or u8t_utils.diff_str(e, a)
)
or 'error'
end
end
-- Derived data required for report.
local member_fails = {}
local failed_members = {}
local total_tests = 0
local failed_tests = 0
for test_member, test_output in pairs(test_results) do
for test_index, test_result in ipairs(test_output) do
-- Result count and status.
total_tests = total_tests + 1
member_fails[test_member] = member_fails[test_member] or 0
test_result.status = (
(test_result.err[1] == test_result.err[2]) and
#test_result.diff_i == 0
)
-- Test logic.
if test_result.status == false then
member_fails[test_member] = member_fails[test_member] + 1
failed_tests = failed_tests + 1
end
end
end
for test_member, fail_count in pairs(member_fails) do
if fail_count ~= 0 then
table.insert(failed_members, test_member)
end
end
-- Generate report.
local suite_status = (failed_tests == 0)
local ersult, result = pcall(render_report, suite_status, total_tests, failed_tests, config, failed_members, time_exec, test_results, member_fails)
if ersult then
return result
else
-- debug
mw.log('run_tests failed to render_report', '\nresult:', result, '\nsuite_status:', suite_status, '\ntotal_tests:', total_tests, '\nfailed_tests:', failed_tests, '\nconfig:', dump(config), '\nfailed_members:', dump(failed_members), '\ntim_exec:', tim_exec, '\ntest_results:', dump(test_results), '\nmember_fails:', dump(member_fails))
-- back to default error handler, if frame is available
-- in order to return most informative msg possible
-- the render_report will be double-processed
return mw.getCurrentFrame() and render_report(suite_status, total_tests, failed_tests, config, failed_members, time_exec, test_results, member_fails) or result
end
end
-- Test harness logic.
-- @section th
-- Code coverage analyser.
-- @param {string} module_name Module name.
-- @param {string} test_module Test module package.
-- @param {string} test_cases Test case data.
-- @returns {table} Code coverage data.
function th.code_cov(module_name, test_module, test_suite)
-- Function variables.
local members = {}
local raw_text = mw.title.new(module_name):getContent()
local package_name = raw_text:match('\nreturn (%S+)\n?') or 'p'
local raw_functions = {}
local coverage = {}
local coverage_old = {}
local positive = {}
local negative = {}
-- Direct coverage.
for member_key, val in pairs(test_module) do
table.insert(members, member_key)
raw_functions[member_key] = raw_text:match(package_name .. '[:.]' .. member_key .. '(.-\n%S+)\n')
if test_suite[member_key] then
coverage[member_key] = true
end
end
-- Source code coverage.
while (coverage_old ~= coverage) do
coverage_old = coverage
for member_key, _ in pairs(coverage) do
for _, p in ipairs({ 'self', package_name }) do
-- Test for code access.
for member_key in raw_functions[member_key]:gmatch(p .. '[[.:"\']+([^\'%s([]+)') do
if test_module[member_key] then
coverage[member_key] = true
end
end
end
end
end
-- Package members with positive coverage.
for member_key, _ in pairs(coverage) do
table.insert(positive, member_key)
end
table.sort(positive)
-- Package members with negative coverage.
for member_key, _ in pairs(test_module) do
if coverage[member_key] == nil then
table.insert(negative, member_key);
end
end
table.sort(negative)
-- Output result.
return {
['pkg'] = package_name,
['neg'] = negative,
['pos'] = positive,
['res'] = tostring(#positive) .. '/' .. tostring(#members)
}
end
-- Test harness function.
-- @param {table} f Frame object.
-- @returns {string} Test report.
function th.run_tests(f)
-- Frame arguments.
frame = f
local module_name = (frame.args or {}).modulename or ''
local test_data = (frame.args or {}).testdata or ''
local differs_at = (frame.args or {}).differs_at or 'true'
-- Argument validation and defaults.
local module_ns = mw.site.namespaces[828].canonicalName
module_name = #module_name ~= 0
and frame.args.modulename
or mw.title.getCurrentTitle().baseText
module_name = module_ns .. ':' .. module_name
test_data = #test_data ~= 0
and module_ns .. ':' .. test_data
or module_name .. '/testcases'
differs_at = (yesno(differs_at) == true or #differs_at == 0)
and true
or false
-- Load module package.
local test_module = require(module_name)
-- Load test cases.
local _, test_suite = pcall(require, test_data)
test_suite = test_suite or {}
-- Remove string members.
for member_key, val in pairs(test_module) do
if type(val) == 'string' then
test_module[member_key] = nil
end
end
-- Check execution validity.
for test_member, test_data in pairs(test_suite) do
if
not test_module[test_member] or -- Nonexistent member.
type(test_data) ~= 'table' or -- Old UnitTests format.
not (test_data.options or {}).mode -- Unconfigured tests.
then
test_suite[test_member] = nil
end
end
-- Unit test configuration.
local config = th.code_cov(module_name, test_module, test_suite)
config['diff'] = differs_at
config['name'] = module_name
-- Execute tests & generate report.
return run_tests(test_suite or {}, test_module, config)
end
return th
-- </nowiki>