diff --git a/.gitignore b/.gitignore index 3f5288d45..2721f6963 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,4 @@ stamp-h1 /testing/output/*.html /testing/output/*.md /testing/output/actual/*.png +/testing/output/difference/*.png diff --git a/testing/classes/TestMethod.lua b/testing/classes/TestMethod.lua index 9838b9f67..405b9275b 100644 --- a/testing/classes/TestMethod.lua +++ b/testing/classes/TestMethod.lua @@ -64,7 +64,7 @@ TestMethod = { table.insert(self.asserts, { key = 'assert ' .. tostring(self.count), passed = expected == actual, - message = 'expected \'' .. tostring(expected) .. '\' got \'' .. + message = 'expected \'' .. tostring(expected) .. '\' got \'' .. tostring(actual) .. '\'', test = label or 'no label given' }) @@ -81,7 +81,7 @@ TestMethod = { table.insert(self.asserts, { key = 'assert ' .. tostring(self.count), passed = value == true, - message = 'expected \'true\' got \'' .. + message = 'expected \'true\' got \'' .. tostring(value) .. '\'', test = label or 'no label given' }) @@ -98,7 +98,7 @@ TestMethod = { table.insert(self.asserts, { key = 'assert ' .. tostring(self.count), passed = value == false, - message = 'expected \'false\' got \'' .. + message = 'expected \'false\' got \'' .. tostring(value) .. '\'', test = label or 'no label given' }) @@ -165,7 +165,7 @@ TestMethod = { -- @method - TestMethod:assertGreaterEqual() - -- @desc - used to check a value is >= than a certain target value + -- @desc - used to check a value is >= than a certain target value -- @param {any} target - value to check the test agaisnt -- @param {any} actual - actual value of the test -- @param {string} label - label for this test to use in exports @@ -187,7 +187,7 @@ TestMethod = { -- @method - TestMethod:assertLessEqual() - -- @desc - used to check a value is <= than a certain target value + -- @desc - used to check a value is <= than a certain target value -- @param {any} target - value to check the test agaisnt -- @param {any} actual - actual value of the test -- @param {string} label - label for this test to use in exports @@ -209,7 +209,7 @@ TestMethod = { -- @method - TestMethod:assertObject() - -- @desc - used to check a table is a love object, this runs 3 seperate + -- @desc - used to check a table is a love object, this runs 3 seperate -- tests to check table has the basic properties of an object -- @note - actual object functionality tests have their own methods -- @param {table} obj - table to check is a valid love object @@ -238,7 +238,7 @@ TestMethod = { table.insert(self.asserts, { key = 'assert ' .. tostring(self.count), passed = passing, - message = 'expected \'' .. tostring(expected[1]) .. 'x,' .. + message = 'expected \'' .. tostring(expected[1]) .. 'x,' .. tostring(expected[2]) .. 'y\' got \'' .. tostring(actual[1]) .. 'x,' .. tostring(actual[2]) .. 'y\'', test = label or 'no label given' @@ -247,7 +247,7 @@ TestMethod = { -- @method - TestMethod:assertNotNil() - -- @desc - quick assert for value not nil + -- @desc - quick assert for value not nil -- @param {any} value - value to check not nil -- @return {nil} assertNotNil = function (self, value, err) @@ -264,8 +264,8 @@ TestMethod = { -- @method - TestMethod:compareImg() - -- @desc - compares a given image to the 'expected' version, with a tolerance of - -- 1px in any direction, and then saves it as the 'actual' version for + -- @desc - compares a given image to the 'expected' version, with a tolerance of + -- 1px in any direction, and then saves it as the 'actual' version for -- report viewing -- @param {table} imgdata - imgdata to save as a png -- @return {nil} @@ -275,24 +275,28 @@ TestMethod = { local ok, chunk, _ = pcall(love.image.newImageData, expected_path) if ok == false then return self:assertEquals(true, false, chunk) end local expected = chunk - local iw = imgdata:getWidth()-2 - local ih = imgdata:getHeight()-2 + local iw = imgdata:getWidth()-1 + local ih = imgdata:getHeight()-1 + local differences = {} local rgba_tolerance = self.rgba_tolerance * (1/255) - for ix=2,iw do - for iy=2,ih do + + -- for each pixel, compare the expected vs the actual pixel data + -- by default rgba_tolerance is 0 + for ix=0,iw do + for iy=0,ih do local ir, ig, ib, ia = imgdata:getPixel(ix, iy) local points = { {expected:getPixel(ix, iy)} } if self.pixel_tolerance > 0 then - table.insert(points, {expected:getPixel(ix-1, iy+1)}) - table.insert(points, {expected:getPixel(ix-1, iy)}) - table.insert(points, {expected:getPixel(ix-1, iy-1)}) - table.insert(points, {expected:getPixel(ix, iy+1)}) - table.insert(points, {expected:getPixel(ix, iy-1)}) - table.insert(points, {expected:getPixel(ix+1, iy+1)}) - table.insert(points, {expected:getPixel(ix+1, iy)}) - table.insert(points, {expected:getPixel(ix+1, iy-1)}) + if ix > 0 and iy < ih-1 then table.insert(points, {expected:getPixel(ix-1, iy+1)}) end + if ix > 0 then table.insert(points, {expected:getPixel(ix-1, iy)}) end + if ix > 0 and iy > 0 then table.insert(points, {expected:getPixel(ix-1, iy-1)}) end + if iy < ih-1 then table.insert(points, {expected:getPixel(ix, iy+1)}) end + if iy > 0 then table.insert(points, {expected:getPixel(ix, iy-1)}) end + if ix < iw-1 and iy < ih-1 then table.insert(points, {expected:getPixel(ix+1, iy+1)}) end + if ix < iw-1 then table.insert(points, {expected:getPixel(ix+1, iy)}) end + if ix < iw-1 and iy > 0 then table.insert(points, {expected:getPixel(ix+1, iy-1)}) end end local has_match_r = false local has_match_g = false @@ -317,23 +321,48 @@ TestMethod = { tostring(ix) .. ',' .. tostring(iy) .. ', matching = ' .. ymatch .. ', not matching = ' .. nmatch .. ' (' .. self.method .. '-' .. tostring(self.imgs) .. ')' ) + -- add difference co-ord for rendering later + if matching ~= true then + table.insert(differences, ix+1) + table.insert(differences, iy+1) + end end end local path = 'tempoutput/actual/love.test.graphics.' .. self.method .. '-' .. tostring(self.imgs) .. '.png' imgdata:encode('png', path) + + -- if we have differences draw them to a new canvas to display in HTML report + local dpath = 'tempoutput/difference/love.test.graphics.' .. + self.method .. '-' .. tostring(self.imgs) .. '.png' + if #differences > 0 then + local difference = love.graphics.newCanvas(iw+1, ih+1) + love.graphics.setCanvas(difference) + love.graphics.clear(0, 0, 0, 1) + love.graphics.setColor(1, 0, 1, 1) + love.graphics.points(differences) + love.graphics.setColor(1, 1, 1, 1) + love.graphics.setCanvas() + love.graphics.readbackTexture(difference):encode('png', dpath) + + -- otherwise clear the old difference file (if any) to stop it coming up + -- in future reports when there's no longer a difference + elseif love.filesystem.openFile(dpath, 'r') then + love.filesystem.remove(dpath) + end + self.imgs = self.imgs + 1 end, -- @method - TestMethod:exportImg() -- @desc - exports the given imgdata to the 'output/expected/' folder, to use when - -- writing new graphics tests to set the expected image output - -- @NOTE - you should not leave this method in when you are finished this is + -- writing new graphics tests to set the expected image output + -- @NOTE - you should not leave this method in when you are finished this is -- for test writing only -- @param {table} imgdata - imgdata to save as a png -- @param {integer} imgdata - index of the png, graphic tests are run sequentially - -- and each test image is numbered in order that its + -- and each test image is numbered in order that its -- compared to, so set the number here to match -- @return {nil} exportImg = function(self, imgdata, index) @@ -386,6 +415,21 @@ TestMethod = { return false end, + -- @method - TestMethod:isLuaVersion() + -- @desc - checks for a specific Lua version (or list of versions) + -- @param {number} - the minimum Lua version to check against + -- @return {boolean} - returns true if the current Lua version is at least the given version + isAtLeastLuaVersion = function(self, version) + return love.test.lua_version >= version + end, + + -- @method - TestMethod:isLuaJITEnabled() + -- @desc - checks if LuaJIT is enabled + -- @return {boolean} - returns true if LuaJIT is enabled + isLuaJITEnabled = function(self) + return love.test.has_lua_jit + end, + -- @method - TestMethod:evaluateTest() -- @desc - evaluates the results of all assertions for a final restult -- @return {nil} @@ -410,10 +454,10 @@ TestMethod = { if self.skipped then self.testmodule.skipped = self.testmodule.skipped + 1 love.test.totals[3] = love.test.totals[3] + 1 - self.result = { - total = '', - result = "SKIP", - passed = false, + self.result = { + total = '', + result = "SKIP", + passed = false, message = '(0/0) - method skipped [' .. self.skipreason .. ']', failures = {} } @@ -424,10 +468,10 @@ TestMethod = { self.passed = true self.testmodule.passed = self.testmodule.passed + 1 love.test.totals[1] = love.test.totals[1] + 1 - self.result = { - total = total, - result = 'PASS', - passed = true, + self.result = { + total = total, + result = 'PASS', + passed = true, message = nil, failures = {} } @@ -517,25 +561,25 @@ TestMethod = { local preview = '' if self.testmodule.module == 'graphics' then local filename = 'love.test.graphics.' .. self.method - if love.filesystem.openFile('tempoutput/actual/' .. filename .. '-1.png', 'r') then - preview = '
' .. '

Expected

' .. - '
' .. '

Actual

' - end - if love.filesystem.openFile('tempoutput/actual/' .. filename .. '-2.png', 'r') then - preview = preview .. '
' .. '

Expected

' .. - '
' .. '

Actual

' - end - if love.filesystem.openFile('tempoutput/actual/' .. filename .. '-3.png', 'r') then - preview = preview .. '
' .. '

Expected

' .. - '
' .. '

Actual

' + for f=1,5 do + local fstr = tostring(f) + if love.filesystem.openFile('tempoutput/actual/' .. filename .. '-' .. fstr .. '.png', 'r') then + preview = preview .. '
' + preview = preview .. '
' .. '

Expected

' .. + '
' .. '

Actual

' + if love.filesystem.openFile('tempoutput/difference/' .. filename .. '-' .. fstr .. '.png', 'r') then + preview = preview .. '
' .. '

Difference

' + end + preview = preview .. '
' + end end end - -- append HTML for the test class result - local status = '🔴' + -- append HTML for the test class result + local status = '' local cls = 'red' - if self.passed then status = '🟢'; cls = '' end - if self.skipped then status = '🟡'; cls = '' end + if self.passed then status = '
'; cls = 'green' end + if self.skipped then status = ''; cls = 'yellow' end self.testmodule.html = self.testmodule.html .. '' .. '' .. status .. '' .. @@ -565,7 +609,7 @@ TestMethod = { self.result.total .. msg ) - -- if we failed on multiple asserts, list them here - makes it easier for + -- if we failed on multiple asserts, list them here - makes it easier for -- debugging new methods added that are failing multiple asserts if #self.result.failures > 1 then for f=2,#self.result.failures do diff --git a/testing/classes/TestModule.lua b/testing/classes/TestModule.lua index f66d247ed..e4a446116 100644 --- a/testing/classes/TestModule.lua +++ b/testing/classes/TestModule.lua @@ -82,8 +82,8 @@ TestModule = { -- @return {nil} printResult = function(self) local finaltime = UtilTimeFormat(self.time) - local status = '🔴' - if self.failed == 0 then status = '🟢' end + local status = '
' + if self.failed == 0 then status = '
' end -- add md row to main output love.test.mdrows = love.test.mdrows .. '| ' .. status .. ' ' .. self.module .. @@ -98,13 +98,23 @@ TestModule = { '" skipped="' .. tostring(self.skipped) .. '" time="' .. finaltime .. '">\n' .. self.xml .. '\t\n' -- add html to main output - love.test.html = love.test.html .. '

' .. status .. ' love.' .. self.module .. '