diff --git a/.gitignore b/.gitignore index 3f5288d45..2721f6963 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,4 @@ stamp-h1 /testing/output/*.html /testing/output/*.md /testing/output/actual/*.png +/testing/output/difference/*.png diff --git a/testing/classes/TestMethod.lua b/testing/classes/TestMethod.lua index 9838b9f67..30dd8e1cc 100644 --- a/testing/classes/TestMethod.lua +++ b/testing/classes/TestMethod.lua @@ -64,7 +64,7 @@ TestMethod = { table.insert(self.asserts, { key = 'assert ' .. tostring(self.count), passed = expected == actual, - message = 'expected \'' .. tostring(expected) .. '\' got \'' .. + message = 'expected \'' .. tostring(expected) .. '\' got \'' .. tostring(actual) .. '\'', test = label or 'no label given' }) @@ -81,7 +81,7 @@ TestMethod = { table.insert(self.asserts, { key = 'assert ' .. tostring(self.count), passed = value == true, - message = 'expected \'true\' got \'' .. + message = 'expected \'true\' got \'' .. tostring(value) .. '\'', test = label or 'no label given' }) @@ -98,7 +98,7 @@ TestMethod = { table.insert(self.asserts, { key = 'assert ' .. tostring(self.count), passed = value == false, - message = 'expected \'false\' got \'' .. + message = 'expected \'false\' got \'' .. tostring(value) .. '\'', test = label or 'no label given' }) @@ -165,7 +165,7 @@ TestMethod = { -- @method - TestMethod:assertGreaterEqual() - -- @desc - used to check a value is >= than a certain target value + -- @desc - used to check a value is >= than a certain target value -- @param {any} target - value to check the test agaisnt -- @param {any} actual - actual value of the test -- @param {string} label - label for this test to use in exports @@ -187,7 +187,7 @@ TestMethod = { -- @method - TestMethod:assertLessEqual() - -- @desc - used to check a value is <= than a certain target value + -- @desc - used to check a value is <= than a certain target value -- @param {any} target - value to check the test agaisnt -- @param {any} actual - actual value of the test -- @param {string} label - label for this test to use in exports @@ -209,7 +209,7 @@ TestMethod = { -- @method - TestMethod:assertObject() - -- @desc - used to check a table is a love object, this runs 3 seperate + -- @desc - used to check a table is a love object, this runs 3 seperate -- tests to check table has the basic properties of an object -- @note - actual object functionality tests have their own methods -- @param {table} obj - table to check is a valid love object @@ -238,7 +238,7 @@ TestMethod = { table.insert(self.asserts, { key = 'assert ' .. tostring(self.count), passed = passing, - message = 'expected \'' .. tostring(expected[1]) .. 'x,' .. + message = 'expected \'' .. tostring(expected[1]) .. 'x,' .. tostring(expected[2]) .. 'y\' got \'' .. tostring(actual[1]) .. 'x,' .. tostring(actual[2]) .. 'y\'', test = label or 'no label given' @@ -247,7 +247,7 @@ TestMethod = { -- @method - TestMethod:assertNotNil() - -- @desc - quick assert for value not nil + -- @desc - quick assert for value not nil -- @param {any} value - value to check not nil -- @return {nil} assertNotNil = function (self, value, err) @@ -264,8 +264,8 @@ TestMethod = { -- @method - TestMethod:compareImg() - -- @desc - compares a given image to the 'expected' version, with a tolerance of - -- 1px in any direction, and then saves it as the 'actual' version for + -- @desc - compares a given image to the 'expected' version, with a tolerance of + -- 1px in any direction, and then saves it as the 'actual' version for -- report viewing -- @param {table} imgdata - imgdata to save as a png -- @return {nil} @@ -275,24 +275,28 @@ TestMethod = { local ok, chunk, _ = pcall(love.image.newImageData, expected_path) if ok == false then return self:assertEquals(true, false, chunk) end local expected = chunk - local iw = imgdata:getWidth()-2 - local ih = imgdata:getHeight()-2 + local iw = imgdata:getWidth()-1 + local ih = imgdata:getHeight()-1 + local differences = {} local rgba_tolerance = self.rgba_tolerance * (1/255) - for ix=2,iw do - for iy=2,ih do + + -- for each pixel, compare the expected vs the actual pixel data + -- by default rgba_tolerance is 0 + for ix=0,iw do + for iy=0,ih do local ir, ig, ib, ia = imgdata:getPixel(ix, iy) local points = { {expected:getPixel(ix, iy)} } if self.pixel_tolerance > 0 then - table.insert(points, {expected:getPixel(ix-1, iy+1)}) - table.insert(points, {expected:getPixel(ix-1, iy)}) - table.insert(points, {expected:getPixel(ix-1, iy-1)}) - table.insert(points, {expected:getPixel(ix, iy+1)}) - table.insert(points, {expected:getPixel(ix, iy-1)}) - table.insert(points, {expected:getPixel(ix+1, iy+1)}) - table.insert(points, {expected:getPixel(ix+1, iy)}) - table.insert(points, {expected:getPixel(ix+1, iy-1)}) + if ix > 0 and iy < ih-1 then table.insert(points, {expected:getPixel(ix-1, iy+1)}) end + if ix > 0 then table.insert(points, {expected:getPixel(ix-1, iy)}) end + if ix > 0 and iy > 0 then table.insert(points, {expected:getPixel(ix-1, iy-1)}) end + if iy < ih-1 then table.insert(points, {expected:getPixel(ix, iy+1)}) end + if iy > 0 then table.insert(points, {expected:getPixel(ix, iy-1)}) end + if ix < iw-1 and iy < ih-1 then table.insert(points, {expected:getPixel(ix+1, iy+1)}) end + if ix < iw-1 then table.insert(points, {expected:getPixel(ix+1, iy)}) end + if ix < iw-1 and iy > 0 then table.insert(points, {expected:getPixel(ix+1, iy-1)}) end end local has_match_r = false local has_match_g = false @@ -317,23 +321,48 @@ TestMethod = { tostring(ix) .. ',' .. tostring(iy) .. ', matching = ' .. ymatch .. ', not matching = ' .. nmatch .. ' (' .. self.method .. '-' .. tostring(self.imgs) .. ')' ) + -- add difference co-ord for rendering later + if matching ~= true then + table.insert(differences, ix+1) + table.insert(differences, iy+1) + end end end local path = 'tempoutput/actual/love.test.graphics.' .. self.method .. '-' .. tostring(self.imgs) .. '.png' imgdata:encode('png', path) + + -- if we have differences draw them to a new canvas to display in HTML report + local dpath = 'tempoutput/difference/love.test.graphics.' .. + self.method .. '-' .. tostring(self.imgs) .. '.png' + if #differences > 0 then + local difference = love.graphics.newCanvas(iw+1, ih+1) + love.graphics.setCanvas(difference) + love.graphics.clear(0, 0, 0, 1) + love.graphics.setColor(1, 0, 1, 1) + love.graphics.points(differences) + love.graphics.setColor(1, 1, 1, 1) + love.graphics.setCanvas() + love.graphics.readbackTexture(difference):encode('png', dpath) + + -- otherwise clear the old difference file (if any) to stop it coming up + -- in future reports when there's no longer a difference + elseif love.filesystem.openFile(dpath, 'r') then + love.filesystem.remove(dpath) + end + self.imgs = self.imgs + 1 end, -- @method - TestMethod:exportImg() -- @desc - exports the given imgdata to the 'output/expected/' folder, to use when - -- writing new graphics tests to set the expected image output - -- @NOTE - you should not leave this method in when you are finished this is + -- writing new graphics tests to set the expected image output + -- @NOTE - you should not leave this method in when you are finished this is -- for test writing only -- @param {table} imgdata - imgdata to save as a png -- @param {integer} imgdata - index of the png, graphic tests are run sequentially - -- and each test image is numbered in order that its + -- and each test image is numbered in order that its -- compared to, so set the number here to match -- @return {nil} exportImg = function(self, imgdata, index) @@ -386,6 +415,21 @@ TestMethod = { return false end, + -- @method - TestMethod:isLuaVersion() + -- @desc - checks for a specific Lua version (or list of versions) + -- @param {number} - the minimum Lua version to check against + -- @return {boolean} - returns true if the current Lua version is at least the given version + isAtLeastLuaVersion = function(self, version) + return love.test.lua_version >= version + end, + + -- @method - TestMethod:isLuaJITEnabled() + -- @desc - checks if LuaJIT is enabled + -- @return {boolean} - returns true if LuaJIT is enabled + isLuaJITEnabled = function(self) + return love.test.has_lua_jit + end, + -- @method - TestMethod:evaluateTest() -- @desc - evaluates the results of all assertions for a final restult -- @return {nil} @@ -410,10 +454,10 @@ TestMethod = { if self.skipped then self.testmodule.skipped = self.testmodule.skipped + 1 love.test.totals[3] = love.test.totals[3] + 1 - self.result = { - total = '', - result = "SKIP", - passed = false, + self.result = { + total = '', + result = "SKIP", + passed = false, message = '(0/0) - method skipped [' .. self.skipreason .. ']', failures = {} } @@ -424,10 +468,10 @@ TestMethod = { self.passed = true self.testmodule.passed = self.testmodule.passed + 1 love.test.totals[1] = love.test.totals[1] + 1 - self.result = { - total = total, - result = 'PASS', - passed = true, + self.result = { + total = total, + result = 'PASS', + passed = true, message = nil, failures = {} } @@ -517,21 +561,21 @@ TestMethod = { local preview = '' if self.testmodule.module == 'graphics' then local filename = 'love.test.graphics.' .. self.method - if love.filesystem.openFile('tempoutput/actual/' .. filename .. '-1.png', 'r') then - preview = '
' .. '

Expected

' .. - '
' .. '

Actual

' - end - if love.filesystem.openFile('tempoutput/actual/' .. filename .. '-2.png', 'r') then - preview = preview .. '
' .. '

Expected

' .. - '
' .. '

Actual

' - end - if love.filesystem.openFile('tempoutput/actual/' .. filename .. '-3.png', 'r') then - preview = preview .. '
' .. '

Expected

' .. - '
' .. '

Actual

' + for f=1,5 do + local fstr = tostring(f) + if love.filesystem.openFile('tempoutput/actual/' .. filename .. '-' .. fstr .. '.png', 'r') then + preview = preview .. '
' + preview = preview .. '
' .. '

Expected

' .. + '
' .. '

Actual

' + if love.filesystem.openFile('tempoutput/difference/' .. filename .. '-' .. fstr .. '.png', 'r') then + preview = preview .. '
' .. '

Difference

' + end + preview = preview .. '
' + end end end - -- append HTML for the test class result + -- append HTML for the test class result local status = '🔴' local cls = 'red' if self.passed then status = '🟢'; cls = '' end @@ -565,7 +609,7 @@ TestMethod = { self.result.total .. msg ) - -- if we failed on multiple asserts, list them here - makes it easier for + -- if we failed on multiple asserts, list them here - makes it easier for -- debugging new methods added that are failing multiple asserts if #self.result.failures > 1 then for f=2,#self.result.failures do diff --git a/testing/classes/TestSuite.lua b/testing/classes/TestSuite.lua index 4777281e6..6457f8ea2 100644 --- a/testing/classes/TestSuite.lua +++ b/testing/classes/TestSuite.lua @@ -24,6 +24,8 @@ TestSuite = { fakequit = false, windowmode = true, current_os = love._os, + lua_version = tonumber(_VERSION:match("%d%.%d")), + has_lua_jit = type(jit) == 'table', -- love modules to test audio = {}, @@ -98,7 +100,7 @@ TestSuite = { -- when wait finished (or no yields) if coroutine.status(self.test.co) == 'dead' then - -- now we're all done evaluate the test + -- now we're all done evaluate the test local ok, chunk, err = pcall(self.test.evaluateTest, self.test) if ok == false then self.test.passed = false @@ -129,7 +131,7 @@ TestSuite = { self:printResult() love.event.quit(0) end - + end end end @@ -152,7 +154,7 @@ TestSuite = { if love.graphics then name, version, vendor, device = love.graphics.getRendererInfo() end - + local md = '