From 24d84418c7d938714bc3fcbdf83257de6dfac178 Mon Sep 17 00:00:00 2001 From: ell <77150506+ellraiser@users.noreply.github.com> Date: Fri, 22 Mar 2024 12:17:15 +0000 Subject: [PATCH 1/2] testsuite updates - added some additional checks to check for lua version/jit enabled for use with other repos (namely love potion) - updated HTML report CSS to group each image compare and fix pixel rendering - fixed not showing image checks for the 4th compareImg or more in a given test method - fixed pixel tolerance usage crashing when looking at images on the boundary - fixed compare img not checking the first 2 rows of pixels on the x + y -graphic tests that fail will now show a third "difference" image to help identify what isn't matching up --- .gitignore | 1 + testing/classes/TestMethod.lua | 136 ++++++++++++++++++---------- testing/classes/TestSuite.lua | 20 ++-- testing/output/difference/notes.txt | 3 + testing/tests/filesystem.lua | 37 +++++--- testing/todo.md | 2 - 6 files changed, 129 insertions(+), 70 deletions(-) create mode 100644 testing/output/difference/notes.txt diff --git a/.gitignore b/.gitignore index 3f5288d45..2721f6963 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,4 @@ stamp-h1 /testing/output/*.html /testing/output/*.md /testing/output/actual/*.png +/testing/output/difference/*.png diff --git a/testing/classes/TestMethod.lua b/testing/classes/TestMethod.lua index 9838b9f67..30dd8e1cc 100644 --- a/testing/classes/TestMethod.lua +++ b/testing/classes/TestMethod.lua @@ -64,7 +64,7 @@ TestMethod = { table.insert(self.asserts, { key = 'assert ' .. tostring(self.count), passed = expected == actual, - message = 'expected \'' .. tostring(expected) .. '\' got \'' .. + message = 'expected \'' .. tostring(expected) .. '\' got \'' .. tostring(actual) .. '\'', test = label or 'no label given' }) @@ -81,7 +81,7 @@ TestMethod = { table.insert(self.asserts, { key = 'assert ' .. tostring(self.count), passed = value == true, - message = 'expected \'true\' got \'' .. + message = 'expected \'true\' got \'' .. tostring(value) .. '\'', test = label or 'no label given' }) @@ -98,7 +98,7 @@ TestMethod = { table.insert(self.asserts, { key = 'assert ' .. tostring(self.count), passed = value == false, - message = 'expected \'false\' got \'' .. + message = 'expected \'false\' got \'' .. tostring(value) .. '\'', test = label or 'no label given' }) @@ -165,7 +165,7 @@ TestMethod = { -- @method - TestMethod:assertGreaterEqual() - -- @desc - used to check a value is >= than a certain target value + -- @desc - used to check a value is >= than a certain target value -- @param {any} target - value to check the test agaisnt -- @param {any} actual - actual value of the test -- @param {string} label - label for this test to use in exports @@ -187,7 +187,7 @@ TestMethod = { -- @method - TestMethod:assertLessEqual() - -- @desc - used to check a value is <= than a certain target value + -- @desc - used to check a value is <= than a certain target value -- @param {any} target - value to check the test agaisnt -- @param {any} actual - actual value of the test -- @param {string} label - label for this test to use in exports @@ -209,7 +209,7 @@ TestMethod = { -- @method - TestMethod:assertObject() - -- @desc - used to check a table is a love object, this runs 3 seperate + -- @desc - used to check a table is a love object, this runs 3 seperate -- tests to check table has the basic properties of an object -- @note - actual object functionality tests have their own methods -- @param {table} obj - table to check is a valid love object @@ -238,7 +238,7 @@ TestMethod = { table.insert(self.asserts, { key = 'assert ' .. tostring(self.count), passed = passing, - message = 'expected \'' .. tostring(expected[1]) .. 'x,' .. + message = 'expected \'' .. tostring(expected[1]) .. 'x,' .. tostring(expected[2]) .. 'y\' got \'' .. tostring(actual[1]) .. 'x,' .. tostring(actual[2]) .. 'y\'', test = label or 'no label given' @@ -247,7 +247,7 @@ TestMethod = { -- @method - TestMethod:assertNotNil() - -- @desc - quick assert for value not nil + -- @desc - quick assert for value not nil -- @param {any} value - value to check not nil -- @return {nil} assertNotNil = function (self, value, err) @@ -264,8 +264,8 @@ TestMethod = { -- @method - TestMethod:compareImg() - -- @desc - compares a given image to the 'expected' version, with a tolerance of - -- 1px in any direction, and then saves it as the 'actual' version for + -- @desc - compares a given image to the 'expected' version, with a tolerance of + -- 1px in any direction, and then saves it as the 'actual' version for -- report viewing -- @param {table} imgdata - imgdata to save as a png -- @return {nil} @@ -275,24 +275,28 @@ TestMethod = { local ok, chunk, _ = pcall(love.image.newImageData, expected_path) if ok == false then return self:assertEquals(true, false, chunk) end local expected = chunk - local iw = imgdata:getWidth()-2 - local ih = imgdata:getHeight()-2 + local iw = imgdata:getWidth()-1 + local ih = imgdata:getHeight()-1 + local differences = {} local rgba_tolerance = self.rgba_tolerance * (1/255) - for ix=2,iw do - for iy=2,ih do + + -- for each pixel, compare the expected vs the actual pixel data + -- by default rgba_tolerance is 0 + for ix=0,iw do + for iy=0,ih do local ir, ig, ib, ia = imgdata:getPixel(ix, iy) local points = { {expected:getPixel(ix, iy)} } if self.pixel_tolerance > 0 then - table.insert(points, {expected:getPixel(ix-1, iy+1)}) - table.insert(points, {expected:getPixel(ix-1, iy)}) - table.insert(points, {expected:getPixel(ix-1, iy-1)}) - table.insert(points, {expected:getPixel(ix, iy+1)}) - table.insert(points, {expected:getPixel(ix, iy-1)}) - table.insert(points, {expected:getPixel(ix+1, iy+1)}) - table.insert(points, {expected:getPixel(ix+1, iy)}) - table.insert(points, {expected:getPixel(ix+1, iy-1)}) + if ix > 0 and iy < ih-1 then table.insert(points, {expected:getPixel(ix-1, iy+1)}) end + if ix > 0 then table.insert(points, {expected:getPixel(ix-1, iy)}) end + if ix > 0 and iy > 0 then table.insert(points, {expected:getPixel(ix-1, iy-1)}) end + if iy < ih-1 then table.insert(points, {expected:getPixel(ix, iy+1)}) end + if iy > 0 then table.insert(points, {expected:getPixel(ix, iy-1)}) end + if ix < iw-1 and iy < ih-1 then table.insert(points, {expected:getPixel(ix+1, iy+1)}) end + if ix < iw-1 then table.insert(points, {expected:getPixel(ix+1, iy)}) end + if ix < iw-1 and iy > 0 then table.insert(points, {expected:getPixel(ix+1, iy-1)}) end end local has_match_r = false local has_match_g = false @@ -317,23 +321,48 @@ TestMethod = { tostring(ix) .. ',' .. tostring(iy) .. ', matching = ' .. ymatch .. ', not matching = ' .. nmatch .. ' (' .. self.method .. '-' .. tostring(self.imgs) .. ')' ) + -- add difference co-ord for rendering later + if matching ~= true then + table.insert(differences, ix+1) + table.insert(differences, iy+1) + end end end local path = 'tempoutput/actual/love.test.graphics.' .. self.method .. '-' .. tostring(self.imgs) .. '.png' imgdata:encode('png', path) + + -- if we have differences draw them to a new canvas to display in HTML report + local dpath = 'tempoutput/difference/love.test.graphics.' .. + self.method .. '-' .. tostring(self.imgs) .. '.png' + if #differences > 0 then + local difference = love.graphics.newCanvas(iw+1, ih+1) + love.graphics.setCanvas(difference) + love.graphics.clear(0, 0, 0, 1) + love.graphics.setColor(1, 0, 1, 1) + love.graphics.points(differences) + love.graphics.setColor(1, 1, 1, 1) + love.graphics.setCanvas() + love.graphics.readbackTexture(difference):encode('png', dpath) + + -- otherwise clear the old difference file (if any) to stop it coming up + -- in future reports when there's no longer a difference + elseif love.filesystem.openFile(dpath, 'r') then + love.filesystem.remove(dpath) + end + self.imgs = self.imgs + 1 end, -- @method - TestMethod:exportImg() -- @desc - exports the given imgdata to the 'output/expected/' folder, to use when - -- writing new graphics tests to set the expected image output - -- @NOTE - you should not leave this method in when you are finished this is + -- writing new graphics tests to set the expected image output + -- @NOTE - you should not leave this method in when you are finished this is -- for test writing only -- @param {table} imgdata - imgdata to save as a png -- @param {integer} imgdata - index of the png, graphic tests are run sequentially - -- and each test image is numbered in order that its + -- and each test image is numbered in order that its -- compared to, so set the number here to match -- @return {nil} exportImg = function(self, imgdata, index) @@ -386,6 +415,21 @@ TestMethod = { return false end, + -- @method - TestMethod:isLuaVersion() + -- @desc - checks for a specific Lua version (or list of versions) + -- @param {number} - the minimum Lua version to check against + -- @return {boolean} - returns true if the current Lua version is at least the given version + isAtLeastLuaVersion = function(self, version) + return love.test.lua_version >= version + end, + + -- @method - TestMethod:isLuaJITEnabled() + -- @desc - checks if LuaJIT is enabled + -- @return {boolean} - returns true if LuaJIT is enabled + isLuaJITEnabled = function(self) + return love.test.has_lua_jit + end, + -- @method - TestMethod:evaluateTest() -- @desc - evaluates the results of all assertions for a final restult -- @return {nil} @@ -410,10 +454,10 @@ TestMethod = { if self.skipped then self.testmodule.skipped = self.testmodule.skipped + 1 love.test.totals[3] = love.test.totals[3] + 1 - self.result = { - total = '', - result = "SKIP", - passed = false, + self.result = { + total = '', + result = "SKIP", + passed = false, message = '(0/0) - method skipped [' .. self.skipreason .. ']', failures = {} } @@ -424,10 +468,10 @@ TestMethod = { self.passed = true self.testmodule.passed = self.testmodule.passed + 1 love.test.totals[1] = love.test.totals[1] + 1 - self.result = { - total = total, - result = 'PASS', - passed = true, + self.result = { + total = total, + result = 'PASS', + passed = true, message = nil, failures = {} } @@ -517,21 +561,21 @@ TestMethod = { local preview = '' if self.testmodule.module == 'graphics' then local filename = 'love.test.graphics.' .. self.method - if love.filesystem.openFile('tempoutput/actual/' .. filename .. '-1.png', 'r') then - preview = '
' .. '

Expected

' .. - '
' .. '

Actual

' - end - if love.filesystem.openFile('tempoutput/actual/' .. filename .. '-2.png', 'r') then - preview = preview .. '
' .. '

Expected

' .. - '
' .. '

Actual

' - end - if love.filesystem.openFile('tempoutput/actual/' .. filename .. '-3.png', 'r') then - preview = preview .. '
' .. '

Expected

' .. - '
' .. '

Actual

' + for f=1,5 do + local fstr = tostring(f) + if love.filesystem.openFile('tempoutput/actual/' .. filename .. '-' .. fstr .. '.png', 'r') then + preview = preview .. '
' + preview = preview .. '
' .. '

Expected

' .. + '
' .. '

Actual

' + if love.filesystem.openFile('tempoutput/difference/' .. filename .. '-' .. fstr .. '.png', 'r') then + preview = preview .. '
' .. '

Difference

' + end + preview = preview .. '
' + end end end - -- append HTML for the test class result + -- append HTML for the test class result local status = '🔴' local cls = 'red' if self.passed then status = '🟢'; cls = '' end @@ -565,7 +609,7 @@ TestMethod = { self.result.total .. msg ) - -- if we failed on multiple asserts, list them here - makes it easier for + -- if we failed on multiple asserts, list them here - makes it easier for -- debugging new methods added that are failing multiple asserts if #self.result.failures > 1 then for f=2,#self.result.failures do diff --git a/testing/classes/TestSuite.lua b/testing/classes/TestSuite.lua index 4777281e6..6457f8ea2 100644 --- a/testing/classes/TestSuite.lua +++ b/testing/classes/TestSuite.lua @@ -24,6 +24,8 @@ TestSuite = { fakequit = false, windowmode = true, current_os = love._os, + lua_version = tonumber(_VERSION:match("%d%.%d")), + has_lua_jit = type(jit) == 'table', -- love modules to test audio = {}, @@ -98,7 +100,7 @@ TestSuite = { -- when wait finished (or no yields) if coroutine.status(self.test.co) == 'dead' then - -- now we're all done evaluate the test + -- now we're all done evaluate the test local ok, chunk, err = pcall(self.test.evaluateTest, self.test) if ok == false then self.test.passed = false @@ -129,7 +131,7 @@ TestSuite = { self:printResult() love.event.quit(0) end - + end end end @@ -152,7 +154,7 @@ TestSuite = { if love.graphics then name, version, vendor, device = love.graphics.getRendererInfo() end - + local md = ' + ### Info -**344** tests were completed in **14.817s** with **335** passed, **0** failed, and **9** skipped +**355** tests were completed in **14.567s** with **343** passed, **2** failed, and **10** skipped Renderer: OpenGL | 4.1 Metal - 76.3 | Apple | Apple M1 Max ### Report | Module | Pass | Fail | Skip | Time | | --------------------- | ------ | ------ | ------- | ------ | -| 🟢 audio | 28 | 0 | 0 | 0.886s | -| 🟢 data | 12 | 0 | 0 | 0.212s | -| 🟢 event | 4 | 0 | 2 | 0.103s | -| 🟢 filesystem | 29 | 0 | 2 | 0.555s | -| 🟢 font | 7 | 0 | 0 | 0.124s | -| 🟢 graphics | 104 | 0 | 1 | 3.006s | -| 🟢 image | 5 | 0 | 0 | 0.085s | -| 🟢 joystick | 6 | 0 | 0 | 0.106s | -| 🟢 keyboard | 9 | 0 | 0 | 0.150s | -| 🟢 love | 6 | 0 | 0 | 0.098s | -| 🟢 math | 20 | 0 | 0 | 0.337s | -| 🟢 mouse | 18 | 0 | 0 | 0.302s | -| 🟢 physics | 26 | 0 | 0 | 0.434s | -| 🟢 sensor | 1 | 0 | 0 | 0.016s | -| 🟢 sound | 4 | 0 | 0 | 0.072s | -| 🟢 system | 6 | 0 | 2 | 0.135s | -| 🟢 thread | 5 | 0 | 0 | 0.363s | -| 🟢 timer | 6 | 0 | 0 | 2.074s | -| 🟢 touch | 3 | 0 | 0 | 0.023s | -| 🟢 video | 2 | 0 | 0 | 0.039s | -| 🟢 window | 34 | 0 | 2 | 5.696s | +| 🟢 audio | 31 | 0 | 0 | 1.328s | +| 🟢 data | 12 | 0 | 0 | 0.197s | +| 🟢 event | 4 | 0 | 2 | 0.100s | +| 🟢 filesystem | 33 | 0 | 2 | 0.601s | +| 🟢 font | 7 | 0 | 0 | 0.116s | +| 🔴 graphics | 104 | 1 | 2 | 3.463s | +| 🟢 image | 5 | 0 | 0 | 0.093s | +| 🟢 joystick | 6 | 0 | 0 | 0.116s | +| 🟢 keyboard | 10 | 0 | 0 | 0.170s | +| 🟢 love | 6 | 0 | 0 | 0.100s | +| 🟢 math | 20 | 0 | 0 | 0.334s | +| 🔴 mouse | 17 | 1 | 0 | 0.301s | +| 🟢 physics | 26 | 0 | 0 | 0.435s | +| 🟢 sensor | 1 | 0 | 0 | 0.017s | +| 🟢 sound | 4 | 0 | 0 | 0.075s | +| 🟢 system | 7 | 0 | 2 | 0.150s | +| 🟢 thread | 5 | 0 | 0 | 0.306s | +| 🟢 timer | 6 | 0 | 0 | 0.298s | +| 🟢 touch | 3 | 0 | 0 | 0.051s | +| 🟢 video | 2 | 0 | 0 | 0.038s | +| 🟢 window | 34 | 0 | 2 | 6.275s | ### Failures +> 🔴 Shader +> assert 4 [check shader valid] expected '' got 'vertex shader: +pixel shader: +' + +> 🔴 setGrabbed +> assert 2 [check now grabbed] expected 'true' got 'false' + diff --git a/testing/examples/lovetest_runAllTests.xml b/testing/examples/lovetest_runAllTests.xml index f6bf47390..888feca6c 100644 --- a/testing/examples/lovetest_runAllTests.xml +++ b/testing/examples/lovetest_runAllTests.xml @@ -1,137 +1,145 @@ - - - + + + - + - + - + - + - + - + - + - + - + - + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + + + - - + + - + - + - + - + - + - + - + - + - + - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + + + - + - + @@ -139,187 +147,203 @@ - + - + - + - + - + + + + + - + - + - + - + - + - + + + - - + + - + - + - + - + - - + + - + - + + + - + - + - + + assert 4 [check shader valid] expected '' got 'vertex shader: +pixel shader: +' - + + - + - + - + - + - + - + - + - + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -327,345 +351,352 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + - + - - + + - + - + - + - + - + - + - + - + - + - + - + + + - - + + - + - + - + - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + - + - + - + - + - + - + + assert 2 [check now grabbed] expected 'true' got 'false' - + - + - + - + - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - + + - + - + - - + + - + - + + + - + - + - + - - + + - + - + - + - + - - + + - + - + - + - + - + - - + + - + - + - - + + - + + + - + @@ -675,67 +706,65 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - + \ No newline at end of file