testsuite updates

- added some additional checks to check for lua version/jit enabled for use with other repos (namely love potion)
- updated HTML report CSS to group each image compare and fix pixel rendering
- fixed not showing image checks for the 4th compareImg or more in a given test method
- fixed pixel tolerance usage crashing when looking at images on the boundary
- fixed compare img not checking the first 2 rows of pixels on the x + y
-graphic tests that fail will now show a third "difference" image to help identify what isn't matching up

.gitignore

@@ -73,3 +73,4 @@ stamp-h1
 /testing/output/*.html
 /testing/output/*.md
 /testing/output/actual/*.png
+/testing/output/difference/*.png

testing/classes/TestMethod.lua

@@ -64,7 +64,7 @@ TestMethod = {
     table.insert(self.asserts, {
       key = 'assert ' .. tostring(self.count),
       passed = expected == actual,
-      message = 'expected \'' .. tostring(expected) .. '\' got \'' .. 
+      message = 'expected \'' .. tostring(expected) .. '\' got \'' ..
         tostring(actual) .. '\'',
       test = label or 'no label given'
     })
@@ -81,7 +81,7 @@ TestMethod = {
     table.insert(self.asserts, {
       key = 'assert ' .. tostring(self.count),
       passed = value == true,
-      message = 'expected \'true\' got \'' .. 
+      message = 'expected \'true\' got \'' ..
         tostring(value) .. '\'',
       test = label or 'no label given'
     })
@@ -98,7 +98,7 @@ TestMethod = {
     table.insert(self.asserts, {
       key = 'assert ' .. tostring(self.count),
       passed = value == false,
-      message = 'expected \'false\' got \'' .. 
+      message = 'expected \'false\' got \'' ..
         tostring(value) .. '\'',
       test = label or 'no label given'
     })
@@ -165,7 +165,7 @@ TestMethod = {
 
 
   -- @method - TestMethod:assertGreaterEqual()
-  -- @desc - used to check a value is >= than a certain target value 
+  -- @desc - used to check a value is >= than a certain target value
   -- @param {any} target - value to check the test agaisnt
   -- @param {any} actual - actual value of the test
   -- @param {string} label - label for this test to use in exports
@@ -187,7 +187,7 @@ TestMethod = {
 
 
   -- @method - TestMethod:assertLessEqual()
-  -- @desc - used to check a value is <= than a certain target value 
+  -- @desc - used to check a value is <= than a certain target value
   -- @param {any} target - value to check the test agaisnt
   -- @param {any} actual - actual value of the test
   -- @param {string} label - label for this test to use in exports
@@ -209,7 +209,7 @@ TestMethod = {
 
 
   -- @method - TestMethod:assertObject()
-  -- @desc - used to check a table is a love object, this runs 3 seperate 
+  -- @desc - used to check a table is a love object, this runs 3 seperate
   --         tests to check table has the basic properties of an object
   -- @note - actual object functionality tests have their own methods
   -- @param {table} obj - table to check is a valid love object
@@ -238,7 +238,7 @@ TestMethod = {
     table.insert(self.asserts, {
       key = 'assert ' .. tostring(self.count),
       passed = passing,
-      message = 'expected \'' .. tostring(expected[1]) .. 'x,' .. 
+      message = 'expected \'' .. tostring(expected[1]) .. 'x,' ..
         tostring(expected[2]) .. 'y\' got \'' ..
         tostring(actual[1]) .. 'x,' .. tostring(actual[2]) .. 'y\'',
       test = label or 'no label given'
@@ -247,7 +247,7 @@ TestMethod = {
 
 
   -- @method - TestMethod:assertNotNil()
-  -- @desc - quick assert for value not nil 
+  -- @desc - quick assert for value not nil
   -- @param {any} value - value to check not nil
   -- @return {nil}
   assertNotNil = function (self, value, err)
@@ -264,8 +264,8 @@ TestMethod = {
 
 
   -- @method - TestMethod:compareImg()
-  -- @desc - compares a given image to the 'expected' version, with a tolerance of 
-  --         1px in any direction, and then saves it as the 'actual' version for 
+  -- @desc - compares a given image to the 'expected' version, with a tolerance of
+  --         1px in any direction, and then saves it as the 'actual' version for
   --         report viewing
   -- @param {table} imgdata - imgdata to save as a png
   -- @return {nil}
@@ -275,24 +275,28 @@ TestMethod = {
     local ok, chunk, _ = pcall(love.image.newImageData, expected_path)
     if ok == false then return self:assertEquals(true, false, chunk) end
     local expected = chunk
-    local iw = imgdata:getWidth()-2
-    local ih = imgdata:getHeight()-2
+    local iw = imgdata:getWidth()-1
+    local ih = imgdata:getHeight()-1
+    local differences = {}
     local rgba_tolerance = self.rgba_tolerance * (1/255)
-    for ix=2,iw do
-      for iy=2,ih do
+
+    -- for each pixel, compare the expected vs the actual pixel data
+    -- by default rgba_tolerance is 0
+    for ix=0,iw do
+      for iy=0,ih do
         local ir, ig, ib, ia = imgdata:getPixel(ix, iy)
         local points = {
           {expected:getPixel(ix, iy)}
         }
         if self.pixel_tolerance > 0 then
-          table.insert(points, {expected:getPixel(ix-1, iy+1)})
-          table.insert(points, {expected:getPixel(ix-1, iy)})
-          table.insert(points, {expected:getPixel(ix-1, iy-1)})
-          table.insert(points, {expected:getPixel(ix, iy+1)})
-          table.insert(points, {expected:getPixel(ix, iy-1)})
-          table.insert(points, {expected:getPixel(ix+1, iy+1)})
-          table.insert(points, {expected:getPixel(ix+1, iy)})
-          table.insert(points, {expected:getPixel(ix+1, iy-1)})
+          if ix > 0 and iy < ih-1 then table.insert(points, {expected:getPixel(ix-1, iy+1)}) end
+          if ix > 0 then table.insert(points, {expected:getPixel(ix-1, iy)}) end
+          if ix > 0 and iy > 0 then table.insert(points, {expected:getPixel(ix-1, iy-1)}) end
+          if iy < ih-1 then table.insert(points, {expected:getPixel(ix, iy+1)}) end
+          if iy > 0 then table.insert(points, {expected:getPixel(ix, iy-1)}) end
+          if ix < iw-1 and iy < ih-1 then table.insert(points, {expected:getPixel(ix+1, iy+1)}) end
+          if ix < iw-1 then table.insert(points, {expected:getPixel(ix+1, iy)}) end
+          if ix < iw-1 and iy > 0 then table.insert(points, {expected:getPixel(ix+1, iy-1)}) end
         end
         local has_match_r = false
         local has_match_g = false
@@ -317,23 +321,48 @@ TestMethod = {
           tostring(ix) .. ',' .. tostring(iy) .. ', matching = ' .. ymatch ..
           ', not matching = ' .. nmatch .. ' (' .. self.method .. '-' .. tostring(self.imgs) .. ')'
         )
+        -- add difference co-ord for rendering later
+        if matching ~= true then
+          table.insert(differences, ix+1)
+          table.insert(differences, iy+1)
+        end
       end
     end
     local path = 'tempoutput/actual/love.test.graphics.' ..
       self.method .. '-' .. tostring(self.imgs) .. '.png'
     imgdata:encode('png', path)
+
+    -- if we have differences draw them to a new canvas to display in HTML report
+    local dpath = 'tempoutput/difference/love.test.graphics.' ..
+      self.method .. '-' .. tostring(self.imgs) .. '.png'
+    if #differences > 0 then
+      local difference = love.graphics.newCanvas(iw+1, ih+1)
+      love.graphics.setCanvas(difference)
+        love.graphics.clear(0, 0, 0, 1)
+        love.graphics.setColor(1, 0, 1, 1)
+        love.graphics.points(differences)
+        love.graphics.setColor(1, 1, 1, 1)
+      love.graphics.setCanvas()
+      love.graphics.readbackTexture(difference):encode('png', dpath)
+
+    -- otherwise clear the old difference file (if any) to stop it coming up 
+    -- in future reports when there's no longer a difference
+    elseif love.filesystem.openFile(dpath, 'r') then
+      love.filesystem.remove(dpath)
+    end
+
     self.imgs = self.imgs + 1
   end,
 
 
   -- @method - TestMethod:exportImg()
   -- @desc - exports the given imgdata to the 'output/expected/' folder, to use when
-  --         writing new graphics tests to set the expected image output 
-  -- @NOTE - you should not leave this method in when you are finished this is 
+  --         writing new graphics tests to set the expected image output
+  -- @NOTE - you should not leave this method in when you are finished this is
   --         for test writing only
   -- @param {table} imgdata - imgdata to save as a png
   -- @param {integer} imgdata - index of the png, graphic tests are run sequentially
-  --                            and each test image is numbered in order that its 
+  --                            and each test image is numbered in order that its
   --                            compared to, so set the number here to match
   -- @return {nil}
   exportImg = function(self, imgdata, index)
@@ -386,6 +415,21 @@ TestMethod = {
     return false
   end,
 
+  -- @method - TestMethod:isLuaVersion()
+  -- @desc - checks for a specific Lua version (or list of versions)
+  -- @param {number} - the minimum Lua version to check against
+  -- @return {boolean} - returns true if the current Lua version is at least the given version
+  isAtLeastLuaVersion = function(self, version)
+    return love.test.lua_version >= version
+  end,
+
+  -- @method - TestMethod:isLuaJITEnabled()
+  -- @desc - checks if LuaJIT is enabled
+  -- @return {boolean} - returns true if LuaJIT is enabled
+  isLuaJITEnabled = function(self)
+    return love.test.has_lua_jit
+  end,
+
   -- @method - TestMethod:evaluateTest()
   -- @desc - evaluates the results of all assertions for a final restult
   -- @return {nil}
@@ -410,10 +454,10 @@ TestMethod = {
     if self.skipped then
       self.testmodule.skipped = self.testmodule.skipped + 1
       love.test.totals[3] = love.test.totals[3] + 1
-      self.result = { 
-        total = '', 
-        result = "SKIP", 
-        passed = false, 
+      self.result = {
+        total = '',
+        result = "SKIP",
+        passed = false,
         message = '(0/0) - method skipped [' .. self.skipreason .. ']',
         failures = {}
       }
@@ -424,10 +468,10 @@ TestMethod = {
         self.passed = true
         self.testmodule.passed = self.testmodule.passed + 1
         love.test.totals[1] = love.test.totals[1] + 1
-        self.result = { 
-          total = total, 
-          result = 'PASS', 
-          passed = true, 
+        self.result = {
+          total = total,
+          result = 'PASS',
+          passed = true,
           message = nil,
           failures = {}
         }
@@ -517,21 +561,21 @@ TestMethod = {
     local preview = ''
     if self.testmodule.module == 'graphics' then
       local filename = 'love.test.graphics.' .. self.method
-      if love.filesystem.openFile('tempoutput/actual/' .. filename .. '-1.png', 'r') then
-        preview = '<div class="preview">' .. '<img src="expected/' .. filename .. '-1.png"/><p>Expected</p></div>' ..
-          '<div class="preview">' .. '<img src="actual/' .. filename .. '-1.png"/><p>Actual</p></div>'
-      end
-      if love.filesystem.openFile('tempoutput/actual/' .. filename .. '-2.png', 'r') then
-        preview = preview .. '<div class="preview">' .. '<img src="expected/' .. filename .. '-2.png"/><p>Expected</p></div>' ..
-          '<div class="preview">' .. '<img src="actual/' .. filename .. '-2.png"/><p>Actual</p></div>'
-      end
-      if love.filesystem.openFile('tempoutput/actual/' .. filename .. '-3.png', 'r') then
-        preview = preview .. '<div class="preview">' .. '<img src="expected/' .. filename .. '-3.png"/><p>Expected</p></div>' ..
-          '<div class="preview">' .. '<img src="actual/' .. filename .. '-3.png"/><p>Actual</p></div>'
+      for f=1,5 do
+        local fstr = tostring(f)
+        if love.filesystem.openFile('tempoutput/actual/' .. filename .. '-' .. fstr .. '.png', 'r') then
+          preview = preview .. '<div class="preview-wrap">'
+          preview = preview .. '<div class="preview">' .. '<img src="expected/' .. filename .. '-' .. fstr .. '.png"/><p>Expected</p></div>' ..
+            '<div class="preview">' .. '<img src="actual/' .. filename .. '-' .. fstr .. '.png"/><p>Actual</p></div>'
+          if love.filesystem.openFile('tempoutput/difference/' .. filename .. '-' .. fstr .. '.png', 'r') then
+            preview = preview .. '<div class="preview">' .. '<img src="difference/' .. filename .. '-' .. fstr .. '.png"/><p>Difference</p></div>'
+          end
+          preview = preview .. '</div>'
+        end
       end
     end
 
-    -- append HTML for the test class result 
+    -- append HTML for the test class result
     local status = '🔴'
     local cls = 'red'
     if self.passed then status = '🟢'; cls = '' end
@@ -565,7 +609,7 @@ TestMethod = {
       self.result.total .. msg
     )
 
-    -- if we failed on multiple asserts, list them here - makes it easier for 
+    -- if we failed on multiple asserts, list them here - makes it easier for
     -- debugging new methods added that are failing multiple asserts
     if #self.result.failures > 1 then
       for f=2,#self.result.failures do

testing/classes/TestSuite.lua

@@ -24,6 +24,8 @@ TestSuite = {
       fakequit = false,
       windowmode = true,
       current_os = love._os,
+      lua_version = tonumber(_VERSION:match("%d%.%d")),
+      has_lua_jit = type(jit) == 'table',
 
       -- love modules to test
       audio = {},
@@ -98,7 +100,7 @@ TestSuite = {
 
               -- when wait finished (or no yields)
               if coroutine.status(self.test.co) == 'dead' then
-                -- now we're all done evaluate the test 
+                -- now we're all done evaluate the test
                 local ok, chunk, err = pcall(self.test.evaluateTest, self.test)
                 if ok == false then
                   self.test.passed = false
@@ -129,7 +131,7 @@ TestSuite = {
               self:printResult()
               love.event.quit(0)
             end
-  
+
           end
         end
       end
@@ -152,7 +154,7 @@ TestSuite = {
     if love.graphics then
       name, version, vendor, device = love.graphics.getRendererInfo()
     end
-    
+
     local md = '<!-- PASSED ' .. tostring(self.totals[1]) ..
       ' || FAILED ' .. tostring(self.totals[2]) ..
       ' || SKIPPED ' .. tostring(self.totals[3]) ..
@@ -161,22 +163,22 @@ TestSuite = {
       finaltime .. 's** with **' ..
       tostring(self.totals[1]) .. '** passed, **' ..
       tostring(self.totals[2]) .. '** failed, and **' ..
-      tostring(self.totals[3]) .. '** skipped\n\n' .. 
+      tostring(self.totals[3]) .. '** skipped\n\n' ..
       'Renderer: ' .. name .. ' | ' .. version .. ' | ' .. vendor .. ' | ' .. device .. '\n\n' ..
       '### Report\n' ..
       '| Module                | Pass | Fail | Skip | Time   |\n' ..
       '| --------------------- | ------ | ------ | ------- | ------ |\n' ..
       self.mdrows .. '\n### Failures\n' .. self.mdfailures
 
-    local xml = '<testsuites name="love.test" tests="' .. tostring(self.totals[1]) .. 
-      '" failures="' .. tostring(self.totals[2]) .. 
-      '" skipped="' .. tostring(self.totals[3]) .. 
+    local xml = '<testsuites name="love.test" tests="' .. tostring(self.totals[1]) ..
+      '" failures="' .. tostring(self.totals[2]) ..
+      '" skipped="' .. tostring(self.totals[3]) ..
       '" time="' .. finaltime .. '">\n'
 
     local status = '🔴'
     if self.totals[2] == 0 then status = '🟢' end
-    local html = '<html><head><style>* { font-family: monospace; margin: 0; font-size: 11px; padding: 0; } body { margin: 50px; } h1 { padding-bottom: 10px; font-size: 13px; } h2 { padding: 20px 0 10px 0; font-size: 12px; } .summary { list-style: none; margin: 0; padding: 0; } .summary li { float: left; background: #eee; padding: 5px; margin-right: 10px; } table { background: #eee; margin-top: 10px; width: 100%; max-width: 800px; border-collapse: collapse } table thead { background: #ddd; } table th, table td { padding: 2px; } tr.red { color: red } .wrap { max-width: 800px; margin: auto; } .preview { width: 64px; height: 80px; float: left; margin-right: 10px; } .preview img { width: 100% } .preview p { text-align: center; }</style></head><body><div class="wrap"><h1>' .. status .. '&nbsp;love.test</h1><ul class="summary">'
-    html = html .. 
+    local html = '<html><head><style>* { font-family: monospace; margin: 0; font-size: 11px; padding: 0; } body { margin: 50px; } h1 { padding-bottom: 10px; font-size: 13px; } h2 { padding: 20px 0 10px 0; font-size: 12px; } .summary { list-style: none; margin: 0; padding: 0; } .summary li { float: left; background: #eee; padding: 5px; margin-right: 10px; } table { background: #eee; margin-top: 10px; width: 100%; max-width: 800px; border-collapse: collapse } table thead { background: #ddd; } table th, table td { padding: 2px; } tr.red { color: red } .wrap { max-width: 800px; margin: auto; } .preview-wrap { display: inline-block; height: 80px; padding: 5px 0 0 5px; margin: 5px; background: #ccc; } .preview { width: 64px; height: 80px; float: left; margin-right: 10px; } .preview:nth-last-child(1) { margin-right: 5px; } .preview img { width: 100%; image-rendering: pixelated; } .preview p { text-align: center; }</style></head><body><div class="wrap"><h1>' .. status .. '&nbsp;love.test</h1><ul class="summary">'
+    html = html ..
       '<li>🟢&nbsp;' .. tostring(self.totals[1]) .. ' Tests</li>' ..
       '<li>🔴&nbsp;' .. tostring(self.totals[2]) .. ' Failures</li>' ..
       '<li>🟡&nbsp;' .. tostring(self.totals[3]) .. ' Skipped</li>' ..

testing/output/difference/notes.txt

@@ -0,0 +1,3 @@
+# Graphic Differences
+If a graphics test fails then a "difference" image will be created to highlight 
+the differences between the actual + expected image, for use in the HTML report

testing/tests/filesystem.lua

@@ -137,7 +137,7 @@ end
 
 -- love.filesystem.createDirectory
 love.test.filesystem.createDirectory = function(test)
-  -- try creating a dir + subdir and check both exist 
+  -- try creating a dir + subdir and check both exist
   local success = love.filesystem.createDirectory('foo/bar')
   test:assertNotEquals(false, success, 'check success')
   test:assertNotEquals(nil, love.filesystem.getInfo('foo', 'directory'), 'check directory created')
@@ -225,7 +225,7 @@ love.test.filesystem.getRealDirectory = function(test)
   love.filesystem.createDirectory('foo')
   love.filesystem.write('foo/test.txt', 'test')
   -- check save dir matches the real dir we just wrote to
-  test:assertEquals(love.filesystem.getSaveDirectory(), 
+  test:assertEquals(love.filesystem.getSaveDirectory(),
     love.filesystem.getRealDirectory('foo/test.txt'), 'check directory matches')
   -- cleanup
   love.filesystem.remove('foo/test.txt')
@@ -321,17 +321,28 @@ love.test.filesystem.load = function(test)
   -- setup some fake lua files
   love.filesystem.write('test1.lua', 'function test()\nreturn 1\nend\nreturn test()')
   love.filesystem.write('test2.lua', 'function test()\nreturn 1')
-  -- check file that doesn't exist
-  local chunk1, errormsg1 = love.filesystem.load('faker.lua', 'b')
-  test:assertEquals(nil, chunk1, 'check file doesnt exist')
-  -- check valid lua file (text load)
-  local chunk2, errormsg2 = love.filesystem.load('test1.lua', 't')
-  test:assertEquals(nil, errormsg2, 'check no error message')
-  test:assertEquals(1, chunk2(), 'check lua file runs')
+
+  if test:isAtLeastLuaVersion(5.2) or test:isLuaJITEnabled() then
+    -- check file that doesn't exist
+    local chunk1, errormsg1 = love.filesystem.load('faker.lua', 'b')
+    test:assertEquals(nil, chunk1, 'check file doesnt exist')
+    -- check valid lua file (text load)
+    local chunk2, errormsg2 = love.filesystem.load('test1.lua', 't')
+    test:assertEquals(nil, errormsg2, 'check no error message')
+    test:assertEquals(1, chunk2(), 'check lua file runs')
+  else
+    local _, errormsg3 = love.filesystem.load('test1.lua', 'b')
+    test:assertNotEquals(nil, errormsg3, 'check for an error message')
+
+    local _, errormsg4 = love.filesystem.load('test1.lua', 't')
+    test:assertNotEquals(nil, errormsg4, 'check for an error message')
+  end
+
   -- check valid lua file (any load)
-  local chunk4, errormsg4 = love.filesystem.load('test1.lua', 'bt')
-  test:assertEquals(nil, errormsg2, 'check no error message')
-  test:assertEquals(1, chunk4(), 'check lua file runs')
+  local chunk5, errormsg5 = love.filesystem.load('test1.lua', 'bt')
+  test:assertEquals(nil, errormsg5, 'check no error message')
+  test:assertEquals(1, chunk5(), 'check lua file runs')
+
   -- check invalid lua file
   local ok, chunk, err = pcall(love.filesystem.load, 'test2.lua')
   test:assertFalse(ok, 'check invalid lua file')
@@ -410,7 +421,7 @@ end
 
 -- love.filesystem.unmountCommonPath
 --love.test.filesystem.unmountCommonPath = function(test)
---  -- check unmounting invalid 
+--  -- check unmounting invalid
 --  local ok = pcall(love.filesystem.unmountCommonPath, 'fakepath')
 --  test:assertFalse(ok, 'check unmount invalid common path')
 --  -- check mounting valid paths

testing/todo.md

@@ -23,8 +23,6 @@ These are all the outstanding methods that require test coverage, along with a f
 - love.graphics.drawIndirect()
 - love.graphics.getQuadIndexBuffer()
 - love.graphics.setBlendState()
-- love.graphics.setOrthoProjection()
-- love.graphics.setPerspectiveProjection()
 - love.graphics.resetProjection()
 - love.graphics.Mesh:getAttachedAttributes()
 - love.graphics.Shader:hasStage()