small update

changed tests so that all assert failures are logged rather than just the first assert that failed, added isOS() to the test method class to be used in tests directly, updated test readme on a couple bits and added newTexture basic check

testing/classes/TestMethod.lua

@@ -374,23 +374,42 @@ TestMethod = {
   end,
 
 
+  -- @method - TestMethod:isOS()
+  -- @desc - checks for a specific OS (or list of OSs)
+  -- @param {string||table} - single OS string or list of OSs that are allowed, 
+  --          these will be checked agaisnt love.system.getOS()'s return value
+  -- @return {boolean} - returns true if one of the OSs given matches actual OS
+  isOS = function(self, oss)
+    if type(oss) == 'table' then
+      for o=1,#oss do
+        if oss[o] == love.test.current_os then return true end
+      end
+    else
+      return love.test.current_os == oss
+    end
+  end,
+
   -- @method - TestMethod:evaluateTest()
   -- @desc - evaluates the results of all assertions for a final restult
   -- @return {nil}
   evaluateTest = function(self)
     local failure = ''
     local failures = 0
+
+    -- check all asserts for failures, additional failures are also printed
+    local assert_failures = {}
     for a=1,#self.asserts do
-      -- @TODO show all failed assertion methods?
-      -- currently just shows the first assert that failed
       if not self.asserts[a].passed and not self.skipped then
         if failure == '' then failure = self.asserts[a] end
+        table.insert(assert_failures, self.asserts[a])
         failures = failures + 1
       end
     end
     if self.fatal ~= '' then failure = self.fatal end
     local passed = tostring(#self.asserts - failures)
     local total = '(' .. passed .. '/' .. tostring(#self.asserts) .. ')'
+
+    -- skipped tests have a special log
     if self.skipped then
       self.testmodule.skipped = self.testmodule.skipped + 1
       love.test.totals[3] = love.test.totals[3] + 1
@@ -398,9 +417,12 @@ TestMethod = {
         total = '', 
         result = "SKIP", 
         passed = false, 
-        message = '(0/0) - method skipped [' .. self.skipreason .. ']'
+        message = '(0/0) - method skipped [' .. self.skipreason .. ']',
+        failures = {}
       }
     else
+
+      -- if no failure but has asserts, then passed
       if failure == '' and #self.asserts > 0 then
         self.passed = true
         self.testmodule.passed = self.testmodule.passed + 1
@@ -409,38 +431,48 @@ TestMethod = {
           total = total, 
           result = 'PASS', 
           passed = true, 
-          message = nil
+          message = nil,
+          failures = {}
         }
+
+      -- otherwise it failed
       else
         self.passed = false
         self.testmodule.failed = self.testmodule.failed + 1
         love.test.totals[2] = love.test.totals[2] + 1
+
+        -- no asserts means invalid test
         if #self.asserts == 0 then
           local msg = 'no asserts defined'
           if self.fatal ~= '' then msg = self.fatal end
-          self.result = { 
-            total = total, 
-            result = 'FAIL', 
-            passed = false, 
-            key = 'test', 
-            message = msg 
+          self.result = {
+            total = total,
+            result = 'FAIL',
+            passed = false,
+            key = 'test',
+            message = msg,
+            failures = {}
           }
+
+        -- otherwise we had failures, log the first and supply the list of
+        -- additional failures if any for printResult()
         else
           local key = failure['key']
           if failure['test'] ~= nil then
             key = key .. ' [' .. failure['test'] .. ']'
           end
           local msg = failure['message']
-          if self.fatal ~= '' then 
+          if self.fatal ~= '' then
             key = 'code'
             msg = self.fatal
           end
-          self.result = { 
-            total = total, 
-            result = 'FAIL', 
-            passed = false, 
+          self.result = {
+            total = total,
+            result = 'FAIL',
+            passed = false,
             key = key,
-            message = msg
+            message = msg,
+            failures = assert_failures
           }
         end
       end
@@ -535,6 +567,21 @@ TestMethod = {
       ' ==> ' .. self.result.result .. ' - ' .. endtime .. 's ' ..
       self.result.total .. msg
     )
+
+    -- if we failed on multiple asserts, list them here - makes it easier for 
+    -- debugging new methods added that are failing multiple asserts
+    if #self.result.failures > 1 then
+      for f=2,#self.result.failures do
+        local addf = self.result.failures[f]
+        self.testmodule:log(
+          self.testmodule.colors[self.result.result],
+          '  ' .. tested .. matching,
+          ' ==> ' ..
+          addf['key'] .. ' [' .. addf['test'] .. '] failed - ' .. addf['message']
+        )
+      end
+    end
+
   end
 
 

testing/classes/TestSuite.lua

@@ -23,6 +23,7 @@ TestSuite = {
       delayed = nil,
       fakequit = false,
       windowmode = true,
+      current_os = love.system.getOS(),
 
       -- love modules to test
       audio = {},
@@ -76,7 +77,10 @@ TestSuite = {
               TextRun = 'love.' .. self.module.module .. '.' .. method
 
               self.test.co = coroutine.create(function()
-                local ok, chunk, err = pcall(love.test[love.test.module.module][method], love.test.test)
+                local ok, chunk, err = pcall(
+                  love.test[love.test.module.module][method],
+                  love.test.test
+                )
                 if ok == false then
                   love.test.test['passed'] = false
                   love.test.test['fatal'] = tostring(chunk) .. tostring(err)
@@ -140,7 +144,14 @@ TestSuite = {
   printResult = function(self)
     local finaltime = UtilTimeFormat(self.time)
 
-    local name, version, vendor, device = love.graphics.getRendererInfo()
+    -- in case we dont have love.graphics loaded, for future module specific disabling
+    local name = 'NONE'
+    local version = 'NONE'
+    local vendor = 'NONE'
+    local device = 'NONE'
+    if love.graphics then
+      name, version, vendor, device = love.graphics.getRendererInfo()
+    end
     
     local md = '<!-- PASSED ' .. tostring(self.totals[1]) ..
       ' || FAILED ' .. tostring(self.totals[2]) ..

testing/main.lua

@@ -41,6 +41,8 @@ love.load = function(args)
       resizable = true,
       centered = true
     })
+
+    -- set up some graphics to draw if enabled
     if love.graphics ~= nil then
       love.graphics.setDefaultFilter("nearest", "nearest")
       love.graphics.setLineStyle('rough')
@@ -54,6 +56,7 @@ love.load = function(args)
       TextCommand = 'Loading...'
       TextRun = ''
     end
+
   end
 
   -- mount for output later

testing/readme.md

@@ -25,15 +25,15 @@ This is the status of all module tests.
 See the **Todo** section for outstanding tasks if you want to contribute!
 | Module            | Done | Skip | Modules          | Done | Skip |
 | ----------------- | ---- | ---- | ---------------- | ---- | ---- |
-| 🟢 audio          |   28 |   0  | 🟢 mouse          |   18 |   0  |
+| 🟢 audio          |   31 |   0  | 🟢 mouse          |   18 |   0  |
 | 🟢 data           |   12 |   0  | 🟢 physics        |   26 |   0  |
 | 🟢 event          |    4 |   2  | 🟢 sensor         |    1 |   0  |
-| 🟢 filesystem     |   29 |   2  | 🟢 sound          |    4 |   0  |
-| 🟢 font           |    7 |   0  | 🟢 system         |    6 |   2  |
-| 🟢 graphics       |  104 |   1  | 🟢 thread         |    5 |   0  |
+| 🟢 filesystem     |   33 |   2  | 🟢 sound          |    4 |   0  |
+| 🟢 font           |    7 |   0  | 🟢 system         |    7 |   2  |
+| 🟢 graphics       |  105 |   1  | 🟢 thread         |    5 |   0  |
 | 🟢 image          |    5 |   0  | 🟢 timer          |    6 |   0  |
 | 🟢 joystick       |    6 |   0  | 🟢 touch          |    3 |   0  |
-| 🟢 keyboard       |    9 |   0  | 🟢 video          |    2 |   0  |
+| 🟢 keyboard       |   10 |   0  | 🟢 video          |    2 |   0  |
 | 🟢 love           |    6 |   0  | 🟢 window         |   34 |   2  |
 | 🟢 math           |   20 |   0  | 
 
@@ -102,7 +102,7 @@ love.test.filesystem.read = function(test)
 end
 ```
 
-Each test is run inside it's own coroutine - you can use `test:waitFrames(frames)` to pause the test for a small period if you need to check things that won't happen for a few seconds.
+Each test is run inside it's own coroutine - you can use `test:waitFrames(frames)` or `test:waitSeconds(seconds)` to pause the test for a small period if you need to check things that won't happen for a few frames/seconds.
 
 After each test method is ran, the assertions are totalled up, printed, and we move onto the next method! Once all methods in the suite are run a total pass/fail/skip is given for that module and we move onto the next module (if any)
 
@@ -133,7 +133,7 @@ The automated tests through Github work for the most part however there are a fe
 
 These exceptions are either skipped, or handled by using a 1px or 1/255rgba tolerance - when run locally on real hardware, these tests pass fine at the default 0 tolerance.  
 You can specify the test suite is being run on a runner by adding the `--isRunner` flag in your workflow file, i.e.:  
-`& 'c:\Program Files\LOVE\love.exe' PATH_TO_TESTING_FOLDER/main.lua --console --runAllTests --isRunner`
+`& 'c:\Program Files\LOVE\love.exe' PATH_TO_TESTING_FOLDER/main.lua --console --all --isRunner`
 | Test                       |    OS     |      Exception      | Reason |
 | -------------------------- | --------- | ------------------- | ------ |
 | love.graphics.setWireframe |   MacOS   |    1px tolerance    | Wireframes are offset by 1,1 when drawn |

testing/tests/audio.lua

@@ -469,7 +469,7 @@ love.test.audio.setPlaybackDevice = function(test)
   -- rn on macos all 3 return false
   -- whereas linux/windows return true for blank/current, which is expected
   -- as openalsoft treats blank as current
-  if love.system.getOS() == 'OS X' then
+  if test:isOS('OS X') then
     test:assertFalse(success1, 'check blank device fails')
     test:assertFalse(success2, 'check invalid device fails')
     test:assertFalse(success3, 'check existing device fails')

testing/tests/filesystem.lua

@@ -393,7 +393,7 @@ love.test.filesystem.mountCommonPath = function(test)
   local mount3 = love.filesystem.mountCommonPath('userhome', 'userhome', 'readwrite')
   local mount4 = love.filesystem.mountCommonPath('userappdata', 'userappdata', 'readwrite')
   -- userdesktop isnt valid on linux
-  if love.system.getOS() ~= 'Linux' then
+  if not test:isOS('Linux') then
     local mount5 = love.filesystem.mountCommonPath('userdesktop', 'userdesktop', 'readwrite')
     test:assertTrue(mount5, 'check mount userdesktop')
   end

testing/tests/graphics.lua

@@ -10,56 +10,53 @@
 
 -- GraphicsBuffer (love.graphics.newBuffer)
 love.test.graphics.Buffer = function(test)
+
+  -- setup vertex data and create some buffers
   local vertexformat = {
     {name="VertexPosition", format="floatvec2"},
     {name="VertexTexCoord", format="floatvec2"},
     {name="VertexColor", format="unorm8vec4"},
   }
-
   local vertexdata = {
     {0,  0,  0, 0, 1, 0, 1, 1},
     {10, 0,  1, 0, 0, 1, 1, 1},
     {10, 10, 1, 1, 0, 0, 1, 1},
     {0,  10, 0, 1, 1, 0, 0, 1},
   }
-
   local flatvertexdata = {}
   for i, vert in ipairs(vertexdata) do
     for j, v in ipairs(vert) do
       table.insert(flatvertexdata, v)
     end
   end
-
   local vertexbuffer1 = love.graphics.newBuffer(vertexformat, 4, {vertex=true, debugname='testvertexbuffer'})
   local vertexbuffer2 = love.graphics.newBuffer(vertexformat, vertexdata, {vertex=true})
-
   test:assertObject(vertexbuffer1)
   test:assertObject(vertexbuffer2)
 
+  -- check buffer properties
   test:assertEquals(4, vertexbuffer1:getElementCount(), 'check vertex count 1')
   test:assertEquals(4, vertexbuffer2:getElementCount(), 'check vertex count 2')
-
   -- vertex buffers have their elements tightly packed.
   test:assertEquals(20, vertexbuffer1:getElementStride(), 'check vertex array stride')
-
   test:assertEquals(20 * 4, vertexbuffer1:getSize(), 'check vertex buffer size')
-
   vertexbuffer1:setArrayData(vertexdata)
   vertexbuffer1:setArrayData(flatvertexdata)
-
   vertexbuffer1:clear(8, 8) -- partial clear (the first texcoord)
 
+  -- check buffer types
   test:assertTrue(vertexbuffer1:isBufferType('vertex'), 'check is vertex buffer')
   test:assertFalse(vertexbuffer1:isBufferType('index'), 'check is not index buffer')
   test:assertFalse(vertexbuffer1:isBufferType('texel'), 'check is not texel buffer')
   test:assertFalse(vertexbuffer1:isBufferType('shaderstorage'), 'check is not shader storage buffer')
 
+  -- check debug name
   test:assertEquals('testvertexbuffer', vertexbuffer1:getDebugName(), 'check buffer debug name')
 
+  -- check buffer format and format properties
   local format = vertexbuffer1:getFormat()
   test:assertEquals('table', type(format), 'check buffer format is table')
   test:assertEquals(#vertexformat, #format, 'check buffer format length')
-
   for i, v in ipairs(vertexformat) do
     test:assertEquals(v.name, format[i].name, string.format('check buffer format %d name', i))
     test:assertEquals(v.format, format[i].format, string.format('check buffer format %d format', i))
@@ -67,8 +64,10 @@ love.test.graphics.Buffer = function(test)
     test:assertNotNil(format[i].offset)
   end
 
+  -- check index buffer
   local indexbuffer = love.graphics.newBuffer('uint16', 128, {index=true})
   test:assertTrue(indexbuffer:isBufferType('index'), 'check is index buffer')
+
 end
 
 
@@ -80,22 +79,22 @@ love.test.graphics.ShaderStorageBuffer = function(test)
     return
   end
 
+  -- setup buffer
   local format = {
     { name="1", format="float" },
     { name="2", format="floatmat4x4" },
     { name="3", format="floatvec2" }
   }
-
   local buffer = love.graphics.newBuffer(format, 1, {shaderstorage = true})
-
   test:assertEquals(96, buffer:getElementStride(), 'check shader storage buffer element stride')
 
+  -- set new data
   local data = {}
   for i = 1, 19 do
     data[i] = 0
   end
-
   buffer:setArrayData(data)
+
 end
 
 
@@ -1170,7 +1169,7 @@ love.test.graphics.arc = function(test)
     love.graphics.setColor(1, 1, 1, 1)
   love.graphics.setCanvas()
   local imgdata3 = love.graphics.readbackTexture(canvas)
-  if GITHUB_RUNNER and love.system.getOS() == 'OS X' then
+  if GITHUB_RUNNER and test:isOS('OS X') then
     -- on macosx runners, the arcs are not drawn as accurately at low res
     -- there's a couple pixels different in the curve of the arc but as we
     -- are at such a low resolution I think that can be expected
@@ -1468,7 +1467,7 @@ love.test.graphics.captureScreenshot = function(test)
   TextCommand = prevtextcommand
   test:assertNotNil(cbdata)
 
-  if love.system.getOS() == "iOS" or love.system.getOS() == "Android" then
+  if test:isOS({'iOS', 'Android'}) then
     -- Mobile operating systems don't let us control the window resolution,
     -- so we can't compare the reference image properly.
     test:assertTrue(true, 'skip test')
@@ -1586,7 +1585,7 @@ love.test.graphics.newSpriteBatch = function(test)
 end
 
 
--- love.graphics.newText
+-- love.graphics.newTextBatch
 -- @NOTE this is just basic nil checking, objs have their own test method
 love.test.graphics.newTextBatch = function(test)
   local font = love.graphics.newFont('resources/font.ttf')
@@ -1594,6 +1593,14 @@ love.test.graphics.newTextBatch = function(test)
 end
 
 
+-- love.graphics.newTexture
+-- @NOTE this is just basic nil checking, objs have their own test method
+love.test.graphics.newTexture = function(test)
+  local imgdata = love.image.newImageData('resources/love.png')
+  test:assertObject(love.graphics.newTexture(imgdata))
+end
+
+
 -- love.graphics.newVideo
 -- @NOTE this is just basic nil checking, objs have their own test method
 love.test.graphics.newVideo = function(test)
@@ -2167,7 +2174,7 @@ love.test.graphics.setLineStyle = function(test)
   love.graphics.setCanvas()
   local imgdata = love.graphics.readbackTexture(canvas)
   -- linux runner needs a 1/255 tolerance for the blend between a rough line + bg 
-  if GITHUB_RUNNER and love.system.getOS() == 'Linux' then
+  if GITHUB_RUNNER and test:isOS('Linux') then
     test.rgba_tolerance = 1
   end
   test:compareImg(imgdata)
@@ -2300,7 +2307,7 @@ love.test.graphics.setWireframe = function(test)
     love.graphics.setWireframe(false)
     local imgdata = love.graphics.readbackTexture(canvas)
     -- on macOS runners wireframes are drawn 1px off from the target
-    if GITHUB_RUNNER and love.system.getOS() == 'OS X' then
+    if GITHUB_RUNNER and test:isOS('OS X') then
       test.pixel_tolerance = 1
     end
     test:compareImg(imgdata)
@@ -2667,8 +2674,8 @@ love.test.graphics.getSupported = function(test)
     'clampzero', 'lighten', 'glsl3', 'instancing', 'fullnpot', 
     'pixelshaderhighp', 'shaderderivatives', 'indirectdraw', 'mipmaprange',
     'copyrendertargettobuffer', 'copytexturetobuffer', 'copybuffer',
-    'indexbuffer32bit', 'multirendertargetformats', 'clampone', 'blendminmax',
-    'glsl4'
+    'copybuffertotexture', 'indexbuffer32bit', 'multirendertargetformats', 
+    'clampone', 'blendminmax', 'glsl4'
   }
   local features = love.graphics.getSupported()
   for g=1,#gfs do

testing/todo.md

@@ -3,7 +3,6 @@ These are all the outstanding methods that require test coverage, along with a f
 
 ## General 
 - ability to test loading different combinations of modules if needed?
-- performance tests? need to discuss what + how, might be better as a seperate thing
 - check expected behaviour of mount + unmount with common path
   try uncommenting love.filesystem.unmountCommonPath and you'll see the issues
 - revisit love.audio.setPlaybackDevice when we update openal soft for MacOS
@@ -19,7 +18,6 @@ These are all the outstanding methods that require test coverage, along with a f
 - love.graphics.newComputeShader()
 - love.graphics.dispatchThreadgroups()
 - love.graphics.dispatchIndirect()
-- love.graphics.newTexture()
 - love.graphics.drawFromShader()
 - love.graphics.drawFromShaderIndirect()
 - love.graphics.drawIndirect()