diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f3922d769..a6da63566 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,7 +3,13 @@ on: [push, pull_request] jobs: linux-os: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 + permissions: + checks: write + pull-requests: write + env: + ALSOFT_CONF: love2d-${{ github.sha }}/testing/resources/alsoft.conf + DISPLAY: :99 steps: - name: Update APT run: sudo apt-get update @@ -16,7 +22,8 @@ jobs: libgl1-mesa-dev libdbus-1-dev libudev-dev libgles2-mesa-dev \ libegl1-mesa-dev libibus-1.0-dev fcitx-libs-dev libsamplerate0-dev \ libsndio-dev libwayland-dev libxkbcommon-dev libdrm-dev libgbm-dev \ - libcurl4-openssl-dev + libcurl4-openssl-dev libfuse2 wmctrl openbox mesa-vulkan-drivers \ + libvulkan1 vulkan-tools vulkan-validationlayers - name: Checkout love-appimage-source uses: actions/checkout@v3 with: @@ -40,6 +47,83 @@ jobs: run: make LOVE_BRANCH=${{ github.sha }} - name: Print LuaJIT branch run: git -C LuaJIT-v2.1 branch -v + # start xvfb for test running + - name: Start xvfb and openbox + run: | + echo "Starting XVFB on $DISPLAY" + Xvfb $DISPLAY -screen 0, 360x240x24 & + echo "XVFBPID=$!" >> $GITHUB_ENV + # wait for xvfb to startup (3s is the same amount xvfb-run waits by default) + sleep 3 + openbox & + echo "OPENBOXPID=$!" >> $GITHUB_ENV + # linux opengl tests + - name: Run Test Suite (opengl) + run: | + chmod a+x love-${{ github.sha }}.AppImage + ./love-${{ github.sha }}.AppImage love2d-${{ github.sha }}/testing/main.lua --runAllTests --isRunner + - name: Love Test Report (opengl) + id: report1 + uses: ellraiser/love-test-report@main + with: + name: Love Testsuite Linux + title: test-report-linux-opengl + path: love2d-${{ github.sha }}/testing/output/lovetest_runAllTests.md + token: ${{ secrets.GITHUB_TOKEN }} + - name: Zip Test Output (opengl) + run: | + 7z a -tzip test-output-linux-opengl.zip love2d-${{ github.sha }}/testing/output/ + - name: Artifact Test Output (opengl) + uses: actions/upload-artifact@v3 + with: + name: test-output-linux-opengl-${{ steps.report1.outputs.conclusion }} + path: test-output-linux-opengl.zip + # linux opengles tests + - name: Run Test Suite (opengles) + run: | + export LOVE_GRAPHICS_USE_OPENGLES=1 + ./love-${{ github.sha }}.AppImage love2d-${{ github.sha }}/testing/main.lua --runAllTests --isRunner + - name: Love Test Report (opengles) + uses: ellraiser/love-test-report@main + id: report2 + with: + name: Love Testsuite Linux + title: test-report-linux-opengles + path: love2d-${{ github.sha }}/testing/output/lovetest_runAllTests.md + token: ${{ secrets.GITHUB_TOKEN }} + - name: Zip Test Output (opengles) + run: | + 7z a -tzip test-output-linux-opengles.zip love2d-${{ github.sha }}/testing/output/ + - name: Artifact Test Output (opengles) + uses: actions/upload-artifact@v3 + with: + name: test-output-linux-opengles-${{ steps.report2.outputs.conclusion }} + path: test-output-linux-opengles.zip +# # linux vulkan tests +# - name: Run Test Suite (vulkan) +# run: | +# export LOVE_GRAPHICS_DEBUG=1 +# ./love-${{ github.sha }}.AppImage love2d-${{ github.sha }}/testing/main.lua --runAllTests --isRunner --renderers vulkan +# - name: Love Test Report (vulkan) +# uses: ellraiser/love-test-report@main +# with: +# name: Love Testsuite Linux +# title: test-report-linux-vulkan +# path: love2d-${{ github.sha }}/testing/output/lovetest_runAllTests.md +# - name: Zip Test Output (vulkan) +# run: | +# 7z a -tzip test-output-linux-vulkan.zip love2d-${{ github.sha }}/testing/output/ +# - name: Artifact Test Output (vulkan) +# uses: actions/upload-artifact@v3 +# with: +# name: test-output-linux-vulkan +# path: test-output-linux-vulkan.zip + - name: Stop xvfb and openbox + # should always stop xvfb and openbox even if other steps failed + if: always() + run: | + kill $XVFBPID + kill $OPENBOXPID - name: Artifact uses: actions/upload-artifact@v3 with: @@ -50,8 +134,21 @@ jobs: with: name: love-x86_64-AppImage-debug path: love-${{ github.sha }}.AppImage-debug.tar.gz + - name: Check Tests Passing + if: steps.report1.outputs.conclusion == 'failure' || steps.report2.outputs.conclusion == 'failure' + run: | + echo "${{ steps.report1.outputs.failed }} opengl tests failed" + echo "${{ steps.report2.outputs.failed }} opengles tests failed" + exit 1 windows-os: runs-on: windows-latest + permissions: + checks: write + pull-requests: write + env: + ALSOFT_CONF: megasource/libs/love/testing/resources/alsoft.conf + VK_ICD_FILENAMES: ${{ github.workspace }}\mesa\x64\lvp_icd.x86_64.json + VULKAN_SDK: C:/VulkanSDK/1.3.231.1 strategy: matrix: platform: [Win32, x64, ARM64] @@ -206,8 +303,115 @@ jobs: with: name: love-windows-${{ steps.vars.outputs.arch }}${{ steps.vars.outputs.compatname }}-dbg path: pdb/Release/*.pdb + # install mesa for graphic tests + - name: Install Mesa + if: steps.vars.outputs.arch != 'ARM64' && steps.vars.outputs.compatname != '-compat' + run: | + curl -L --output mesa.7z --url https://github.com/pal1000/mesa-dist-win/releases/download/23.2.1/mesa3d-23.2.1-release-msvc.7z + 7z x mesa.7z -o* + powershell.exe mesa\systemwidedeploy.cmd 1 + # build love to use for the tests + - name: Build Test Exe + if: steps.vars.outputs.arch != 'ARM64' && steps.vars.outputs.compatname != '-compat' + run: cmake --build build --config Release --target install + # windows opengl tests + - name: Run Tests (opengl) + if: steps.vars.outputs.arch != 'ARM64' && steps.vars.outputs.compatname != '-compat' + run: | + echo 'check dir' + ls + powershell.exe ./install/lovec.exe ./megasource/libs/love/testing/main.lua --runAllTests --isRunner + - name: Love Test Report (opengl) + id: report1 + if: steps.vars.outputs.arch != 'ARM64' && steps.vars.outputs.compatname != '-compat' + uses: ellraiser/love-test-report@main + with: + name: Love Testsuite Windows ${{ steps.vars.outputs.arch }} ${{ steps.vars.outputs.compatname }} (opengl) + title: test-report-windows-${{ steps.vars.outputs.arch }}${{ steps.vars.outputs.compatname }}-opengl + path: megasource/libs/love/testing/output/lovetest_runAllTests.md + token: ${{ secrets.GITHUB_TOKEN }} + - name: Zip Test Output (opengl) + if: steps.vars.outputs.arch != 'ARM64' && steps.vars.outputs.compatname != '-compat' + run: | + 7z a -tzip test-output-windows-${{ steps.vars.outputs.arch }}${{ steps.vars.outputs.compatname }}-opengl.zip megasource/libs/love/testing/output/ + - name: Artifact Test Output (opengl) + if: steps.vars.outputs.arch != 'ARM64' && steps.vars.outputs.compatname != '-compat' + uses: actions/upload-artifact@v3 + with: + name: test-output-windows-${{ steps.vars.outputs.arch }}-opengl-${{ steps.report1.outputs.conclusion }} + path: test-output-windows-${{ steps.vars.outputs.arch }}-opengl.zip + # windows opengles tests + - name: Run Tests (opengles) + if: steps.vars.outputs.arch != 'ARM64' && steps.vars.outputs.compatname != '-compat' + run: | + $ENV:LOVE_GRAPHICS_USE_OPENGLES=1 + powershell.exe ./install/lovec.exe ./megasource/libs/love/testing/main.lua --runAllTests --isRunner + - name: Love Test Report (opengles) + id: report2 + if: steps.vars.outputs.arch != 'ARM64' && steps.vars.outputs.compatname != '-compat' + uses: ellraiser/love-test-report@main + with: + name: Love Testsuite Windows ${{ steps.vars.outputs.arch }} ${{ steps.vars.outputs.compatname }} (opengles) + title: test-report-windows-${{ steps.vars.outputs.arch }}${{ steps.vars.outputs.compatname }}-opengles + path: megasource/libs/love/testing/output/lovetest_runAllTests.md + token: ${{ secrets.GITHUB_TOKEN }} + - name: Zip Test Output (opengles) + if: steps.vars.outputs.arch != 'ARM64' && steps.vars.outputs.compatname != '-compat' + run: | + 7z a -tzip test-output-windows-${{ steps.vars.outputs.arch }}${{ steps.vars.outputs.compatname }}-opengles.zip megasource/libs/love/testing/output/ + - name: Artifact Test Output (opengles) + if: steps.vars.outputs.arch != 'ARM64' && steps.vars.outputs.compatname != '-compat' + uses: actions/upload-artifact@v3 + with: + name: test-output-windows-${{ steps.vars.outputs.arch }}-opengles-${{ steps.report2.outputs.conclusion }} + path: test-output-windows-${{ steps.vars.outputs.arch }}-opengles.zip + - name: Check Tests Passing + if: steps.report1.outputs.conclusion == 'failure' || steps.report2.outputs.conclusion == 'failure' + run: | + echo "${{ steps.report1.outputs.failed }} opengl tests failed" + echo "${{ steps.report2.outputs.failed }} opengles tests failed" + exit 1 +# # install vulkan +# - name: Install Vulkan +# if: steps.vars.outputs.arch != 'ARM64' +# run: | +# curl -L --show-error --output VulkanSDK.exe https://sdk.lunarg.com/sdk/download/1.3.231.1/windows/VulkanSDK-1.3.231.1-Installer.exe +# ./VulkanSDK.exe --root C:/VulkanSDK/1.3.231.1 --accept-licenses --default-answer --confirm-command install com.lunarg.vulkan.core com.lunarg.vulkan.vma +# curl -L --show-error --output vulkan-runtime.zip https://sdk.lunarg.com/sdk/download/1.3.231.1/windows/vulkan-runtime-components.zip +# 7z e vulkan-runtime.zip -o"C:/VulkanSDK/1.3.231.1/runtime/x64" */x64 +# copy "C:/VulkanSDK/1.3.231.1/runtime/x64/vulkan-1.dll" "mesa/x64" +# copy "C:/VulkanSDK/1.3.231.1/runtime/x64/vulkan-1.dll" "C:/Windows/System32" +# copy "C:/VulkanSDK/1.3.231.1/runtime/x64/vulkan-1.dll" "love-12.0-win64/love-12.0-win64" +# reg add HKEY_LOCAL_MACHINE\SOFTWARE\Khronos\Vulkan\Drivers /v "${{ github.workspace }}\mesa\x64\lvp_icd.x86_64.json" /t REG_DWORD /d 0 +# powershell.exe C:/VulkanSDK/1.3.231.1/runtime/x64/vulkaninfo.exe --summary +# # windows vulkan tests +# - name: Run Tests (vulkan) +# if: steps.vars.outputs.arch != 'ARM64' +# run: | +# $ENV:LOVE_GRAPHICS_DEBUG=1 +# powershell.exe ./install/lovec.exe ./megasource/libs/love/testing/main.lua --runAllTests --isRunner --renderers vulkan +# - name: Love Test Report (vulkan) +# if: steps.vars.outputs.arch != 'ARM64' +# uses: ellraiser/love-test-report@main +# with: +# name: Love Testsuite Windows ${{ steps.vars.outputs.arch }} ${{ steps.vars.outputs.compatname }} (vulkan) +# title: test-report-windows-${{ steps.vars.outputs.arch }}${{ steps.vars.outputs.compatname }}-vulkan +# path: megasource/libs/love/testing/output/lovetest_runAllTests.md +# - name: Zip Test Output (vulkan) +# if: steps.vars.outputs.arch != 'ARM64' +# run: | +# 7z a -tzip test-output-windows-${{ steps.vars.outputs.arch }}${{ steps.vars.outputs.compatname }}-vulkan.zip megasource/libs/love/testing/output/ +# - name: Artifact Test Output (vulkan) +# if: steps.vars.outputs.arch != 'ARM64' +# uses: actions/upload-artifact@v3 +# with: +# name: test-output-windows-${{ steps.vars.outputs.arch }}${{ steps.vars.outputs.compatname }}-vulkan +# path: test-output-windows-${{ steps.vars.outputs.arch }}${{ steps.vars.outputs.compatname }}-vulkan.zip macOS: runs-on: macos-latest + permissions: + checks: write + pull-requests: write steps: - name: Checkout uses: actions/checkout@v3 @@ -234,6 +438,32 @@ jobs: with: name: love-macos path: love-macos.zip + # macos opengl tests (metal not supported on runners) + - name: Run Test Suite + run: | + ls + love-macos/love.app/Contents/MacOS/love ./testing/main.lua --runAllTests --isRunner + - name: Love Test Report + id: report1 + uses: ellraiser/love-test-report@main + with: + name: Love Testsuite MacOS + title: test-report-macos + path: testing/output/lovetest_runAllTests.md + token: ${{ secrets.GITHUB_TOKEN }} + - name: Zip Test Output + run: | + 7z a -tzip test-output-macos-opengl.zip ./testing/output/ + - name: Artifact Test Output + uses: actions/upload-artifact@v3 + with: + name: test-output-macos-opengl-${{ steps.report1.outputs.conclusion }} + path: test-output-macos-opengl.zip + - name: Check Tests Passing + if: steps.report1.outputs.conclusion == 'failure' + run: | + echo "${{ steps.report1.outputs.failed }} opengl tests failed" + exit 1 iOS-Simulator: runs-on: macos-latest steps: diff --git a/.gitignore b/.gitignore index a8b316a2d..3f5288d45 100644 --- a/.gitignore +++ b/.gitignore @@ -68,4 +68,8 @@ stamp-h1 /src/love /src/tags .vs/ -.vscode/ \ No newline at end of file +.vscode/ +/testing/output/*.xml +/testing/output/*.html +/testing/output/*.md +/testing/output/actual/*.png diff --git a/testing/classes/TestMethod.lua b/testing/classes/TestMethod.lua new file mode 100644 index 000000000..816c6bd74 --- /dev/null +++ b/testing/classes/TestMethod.lua @@ -0,0 +1,540 @@ +-- @class - TestMethod +-- @desc - used to run a specific method from a module's /test/ suite +-- each assertion is tracked and then printed to output +TestMethod = { + + + -- @method - TestMethod:new() + -- @desc - create a new TestMethod object + -- @param {string} method - string of method name to run + -- @param {TestMethod} testmethod - parent testmethod this test belongs to + -- @return {table} - returns the new Test object + new = function(self, method, testmodule) + local test = { + testmodule = testmodule, + method = method, + asserts = {}, + start = love.timer.getTime(), + finish = 0, + count = 0, + passed = false, + skipped = false, + skipreason = '', + rgba_tolerance = 0, + pixel_tolerance = 0, + fatal = '', + message = nil, + result = {}, + colors = { + red = {1, 0, 0, 1}, + redpale = {1, 0.5, 0.5, 1}, + red07 = {0.7, 0, 0, 1}, + green = {0, 1, 0, 1}, + greenhalf = {0, 0.5, 0, 1}, + greenfade = {0, 1, 0, 0.5}, + blue = {0, 0, 1, 1}, + bluefade = {0, 0, 1, 0.5}, + yellow = {1, 1, 0, 1}, + pink = {1, 0, 1, 1}, + black = {0, 0, 0, 1}, + white = {1, 1, 1, 1}, + lovepink = {214/255, 86/255, 151/255, 1}, + loveblue = {83/255, 168/255, 220/255, 1} + }, + imgs = 1, + delay = 0, + delayed = false, + store = {}, + co = nil + } + setmetatable(test, self) + self.__index = self + return test + end, + + + -- @method - TestMethod:assertEquals() + -- @desc - used to assert two values are equals + -- @param {any} expected - expected value of the test + -- @param {any} actual - actual value of the test + -- @param {string} label - label for this test to use in exports + -- @return {nil} + assertEquals = function(self, expected, actual, label) + self.count = self.count + 1 + table.insert(self.asserts, { + key = 'assert ' .. tostring(self.count), + passed = expected == actual, + message = 'expected \'' .. tostring(expected) .. '\' got \'' .. + tostring(actual) .. '\'', + test = label or 'no label given' + }) + end, + + + -- @method - TestMethod:assertTrue() + -- @desc - used to assert a value is true + -- @param {any} value - value to test + -- @param {string} label - label for this test to use in exports + -- @return {nil} + assertTrue = function(self, value, label) + self.count = self.count + 1 + table.insert(self.asserts, { + key = 'assert ' .. tostring(self.count), + passed = value == true, + message = 'expected \'true\' got \'' .. + tostring(value) .. '\'', + test = label or 'no label given' + }) + end, + + + -- @method - TestMethod:assertFalse() + -- @desc - used to assert a value is false + -- @param {any} value - value to test + -- @param {string} label - label for this test to use in exports + -- @return {nil} + assertFalse = function(self, value, label) + self.count = self.count + 1 + table.insert(self.asserts, { + key = 'assert ' .. tostring(self.count), + passed = value == false, + message = 'expected \'false\' got \'' .. + tostring(value) .. '\'', + test = label or 'no label given' + }) + end, + + + -- @method - TestMethod:assertNotEquals() + -- @desc - used to assert two values are not equal + -- @param {any} expected - expected value of the test + -- @param {any} actual - actual value of the test + -- @param {string} label - label for this test to use in exports + -- @return {nil} + assertNotEquals = function(self, expected, actual, label) + self.count = self.count + 1 + table.insert(self.asserts, { + key = 'assert ' .. tostring(self.count), + passed = expected ~= actual, + message = 'avoiding \'' .. tostring(expected) .. '\' got \'' .. + tostring(actual) .. '\'', + test = label or 'no label given' + }) + end, + + + -- @method - TestMethod:assertPixels() + -- @desc - checks a list of coloured pixels agaisnt given imgdata + -- @param {ImageData} imgdata - image data to check + -- @param {table} pixels - map of colors to list of pixel coords, i.e. + -- { blue = { {1, 1}, {2, 2}, {3, 4} } } + -- @return {nil} + assertPixels = function(self, imgdata, pixels, label) + for i, v in pairs(pixels) do + local col = self.colors[i] + local pixels = v + for p=1,#pixels do + local coord = pixels[p] + local tr, tg, tb, ta = imgdata:getPixel(coord[1], coord[2]) + local compare_id = tostring(coord[1]) .. ',' .. tostring(coord[2]) + -- prevent us getting stuff like 0.501960785 for 0.5 red + tr = math.floor((tr*10)+0.5)/10 + tg = math.floor((tg*10)+0.5)/10 + tb = math.floor((tb*10)+0.5)/10 + ta = math.floor((ta*10)+0.5)/10 + col[1] = math.floor((col[1]*10)+0.5)/10 + col[2] = math.floor((col[2]*10)+0.5)/10 + col[3] = math.floor((col[3]*10)+0.5)/10 + col[4] = math.floor((col[4]*10)+0.5)/10 + self:assertEquals(col[1], tr, 'check pixel r for ' .. i .. ' at ' .. compare_id .. '(' .. label .. ')') + self:assertEquals(col[2], tg, 'check pixel g for ' .. i .. ' at ' .. compare_id .. '(' .. label .. ')') + self:assertEquals(col[3], tb, 'check pixel b for ' .. i .. ' at ' .. compare_id .. '(' .. label .. ')') + self:assertEquals(col[4], ta, 'check pixel a for ' .. i .. ' at ' .. compare_id .. '(' .. label .. ')') + end + end + end, + + + -- @method - TestMethod:assertRange() + -- @desc - used to check a value is within an expected range + -- @param {number} actual - actual value of the test + -- @param {number} min - minimum value the actual should be >= to + -- @param {number} max - maximum value the actual should be <= to + -- @param {string} label - label for this test to use in exports + -- @return {nil} + assertRange = function(self, actual, min, max, label) + self.count = self.count + 1 + table.insert(self.asserts, { + key = 'assert ' .. tostring(self.count), + passed = actual >= min and actual <= max, + message = 'value \'' .. tostring(actual) .. '\' out of range \'' .. + tostring(min) .. '-' .. tostring(max) .. '\'', + test = label or 'no label given' + }) + end, + + + -- @method - TestMethod:assertMatch() + -- @desc - used to check a value is within a list of values + -- @param {number} list - list of valid values for the test + -- @param {number} actual - actual value of the test to check is in the list + -- @param {string} label - label for this test to use in exports + -- @return {nil} + assertMatch = function(self, list, actual, label) + self.count = self.count + 1 + local found = false + for l=1,#list do + if list[l] == actual then found = true end; + end + table.insert(self.asserts, { + key = 'assert ' .. tostring(self.count), + passed = found == true, + message = 'value \'' .. tostring(actual) .. '\' not found in \'' .. + table.concat(list, ',') .. '\'', + test = label or 'no label given' + }) + end, + + + -- @method - TestMethod:assertGreaterEqual() + -- @desc - used to check a value is >= than a certain target value + -- @param {any} target - value to check the test agaisnt + -- @param {any} actual - actual value of the test + -- @param {string} label - label for this test to use in exports + -- @return {nil} + assertGreaterEqual = function(self, target, actual, label) + self.count = self.count + 1 + local passing = false + if target ~= nil and actual ~= nil then + passing = actual >= target + end + table.insert(self.asserts, { + key = 'assert ' .. tostring(self.count), + passed = passing, + message = 'value \'' .. tostring(actual) .. '\' not >= \'' .. + tostring(target) .. '\'', + test = label or 'no label given' + }) + end, + + + -- @method - TestMethod:assertLessEqual() + -- @desc - used to check a value is <= than a certain target value + -- @param {any} target - value to check the test agaisnt + -- @param {any} actual - actual value of the test + -- @param {string} label - label for this test to use in exports + -- @return {nil} + assertLessEqual = function(self, target, actual, label) + self.count = self.count + 1 + local passing = false + if target ~= nil and actual ~= nil then + passing = actual <= target + end + table.insert(self.asserts, { + key = 'assert ' .. tostring(self.count), + passed = passing, + message = 'value \'' .. tostring(actual) .. '\' not <= \'' .. + tostring(target) .. '\'', + test = label or 'no label given' + }) + end, + + + -- @method - TestMethod:assertObject() + -- @desc - used to check a table is a love object, this runs 3 seperate + -- tests to check table has the basic properties of an object + -- @note - actual object functionality tests have their own methods + -- @param {table} obj - table to check is a valid love object + -- @return {nil} + assertObject = function(self, obj) + self:assertNotNil(obj) + self:assertEquals('userdata', type(obj), 'check is userdata') + if obj ~= nil then + self:assertNotEquals(nil, obj:type(), 'check has :type()') + end + end, + + + -- @method - TestMethod:assertCoords() + -- @desc - used to check a pair of values (usually coordinates) + -- @param {table} obj - table to check is a valid love object + -- @return {nil} + assertCoords = function(self, expected, actual, label) + self.count = self.count + 1 + local passing = false + if expected ~= nil and actual ~= nil then + if expected[1] == actual[1] and expected[2] == actual[2] then + passing = true + end + end + table.insert(self.asserts, { + key = 'assert ' .. tostring(self.count), + passed = passing, + message = 'expected \'' .. tostring(expected[1]) .. 'x,' .. + tostring(expected[2]) .. 'y\' got \'' .. + tostring(actual[1]) .. 'x,' .. tostring(actual[2]) .. 'y\'', + test = label or 'no label given' + }) + end, + + + -- @method - TestMethod:assertNotNil() + -- @desc - quick assert for value not nil + -- @param {any} value - value to check not nil + -- @return {nil} + assertNotNil = function (self, value, err) + self:assertNotEquals(nil, value, 'check not nil') + if err ~= nil then + table.insert(self.asserts, { + key = 'assert ' .. tostring(self.count), + passed = false, + message = err, + test = 'assert not nil catch' + }) + end + end, + + + -- @method - TestMethod:compareImg() + -- @desc - compares a given image to the 'expected' version, with a tolerance of + -- 1px in any direction, and then saves it as the 'actual' version for + -- report viewing + -- @param {table} imgdata - imgdata to save as a png + -- @return {nil} + compareImg = function(self, imgdata) + local expected = love.image.newImageData( + 'tempoutput/expected/love.test.graphics.' .. self.method .. '-' .. + tostring(self.imgs) .. '.png' + ) + local iw = imgdata:getWidth()-2 + local ih = imgdata:getHeight()-2 + local rgba_tolerance = self.rgba_tolerance * (1/255) + for ix=2,iw do + for iy=2,ih do + local ir, ig, ib, ia = imgdata:getPixel(ix, iy) + local points = { + {expected:getPixel(ix, iy)} + } + if self.pixel_tolerance > 0 then + table.insert(points, {expected:getPixel(ix-1, iy+1)}) + table.insert(points, {expected:getPixel(ix-1, iy)}) + table.insert(points, {expected:getPixel(ix-1, iy-1)}) + table.insert(points, {expected:getPixel(ix, iy+1)}) + table.insert(points, {expected:getPixel(ix, iy-1)}) + table.insert(points, {expected:getPixel(ix+1, iy+1)}) + table.insert(points, {expected:getPixel(ix+1, iy)}) + table.insert(points, {expected:getPixel(ix+1, iy-1)}) + end + local has_match_r = false + local has_match_g = false + local has_match_b = false + local has_match_a = false + for t=1,#points do + local epoint = points[t] + if ir >= epoint[1] - rgba_tolerance and ir <= epoint[1] + rgba_tolerance then has_match_r = true; end + if ig >= epoint[2] - rgba_tolerance and ig <= epoint[2] + rgba_tolerance then has_match_g = true; end + if ib >= epoint[3] - rgba_tolerance and ib <= epoint[3] + rgba_tolerance then has_match_b = true; end + if ia >= epoint[4] - rgba_tolerance and ia <= epoint[4] + rgba_tolerance then has_match_a = true; end + end + local matching = has_match_r and has_match_g and has_match_b and has_match_a + local ymatch = '' + local nmatch = '' + if has_match_r then ymatch = ymatch .. 'r' else nmatch = nmatch .. 'r' end + if has_match_g then ymatch = ymatch .. 'g' else nmatch = nmatch .. 'g' end + if has_match_b then ymatch = ymatch .. 'b' else nmatch = nmatch .. 'b' end + if has_match_a then ymatch = ymatch .. 'a' else nmatch = nmatch .. 'a' end + local pixel = tostring(ir)..','..tostring(ig)..','..tostring(ib)..','..tostring(ia) + self:assertEquals(true, matching, 'compare image pixel (' .. pixel .. ') at ' .. + tostring(ix) .. ',' .. tostring(iy) .. ', matching = ' .. ymatch .. + ', not matching = ' .. nmatch .. ' (' .. self.method .. '-' .. tostring(self.imgs) .. ')' + ) + end + end + local path = 'tempoutput/actual/love.test.graphics.' .. + self.method .. '-' .. tostring(self.imgs) .. '.png' + imgdata:encode('png', path) + self.imgs = self.imgs + 1 + end, + + + -- @method - TestMethod:skipTest() + -- @desc - used to mark this test as skipped for a specific reason + -- @param {string} reason - reason why method is being skipped + -- @return {nil} + skipTest = function(self, reason) + self.skipped = true + self.skipreason = reason + end, + + + waitFrames = function(self, frames) + for i=1,frames do coroutine.yield() end + end, + + + -- @method - TestMethod:evaluateTest() + -- @desc - evaluates the results of all assertions for a final restult + -- @return {nil} + evaluateTest = function(self) + local failure = '' + local failures = 0 + for a=1,#self.asserts do + -- @TODO just return first failed assertion msg? or all? + -- currently just shows the first assert that failed + if self.asserts[a].passed == false and self.skipped == false then + if failure == '' then failure = self.asserts[a] end + failures = failures + 1 + end + end + if self.fatal ~= '' then failure = self.fatal end + local passed = tostring(#self.asserts - failures) + local total = '(' .. passed .. '/' .. tostring(#self.asserts) .. ')' + if self.skipped == true then + self.testmodule.skipped = self.testmodule.skipped + 1 + love.test.totals[3] = love.test.totals[3] + 1 + self.result = { + total = '', + result = "SKIP", + passed = false, + message = '(0/0) - method skipped [' .. self.skipreason .. ']' + } + else + if failure == '' and #self.asserts > 0 then + self.passed = true + self.testmodule.passed = self.testmodule.passed + 1 + love.test.totals[1] = love.test.totals[1] + 1 + self.result = { + total = total, + result = 'PASS', + passed = true, + message = nil + } + else + self.passed = false + self.testmodule.failed = self.testmodule.failed + 1 + love.test.totals[2] = love.test.totals[2] + 1 + if #self.asserts == 0 then + local msg = 'no asserts defined' + if self.fatal ~= '' then msg = self.fatal end + self.result = { + total = total, + result = 'FAIL', + passed = false, + key = 'test', + message = msg + } + else + local key = failure['key'] + if failure['test'] ~= nil then + key = key .. ' [' .. failure['test'] .. ']' + end + local msg = failure['message'] + if self.fatal ~= '' then + key = 'code' + msg = self.fatal + end + self.result = { + total = total, + result = 'FAIL', + passed = false, + key = key, + message = msg + } + end + end + end + self:printResult() + end, + + + -- @method - TestMethod:printResult() + -- @desc - prints the result of the test to the console as well as appends + -- the XML + HTML for the test to the testsuite output + -- @return {nil} + printResult = function(self) + + -- get total timestamp + -- @TODO make nicer, just need a 3DP ms value + self.finish = love.timer.getTime() - self.start + love.test.time = love.test.time + self.finish + self.testmodule.time = self.testmodule.time + self.finish + local endtime = UtilTimeFormat(love.timer.getTime() - self.start) + + -- get failure/skip message for output (if any) + local failure = '' + local output = '' + if self.passed == false and self.skipped == false then + failure = '\t\t\t' .. self.result.key .. ' ' .. self.result.message .. '\n' + output = self.result.key .. ' ' .. self.result.message + -- append failures if any to report md + love.test.mdfailures = love.test.mdfailures .. '> 🔴 ' .. self.method .. ' \n' .. + '> ' .. output .. ' \n\n' + end + if output == '' and self.skipped == true then + failure = '\t\t\t\n' + output = self.skipreason + end + + + -- append XML for the test class result + self.testmodule.xml = self.testmodule.xml .. '\t\t\n' .. + failure .. '\t\t\n' + + -- unused currently, adds a preview image for certain graphics methods to the output + local preview = '' + if self.testmodule.module == 'graphics' then + local filename = 'love.test.graphics.' .. self.method + if love.filesystem.openFile('tempoutput/actual/' .. filename .. '-1.png', 'r') then + preview = '
' .. '

Expected

' .. + '
' .. '

Actual

' + end + if love.filesystem.openFile('tempoutput/actual/' .. filename .. '-2.png', 'r') then + preview = preview .. '
' .. '

Expected

' .. + '
' .. '

Actual

' + end + if love.filesystem.openFile('tempoutput/actual/' .. filename .. '-3.png', 'r') then + preview = preview .. '
' .. '

Expected

' .. + '
' .. '

Actual

' + end + end + + -- append HTML for the test class result + local status = '🔴' + local cls = 'red' + if self.passed == true then status = '🟢'; cls = '' end + if self.skipped == true then status = '🟡'; cls = '' end + self.testmodule.html = self.testmodule.html .. + '' .. + '' .. status .. '' .. + '' .. self.method .. '' .. + '' .. endtime .. 's' .. + '' .. output .. preview .. '' .. + '' + + -- add message if assert failed + local msg = '' + if self.result.message ~= nil and self.skipped == false then + msg = ' - ' .. self.result.key .. + ' failed - (' .. self.result.message .. ')' + end + if self.skipped == true then + msg = self.result.message + end + + -- log final test result to console + -- i know its hacky but its neat soz + local tested = 'love.' .. self.testmodule.module .. '.' .. self.method .. '()' + local matching = string.sub(self.testmodule.spacer, string.len(tested), 40) + self.testmodule:log( + self.testmodule.colors[self.result.result], + ' ' .. tested .. matching, + ' ==> ' .. self.result.result .. ' - ' .. endtime .. 's ' .. + self.result.total .. msg + ) + end + + +} diff --git a/testing/classes/TestModule.lua b/testing/classes/TestModule.lua new file mode 100644 index 000000000..f66d247ed --- /dev/null +++ b/testing/classes/TestModule.lua @@ -0,0 +1,120 @@ +-- @class - TestModule +-- @desc - used to run tests for a given module, each test method will spawn +-- a love.test.Test object +TestModule = { + + + -- @method - TestModule:new() + -- @desc - create a new Suite object + -- @param {string} module - string of love module the suite is for + -- @return {table} - returns the new Suite object + new = function(self, module, method) + local testmodule = { + time = 0, + spacer = ' ', + colors = { + PASS = 'green', FAIL = 'red', SKIP = 'grey' + }, + colormap = { + grey = '\27[37m', + green = '\27[32m', + red = '\27[31m', + yellow = '\27[33m' + }, + xml = '', + html = '', + tests = {}, + running = {}, + called = {}, + passed = 0, + failed = 0, + skipped = 0, + module = module, + method = method, + index = 1, + start = false, + } + setmetatable(testmodule, self) + self.__index = self + return testmodule + end, + + + -- @method - TestModule:log() + -- @desc - log to console with specific colors, split out to make it easier + -- to adjust all console output across the tests + -- @param {string} color - color key to use for the log + -- @param {string} line - main message to write (LHS) + -- @param {string} result - result message to write (RHS) + -- @return {nil} + log = function(self, color, line, result) + if result == nil then result = '' end + print(self.colormap[color] .. line .. result) + end, + + + -- @method - TestModule:runTests() + -- @desc - starts the running of tests and sets up the list of methods to test + -- @param {string} module - module to set for the test suite + -- @param {string} method - specific method to test, if nil all methods tested + -- @return {nil} + runTests = function(self) + self.running = {} + self.passed = 0 + self.failed = 0 + if self.method ~= nil then + table.insert(self.running, self.method) + else + for i,_ in pairs(love.test[self.module]) do + table.insert(self.running, i) + end + table.sort(self.running) + end + self.index = 1 + self.start = true + self:log('yellow', '\nlove.' .. self.module .. '.testmodule.start') + end, + + + -- @method - TestModule:printResult() + -- @desc - prints the result of the module to the console as well as appends + -- the XML + HTML for the test to the testsuite output + -- @return {nil} + printResult = function(self) + local finaltime = UtilTimeFormat(self.time) + local status = '🔴' + if self.failed == 0 then status = '🟢' end + -- add md row to main output + love.test.mdrows = love.test.mdrows .. '| ' .. status .. + ' ' .. self.module .. + ' | ' .. tostring(self.passed) .. + ' | ' .. tostring(self.failed) .. + ' | ' .. tostring(self.skipped) .. + ' | ' .. finaltime .. 's |' .. '\n' + -- add xml to main output + love.test.xml = love.test.xml .. '\t\n' .. self.xml .. '\t\n' + -- add html to main output + love.test.html = love.test.html .. '

' .. status .. ' love.' .. self.module .. '