Taiko-Accessibility plugin issue

[harimadusumilli]

Taiko-Accessibility plugin appears to have discrepancies in the results that it provides when compared to selenium-accessibility using Axe-Core library by Deque labs.

I have posted this issue here as it appears that the spectrum forum for gauge recommends this place to post the questions.

This is in continuation with the issue that is also posted in spectrum forum. Any help and advise is appreciated. I am unable to upload the whole project to the git-hub as there is a security constraint in the place I work. But the complete code and the plugins used are below and I hope you are able to reproduce and see the issue.

Here is what we did:
Installed the following software:
Gauge version: 1.0.7
Commit Hash: ed7b4fd6

### **Plugins**
-------
dotnet (0.1.3)
html-report (4.0.9)
js (2.3.7)
json-report (0.3.3)
screenshot (0.0.1)
xml-report (0.2.3)

**Taiko - Version: 1.0.28 (Chromium: 88.0.4299.0) RELEASE**

Installed taiko-accessibility plugin

      **Installed the axe-core plugin version - version 4.0.2**

Added the following piece of code in index.js present in the taiko-accessibility\node_modules\axe-core. The code that has been modified in order to get results from runAudit method of taiko-accessibility plugin is as follows:

const runAudit = async () => {
    try{
    await injectAxe();
    const response = await _runtime.evaluate({expression: '{testResult: axe.run({resultTypes: ["violations"]})}', awaitPromise: true, returnByValue: true});
  //console.log(response);
    const result = response.result.value;
    
    return {
        score: calculateScore(result.passes, result.violations),
        violations: result.violations, incomplete: result.incomplete, passes: result.passes, inapplicable: result.inapplicable, url: result.url, timestamp: result.timestamp, testEngine: result.testEngine, testEnvironment: result.testEnvironment
    };
}
catch(error){
    console.log("ErrorAudit" + error);
}
};

Now, after adding this piece of code we created the following spec file and its implementation as follows:

Spec file :

# Testing Axe Taiko plugin

## Scenario of Taiko - Axe Plugin

* Launch Application url: "http://puppies.herokuapp.com/"

Implementation file Code (js) file is as given below:

"use strict";
var assert = require("assert");
const fs = require('fs');
var axe = require('axe-core');
let _runtime;

const {accessibility, openBrowser, goto, currentURL,write, click, link, into, textBox, button,closeBrowser, waitFor} = require('taiko');

step("Launch Application url: ", async function(url) {
    await openBrowser({headless:false,args:['--start-maximized']});
    await goto(url);
    waitFor(20000);
    await accessibilityReport();
      gauge.screenshot();
    assert.ok(await currentURL(),url);
});
const accessibilityReport = async() => {
    const audit = await accessibility.runAudit();
    console.log("Accessibility Check");
    console.log("Context:");
    console.log("Url : " + audit.url);
    console.log("Orientation : " + audit.testEnvironment.orientationType);
    console.log("Size : " + audit.testEnvironment.windowWidth + " X " + audit.testEnvironment.windowHeight);
    console.log("Time : " + audit.timestamp);
    console.log("User agent : " + audit.testEnvironment.userAgent);
    console.log("Using : " + audit.testEngine.name + " " + audit.testEngine.version);
    console.log("Counts:");
    console.log("score : " + audit.score);
    console.log("violations : " + audit.violations.length);
    console.log("incomplete : " + audit.incomplete.length);
    console.log("passes : " + audit.passes.length);
    console.log("inapplicable : " + audit.inapplicable.length);
    /console.log("Violations : ");
    for(let val of audit.violations) {
        console.log("Description : " + val.description);
        console.log("Help : " + val.help);
        console.log("HelpUrl : " + val.helpUrl);
        console.log("Impact : " + val.impact);
        console.log("Tags : " + val.tags);
        console.log("Nodes : " + val.nodes.length);
        for(let a of val.nodes) {
            console.log("Html : " + a.html);
            console.log("Selector(s): " + a.target[0]);
        }
    }
    console.log("incomplete : ");
    for(let val of audit.incomplete) {
        console.log("Description : " + val.description);
        console.log("Help : " + val.help);
        console.log("HelpUrl : " + val.helpUrl);
        console.log("Impact : " + val.impact);
        console.log("Tags : " + val.tags);
        console.log("Nodes : " + val.nodes.length);
        for(let a of val.nodes) {
            console.log("Html : " + a.html);
            console.log("Selector(s): " + a.target[0]);
        }
    }
    console.log("passes : ");
    for(let val of audit.passes) {
        console.log("Description : " + val.description);
        console.log("Help : " + val.help);
        console.log("HelpUrl : " + val.helpUrl);
        console.log("Tags : " + val.tags);
        console.log("Nodes : " + val.nodes.length);
        for(let a of val.nodes) {
            console.log("Html : " + a.html);
            console.log("Selector(s): " + a.target[0]);
        }
    }
    console.log("inapplicable : ");
    for(let val of audit.inapplicable) {
        console.log("Description : " + val.description);
        console.log("Help : " + val.help);
        console.log("HelpUrl : " + val.helpUrl);
        console.log("Tags : " + val.tags);
    }/
};

Output of the run is as given below:
Accessibility Check
Context:
Url : http://puppies.herokuapp.com/
Orientation : landscape-primary
Size : 1920 X 877
Time : 2020-12-18T16:33:18.341Z
User agent : Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Edg/87.0.664.60
Using : axe-core 4.0.2
Counts:
score : 75
violations : 6
incomplete : 1
passes : 18
inapplicable : 53

When the same test is run using Selenium axe plugin we get the following results:
Environment: Gauge + Selenium + Axe-Core + Dotnet Core Context: Url: http://puppies.herokuapp.com/ Orientation: landscape-primary Size: 1280 x 565 Time: 12/7/2020 2:37:11 PM +00:00 User agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Using: axe-core (4.0.2)
Counts: Violation: 11
Incomplete: 14
Pass: 191
Inapplicable: 53

Notice that except for the inapplicable all the other metrics such as Violations, Incomplete and passes do not match there is a huge discrepancy.

[sriv]

/cc @saikrishna321

[harimadusumilli]

Circling back to see if anyone had a chance to look this issue that I have raised in the discussion group. I am assuming that most of the folks are on vacation at this time of the year. I will try and request once again after the Dec 31st. Thanks in advance.