Skip to content

Commit

Permalink
nunit#14 Erroneous TeamCity service messages with multiple test asse…
Browse files Browse the repository at this point in the history 
…mblies and (default) parallel execution - use test's id as a base for flowId for TC service messages in the case when NUnit 3 console runs test for nunit.framework.dll v2 in parallel mode
  • Loading branch information
Nikolay Pianikov authored and Nikolay Pianikov committed Sep 8, 2016
1 parent 7cd28d0 commit 39f7e18
Show file tree
Hide file tree
Showing 7 changed files with 748 additions and 156 deletions.
54 changes: 38 additions & 16 deletions src/extension/TeamCityEventListener.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,13 @@ public class TeamCityEventListener : ITestEventListener
private static readonly ServiceMessageWriter ServiceMessageWriter = new ServiceMessageWriter();
private readonly TextWriter _outWriter;
private readonly Dictionary<string, string> _refs = new Dictionary<string, string>();
private int _blockCounter;
private string _rootFlowId;
private readonly Dictionary<string, int> _blockCounters = new Dictionary<string, int>();

public TeamCityEventListener() : this(Console.Out) { }

public TeamCityEventListener(TextWriter outWriter)
{
if (outWriter == null) throw new ArgumentNullException("outWriter");
if (outWriter == null) throw new ArgumentNullException("outWriter");

_outWriter = outWriter;
}
Expand Down Expand Up @@ -93,8 +92,13 @@ public void RegisterMessage(XmlNode testEvent)
}

var id = testEvent.GetAttribute("id");
if (id == null)
{
id = string.Empty;
}

var parentId = testEvent.GetAttribute("parentId");
string flowId;
var flowId = ".";
if (parentId != null)
{
// NUnit 3 case
Expand All @@ -104,7 +108,14 @@ public void RegisterMessage(XmlNode testEvent)
else
{
// NUnit 2 case
flowId = _rootFlowId;
if (!string.IsNullOrEmpty(id))
{
var idParts = id.Split('-');
if (idParts.Length == 2)
{
flowId = idParts[0];
}
}
}

string testFlowId;
Expand All @@ -125,12 +136,12 @@ public void RegisterMessage(XmlNode testEvent)
{
case "start-suite":
_refs[id] = parentId;
StartSuiteCase(id, parentId, flowId, fullName);
StartSuiteCase(parentId, flowId, fullName);
break;

case "test-suite":
_refs.Remove(id);
TestSuiteCase(id, parentId, flowId, fullName);
TestSuiteCase(parentId, flowId, fullName);
break;

case "start-test":
Expand Down Expand Up @@ -194,7 +205,7 @@ private void CaseStartTest(string id, string flowId, string parentId, string tes
OnTestStart(testFlowId, fullName);
}

private void TestSuiteCase(string id, string parentId, string flowId, string fullName)
private void TestSuiteCase(string parentId, string flowId, string fullName)
{
// NUnit 3 case
if (parentId == string.Empty)
Expand All @@ -205,15 +216,14 @@ private void TestSuiteCase(string id, string parentId, string flowId, string ful
// NUnit 2 case
if (parentId == null)
{
if (--_blockCounter == 0)
if (ChangeBlockCounter(flowId, -1) == 0)
{
_rootFlowId = null;
OnRootSuiteFinish(id, fullName);
OnRootSuiteFinish(flowId, fullName);
}
}
}

private void StartSuiteCase(string id, string parentId, string flowId, string fullName)
private void StartSuiteCase(string parentId, string flowId, string fullName)
{
// NUnit 3 case
if (parentId == string.Empty)
Expand All @@ -224,14 +234,26 @@ private void StartSuiteCase(string id, string parentId, string flowId, string fu
// NUnit 2 case
if (parentId == null)
{
if (_blockCounter++ == 0)
if (ChangeBlockCounter(flowId, 1) == 1)
{
_rootFlowId = id;
OnRootSuiteStart(id, fullName);
OnRootSuiteStart(flowId, fullName);
}
}
}

private int ChangeBlockCounter(string flowId, int changeValue)
{
int currentBlockCounter;
if (!_blockCounters.TryGetValue(flowId, out currentBlockCounter))
{
currentBlockCounter = 0;
}

currentBlockCounter += changeValue;
_blockCounters[flowId] = currentBlockCounter;
return currentBlockCounter;
}

private bool TryFindParentId(string id, out string parentId)
{
if (id == null)
Expand Down Expand Up @@ -278,7 +300,7 @@ private void TrySendOutput(string flowId, XmlNode message, string fullName)
new ServiceMessageAttr(ServiceMessageAttr.Names.Name, fullName),
new ServiceMessageAttr(ServiceMessageAttr.Names.Out, outputStr),
new ServiceMessageAttr(ServiceMessageAttr.Names.FlowId, flowId),
new ServiceMessageAttr(ServiceMessageAttr.Names.TcTags, "tc:parseServiceMessagesInside")));
new ServiceMessageAttr(ServiceMessageAttr.Names.TcTags, "tc:parseServiceMessagesInside")));
}

private void OnRootSuiteStart(string flowId, string assemblyName)
Expand Down
2 changes: 1 addition & 1 deletion src/extension/teamcity-event-listener.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>..\..\bin\nunit\NUnit.Extension.TeamCityEventListener\tools\</OutputPath>
<OutputPath>..\..\..\nunit-console\bin\Debug\addins\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
Expand Down
149 changes: 148 additions & 1 deletion src/nunit.integration.tests/RunTests.feature
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,153 @@
Background:
Given NUnit path is ..\nunit\

Scenario Outline: User runs tests for several assemblies
Given Framework version is <frameworkVersion>
And I have created the folder mocks
And I have copied NUnit framework references to folder mocks
And I have added successful method as SuccessfulTest to the class Foo.Tests.UnitTests1 for foo.tests
And I have added NUnit framework references to foo.tests

And I have compiled the assembly foo.tests to file mocks\foo1.tests.dll
And I have added the assembly mocks\foo1.tests.dll to the list of testing assemblies

And I have compiled the assembly foo.tests to file mocks\foo2.tests.dll
And I have added the assembly mocks\foo2.tests.dll to the list of testing assemblies

And I have compiled the assembly foo.tests to file mocks\foo3.tests.dll
And I have added the assembly mocks\foo3.tests.dll to the list of testing assemblies

And I have compiled the assembly foo.tests to file mocks\foo4.tests.dll
And I have added the assembly mocks\foo4.tests.dll to the list of testing assemblies

And I have compiled the assembly foo.tests to file mocks\foo5.tests.dll
And I have added the assembly mocks\foo5.tests.dll to the list of testing assemblies

And I have compiled the assembly foo.tests to file mocks\foo6.tests.dll
And I have added the assembly mocks\foo6.tests.dll to the list of testing assemblies

And I want to use <configurationType> configuration type
And I have added the arg workers=10 to NUnit console command line
And I have added the arg agents=<agents> to NUnit console command line
And I have added the arg process=<process> to NUnit console command line
And I have added the arg domain=<domain> to NUnit console command line

When I run NUnit console
Then the exit code should be 0
And the output should contain correct set of TeamCity service messages
And the Test Run Summary should has following:
| field | value |
| Test Count | 6 |
| Passed | 6 |
| Failed | 0 |
| Inconclusive | 0 |
| Skipped | 0 |

Examples:
| frameworkVersion | process | domain | agents | platform | configurationType |
| Version45 | InProcess | Single | 10 | AnyCpu | ProjectFile |
| Version40 | InProcess | Single | 10 | AnyCpu | ProjectFile |
| Version45 | Separate | Single | 10 | AnyCpu | ProjectFile |
| Version40 | Separate | Single | 10 | AnyCpu | ProjectFile |
| Version45 | Multiple | Single | 10 | AnyCpu | ProjectFile |
| Version40 | Multiple | Single | 10 | AnyCpu | ProjectFile |
| Version45 | InProcess | Multiple | 10 | AnyCpu | ProjectFile |
| Version40 | InProcess | Multiple | 10 | AnyCpu | ProjectFile |
| Version45 | Separate | Multiple | 10 | AnyCpu | ProjectFile |
| Version40 | Separate | Multiple | 10 | AnyCpu | ProjectFile |
| Version45 | InProcess | Single | 10 | AnyCpu | ProjectFile |
| Version40 | InProcess | Single | 2 | AnyCpu | ProjectFile |
| Version45 | Separate | Single | 2 | AnyCpu | ProjectFile |
| Version40 | Separate | Single | 2 | AnyCpu | ProjectFile |
| Version45 | Multiple | Single | 2 | AnyCpu | ProjectFile |
| Version40 | Multiple | Single | 2 | AnyCpu | ProjectFile |
| Version45 | InProcess | Multiple | 2 | AnyCpu | ProjectFile |
| Version40 | InProcess | Multiple | 2 | AnyCpu | ProjectFile |
| Version45 | Separate | Multiple | 2 | AnyCpu | ProjectFile |
| Version40 | Separate | Multiple | 2 | AnyCpu | ProjectFile |
| Version45 | InProcess | Single | 1 | AnyCpu | ProjectFile |
| Version40 | InProcess | Single | 1 | AnyCpu | ProjectFile |
| Version45 | Separate | Single | 1 | AnyCpu | ProjectFile |
| Version40 | Separate | Single | 1 | AnyCpu | ProjectFile |
| Version45 | Multiple | Single | 1 | AnyCpu | ProjectFile |
| Version40 | Multiple | Single | 1 | AnyCpu | ProjectFile |
| Version45 | InProcess | Multiple | 1 | AnyCpu | ProjectFile |
| Version40 | InProcess | Multiple | 1 | AnyCpu | ProjectFile |
| Version45 | Separate | Multiple | 1 | AnyCpu | ProjectFile |
| Version40 | Separate | Multiple | 1 | AnyCpu | ProjectFile |
| Version45 | Separate | Single | 10 | X86 | ProjectFile |
| Version40 | Separate | Single | 10 | X86 | ProjectFile |
| Version45 | Multiple | Single | 10 | X86 | ProjectFile |
| Version40 | Multiple | Single | 10 | X86 | ProjectFile |
| Version45 | Separate | Multiple | 10 | X86 | ProjectFile |
| Version40 | Separate | Multiple | 10 | X86 | ProjectFile |
| Version45 | Separate | Single | 1 | X86 | ProjectFile |
| Version40 | Separate | Single | 1 | X86 | ProjectFile |
| Version45 | Multiple | Single | 1 | X86 | ProjectFile |
| Version40 | Multiple | Single | 1 | X86 | ProjectFile |
| Version45 | Separate | Multiple | 1 | X86 | ProjectFile |
| Version40 | Separate | Multiple | 1 | X86 | ProjectFile |
| Version40 | InProcess | Single | 2 | X86 | ProjectFile |
| Version45 | Separate | Single | 2 | X86 | ProjectFile |
| Version40 | Separate | Single | 2 | X86 | ProjectFile |
| Version45 | Multiple | Single | 2 | X86 | ProjectFile |
| Version40 | Multiple | Single | 2 | X86 | ProjectFile |
| Version45 | InProcess | Multiple | 2 | X86 | ProjectFile |
| Version40 | InProcess | Multiple | 2 | X86 | ProjectFile |
| Version45 | Separate | Multiple | 2 | X86 | ProjectFile |
| Version40 | Separate | Multiple | 2 | X86 | ProjectFile |
| Version45 | InProcess | Single | 10 | AnyCpu | CmdArguments |
| Version40 | InProcess | Single | 10 | AnyCpu | CmdArguments |
| Version45 | Separate | Single | 10 | AnyCpu | CmdArguments |
| Version40 | Separate | Single | 10 | AnyCpu | CmdArguments |
| Version45 | Multiple | Single | 10 | AnyCpu | CmdArguments |
| Version40 | Multiple | Single | 10 | AnyCpu | CmdArguments |
| Version45 | InProcess | Multiple | 10 | AnyCpu | CmdArguments |
| Version40 | InProcess | Multiple | 10 | AnyCpu | CmdArguments |
| Version45 | Separate | Multiple | 10 | AnyCpu | CmdArguments |
| Version40 | Separate | Multiple | 10 | AnyCpu | CmdArguments |
| Version45 | InProcess | Single | 10 | AnyCpu | CmdArguments |
| Version40 | InProcess | Single | 2 | AnyCpu | CmdArguments |
| Version45 | Separate | Single | 2 | AnyCpu | CmdArguments |
| Version40 | Separate | Single | 2 | AnyCpu | CmdArguments |
| Version45 | Multiple | Single | 2 | AnyCpu | CmdArguments |
| Version40 | Multiple | Single | 2 | AnyCpu | CmdArguments |
| Version45 | InProcess | Multiple | 2 | AnyCpu | CmdArguments |
| Version40 | InProcess | Multiple | 2 | AnyCpu | CmdArguments |
| Version45 | Separate | Multiple | 2 | AnyCpu | CmdArguments |
| Version40 | Separate | Multiple | 2 | AnyCpu | CmdArguments |
| Version45 | InProcess | Single | 1 | AnyCpu | CmdArguments |
| Version40 | InProcess | Single | 1 | AnyCpu | CmdArguments |
| Version45 | Separate | Single | 1 | AnyCpu | CmdArguments |
| Version40 | Separate | Single | 1 | AnyCpu | CmdArguments |
| Version45 | Multiple | Single | 1 | AnyCpu | CmdArguments |
| Version40 | Multiple | Single | 1 | AnyCpu | CmdArguments |
| Version45 | InProcess | Multiple | 1 | AnyCpu | CmdArguments |
| Version40 | InProcess | Multiple | 1 | AnyCpu | CmdArguments |
| Version45 | Separate | Multiple | 1 | AnyCpu | CmdArguments |
| Version40 | Separate | Multiple | 1 | AnyCpu | CmdArguments |
| Version45 | Separate | Single | 10 | X86 | CmdArguments |
| Version40 | Separate | Single | 10 | X86 | CmdArguments |
| Version45 | Multiple | Single | 10 | X86 | CmdArguments |
| Version40 | Multiple | Single | 10 | X86 | CmdArguments |
| Version45 | Separate | Multiple | 10 | X86 | CmdArguments |
| Version40 | Separate | Multiple | 10 | X86 | CmdArguments |
| Version45 | Separate | Single | 1 | X86 | CmdArguments |
| Version40 | Separate | Single | 1 | X86 | CmdArguments |
| Version45 | Multiple | Single | 1 | X86 | CmdArguments |
| Version40 | Multiple | Single | 1 | X86 | CmdArguments |
| Version45 | Separate | Multiple | 1 | X86 | CmdArguments |
| Version40 | Separate | Multiple | 1 | X86 | CmdArguments |
| Version40 | InProcess | Single | 2 | X86 | CmdArguments |
| Version45 | Separate | Single | 2 | X86 | CmdArguments |
| Version40 | Separate | Single | 2 | X86 | CmdArguments |
| Version45 | Multiple | Single | 2 | X86 | CmdArguments |
| Version40 | Multiple | Single | 2 | X86 | CmdArguments |
| Version45 | InProcess | Multiple | 2 | X86 | CmdArguments |
| Version40 | InProcess | Multiple | 2 | X86 | CmdArguments |
| Version45 | Separate | Multiple | 2 | X86 | CmdArguments |
| Version40 | Separate | Multiple | 2 | X86 | CmdArguments |

Scenario Outline: User runs parallelizable tests
Given Framework version is <frameworkVersion>
And I have added SuccessfulParallelizable method as SuccessfulParallelizable1 to the class Foo.Tests.UnitTests1 for foo1.tests
Expand Down Expand Up @@ -151,4 +298,4 @@ Examples:
| Version45 | Separate | Multiple | 1 | X86 |
| Version40 | Separate | Multiple | 1 | X86 |
# | Version45 | Multiple | Multiple | 1 | X86 |
# | Version40 | Multiple | Multiple | 1 | X86 |
# | Version40 | Multiple | Multiple | 1 | X86 |
Loading

0 comments on commit 39f7e18

Please sign in to comment.