From 2c5c49ad4151cb8672195ea0df3d5a54385c43a2 Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Thu, 24 May 2018 16:49:02 -0700 Subject: [PATCH 1/7] Return the stdout messages --- src/OmniSharp.DotNetTest/Models/DotNetTestResult.cs | 3 ++- src/OmniSharp.DotNetTest/VSTestManager.cs | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/OmniSharp.DotNetTest/Models/DotNetTestResult.cs b/src/OmniSharp.DotNetTest/Models/DotNetTestResult.cs index d8bad6b5a3..ee6be1c3fb 100644 --- a/src/OmniSharp.DotNetTest/Models/DotNetTestResult.cs +++ b/src/OmniSharp.DotNetTest/Models/DotNetTestResult.cs @@ -6,5 +6,6 @@ public class DotNetTestResult public string Outcome { get; set; } public string ErrorMessage { get; set; } public string ErrorStackTrace { get; set; } + public string[] StdOutMessages {get; set; } } -} \ No newline at end of file +} diff --git a/src/OmniSharp.DotNetTest/VSTestManager.cs b/src/OmniSharp.DotNetTest/VSTestManager.cs index ce532dea8a..7d3746af42 100644 --- a/src/OmniSharp.DotNetTest/VSTestManager.cs +++ b/src/OmniSharp.DotNetTest/VSTestManager.cs @@ -240,7 +240,11 @@ public override RunTestResponse RunTest(string[] methodNames, string testFramewo MethodName = testResult.TestCase.FullyQualifiedName, Outcome = testResult.Outcome.ToString().ToLowerInvariant(), ErrorMessage = testResult.ErrorMessage, - ErrorStackTrace = testResult.ErrorStackTrace + ErrorStackTrace = testResult.ErrorStackTrace, + StdOutMessages = testResult.Messages + .Where(message => message.Category == TestResultMessage.StandardOutCategory) + .Select(message => message.Text).ToArray() + }); return new RunTestResponse From bb1a4e45a02a9b1e715aa911b0db5de5185e348c Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Fri, 25 May 2018 11:37:21 -0700 Subject: [PATCH 2/7] StandardErrorMessages --- src/OmniSharp.DotNetTest/Models/DotNetTestResult.cs | 3 ++- src/OmniSharp.DotNetTest/VSTestManager.cs | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/OmniSharp.DotNetTest/Models/DotNetTestResult.cs b/src/OmniSharp.DotNetTest/Models/DotNetTestResult.cs index ee6be1c3fb..10e8261449 100644 --- a/src/OmniSharp.DotNetTest/Models/DotNetTestResult.cs +++ b/src/OmniSharp.DotNetTest/Models/DotNetTestResult.cs @@ -6,6 +6,7 @@ public class DotNetTestResult public string Outcome { get; set; } public string ErrorMessage { get; set; } public string ErrorStackTrace { get; set; } - public string[] StdOutMessages {get; set; } + public string[] StandardOutputMessages { get; set; } + public string[] StandardErrorMessages { get; set; } } } diff --git a/src/OmniSharp.DotNetTest/VSTestManager.cs b/src/OmniSharp.DotNetTest/VSTestManager.cs index 7d3746af42..3dda122cba 100644 --- a/src/OmniSharp.DotNetTest/VSTestManager.cs +++ b/src/OmniSharp.DotNetTest/VSTestManager.cs @@ -241,10 +241,11 @@ public override RunTestResponse RunTest(string[] methodNames, string testFramewo Outcome = testResult.Outcome.ToString().ToLowerInvariant(), ErrorMessage = testResult.ErrorMessage, ErrorStackTrace = testResult.ErrorStackTrace, - StdOutMessages = testResult.Messages + StandardOutputMessages = testResult.Messages .Where(message => message.Category == TestResultMessage.StandardOutCategory) - .Select(message => message.Text).ToArray() - + .Select(message => message.Text).ToArray(), + StandardErrorMessages = testResult.Messages.Where(message => message.Category == TestResultMessage.StandardErrorCategory) + .Select(message => message.Text).ToArray() }); return new RunTestResponse From 09fdc3771af6609efe2947b9f9fa1b66602cc90a Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Mon, 28 May 2018 11:28:09 +0530 Subject: [PATCH 3/7] Rename --- src/OmniSharp.DotNetTest/Models/DotNetTestResult.cs | 4 ++-- src/OmniSharp.DotNetTest/VSTestManager.cs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/OmniSharp.DotNetTest/Models/DotNetTestResult.cs b/src/OmniSharp.DotNetTest/Models/DotNetTestResult.cs index 10e8261449..7a23a149e4 100644 --- a/src/OmniSharp.DotNetTest/Models/DotNetTestResult.cs +++ b/src/OmniSharp.DotNetTest/Models/DotNetTestResult.cs @@ -6,7 +6,7 @@ public class DotNetTestResult public string Outcome { get; set; } public string ErrorMessage { get; set; } public string ErrorStackTrace { get; set; } - public string[] StandardOutputMessages { get; set; } - public string[] StandardErrorMessages { get; set; } + public string[] StandardOutput { get; set; } + public string[] StandardError { get; set; } } } diff --git a/src/OmniSharp.DotNetTest/VSTestManager.cs b/src/OmniSharp.DotNetTest/VSTestManager.cs index 3dda122cba..2ecf992ca5 100644 --- a/src/OmniSharp.DotNetTest/VSTestManager.cs +++ b/src/OmniSharp.DotNetTest/VSTestManager.cs @@ -241,10 +241,10 @@ public override RunTestResponse RunTest(string[] methodNames, string testFramewo Outcome = testResult.Outcome.ToString().ToLowerInvariant(), ErrorMessage = testResult.ErrorMessage, ErrorStackTrace = testResult.ErrorStackTrace, - StandardOutputMessages = testResult.Messages + StandardOutput = testResult.Messages .Where(message => message.Category == TestResultMessage.StandardOutCategory) .Select(message => message.Text).ToArray(), - StandardErrorMessages = testResult.Messages.Where(message => message.Category == TestResultMessage.StandardErrorCategory) + StandardError = testResult.Messages.Where(message => message.Category == TestResultMessage.StandardErrorCategory) .Select(message => message.Text).ToArray() }); From d0e275fb31befac8c12989864ef2d385463bbd3d Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Mon, 28 May 2018 13:50:21 -0700 Subject: [PATCH 4/7] Tests for error message ans stack trace --- .../MSTestProject/TestProgram.cs | 6 +++ .../NUnitTestProject/TestProgram.cs | 6 +++ .../XunitTestProject/TestProgram.cs | 14 +++++-- .../RunTestFacts.cs | 42 +++++++++++++++++++ 4 files changed, 64 insertions(+), 4 deletions(-) diff --git a/test-assets/test-projects/MSTestProject/TestProgram.cs b/test-assets/test-projects/MSTestProject/TestProgram.cs index 832569d94e..8f9378b574 100644 --- a/test-assets/test-projects/MSTestProject/TestProgram.cs +++ b/test-assets/test-projects/MSTestProject/TestProgram.cs @@ -27,6 +27,12 @@ public void DataDrivenTest2(int i) Assert.IsTrue(i >= 0); } + [TestMethod] + public void FailingTest() + { + Assert.Equals(1, 2); + } + private void UtilityFunction() { diff --git a/test-assets/test-projects/NUnitTestProject/TestProgram.cs b/test-assets/test-projects/NUnitTestProject/TestProgram.cs index 3a93c12101..8b2def378b 100644 --- a/test-assets/test-projects/NUnitTestProject/TestProgram.cs +++ b/test-assets/test-projects/NUnitTestProject/TestProgram.cs @@ -30,6 +30,12 @@ public void SourceDataDrivenTest(int i) Assert.True(i > 0); } + [Test] + public void FailingTest() + { + Assert.Equals(1, 2); + } + public void UtilityFunction() { diff --git a/test-assets/test-projects/XunitTestProject/TestProgram.cs b/test-assets/test-projects/XunitTestProject/TestProgram.cs index 10e79a27a0..64da6cf5ee 100644 --- a/test-assets/test-projects/XunitTestProject/TestProgram.cs +++ b/test-assets/test-projects/XunitTestProject/TestProgram.cs @@ -9,7 +9,7 @@ public void Test() { Assert.True(true); } - + [Theory] [InlineData(0)] [InlineData(1)] @@ -17,7 +17,7 @@ public void DataDrivenTest1(int i) { Assert.True(i > 0); } - + [Theory] [InlineData(0)] [InlineData(1)] @@ -25,10 +25,10 @@ public void DataDrivenTest2(int i) { Assert.True(i >= 0); } - + private void UtilityFunction() { - + } [Fact(DisplayName = "My Test Name")] @@ -48,5 +48,11 @@ public void TestWithSimilarNameFooBar() { Assert.True(true); } + + [Fact] + public void FailingTest() + { + Assert.Equal(1, 2); + } } } diff --git a/tests/OmniSharp.DotNetTest.Tests/RunTestFacts.cs b/tests/OmniSharp.DotNetTest.Tests/RunTestFacts.cs index 5bb839a662..3625482da5 100644 --- a/tests/OmniSharp.DotNetTest.Tests/RunTestFacts.cs +++ b/tests/OmniSharp.DotNetTest.Tests/RunTestFacts.cs @@ -66,6 +66,20 @@ public async Task RunXunitTestWithSimilarName() Assert.Single(response.Results); } + [Fact] + public async Task RunXunitFailingTest() + { + var response = await RunDotNetTestAsync( + XunitTestProject, + methodName: "Main.Test.MainTest.FailingTest", + testFramework: "xunit", + shouldPass: false); + + Assert.Single(response.Results); + Assert.NotEmpty(response.Results[0].ErrorMessage); + Assert.NotEmpty(response.Results[0].ErrorStackTrace); + } + [Fact] public async Task RunNunitTest() { @@ -106,6 +120,20 @@ await RunDotNetTestAsync( shouldPass: true); } + [Fact] + public async Task RunNunitFailingTest() + { + var response = await RunDotNetTestAsync( + NUnitTestProject, + methodName: "Main.Test.MainTest.FailingTest", + testFramework: "xunit", + shouldPass: false); + + Assert.Single(response.Results); + Assert.NotEmpty(response.Results[0].ErrorMessage); + Assert.NotEmpty(response.Results[0].ErrorStackTrace); + } + [Fact] public async Task RunMSTestTest() { @@ -135,5 +163,19 @@ await RunDotNetTestAsync( testFramework: "mstest", shouldPass: true); } + + [Fact] + public async Task RunMSTestFailingTest() + { + var response = await RunDotNetTestAsync( + MSTestProject, + methodName: "Main.Test.MainTest.FailingTest", + testFramework: "xunit", + shouldPass: false); + + Assert.Single(response.Results); + Assert.NotEmpty(response.Results[0].ErrorMessage); + Assert.NotEmpty(response.Results[0].ErrorStackTrace); + } } } From 9e8af316cd7af1d678303e986309308624fe0999 Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Mon, 28 May 2018 14:41:28 -0700 Subject: [PATCH 5/7] Standard output is returned --- .../MSTestProject/TestProgram.cs | 11 ++++- .../NUnitTestProject/TestProgram.cs | 11 ++++- .../XunitTestProject/TestProgram.cs | 9 ++++ .../RunTestFacts.cs | 43 ++++++++++++++++++- 4 files changed, 70 insertions(+), 4 deletions(-) diff --git a/test-assets/test-projects/MSTestProject/TestProgram.cs b/test-assets/test-projects/MSTestProject/TestProgram.cs index 8f9378b574..b1f1afa653 100644 --- a/test-assets/test-projects/MSTestProject/TestProgram.cs +++ b/test-assets/test-projects/MSTestProject/TestProgram.cs @@ -1,3 +1,4 @@ +using System; using Microsoft.VisualStudio.TestTools.UnitTesting; namespace Main.Test @@ -30,7 +31,15 @@ public void DataDrivenTest2(int i) [TestMethod] public void FailingTest() { - Assert.Equals(1, 2); + Assert.AreEqual(1, 2); + } + + [TestMethod] + public void CheckStandardOutput() + { + int a = 1, b = 1; + Console.WriteLine($"a = {a}, b = {b}"); + Assert.AreEqual(a,b); } private void UtilityFunction() diff --git a/test-assets/test-projects/NUnitTestProject/TestProgram.cs b/test-assets/test-projects/NUnitTestProject/TestProgram.cs index 8b2def378b..493c2471e6 100644 --- a/test-assets/test-projects/NUnitTestProject/TestProgram.cs +++ b/test-assets/test-projects/NUnitTestProject/TestProgram.cs @@ -1,4 +1,5 @@ using NUnit.Framework; +using System; namespace Main.Test { @@ -33,7 +34,15 @@ public void SourceDataDrivenTest(int i) [Test] public void FailingTest() { - Assert.Equals(1, 2); + Assert.AreEqual(1, 2); + } + + [Test] + public void CheckStandardOutput() + { + int a = 1, b = 1; + Console.WriteLine($"a = {a}, b = {b}"); + Assert.AreEqual(a,b); } public void UtilityFunction() diff --git a/test-assets/test-projects/XunitTestProject/TestProgram.cs b/test-assets/test-projects/XunitTestProject/TestProgram.cs index 64da6cf5ee..f01ceea560 100644 --- a/test-assets/test-projects/XunitTestProject/TestProgram.cs +++ b/test-assets/test-projects/XunitTestProject/TestProgram.cs @@ -1,3 +1,4 @@ +using System; using Xunit; namespace Main.Test @@ -54,5 +55,13 @@ public void FailingTest() { Assert.Equal(1, 2); } + + [Fact] + public void CheckStandardOutput() + { + int a = 1, b = 1; + Console.WriteLine($"a = {a}, b = {b}"); + Assert.Equal(a,b); + } } } diff --git a/tests/OmniSharp.DotNetTest.Tests/RunTestFacts.cs b/tests/OmniSharp.DotNetTest.Tests/RunTestFacts.cs index 3625482da5..b367692468 100644 --- a/tests/OmniSharp.DotNetTest.Tests/RunTestFacts.cs +++ b/tests/OmniSharp.DotNetTest.Tests/RunTestFacts.cs @@ -80,6 +80,19 @@ public async Task RunXunitFailingTest() Assert.NotEmpty(response.Results[0].ErrorStackTrace); } + [Fact] + public async Task RunXunitStandardOutputIsReturned() + { + var response = await RunDotNetTestAsync( + NUnitTestProject, + methodName: "Main.Test.MainTest.CheckStandardOutput", + testFramework: "xunit", + shouldPass: true); + + Assert.Single(response.Results); + Assert.NotEmpty(response.Results[0].StandardOutput); + } + [Fact] public async Task RunNunitTest() { @@ -126,7 +139,7 @@ public async Task RunNunitFailingTest() var response = await RunDotNetTestAsync( NUnitTestProject, methodName: "Main.Test.MainTest.FailingTest", - testFramework: "xunit", + testFramework: "nunit", shouldPass: false); Assert.Single(response.Results); @@ -134,6 +147,19 @@ public async Task RunNunitFailingTest() Assert.NotEmpty(response.Results[0].ErrorStackTrace); } + [Fact] + public async Task RunNunitStandardOutputIsReturned() + { + var response = await RunDotNetTestAsync( + NUnitTestProject, + methodName: "Main.Test.MainTest.CheckStandardOutput", + testFramework: "nunit", + shouldPass: true); + + Assert.Single(response.Results); + Assert.NotEmpty(response.Results[0].StandardOutput); + } + [Fact] public async Task RunMSTestTest() { @@ -170,12 +196,25 @@ public async Task RunMSTestFailingTest() var response = await RunDotNetTestAsync( MSTestProject, methodName: "Main.Test.MainTest.FailingTest", - testFramework: "xunit", + testFramework: "mstest", shouldPass: false); Assert.Single(response.Results); Assert.NotEmpty(response.Results[0].ErrorMessage); Assert.NotEmpty(response.Results[0].ErrorStackTrace); } + + [Fact] + public async Task RunMSTestStandardOutputIsReturned() + { + var response = await RunDotNetTestAsync( + MSTestProject, + methodName: "Main.Test.MainTest.CheckStandardOutput", + testFramework: "mstest", + shouldPass: true); + + Assert.Single(response.Results); + Assert.NotEmpty(response.Results[0].StandardOutput); + } } } From f19d4305dcbbc6644017f5c8753b24f13aeb8ad6 Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Mon, 28 May 2018 15:35:57 -0700 Subject: [PATCH 6/7] Modifed the position int the tests --- .../TestDiscoveryFacts.cs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/OmniSharp.DotNetTest.Tests/TestDiscoveryFacts.cs b/tests/OmniSharp.DotNetTest.Tests/TestDiscoveryFacts.cs index d9ccc2b430..b6f47a5672 100644 --- a/tests/OmniSharp.DotNetTest.Tests/TestDiscoveryFacts.cs +++ b/tests/OmniSharp.DotNetTest.Tests/TestDiscoveryFacts.cs @@ -20,15 +20,15 @@ public TestDiscoveryFacts(ITestOutputHelper output) } [Theory] - [InlineData("XunitTestProject", "TestProgram.cs", 7, 20, true, "XunitTestMethod", "Main.Test.MainTest.Test")] - [InlineData("XunitTestProject", "TestProgram.cs", 15, 20, true, "XunitTestMethod", "Main.Test.MainTest.DataDrivenTest1")] - [InlineData("XunitTestProject", "TestProgram.cs", 23, 20, true, "XunitTestMethod", "Main.Test.MainTest.DataDrivenTest2")] - [InlineData("XunitTestProject", "TestProgram.cs", 28, 21, false, "", "")] - [InlineData("NUnitTestProject", "TestProgram.cs", 7, 20, true, "NUnitTestMethod", "Main.Test.MainTest.Test")] - [InlineData("NUnitTestProject", "TestProgram.cs", 14, 20, true, "NUnitTestMethod", "Main.Test.MainTest.DataDrivenTest1")] - [InlineData("NUnitTestProject", "TestProgram.cs", 21, 20, true, "NUnitTestMethod", "Main.Test.MainTest.DataDrivenTest2")] - [InlineData("NUnitTestProject", "TestProgram.cs", 27, 20, true, "NUnitTestMethod", "Main.Test.MainTest.SourceDataDrivenTest")] - [InlineData("NUnitTestProject", "TestProgram.cs", 32, 20, false, "", "")] + [InlineData("XunitTestProject", "TestProgram.cs", 8, 20, true, "XunitTestMethod", "Main.Test.MainTest.Test")] + [InlineData("XunitTestProject", "TestProgram.cs", 16, 20, true, "XunitTestMethod", "Main.Test.MainTest.DataDrivenTest1")] + [InlineData("XunitTestProject", "TestProgram.cs", 24, 20, true, "XunitTestMethod", "Main.Test.MainTest.DataDrivenTest2")] + [InlineData("XunitTestProject", "TestProgram.cs", 29, 21, false, "", "")] + [InlineData("NUnitTestProject", "TestProgram.cs", 8, 20, true, "NUnitTestMethod", "Main.Test.MainTest.Test")] + [InlineData("NUnitTestProject", "TestProgram.cs", 15, 20, true, "NUnitTestMethod", "Main.Test.MainTest.DataDrivenTest1")] + [InlineData("NUnitTestProject", "TestProgram.cs", 22, 20, true, "NUnitTestMethod", "Main.Test.MainTest.DataDrivenTest2")] + [InlineData("NUnitTestProject", "TestProgram.cs", 28, 20, true, "NUnitTestMethod", "Main.Test.MainTest.SourceDataDrivenTest")] + [InlineData("NUnitTestProject", "TestProgram.cs", 34, 20, true, "NUnitTestMethod", "Main.Test.MainTest.FailingTest")] public async Task FindTestMethods(string projectName, string fileName, int line, int column, bool found, string expectedFeatureName, string expectedMethodName) { using (var testProject = await this._testAssets.GetTestProjectAsync(projectName)) From 1e086d4c335944f19fda9b035de252fcc9e43413 Mon Sep 17 00:00:00 2001 From: Akshita Agarwal Date: Tue, 29 May 2018 02:09:34 -0700 Subject: [PATCH 7/7] Modify test discovery facts --- tests/OmniSharp.DotNetTest.Tests/TestDiscoveryFacts.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/OmniSharp.DotNetTest.Tests/TestDiscoveryFacts.cs b/tests/OmniSharp.DotNetTest.Tests/TestDiscoveryFacts.cs index b6f47a5672..1d84e0987d 100644 --- a/tests/OmniSharp.DotNetTest.Tests/TestDiscoveryFacts.cs +++ b/tests/OmniSharp.DotNetTest.Tests/TestDiscoveryFacts.cs @@ -23,12 +23,15 @@ public TestDiscoveryFacts(ITestOutputHelper output) [InlineData("XunitTestProject", "TestProgram.cs", 8, 20, true, "XunitTestMethod", "Main.Test.MainTest.Test")] [InlineData("XunitTestProject", "TestProgram.cs", 16, 20, true, "XunitTestMethod", "Main.Test.MainTest.DataDrivenTest1")] [InlineData("XunitTestProject", "TestProgram.cs", 24, 20, true, "XunitTestMethod", "Main.Test.MainTest.DataDrivenTest2")] + [InlineData("XunitTestProject", "TestProgram.cs", 53, 20, true, "XunitTestMethod", "Main.Test.MainTest.FailingTest")] + [InlineData("XunitTestProject", "TestProgram.cs", 59, 20, true, "XunitTestMethod", "Main.Test.MainTest.CheckStandardOutput")] [InlineData("XunitTestProject", "TestProgram.cs", 29, 21, false, "", "")] [InlineData("NUnitTestProject", "TestProgram.cs", 8, 20, true, "NUnitTestMethod", "Main.Test.MainTest.Test")] [InlineData("NUnitTestProject", "TestProgram.cs", 15, 20, true, "NUnitTestMethod", "Main.Test.MainTest.DataDrivenTest1")] [InlineData("NUnitTestProject", "TestProgram.cs", 22, 20, true, "NUnitTestMethod", "Main.Test.MainTest.DataDrivenTest2")] [InlineData("NUnitTestProject", "TestProgram.cs", 28, 20, true, "NUnitTestMethod", "Main.Test.MainTest.SourceDataDrivenTest")] [InlineData("NUnitTestProject", "TestProgram.cs", 34, 20, true, "NUnitTestMethod", "Main.Test.MainTest.FailingTest")] + [InlineData("NUnitTestProject", "TestProgram.cs", 47, 20, false, "", "")] public async Task FindTestMethods(string projectName, string fileName, int line, int column, bool found, string expectedFeatureName, string expectedMethodName) { using (var testProject = await this._testAssets.GetTestProjectAsync(projectName))