diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs index e94f8fd47d..d4825ba58b 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs @@ -663,7 +663,7 @@ private static void WarnAboutNotFoundRuntimeProvidersOrThrowWhenNoneAreFound(Lis throw new ArgumentException(null, nameof(testRuntimeProviders)); // Throw when we did not find any runtime provider for any of the provided sources. - var shouldThrow = testRuntimeProviders.All(runtimeProvider => runtimeProvider == null); + var shouldThrow = testRuntimeProviders.All(runtimeProvider => runtimeProvider.Type == null); var missingRuntimeProviders = testRuntimeProviders.Where(p => p.Type == null); if (missingRuntimeProviders.Any()) diff --git a/src/vstest.console/CommandLine/CommandLineOptions.cs b/src/vstest.console/CommandLine/CommandLineOptions.cs index 7e794e4bed..3ba620853c 100644 --- a/src/vstest.console/CommandLine/CommandLineOptions.cs +++ b/src/vstest.console/CommandLine/CommandLineOptions.cs @@ -8,6 +8,7 @@ using System.Linq; using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors; +using Microsoft.VisualStudio.TestPlatform.CommandLine.TestPlatformHelpers; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers; using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces; @@ -292,7 +293,8 @@ public void AddSource(string source) string.Format(CultureInfo.CurrentCulture, CommandLineResources.InvalidArgument, source), ex); } // Add the matching files to source list - _sources = _sources.Union(matchingFiles).ToList(); + var filteredFiles = KnownPlatformSourceFilter.FilterKnownPlatformSources(matchingFiles); + _sources = _sources.Union(filteredFiles).ToList(); } /// diff --git a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs index 34d48d3ea5..7573928d13 100644 --- a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs +++ b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs @@ -167,7 +167,7 @@ public void DiscoverTests( EqtTrace.Info("TestRequestManager.DiscoverTests: Discovery tests started."); // TODO: Normalize rest of the data on the request as well - discoveryPayload.Sources = discoveryPayload.Sources?.Distinct().ToList() ?? new List(); + discoveryPayload.Sources = KnownPlatformSourceFilter.FilterKnownPlatformSources(discoveryPayload.Sources?.Distinct().ToList()); discoveryPayload.RunSettings ??= ""; var runsettings = discoveryPayload.RunSettings; @@ -275,6 +275,11 @@ public void RunTests( { EqtTrace.Info("TestRequestManager.RunTests: run tests started."); testRunRequestPayload.RunSettings ??= ""; + if (testRunRequestPayload.Sources != null) + { + testRunRequestPayload.Sources = KnownPlatformSourceFilter.FilterKnownPlatformSources(testRunRequestPayload.Sources); + } + var runsettings = testRunRequestPayload.RunSettings; if (testRunRequestPayload.TestPlatformOptions != null) @@ -1465,3 +1470,63 @@ private static List GetSources(TestRunRequestPayload testRunRequestPaylo return sources; } } + +internal static class KnownPlatformSourceFilter +{ + + // Running tests on AzureDevops, many projects use the default filter + // which includes all *test*.dll, this includes many of the TestPlatform dlls, + // which we cannot run, and don't want to attempt to run. + // The default filter also filters out !*TestAdapter*.dll but it is easy to forget + // so we skip the most used adapters here as well. + private static readonly HashSet KnownPlatformSources = new(new string[] + { + "Microsoft.TestPlatform.CommunicationUtilities.dll", + "Microsoft.TestPlatform.CoreUtilities.dll", + "Microsoft.TestPlatform.CrossPlatEngine.dll", + "Microsoft.TestPlatform.PlatformAbstractions.dll", + "Microsoft.TestPlatform.Utilities.dll", + "Microsoft.VisualStudio.TestPlatform.Common.dll", + "Microsoft.VisualStudio.TestPlatform.ObjectModel.dll", + "testhost.dll", + "Microsoft.TestPlatform.AdapterUtilities.dll", + + // NUnit + "NUnit3.TestAdapter.dll", + + // XUnit + "xunit.runner.visualstudio.testadapter.dll", + "xunit.runner.visualstudio.dotnetcore.testadapter.dll", + + // MSTest + "Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.dll", + "Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll", + "Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface.dll", + "Microsoft.VisualStudio.TestPlatform.TestFramework.dll", + "Microsoft.VisualStudio.TestPlatform.TestFramework.Extensions.dll", + }, StringComparer.OrdinalIgnoreCase); + + + internal static List FilterKnownPlatformSources(List? sources) + { + if (sources == null) + { + return new List(); + } + + var filteredSources = new List(); + foreach (string source in sources) + { + if (KnownPlatformSources.Contains(Path.GetFileName(source))) + { + EqtTrace.Info($"TestRequestManager.FilterKnownPlatformSources: Known platform dll was provided in sources, removing it '{source}'"); + } + else + { + filteredSources.Add(source); + } + } + + return filteredSources; + } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index 69a8e8f227..142048b2c0 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -9,6 +9,9 @@ using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Extensions; using TestPlatform.TestUtilities; +using System.Linq; +using Microsoft.VisualStudio.TestPlatform.Common; +using FluentAssertions; namespace Microsoft.TestPlatform.AcceptanceTests; @@ -396,4 +399,100 @@ public void ExitCodeShouldNotDependOnFailTreatNoTestsAsErrorFalseValueWhenThereA // Returning 1 because of failing test in test assembly (SimpleTestProject2.dll) ExitCodeEquals(1); } + + [TestMethod] + [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] + [NetCoreTargetFrameworkDataSource] + public void RunXunitTestsWhenProvidingAllDllsInBin(RunnerInfo runnerInfo) + { + // This is the default filter of AzDo VSTest task: + // testAssemblyVer2: | + // **\*test *.dll + // ! * *\*TestAdapter.dll + // ! * *\obj\** + // Because of this in typical run we get a lot of dlls that we are sure don't have tests, like Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll + // or testhost.dll. Those dlls are built for netcoreapp3.1 tfm, so theoretically they should be tests, but attempting to run them fails to find runtimeconfig.json + // or deps.json, and fails the run. + SetTestEnvironment(_testEnvironment, runnerInfo); + + var testAssemblyPath = _testEnvironment.GetTestAsset("XUTestProject.dll"); + var allDllsMatchingTestPattern = Directory.GetFiles(Path.GetDirectoryName(testAssemblyPath)!, "*test*.dll"); + + string assemblyPaths = string.Join(" ", allDllsMatchingTestPattern.Concat(new[] { testAssemblyPath }).Select(s => s.AddDoubleQuote())); + InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty); + var fails = this.StdErrWithWhiteSpace.Split('\n').Where(s => !s.IsNullOrWhiteSpace()).Select(s => s.Trim()).ToList(); + fails.Should().HaveCount(2, "because there is 1 failed test, and one message that tests failed."); + fails.Last().Should().Be("Test Run Failed."); + } + + [TestMethod] + [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] + [NetCoreTargetFrameworkDataSource()] + public void RunMstestTestsWhenProvidingAllDllsInBin(RunnerInfo runnerInfo) + { + // This is the default filter of AzDo VSTest task: + // testAssemblyVer2: | + // **\*test *.dll + // ! * *\*TestAdapter.dll + // ! * *\obj\** + // Because of this in typical run we get a lot of dlls that we are sure don't have tests, like Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll + // or testhost.dll. Those dlls are built for netcoreapp3.1 tfm, so theoretically they should be tests, but attempting to run them fails to find runtimeconfig.json + // or deps.json, and fails the run. + SetTestEnvironment(_testEnvironment, runnerInfo); + + var testAssemblyPath = _testEnvironment.GetTestAsset("SimpleTestProject.dll"); + var allDllsMatchingTestPattern = Directory.GetFiles(Path.GetDirectoryName(testAssemblyPath)!, "*test*.dll"); + + string assemblyPaths = string.Join(" ", allDllsMatchingTestPattern.Concat(new[] { testAssemblyPath }).Select(s => s.AddDoubleQuote())); + InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty); + + StdErrHasTestRunFailedMessageButNoOtherError(); + } + + [TestMethod] + [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] + [NetCoreTargetFrameworkDataSource()] + public void RunNunitTestsWhenProvidingAllDllsInBin(RunnerInfo runnerInfo) + { + // This is the default filter of AzDo VSTest task: + // testAssemblyVer2: | + // **\*test *.dll + // ! * *\*TestAdapter.dll + // ! * *\obj\** + // Because of this in typical run we get a lot of dlls that we are sure don't have tests, like Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll + // or testhost.dll. Those dlls are built for netcoreapp3.1 tfm, so theoretically they should be tests, but attempting to run them fails to find runtimeconfig.json + // or deps.json, and fails the run. + SetTestEnvironment(_testEnvironment, runnerInfo); + + var testAssemblyPath = _testEnvironment.GetTestAsset("NUTestProject.dll"); + var allDllsMatchingTestPattern = Directory.GetFiles(Path.GetDirectoryName(testAssemblyPath)!, "*test*.dll"); + + string assemblyPaths = string.Join(" ", allDllsMatchingTestPattern.Concat(new[] { testAssemblyPath }).Select(s => s.AddDoubleQuote())); + InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty); + + StdErrHasTestRunFailedMessageButNoOtherError(); + } + + [TestMethod] + [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)] + public void RunTestsWhenProvidingJustPlatformDllsFailsTheRun(RunnerInfo runnerInfo) + { + // This is the default filter of AzDo VSTest task: + // testAssemblyVer2: | + // **\*test *.dll + // ! * *\*TestAdapter.dll + // ! * *\obj\** + // Because of this in typical run we get a lot of dlls that we are sure don't have tests, like Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll + // or testhost.dll. Those dlls are built for netcoreapp3.1 tfm, so theoretically they should be tests, but attempting to run them fails to find runtimeconfig.json + // or deps.json, and fails the run. + SetTestEnvironment(_testEnvironment, runnerInfo); + + var xunitAssemblyPath = _testEnvironment.GetTestAsset("SimpleTestProject.dll"); + var allDllsMatchingTestPattern = Directory.GetFiles(Path.GetDirectoryName(xunitAssemblyPath)!, "*test*.dll").Where(f => !f.EndsWith("SimpleTestProject.dll")); + + string assemblyPaths = string.Join(" ", allDllsMatchingTestPattern.Select(s => s.AddDoubleQuote())); + InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty); + + StdErr.Should().Be("No test source files were specified. ", "because all platform files we provided were filtered out"); + } }