Support for manipulating generated files (#128)

* Get generated files binary log

closes #125

* Command for dumping generated files to disk

* Generated command

* more tests

* more tests

src/Basic.CompilerLog.UnitTests/BinaryLogReaderTests.cs

@@ -147,5 +147,18 @@ public void ReadDeletedPdb()
         var data = reader.ReadAllCompilationData().Single();
         var diagnostic = data.GetDiagnostics().Where(x => x.Severity == DiagnosticSeverity.Error).Single();
         Assert.Contains("Can't find portable pdb file for", diagnostic.GetMessage());
+
+        Assert.Throws<InvalidOperationException>(() => reader.ReadAllGeneratedFiles(data.CompilerCall));
+    }
+
+    [Fact]
+    public void ReadGeneratedFiles()
+    {
+        using var reader = BinaryLogReader.Create(Fixture.Console.Value.BinaryLogPath!, BasicAnalyzerKind.None);
+        var compilerCall = reader.ReadAllCompilerCalls().Single();
+        var generatedFiles = reader.ReadAllGeneratedFiles(compilerCall);
+        Assert.Single(generatedFiles);
+        var tuple = generatedFiles.Single();
+        Assert.True(tuple.Stream.TryGetBuffer(out var _));
     }
 }

src/Basic.CompilerLog.UnitTests/CompilationDataTests.cs

@@ -124,4 +124,17 @@ public void GetCompilationAfterGeneratorsDiagnostics()
         _ = data.GetCompilationAfterGenerators(out var diagnostics);
         Assert.NotEmpty(diagnostics);
     }
+
+    [Fact]
+    public void GetGeneratedSyntaxTrees()
+    {
+        using var reader = CompilerLogReader.Create(Fixture.Console.Value.CompilerLogPath);
+        var data = reader.ReadAllCompilationData().Single();
+        var trees = data.GetGeneratedSyntaxTrees();
+        Assert.Single(trees);
+
+        trees = data.GetGeneratedSyntaxTrees(out var diagnostics);
+        Assert.Single(trees);
+        Assert.Empty(diagnostics);
+    }
 }

src/Basic.CompilerLog.UnitTests/ProgramTests.cs

@@ -292,7 +292,7 @@ public void ResponseSingleLine()
     {
         var exitCode = RunCompLog($"rsp {Fixture.SolutionBinaryLogPath} -p console.csproj -s");
         Assert.Equal(Constants.ExitSuccess, exitCode);
-        var rsp = Path.Combine(RootDirectory, @".complog", "console", "build.rsp");
+        var rsp = Path.Combine(RootDirectory, @".complog", "rsp", "console", "build.rsp");
         Assert.True(File.Exists(rsp));
 
         var lines = File.ReadAllLines(rsp);
@@ -316,7 +316,7 @@ public void ResponseProjectFilter()
     {
         var exitCode = RunCompLog($"rsp {Fixture.SolutionBinaryLogPath} -p console.csproj");
         Assert.Equal(Constants.ExitSuccess, exitCode);
-        var rsp = Path.Combine(RootDirectory, @".complog", "console", "build.rsp");
+        var rsp = Path.Combine(RootDirectory, @".complog", "rsp", "console", "build.rsp");
         Assert.True(File.Exists(rsp));
         Assert.Contains("Program.cs", File.ReadAllLines(rsp));
     }
@@ -342,7 +342,7 @@ public void ResponseAll()
     {
         var exitCode = RunCompLog($"rsp {Fixture.SolutionBinaryLogPath}");
         Assert.Equal(Constants.ExitSuccess, exitCode);
-        var rsp = Path.Combine(RootDirectory, @".complog", "console", "build.rsp");
+        var rsp = Path.Combine(RootDirectory, @".complog", "rsp", "console", "build.rsp");
         Assert.True(File.Exists(rsp));
         Assert.Contains("Program.cs", File.ReadAllLines(rsp));
     }
@@ -352,8 +352,8 @@ public void ResponseMultiTarget()
     {
         var exitCode = RunCompLog($"rsp {Fixture.ClassLibMultiProjectPath}");
         Assert.Equal(Constants.ExitSuccess, exitCode);
-        Assert.True(File.Exists(Path.Combine(RootDirectory, @".complog", "classlibmulti-net6.0", "build.rsp")));
-        Assert.True(File.Exists(Path.Combine(RootDirectory, @".complog", "classlibmulti-net7.0", "build.rsp")));
+        Assert.True(File.Exists(Path.Combine(RootDirectory, @".complog", "rsp", "classlibmulti-net6.0", "build.rsp")));
+        Assert.True(File.Exists(Path.Combine(RootDirectory, @".complog", "rsp", "classlibmulti-net7.0", "build.rsp")));
     }
 
     [Fact]
@@ -417,7 +417,7 @@ public void ExportCompilerLog(string arg, BasicAnalyzerKind? expectedKind)
             Assert.Equal(Constants.ExitSuccess, RunCompLog($"export -o {exportDir.DirectoryPath} {arg} {logPath} ", RootDirectory));
 
             // Now run the generated build.cmd and see if it succeeds;
-            var exportPath = Path.Combine(exportDir.DirectoryPath, "console", "export");
+            var exportPath = Path.Combine(exportDir.DirectoryPath, "console");
             var buildResult = RunBuildCmd(exportPath);
             Assert.True(buildResult.Succeeded);
 
@@ -493,9 +493,9 @@ public void ReplayConsoleWithEmit(string arg)
         using var emitDir = new TempDir();
         Assert.Equal(Constants.ExitSuccess, RunCompLog($"replay {arg} -o {emitDir.DirectoryPath} {Fixture.SolutionBinaryLogPath}"));
 
-        AssertOutput(@"console\emit\console.dll");
-        AssertOutput(@"console\emit\console.pdb");
-        AssertOutput(@"console\emit\ref\console.dll");
+        AssertOutput(@"console\console.dll");
+        AssertOutput(@"console\console.pdb");
+        AssertOutput(@"console\ref\console.dll");
 
         void AssertOutput(string relativePath)
         {
@@ -603,6 +603,61 @@ public void ReplayWithProject()
         Assert.Equal(Constants.ExitSuccess, RunCompLog($"replay {Fixture.ConsoleProjectPath}"));
     }
 
+    [Theory]
+    [CombinatorialData]
+    public void GeneratedBoth(BasicAnalyzerKind basicAnalyzerKind)
+    {
+        RunWithBoth(logPath =>
+        {
+            AssertCompilerCallReader(void (ICompilerCallReader reader) => AssertCorrectReader(reader, logPath));
+            var dir = Root.NewDirectory("generated");
+            var (exitCode, output) = RunCompLogEx($"generated {logPath} -p console.csproj -a {basicAnalyzerKind} -o {dir}");
+            Assert.Equal(Constants.ExitSuccess, exitCode);
+            Assert.Single(Directory.EnumerateFiles(dir, "RegexGenerator.g.cs", SearchOption.AllDirectories));
+        });
+    }
+
+    [Fact]
+    public void GeneratedBadFilter()
+    {
+        RunWithBoth(logPath =>
+        {
+            AssertCompilerCallReader(void (ICompilerCallReader reader) => AssertCorrectReader(reader, logPath));
+            var (exitCode, _) = RunCompLogEx($"generated {logPath} -p console-does-not-exist.csproj");
+            Assert.Equal(Constants.ExitFailure, exitCode);
+        });
+    }
+
+    [Fact]
+    public void GeneratePdbMissing()
+    {
+        var dir = Root.NewDirectory();
+        RunDotNet($"new console --name example --output .", dir);
+        RunDotNet("build -bl -nr:false", dir);
+
+        // Delete the PDB
+        Directory.EnumerateFiles(dir, "*.pdb", SearchOption.AllDirectories).ForEach(File.Delete);
+
+        var (exitCode, output) = RunCompLogEx($"generated {dir} -a None");
+        Assert.Equal(Constants.ExitSuccess, exitCode);
+        Assert.Contains("BCLA0001", output);
+    }
+
+    [Fact]
+    public void GeneratedHelp()
+    {
+        var (exitCode, output) = RunCompLogEx($"generated -h");
+        Assert.Equal(Constants.ExitSuccess, exitCode);
+        Assert.StartsWith("complog generated [OPTIONS]", output);
+    }
+
+    [Fact]
+    public void GeneratedBadArg()
+    {
+        var (exitCode, _) = RunCompLogEx($"generated -o");
+        Assert.Equal(Constants.ExitFailure, exitCode);
+    }
+
     [Fact]
     public void PrintAll()
     {

src/Basic.CompilerLog.UnitTests/SolutionFixture.cs

@@ -67,6 +67,19 @@ public SolutionFixture(IMessageSink messageSink)
         ConsoleProjectPath = WithProject("console", string (string dir) =>
         {
             RunDotnetCommand("new console --name console -o .", dir);
+            var program = """
+                using System;
+                using System.Text.RegularExpressions;
+                // This is an amazing resource
+                var r = Util.GetRegex();
+                Console.WriteLine(r);
+
+                partial class Util {
+                    [GeneratedRegex("abc|def", RegexOptions.IgnoreCase, "en-US")]
+                    internal static partial Regex GetRegex();
+                }
+                """;
+            File.WriteAllText(Path.Combine(dir, "Program.cs"), program, TestBase.DefaultEncoding);
             return Path.Combine(dir, "console.csproj");
         });
 

src/Basic.CompilerLog.Util/BinaryLogReader.cs

@@ -305,6 +305,20 @@ public List<ReferenceData> ReadAllReferenceData(CompilerCall compilerCall)
         return ReadAllReferenceDataCore(args.MetadataReferences.Select(x => x.Reference), args.MetadataReferences.Length);
     }
 
+    /// <summary>
+    /// Attempt to add all the generated files from generators. When successful the generators
+    /// don't need to be run when re-hydrating the compilation.
+    /// </summary>
+    /// <remarks>
+    /// This method will throw if the compilation does not have a PDB compatible with generated files
+    /// available to read
+    /// </remarks>
+    public List<(string FilePath, MemoryStream Stream)> ReadAllGeneratedFiles(CompilerCall compilerCall)
+    {
+        var args = ReadCommandLineArguments(compilerCall);
+        return RoslynUtil.ReadGeneratedFiles(compilerCall, args);
+    }
+
     private List<ReferenceData> ReadAllReferenceDataCore(IEnumerable<string> filePaths, int count)
     {
         var list = new List<ReferenceData>(capacity: count);

src/Basic.CompilerLog.Util/CompilationData.cs

@@ -119,7 +119,9 @@ public ImmutableArray<ISourceGenerator> GetGenerators()
     public Compilation GetCompilationAfterGenerators(CancellationToken cancellationToken = default) =>
         GetCompilationAfterGenerators(out _, cancellationToken);
 
-    public Compilation GetCompilationAfterGenerators(out ImmutableArray<Diagnostic> diagnostics, CancellationToken cancellationToken = default)
+    public Compilation GetCompilationAfterGenerators(
+        out ImmutableArray<Diagnostic> diagnostics,
+        CancellationToken cancellationToken = default)
     {
         if (_afterGenerators is { } tuple)
         {
@@ -141,6 +143,28 @@ public Compilation GetCompilationAfterGenerators(out ImmutableArray<Diagnostic>
         return tuple.Item1;
     }
 
+    public List<SyntaxTree> GetGeneratedSyntaxTrees(CancellationToken cancellationToken = default) =>
+        GetGeneratedSyntaxTrees(out _, cancellationToken);
+
+    public List<SyntaxTree> GetGeneratedSyntaxTrees(
+        out ImmutableArray<Diagnostic> diagnostics,
+        CancellationToken cancellationToken = default)
+    {
+        var afterCompilation = GetCompilationAfterGenerators(out diagnostics, cancellationToken);
+
+        // This is a bit of a hack to get the number of syntax trees before running the generators. It feels
+        // a bit disjoint that we have to think of the None case differently here. Possible it may be simpler
+        // to have the None host go back to faking a ISourceGenerator in memory that just adds the files
+        // directly.
+        var originalCount = Compilation.SyntaxTrees.Count();
+        if (BasicAnalyzerHost is BasicAnalyzerHostNone none)
+        {
+            var generatedCount = none.GeneratedSourceTexts.Length;
+            originalCount -= generatedCount;
+        }
+        return afterCompilation.SyntaxTrees.Skip(originalCount).ToList();
+    }
+
     private void EnsureAnalyzersLoaded()
     {
         if (!_analyzers.IsDefault)

src/Basic.CompilerLog/Constants.cs

@@ -14,5 +14,4 @@ internal static class Constants
     internal static TextWriter Out { get; set; } = Console.Out;
 
     internal static Action<ICompilerCallReader> OnCompilerCallReader = _ => { };
-
 }