Separate error reporting, string baselining from compiler baselining (#217)

Author: andrewbranch

.gitignore

@@ -175,6 +175,7 @@ go.work.sum
 
 # Local baselines
 testdata/baselines/local
+testdata/baselines/tmp
 
 custom-gcl
 custom-gcl.exe

cmd/tsgo/main.go

@@ -15,6 +15,7 @@ import (
 	"github.com/microsoft/typescript-go/internal/bundled"
 	ts "github.com/microsoft/typescript-go/internal/compiler"
 	"github.com/microsoft/typescript-go/internal/core"
+	"github.com/microsoft/typescript-go/internal/diagnosticwriter"
 	"github.com/microsoft/typescript-go/internal/scanner"
 	"github.com/microsoft/typescript-go/internal/tspath"
 	"github.com/microsoft/typescript-go/internal/vfs"
@@ -141,13 +142,13 @@ func main() {
 			UseCaseSensitiveFileNames: useCaseSensitiveFileNames,
 		}
 		if pretty {
-			formatOpts := ts.DiagnosticsFormattingOptions{
+			formatOpts := diagnosticwriter.FormattingOptions{
 				NewLine:             "\n",
 				ComparePathsOptions: comparePathOptions,
 			}
-			ts.FormatDiagnosticsWithColorAndContext(os.Stdout, diagnostics, &formatOpts)
+			diagnosticwriter.FormatDiagnosticsWithColorAndContext(os.Stdout, diagnostics, &formatOpts)
 			fmt.Fprintln(os.Stdout)
-			ts.WriteErrorSummaryText(os.Stdout, diagnostics, &formatOpts)
+			diagnosticwriter.WriteErrorSummaryText(os.Stdout, diagnostics, &formatOpts)
 		} else {
 			for _, diagnostic := range diagnostics {
 				printDiagnostic(diagnostic, 0, comparePathOptions)

internal/compiler/error_reporting.go renamed to internal/diagnosticwriter/diagnosticwriter.go

@@ -1,4 +1,4 @@
-package compiler
+package diagnosticwriter
 
 import (
 	"fmt"
@@ -15,7 +15,7 @@ import (
 	"github.com/microsoft/typescript-go/internal/tspath"
 )
 
-type DiagnosticsFormattingOptions struct {
+type FormattingOptions struct {
 	tspath.ComparePathsOptions
 	NewLine string
 }
@@ -35,7 +35,7 @@ const (
 	ellipsis            = "..."
 )
 
-func FormatDiagnosticsWithColorAndContext(output io.Writer, diags []*ast.Diagnostic, formatOpts *DiagnosticsFormattingOptions) {
+func FormatDiagnosticsWithColorAndContext(output io.Writer, diags []*ast.Diagnostic, formatOpts *FormattingOptions) {
 	if len(diags) == 0 {
 		return
 	}
@@ -80,7 +80,7 @@ func FormatDiagnosticsWithColorAndContext(output io.Writer, diags []*ast.Diagnos
 	}
 }
 
-func writeCodeSnippet(writer io.Writer, sourceFile *ast.SourceFile, start int, length int, squiggleColor string, formatOpts *DiagnosticsFormattingOptions) {
+func writeCodeSnippet(writer io.Writer, sourceFile *ast.SourceFile, start int, length int, squiggleColor string, formatOpts *FormattingOptions) {
 	firstLine, firstLineChar := scanner.GetLineAndCharacterOfPosition(sourceFile, start)
 	lastLine, lastLineChar := scanner.GetLineAndCharacterOfPosition(sourceFile, start+length)
 
@@ -196,7 +196,7 @@ func writeWithStyleAndReset(output io.Writer, text string, formatStyle string) {
 	fmt.Fprint(output, resetEscapeSequence)
 }
 
-func WriteLocation(output io.Writer, file *ast.SourceFile, pos int, formatOpts *DiagnosticsFormattingOptions, writeWithStyleAndReset FormattedWriter) {
+func WriteLocation(output io.Writer, file *ast.SourceFile, pos int, formatOpts *FormattingOptions, writeWithStyleAndReset FormattedWriter) {
 	firstLine, firstChar := scanner.GetLineAndCharacterOfPosition(file, pos)
 	var relativeFileName string
 	if formatOpts != nil {
@@ -221,7 +221,7 @@ type ErrorSummary struct {
 	SortedFileList  []*ast.SourceFile
 }
 
-func WriteErrorSummaryText(output io.Writer, allDiagnostics []*ast.Diagnostic, formatOpts *DiagnosticsFormattingOptions) {
+func WriteErrorSummaryText(output io.Writer, allDiagnostics []*ast.Diagnostic, formatOpts *FormattingOptions) {
 	// Roughly corresponds to 'getErrorSummaryText' from watch.ts
 
 	errorSummary := getErrorSummary(allDiagnostics)
@@ -299,7 +299,7 @@ func getErrorSummary(diags []*ast.Diagnostic) *ErrorSummary {
 	}
 }
 
-func writeTabularErrorsDisplay(output io.Writer, errorSummary *ErrorSummary, formatOpts *DiagnosticsFormattingOptions) {
+func writeTabularErrorsDisplay(output io.Writer, errorSummary *ErrorSummary, formatOpts *FormattingOptions) {
 	sortedFiles := errorSummary.SortedFileList
 
 	maxErrors := 0
@@ -330,7 +330,7 @@ func writeTabularErrorsDisplay(output io.Writer, errorSummary *ErrorSummary, for
 	}
 }
 
-func prettyPathForFileError(file *ast.SourceFile, fileErrors []*ast.Diagnostic, formatOpts *DiagnosticsFormattingOptions) string {
+func prettyPathForFileError(file *ast.SourceFile, fileErrors []*ast.Diagnostic, formatOpts *FormattingOptions) string {
 	line, _ := scanner.GetLineAndCharacterOfPosition(file, fileErrors[0].Loc().Pos())
 	fileName := file.FileName()
 	if tspath.PathIsAbsolute(fileName) && tspath.PathIsAbsolute(formatOpts.CurrentDirectory) {
@@ -344,13 +344,13 @@ func prettyPathForFileError(file *ast.SourceFile, fileErrors []*ast.Diagnostic,
 	)
 }
 
-func WriteFormatDiagnostics(output io.Writer, diagnostics []*ast.Diagnostic, formatOpts *DiagnosticsFormattingOptions) {
+func WriteFormatDiagnostics(output io.Writer, diagnostics []*ast.Diagnostic, formatOpts *FormattingOptions) {
 	for _, diagnostic := range diagnostics {
 		WriteFormatDiagnostic(output, diagnostic, formatOpts)
 	}
 }
 
-func WriteFormatDiagnostic(output io.Writer, diagnostic *ast.Diagnostic, formatOpts *DiagnosticsFormattingOptions) {
+func WriteFormatDiagnostic(output io.Writer, diagnostic *ast.Diagnostic, formatOpts *FormattingOptions) {
 	if diagnostic.File() != nil {
 		line, character := scanner.GetLineAndCharacterOfPosition(diagnostic.File(), diagnostic.Loc().Pos())
 		fileName := diagnostic.File().FileName()

internal/testutil/baseline/baseline.go

@@ -16,21 +16,35 @@ type Options struct {
 const NoContent = "<no content>"
 
 func Run(t *testing.T, fileName string, actual string, opts Options) {
-	writeComparison(t, actual, fileName, opts)
+	writeComparison(t, actual, fileName, false /*useSubmodule*/, opts)
 }
 
-func writeComparison(t *testing.T, actual string, relativeFileName string, opts Options) {
+func RunAgainstSubmodule(t *testing.T, fileName string, actual string, opts Options) {
+	writeComparison(t, actual, fileName, true /*useSubmodule*/, opts)
+}
+
+func writeComparison(t *testing.T, actual string, relativeFileName string, useSubmodule bool, opts Options) {
 	if actual == "" {
-		panic("The generated content was \"\". Return 'baseline.NoContent' if no baselining is required.")
+		panic("the generated content was \"\". Return 'baseline.NoContent' if no baselining is required.")
 	}
 	var (
 		localFileName     string
 		referenceFileName string
 	)
 
-	localFileName = localPath(relativeFileName, opts.Subfolder)
-	referenceFileName = referencePath(relativeFileName, opts.Subfolder)
-	expected := getExpectedContent(relativeFileName, opts)
+	if useSubmodule {
+		localFileName = submoduleLocalPath(relativeFileName, opts.Subfolder)
+		referenceFileName = submoduleReferencePath(relativeFileName, opts.Subfolder)
+	} else {
+		localFileName = localPath(relativeFileName, opts.Subfolder)
+		referenceFileName = referencePath(relativeFileName, opts.Subfolder)
+	}
+
+	expected := NoContent
+	if content, err := os.ReadFile(referenceFileName); err == nil {
+		expected = string(content)
+	}
+
 	if _, err := os.Stat(localFileName); err == nil {
 		if err := os.Remove(localFileName); err != nil {
 			t.Fatal(fmt.Errorf("failed to remove the local baseline file %s: %w", localFileName, err))
@@ -49,26 +63,31 @@ func writeComparison(t *testing.T, actual string, relativeFileName string, opts
 		}
 
 		if _, err := os.Stat(referenceFileName); err != nil {
-			t.Errorf("New baseline created at %s.", localFileName)
+			if useSubmodule {
+				t.Errorf("the baseline file %s does not exist in the TypeScript submodule", referenceFileName)
+			} else {
+				t.Errorf("new baseline created at %s.", localFileName)
+			}
+		} else if useSubmodule {
+			t.Errorf("the baseline file %s does not match the reference in the TypeScript submodule", relativeFileName)
 		} else {
-			t.Errorf("The baseline file %s has changed. (Run `hereby baseline-accept` if the new baseline is correct.)", relativeFileName)
+			t.Errorf("the baseline file %s has changed. (Run `hereby baseline-accept` if the new baseline is correct.)", relativeFileName)
 		}
 	}
 }
 
-func getExpectedContent(relativeFileName string, opts Options) string {
-	refFileName := referencePath(relativeFileName, opts.Subfolder)
-	expected := NoContent
-	if content, err := os.ReadFile(refFileName); err == nil {
-		expected = string(content)
-	}
-	return expected
-}
-
 func localPath(fileName string, subfolder string) string {
 	return filepath.Join(repo.TestDataPath, "baselines", "local", subfolder, fileName)
 }
 
+func submoduleLocalPath(fileName string, subfolder string) string {
+	return filepath.Join(repo.TestDataPath, "baselines", "tmp", subfolder, fileName)
+}
+
 func referencePath(fileName string, subfolder string) string {
 	return filepath.Join(repo.TestDataPath, "baselines", "reference", subfolder, fileName)
 }
+
+func submoduleReferencePath(fileName string, subfolder string) string {
+	return filepath.Join(repo.TypeScriptSubmodulePath, "tests", "baselines", "reference", subfolder, fileName)
+}

internal/testutil/baseline/error_baseline.go renamed to internal/testutil/tsbaseline/error_baseline.go

@@ -1,4 +1,4 @@
-package baseline
+package tsbaseline
 
 import (
 	"fmt"
@@ -11,6 +11,8 @@ import (
 	"github.com/microsoft/typescript-go/internal/ast"
 	"github.com/microsoft/typescript-go/internal/compiler"
 	"github.com/microsoft/typescript-go/internal/core"
+	"github.com/microsoft/typescript-go/internal/diagnosticwriter"
+	"github.com/microsoft/typescript-go/internal/testutil/baseline"
 	"github.com/microsoft/typescript-go/internal/tspath"
 	"gotest.tools/v3/assert"
 	"gotest.tools/v3/assert/cmp"
@@ -19,7 +21,7 @@ import (
 // IO
 const harnessNewLine = "\r\n"
 
-var formatOpts = &compiler.DiagnosticsFormattingOptions{
+var formatOpts = &diagnosticwriter.FormattingOptions{
 	NewLine: harnessNewLine,
 }
 
@@ -40,17 +42,17 @@ func DoErrorBaseline(t *testing.T, baselinePath string, inputFiles []*TestFile,
 	if len(errors) > 0 {
 		errorBaseline = getErrorBaseline(t, inputFiles, errors, pretty)
 	} else {
-		errorBaseline = NoContent
+		errorBaseline = baseline.NoContent
 	}
-	Run(t, baselinePath, errorBaseline, Options{})
+	baseline.Run(t, baselinePath, errorBaseline, baseline.Options{})
 }
 
 func minimalDiagnosticsToString(diagnostics []*ast.Diagnostic, pretty bool) string {
 	var output strings.Builder
 	if pretty {
-		compiler.FormatDiagnosticsWithColorAndContext(&output, diagnostics, formatOpts)
+		diagnosticwriter.FormatDiagnosticsWithColorAndContext(&output, diagnostics, formatOpts)
 	} else {
-		compiler.WriteFormatDiagnostics(&output, diagnostics, formatOpts)
+		diagnosticwriter.WriteFormatDiagnostics(&output, diagnostics, formatOpts)
 	}
 	return output.String()
 }
@@ -61,7 +63,7 @@ func getErrorBaseline(t *testing.T, inputFiles []*TestFile, diagnostics []*ast.D
 
 	if pretty {
 		var summaryBuilder strings.Builder
-		compiler.WriteErrorSummaryText(
+		diagnosticwriter.WriteErrorSummaryText(
 			&summaryBuilder,
 			diagnostics,
 			formatOpts)
@@ -248,12 +250,12 @@ func iterateErrorBaseline(t *testing.T, inputFiles []*TestFile, inputDiagnostics
 
 func flattenDiagnosticMessage(d *ast.Diagnostic, newLine string) string {
 	var output strings.Builder
-	compiler.WriteFlattenedDiagnosticMessage(&output, d, newLine)
+	diagnosticwriter.WriteFlattenedDiagnosticMessage(&output, d, newLine)
 	return output.String()
 }
 
-func formatLocation(file *ast.SourceFile, pos int, formatOpts *compiler.DiagnosticsFormattingOptions, writeWithStyleAndReset compiler.FormattedWriter) string {
+func formatLocation(file *ast.SourceFile, pos int, formatOpts *diagnosticwriter.FormattingOptions, writeWithStyleAndReset diagnosticwriter.FormattedWriter) string {
 	var output strings.Builder
-	compiler.WriteLocation(&output, file, pos, formatOpts, writeWithStyleAndReset)
+	diagnosticwriter.WriteLocation(&output, file, pos, formatOpts, writeWithStyleAndReset)
 	return output.String()
 }

internal/testutil/baseline/symbol_baseline.go renamed to internal/testutil/tsbaseline/symbol_baseline.go

@@ -1,4 +1,4 @@
-package baseline
+package tsbaseline
 
 import (
 	"fmt"
@@ -12,6 +12,7 @@ import (
 	"github.com/microsoft/typescript-go/internal/compiler"
 	"github.com/microsoft/typescript-go/internal/core"
 	"github.com/microsoft/typescript-go/internal/scanner"
+	"github.com/microsoft/typescript-go/internal/testutil/baseline"
 	"github.com/microsoft/typescript-go/internal/tspath"
 )
 
@@ -27,7 +28,7 @@ func DoTypeAndSymbolBaseline(
 	header string,
 	program *compiler.Program,
 	allFiles []*TestFile,
-	opts Options,
+	opts baseline.Options,
 	skipTypeBaselines bool,
 	skipSymbolBaselines bool,
 	hasErrorBaseline bool,
@@ -63,13 +64,13 @@ func checkBaselines(
 	allFiles []*TestFile,
 	fullWalker *typeWriterWalker,
 	header string,
-	opts Options,
+	opts baseline.Options,
 	isSymbolBaseline bool,
 ) {
 	fullExtension := core.IfElse(isSymbolBaseline, ".symbols", ".types")
 	outputFileName := tspath.RemoveFileExtension(baselinePath)
 	fullBaseline := generateBaseline(allFiles, fullWalker, header, isSymbolBaseline)
-	Run(t, outputFileName+fullExtension, fullBaseline, opts)
+	baseline.Run(t, outputFileName+fullExtension, fullBaseline, opts)
 }
 
 func generateBaseline(

internal/testutil/baseline/util.go renamed to internal/testutil/tsbaseline/util.go

@@ -1,4 +1,4 @@
-package baseline
+package tsbaseline
 
 import (
 	"regexp"