From c387ef28bca1fd07b481642a6ec57d1fd1f94645 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Sun, 12 Oct 2025 04:13:21 -0400 Subject: [PATCH 1/2] fix: Fix incremental compilation of macros **Problem** sbt-deps phase runs after the Typer, but prior to the Inliner. This means that the Zinc won't be able to track the symbolic dependency from the use site to the generated code, unless the code exists as a form of quotation. This often results in under-compilation of the macro use sites. **Solution** 1. Move the sbt-deps phase after the Inliner, which expands the macros 2. Traverse the Inlined tree, and treat it like a normal code --- compiler/src/dotty/tools/dotc/Compiler.scala | 2 +- .../dotty/tools/dotc/sbt/ExtractDependencies.scala | 6 ++++-- .../macro-expansion-dependencies-4/D0.scala | 3 +++ .../macro-expansion-dependencies-4/D1.scala | 3 +++ .../macro-expansion-dependencies-4/D2.scala | 3 +++ .../macro-expansion-dependencies-4/Test.scala | 12 ++++++++++++ .../macro-expansion-dependencies-4/build.sbt | 6 ++++++ .../macro-expansion-dependencies-4/changes/D0.scala | 3 +++ .../project/DottyInjectedPlugin.scala | 11 +++++++++++ .../macro-expansion-dependencies-4/test | 5 +++++ 10 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 sbt-test/source-dependencies/macro-expansion-dependencies-4/D0.scala create mode 100644 sbt-test/source-dependencies/macro-expansion-dependencies-4/D1.scala create mode 100644 sbt-test/source-dependencies/macro-expansion-dependencies-4/D2.scala create mode 100644 sbt-test/source-dependencies/macro-expansion-dependencies-4/Test.scala create mode 100644 sbt-test/source-dependencies/macro-expansion-dependencies-4/build.sbt create mode 100644 sbt-test/source-dependencies/macro-expansion-dependencies-4/changes/D0.scala create mode 100644 sbt-test/source-dependencies/macro-expansion-dependencies-4/project/DottyInjectedPlugin.scala create mode 100644 sbt-test/source-dependencies/macro-expansion-dependencies-4/test diff --git a/compiler/src/dotty/tools/dotc/Compiler.scala b/compiler/src/dotty/tools/dotc/Compiler.scala index 66044dd9462d..a23671e78c5a 100644 --- a/compiler/src/dotty/tools/dotc/Compiler.scala +++ b/compiler/src/dotty/tools/dotc/Compiler.scala @@ -35,7 +35,6 @@ class Compiler { List(new TyperPhase) :: // Compiler frontend: namer, typer List(CheckUnused.PostTyper(), CheckShadowing()) :: // Check for unused, shadowed elements List(new YCheckPositions) :: // YCheck positions - List(new sbt.ExtractDependencies) :: // Sends information on classes' dependencies to sbt via callbacks List(new semanticdb.ExtractSemanticDB.ExtractSemanticInfo) :: // Extract info into .semanticdb files List(new PostTyper) :: // Additional checks and cleanups after type checking List(new UnrollDefinitions) :: // Unroll annotated methods if detected in PostTyper @@ -48,6 +47,7 @@ class Compiler { List(new Pickler) :: // Generate TASTY info List(new sbt.ExtractAPI) :: // Sends a representation of the API of classes to sbt via callbacks List(new Inlining) :: // Inline and execute macros + List(new sbt.ExtractDependencies) :: // Sends information on classes' dependencies to sbt via callbacks List(new PostInlining) :: // Add mirror support for inlined code List(new Staging) :: // Check staging levels and heal staged types List(new Splicing) :: // Replace level 1 splices with holes diff --git a/compiler/src/dotty/tools/dotc/sbt/ExtractDependencies.scala b/compiler/src/dotty/tools/dotc/sbt/ExtractDependencies.scala index 8b3b217cb0fc..d86121a7fc3c 100644 --- a/compiler/src/dotty/tools/dotc/sbt/ExtractDependencies.scala +++ b/compiler/src/dotty/tools/dotc/sbt/ExtractDependencies.scala @@ -227,10 +227,12 @@ private class ExtractDependenciesCollector(rec: DependencyRecorder) extends tpd. } tree match { - case tree: Inlined if !tree.inlinedFromOuterScope => + case tree: Inlined => // The inlined call is normally ignored by TreeTraverser but we need to // record it as a dependency - traverse(tree.call) + if !tree.inlinedFromOuterScope then + traverse(tree.call) + traverseChildren(tree) case vd: ValDef if vd.symbol.is(ModuleVal) => // Don't visit module val case t: Template if t.symbol.owner.is(ModuleClass) => diff --git a/sbt-test/source-dependencies/macro-expansion-dependencies-4/D0.scala b/sbt-test/source-dependencies/macro-expansion-dependencies-4/D0.scala new file mode 100644 index 000000000000..f7b2b57929ed --- /dev/null +++ b/sbt-test/source-dependencies/macro-expansion-dependencies-4/D0.scala @@ -0,0 +1,3 @@ +package example + +class D0 diff --git a/sbt-test/source-dependencies/macro-expansion-dependencies-4/D1.scala b/sbt-test/source-dependencies/macro-expansion-dependencies-4/D1.scala new file mode 100644 index 000000000000..e72bb609ec16 --- /dev/null +++ b/sbt-test/source-dependencies/macro-expansion-dependencies-4/D1.scala @@ -0,0 +1,3 @@ +package example + +class D1 diff --git a/sbt-test/source-dependencies/macro-expansion-dependencies-4/D2.scala b/sbt-test/source-dependencies/macro-expansion-dependencies-4/D2.scala new file mode 100644 index 000000000000..06fcc2464380 --- /dev/null +++ b/sbt-test/source-dependencies/macro-expansion-dependencies-4/D2.scala @@ -0,0 +1,3 @@ +package example + +class D2 diff --git a/sbt-test/source-dependencies/macro-expansion-dependencies-4/Test.scala b/sbt-test/source-dependencies/macro-expansion-dependencies-4/Test.scala new file mode 100644 index 000000000000..248ba48583b1 --- /dev/null +++ b/sbt-test/source-dependencies/macro-expansion-dependencies-4/Test.scala @@ -0,0 +1,12 @@ +package example + +import com.softwaremill.macwire.* + +object Test: + try + val d = wire[D0] + val d1 = wire[D1] + val d2 = wire[D2] + catch + case e: Throwable => + e.printStackTrace() diff --git a/sbt-test/source-dependencies/macro-expansion-dependencies-4/build.sbt b/sbt-test/source-dependencies/macro-expansion-dependencies-4/build.sbt new file mode 100644 index 000000000000..f660b48ba4ba --- /dev/null +++ b/sbt-test/source-dependencies/macro-expansion-dependencies-4/build.sbt @@ -0,0 +1,6 @@ +name := "add-dep" +libraryDependencies ++= Seq( + "com.softwaremill.macwire" %% "macros" % "2.6.6" % Provided, + "com.softwaremill.macwire" %% "util" % "2.6.6", +) +Compile / incOptions ~= { _.withRecompileAllFraction(1.0) } diff --git a/sbt-test/source-dependencies/macro-expansion-dependencies-4/changes/D0.scala b/sbt-test/source-dependencies/macro-expansion-dependencies-4/changes/D0.scala new file mode 100644 index 000000000000..28fac7809f2a --- /dev/null +++ b/sbt-test/source-dependencies/macro-expansion-dependencies-4/changes/D0.scala @@ -0,0 +1,3 @@ +package example + +class D0(de1: D1) diff --git a/sbt-test/source-dependencies/macro-expansion-dependencies-4/project/DottyInjectedPlugin.scala b/sbt-test/source-dependencies/macro-expansion-dependencies-4/project/DottyInjectedPlugin.scala new file mode 100644 index 000000000000..fb946c4b8c61 --- /dev/null +++ b/sbt-test/source-dependencies/macro-expansion-dependencies-4/project/DottyInjectedPlugin.scala @@ -0,0 +1,11 @@ +import sbt._ +import Keys._ + +object DottyInjectedPlugin extends AutoPlugin { + override def requires = plugins.JvmPlugin + override def trigger = allRequirements + + override val projectSettings = Seq( + scalaVersion := sys.props("plugin.scalaVersion") + ) +} diff --git a/sbt-test/source-dependencies/macro-expansion-dependencies-4/test b/sbt-test/source-dependencies/macro-expansion-dependencies-4/test new file mode 100644 index 000000000000..3ea01e97c40c --- /dev/null +++ b/sbt-test/source-dependencies/macro-expansion-dependencies-4/test @@ -0,0 +1,5 @@ +> compile + +$ copy-file changes/D0.scala D0.scala + +-> compile From e9a61dfb94a2383175c9c13fa37543003c77fc69 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Sun, 19 Oct 2025 19:21:48 -0400 Subject: [PATCH 2/2] Split the dependency collection Instead of moving the entire sbt-deps phase, this leaves the phase as-is, and adds new instrumentation in the Inlining phase for the inlined code. --- compiler/src/dotty/tools/dotc/Compiler.scala | 2 +- .../tools/dotc/sbt/ExtractDependencies.scala | 101 ++++++++---------- .../dotty/tools/dotc/transform/Inlining.scala | 81 ++++++++++++-- 3 files changed, 122 insertions(+), 62 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/Compiler.scala b/compiler/src/dotty/tools/dotc/Compiler.scala index a23671e78c5a..66044dd9462d 100644 --- a/compiler/src/dotty/tools/dotc/Compiler.scala +++ b/compiler/src/dotty/tools/dotc/Compiler.scala @@ -35,6 +35,7 @@ class Compiler { List(new TyperPhase) :: // Compiler frontend: namer, typer List(CheckUnused.PostTyper(), CheckShadowing()) :: // Check for unused, shadowed elements List(new YCheckPositions) :: // YCheck positions + List(new sbt.ExtractDependencies) :: // Sends information on classes' dependencies to sbt via callbacks List(new semanticdb.ExtractSemanticDB.ExtractSemanticInfo) :: // Extract info into .semanticdb files List(new PostTyper) :: // Additional checks and cleanups after type checking List(new UnrollDefinitions) :: // Unroll annotated methods if detected in PostTyper @@ -47,7 +48,6 @@ class Compiler { List(new Pickler) :: // Generate TASTY info List(new sbt.ExtractAPI) :: // Sends a representation of the API of classes to sbt via callbacks List(new Inlining) :: // Inline and execute macros - List(new sbt.ExtractDependencies) :: // Sends information on classes' dependencies to sbt via callbacks List(new PostInlining) :: // Add mirror support for inlined code List(new Staging) :: // Check staging levels and heal staged types List(new Splicing) :: // Replace level 1 splices with holes diff --git a/compiler/src/dotty/tools/dotc/sbt/ExtractDependencies.scala b/compiler/src/dotty/tools/dotc/sbt/ExtractDependencies.scala index d86121a7fc3c..06e01b40acf6 100644 --- a/compiler/src/dotty/tools/dotc/sbt/ExtractDependencies.scala +++ b/compiler/src/dotty/tools/dotc/sbt/ExtractDependencies.scala @@ -5,7 +5,7 @@ import scala.language.unsafeNulls import java.io.File import java.nio.file.Path -import java.util.{Arrays, EnumSet} +import java.util.EnumSet import dotty.tools.dotc.ast.tpd import dotty.tools.dotc.classpath.FileUtils.{hasClassExtension, hasTastyExtension} @@ -46,7 +46,6 @@ import scala.compiletime.uninitialized * * The following flags affect this phase: * -Yforce-sbt-phases - * -Ydump-sbt-inc * * @see ExtractAPI */ @@ -77,27 +76,6 @@ class ExtractDependencies extends Phase { val rec = unit.depRecorder val collector = ExtractDependenciesCollector(rec) collector.traverse(unit.tpdTree) - - if (ctx.settings.YdumpSbtInc.value) { - val deps = rec.foundDeps.iterator.map { case (clazz, found) => s"$clazz: ${found.classesString}" }.toArray[Object] - val names = rec.foundDeps.iterator.map { case (clazz, found) => s"$clazz: ${found.namesString}" }.toArray[Object] - Arrays.sort(deps) - Arrays.sort(names) - - val pw = io.File(unit.source.file.jpath).changeExtension(FileExtension.Inc).toFile.printWriter() - // val pw = Console.out - try { - pw.println("Used Names:") - pw.println("===========") - names.foreach(pw.println) - pw.println() - pw.println("Dependencies:") - pw.println("=============") - deps.foreach(pw.println) - } finally pw.close() - } - - rec.sendToZinc() } } @@ -129,6 +107,47 @@ object ExtractDependencies { report.error(em"Internal error in the incremental compiler while compiling ${ctx.compilationUnit.source}: $msg", pos) } +/** Extract the dependency information of a compilation unit. + * + * This extracts the symbol dependencies in the written code. + * There are further extractions performed in the Inlining phase later. + * + * To understand why we track the used names see the section "Name hashing + * algorithm" in http://www.scala-sbt.org/0.13/docs/Understanding-Recompilation.html + * To understand why we need to track dependencies introduced by inheritance + * specially, see the subsection "Dependencies introduced by member reference and + * inheritance" in the "Name hashing algorithm" section. + */ +private class ExtractDependenciesCollector(rec: DependencyRecorder) extends AbstractExtractDependenciesCollector(rec): + import tpd.* + + /** Traverse the tree of a source file and record the dependencies and used names which + * can be retrieved using DependencyRecorder. + */ + override def traverse(tree: Tree)(using Context): Unit = + try + recordTree(tree) + tree match + case tree: Inlined if !tree.inlinedFromOuterScope => + // The inlined call is normally ignored by TreeTraverser but we need to + // record it as a dependency + traverse(tree.call) + // traverseChildren(tree) + case vd: ValDef if vd.symbol.is(ModuleVal) => + // Don't visit module val + case t: Template if t.symbol.owner.is(ModuleClass) => + // Don't visit self type of module class + traverse(t.constr) + t.parents.foreach(traverse) + t.body.foreach(traverse) + case _ => + traverseChildren(tree) + catch + case ex: AssertionError => + println(i"asserted failed while traversing $tree") + throw ex +end ExtractDependenciesCollector + /** Extract the dependency information of a compilation unit. * * To understand why we track the used names see the section "Name hashing @@ -137,7 +156,7 @@ object ExtractDependencies { * specially, see the subsection "Dependencies introduced by member reference and * inheritance" in the "Name hashing algorithm" section. */ -private class ExtractDependenciesCollector(rec: DependencyRecorder) extends tpd.TreeTraverser { thisTreeTraverser => +trait AbstractExtractDependenciesCollector(rec: DependencyRecorder) extends tpd.TreeTraverser { thisTreeTraverser => import tpd.* private def addMemberRefDependency(sym: Symbol)(using Context): Unit = @@ -181,12 +200,8 @@ private class ExtractDependenciesCollector(rec: DependencyRecorder) extends tpd. // can happen for constructor proxies. Test case is pos-macros/i13532. true - - /** Traverse the tree of a source file and record the dependencies and used names which - * can be retrieved using `foundDeps`. - */ - override def traverse(tree: Tree)(using Context): Unit = try { - tree match { + protected def recordTree(tree: Tree)(using Context): Unit = + tree match case Match(selector, _) => addPatMatDependency(selector.tpe) case Import(expr, selectors) => @@ -223,31 +238,7 @@ private class ExtractDependenciesCollector(rec: DependencyRecorder) extends tpd. addInheritanceDependencies(t) case t: Template => addInheritanceDependencies(t) - case _ => - } - - tree match { - case tree: Inlined => - // The inlined call is normally ignored by TreeTraverser but we need to - // record it as a dependency - if !tree.inlinedFromOuterScope then - traverse(tree.call) - traverseChildren(tree) - case vd: ValDef if vd.symbol.is(ModuleVal) => - // Don't visit module val - case t: Template if t.symbol.owner.is(ModuleClass) => - // Don't visit self type of module class - traverse(t.constr) - t.parents.foreach(traverse) - t.body.foreach(traverse) - case _ => - traverseChildren(tree) - } - } catch { - case ex: AssertionError => - println(i"asserted failed while traversing $tree") - throw ex - } + case _ => () /**Reused EqHashSet, safe to use as each TypeDependencyTraverser is used atomically * Avoid cycles by remembering both the types (testcase: diff --git a/compiler/src/dotty/tools/dotc/transform/Inlining.scala b/compiler/src/dotty/tools/dotc/transform/Inlining.scala index 751636c7d806..ffc92f7908d3 100644 --- a/compiler/src/dotty/tools/dotc/transform/Inlining.scala +++ b/compiler/src/dotty/tools/dotc/transform/Inlining.scala @@ -1,6 +1,10 @@ package dotty.tools.dotc package transform + +import java.util.Arrays + +import dotty.tools.io import ast.tpd import ast.Trees.* import ast.TreeMapWithTrackedStats @@ -15,12 +19,20 @@ import DenotTransformers.IdentityDenotTransformer import MacroAnnotations.hasMacroAnnotation import inlines.Inlines import quoted.* +import sbt.{ AbstractExtractDependenciesCollector, DependencyRecorder } import staging.StagingLevel import util.Property import scala.collection.mutable - -/** Inlines all calls to inline methods that are not in an inline method or a quote */ +import scala.io.Codec + +/** + * Inlines all calls to inline methods that are not in an inline method or a quote. + * + * The following flags affect this phase: + * -Ydump-sbt-inc + * + */ class Inlining extends MacroTransform, IdentityDenotTransformer { self => @@ -35,8 +47,32 @@ class Inlining extends MacroTransform, IdentityDenotTransformer { override def changesMembers: Boolean = true override def run(using Context): Unit = - if ctx.compilationUnit.needsInlining || ctx.compilationUnit.hasMacroAnnotations then + val unit = ctx.compilationUnit + val rec = ctx.compilationUnit.depRecorder + if unit.needsInlining || unit.hasMacroAnnotations then super.run + rec.sendToZinc() + + if ctx.settings.YdumpSbtInc.value then + val deps = rec.foundDeps.iterator.map { case (clazz, found) => s"$clazz: ${found.classesString}" }.toArray[Object] + val names = rec.foundDeps.iterator.map { case (clazz, found) => s"$clazz: ${found.namesString}" }.toArray[Object] + Arrays.sort(deps) + Arrays.sort(names) + + unit.source.file.jpath match + case jpath: io.JPath => + val pw = io.File(jpath)(using Codec.UTF8).changeExtension(io.FileExtension.Inc).toFile.printWriter() + // val pw = Console.out + try + pw.println("Used Names:") + pw.println("===========") + names.foreach(pw.println) + pw.println() + pw.println("Dependencies:") + pw.println("=============") + deps.foreach(pw.println) + finally pw.close() + case null => () override def checkPostCondition(tree: Tree)(using Context): Unit = tree match { @@ -54,18 +90,49 @@ class Inlining extends MacroTransform, IdentityDenotTransformer { def newTransformer(using Context): Transformer = new Transformer { override def transform(tree: tpd.Tree)(using Context): tpd.Tree = - InliningTreeMap().transform(tree) + val rec = ctx.compilationUnit.depRecorder + val collector = ExtractInlineDependenciesCollector(rec) + InliningTreeMap(collector).transform(tree) } - private class InliningTreeMap extends TreeMapWithTrackedStats { + private class ExtractInlineDependenciesCollector(rec: DependencyRecorder) extends AbstractExtractDependenciesCollector(rec): + /** Traverse the tree of a source file and record the dependencies and used names which + * can be retrieved using `rec`. + */ + override def traverse(tree: Tree)(using Context): Unit = + recordTree(tree) + traverseChildren(tree) + end ExtractInlineDependenciesCollector + + private class InliningTreeMap(collector: ExtractInlineDependenciesCollector) extends TreeMapWithTrackedStats { /** List of top level classes added by macro annotation in a package object. * These are added to the PackageDef that owns this particular package object. */ private val newTopClasses = MutableSymbolMap[mutable.ListBuffer[Tree]]() + val inlineFinder = new tpd.TreeTraverser: + override def traverse(tree: Tree)(using Context): Unit = + try + tree match + case tree: Inlined => + collector.traverse(tree) + case vd: ValDef if vd.symbol.is(ModuleVal) => + // Don't visit module val + case t: Template if t.symbol.owner.is(ModuleClass) => + // Don't visit self type of module class + traverse(t.constr) + t.parents.foreach(traverse) + t.body.foreach(traverse) + case _ => + traverseChildren(tree) + catch + case ex: AssertionError => + println(i"asserted failed while traversing $tree") + throw ex + override def transform(tree: Tree)(using Context): Tree = { - tree match + val result = tree match case tree: MemberDef => // Fetch the latest tracked tree (It might have already been transformed by its companion) transformMemberDef(getTracked(tree.symbol).getOrElse(tree)) @@ -93,6 +160,8 @@ class Inlining extends MacroTransform, IdentityDenotTransformer { if tree1.tpe.isError then tree1 else Inlines.inlineCall(tree1) else super.transform(tree) + inlineFinder.traverse(result) + result } private def transformMemberDef(tree: MemberDef)(using Context) : Tree =