Avoid inf recursion in provablyDisjointClasses (#22489)

Author: dwijnand

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -2983,11 +2983,11 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
    *
    *  1. Single inheritance of classes
    *  2. Final classes cannot be extended
-   *  3. ConstantTypes with distinct values are non intersecting
-   *  4. TermRefs with distinct values are non intersecting
+   *  3. ConstantTypes with distinct values are non-intersecting
+   *  4. TermRefs with distinct values are non-intersecting
    *  5. There is no value of type Nothing
    *
-   *  Note on soundness: the correctness of match types relies on on the
+   *  Note on soundness: the correctness of match types relies on the
    *  property that in all possible contexts, the same match type expression
    *  is either stuck or reduces to the same case.
    *
@@ -3206,22 +3206,38 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
     end match
   }
 
+  /** Are `cls1` and `cls1` provablyDisjoint classes, i.e., is `cls1 ⋔ cls2` true?
+   *
+   *  Note that "class" where includes traits, module classes, and (in the recursive case)
+   *  enum value term symbols.
+   */
   private def provablyDisjointClasses(cls1: Symbol, cls2: Symbol)(using Context): Boolean =
     def isDecomposable(cls: Symbol): Boolean =
       cls.is(Sealed) && !cls.hasAnonymousChild
 
     def decompose(cls: Symbol): List[Symbol] =
-      cls.children.flatMap { child =>
+      cls.children.map: child =>
         if child.isTerm then
-          child.info.classSymbols // allow enum vals to be decomposed to their enum class (then filtered out) and any mixins
-        else child :: Nil
-      }.filter(child => child.exists && child != cls)
+          // Enum vals with mixins, such as in i21860 or i22266,
+          // don't have a single class symbol.
+          // So instead of decomposing to NoSymbol
+          // (which leads to erroneously considering an enum type
+          // as disjoint from one of the mixin, eg. i21860.scala),
+          // or instead of decomposing to all the class symbols of
+          // the enum value (which leads to other mixins being decomposed,
+          // and infinite recursion, eg. i22266),
+          // we decompose to the enum value term symbol, and handle
+          // that within the rest of provablyDisjointClasses.
+          child.info.classSymbol.orElse(child)
+        else child
+      .filter(child => child.exists && child != cls)
 
     def eitherDerivesFromOther(cls1: Symbol, cls2: Symbol): Boolean =
       cls1.derivesFrom(cls2) || cls2.derivesFrom(cls1)
 
     def smallestNonTraitBase(cls: Symbol): Symbol =
-      cls.asClass.baseClasses.find(!_.is(Trait)).get
+      val classes = if cls.isClass then cls.asClass.baseClasses else cls.info.classSymbols
+      classes.find(!_.is(Trait)).get
 
     if (eitherDerivesFromOther(cls1, cls2))
       false

docs/_spec/03-types.md

@@ -1071,7 +1071,7 @@ The following properties hold about ´⌈X⌉´ (we have paper proofs for those)
 The "lower-bound rule" states that ´S <: T´ if ´T = q.X´ and ´q.X´ is a non-class type designator and ´S <: L´ where ´L´ is the lower bound of the underlying type definition of ´q.X´".
 That rule is known to break transitivy of subtyping in Scala already.
 
-Second, we define the relation ´⋔´ on *classes* (including traits and hidden classes of objects) as:
+Second, we define the relation ´⋔´ on *classes* (including traits, hidden classes of objects, and enum terms) as:
 
 - ´C ⋔ D´ if `´C ∉´ baseClasses´(D)´` and ´D´ is `final`
 - ´C ⋔ D´ if `´D ∉´ baseClasses´(C)´` and ´C´ is `final`

tests/pos/i22266.scala

@@ -0,0 +1,22 @@
+sealed trait NonPolygon
+sealed trait Polygon
+
+sealed trait SymmetryAspect
+sealed trait RotationalSymmetry      extends SymmetryAspect
+sealed trait MaybeRotationalSymmetry extends SymmetryAspect
+
+enum Shape:
+  case Circle   extends Shape with NonPolygon with RotationalSymmetry
+  case Triangle extends Shape with Polygon with MaybeRotationalSymmetry
+  case Square   extends Shape with Polygon with RotationalSymmetry
+
+object Shape:
+
+  def hasPolygon(
+    rotationalSyms: Vector[Shape & RotationalSymmetry],
+    maybeSyms: Vector[Shape & MaybeRotationalSymmetry]
+  ): Boolean =
+    val all = rotationalSyms.concat(maybeSyms)
+    all.exists:
+      case _: Polygon => true
+      case _          => false

tests/pos/i22266.unenum.scala

@@ -0,0 +1,22 @@
+sealed trait NonPolygon
+sealed trait Polygon
+
+sealed trait SymmetryAspect
+sealed trait RotationalSymmetry      extends SymmetryAspect
+sealed trait MaybeRotationalSymmetry extends SymmetryAspect
+
+sealed abstract class Shape
+
+object Shape:
+  case object Circle   extends Shape with NonPolygon with RotationalSymmetry
+  case object Triangle extends Shape with Polygon with MaybeRotationalSymmetry
+  case object Square   extends Shape with Polygon with RotationalSymmetry
+
+  def hasPolygon(
+    rotationalSyms: Vector[Shape & RotationalSymmetry],
+    maybeSyms: Vector[Shape & MaybeRotationalSymmetry]
+  ): Boolean =
+    val all = rotationalSyms.concat(maybeSyms)
+    all.exists:
+      case _: Polygon => true
+      case _          => false

tests/warn/i21860.unenum.scala

@@ -3,7 +3,7 @@ sealed trait Corners { self: Figure => }
 
 sealed abstract class Shape extends Figure
 object Shape:
-  case object Triange  extends Shape with Corners
+  case object Triangle extends Shape with Corners
   case object Square   extends Shape with Corners
   case object Circle   extends Shape
   case object Ellipsis extends Shape