Skip to content

Compile benchmark with Dotty, fixes #29. #31

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Aug 23, 2017
2 changes: 1 addition & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ language: scala
env:
global:

script: sbt test:compile "hot -psource=scalap -w1 -f1" "micro/jmh:run -w1 -f1"
script: sbt testAll

jdk:
- oraclejdk8
Expand Down
29 changes: 26 additions & 3 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,21 @@ name := "compiler-benchmark"

version := "1.0-SNAPSHOT"

scalaVersion in ThisBuild := "2.11.8"
def scala211 = "2.11.11"
def dottyLatest = "0.2.0-RC1"
scalaVersion in ThisBuild := scala211

commands += Command.command("testAll") { s =>
"test:compile" ::
"compilation/test" ::
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should this be done on dotty too?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is done in Dotty as well, see L14.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

+compilation/test would not run on Dotty because it uses crossScalaVersions from the root project. Alternatives are

  • project compilation; +test ; project compiler-benchmark
  • add the sbt-doge plugin and run such compilation/test
  • add 0.2.0-RC1 to compiler-benchmark/crossScalaVersions, but that would be incorrect.

I opted for duplication since it's the dumbest solution.

"hot -psource=scalap -wi 1 -i 1 -f1" ::
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note. I bumped down the warmup iterations from 10 to 1 to make the tests run faster. The benchmark numbers on Travis are too unreliable anyways.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

s"++$dottyLatest" ::
"compilation/test" ::
"hot -psource=vector -wi 1 -i 1 -f1" ::
s"++$scala211" ::
"micro/jmh:run -w1 -f1" ::
s
}

resolvers += "scala-integration" at "https://scala-ci.typesafe.com/artifactory/scala-integration/"

Expand Down Expand Up @@ -36,8 +50,17 @@ lazy val compilation = addJmh(project).settings(
// We should be able to switch this project to a broad range of Scala versions for comparative
// benchmarking. As such, this project should only depend on the high level `MainClass` compiler API.
description := "Black box benchmark of the compiler",
libraryDependencies += scalaOrganization.value % "scala-compiler" % scalaVersion.value,
mainClass in (Jmh, run) := Some("scala.bench.ScalacBenchmarkRunner")
libraryDependencies += {
if (isDotty.value) "ch.epfl.lamp" %% "dotty-compiler" % scalaVersion.value
else scalaOrganization.value % "scala-compiler" % scalaVersion.value
},
crossScalaVersions := List(scala211, dottyLatest),
unmanagedSourceDirectories.in(Compile) +=
sourceDirectory.in(Compile).value / (if (isDotty.value) "dotc" else "scalac"),
mainClass in (Jmh, run) := Some("scala.bench.ScalacBenchmarkRunner"),
libraryDependencies += "com.novocode" % "junit-interface" % "0.11" % Test,
testOptions in Test += Tests.Argument(TestFrameworks.JUnit),
fork in (Test, test) := true // jmh scoped tasks run with fork := true.
).settings(addJavaOptions).dependsOn(infrastructure)

lazy val micro = addJmh(project).settings(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package scala.tools.benchmark

import java.io.File
import scala.tools.nsc.BaseBenchmarkDriver
import dotty.tools.dotc.core.Contexts.ContextBase

trait BenchmarkDriver extends BaseBenchmarkDriver {
def compileImpl(): Unit = {
implicit val ctx = new ContextBase().initialCtx.fresh
ctx.setSetting(ctx.settings.usejavacp, true)
if (depsClasspath != null) {
ctx.setSetting(ctx.settings.classpath,
depsClasspath.mkString(File.pathSeparator))
}
ctx.setSetting(ctx.settings.migration, true)
ctx.setSetting(ctx.settings.d, tempDir.getAbsolutePath)
ctx.setSetting(ctx.settings.language, List("Scala2"))
val compiler = new dotty.tools.dotc.Compiler
val reporter = dotty.tools.dotc.Bench.doCompile(compiler, allArgs)
assert(!reporter.hasErrors)
}
}
75 changes: 29 additions & 46 deletions compilation/src/main/scala/scala/tools/nsc/ScalacBenchmark.scala
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,23 @@ import org.openjdk.jmh.annotations.Mode._
import org.openjdk.jmh.annotations._

import scala.collection.JavaConverters._
import scala.tools.benchmark.BenchmarkDriver

trait BaseBenchmarkDriver {
def source: String
def extraArgs: String
def extras: List[String] = if (extraArgs != null && extraArgs != "") extraArgs.split('|').toList else Nil
def allArgs: List[String] = compilerArgs ++ extras ++ sourceFiles
def corpusVersion: String
def depsClasspath: String
def tempDir: File
def corpusSourcePath: Path
def compilerArgs: List[String]
def sourceFiles: List[String]
}

@State(Scope.Benchmark)
class ScalacBenchmark {
class ScalacBenchmark extends BenchmarkDriver {
@Param(value = Array())
var source: String = _

Expand All @@ -26,54 +40,23 @@ class ScalacBenchmark {
@Param(value = Array("latest"))
var corpusVersion: String = _

var driver: Driver = _

var depsClasspath: String = _

def compileImpl(): Unit = {
val (compilerArgs, sourceFiles) =
if (source.startsWith("@")) (List(source), List[String]())
else {
import scala.collection.JavaConverters._
val allFiles = Files.walk(findSourceDir, FileVisitOption.FOLLOW_LINKS).collect(Collectors.toList[Path]).asScala.toList
val files = allFiles.filter(f => {
val name = f.getFileName.toString
name.endsWith(".scala") || name.endsWith(".java")
}).map(_.toAbsolutePath.normalize.toString).toList
(List[String](), files)
}

// MainClass is copy-pasted from compiler for source compatibility with 2.10.x - 2.13.x
class MainClass extends Driver with EvalLoop {
def resident(compiler: Global): Unit = loop { line =>
val command = new CompilerCommand(line split "\\s+" toList, new Settings(scalacError))
compiler.reporter.reset()
new compiler.Run() compile command.files
}

override def newCompiler(): Global = Global(settings, reporter)

override protected def processSettingsHook(): Boolean = {
if (source == "scala")
settings.sourcepath.value = Paths.get(s"../corpus/$source/$corpusVersion/library").toAbsolutePath.normalize.toString
else
settings.usejavacp.value = true
settings.outdir.value = tempDir.getAbsolutePath
settings.nowarn.value = true
if (depsClasspath != null)
settings.processArgumentString(s"-cp $depsClasspath")
true
}
def compilerArgs: List[String] = if (source.startsWith("@")) source :: Nil else Nil

def sourceFiles: List[String] =
if (source.startsWith("@")) Nil
else {
import scala.collection.JavaConverters._
val allFiles = Files.walk(findSourceDir, FileVisitOption.FOLLOW_LINKS).collect(Collectors.toList[Path]).asScala.toList
val files = allFiles.filter(f => {
val name = f.getFileName.toString
name.endsWith(".scala") || name.endsWith(".java")
}).map(_.toAbsolutePath.normalize.toString)
files
}
val driver = new MainClass

val extras = if (extraArgs != null && extraArgs != "") extraArgs.split('|').toList else Nil
val allArgs = compilerArgs ++ extras ++ sourceFiles
driver.process(allArgs.toArray)
assert(!driver.reporter.hasErrors)
}

private var tempDir: File = null
var tempDir: File = null

// Executed once per fork
@Setup(Level.Trial) def initTemp(): Unit = {
Expand All @@ -86,7 +69,7 @@ class ScalacBenchmark {
BenchmarkUtils.deleteRecursive(tempDir.toPath)
}

private def corpusSourcePath = Paths.get(s"../corpus/$source/$corpusVersion")
def corpusSourcePath: Path = Paths.get(s"../corpus/$source/$corpusVersion")

@Setup(Level.Trial) def initDepsClasspath(): Unit = {
val classPath = BenchmarkUtils.initDeps(corpusSourcePath)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package scala.tools.benchmark

import java.nio.file._
import scala.tools.nsc._

trait BenchmarkDriver extends BaseBenchmarkDriver {
def compileImpl(): Unit = {
// MainClass is copy-pasted from compiler for source compatibility with 2.10.x - 2.13.x
class MainClass extends Driver with EvalLoop {
def resident(compiler: Global): Unit = loop { line =>
val command = new CompilerCommand(line split "\\s+" toList, new Settings(scalacError))
compiler.reporter.reset()
new compiler.Run() compile command.files
}

override def newCompiler(): Global = Global(settings, reporter)

override protected def processSettingsHook(): Boolean = {
if (source == "scala")
settings.sourcepath.value = Paths.get(s"../corpus/$source/$corpusVersion/library").toAbsolutePath.normalize.toString
else
settings.usejavacp.value = true
settings.outdir.value = tempDir.getAbsolutePath
settings.nowarn.value = true
if (depsClasspath != null)
settings.processArgumentString(s"-cp $depsClasspath")
true
}
}
val driver = new MainClass
driver.process(allArgs.toArray)
assert(!driver.reporter.hasErrors)
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package scala.tools.benchmark

import scala.tools.nsc.ScalacBenchmark
import org.junit.Test

class BenchmarkTest {
@Test def compilesOK() = {
val bench = new ScalacBenchmark
bench.source = "../corpus/vector"
bench.corpusVersion = "latest"
bench.initTemp()
bench.compileImpl()
bench.clearTemp()
}
}
2 changes: 1 addition & 1 deletion corpus/vector/fb04376/Vector.scala
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ override def companion: GenericCompanion[Vector] = Vector

override def lengthCompare(len: Int): Int = length - len

private[collection] final def initIterator[B >: A](s: VectorIterator[B]) {
private[collection] final def initIterator[B >: A](s: VectorIterator[B]): Unit = {
s.initFrom(this)
if (dirty) s.stabilize(focus)
if (s.depth > 1) s.gotoPos(startIndex, startIndex ^ focus)
Expand Down
3 changes: 3 additions & 0 deletions project/plugins.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,6 @@ logLevel := Level.Warn

// sbt-jmh plugin - pulls in JMH dependencies too
addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.2.25")

// sbt-dotty plugin - to support `scalaVersion := "0.x"`
addSbtPlugin("ch.epfl.lamp" % "sbt-dotty" % "0.1.2")