diff --git a/akka-remote-tests/src/multi-jvm/scala/akka/remote/artery/BenchmarkFileReporter.scala b/akka-remote-tests/src/multi-jvm/scala/akka/remote/artery/BenchmarkFileReporter.scala new file mode 100644 index 0000000000..11b8fea0df --- /dev/null +++ b/akka-remote-tests/src/multi-jvm/scala/akka/remote/artery/BenchmarkFileReporter.scala @@ -0,0 +1,79 @@ +/** + * Copyright (C) 2016 Lightbend Inc. + */ +package akka.remote.artery + +import java.io.File +import java.io.FileOutputStream +import java.text.SimpleDateFormat +import java.util.Date + +import akka.actor.ActorSystem + +import scala.util.Try + +/** + * Simple to file logger for benchmark results. Will log relevant settings first to make sure + * results can be understood later. + */ +trait BenchmarkFileReporter { + def reportResults(result: String): Unit + def close(): Unit +} +object BenchmarkFileReporter { + val targetDirectory = { + val target = new File("akka-stream-tests/target/benchmark-results") + target.mkdirs() + target + } + + def apply(testName: String, system: ActorSystem): BenchmarkFileReporter = + new BenchmarkFileReporter { + val gitCommit = { + import sys.process._ + Try("git describe".!!.trim).getOrElse("[unknown]") + } + val testResultFile: File = { + val format = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss") + val fileName = s"${format.format(new Date())}-Artery-$testName-$gitCommit-results.txt" + new File(targetDirectory, fileName) + } + val config = system.settings.config + + val fos = new FileOutputStream(testResultFile) + reportResults(s"Git commit: $gitCommit") + + val settingsToReport = + Seq( + "akka.test.MaxThroughputSpec.totalMessagesFactor", + "akka.test.MaxThroughputSpec.real-message", + "akka.test.LatencySpec.totalMessagesFactor", + "akka.test.LatencySpec.repeatCount", + "akka.test.LatencySpec.real-message", + "akka.remote.artery.enabled", + "akka.remote.artery.advanced.inbound-lanes", + "akka.remote.artery.advanced.idle-cpu-level", + "akka.remote.artery.advanced.buffer-pool-size", + "akka.remote.artery.advanced.embedded-media-driver", + "akka.remote.default-remote-dispatcher.throughput", + "akka.remote.default-remote-dispatcher.fork-join-executor.parallelism-factor", + "akka.remote.default-remote-dispatcher.fork-join-executor.parallelism-min", + "akka.remote.default-remote-dispatcher.fork-join-executor.parallelism-max" + ) + settingsToReport.foreach(reportSetting) + + def reportResults(result: String): Unit = synchronized { + println(result) + fos.write(result.getBytes("utf8")) + fos.write('\n') + fos.flush() + } + + def reportSetting(name: String): Unit = { + val value = if (config.hasPath(name)) config.getString(name) else "[unset]" + reportResults(s"$name: $value") + } + + def close(): Unit = fos.close() + } +} \ No newline at end of file diff --git a/akka-remote-tests/src/multi-jvm/scala/akka/remote/artery/LatencySpec.scala b/akka-remote-tests/src/multi-jvm/scala/akka/remote/artery/LatencySpec.scala index 5e40c11521..2abcaa0d0e 100644 --- a/akka-remote-tests/src/multi-jvm/scala/akka/remote/artery/LatencySpec.scala +++ b/akka-remote-tests/src/multi-jvm/scala/akka/remote/artery/LatencySpec.scala @@ -83,11 +83,11 @@ object LatencySpec extends MultiNodeConfig { } def receiverProps(reporter: RateReporter, settings: TestSettings, totalMessages: Int, - sendTimes: AtomicLongArray, histogram: Histogram, plotsRef: ActorRef): Props = - Props(new Receiver(reporter, settings, totalMessages, sendTimes, histogram, plotsRef)) + sendTimes: AtomicLongArray, histogram: Histogram, plotsRef: ActorRef, BenchmarkFileReporter: BenchmarkFileReporter): Props = + Props(new Receiver(reporter, settings, totalMessages, sendTimes, histogram, plotsRef, BenchmarkFileReporter)) class Receiver(reporter: RateReporter, settings: TestSettings, totalMessages: Int, - sendTimes: AtomicLongArray, histogram: Histogram, plotsRef: ActorRef) extends Actor { + sendTimes: AtomicLongArray, histogram: Histogram, plotsRef: ActorRef, BenchmarkFileReporter: BenchmarkFileReporter) extends Actor { import settings._ var count = 0 @@ -122,12 +122,12 @@ object LatencySpec extends MultiNodeConfig { } } if (count == totalMessages) { - printTotal(testName, size, histogram, System.nanoTime() - startTime) + printTotal(testName, size, histogram, System.nanoTime() - startTime, BenchmarkFileReporter) context.stop(self) } } - def printTotal(testName: String, payloadSize: Long, histogram: Histogram, totalDurationNanos: Long): Unit = { + def printTotal(testName: String, payloadSize: Long, histogram: Histogram, totalDurationNanos: Long, reporter: BenchmarkFileReporter): Unit = { import scala.collection.JavaConverters._ val percentiles = histogram.percentiles(5) def percentile(p: Double): Double = @@ -138,7 +138,7 @@ object LatencySpec extends MultiNodeConfig { val throughput = 1000.0 * histogram.getTotalCount / math.max(1, totalDurationNanos.nanos.toMillis) - println(s"=== Latency $testName: RTT " + + reporter.reportResults(s"=== Latency $testName: RTT " + f"50%%ile: ${percentile(50.0)}%.0f µs, " + f"90%%ile: ${percentile(90.0)}%.0f µs, " + f"99%%ile: ${percentile(99.0)}%.0f µs, " + @@ -244,7 +244,7 @@ abstract class LatencySpec repeat = repeatCount, realMessage)) - def test(testSettings: TestSettings): Unit = { + def test(testSettings: TestSettings, BenchmarkFileReporter: BenchmarkFileReporter): Unit = { import testSettings._ runOn(first) { @@ -271,7 +271,7 @@ abstract class LatencySpec echo ! Reset expectMsg(Reset) histogram.reset() - val receiver = system.actorOf(receiverProps(rep, testSettings, totalMessages, sendTimes, histogram, plotProbe.ref)) + val receiver = system.actorOf(receiverProps(rep, testSettings, totalMessages, sendTimes, histogram, plotProbe.ref, BenchmarkFileReporter)) // warmup for 3 seconds to init compression val warmup = Source(1 to 30) @@ -335,6 +335,7 @@ abstract class LatencySpec } "Latency of Artery" must { + val reporter = BenchmarkFileReporter("LatencySpec", system) "start echo" in { runOn(second) { @@ -345,7 +346,7 @@ abstract class LatencySpec } for (s ← scenarios) { - s"be low for ${s.testName}, at ${s.messageRate} msg/s, payloadSize = ${s.payloadSize}" in test(s) + s"be low for ${s.testName}, at ${s.messageRate} msg/s, payloadSize = ${s.payloadSize}" in test(s, reporter) } // TODO add more tests diff --git a/akka-remote-tests/src/multi-jvm/scala/akka/remote/artery/MaxThroughputSpec.scala b/akka-remote-tests/src/multi-jvm/scala/akka/remote/artery/MaxThroughputSpec.scala index 1a91420b7c..51605c6bad 100644 --- a/akka-remote-tests/src/multi-jvm/scala/akka/remote/artery/MaxThroughputSpec.scala +++ b/akka-remote-tests/src/multi-jvm/scala/akka/remote/artery/MaxThroughputSpec.scala @@ -3,13 +3,16 @@ */ package akka.remote.artery +import java.io.FileOutputStream import java.nio.ByteBuffer +import java.text.SimpleDateFormat +import java.util.Date import java.util.concurrent.Executors import java.util.concurrent.TimeUnit.NANOSECONDS import scala.concurrent.duration._ import akka.actor._ -import akka.remote.{ RemotingMultiNodeSpec, RARP, RemoteActorRefProvider } +import akka.remote.{ RARP, RemoteActorRefProvider, RemotingMultiNodeSpec } import akka.remote.testconductor.RoleName import akka.remote.testkit.MultiNodeConfig import akka.remote.testkit.MultiNodeSpec @@ -108,10 +111,10 @@ object MaxThroughputSpec extends MultiNodeConfig { } def senderProps(target: ActorRef, testSettings: TestSettings, plotRef: ActorRef, - printTaskRunnerMetrics: Boolean): Props = - Props(new Sender(target, testSettings, plotRef, printTaskRunnerMetrics)) + printTaskRunnerMetrics: Boolean, reporter: BenchmarkFileReporter): Props = + Props(new Sender(target, testSettings, plotRef, printTaskRunnerMetrics, reporter)) - class Sender(target: ActorRef, testSettings: TestSettings, plotRef: ActorRef, printTaskRunnerMetrics: Boolean) + class Sender(target: ActorRef, testSettings: TestSettings, plotRef: ActorRef, printTaskRunnerMetrics: Boolean, reporter: BenchmarkFileReporter) extends Actor { import testSettings._ val payload = ("0" * testSettings.payloadSize).getBytes("utf-8") @@ -176,7 +179,8 @@ object MaxThroughputSpec extends MultiNodeConfig { case EndResult(totalReceived) ⇒ val took = NANOSECONDS.toMillis(System.nanoTime - startTime) val throughput = (totalReceived * 1000.0 / took) - println( + + reporter.reportResults( s"=== MaxThroughput ${self.path.name}: " + f"throughput ${throughput * testSettings.senderReceiverPairs}%,.0f msg/s, " + f"${throughput * payloadSize * testSettings.senderReceiverPairs}%,.0f bytes/s (payload), " + @@ -351,7 +355,7 @@ abstract class MaxThroughputSpec extends RemotingMultiNodeSpec(MaxThroughputSpec senderReceiverPairs = 5, realMessage)) - def test(testSettings: TestSettings): Unit = { + def test(testSettings: TestSettings, resultReporter: BenchmarkFileReporter): Unit = { import testSettings._ val receiverName = testName + "-rcv" @@ -376,7 +380,7 @@ abstract class MaxThroughputSpec extends RemotingMultiNodeSpec(MaxThroughputSpec val receiver = identifyReceiver(receiverName + n) val plotProbe = TestProbe() val snd = system.actorOf( - senderProps(receiver, testSettings, plotProbe.ref, printTaskRunnerMetrics = n == 1), + senderProps(receiver, testSettings, plotProbe.ref, printTaskRunnerMetrics = n == 1, resultReporter), testName + "-snd" + n) val terminationProbe = TestProbe() terminationProbe.watch(snd) @@ -399,10 +403,9 @@ abstract class MaxThroughputSpec extends RemotingMultiNodeSpec(MaxThroughputSpec } "Max throughput of Artery" must { - + val reporter = BenchmarkFileReporter("MaxThroughputSpec", system) for (s ← scenarios) { - s"be great for ${s.testName}, burstSize = ${s.burstSize}, payloadSize = ${s.payloadSize}" in test(s) + s"be great for ${s.testName}, burstSize = ${s.burstSize}, payloadSize = ${s.payloadSize}" in test(s, reporter) } - } }