From 418b11d5539bfa69eabdcce5f1ad0e77550a5154 Mon Sep 17 00:00:00 2001 From: Roland Date: Wed, 14 Sep 2011 16:09:17 +0200 Subject: [PATCH 01/36] first step towards TestConductor - it compiles - server side ("Conductor") functions almost there - client side ("Player") sketched, but missing network failures - no internal failure handling whatsoever, waiting for Project DeathWatch - not yet possible to shutdown, need to kill VM - next step is to hook into the NettyRemoteSupport for failure injection --- .../testconductor/TestConductorProtocol.java | 2610 +++++++++++++++++ .../main/protocol/TestConductorProtocol.proto | 48 + .../akka/remote/testconductor/Conductor.scala | 268 ++ .../akka/remote/testconductor/DataTypes.scala | 13 + .../akka/remote/testconductor/Features.scala | 74 + .../NetworkFailureInjector.scala | 161 + .../akka/remote/testconductor/Player.scala | 133 + .../testconductor/RemoteConnection.scala | 52 + 8 files changed, 3359 insertions(+) create mode 100644 akka-remote/src/main/java/akka/remote/testconductor/TestConductorProtocol.java create mode 100644 akka-remote/src/main/protocol/TestConductorProtocol.proto create mode 100644 akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala create mode 100644 akka-remote/src/main/scala/akka/remote/testconductor/DataTypes.scala create mode 100644 akka-remote/src/main/scala/akka/remote/testconductor/Features.scala create mode 100644 akka-remote/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala create mode 100644 akka-remote/src/main/scala/akka/remote/testconductor/Player.scala create mode 100644 akka-remote/src/main/scala/akka/remote/testconductor/RemoteConnection.scala diff --git a/akka-remote/src/main/java/akka/remote/testconductor/TestConductorProtocol.java b/akka-remote/src/main/java/akka/remote/testconductor/TestConductorProtocol.java new file mode 100644 index 0000000000..e9065b53e4 --- /dev/null +++ b/akka-remote/src/main/java/akka/remote/testconductor/TestConductorProtocol.java @@ -0,0 +1,2610 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: TestConductorProtocol.proto + +package akka.remote.testconductor; + +public final class TestConductorProtocol { + private TestConductorProtocol() {} + public static void registerAllExtensions( + com.google.protobuf.ExtensionRegistry registry) { + } + public enum FailType + implements com.google.protobuf.ProtocolMessageEnum { + Throttle(0, 1), + Disconnect(1, 2), + Abort(2, 3), + Shutdown(3, 4), + ; + + public static final int Throttle_VALUE = 1; + public static final int Disconnect_VALUE = 2; + public static final int Abort_VALUE = 3; + public static final int Shutdown_VALUE = 4; + + + public final int getNumber() { return value; } + + public static FailType valueOf(int value) { + switch (value) { + case 1: return Throttle; + case 2: return Disconnect; + case 3: return Abort; + case 4: return Shutdown; + default: return null; + } + } + + public static com.google.protobuf.Internal.EnumLiteMap + internalGetValueMap() { + return internalValueMap; + } + private static com.google.protobuf.Internal.EnumLiteMap + internalValueMap = + new com.google.protobuf.Internal.EnumLiteMap() { + public FailType findValueByNumber(int number) { + return FailType.valueOf(number); + } + }; + + public final com.google.protobuf.Descriptors.EnumValueDescriptor + getValueDescriptor() { + return getDescriptor().getValues().get(index); + } + public final com.google.protobuf.Descriptors.EnumDescriptor + getDescriptorForType() { + return getDescriptor(); + } + public static final com.google.protobuf.Descriptors.EnumDescriptor + getDescriptor() { + return akka.remote.testconductor.TestConductorProtocol.getDescriptor().getEnumTypes().get(0); + } + + private static final FailType[] VALUES = { + Throttle, Disconnect, Abort, Shutdown, + }; + + public static FailType valueOf( + com.google.protobuf.Descriptors.EnumValueDescriptor desc) { + if (desc.getType() != getDescriptor()) { + throw new java.lang.IllegalArgumentException( + "EnumValueDescriptor is not for this type."); + } + return VALUES[desc.getIndex()]; + } + + private final int index; + private final int value; + + private FailType(int index, int value) { + this.index = index; + this.value = value; + } + + // @@protoc_insertion_point(enum_scope:FailType) + } + + public enum Direction + implements com.google.protobuf.ProtocolMessageEnum { + Send(0, 1), + Receive(1, 2), + ; + + public static final int Send_VALUE = 1; + public static final int Receive_VALUE = 2; + + + public final int getNumber() { return value; } + + public static Direction valueOf(int value) { + switch (value) { + case 1: return Send; + case 2: return Receive; + default: return null; + } + } + + public static com.google.protobuf.Internal.EnumLiteMap + internalGetValueMap() { + return internalValueMap; + } + private static com.google.protobuf.Internal.EnumLiteMap + internalValueMap = + new com.google.protobuf.Internal.EnumLiteMap() { + public Direction findValueByNumber(int number) { + return Direction.valueOf(number); + } + }; + + public final com.google.protobuf.Descriptors.EnumValueDescriptor + getValueDescriptor() { + return getDescriptor().getValues().get(index); + } + public final com.google.protobuf.Descriptors.EnumDescriptor + getDescriptorForType() { + return getDescriptor(); + } + public static final com.google.protobuf.Descriptors.EnumDescriptor + getDescriptor() { + return akka.remote.testconductor.TestConductorProtocol.getDescriptor().getEnumTypes().get(1); + } + + private static final Direction[] VALUES = { + Send, Receive, + }; + + public static Direction valueOf( + com.google.protobuf.Descriptors.EnumValueDescriptor desc) { + if (desc.getType() != getDescriptor()) { + throw new java.lang.IllegalArgumentException( + "EnumValueDescriptor is not for this type."); + } + return VALUES[desc.getIndex()]; + } + + private final int index; + private final int value; + + private Direction(int index, int value) { + this.index = index; + this.value = value; + } + + // @@protoc_insertion_point(enum_scope:Direction) + } + + public interface WrapperOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // optional .Hello hello = 1; + boolean hasHello(); + akka.remote.testconductor.TestConductorProtocol.Hello getHello(); + akka.remote.testconductor.TestConductorProtocol.HelloOrBuilder getHelloOrBuilder(); + + // optional .EnterBarrier barrier = 2; + boolean hasBarrier(); + akka.remote.testconductor.TestConductorProtocol.EnterBarrier getBarrier(); + akka.remote.testconductor.TestConductorProtocol.EnterBarrierOrBuilder getBarrierOrBuilder(); + + // optional .InjectFailure failure = 3; + boolean hasFailure(); + akka.remote.testconductor.TestConductorProtocol.InjectFailure getFailure(); + akka.remote.testconductor.TestConductorProtocol.InjectFailureOrBuilder getFailureOrBuilder(); + } + public static final class Wrapper extends + com.google.protobuf.GeneratedMessage + implements WrapperOrBuilder { + // Use Wrapper.newBuilder() to construct. + private Wrapper(Builder builder) { + super(builder); + } + private Wrapper(boolean noInit) {} + + private static final Wrapper defaultInstance; + public static Wrapper getDefaultInstance() { + return defaultInstance; + } + + public Wrapper getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return akka.remote.testconductor.TestConductorProtocol.internal_static_Wrapper_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return akka.remote.testconductor.TestConductorProtocol.internal_static_Wrapper_fieldAccessorTable; + } + + private int bitField0_; + // optional .Hello hello = 1; + public static final int HELLO_FIELD_NUMBER = 1; + private akka.remote.testconductor.TestConductorProtocol.Hello hello_; + public boolean hasHello() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public akka.remote.testconductor.TestConductorProtocol.Hello getHello() { + return hello_; + } + public akka.remote.testconductor.TestConductorProtocol.HelloOrBuilder getHelloOrBuilder() { + return hello_; + } + + // optional .EnterBarrier barrier = 2; + public static final int BARRIER_FIELD_NUMBER = 2; + private akka.remote.testconductor.TestConductorProtocol.EnterBarrier barrier_; + public boolean hasBarrier() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public akka.remote.testconductor.TestConductorProtocol.EnterBarrier getBarrier() { + return barrier_; + } + public akka.remote.testconductor.TestConductorProtocol.EnterBarrierOrBuilder getBarrierOrBuilder() { + return barrier_; + } + + // optional .InjectFailure failure = 3; + public static final int FAILURE_FIELD_NUMBER = 3; + private akka.remote.testconductor.TestConductorProtocol.InjectFailure failure_; + public boolean hasFailure() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public akka.remote.testconductor.TestConductorProtocol.InjectFailure getFailure() { + return failure_; + } + public akka.remote.testconductor.TestConductorProtocol.InjectFailureOrBuilder getFailureOrBuilder() { + return failure_; + } + + private void initFields() { + hello_ = akka.remote.testconductor.TestConductorProtocol.Hello.getDefaultInstance(); + barrier_ = akka.remote.testconductor.TestConductorProtocol.EnterBarrier.getDefaultInstance(); + failure_ = akka.remote.testconductor.TestConductorProtocol.InjectFailure.getDefaultInstance(); + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (hasHello()) { + if (!getHello().isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + if (hasBarrier()) { + if (!getBarrier().isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + if (hasFailure()) { + if (!getFailure().isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeMessage(1, hello_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeMessage(2, barrier_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeMessage(3, failure_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(1, hello_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(2, barrier_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(3, failure_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static akka.remote.testconductor.TestConductorProtocol.Wrapper parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.Wrapper parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.Wrapper parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.Wrapper parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.Wrapper parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.Wrapper parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.Wrapper parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static akka.remote.testconductor.TestConductorProtocol.Wrapper parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static akka.remote.testconductor.TestConductorProtocol.Wrapper parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.Wrapper parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(akka.remote.testconductor.TestConductorProtocol.Wrapper prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements akka.remote.testconductor.TestConductorProtocol.WrapperOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return akka.remote.testconductor.TestConductorProtocol.internal_static_Wrapper_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return akka.remote.testconductor.TestConductorProtocol.internal_static_Wrapper_fieldAccessorTable; + } + + // Construct using akka.remote.testconductor.TestConductorProtocol.Wrapper.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + getHelloFieldBuilder(); + getBarrierFieldBuilder(); + getFailureFieldBuilder(); + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + if (helloBuilder_ == null) { + hello_ = akka.remote.testconductor.TestConductorProtocol.Hello.getDefaultInstance(); + } else { + helloBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000001); + if (barrierBuilder_ == null) { + barrier_ = akka.remote.testconductor.TestConductorProtocol.EnterBarrier.getDefaultInstance(); + } else { + barrierBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000002); + if (failureBuilder_ == null) { + failure_ = akka.remote.testconductor.TestConductorProtocol.InjectFailure.getDefaultInstance(); + } else { + failureBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return akka.remote.testconductor.TestConductorProtocol.Wrapper.getDescriptor(); + } + + public akka.remote.testconductor.TestConductorProtocol.Wrapper getDefaultInstanceForType() { + return akka.remote.testconductor.TestConductorProtocol.Wrapper.getDefaultInstance(); + } + + public akka.remote.testconductor.TestConductorProtocol.Wrapper build() { + akka.remote.testconductor.TestConductorProtocol.Wrapper result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private akka.remote.testconductor.TestConductorProtocol.Wrapper buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + akka.remote.testconductor.TestConductorProtocol.Wrapper result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public akka.remote.testconductor.TestConductorProtocol.Wrapper buildPartial() { + akka.remote.testconductor.TestConductorProtocol.Wrapper result = new akka.remote.testconductor.TestConductorProtocol.Wrapper(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + if (helloBuilder_ == null) { + result.hello_ = hello_; + } else { + result.hello_ = helloBuilder_.build(); + } + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + if (barrierBuilder_ == null) { + result.barrier_ = barrier_; + } else { + result.barrier_ = barrierBuilder_.build(); + } + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + if (failureBuilder_ == null) { + result.failure_ = failure_; + } else { + result.failure_ = failureBuilder_.build(); + } + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof akka.remote.testconductor.TestConductorProtocol.Wrapper) { + return mergeFrom((akka.remote.testconductor.TestConductorProtocol.Wrapper)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(akka.remote.testconductor.TestConductorProtocol.Wrapper other) { + if (other == akka.remote.testconductor.TestConductorProtocol.Wrapper.getDefaultInstance()) return this; + if (other.hasHello()) { + mergeHello(other.getHello()); + } + if (other.hasBarrier()) { + mergeBarrier(other.getBarrier()); + } + if (other.hasFailure()) { + mergeFailure(other.getFailure()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + if (hasHello()) { + if (!getHello().isInitialized()) { + + return false; + } + } + if (hasBarrier()) { + if (!getBarrier().isInitialized()) { + + return false; + } + } + if (hasFailure()) { + if (!getFailure().isInitialized()) { + + return false; + } + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 10: { + akka.remote.testconductor.TestConductorProtocol.Hello.Builder subBuilder = akka.remote.testconductor.TestConductorProtocol.Hello.newBuilder(); + if (hasHello()) { + subBuilder.mergeFrom(getHello()); + } + input.readMessage(subBuilder, extensionRegistry); + setHello(subBuilder.buildPartial()); + break; + } + case 18: { + akka.remote.testconductor.TestConductorProtocol.EnterBarrier.Builder subBuilder = akka.remote.testconductor.TestConductorProtocol.EnterBarrier.newBuilder(); + if (hasBarrier()) { + subBuilder.mergeFrom(getBarrier()); + } + input.readMessage(subBuilder, extensionRegistry); + setBarrier(subBuilder.buildPartial()); + break; + } + case 26: { + akka.remote.testconductor.TestConductorProtocol.InjectFailure.Builder subBuilder = akka.remote.testconductor.TestConductorProtocol.InjectFailure.newBuilder(); + if (hasFailure()) { + subBuilder.mergeFrom(getFailure()); + } + input.readMessage(subBuilder, extensionRegistry); + setFailure(subBuilder.buildPartial()); + break; + } + } + } + } + + private int bitField0_; + + // optional .Hello hello = 1; + private akka.remote.testconductor.TestConductorProtocol.Hello hello_ = akka.remote.testconductor.TestConductorProtocol.Hello.getDefaultInstance(); + private com.google.protobuf.SingleFieldBuilder< + akka.remote.testconductor.TestConductorProtocol.Hello, akka.remote.testconductor.TestConductorProtocol.Hello.Builder, akka.remote.testconductor.TestConductorProtocol.HelloOrBuilder> helloBuilder_; + public boolean hasHello() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public akka.remote.testconductor.TestConductorProtocol.Hello getHello() { + if (helloBuilder_ == null) { + return hello_; + } else { + return helloBuilder_.getMessage(); + } + } + public Builder setHello(akka.remote.testconductor.TestConductorProtocol.Hello value) { + if (helloBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + hello_ = value; + onChanged(); + } else { + helloBuilder_.setMessage(value); + } + bitField0_ |= 0x00000001; + return this; + } + public Builder setHello( + akka.remote.testconductor.TestConductorProtocol.Hello.Builder builderForValue) { + if (helloBuilder_ == null) { + hello_ = builderForValue.build(); + onChanged(); + } else { + helloBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000001; + return this; + } + public Builder mergeHello(akka.remote.testconductor.TestConductorProtocol.Hello value) { + if (helloBuilder_ == null) { + if (((bitField0_ & 0x00000001) == 0x00000001) && + hello_ != akka.remote.testconductor.TestConductorProtocol.Hello.getDefaultInstance()) { + hello_ = + akka.remote.testconductor.TestConductorProtocol.Hello.newBuilder(hello_).mergeFrom(value).buildPartial(); + } else { + hello_ = value; + } + onChanged(); + } else { + helloBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000001; + return this; + } + public Builder clearHello() { + if (helloBuilder_ == null) { + hello_ = akka.remote.testconductor.TestConductorProtocol.Hello.getDefaultInstance(); + onChanged(); + } else { + helloBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000001); + return this; + } + public akka.remote.testconductor.TestConductorProtocol.Hello.Builder getHelloBuilder() { + bitField0_ |= 0x00000001; + onChanged(); + return getHelloFieldBuilder().getBuilder(); + } + public akka.remote.testconductor.TestConductorProtocol.HelloOrBuilder getHelloOrBuilder() { + if (helloBuilder_ != null) { + return helloBuilder_.getMessageOrBuilder(); + } else { + return hello_; + } + } + private com.google.protobuf.SingleFieldBuilder< + akka.remote.testconductor.TestConductorProtocol.Hello, akka.remote.testconductor.TestConductorProtocol.Hello.Builder, akka.remote.testconductor.TestConductorProtocol.HelloOrBuilder> + getHelloFieldBuilder() { + if (helloBuilder_ == null) { + helloBuilder_ = new com.google.protobuf.SingleFieldBuilder< + akka.remote.testconductor.TestConductorProtocol.Hello, akka.remote.testconductor.TestConductorProtocol.Hello.Builder, akka.remote.testconductor.TestConductorProtocol.HelloOrBuilder>( + hello_, + getParentForChildren(), + isClean()); + hello_ = null; + } + return helloBuilder_; + } + + // optional .EnterBarrier barrier = 2; + private akka.remote.testconductor.TestConductorProtocol.EnterBarrier barrier_ = akka.remote.testconductor.TestConductorProtocol.EnterBarrier.getDefaultInstance(); + private com.google.protobuf.SingleFieldBuilder< + akka.remote.testconductor.TestConductorProtocol.EnterBarrier, akka.remote.testconductor.TestConductorProtocol.EnterBarrier.Builder, akka.remote.testconductor.TestConductorProtocol.EnterBarrierOrBuilder> barrierBuilder_; + public boolean hasBarrier() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public akka.remote.testconductor.TestConductorProtocol.EnterBarrier getBarrier() { + if (barrierBuilder_ == null) { + return barrier_; + } else { + return barrierBuilder_.getMessage(); + } + } + public Builder setBarrier(akka.remote.testconductor.TestConductorProtocol.EnterBarrier value) { + if (barrierBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + barrier_ = value; + onChanged(); + } else { + barrierBuilder_.setMessage(value); + } + bitField0_ |= 0x00000002; + return this; + } + public Builder setBarrier( + akka.remote.testconductor.TestConductorProtocol.EnterBarrier.Builder builderForValue) { + if (barrierBuilder_ == null) { + barrier_ = builderForValue.build(); + onChanged(); + } else { + barrierBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000002; + return this; + } + public Builder mergeBarrier(akka.remote.testconductor.TestConductorProtocol.EnterBarrier value) { + if (barrierBuilder_ == null) { + if (((bitField0_ & 0x00000002) == 0x00000002) && + barrier_ != akka.remote.testconductor.TestConductorProtocol.EnterBarrier.getDefaultInstance()) { + barrier_ = + akka.remote.testconductor.TestConductorProtocol.EnterBarrier.newBuilder(barrier_).mergeFrom(value).buildPartial(); + } else { + barrier_ = value; + } + onChanged(); + } else { + barrierBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000002; + return this; + } + public Builder clearBarrier() { + if (barrierBuilder_ == null) { + barrier_ = akka.remote.testconductor.TestConductorProtocol.EnterBarrier.getDefaultInstance(); + onChanged(); + } else { + barrierBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000002); + return this; + } + public akka.remote.testconductor.TestConductorProtocol.EnterBarrier.Builder getBarrierBuilder() { + bitField0_ |= 0x00000002; + onChanged(); + return getBarrierFieldBuilder().getBuilder(); + } + public akka.remote.testconductor.TestConductorProtocol.EnterBarrierOrBuilder getBarrierOrBuilder() { + if (barrierBuilder_ != null) { + return barrierBuilder_.getMessageOrBuilder(); + } else { + return barrier_; + } + } + private com.google.protobuf.SingleFieldBuilder< + akka.remote.testconductor.TestConductorProtocol.EnterBarrier, akka.remote.testconductor.TestConductorProtocol.EnterBarrier.Builder, akka.remote.testconductor.TestConductorProtocol.EnterBarrierOrBuilder> + getBarrierFieldBuilder() { + if (barrierBuilder_ == null) { + barrierBuilder_ = new com.google.protobuf.SingleFieldBuilder< + akka.remote.testconductor.TestConductorProtocol.EnterBarrier, akka.remote.testconductor.TestConductorProtocol.EnterBarrier.Builder, akka.remote.testconductor.TestConductorProtocol.EnterBarrierOrBuilder>( + barrier_, + getParentForChildren(), + isClean()); + barrier_ = null; + } + return barrierBuilder_; + } + + // optional .InjectFailure failure = 3; + private akka.remote.testconductor.TestConductorProtocol.InjectFailure failure_ = akka.remote.testconductor.TestConductorProtocol.InjectFailure.getDefaultInstance(); + private com.google.protobuf.SingleFieldBuilder< + akka.remote.testconductor.TestConductorProtocol.InjectFailure, akka.remote.testconductor.TestConductorProtocol.InjectFailure.Builder, akka.remote.testconductor.TestConductorProtocol.InjectFailureOrBuilder> failureBuilder_; + public boolean hasFailure() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public akka.remote.testconductor.TestConductorProtocol.InjectFailure getFailure() { + if (failureBuilder_ == null) { + return failure_; + } else { + return failureBuilder_.getMessage(); + } + } + public Builder setFailure(akka.remote.testconductor.TestConductorProtocol.InjectFailure value) { + if (failureBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + failure_ = value; + onChanged(); + } else { + failureBuilder_.setMessage(value); + } + bitField0_ |= 0x00000004; + return this; + } + public Builder setFailure( + akka.remote.testconductor.TestConductorProtocol.InjectFailure.Builder builderForValue) { + if (failureBuilder_ == null) { + failure_ = builderForValue.build(); + onChanged(); + } else { + failureBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000004; + return this; + } + public Builder mergeFailure(akka.remote.testconductor.TestConductorProtocol.InjectFailure value) { + if (failureBuilder_ == null) { + if (((bitField0_ & 0x00000004) == 0x00000004) && + failure_ != akka.remote.testconductor.TestConductorProtocol.InjectFailure.getDefaultInstance()) { + failure_ = + akka.remote.testconductor.TestConductorProtocol.InjectFailure.newBuilder(failure_).mergeFrom(value).buildPartial(); + } else { + failure_ = value; + } + onChanged(); + } else { + failureBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000004; + return this; + } + public Builder clearFailure() { + if (failureBuilder_ == null) { + failure_ = akka.remote.testconductor.TestConductorProtocol.InjectFailure.getDefaultInstance(); + onChanged(); + } else { + failureBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + public akka.remote.testconductor.TestConductorProtocol.InjectFailure.Builder getFailureBuilder() { + bitField0_ |= 0x00000004; + onChanged(); + return getFailureFieldBuilder().getBuilder(); + } + public akka.remote.testconductor.TestConductorProtocol.InjectFailureOrBuilder getFailureOrBuilder() { + if (failureBuilder_ != null) { + return failureBuilder_.getMessageOrBuilder(); + } else { + return failure_; + } + } + private com.google.protobuf.SingleFieldBuilder< + akka.remote.testconductor.TestConductorProtocol.InjectFailure, akka.remote.testconductor.TestConductorProtocol.InjectFailure.Builder, akka.remote.testconductor.TestConductorProtocol.InjectFailureOrBuilder> + getFailureFieldBuilder() { + if (failureBuilder_ == null) { + failureBuilder_ = new com.google.protobuf.SingleFieldBuilder< + akka.remote.testconductor.TestConductorProtocol.InjectFailure, akka.remote.testconductor.TestConductorProtocol.InjectFailure.Builder, akka.remote.testconductor.TestConductorProtocol.InjectFailureOrBuilder>( + failure_, + getParentForChildren(), + isClean()); + failure_ = null; + } + return failureBuilder_; + } + + // @@protoc_insertion_point(builder_scope:Wrapper) + } + + static { + defaultInstance = new Wrapper(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:Wrapper) + } + + public interface HelloOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // required string name = 1; + boolean hasName(); + String getName(); + + // required string host = 2; + boolean hasHost(); + String getHost(); + + // required int32 port = 3; + boolean hasPort(); + int getPort(); + } + public static final class Hello extends + com.google.protobuf.GeneratedMessage + implements HelloOrBuilder { + // Use Hello.newBuilder() to construct. + private Hello(Builder builder) { + super(builder); + } + private Hello(boolean noInit) {} + + private static final Hello defaultInstance; + public static Hello getDefaultInstance() { + return defaultInstance; + } + + public Hello getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return akka.remote.testconductor.TestConductorProtocol.internal_static_Hello_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return akka.remote.testconductor.TestConductorProtocol.internal_static_Hello_fieldAccessorTable; + } + + private int bitField0_; + // required string name = 1; + public static final int NAME_FIELD_NUMBER = 1; + private java.lang.Object name_; + public boolean hasName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getName() { + java.lang.Object ref = name_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + name_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getNameBytes() { + java.lang.Object ref = name_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + name_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // required string host = 2; + public static final int HOST_FIELD_NUMBER = 2; + private java.lang.Object host_; + public boolean hasHost() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public String getHost() { + java.lang.Object ref = host_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + host_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getHostBytes() { + java.lang.Object ref = host_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + host_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // required int32 port = 3; + public static final int PORT_FIELD_NUMBER = 3; + private int port_; + public boolean hasPort() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public int getPort() { + return port_; + } + + private void initFields() { + name_ = ""; + host_ = ""; + port_ = 0; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!hasName()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasHost()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasPort()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(1, getNameBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeBytes(2, getHostBytes()); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeInt32(3, port_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(1, getNameBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(2, getHostBytes()); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(3, port_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static akka.remote.testconductor.TestConductorProtocol.Hello parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.Hello parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.Hello parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.Hello parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.Hello parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.Hello parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.Hello parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static akka.remote.testconductor.TestConductorProtocol.Hello parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static akka.remote.testconductor.TestConductorProtocol.Hello parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.Hello parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(akka.remote.testconductor.TestConductorProtocol.Hello prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements akka.remote.testconductor.TestConductorProtocol.HelloOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return akka.remote.testconductor.TestConductorProtocol.internal_static_Hello_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return akka.remote.testconductor.TestConductorProtocol.internal_static_Hello_fieldAccessorTable; + } + + // Construct using akka.remote.testconductor.TestConductorProtocol.Hello.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + name_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + host_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + port_ = 0; + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return akka.remote.testconductor.TestConductorProtocol.Hello.getDescriptor(); + } + + public akka.remote.testconductor.TestConductorProtocol.Hello getDefaultInstanceForType() { + return akka.remote.testconductor.TestConductorProtocol.Hello.getDefaultInstance(); + } + + public akka.remote.testconductor.TestConductorProtocol.Hello build() { + akka.remote.testconductor.TestConductorProtocol.Hello result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private akka.remote.testconductor.TestConductorProtocol.Hello buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + akka.remote.testconductor.TestConductorProtocol.Hello result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public akka.remote.testconductor.TestConductorProtocol.Hello buildPartial() { + akka.remote.testconductor.TestConductorProtocol.Hello result = new akka.remote.testconductor.TestConductorProtocol.Hello(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.name_ = name_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.host_ = host_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.port_ = port_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof akka.remote.testconductor.TestConductorProtocol.Hello) { + return mergeFrom((akka.remote.testconductor.TestConductorProtocol.Hello)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(akka.remote.testconductor.TestConductorProtocol.Hello other) { + if (other == akka.remote.testconductor.TestConductorProtocol.Hello.getDefaultInstance()) return this; + if (other.hasName()) { + setName(other.getName()); + } + if (other.hasHost()) { + setHost(other.getHost()); + } + if (other.hasPort()) { + setPort(other.getPort()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + if (!hasName()) { + + return false; + } + if (!hasHost()) { + + return false; + } + if (!hasPort()) { + + return false; + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 10: { + bitField0_ |= 0x00000001; + name_ = input.readBytes(); + break; + } + case 18: { + bitField0_ |= 0x00000002; + host_ = input.readBytes(); + break; + } + case 24: { + bitField0_ |= 0x00000004; + port_ = input.readInt32(); + break; + } + } + } + } + + private int bitField0_; + + // required string name = 1; + private java.lang.Object name_ = ""; + public boolean hasName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getName() { + java.lang.Object ref = name_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + name_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setName(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + name_ = value; + onChanged(); + return this; + } + public Builder clearName() { + bitField0_ = (bitField0_ & ~0x00000001); + name_ = getDefaultInstance().getName(); + onChanged(); + return this; + } + void setName(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000001; + name_ = value; + onChanged(); + } + + // required string host = 2; + private java.lang.Object host_ = ""; + public boolean hasHost() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public String getHost() { + java.lang.Object ref = host_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + host_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setHost(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + host_ = value; + onChanged(); + return this; + } + public Builder clearHost() { + bitField0_ = (bitField0_ & ~0x00000002); + host_ = getDefaultInstance().getHost(); + onChanged(); + return this; + } + void setHost(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000002; + host_ = value; + onChanged(); + } + + // required int32 port = 3; + private int port_ ; + public boolean hasPort() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public int getPort() { + return port_; + } + public Builder setPort(int value) { + bitField0_ |= 0x00000004; + port_ = value; + onChanged(); + return this; + } + public Builder clearPort() { + bitField0_ = (bitField0_ & ~0x00000004); + port_ = 0; + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:Hello) + } + + static { + defaultInstance = new Hello(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:Hello) + } + + public interface EnterBarrierOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // required string name = 1; + boolean hasName(); + String getName(); + } + public static final class EnterBarrier extends + com.google.protobuf.GeneratedMessage + implements EnterBarrierOrBuilder { + // Use EnterBarrier.newBuilder() to construct. + private EnterBarrier(Builder builder) { + super(builder); + } + private EnterBarrier(boolean noInit) {} + + private static final EnterBarrier defaultInstance; + public static EnterBarrier getDefaultInstance() { + return defaultInstance; + } + + public EnterBarrier getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return akka.remote.testconductor.TestConductorProtocol.internal_static_EnterBarrier_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return akka.remote.testconductor.TestConductorProtocol.internal_static_EnterBarrier_fieldAccessorTable; + } + + private int bitField0_; + // required string name = 1; + public static final int NAME_FIELD_NUMBER = 1; + private java.lang.Object name_; + public boolean hasName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getName() { + java.lang.Object ref = name_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + name_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getNameBytes() { + java.lang.Object ref = name_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + name_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private void initFields() { + name_ = ""; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!hasName()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(1, getNameBytes()); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(1, getNameBytes()); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static akka.remote.testconductor.TestConductorProtocol.EnterBarrier parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.EnterBarrier parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.EnterBarrier parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.EnterBarrier parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.EnterBarrier parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.EnterBarrier parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.EnterBarrier parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static akka.remote.testconductor.TestConductorProtocol.EnterBarrier parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static akka.remote.testconductor.TestConductorProtocol.EnterBarrier parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.EnterBarrier parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(akka.remote.testconductor.TestConductorProtocol.EnterBarrier prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements akka.remote.testconductor.TestConductorProtocol.EnterBarrierOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return akka.remote.testconductor.TestConductorProtocol.internal_static_EnterBarrier_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return akka.remote.testconductor.TestConductorProtocol.internal_static_EnterBarrier_fieldAccessorTable; + } + + // Construct using akka.remote.testconductor.TestConductorProtocol.EnterBarrier.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + name_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return akka.remote.testconductor.TestConductorProtocol.EnterBarrier.getDescriptor(); + } + + public akka.remote.testconductor.TestConductorProtocol.EnterBarrier getDefaultInstanceForType() { + return akka.remote.testconductor.TestConductorProtocol.EnterBarrier.getDefaultInstance(); + } + + public akka.remote.testconductor.TestConductorProtocol.EnterBarrier build() { + akka.remote.testconductor.TestConductorProtocol.EnterBarrier result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private akka.remote.testconductor.TestConductorProtocol.EnterBarrier buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + akka.remote.testconductor.TestConductorProtocol.EnterBarrier result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public akka.remote.testconductor.TestConductorProtocol.EnterBarrier buildPartial() { + akka.remote.testconductor.TestConductorProtocol.EnterBarrier result = new akka.remote.testconductor.TestConductorProtocol.EnterBarrier(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.name_ = name_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof akka.remote.testconductor.TestConductorProtocol.EnterBarrier) { + return mergeFrom((akka.remote.testconductor.TestConductorProtocol.EnterBarrier)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(akka.remote.testconductor.TestConductorProtocol.EnterBarrier other) { + if (other == akka.remote.testconductor.TestConductorProtocol.EnterBarrier.getDefaultInstance()) return this; + if (other.hasName()) { + setName(other.getName()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + if (!hasName()) { + + return false; + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 10: { + bitField0_ |= 0x00000001; + name_ = input.readBytes(); + break; + } + } + } + } + + private int bitField0_; + + // required string name = 1; + private java.lang.Object name_ = ""; + public boolean hasName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getName() { + java.lang.Object ref = name_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + name_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setName(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + name_ = value; + onChanged(); + return this; + } + public Builder clearName() { + bitField0_ = (bitField0_ & ~0x00000001); + name_ = getDefaultInstance().getName(); + onChanged(); + return this; + } + void setName(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000001; + name_ = value; + onChanged(); + } + + // @@protoc_insertion_point(builder_scope:EnterBarrier) + } + + static { + defaultInstance = new EnterBarrier(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:EnterBarrier) + } + + public interface InjectFailureOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // required .FailType failure = 1; + boolean hasFailure(); + akka.remote.testconductor.TestConductorProtocol.FailType getFailure(); + + // optional .Direction direction = 2; + boolean hasDirection(); + akka.remote.testconductor.TestConductorProtocol.Direction getDirection(); + + // optional string host = 3; + boolean hasHost(); + String getHost(); + + // optional int32 port = 4; + boolean hasPort(); + int getPort(); + + // optional float rateMBit = 5; + boolean hasRateMBit(); + float getRateMBit(); + + // optional int32 exitValue = 6; + boolean hasExitValue(); + int getExitValue(); + } + public static final class InjectFailure extends + com.google.protobuf.GeneratedMessage + implements InjectFailureOrBuilder { + // Use InjectFailure.newBuilder() to construct. + private InjectFailure(Builder builder) { + super(builder); + } + private InjectFailure(boolean noInit) {} + + private static final InjectFailure defaultInstance; + public static InjectFailure getDefaultInstance() { + return defaultInstance; + } + + public InjectFailure getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return akka.remote.testconductor.TestConductorProtocol.internal_static_InjectFailure_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return akka.remote.testconductor.TestConductorProtocol.internal_static_InjectFailure_fieldAccessorTable; + } + + private int bitField0_; + // required .FailType failure = 1; + public static final int FAILURE_FIELD_NUMBER = 1; + private akka.remote.testconductor.TestConductorProtocol.FailType failure_; + public boolean hasFailure() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public akka.remote.testconductor.TestConductorProtocol.FailType getFailure() { + return failure_; + } + + // optional .Direction direction = 2; + public static final int DIRECTION_FIELD_NUMBER = 2; + private akka.remote.testconductor.TestConductorProtocol.Direction direction_; + public boolean hasDirection() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public akka.remote.testconductor.TestConductorProtocol.Direction getDirection() { + return direction_; + } + + // optional string host = 3; + public static final int HOST_FIELD_NUMBER = 3; + private java.lang.Object host_; + public boolean hasHost() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public String getHost() { + java.lang.Object ref = host_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + host_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getHostBytes() { + java.lang.Object ref = host_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + host_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // optional int32 port = 4; + public static final int PORT_FIELD_NUMBER = 4; + private int port_; + public boolean hasPort() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public int getPort() { + return port_; + } + + // optional float rateMBit = 5; + public static final int RATEMBIT_FIELD_NUMBER = 5; + private float rateMBit_; + public boolean hasRateMBit() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + public float getRateMBit() { + return rateMBit_; + } + + // optional int32 exitValue = 6; + public static final int EXITVALUE_FIELD_NUMBER = 6; + private int exitValue_; + public boolean hasExitValue() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + public int getExitValue() { + return exitValue_; + } + + private void initFields() { + failure_ = akka.remote.testconductor.TestConductorProtocol.FailType.Throttle; + direction_ = akka.remote.testconductor.TestConductorProtocol.Direction.Send; + host_ = ""; + port_ = 0; + rateMBit_ = 0F; + exitValue_ = 0; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!hasFailure()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeEnum(1, failure_.getNumber()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeEnum(2, direction_.getNumber()); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeBytes(3, getHostBytes()); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeInt32(4, port_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + output.writeFloat(5, rateMBit_); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { + output.writeInt32(6, exitValue_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeEnumSize(1, failure_.getNumber()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeEnumSize(2, direction_.getNumber()); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(3, getHostBytes()); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(4, port_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + size += com.google.protobuf.CodedOutputStream + .computeFloatSize(5, rateMBit_); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(6, exitValue_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static akka.remote.testconductor.TestConductorProtocol.InjectFailure parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.InjectFailure parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.InjectFailure parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.InjectFailure parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.InjectFailure parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.InjectFailure parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.InjectFailure parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static akka.remote.testconductor.TestConductorProtocol.InjectFailure parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static akka.remote.testconductor.TestConductorProtocol.InjectFailure parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.InjectFailure parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(akka.remote.testconductor.TestConductorProtocol.InjectFailure prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements akka.remote.testconductor.TestConductorProtocol.InjectFailureOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return akka.remote.testconductor.TestConductorProtocol.internal_static_InjectFailure_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return akka.remote.testconductor.TestConductorProtocol.internal_static_InjectFailure_fieldAccessorTable; + } + + // Construct using akka.remote.testconductor.TestConductorProtocol.InjectFailure.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + failure_ = akka.remote.testconductor.TestConductorProtocol.FailType.Throttle; + bitField0_ = (bitField0_ & ~0x00000001); + direction_ = akka.remote.testconductor.TestConductorProtocol.Direction.Send; + bitField0_ = (bitField0_ & ~0x00000002); + host_ = ""; + bitField0_ = (bitField0_ & ~0x00000004); + port_ = 0; + bitField0_ = (bitField0_ & ~0x00000008); + rateMBit_ = 0F; + bitField0_ = (bitField0_ & ~0x00000010); + exitValue_ = 0; + bitField0_ = (bitField0_ & ~0x00000020); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return akka.remote.testconductor.TestConductorProtocol.InjectFailure.getDescriptor(); + } + + public akka.remote.testconductor.TestConductorProtocol.InjectFailure getDefaultInstanceForType() { + return akka.remote.testconductor.TestConductorProtocol.InjectFailure.getDefaultInstance(); + } + + public akka.remote.testconductor.TestConductorProtocol.InjectFailure build() { + akka.remote.testconductor.TestConductorProtocol.InjectFailure result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private akka.remote.testconductor.TestConductorProtocol.InjectFailure buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + akka.remote.testconductor.TestConductorProtocol.InjectFailure result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public akka.remote.testconductor.TestConductorProtocol.InjectFailure buildPartial() { + akka.remote.testconductor.TestConductorProtocol.InjectFailure result = new akka.remote.testconductor.TestConductorProtocol.InjectFailure(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.failure_ = failure_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.direction_ = direction_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.host_ = host_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000008; + } + result.port_ = port_; + if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000010; + } + result.rateMBit_ = rateMBit_; + if (((from_bitField0_ & 0x00000020) == 0x00000020)) { + to_bitField0_ |= 0x00000020; + } + result.exitValue_ = exitValue_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof akka.remote.testconductor.TestConductorProtocol.InjectFailure) { + return mergeFrom((akka.remote.testconductor.TestConductorProtocol.InjectFailure)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(akka.remote.testconductor.TestConductorProtocol.InjectFailure other) { + if (other == akka.remote.testconductor.TestConductorProtocol.InjectFailure.getDefaultInstance()) return this; + if (other.hasFailure()) { + setFailure(other.getFailure()); + } + if (other.hasDirection()) { + setDirection(other.getDirection()); + } + if (other.hasHost()) { + setHost(other.getHost()); + } + if (other.hasPort()) { + setPort(other.getPort()); + } + if (other.hasRateMBit()) { + setRateMBit(other.getRateMBit()); + } + if (other.hasExitValue()) { + setExitValue(other.getExitValue()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + if (!hasFailure()) { + + return false; + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 8: { + int rawValue = input.readEnum(); + akka.remote.testconductor.TestConductorProtocol.FailType value = akka.remote.testconductor.TestConductorProtocol.FailType.valueOf(rawValue); + if (value == null) { + unknownFields.mergeVarintField(1, rawValue); + } else { + bitField0_ |= 0x00000001; + failure_ = value; + } + break; + } + case 16: { + int rawValue = input.readEnum(); + akka.remote.testconductor.TestConductorProtocol.Direction value = akka.remote.testconductor.TestConductorProtocol.Direction.valueOf(rawValue); + if (value == null) { + unknownFields.mergeVarintField(2, rawValue); + } else { + bitField0_ |= 0x00000002; + direction_ = value; + } + break; + } + case 26: { + bitField0_ |= 0x00000004; + host_ = input.readBytes(); + break; + } + case 32: { + bitField0_ |= 0x00000008; + port_ = input.readInt32(); + break; + } + case 45: { + bitField0_ |= 0x00000010; + rateMBit_ = input.readFloat(); + break; + } + case 48: { + bitField0_ |= 0x00000020; + exitValue_ = input.readInt32(); + break; + } + } + } + } + + private int bitField0_; + + // required .FailType failure = 1; + private akka.remote.testconductor.TestConductorProtocol.FailType failure_ = akka.remote.testconductor.TestConductorProtocol.FailType.Throttle; + public boolean hasFailure() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public akka.remote.testconductor.TestConductorProtocol.FailType getFailure() { + return failure_; + } + public Builder setFailure(akka.remote.testconductor.TestConductorProtocol.FailType value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + failure_ = value; + onChanged(); + return this; + } + public Builder clearFailure() { + bitField0_ = (bitField0_ & ~0x00000001); + failure_ = akka.remote.testconductor.TestConductorProtocol.FailType.Throttle; + onChanged(); + return this; + } + + // optional .Direction direction = 2; + private akka.remote.testconductor.TestConductorProtocol.Direction direction_ = akka.remote.testconductor.TestConductorProtocol.Direction.Send; + public boolean hasDirection() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public akka.remote.testconductor.TestConductorProtocol.Direction getDirection() { + return direction_; + } + public Builder setDirection(akka.remote.testconductor.TestConductorProtocol.Direction value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + direction_ = value; + onChanged(); + return this; + } + public Builder clearDirection() { + bitField0_ = (bitField0_ & ~0x00000002); + direction_ = akka.remote.testconductor.TestConductorProtocol.Direction.Send; + onChanged(); + return this; + } + + // optional string host = 3; + private java.lang.Object host_ = ""; + public boolean hasHost() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public String getHost() { + java.lang.Object ref = host_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + host_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setHost(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + host_ = value; + onChanged(); + return this; + } + public Builder clearHost() { + bitField0_ = (bitField0_ & ~0x00000004); + host_ = getDefaultInstance().getHost(); + onChanged(); + return this; + } + void setHost(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000004; + host_ = value; + onChanged(); + } + + // optional int32 port = 4; + private int port_ ; + public boolean hasPort() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public int getPort() { + return port_; + } + public Builder setPort(int value) { + bitField0_ |= 0x00000008; + port_ = value; + onChanged(); + return this; + } + public Builder clearPort() { + bitField0_ = (bitField0_ & ~0x00000008); + port_ = 0; + onChanged(); + return this; + } + + // optional float rateMBit = 5; + private float rateMBit_ ; + public boolean hasRateMBit() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + public float getRateMBit() { + return rateMBit_; + } + public Builder setRateMBit(float value) { + bitField0_ |= 0x00000010; + rateMBit_ = value; + onChanged(); + return this; + } + public Builder clearRateMBit() { + bitField0_ = (bitField0_ & ~0x00000010); + rateMBit_ = 0F; + onChanged(); + return this; + } + + // optional int32 exitValue = 6; + private int exitValue_ ; + public boolean hasExitValue() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + public int getExitValue() { + return exitValue_; + } + public Builder setExitValue(int value) { + bitField0_ |= 0x00000020; + exitValue_ = value; + onChanged(); + return this; + } + public Builder clearExitValue() { + bitField0_ = (bitField0_ & ~0x00000020); + exitValue_ = 0; + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:InjectFailure) + } + + static { + defaultInstance = new InjectFailure(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:InjectFailure) + } + + private static com.google.protobuf.Descriptors.Descriptor + internal_static_Wrapper_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_Wrapper_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_Hello_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_Hello_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_EnterBarrier_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_EnterBarrier_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_InjectFailure_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_InjectFailure_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor + getDescriptor() { + return descriptor; + } + private static com.google.protobuf.Descriptors.FileDescriptor + descriptor; + static { + java.lang.String[] descriptorData = { + "\n\033TestConductorProtocol.proto\"a\n\007Wrapper" + + "\022\025\n\005hello\030\001 \001(\0132\006.Hello\022\036\n\007barrier\030\002 \001(\013" + + "2\r.EnterBarrier\022\037\n\007failure\030\003 \001(\0132\016.Injec" + + "tFailure\"1\n\005Hello\022\014\n\004name\030\001 \002(\t\022\014\n\004host\030" + + "\002 \002(\t\022\014\n\004port\030\003 \002(\005\"\034\n\014EnterBarrier\022\014\n\004n" + + "ame\030\001 \002(\t\"\213\001\n\rInjectFailure\022\032\n\007failure\030\001" + + " \002(\0162\t.FailType\022\035\n\tdirection\030\002 \001(\0162\n.Dir" + + "ection\022\014\n\004host\030\003 \001(\t\022\014\n\004port\030\004 \001(\005\022\020\n\010ra" + + "teMBit\030\005 \001(\002\022\021\n\texitValue\030\006 \001(\005*A\n\010FailT" + + "ype\022\014\n\010Throttle\020\001\022\016\n\nDisconnect\020\002\022\t\n\005Abo", + "rt\020\003\022\014\n\010Shutdown\020\004*\"\n\tDirection\022\010\n\004Send\020" + + "\001\022\013\n\007Receive\020\002B\035\n\031akka.remote.testconduc" + + "torH\001" + }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = + new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors( + com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + internal_static_Wrapper_descriptor = + getDescriptor().getMessageTypes().get(0); + internal_static_Wrapper_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_Wrapper_descriptor, + new java.lang.String[] { "Hello", "Barrier", "Failure", }, + akka.remote.testconductor.TestConductorProtocol.Wrapper.class, + akka.remote.testconductor.TestConductorProtocol.Wrapper.Builder.class); + internal_static_Hello_descriptor = + getDescriptor().getMessageTypes().get(1); + internal_static_Hello_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_Hello_descriptor, + new java.lang.String[] { "Name", "Host", "Port", }, + akka.remote.testconductor.TestConductorProtocol.Hello.class, + akka.remote.testconductor.TestConductorProtocol.Hello.Builder.class); + internal_static_EnterBarrier_descriptor = + getDescriptor().getMessageTypes().get(2); + internal_static_EnterBarrier_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_EnterBarrier_descriptor, + new java.lang.String[] { "Name", }, + akka.remote.testconductor.TestConductorProtocol.EnterBarrier.class, + akka.remote.testconductor.TestConductorProtocol.EnterBarrier.Builder.class); + internal_static_InjectFailure_descriptor = + getDescriptor().getMessageTypes().get(3); + internal_static_InjectFailure_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_InjectFailure_descriptor, + new java.lang.String[] { "Failure", "Direction", "Host", "Port", "RateMBit", "ExitValue", }, + akka.remote.testconductor.TestConductorProtocol.InjectFailure.class, + akka.remote.testconductor.TestConductorProtocol.InjectFailure.Builder.class); + return null; + } + }; + com.google.protobuf.Descriptors.FileDescriptor + .internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { + }, assigner); + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git a/akka-remote/src/main/protocol/TestConductorProtocol.proto b/akka-remote/src/main/protocol/TestConductorProtocol.proto new file mode 100644 index 0000000000..1db35a7516 --- /dev/null +++ b/akka-remote/src/main/protocol/TestConductorProtocol.proto @@ -0,0 +1,48 @@ +/** + * Copyright (C) 2009-2011 Typesafe Inc. + */ + +option java_package = "akka.remote.testconductor"; +option optimize_for = SPEED; + +/****************************************** + Compile with: + cd ./akka-remote/src/main/protocol + protoc TestConductorProtocol.proto --java_out ../java +*******************************************/ + +message Wrapper { + optional Hello hello = 1; + optional EnterBarrier barrier = 2; + optional InjectFailure failure = 3; +} + +message Hello { + required string name = 1; + required string host = 2; + required int32 port = 3; +} + +message EnterBarrier { + required string name = 1; +} + +enum FailType { + Throttle = 1; + Disconnect = 2; + Abort = 3; + Shutdown = 4; +} +enum Direction { + Send = 1; + Receive = 2; +} +message InjectFailure { + required FailType failure = 1; + optional Direction direction = 2; + optional string host = 3; + optional int32 port = 4; + optional float rateMBit = 5; + optional int32 exitValue = 6; +} + diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala b/akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala new file mode 100644 index 0000000000..58a6a5f88e --- /dev/null +++ b/akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala @@ -0,0 +1,268 @@ +/** + * Copyright (C) 2009-2011 Typesafe Inc. + */ +package akka.remote.testconductor + +import akka.actor.{ Actor, ActorRef, LoggingFSM, Timeout, UntypedChannel } +import akka.event.EventHandler +import RemoteConnection.getAddrString +import akka.util.duration._ +import TestConductorProtocol._ +import akka.NoStackTrace +import org.jboss.netty.channel.{ Channel, SimpleChannelUpstreamHandler, ChannelHandlerContext, ChannelStateEvent, MessageEvent } + +object Conductor extends RunControl with FailureInject with BarrierSync { + + import Controller._ + + private val controller = Actor.actorOf[Controller] + controller ! ClientConnected + + override def enter(name: String*) { + implicit val timeout = Timeout(30 seconds) + name foreach (b ⇒ (controller ? EnterBarrier(b)).get) + } + + override def throttle(node: String, target: String, direction: Direction, rateMBit: Float) { + controller ! Throttle(node, target, direction, rateMBit) + } + + override def blackhole(node: String, target: String, direction: Direction) { + controller ! Throttle(node, target, direction, 0f) + } + + override def disconnect(node: String, target: String) { + controller ! Disconnect(node, target, false) + } + + override def abort(node: String, target: String) { + controller ! Disconnect(node, target, true) + } + + override def shutdown(node: String, exitValue: Int) { + controller ! Terminate(node, exitValue) + } + + override def kill(node: String) { + controller ! Terminate(node, -1) + } + + override def getNodes = (controller ? GetNodes).as[List[String]].get + + override def removeNode(node: String) { + controller ! Remove(node) + } + +} + +class ConductorHandler(controller: ActorRef) extends SimpleChannelUpstreamHandler { + + var clients = Map[Channel, ActorRef]() + + override def channelConnected(ctx: ChannelHandlerContext, event: ChannelStateEvent) = { + val channel = event.getChannel + EventHandler.debug(this, "connection from " + getAddrString(channel)) + val fsm = Actor.actorOf(new ServerFSM(controller, channel)) + clients += channel -> fsm + } + + override def channelDisconnected(ctx: ChannelHandlerContext, event: ChannelStateEvent) = { + val channel = event.getChannel + EventHandler.debug(this, "disconnect from " + getAddrString(channel)) + val fsm = clients(channel) + fsm.stop() + clients -= channel + } + + override def messageReceived(ctx: ChannelHandlerContext, event: MessageEvent) = { + val channel = event.getChannel + EventHandler.debug(this, "message from " + getAddrString(channel) + ": " + event.getMessage) + event.getMessage match { + case msg: Wrapper if msg.getAllFields.size == 1 ⇒ + clients(channel) ! msg + case msg ⇒ + EventHandler.info(this, "client " + getAddrString(channel) + " sent garbage '" + msg + "', disconnecting") + channel.close() + } + } + +} + +object ServerFSM { + sealed trait State + case object Initial extends State + case object Ready extends State + + case class Send(msg: Wrapper) +} + +class ServerFSM(val controller: ActorRef, val channel: Channel) extends Actor with LoggingFSM[ServerFSM.State, Null] { + import ServerFSM._ + import akka.actor.FSM._ + import Controller._ + + startWith(Initial, null) + + when(Initial, stateTimeout = 10 seconds) { + case Ev(msg: Wrapper) ⇒ + if (msg.hasHello) { + val hello = msg.getHello + controller ! ClientConnected(hello.getName, hello.getHost, hello.getPort) + goto(Ready) + } else { + EventHandler.warning(this, "client " + getAddrString(channel) + " sent no Hello in first message, disconnecting") + channel.close() + stop() + } + case Ev(StateTimeout) ⇒ + EventHandler.info(this, "closing channel to " + getAddrString(channel) + " because of Hello timeout") + channel.close() + stop() + } + + when(Ready) { + case Ev(msg: Wrapper) ⇒ + if (msg.hasBarrier) { + val barrier = msg.getBarrier + controller ! EnterBarrier(barrier.getName) + } else { + EventHandler.warning(this, "client " + getAddrString(channel) + " sent unsupported message " + msg) + } + stay + case Ev(Send(msg)) ⇒ + channel.write(msg) + stay + case Ev(EnterBarrier(name)) ⇒ + val barrier = TestConductorProtocol.EnterBarrier.newBuilder.setName(name).build + channel.write(Wrapper.newBuilder.setBarrier(barrier).build) + stay + } + + initialize +} + +object Controller { + case class ClientConnected(name: String, host: String, port: Int) + case class ClientDisconnected(name: String) + case object GetNodes + + case class NodeInfo(name: String, host: String, port: Int, fsm: ActorRef) +} + +class Controller extends Actor { + import Controller._ + + val host = System.getProperty("akka.testconductor.host", "localhost") + val port = Integer.getInteger("akka.testconductor.port", 4545) + val connection = RemoteConnection(Server, host, port, new ConductorHandler(self)) + + val barrier = Actor.actorOf[BarrierCoordinator] + var nodes = Map[String, NodeInfo]() + + override def receive = Actor.loggable(this) { + case ClientConnected(name, host, port) ⇒ + self.channel match { + case ref: ActorRef ⇒ nodes += name -> NodeInfo(name, host, port, ref) + } + barrier forward ClientConnected + case ClientConnected ⇒ + barrier forward ClientConnected + case ClientDisconnected(name) ⇒ + nodes -= name + barrier forward ClientDisconnected + case e @ EnterBarrier(name) ⇒ + barrier forward e + case Throttle(node, target, direction, rateMBit) ⇒ + val t = nodes(target) + val throttle = + InjectFailure.newBuilder + .setFailure(FailType.Throttle) + .setDirection(TestConductorProtocol.Direction.valueOf(direction.toString)) + .setHost(t.host) + .setPort(t.port) + .setRateMBit(rateMBit) + .build + nodes(node).fsm ! ServerFSM.Send(Wrapper.newBuilder.setFailure(throttle).build) + case Disconnect(node, target, abort) ⇒ + val t = nodes(target) + val disconnect = + InjectFailure.newBuilder + .setFailure(if (abort) FailType.Abort else FailType.Disconnect) + .setHost(t.host) + .setPort(t.port) + .build + nodes(node).fsm ! ServerFSM.Send(Wrapper.newBuilder.setFailure(disconnect).build) + case Terminate(node, exitValueOrKill) ⇒ + if (exitValueOrKill < 0) { + // TODO: kill via SBT + } else { + val shutdown = InjectFailure.newBuilder.setFailure(FailType.Shutdown).setExitValue(exitValueOrKill).build + nodes(node).fsm ! ServerFSM.Send(Wrapper.newBuilder.setFailure(shutdown).build) + } + // TODO: properly remove node from BarrierCoordinator + // case Remove(node) => + // nodes -= node + case GetNodes ⇒ self reply nodes.keys + } +} + +object BarrierCoordinator { + sealed trait State + case object Idle extends State + case object Waiting extends State + + case class Data(clients: Int, barrier: String, arrived: List[UntypedChannel]) + class BarrierTimeoutException(msg: String) extends RuntimeException(msg) with NoStackTrace +} + +class BarrierCoordinator extends Actor with LoggingFSM[BarrierCoordinator.State, BarrierCoordinator.Data] { + import BarrierCoordinator._ + import akka.actor.FSM._ + import Controller._ + + startWith(Idle, Data(0, "", Nil)) + + when(Idle) { + case Event(EnterBarrier(name), Data(num, _, _)) ⇒ + if (num == 0) throw new IllegalStateException("no client expected yet") + goto(Waiting) using Data(num, name, self.channel :: Nil) + case Event(ClientConnected, d @ Data(num, _, _)) ⇒ + stay using d.copy(clients = num + 1) + case Event(ClientDisconnected, d @ Data(num, _, _)) ⇒ + if (num == 0) throw new IllegalStateException("no client to disconnect") + stay using d.copy(clients = num - 1) + } + + onTransition { + case Idle -> Waiting ⇒ setTimer("Timeout", StateTimeout, 30 seconds, false) + case Waiting -> Idle ⇒ cancelTimer("Timeout") + } + + when(Waiting) { + case Event(e @ EnterBarrier(name), d @ Data(num, barrier, arrived)) ⇒ + if (name != barrier) throw new IllegalStateException("trying enter barrier '" + name + "' while barrier '" + barrier + "' is active") + val together = self.channel :: arrived + if (together.size == num) { + together foreach (_ ! e) + goto(Idle) using Data(num, "", Nil) + } else { + stay using d.copy(arrived = together) + } + case Event(ClientConnected, d @ Data(num, _, _)) ⇒ + stay using d.copy(clients = num + 1) + case Event(ClientDisconnected, d @ Data(num, barrier, arrived)) ⇒ + val expected = num - 1 + if (arrived.size == expected) { + val e = EnterBarrier(barrier) + self.channel :: arrived foreach (_ ! e) + goto(Idle) using Data(expected, "", Nil) + } else { + stay using d.copy(clients = expected) + } + case Event(StateTimeout, Data(num, barrier, arrived)) ⇒ + throw new BarrierTimeoutException("only " + arrived.size + " of " + num + " arrived at barrier " + barrier) + } + + initialize +} + diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/DataTypes.scala b/akka-remote/src/main/scala/akka/remote/testconductor/DataTypes.scala new file mode 100644 index 0000000000..2b54ea1018 --- /dev/null +++ b/akka-remote/src/main/scala/akka/remote/testconductor/DataTypes.scala @@ -0,0 +1,13 @@ +/** + * Copyright (C) 2009-2011 Typesafe Inc. + */ +package akka.remote.testconductor + +sealed trait ClientOp +sealed trait ServerOp + +case class EnterBarrier(name: String) extends ClientOp with ServerOp +case class Throttle(node: String, target: String, direction: Direction, rateMBit: Float) extends ServerOp +case class Disconnect(node: String, target: String, abort: Boolean) extends ServerOp +case class Terminate(node: String, exitValueOrKill: Int) extends ServerOp +case class Remove(node: String) extends ServerOp diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/Features.scala b/akka-remote/src/main/scala/akka/remote/testconductor/Features.scala new file mode 100644 index 0000000000..399b58337b --- /dev/null +++ b/akka-remote/src/main/scala/akka/remote/testconductor/Features.scala @@ -0,0 +1,74 @@ +/** + * Copyright (C) 2009-2011 Typesafe Inc. + */ +package akka.remote.testconductor + +trait BarrierSync { + /** + * Enter all given barriers in the order in which they were given. + */ + def enter(name: String*): Unit +} + +sealed trait Direction +case object Send extends Direction +case object Receive extends Direction +case object Both extends Direction + +trait FailureInject { + + /** + * Make the remoting pipeline on the node throttle data sent to or received + * from the given remote peer. + */ + def throttle(node: String, target: String, direction: Direction, rateMBit: Float): Unit + + /** + * Switch the Netty pipeline of the remote support into blackhole mode for + * sending and/or receiving: it will just drop all messages right before + * submitting them to the Socket or right after receiving them from the + * Socket. + */ + def blackhole(node: String, target: String, direction: Direction): Unit + + /** + * Tell the remote support to shutdown the connection to the given remote + * peer. It works regardless of whether the recipient was initiator or + * responder. + */ + def disconnect(node: String, target: String): Unit + + /** + * Tell the remote support to TCP_RESET the connection to the given remote + * peer. It works regardless of whether the recipient was initiator or + * responder. + */ + def abort(node: String, target: String): Unit + +} + +trait RunControl { + + /** + * Tell the remote node to shut itself down using System.exit with the given + * exitValue. + */ + def shutdown(node: String, exitValue: Int): Unit + + /** + * Tell the SBT plugin to forcibly terminate the given remote node using Process.destroy. + */ + def kill(node: String): Unit + + /** + * Obtain the list of remote host names currently registered. + */ + def getNodes: List[String] + + /** + * Remove a remote host from the list, so that the remaining nodes may still + * pass subsequent barriers. + */ + def removeNode(node: String): Unit + +} diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala b/akka-remote/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala new file mode 100644 index 0000000000..eec6a2cbf1 --- /dev/null +++ b/akka-remote/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala @@ -0,0 +1,161 @@ +/** + * Copyright (C) 2009-2011 Typesafe Inc. + */ +package akka.remote.testconductor + +import java.net.InetSocketAddress + +import scala.collection.immutable.Queue + +import org.jboss.netty.buffer.ChannelBuffer +import org.jboss.netty.channel.ChannelState.BOUND +import org.jboss.netty.channel.ChannelState.OPEN +import org.jboss.netty.channel.Channel +import org.jboss.netty.channel.ChannelDownstreamHandler +import org.jboss.netty.channel.ChannelEvent +import org.jboss.netty.channel.ChannelHandlerContext +import org.jboss.netty.channel.ChannelStateEvent +import org.jboss.netty.channel.ChannelUpstreamHandler +import org.jboss.netty.channel.MessageEvent + +import akka.actor.FSM +import akka.actor.Actor +import akka.util.duration.doubleToDurationDouble +import akka.util.Index +import akka.util.RemoteAddress + +object NetworkFailureInjector { + + val channels = new Index[RemoteAddress, Channel]() + + def close(remote: RemoteAddress): Unit = { + val set = channels.remove(remote) + // channels will be cleaned up by the handler + set foreach (_.close()) + } +} + +class NetworkFailureInjector extends ChannelUpstreamHandler with ChannelDownstreamHandler { + + import NetworkFailureInjector._ + + // local cache of remote address + private var remote: Option[RemoteAddress] = None + + // everything goes via these Throttle actors to enable easy steering + private val sender = Actor.actorOf(new Throttle(_.sendDownstream(_))) + private val receiver = Actor.actorOf(new Throttle(_.sendUpstream(_))) + + /* + * State, Data and Messages for the internal Throttle actor + */ + sealed private trait State + private case object PassThrough extends State + private case object Throttle extends State + private case object Blackhole extends State + + private case class Data(ctx: ChannelHandlerContext, rateMBit: Float, queue: Queue[MessageEvent]) + + private case class SetRate(rateMBit: Float) + private case class Send(ctx: ChannelHandlerContext, msg: MessageEvent) + private case object Tick + + private class Throttle(send: (ChannelHandlerContext, MessageEvent) ⇒ Unit) extends Actor with FSM[State, Data] { + import FSM._ + + startWith(PassThrough, Data(null, -1, Queue())) + + when(PassThrough) { + case Event(Send(ctx, msg), d) ⇒ + send(ctx, msg) + stay + } + + when(Throttle) { + case Event(Send(ctx, msg), d) ⇒ + if (!timerActive_?("send")) { + setTimer("send", Tick, (size(msg) / d.rateMBit) microseconds, false) + } + stay using d.copy(ctx = ctx, queue = d.queue.enqueue(msg)) + case Event(Tick, d) ⇒ + val (msg, queue) = d.queue.dequeue + send(d.ctx, msg) + if (queue.nonEmpty) setTimer("send", Tick, (size(queue.head) / d.rateMBit) microseconds, false) + stay using d.copy(queue = queue) + } + + onTransition { + case Throttle -> PassThrough ⇒ + stateData.queue foreach (send(stateData.ctx, _)) + cancelTimer("send") + case Throttle -> Blackhole ⇒ + cancelTimer("send") + } + + when(Blackhole) { + case Event(Send(_, _), _) ⇒ + stay + } + + whenUnhandled { + case Event(SetRate(rate), d) ⇒ + if (rate > 0) { + goto(Throttle) using d.copy(rateMBit = rate, queue = Queue()) + } else if (rate == 0) { + goto(Blackhole) + } else { + goto(PassThrough) + } + } + + initialize + + private def size(msg: MessageEvent) = msg.getMessage() match { + case b: ChannelBuffer ⇒ b.readableBytes() * 8 + case _ ⇒ throw new UnsupportedOperationException("NetworkFailureInjector only supports ChannelBuffer messages") + } + } + + def throttleSend(rateMBit: Float) { + sender ! SetRate(rateMBit) + } + + def throttleReceive(rateMBit: Float) { + receiver ! SetRate(rateMBit) + } + + override def handleUpstream(ctx: ChannelHandlerContext, evt: ChannelEvent) { + evt match { + case msg: MessageEvent ⇒ + receiver ! Send(ctx, msg) + case state: ChannelStateEvent ⇒ + state.getState match { + case BOUND ⇒ + state.getValue match { + case null ⇒ + remote = remote flatMap { a ⇒ channels.remove(a, state.getChannel); None } + case a: InetSocketAddress ⇒ + val addr = RemoteAddress(a) + channels.put(addr, state.getChannel) + remote = Some(addr) + } + case OPEN if state.getValue == false ⇒ + remote = remote flatMap { a ⇒ channels.remove(a, state.getChannel); None } + } + ctx.sendUpstream(evt) + case _ ⇒ + ctx.sendUpstream(evt) + } + } + + override def handleDownstream(ctx: ChannelHandlerContext, evt: ChannelEvent) { + evt match { + case msg: MessageEvent ⇒ + sender ! Send(ctx, msg) + case _ ⇒ + ctx.sendUpstream(evt) + } + } + +} + diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/Player.scala b/akka-remote/src/main/scala/akka/remote/testconductor/Player.scala new file mode 100644 index 0000000000..16abe5bb27 --- /dev/null +++ b/akka-remote/src/main/scala/akka/remote/testconductor/Player.scala @@ -0,0 +1,133 @@ +/** + * Copyright (C) 2009-2011 Typesafe Inc. + */ +package akka.remote.testconductor + +import akka.actor.{ Actor, ActorRef, LoggingFSM, Timeout, UntypedChannel } +import akka.event.EventHandler +import RemoteConnection.getAddrString +import akka.util.duration._ +import TestConductorProtocol._ +import akka.NoStackTrace +import org.jboss.netty.channel.{ Channel, SimpleChannelUpstreamHandler, ChannelHandlerContext, ChannelStateEvent, MessageEvent } +import com.eaio.uuid.UUID + +object Player extends BarrierSync { + + private val server = Actor.actorOf[ClientFSM] + + override def enter(name: String*) { + EventHandler.debug(this, "entering barriers " + name.mkString("(", ", ", ")")) + implicit val timeout = Timeout(30 seconds) + name foreach { b ⇒ + (server ? EnterBarrier(b)).get + EventHandler.debug(this, "passed barrier " + b) + } + } +} + +object ClientFSM { + sealed trait State + case object Connecting extends State + case object Connected extends State + + case class Data(channel: Channel, msg: Either[List[ClientOp], (String, UntypedChannel)]) + + class ConnectionFailure(msg: String) extends RuntimeException(msg) with NoStackTrace + case object Disconnected +} + +class ClientFSM extends Actor with LoggingFSM[ClientFSM.State, ClientFSM.Data] { + import ClientFSM._ + import akka.actor.FSM._ + + val name = System.getProperty("akka.testconductor.name", (new UUID).toString) + val host = System.getProperty("akka.testconductor.host", "localhost") + val port = Integer.getInteger("akka.testconductor.port", 4545) + val handler = new PlayerHandler(self) + + val myself = Actor.remote.address + + startWith(Connecting, Data(RemoteConnection(Client, host, port, handler), Left(Nil))) + + when(Connecting, stateTimeout = 10 seconds) { + case Event(msg: ClientOp, Data(channel, Left(msgs))) ⇒ + stay using Data(channel, Left(msg :: msgs)) + case Event(Connected, Data(channel, Left(msgs))) ⇒ + val hello = Hello.newBuilder.setName(name).setHost(myself.getAddress.getHostAddress).setPort(myself.getPort).build + channel.write(Wrapper.newBuilder.setHello(hello).build) + msgs.reverse foreach sendMsg(channel) + goto(Connected) using Data(channel, Left(Nil)) + case Event(_: ConnectionFailure, _) ⇒ + // System.exit(1) + stop + case Event(StateTimeout, _) ⇒ + EventHandler.error(this, "connect timeout to TestConductor") + // System.exit(1) + stop + } + + when(Connected) { + case Event(Disconnected, _) ⇒ + EventHandler.info(this, "disconnected from TestConductor") + throw new ConnectionFailure("disconnect") + case Event(msg: EnterBarrier, Data(channel, _)) ⇒ + sendMsg(channel)(msg) + stay using Data(channel, Right((msg.name, self.channel))) + case Event(msg: Wrapper, Data(channel, Right((barrier, sender)))) if msg.getAllFields.size == 1 ⇒ + if (msg.hasBarrier) { + val b = msg.getBarrier.getName + if (b != barrier) { + sender.sendException(new RuntimeException("wrong barrier " + b + " received while waiting for " + barrier)) + } else { + sender ! b + } + } + stay using Data(channel, Left(Nil)) + } + + onTermination { + case StopEvent(_, _, Data(channel, _)) ⇒ + channel.close() + } + + private def sendMsg(channel: Channel)(msg: ClientOp) { + msg match { + case EnterBarrier(name) ⇒ + val enter = TestConductorProtocol.EnterBarrier.newBuilder.setName(name).build + channel.write(Wrapper.newBuilder.setBarrier(enter).build) + } + } + +} + +class PlayerHandler(fsm: ActorRef) extends SimpleChannelUpstreamHandler { + + import ClientFSM._ + + override def channelConnected(ctx: ChannelHandlerContext, event: ChannelStateEvent) = { + val channel = event.getChannel + EventHandler.debug(this, "connected to " + getAddrString(channel)) + while (!fsm.isRunning) Thread.sleep(100) + fsm ! Connected + } + + override def channelDisconnected(ctx: ChannelHandlerContext, event: ChannelStateEvent) = { + val channel = event.getChannel + EventHandler.debug(this, "disconnected from " + getAddrString(channel)) + fsm.stop() + } + + override def messageReceived(ctx: ChannelHandlerContext, event: MessageEvent) = { + val channel = event.getChannel + EventHandler.debug(this, "message from " + getAddrString(channel) + ": " + event.getMessage) + event.getMessage match { + case msg: Wrapper if msg.getAllFields.size == 1 ⇒ + fsm ! msg + case msg ⇒ + EventHandler.info(this, "server " + getAddrString(channel) + " sent garbage '" + msg + "', disconnecting") + channel.close() + } + } +} + diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/RemoteConnection.scala b/akka-remote/src/main/scala/akka/remote/testconductor/RemoteConnection.scala new file mode 100644 index 0000000000..a92b6295e2 --- /dev/null +++ b/akka-remote/src/main/scala/akka/remote/testconductor/RemoteConnection.scala @@ -0,0 +1,52 @@ +/** + * Copyright (C) 2009-2011 Typesafe Inc. + */ +package akka.remote.testconductor + +import org.jboss.netty.channel.{ Channel, ChannelPipeline, ChannelPipelineFactory, ChannelUpstreamHandler, SimpleChannelUpstreamHandler, StaticChannelPipeline } +import org.jboss.netty.channel.socket.nio.{ NioClientSocketChannelFactory, NioServerSocketChannelFactory } +import org.jboss.netty.bootstrap.{ ClientBootstrap, ServerBootstrap } +import org.jboss.netty.handler.codec.frame.{ LengthFieldBasedFrameDecoder, LengthFieldPrepender } +import org.jboss.netty.handler.codec.compression.{ ZlibDecoder, ZlibEncoder } +import org.jboss.netty.handler.codec.protobuf.{ ProtobufDecoder, ProtobufEncoder } +import org.jboss.netty.handler.timeout.{ ReadTimeoutHandler, ReadTimeoutException } +import java.net.InetSocketAddress +import java.util.concurrent.Executors + +class TestConductorPipelineFactory(handler: ChannelUpstreamHandler) extends ChannelPipelineFactory { + def getPipeline: ChannelPipeline = { + val encap = List(new LengthFieldPrepender(4), new LengthFieldBasedFrameDecoder(10000, 0, 4, 0, 4)) + val proto = List(new ProtobufEncoder, new ProtobufDecoder(TestConductorProtocol.Wrapper.getDefaultInstance)) + new StaticChannelPipeline(encap ::: proto ::: handler :: Nil: _*) + } +} + +sealed trait Role +case object Client extends Role +case object Server extends Role + +object RemoteConnection { + def apply(role: Role, host: String, port: Int, handler: ChannelUpstreamHandler): Channel = { + val sockaddr = new InetSocketAddress(host, port) + role match { + case Client ⇒ + val socketfactory = new NioClientSocketChannelFactory(Executors.newCachedThreadPool, Executors.newCachedThreadPool) + val bootstrap = new ClientBootstrap(socketfactory) + bootstrap.setPipelineFactory(new TestConductorPipelineFactory(handler)) + bootstrap.setOption("tcpNoDelay", true) + bootstrap.connect(sockaddr).getChannel + case Server ⇒ + val socketfactory = new NioServerSocketChannelFactory(Executors.newCachedThreadPool, Executors.newCachedThreadPool) + val bootstrap = new ServerBootstrap(socketfactory) + bootstrap.setPipelineFactory(new TestConductorPipelineFactory(handler)) + bootstrap.setOption("reuseAddress", true) + bootstrap.setOption("child.tcpNoDelay", true) + bootstrap.bind(sockaddr) + } + } + + def getAddrString(channel: Channel) = channel.getRemoteAddress match { + case i: InetSocketAddress ⇒ i.toString + case _ ⇒ "[unknown]" + } +} From 6c786d20b808b4b5989076223fcc7da0d27e71f9 Mon Sep 17 00:00:00 2001 From: Roland Date: Wed, 2 May 2012 21:56:26 +0200 Subject: [PATCH 02/36] porting to 2.0, making it compile: UNTESTED! --- .../remote/netty/NettyRemoteSupport.scala | 24 ++++- .../main/scala/akka/remote/netty/Server.scala | 4 +- .../akka/remote/testconductor/Conductor.scala | 93 ++++++++++++------- .../NetworkFailureInjector.scala | 24 +++-- .../akka/remote/testconductor/Player.scala | 72 ++++++++------ 5 files changed, 135 insertions(+), 82 deletions(-) diff --git a/akka-remote/src/main/scala/akka/remote/netty/NettyRemoteSupport.scala b/akka-remote/src/main/scala/akka/remote/netty/NettyRemoteSupport.scala index 8acd33c7fb..55e2d95636 100644 --- a/akka-remote/src/main/scala/akka/remote/netty/NettyRemoteSupport.scala +++ b/akka-remote/src/main/scala/akka/remote/netty/NettyRemoteSupport.scala @@ -31,9 +31,11 @@ class NettyRemoteTransport(val remoteSettings: RemoteSettings, val system: Actor val settings = new NettySettings(remoteSettings.config.getConfig("akka.remote.netty"), remoteSettings.systemName) + // TODO replace by system.scheduler val timer: HashedWheelTimer = new HashedWheelTimer(system.threadFactory) - val executor = new OrderedMemoryAwareThreadPoolExecutor( + // TODO make configurable + lazy val executor = new OrderedMemoryAwareThreadPoolExecutor( settings.ExecutionPoolSize, settings.MaxChannelMemorySize, settings.MaxTotalMemorySize, @@ -41,6 +43,7 @@ class NettyRemoteTransport(val remoteSettings: RemoteSettings, val system: Actor settings.ExecutionPoolKeepalive.unit, system.threadFactory) + // TODO make configurable/shareable with server socket factory val clientChannelFactory = new NioClientSocketChannelFactory( Executors.newCachedThreadPool(system.threadFactory), Executors.newCachedThreadPool(system.threadFactory)) @@ -50,9 +53,20 @@ class NettyRemoteTransport(val remoteSettings: RemoteSettings, val system: Actor override protected def useUntrustedMode = remoteSettings.UntrustedMode - val server = try new NettyRemoteServer(this) catch { - case ex ⇒ shutdown(); throw ex - } + val server: NettyRemoteServer = try createServer() catch { case NonFatal(ex) ⇒ shutdown(); throw ex } + + /** + * Override this method to inject a subclass of NettyRemoteServer instead of + * the normal one, e.g. for altering the pipeline. + */ + protected def createServer(): NettyRemoteServer = new NettyRemoteServer(this) + + /** + * Override this method to inject a subclass of RemoteClient instead of + * the normal one, e.g. for altering the pipeline. Get this transport’s + * address from `this.address`. + */ + protected def createClient(recipient: Address): RemoteClient = new ActiveRemoteClient(this, recipient, address) // the address is set in start() or from the RemoteServerHandler, whichever comes first private val _address = new AtomicReference[Address] @@ -121,7 +135,7 @@ class NettyRemoteTransport(val remoteSettings: RemoteSettings, val system: Actor //Recheck for addition, race between upgrades case Some(client) ⇒ client //If already populated by other writer case None ⇒ //Populate map - val client = new ActiveRemoteClient(this, recipientAddress, address) + val client = createClient(recipientAddress) client.connect() remoteClients += recipientAddress -> client client diff --git a/akka-remote/src/main/scala/akka/remote/netty/Server.scala b/akka-remote/src/main/scala/akka/remote/netty/Server.scala index 1f18b27c8c..97d3f194f3 100644 --- a/akka-remote/src/main/scala/akka/remote/netty/Server.scala +++ b/akka-remote/src/main/scala/akka/remote/netty/Server.scala @@ -37,13 +37,15 @@ class NettyRemoteServer(val netty: NettyRemoteTransport) { private val bootstrap = { val b = new ServerBootstrap(factory) - b.setPipelineFactory(new RemoteServerPipelineFactory(openChannels, executionHandler, netty)) + b.setPipelineFactory(makePipeline()) b.setOption("backlog", settings.Backlog) b.setOption("tcpNoDelay", true) b.setOption("child.keepAlive", true) b.setOption("reuseAddress", true) b } + + protected def makePipeline(): ChannelPipelineFactory = new RemoteServerPipelineFactory(openChannels, executionHandler, netty) @volatile private[akka] var channel: Channel = _ diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala b/akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala index 58a6a5f88e..3265fc8808 100644 --- a/akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala +++ b/akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala @@ -3,24 +3,41 @@ */ package akka.remote.testconductor -import akka.actor.{ Actor, ActorRef, LoggingFSM, Timeout, UntypedChannel } -import akka.event.EventHandler +import akka.actor.{ Actor, ActorRef, ActorSystem, LoggingFSM, Props } import RemoteConnection.getAddrString -import akka.util.duration._ import TestConductorProtocol._ -import akka.NoStackTrace import org.jboss.netty.channel.{ Channel, SimpleChannelUpstreamHandler, ChannelHandlerContext, ChannelStateEvent, MessageEvent } +import com.typesafe.config.ConfigFactory +import akka.util.Timeout +import akka.util.Duration +import akka.util.duration._ +import akka.pattern.ask +import java.util.concurrent.TimeUnit.MILLISECONDS +import akka.dispatch.Await +import akka.event.LoggingAdapter +import akka.actor.PoisonPill +import akka.event.Logging +import scala.util.control.NoStackTrace object Conductor extends RunControl with FailureInject with BarrierSync { + val system = ActorSystem("conductor", ConfigFactory.load().getConfig("conductor")) + + object Settings { + val config = system.settings.config + + implicit val BarrierTimeout = Timeout(Duration(config.getMilliseconds("barrier-timeout"), MILLISECONDS)) + implicit val QueryTimeout = Timeout(Duration(config.getMilliseconds("query-timeout"), MILLISECONDS)) + } + import Controller._ - private val controller = Actor.actorOf[Controller] + private val controller = system.actorOf(Props[Controller], "controller") controller ! ClientConnected override def enter(name: String*) { - implicit val timeout = Timeout(30 seconds) - name foreach (b ⇒ (controller ? EnterBarrier(b)).get) + import Settings.BarrierTimeout + name foreach (b ⇒ Await.result(controller ? EnterBarrier(b), Duration.Inf)) } override def throttle(node: String, target: String, direction: Direction, rateMBit: Float) { @@ -47,7 +64,10 @@ object Conductor extends RunControl with FailureInject with BarrierSync { controller ! Terminate(node, -1) } - override def getNodes = (controller ? GetNodes).as[List[String]].get + override def getNodes = { + import Settings.QueryTimeout + Await.result(controller ? GetNodes mapTo manifest[List[String]], Duration.Inf) + } override def removeNode(node: String) { controller ! Remove(node) @@ -55,33 +75,33 @@ object Conductor extends RunControl with FailureInject with BarrierSync { } -class ConductorHandler(controller: ActorRef) extends SimpleChannelUpstreamHandler { +class ConductorHandler(system: ActorSystem, controller: ActorRef, log: LoggingAdapter) extends SimpleChannelUpstreamHandler { var clients = Map[Channel, ActorRef]() override def channelConnected(ctx: ChannelHandlerContext, event: ChannelStateEvent) = { val channel = event.getChannel - EventHandler.debug(this, "connection from " + getAddrString(channel)) - val fsm = Actor.actorOf(new ServerFSM(controller, channel)) + log.debug("connection from {}", getAddrString(channel)) + val fsm = system.actorOf(Props(new ServerFSM(controller, channel))) clients += channel -> fsm } override def channelDisconnected(ctx: ChannelHandlerContext, event: ChannelStateEvent) = { val channel = event.getChannel - EventHandler.debug(this, "disconnect from " + getAddrString(channel)) + log.debug("disconnect from {}", getAddrString(channel)) val fsm = clients(channel) - fsm.stop() + fsm ! PoisonPill clients -= channel } override def messageReceived(ctx: ChannelHandlerContext, event: MessageEvent) = { val channel = event.getChannel - EventHandler.debug(this, "message from " + getAddrString(channel) + ": " + event.getMessage) + log.debug("message from {}: {}", getAddrString(channel), event.getMessage) event.getMessage match { case msg: Wrapper if msg.getAllFields.size == 1 ⇒ clients(channel) ! msg case msg ⇒ - EventHandler.info(this, "client " + getAddrString(channel) + " sent garbage '" + msg + "', disconnecting") + log.info("client {} sent garbage '{}', disconnecting", getAddrString(channel), msg) channel.close() } } @@ -104,35 +124,35 @@ class ServerFSM(val controller: ActorRef, val channel: Channel) extends Actor wi startWith(Initial, null) when(Initial, stateTimeout = 10 seconds) { - case Ev(msg: Wrapper) ⇒ + case Event(msg: Wrapper, _) ⇒ if (msg.hasHello) { val hello = msg.getHello controller ! ClientConnected(hello.getName, hello.getHost, hello.getPort) goto(Ready) } else { - EventHandler.warning(this, "client " + getAddrString(channel) + " sent no Hello in first message, disconnecting") + log.warning("client {} sent no Hello in first message, disconnecting", getAddrString(channel)) channel.close() stop() } - case Ev(StateTimeout) ⇒ - EventHandler.info(this, "closing channel to " + getAddrString(channel) + " because of Hello timeout") + case Event(StateTimeout, _) ⇒ + log.info("closing channel to {} because of Hello timeout", getAddrString(channel)) channel.close() stop() } when(Ready) { - case Ev(msg: Wrapper) ⇒ + case Event(msg: Wrapper, _) ⇒ if (msg.hasBarrier) { val barrier = msg.getBarrier controller ! EnterBarrier(barrier.getName) } else { - EventHandler.warning(this, "client " + getAddrString(channel) + " sent unsupported message " + msg) + log.warning("client {} sent unsupported message {}", getAddrString(channel), msg) } stay - case Ev(Send(msg)) ⇒ + case Event(Send(msg), _) ⇒ channel.write(msg) stay - case Ev(EnterBarrier(name)) ⇒ + case Event(EnterBarrier(name), _) ⇒ val barrier = TestConductorProtocol.EnterBarrier.newBuilder.setName(name).build channel.write(Wrapper.newBuilder.setBarrier(barrier).build) stay @@ -152,18 +172,19 @@ object Controller { class Controller extends Actor { import Controller._ - val host = System.getProperty("akka.testconductor.host", "localhost") - val port = Integer.getInteger("akka.testconductor.port", 4545) - val connection = RemoteConnection(Server, host, port, new ConductorHandler(self)) + val config = context.system.settings.config - val barrier = Actor.actorOf[BarrierCoordinator] + val host = config.getString("akka.testconductor.host") + val port = config.getInt("akka.testconductor.port") + val connection = RemoteConnection(Server, host, port, + new ConductorHandler(context.system, self, Logging(context.system, "ConductorHandler"))) + + val barrier = context.actorOf(Props[BarrierCoordinator], "barriers") var nodes = Map[String, NodeInfo]() - override def receive = Actor.loggable(this) { + override def receive = { case ClientConnected(name, host, port) ⇒ - self.channel match { - case ref: ActorRef ⇒ nodes += name -> NodeInfo(name, host, port, ref) - } + nodes += name -> NodeInfo(name, host, port, sender) barrier forward ClientConnected case ClientConnected ⇒ barrier forward ClientConnected @@ -202,7 +223,7 @@ class Controller extends Actor { // TODO: properly remove node from BarrierCoordinator // case Remove(node) => // nodes -= node - case GetNodes ⇒ self reply nodes.keys + case GetNodes ⇒ sender ! nodes.keys } } @@ -211,7 +232,7 @@ object BarrierCoordinator { case object Idle extends State case object Waiting extends State - case class Data(clients: Int, barrier: String, arrived: List[UntypedChannel]) + case class Data(clients: Int, barrier: String, arrived: List[ActorRef]) class BarrierTimeoutException(msg: String) extends RuntimeException(msg) with NoStackTrace } @@ -225,7 +246,7 @@ class BarrierCoordinator extends Actor with LoggingFSM[BarrierCoordinator.State, when(Idle) { case Event(EnterBarrier(name), Data(num, _, _)) ⇒ if (num == 0) throw new IllegalStateException("no client expected yet") - goto(Waiting) using Data(num, name, self.channel :: Nil) + goto(Waiting) using Data(num, name, sender :: Nil) case Event(ClientConnected, d @ Data(num, _, _)) ⇒ stay using d.copy(clients = num + 1) case Event(ClientDisconnected, d @ Data(num, _, _)) ⇒ @@ -241,7 +262,7 @@ class BarrierCoordinator extends Actor with LoggingFSM[BarrierCoordinator.State, when(Waiting) { case Event(e @ EnterBarrier(name), d @ Data(num, barrier, arrived)) ⇒ if (name != barrier) throw new IllegalStateException("trying enter barrier '" + name + "' while barrier '" + barrier + "' is active") - val together = self.channel :: arrived + val together = sender :: arrived if (together.size == num) { together foreach (_ ! e) goto(Idle) using Data(num, "", Nil) @@ -254,7 +275,7 @@ class BarrierCoordinator extends Actor with LoggingFSM[BarrierCoordinator.State, val expected = num - 1 if (arrived.size == expected) { val e = EnterBarrier(barrier) - self.channel :: arrived foreach (_ ! e) + sender :: arrived foreach (_ ! e) goto(Idle) using Data(expected, "", Nil) } else { stay using d.copy(clients = expected) diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala b/akka-remote/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala index eec6a2cbf1..88102b5e86 100644 --- a/akka-remote/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala +++ b/akka-remote/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala @@ -4,9 +4,7 @@ package akka.remote.testconductor import java.net.InetSocketAddress - import scala.collection.immutable.Queue - import org.jboss.netty.buffer.ChannelBuffer import org.jboss.netty.channel.ChannelState.BOUND import org.jboss.netty.channel.ChannelState.OPEN @@ -17,34 +15,34 @@ import org.jboss.netty.channel.ChannelHandlerContext import org.jboss.netty.channel.ChannelStateEvent import org.jboss.netty.channel.ChannelUpstreamHandler import org.jboss.netty.channel.MessageEvent - import akka.actor.FSM import akka.actor.Actor import akka.util.duration.doubleToDurationDouble import akka.util.Index -import akka.util.RemoteAddress +import akka.actor.Address +import akka.actor.ActorSystem +import akka.actor.Props object NetworkFailureInjector { - val channels = new Index[RemoteAddress, Channel]() + val channels = new Index[Address, Channel](16, (c1, c2) => c1 compareTo c2) - def close(remote: RemoteAddress): Unit = { - val set = channels.remove(remote) + def close(remote: Address): Unit = { // channels will be cleaned up by the handler - set foreach (_.close()) + for (chs <- channels.remove(remote); c <- chs) c.close() } } -class NetworkFailureInjector extends ChannelUpstreamHandler with ChannelDownstreamHandler { +class NetworkFailureInjector(system: ActorSystem) extends ChannelUpstreamHandler with ChannelDownstreamHandler { import NetworkFailureInjector._ // local cache of remote address - private var remote: Option[RemoteAddress] = None + private var remote: Option[Address] = None // everything goes via these Throttle actors to enable easy steering - private val sender = Actor.actorOf(new Throttle(_.sendDownstream(_))) - private val receiver = Actor.actorOf(new Throttle(_.sendUpstream(_))) + private val sender = system.actorOf(Props(new Throttle(_.sendDownstream(_)))) + private val receiver = system.actorOf(Props(new Throttle(_.sendUpstream(_)))) /* * State, Data and Messages for the internal Throttle actor @@ -135,7 +133,7 @@ class NetworkFailureInjector extends ChannelUpstreamHandler with ChannelDownstre case null ⇒ remote = remote flatMap { a ⇒ channels.remove(a, state.getChannel); None } case a: InetSocketAddress ⇒ - val addr = RemoteAddress(a) + val addr = Address("akka", "XXX", a.getHostName, a.getPort) channels.put(addr, state.getChannel) remote = Some(addr) } diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/Player.scala b/akka-remote/src/main/scala/akka/remote/testconductor/Player.scala index 16abe5bb27..029045394c 100644 --- a/akka-remote/src/main/scala/akka/remote/testconductor/Player.scala +++ b/akka-remote/src/main/scala/akka/remote/testconductor/Player.scala @@ -3,25 +3,42 @@ */ package akka.remote.testconductor -import akka.actor.{ Actor, ActorRef, LoggingFSM, Timeout, UntypedChannel } -import akka.event.EventHandler +import akka.actor.{ Actor, ActorRef, ActorSystem, LoggingFSM, Props } import RemoteConnection.getAddrString import akka.util.duration._ import TestConductorProtocol._ -import akka.NoStackTrace import org.jboss.netty.channel.{ Channel, SimpleChannelUpstreamHandler, ChannelHandlerContext, ChannelStateEvent, MessageEvent } import com.eaio.uuid.UUID +import com.typesafe.config.ConfigFactory +import akka.util.Timeout +import akka.util.Duration +import java.util.concurrent.TimeUnit.MILLISECONDS +import akka.pattern.ask +import akka.dispatch.Await +import scala.util.control.NoStackTrace +import akka.actor.Status +import akka.event.LoggingAdapter +import akka.actor.PoisonPill +import akka.event.Logging object Player extends BarrierSync { - private val server = Actor.actorOf[ClientFSM] + val system = ActorSystem("Player", ConfigFactory.load().getConfig("player")) + + object Settings { + val config = system.settings.config + + implicit val BarrierTimeout = Timeout(Duration(config.getMilliseconds("barrier-timeout"), MILLISECONDS)) + } + + private val server = system.actorOf(Props[ClientFSM], "client") override def enter(name: String*) { - EventHandler.debug(this, "entering barriers " + name.mkString("(", ", ", ")")) - implicit val timeout = Timeout(30 seconds) + system.log.debug("entering barriers " + name.mkString("(", ", ", ")")) name foreach { b ⇒ - (server ? EnterBarrier(b)).get - EventHandler.debug(this, "passed barrier " + b) + import Settings.BarrierTimeout + Await.result(server ? EnterBarrier(b), Duration.Inf) + system.log.debug("passed barrier {}", b) } } } @@ -31,7 +48,7 @@ object ClientFSM { case object Connecting extends State case object Connected extends State - case class Data(channel: Channel, msg: Either[List[ClientOp], (String, UntypedChannel)]) + case class Data(channel: Channel, msg: Either[List[ClientOp], (String, ActorRef)]) class ConnectionFailure(msg: String) extends RuntimeException(msg) with NoStackTrace case object Disconnected @@ -39,14 +56,16 @@ object ClientFSM { class ClientFSM extends Actor with LoggingFSM[ClientFSM.State, ClientFSM.Data] { import ClientFSM._ - import akka.actor.FSM._ - val name = System.getProperty("akka.testconductor.name", (new UUID).toString) - val host = System.getProperty("akka.testconductor.host", "localhost") - val port = Integer.getInteger("akka.testconductor.port", 4545) - val handler = new PlayerHandler(self) + val config = context.system.settings.config - val myself = Actor.remote.address + val name = config.getString("akka.testconductor.name") + val host = config.getString("akka.testconductor.host") + val port = config.getInt("akka.testconductor.port") + val handler = new PlayerHandler(self, Logging(context.system, "PlayerHandler")) + + val myself = "XXX" + val myport = 12345 startWith(Connecting, Data(RemoteConnection(Client, host, port, handler), Left(Nil))) @@ -54,7 +73,7 @@ class ClientFSM extends Actor with LoggingFSM[ClientFSM.State, ClientFSM.Data] { case Event(msg: ClientOp, Data(channel, Left(msgs))) ⇒ stay using Data(channel, Left(msg :: msgs)) case Event(Connected, Data(channel, Left(msgs))) ⇒ - val hello = Hello.newBuilder.setName(name).setHost(myself.getAddress.getHostAddress).setPort(myself.getPort).build + val hello = Hello.newBuilder.setName(name).setHost(myself).setPort(myport).build channel.write(Wrapper.newBuilder.setHello(hello).build) msgs.reverse foreach sendMsg(channel) goto(Connected) using Data(channel, Left(Nil)) @@ -62,23 +81,23 @@ class ClientFSM extends Actor with LoggingFSM[ClientFSM.State, ClientFSM.Data] { // System.exit(1) stop case Event(StateTimeout, _) ⇒ - EventHandler.error(this, "connect timeout to TestConductor") + log.error("connect timeout to TestConductor") // System.exit(1) stop } when(Connected) { case Event(Disconnected, _) ⇒ - EventHandler.info(this, "disconnected from TestConductor") + log.info("disconnected from TestConductor") throw new ConnectionFailure("disconnect") case Event(msg: EnterBarrier, Data(channel, _)) ⇒ sendMsg(channel)(msg) - stay using Data(channel, Right((msg.name, self.channel))) + stay using Data(channel, Right((msg.name, sender))) case Event(msg: Wrapper, Data(channel, Right((barrier, sender)))) if msg.getAllFields.size == 1 ⇒ if (msg.hasBarrier) { val b = msg.getBarrier.getName if (b != barrier) { - sender.sendException(new RuntimeException("wrong barrier " + b + " received while waiting for " + barrier)) + sender ! Status.Failure(new RuntimeException("wrong barrier " + b + " received while waiting for " + barrier)) } else { sender ! b } @@ -101,31 +120,30 @@ class ClientFSM extends Actor with LoggingFSM[ClientFSM.State, ClientFSM.Data] { } -class PlayerHandler(fsm: ActorRef) extends SimpleChannelUpstreamHandler { +class PlayerHandler(fsm: ActorRef, log: LoggingAdapter) extends SimpleChannelUpstreamHandler { import ClientFSM._ override def channelConnected(ctx: ChannelHandlerContext, event: ChannelStateEvent) = { val channel = event.getChannel - EventHandler.debug(this, "connected to " + getAddrString(channel)) - while (!fsm.isRunning) Thread.sleep(100) + log.debug("connected to {}", getAddrString(channel)) fsm ! Connected } override def channelDisconnected(ctx: ChannelHandlerContext, event: ChannelStateEvent) = { val channel = event.getChannel - EventHandler.debug(this, "disconnected from " + getAddrString(channel)) - fsm.stop() + log.debug("disconnected from {}", getAddrString(channel)) + fsm ! PoisonPill } override def messageReceived(ctx: ChannelHandlerContext, event: MessageEvent) = { val channel = event.getChannel - EventHandler.debug(this, "message from " + getAddrString(channel) + ": " + event.getMessage) + log.debug("message from {}: {}", getAddrString(channel), event.getMessage) event.getMessage match { case msg: Wrapper if msg.getAllFields.size == 1 ⇒ fsm ! msg case msg ⇒ - EventHandler.info(this, "server " + getAddrString(channel) + " sent garbage '" + msg + "', disconnecting") + log.info("server {} sent garbage '{}', disconnecting", getAddrString(channel), msg) channel.close() } } From 5cf0fa66f803caffee4100507263ab87a8adf71a Mon Sep 17 00:00:00 2001 From: Roland Date: Thu, 3 May 2012 20:48:27 +0200 Subject: [PATCH 03/36] TestConductor: convert to Akka Extension and add test - make start-up synchronous and explicit for client and server - server can be queried for actual port, client requires explicit port - simple multi-jvm-test for verification of TestConductor barriers --- .../testconductor/TestConductorProtocol.java | 1230 +++++++++++++---- .../main/protocol/TestConductorProtocol.proto | 17 +- akka-remote/src/main/resources/reference.conf | 21 + .../remote/netty/NettyRemoteSupport.scala | 6 +- .../main/scala/akka/remote/netty/Server.scala | 2 +- .../akka/remote/testconductor/Conductor.scala | 63 +- .../akka/remote/testconductor/Extension.scala | 31 + .../akka/remote/testconductor/Features.scala | 10 + .../NetworkFailureInjector.scala | 4 +- .../akka/remote/testconductor/Player.scala | 68 +- .../akka/remote/testconductor/package.scala | 19 + .../testconductor/TestConductorSpec.scala | 52 + 12 files changed, 1170 insertions(+), 353 deletions(-) create mode 100644 akka-remote/src/main/scala/akka/remote/testconductor/Extension.scala create mode 100644 akka-remote/src/main/scala/akka/remote/testconductor/package.scala create mode 100644 akka-remote/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala diff --git a/akka-remote/src/main/java/akka/remote/testconductor/TestConductorProtocol.java b/akka-remote/src/main/java/akka/remote/testconductor/TestConductorProtocol.java index e9065b53e4..0b2950018f 100644 --- a/akka-remote/src/main/java/akka/remote/testconductor/TestConductorProtocol.java +++ b/akka-remote/src/main/java/akka/remote/testconductor/TestConductorProtocol.java @@ -905,13 +905,10 @@ public final class TestConductorProtocol { boolean hasName(); String getName(); - // required string host = 2; - boolean hasHost(); - String getHost(); - - // required int32 port = 3; - boolean hasPort(); - int getPort(); + // required .Address address = 2; + boolean hasAddress(); + akka.remote.testconductor.TestConductorProtocol.Address getAddress(); + akka.remote.testconductor.TestConductorProtocol.AddressOrBuilder getAddressOrBuilder(); } public static final class Hello extends com.google.protobuf.GeneratedMessage @@ -974,52 +971,22 @@ public final class TestConductorProtocol { } } - // required string host = 2; - public static final int HOST_FIELD_NUMBER = 2; - private java.lang.Object host_; - public boolean hasHost() { + // required .Address address = 2; + public static final int ADDRESS_FIELD_NUMBER = 2; + private akka.remote.testconductor.TestConductorProtocol.Address address_; + public boolean hasAddress() { return ((bitField0_ & 0x00000002) == 0x00000002); } - public String getHost() { - java.lang.Object ref = host_; - if (ref instanceof String) { - return (String) ref; - } else { - com.google.protobuf.ByteString bs = - (com.google.protobuf.ByteString) ref; - String s = bs.toStringUtf8(); - if (com.google.protobuf.Internal.isValidUtf8(bs)) { - host_ = s; - } - return s; - } + public akka.remote.testconductor.TestConductorProtocol.Address getAddress() { + return address_; } - private com.google.protobuf.ByteString getHostBytes() { - java.lang.Object ref = host_; - if (ref instanceof String) { - com.google.protobuf.ByteString b = - com.google.protobuf.ByteString.copyFromUtf8((String) ref); - host_ = b; - return b; - } else { - return (com.google.protobuf.ByteString) ref; - } - } - - // required int32 port = 3; - public static final int PORT_FIELD_NUMBER = 3; - private int port_; - public boolean hasPort() { - return ((bitField0_ & 0x00000004) == 0x00000004); - } - public int getPort() { - return port_; + public akka.remote.testconductor.TestConductorProtocol.AddressOrBuilder getAddressOrBuilder() { + return address_; } private void initFields() { name_ = ""; - host_ = ""; - port_ = 0; + address_ = akka.remote.testconductor.TestConductorProtocol.Address.getDefaultInstance(); } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -1030,11 +997,11 @@ public final class TestConductorProtocol { memoizedIsInitialized = 0; return false; } - if (!hasHost()) { + if (!hasAddress()) { memoizedIsInitialized = 0; return false; } - if (!hasPort()) { + if (!getAddress().isInitialized()) { memoizedIsInitialized = 0; return false; } @@ -1049,10 +1016,7 @@ public final class TestConductorProtocol { output.writeBytes(1, getNameBytes()); } if (((bitField0_ & 0x00000002) == 0x00000002)) { - output.writeBytes(2, getHostBytes()); - } - if (((bitField0_ & 0x00000004) == 0x00000004)) { - output.writeInt32(3, port_); + output.writeMessage(2, address_); } getUnknownFields().writeTo(output); } @@ -1069,11 +1033,7 @@ public final class TestConductorProtocol { } if (((bitField0_ & 0x00000002) == 0x00000002)) { size += com.google.protobuf.CodedOutputStream - .computeBytesSize(2, getHostBytes()); - } - if (((bitField0_ & 0x00000004) == 0x00000004)) { - size += com.google.protobuf.CodedOutputStream - .computeInt32Size(3, port_); + .computeMessageSize(2, address_); } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; @@ -1191,6 +1151,7 @@ public final class TestConductorProtocol { } private void maybeForceBuilderInitialization() { if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + getAddressFieldBuilder(); } } private static Builder create() { @@ -1201,10 +1162,12 @@ public final class TestConductorProtocol { super.clear(); name_ = ""; bitField0_ = (bitField0_ & ~0x00000001); - host_ = ""; + if (addressBuilder_ == null) { + address_ = akka.remote.testconductor.TestConductorProtocol.Address.getDefaultInstance(); + } else { + addressBuilder_.clear(); + } bitField0_ = (bitField0_ & ~0x00000002); - port_ = 0; - bitField0_ = (bitField0_ & ~0x00000004); return this; } @@ -1250,11 +1213,11 @@ public final class TestConductorProtocol { if (((from_bitField0_ & 0x00000002) == 0x00000002)) { to_bitField0_ |= 0x00000002; } - result.host_ = host_; - if (((from_bitField0_ & 0x00000004) == 0x00000004)) { - to_bitField0_ |= 0x00000004; + if (addressBuilder_ == null) { + result.address_ = address_; + } else { + result.address_ = addressBuilder_.build(); } - result.port_ = port_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -1274,11 +1237,8 @@ public final class TestConductorProtocol { if (other.hasName()) { setName(other.getName()); } - if (other.hasHost()) { - setHost(other.getHost()); - } - if (other.hasPort()) { - setPort(other.getPort()); + if (other.hasAddress()) { + mergeAddress(other.getAddress()); } this.mergeUnknownFields(other.getUnknownFields()); return this; @@ -1289,11 +1249,11 @@ public final class TestConductorProtocol { return false; } - if (!hasHost()) { + if (!hasAddress()) { return false; } - if (!hasPort()) { + if (!getAddress().isInitialized()) { return false; } @@ -1329,13 +1289,12 @@ public final class TestConductorProtocol { break; } case 18: { - bitField0_ |= 0x00000002; - host_ = input.readBytes(); - break; - } - case 24: { - bitField0_ |= 0x00000004; - port_ = input.readInt32(); + akka.remote.testconductor.TestConductorProtocol.Address.Builder subBuilder = akka.remote.testconductor.TestConductorProtocol.Address.newBuilder(); + if (hasAddress()) { + subBuilder.mergeFrom(getAddress()); + } + input.readMessage(subBuilder, extensionRegistry); + setAddress(subBuilder.buildPartial()); break; } } @@ -1380,62 +1339,95 @@ public final class TestConductorProtocol { onChanged(); } - // required string host = 2; - private java.lang.Object host_ = ""; - public boolean hasHost() { + // required .Address address = 2; + private akka.remote.testconductor.TestConductorProtocol.Address address_ = akka.remote.testconductor.TestConductorProtocol.Address.getDefaultInstance(); + private com.google.protobuf.SingleFieldBuilder< + akka.remote.testconductor.TestConductorProtocol.Address, akka.remote.testconductor.TestConductorProtocol.Address.Builder, akka.remote.testconductor.TestConductorProtocol.AddressOrBuilder> addressBuilder_; + public boolean hasAddress() { return ((bitField0_ & 0x00000002) == 0x00000002); } - public String getHost() { - java.lang.Object ref = host_; - if (!(ref instanceof String)) { - String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); - host_ = s; - return s; + public akka.remote.testconductor.TestConductorProtocol.Address getAddress() { + if (addressBuilder_ == null) { + return address_; } else { - return (String) ref; + return addressBuilder_.getMessage(); } } - public Builder setHost(String value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00000002; - host_ = value; - onChanged(); - return this; - } - public Builder clearHost() { - bitField0_ = (bitField0_ & ~0x00000002); - host_ = getDefaultInstance().getHost(); - onChanged(); - return this; - } - void setHost(com.google.protobuf.ByteString value) { + public Builder setAddress(akka.remote.testconductor.TestConductorProtocol.Address value) { + if (addressBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + address_ = value; + onChanged(); + } else { + addressBuilder_.setMessage(value); + } bitField0_ |= 0x00000002; - host_ = value; - onChanged(); - } - - // required int32 port = 3; - private int port_ ; - public boolean hasPort() { - return ((bitField0_ & 0x00000004) == 0x00000004); - } - public int getPort() { - return port_; - } - public Builder setPort(int value) { - bitField0_ |= 0x00000004; - port_ = value; - onChanged(); return this; } - public Builder clearPort() { - bitField0_ = (bitField0_ & ~0x00000004); - port_ = 0; - onChanged(); + public Builder setAddress( + akka.remote.testconductor.TestConductorProtocol.Address.Builder builderForValue) { + if (addressBuilder_ == null) { + address_ = builderForValue.build(); + onChanged(); + } else { + addressBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000002; return this; } + public Builder mergeAddress(akka.remote.testconductor.TestConductorProtocol.Address value) { + if (addressBuilder_ == null) { + if (((bitField0_ & 0x00000002) == 0x00000002) && + address_ != akka.remote.testconductor.TestConductorProtocol.Address.getDefaultInstance()) { + address_ = + akka.remote.testconductor.TestConductorProtocol.Address.newBuilder(address_).mergeFrom(value).buildPartial(); + } else { + address_ = value; + } + onChanged(); + } else { + addressBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000002; + return this; + } + public Builder clearAddress() { + if (addressBuilder_ == null) { + address_ = akka.remote.testconductor.TestConductorProtocol.Address.getDefaultInstance(); + onChanged(); + } else { + addressBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000002); + return this; + } + public akka.remote.testconductor.TestConductorProtocol.Address.Builder getAddressBuilder() { + bitField0_ |= 0x00000002; + onChanged(); + return getAddressFieldBuilder().getBuilder(); + } + public akka.remote.testconductor.TestConductorProtocol.AddressOrBuilder getAddressOrBuilder() { + if (addressBuilder_ != null) { + return addressBuilder_.getMessageOrBuilder(); + } else { + return address_; + } + } + private com.google.protobuf.SingleFieldBuilder< + akka.remote.testconductor.TestConductorProtocol.Address, akka.remote.testconductor.TestConductorProtocol.Address.Builder, akka.remote.testconductor.TestConductorProtocol.AddressOrBuilder> + getAddressFieldBuilder() { + if (addressBuilder_ == null) { + addressBuilder_ = new com.google.protobuf.SingleFieldBuilder< + akka.remote.testconductor.TestConductorProtocol.Address, akka.remote.testconductor.TestConductorProtocol.Address.Builder, akka.remote.testconductor.TestConductorProtocol.AddressOrBuilder>( + address_, + getParentForChildren(), + isClean()); + address_ = null; + } + return addressBuilder_; + } // @@protoc_insertion_point(builder_scope:Hello) } @@ -1831,6 +1823,658 @@ public final class TestConductorProtocol { // @@protoc_insertion_point(class_scope:EnterBarrier) } + public interface AddressOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // required string protocol = 1; + boolean hasProtocol(); + String getProtocol(); + + // required string system = 2; + boolean hasSystem(); + String getSystem(); + + // required string host = 3; + boolean hasHost(); + String getHost(); + + // required int32 port = 4; + boolean hasPort(); + int getPort(); + } + public static final class Address extends + com.google.protobuf.GeneratedMessage + implements AddressOrBuilder { + // Use Address.newBuilder() to construct. + private Address(Builder builder) { + super(builder); + } + private Address(boolean noInit) {} + + private static final Address defaultInstance; + public static Address getDefaultInstance() { + return defaultInstance; + } + + public Address getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return akka.remote.testconductor.TestConductorProtocol.internal_static_Address_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return akka.remote.testconductor.TestConductorProtocol.internal_static_Address_fieldAccessorTable; + } + + private int bitField0_; + // required string protocol = 1; + public static final int PROTOCOL_FIELD_NUMBER = 1; + private java.lang.Object protocol_; + public boolean hasProtocol() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getProtocol() { + java.lang.Object ref = protocol_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + protocol_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getProtocolBytes() { + java.lang.Object ref = protocol_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + protocol_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // required string system = 2; + public static final int SYSTEM_FIELD_NUMBER = 2; + private java.lang.Object system_; + public boolean hasSystem() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public String getSystem() { + java.lang.Object ref = system_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + system_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getSystemBytes() { + java.lang.Object ref = system_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + system_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // required string host = 3; + public static final int HOST_FIELD_NUMBER = 3; + private java.lang.Object host_; + public boolean hasHost() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public String getHost() { + java.lang.Object ref = host_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + host_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getHostBytes() { + java.lang.Object ref = host_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + host_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // required int32 port = 4; + public static final int PORT_FIELD_NUMBER = 4; + private int port_; + public boolean hasPort() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public int getPort() { + return port_; + } + + private void initFields() { + protocol_ = ""; + system_ = ""; + host_ = ""; + port_ = 0; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!hasProtocol()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasSystem()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasHost()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasPort()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(1, getProtocolBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeBytes(2, getSystemBytes()); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeBytes(3, getHostBytes()); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeInt32(4, port_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(1, getProtocolBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(2, getSystemBytes()); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(3, getHostBytes()); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(4, port_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static akka.remote.testconductor.TestConductorProtocol.Address parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.Address parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.Address parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.Address parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.Address parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.Address parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.Address parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static akka.remote.testconductor.TestConductorProtocol.Address parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static akka.remote.testconductor.TestConductorProtocol.Address parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static akka.remote.testconductor.TestConductorProtocol.Address parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(akka.remote.testconductor.TestConductorProtocol.Address prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements akka.remote.testconductor.TestConductorProtocol.AddressOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return akka.remote.testconductor.TestConductorProtocol.internal_static_Address_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return akka.remote.testconductor.TestConductorProtocol.internal_static_Address_fieldAccessorTable; + } + + // Construct using akka.remote.testconductor.TestConductorProtocol.Address.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + protocol_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + system_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + host_ = ""; + bitField0_ = (bitField0_ & ~0x00000004); + port_ = 0; + bitField0_ = (bitField0_ & ~0x00000008); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return akka.remote.testconductor.TestConductorProtocol.Address.getDescriptor(); + } + + public akka.remote.testconductor.TestConductorProtocol.Address getDefaultInstanceForType() { + return akka.remote.testconductor.TestConductorProtocol.Address.getDefaultInstance(); + } + + public akka.remote.testconductor.TestConductorProtocol.Address build() { + akka.remote.testconductor.TestConductorProtocol.Address result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private akka.remote.testconductor.TestConductorProtocol.Address buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + akka.remote.testconductor.TestConductorProtocol.Address result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public akka.remote.testconductor.TestConductorProtocol.Address buildPartial() { + akka.remote.testconductor.TestConductorProtocol.Address result = new akka.remote.testconductor.TestConductorProtocol.Address(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.protocol_ = protocol_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.system_ = system_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.host_ = host_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000008; + } + result.port_ = port_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof akka.remote.testconductor.TestConductorProtocol.Address) { + return mergeFrom((akka.remote.testconductor.TestConductorProtocol.Address)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(akka.remote.testconductor.TestConductorProtocol.Address other) { + if (other == akka.remote.testconductor.TestConductorProtocol.Address.getDefaultInstance()) return this; + if (other.hasProtocol()) { + setProtocol(other.getProtocol()); + } + if (other.hasSystem()) { + setSystem(other.getSystem()); + } + if (other.hasHost()) { + setHost(other.getHost()); + } + if (other.hasPort()) { + setPort(other.getPort()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + if (!hasProtocol()) { + + return false; + } + if (!hasSystem()) { + + return false; + } + if (!hasHost()) { + + return false; + } + if (!hasPort()) { + + return false; + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 10: { + bitField0_ |= 0x00000001; + protocol_ = input.readBytes(); + break; + } + case 18: { + bitField0_ |= 0x00000002; + system_ = input.readBytes(); + break; + } + case 26: { + bitField0_ |= 0x00000004; + host_ = input.readBytes(); + break; + } + case 32: { + bitField0_ |= 0x00000008; + port_ = input.readInt32(); + break; + } + } + } + } + + private int bitField0_; + + // required string protocol = 1; + private java.lang.Object protocol_ = ""; + public boolean hasProtocol() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getProtocol() { + java.lang.Object ref = protocol_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + protocol_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setProtocol(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + protocol_ = value; + onChanged(); + return this; + } + public Builder clearProtocol() { + bitField0_ = (bitField0_ & ~0x00000001); + protocol_ = getDefaultInstance().getProtocol(); + onChanged(); + return this; + } + void setProtocol(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000001; + protocol_ = value; + onChanged(); + } + + // required string system = 2; + private java.lang.Object system_ = ""; + public boolean hasSystem() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public String getSystem() { + java.lang.Object ref = system_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + system_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setSystem(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + system_ = value; + onChanged(); + return this; + } + public Builder clearSystem() { + bitField0_ = (bitField0_ & ~0x00000002); + system_ = getDefaultInstance().getSystem(); + onChanged(); + return this; + } + void setSystem(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000002; + system_ = value; + onChanged(); + } + + // required string host = 3; + private java.lang.Object host_ = ""; + public boolean hasHost() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public String getHost() { + java.lang.Object ref = host_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + host_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setHost(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + host_ = value; + onChanged(); + return this; + } + public Builder clearHost() { + bitField0_ = (bitField0_ & ~0x00000004); + host_ = getDefaultInstance().getHost(); + onChanged(); + return this; + } + void setHost(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000004; + host_ = value; + onChanged(); + } + + // required int32 port = 4; + private int port_ ; + public boolean hasPort() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public int getPort() { + return port_; + } + public Builder setPort(int value) { + bitField0_ |= 0x00000008; + port_ = value; + onChanged(); + return this; + } + public Builder clearPort() { + bitField0_ = (bitField0_ & ~0x00000008); + port_ = 0; + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:Address) + } + + static { + defaultInstance = new Address(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:Address) + } + public interface InjectFailureOrBuilder extends com.google.protobuf.MessageOrBuilder { @@ -1842,19 +2486,16 @@ public final class TestConductorProtocol { boolean hasDirection(); akka.remote.testconductor.TestConductorProtocol.Direction getDirection(); - // optional string host = 3; - boolean hasHost(); - String getHost(); + // optional .Address address = 3; + boolean hasAddress(); + akka.remote.testconductor.TestConductorProtocol.Address getAddress(); + akka.remote.testconductor.TestConductorProtocol.AddressOrBuilder getAddressOrBuilder(); - // optional int32 port = 4; - boolean hasPort(); - int getPort(); - - // optional float rateMBit = 5; + // optional float rateMBit = 6; boolean hasRateMBit(); float getRateMBit(); - // optional int32 exitValue = 6; + // optional int32 exitValue = 7; boolean hasExitValue(); int getExitValue(); } @@ -1907,63 +2548,34 @@ public final class TestConductorProtocol { return direction_; } - // optional string host = 3; - public static final int HOST_FIELD_NUMBER = 3; - private java.lang.Object host_; - public boolean hasHost() { + // optional .Address address = 3; + public static final int ADDRESS_FIELD_NUMBER = 3; + private akka.remote.testconductor.TestConductorProtocol.Address address_; + public boolean hasAddress() { return ((bitField0_ & 0x00000004) == 0x00000004); } - public String getHost() { - java.lang.Object ref = host_; - if (ref instanceof String) { - return (String) ref; - } else { - com.google.protobuf.ByteString bs = - (com.google.protobuf.ByteString) ref; - String s = bs.toStringUtf8(); - if (com.google.protobuf.Internal.isValidUtf8(bs)) { - host_ = s; - } - return s; - } + public akka.remote.testconductor.TestConductorProtocol.Address getAddress() { + return address_; } - private com.google.protobuf.ByteString getHostBytes() { - java.lang.Object ref = host_; - if (ref instanceof String) { - com.google.protobuf.ByteString b = - com.google.protobuf.ByteString.copyFromUtf8((String) ref); - host_ = b; - return b; - } else { - return (com.google.protobuf.ByteString) ref; - } + public akka.remote.testconductor.TestConductorProtocol.AddressOrBuilder getAddressOrBuilder() { + return address_; } - // optional int32 port = 4; - public static final int PORT_FIELD_NUMBER = 4; - private int port_; - public boolean hasPort() { - return ((bitField0_ & 0x00000008) == 0x00000008); - } - public int getPort() { - return port_; - } - - // optional float rateMBit = 5; - public static final int RATEMBIT_FIELD_NUMBER = 5; + // optional float rateMBit = 6; + public static final int RATEMBIT_FIELD_NUMBER = 6; private float rateMBit_; public boolean hasRateMBit() { - return ((bitField0_ & 0x00000010) == 0x00000010); + return ((bitField0_ & 0x00000008) == 0x00000008); } public float getRateMBit() { return rateMBit_; } - // optional int32 exitValue = 6; - public static final int EXITVALUE_FIELD_NUMBER = 6; + // optional int32 exitValue = 7; + public static final int EXITVALUE_FIELD_NUMBER = 7; private int exitValue_; public boolean hasExitValue() { - return ((bitField0_ & 0x00000020) == 0x00000020); + return ((bitField0_ & 0x00000010) == 0x00000010); } public int getExitValue() { return exitValue_; @@ -1972,8 +2584,7 @@ public final class TestConductorProtocol { private void initFields() { failure_ = akka.remote.testconductor.TestConductorProtocol.FailType.Throttle; direction_ = akka.remote.testconductor.TestConductorProtocol.Direction.Send; - host_ = ""; - port_ = 0; + address_ = akka.remote.testconductor.TestConductorProtocol.Address.getDefaultInstance(); rateMBit_ = 0F; exitValue_ = 0; } @@ -1986,6 +2597,12 @@ public final class TestConductorProtocol { memoizedIsInitialized = 0; return false; } + if (hasAddress()) { + if (!getAddress().isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } memoizedIsInitialized = 1; return true; } @@ -2000,16 +2617,13 @@ public final class TestConductorProtocol { output.writeEnum(2, direction_.getNumber()); } if (((bitField0_ & 0x00000004) == 0x00000004)) { - output.writeBytes(3, getHostBytes()); + output.writeMessage(3, address_); } if (((bitField0_ & 0x00000008) == 0x00000008)) { - output.writeInt32(4, port_); + output.writeFloat(6, rateMBit_); } if (((bitField0_ & 0x00000010) == 0x00000010)) { - output.writeFloat(5, rateMBit_); - } - if (((bitField0_ & 0x00000020) == 0x00000020)) { - output.writeInt32(6, exitValue_); + output.writeInt32(7, exitValue_); } getUnknownFields().writeTo(output); } @@ -2030,19 +2644,15 @@ public final class TestConductorProtocol { } if (((bitField0_ & 0x00000004) == 0x00000004)) { size += com.google.protobuf.CodedOutputStream - .computeBytesSize(3, getHostBytes()); + .computeMessageSize(3, address_); } if (((bitField0_ & 0x00000008) == 0x00000008)) { size += com.google.protobuf.CodedOutputStream - .computeInt32Size(4, port_); + .computeFloatSize(6, rateMBit_); } if (((bitField0_ & 0x00000010) == 0x00000010)) { size += com.google.protobuf.CodedOutputStream - .computeFloatSize(5, rateMBit_); - } - if (((bitField0_ & 0x00000020) == 0x00000020)) { - size += com.google.protobuf.CodedOutputStream - .computeInt32Size(6, exitValue_); + .computeInt32Size(7, exitValue_); } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; @@ -2160,6 +2770,7 @@ public final class TestConductorProtocol { } private void maybeForceBuilderInitialization() { if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + getAddressFieldBuilder(); } } private static Builder create() { @@ -2172,14 +2783,16 @@ public final class TestConductorProtocol { bitField0_ = (bitField0_ & ~0x00000001); direction_ = akka.remote.testconductor.TestConductorProtocol.Direction.Send; bitField0_ = (bitField0_ & ~0x00000002); - host_ = ""; + if (addressBuilder_ == null) { + address_ = akka.remote.testconductor.TestConductorProtocol.Address.getDefaultInstance(); + } else { + addressBuilder_.clear(); + } bitField0_ = (bitField0_ & ~0x00000004); - port_ = 0; - bitField0_ = (bitField0_ & ~0x00000008); rateMBit_ = 0F; - bitField0_ = (bitField0_ & ~0x00000010); + bitField0_ = (bitField0_ & ~0x00000008); exitValue_ = 0; - bitField0_ = (bitField0_ & ~0x00000020); + bitField0_ = (bitField0_ & ~0x00000010); return this; } @@ -2229,18 +2842,18 @@ public final class TestConductorProtocol { if (((from_bitField0_ & 0x00000004) == 0x00000004)) { to_bitField0_ |= 0x00000004; } - result.host_ = host_; + if (addressBuilder_ == null) { + result.address_ = address_; + } else { + result.address_ = addressBuilder_.build(); + } if (((from_bitField0_ & 0x00000008) == 0x00000008)) { to_bitField0_ |= 0x00000008; } - result.port_ = port_; + result.rateMBit_ = rateMBit_; if (((from_bitField0_ & 0x00000010) == 0x00000010)) { to_bitField0_ |= 0x00000010; } - result.rateMBit_ = rateMBit_; - if (((from_bitField0_ & 0x00000020) == 0x00000020)) { - to_bitField0_ |= 0x00000020; - } result.exitValue_ = exitValue_; result.bitField0_ = to_bitField0_; onBuilt(); @@ -2264,11 +2877,8 @@ public final class TestConductorProtocol { if (other.hasDirection()) { setDirection(other.getDirection()); } - if (other.hasHost()) { - setHost(other.getHost()); - } - if (other.hasPort()) { - setPort(other.getPort()); + if (other.hasAddress()) { + mergeAddress(other.getAddress()); } if (other.hasRateMBit()) { setRateMBit(other.getRateMBit()); @@ -2285,6 +2895,12 @@ public final class TestConductorProtocol { return false; } + if (hasAddress()) { + if (!getAddress().isInitialized()) { + + return false; + } + } return true; } @@ -2334,22 +2950,21 @@ public final class TestConductorProtocol { break; } case 26: { - bitField0_ |= 0x00000004; - host_ = input.readBytes(); + akka.remote.testconductor.TestConductorProtocol.Address.Builder subBuilder = akka.remote.testconductor.TestConductorProtocol.Address.newBuilder(); + if (hasAddress()) { + subBuilder.mergeFrom(getAddress()); + } + input.readMessage(subBuilder, extensionRegistry); + setAddress(subBuilder.buildPartial()); break; } - case 32: { + case 53: { bitField0_ |= 0x00000008; - port_ = input.readInt32(); - break; - } - case 45: { - bitField0_ |= 0x00000010; rateMBit_ = input.readFloat(); break; } - case 48: { - bitField0_ |= 0x00000020; + case 56: { + bitField0_ |= 0x00000010; exitValue_ = input.readInt32(); break; } @@ -2407,100 +3022,133 @@ public final class TestConductorProtocol { return this; } - // optional string host = 3; - private java.lang.Object host_ = ""; - public boolean hasHost() { + // optional .Address address = 3; + private akka.remote.testconductor.TestConductorProtocol.Address address_ = akka.remote.testconductor.TestConductorProtocol.Address.getDefaultInstance(); + private com.google.protobuf.SingleFieldBuilder< + akka.remote.testconductor.TestConductorProtocol.Address, akka.remote.testconductor.TestConductorProtocol.Address.Builder, akka.remote.testconductor.TestConductorProtocol.AddressOrBuilder> addressBuilder_; + public boolean hasAddress() { return ((bitField0_ & 0x00000004) == 0x00000004); } - public String getHost() { - java.lang.Object ref = host_; - if (!(ref instanceof String)) { - String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); - host_ = s; - return s; + public akka.remote.testconductor.TestConductorProtocol.Address getAddress() { + if (addressBuilder_ == null) { + return address_; } else { - return (String) ref; + return addressBuilder_.getMessage(); } } - public Builder setHost(String value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00000004; - host_ = value; - onChanged(); - return this; - } - public Builder clearHost() { - bitField0_ = (bitField0_ & ~0x00000004); - host_ = getDefaultInstance().getHost(); - onChanged(); - return this; - } - void setHost(com.google.protobuf.ByteString value) { + public Builder setAddress(akka.remote.testconductor.TestConductorProtocol.Address value) { + if (addressBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + address_ = value; + onChanged(); + } else { + addressBuilder_.setMessage(value); + } bitField0_ |= 0x00000004; - host_ = value; - onChanged(); - } - - // optional int32 port = 4; - private int port_ ; - public boolean hasPort() { - return ((bitField0_ & 0x00000008) == 0x00000008); - } - public int getPort() { - return port_; - } - public Builder setPort(int value) { - bitField0_ |= 0x00000008; - port_ = value; - onChanged(); return this; } - public Builder clearPort() { - bitField0_ = (bitField0_ & ~0x00000008); - port_ = 0; - onChanged(); + public Builder setAddress( + akka.remote.testconductor.TestConductorProtocol.Address.Builder builderForValue) { + if (addressBuilder_ == null) { + address_ = builderForValue.build(); + onChanged(); + } else { + addressBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000004; return this; } + public Builder mergeAddress(akka.remote.testconductor.TestConductorProtocol.Address value) { + if (addressBuilder_ == null) { + if (((bitField0_ & 0x00000004) == 0x00000004) && + address_ != akka.remote.testconductor.TestConductorProtocol.Address.getDefaultInstance()) { + address_ = + akka.remote.testconductor.TestConductorProtocol.Address.newBuilder(address_).mergeFrom(value).buildPartial(); + } else { + address_ = value; + } + onChanged(); + } else { + addressBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000004; + return this; + } + public Builder clearAddress() { + if (addressBuilder_ == null) { + address_ = akka.remote.testconductor.TestConductorProtocol.Address.getDefaultInstance(); + onChanged(); + } else { + addressBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + public akka.remote.testconductor.TestConductorProtocol.Address.Builder getAddressBuilder() { + bitField0_ |= 0x00000004; + onChanged(); + return getAddressFieldBuilder().getBuilder(); + } + public akka.remote.testconductor.TestConductorProtocol.AddressOrBuilder getAddressOrBuilder() { + if (addressBuilder_ != null) { + return addressBuilder_.getMessageOrBuilder(); + } else { + return address_; + } + } + private com.google.protobuf.SingleFieldBuilder< + akka.remote.testconductor.TestConductorProtocol.Address, akka.remote.testconductor.TestConductorProtocol.Address.Builder, akka.remote.testconductor.TestConductorProtocol.AddressOrBuilder> + getAddressFieldBuilder() { + if (addressBuilder_ == null) { + addressBuilder_ = new com.google.protobuf.SingleFieldBuilder< + akka.remote.testconductor.TestConductorProtocol.Address, akka.remote.testconductor.TestConductorProtocol.Address.Builder, akka.remote.testconductor.TestConductorProtocol.AddressOrBuilder>( + address_, + getParentForChildren(), + isClean()); + address_ = null; + } + return addressBuilder_; + } - // optional float rateMBit = 5; + // optional float rateMBit = 6; private float rateMBit_ ; public boolean hasRateMBit() { - return ((bitField0_ & 0x00000010) == 0x00000010); + return ((bitField0_ & 0x00000008) == 0x00000008); } public float getRateMBit() { return rateMBit_; } public Builder setRateMBit(float value) { - bitField0_ |= 0x00000010; + bitField0_ |= 0x00000008; rateMBit_ = value; onChanged(); return this; } public Builder clearRateMBit() { - bitField0_ = (bitField0_ & ~0x00000010); + bitField0_ = (bitField0_ & ~0x00000008); rateMBit_ = 0F; onChanged(); return this; } - // optional int32 exitValue = 6; + // optional int32 exitValue = 7; private int exitValue_ ; public boolean hasExitValue() { - return ((bitField0_ & 0x00000020) == 0x00000020); + return ((bitField0_ & 0x00000010) == 0x00000010); } public int getExitValue() { return exitValue_; } public Builder setExitValue(int value) { - bitField0_ |= 0x00000020; + bitField0_ |= 0x00000010; exitValue_ = value; onChanged(); return this; } public Builder clearExitValue() { - bitField0_ = (bitField0_ & ~0x00000020); + bitField0_ = (bitField0_ & ~0x00000010); exitValue_ = 0; onChanged(); return this; @@ -2532,6 +3180,11 @@ public final class TestConductorProtocol { private static com.google.protobuf.GeneratedMessage.FieldAccessorTable internal_static_EnterBarrier_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_Address_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_Address_fieldAccessorTable; private static com.google.protobuf.Descriptors.Descriptor internal_static_InjectFailure_descriptor; private static @@ -2549,16 +3202,17 @@ public final class TestConductorProtocol { "\n\033TestConductorProtocol.proto\"a\n\007Wrapper" + "\022\025\n\005hello\030\001 \001(\0132\006.Hello\022\036\n\007barrier\030\002 \001(\013" + "2\r.EnterBarrier\022\037\n\007failure\030\003 \001(\0132\016.Injec" + - "tFailure\"1\n\005Hello\022\014\n\004name\030\001 \002(\t\022\014\n\004host\030" + - "\002 \002(\t\022\014\n\004port\030\003 \002(\005\"\034\n\014EnterBarrier\022\014\n\004n" + - "ame\030\001 \002(\t\"\213\001\n\rInjectFailure\022\032\n\007failure\030\001" + - " \002(\0162\t.FailType\022\035\n\tdirection\030\002 \001(\0162\n.Dir" + - "ection\022\014\n\004host\030\003 \001(\t\022\014\n\004port\030\004 \001(\005\022\020\n\010ra" + - "teMBit\030\005 \001(\002\022\021\n\texitValue\030\006 \001(\005*A\n\010FailT" + - "ype\022\014\n\010Throttle\020\001\022\016\n\nDisconnect\020\002\022\t\n\005Abo", - "rt\020\003\022\014\n\010Shutdown\020\004*\"\n\tDirection\022\010\n\004Send\020" + - "\001\022\013\n\007Receive\020\002B\035\n\031akka.remote.testconduc" + - "torH\001" + "tFailure\"0\n\005Hello\022\014\n\004name\030\001 \002(\t\022\031\n\007addre" + + "ss\030\002 \002(\0132\010.Address\"\034\n\014EnterBarrier\022\014\n\004na" + + "me\030\001 \002(\t\"G\n\007Address\022\020\n\010protocol\030\001 \002(\t\022\016\n" + + "\006system\030\002 \002(\t\022\014\n\004host\030\003 \002(\t\022\014\n\004port\030\004 \002(" + + "\005\"\212\001\n\rInjectFailure\022\032\n\007failure\030\001 \002(\0162\t.F" + + "ailType\022\035\n\tdirection\030\002 \001(\0162\n.Direction\022\031" + + "\n\007address\030\003 \001(\0132\010.Address\022\020\n\010rateMBit\030\006 ", + "\001(\002\022\021\n\texitValue\030\007 \001(\005*A\n\010FailType\022\014\n\010Th" + + "rottle\020\001\022\016\n\nDisconnect\020\002\022\t\n\005Abort\020\003\022\014\n\010S" + + "hutdown\020\004*\"\n\tDirection\022\010\n\004Send\020\001\022\013\n\007Rece" + + "ive\020\002B\035\n\031akka.remote.testconductorH\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -2578,7 +3232,7 @@ public final class TestConductorProtocol { internal_static_Hello_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_Hello_descriptor, - new java.lang.String[] { "Name", "Host", "Port", }, + new java.lang.String[] { "Name", "Address", }, akka.remote.testconductor.TestConductorProtocol.Hello.class, akka.remote.testconductor.TestConductorProtocol.Hello.Builder.class); internal_static_EnterBarrier_descriptor = @@ -2589,12 +3243,20 @@ public final class TestConductorProtocol { new java.lang.String[] { "Name", }, akka.remote.testconductor.TestConductorProtocol.EnterBarrier.class, akka.remote.testconductor.TestConductorProtocol.EnterBarrier.Builder.class); - internal_static_InjectFailure_descriptor = + internal_static_Address_descriptor = getDescriptor().getMessageTypes().get(3); + internal_static_Address_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_Address_descriptor, + new java.lang.String[] { "Protocol", "System", "Host", "Port", }, + akka.remote.testconductor.TestConductorProtocol.Address.class, + akka.remote.testconductor.TestConductorProtocol.Address.Builder.class); + internal_static_InjectFailure_descriptor = + getDescriptor().getMessageTypes().get(4); internal_static_InjectFailure_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_InjectFailure_descriptor, - new java.lang.String[] { "Failure", "Direction", "Host", "Port", "RateMBit", "ExitValue", }, + new java.lang.String[] { "Failure", "Direction", "Address", "RateMBit", "ExitValue", }, akka.remote.testconductor.TestConductorProtocol.InjectFailure.class, akka.remote.testconductor.TestConductorProtocol.InjectFailure.Builder.class); return null; diff --git a/akka-remote/src/main/protocol/TestConductorProtocol.proto b/akka-remote/src/main/protocol/TestConductorProtocol.proto index 1db35a7516..213820e687 100644 --- a/akka-remote/src/main/protocol/TestConductorProtocol.proto +++ b/akka-remote/src/main/protocol/TestConductorProtocol.proto @@ -19,14 +19,20 @@ message Wrapper { message Hello { required string name = 1; - required string host = 2; - required int32 port = 3; + required Address address = 2; } message EnterBarrier { required string name = 1; } +message Address { + required string protocol = 1; + required string system = 2; + required string host = 3; + required int32 port = 4; +} + enum FailType { Throttle = 1; Disconnect = 2; @@ -40,9 +46,8 @@ enum Direction { message InjectFailure { required FailType failure = 1; optional Direction direction = 2; - optional string host = 3; - optional int32 port = 4; - optional float rateMBit = 5; - optional int32 exitValue = 6; + optional Address address = 3; + optional float rateMBit = 6; + optional int32 exitValue = 7; } diff --git a/akka-remote/src/main/resources/reference.conf b/akka-remote/src/main/resources/reference.conf index 1438904fe2..384d00b55d 100644 --- a/akka-remote/src/main/resources/reference.conf +++ b/akka-remote/src/main/resources/reference.conf @@ -155,4 +155,25 @@ akka { type = PinnedDispatcher } } + + testconductor { + + # Timeout for joining a barrier: this is the maximum time any participants + # waits for everybody else to join a named barrier. + barrier-timeout = 30s + + # Timeout for interrogation of TestConductor’s Controller actor + query-timeout = 5s + + # Default port to start the conductor on; 0 means + port = 0 + + # Hostname of the TestConductor server, used by the server to bind to the IP + # and by the client to connect to it. + host = localhost + + # Name of the TestConductor client (for identification on the server e.g. for + # failure injection) + name = "noname" + } } diff --git a/akka-remote/src/main/scala/akka/remote/netty/NettyRemoteSupport.scala b/akka-remote/src/main/scala/akka/remote/netty/NettyRemoteSupport.scala index 55e2d95636..c3a41f8275 100644 --- a/akka-remote/src/main/scala/akka/remote/netty/NettyRemoteSupport.scala +++ b/akka-remote/src/main/scala/akka/remote/netty/NettyRemoteSupport.scala @@ -56,13 +56,13 @@ class NettyRemoteTransport(val remoteSettings: RemoteSettings, val system: Actor val server: NettyRemoteServer = try createServer() catch { case NonFatal(ex) ⇒ shutdown(); throw ex } /** - * Override this method to inject a subclass of NettyRemoteServer instead of + * Override this method to inject a subclass of NettyRemoteServer instead of * the normal one, e.g. for altering the pipeline. */ protected def createServer(): NettyRemoteServer = new NettyRemoteServer(this) - + /** - * Override this method to inject a subclass of RemoteClient instead of + * Override this method to inject a subclass of RemoteClient instead of * the normal one, e.g. for altering the pipeline. Get this transport’s * address from `this.address`. */ diff --git a/akka-remote/src/main/scala/akka/remote/netty/Server.scala b/akka-remote/src/main/scala/akka/remote/netty/Server.scala index 97d3f194f3..ac4289e8ae 100644 --- a/akka-remote/src/main/scala/akka/remote/netty/Server.scala +++ b/akka-remote/src/main/scala/akka/remote/netty/Server.scala @@ -44,7 +44,7 @@ class NettyRemoteServer(val netty: NettyRemoteTransport) { b.setOption("reuseAddress", true) b } - + protected def makePipeline(): ChannelPipelineFactory = new RemoteServerPipelineFactory(openChannels, executionHandler, netty) @volatile diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala b/akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala index 3265fc8808..c46e22eb9f 100644 --- a/akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala +++ b/akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala @@ -18,26 +18,30 @@ import akka.event.LoggingAdapter import akka.actor.PoisonPill import akka.event.Logging import scala.util.control.NoStackTrace +import akka.event.LoggingReceive +import akka.actor.Address +import java.net.InetSocketAddress -object Conductor extends RunControl with FailureInject with BarrierSync { - - val system = ActorSystem("conductor", ConfigFactory.load().getConfig("conductor")) - - object Settings { - val config = system.settings.config - - implicit val BarrierTimeout = Timeout(Duration(config.getMilliseconds("barrier-timeout"), MILLISECONDS)) - implicit val QueryTimeout = Timeout(Duration(config.getMilliseconds("query-timeout"), MILLISECONDS)) - } +trait Conductor extends RunControl with FailureInject { this: TestConductorExt ⇒ import Controller._ - private val controller = system.actorOf(Props[Controller], "controller") - controller ! ClientConnected + private var _controller: ActorRef = _ + private def controller: ActorRef = _controller match { + case null ⇒ throw new RuntimeException("TestConductorServer was not started") + case x ⇒ x + } - override def enter(name: String*) { + override def startController() { + if (_controller ne null) throw new RuntimeException("TestConductorServer was already started") + _controller = system.actorOf(Props[Controller], "controller") import Settings.BarrierTimeout - name foreach (b ⇒ Await.result(controller ? EnterBarrier(b), Duration.Inf)) + startClient(Await.result(controller ? GetPort mapTo, Duration.Inf)) + } + + override def port: Int = { + import Settings.QueryTimeout + Await.result(controller ? GetPort mapTo, Duration.Inf) } override def throttle(node: String, target: String, direction: Direction, rateMBit: Float) { @@ -127,7 +131,7 @@ class ServerFSM(val controller: ActorRef, val channel: Channel) extends Actor wi case Event(msg: Wrapper, _) ⇒ if (msg.hasHello) { val hello = msg.getHello - controller ! ClientConnected(hello.getName, hello.getHost, hello.getPort) + controller ! ClientConnected(hello.getName, hello.getAddress) goto(Ready) } else { log.warning("client {} sent no Hello in first message, disconnecting", getAddrString(channel)) @@ -162,29 +166,28 @@ class ServerFSM(val controller: ActorRef, val channel: Channel) extends Actor wi } object Controller { - case class ClientConnected(name: String, host: String, port: Int) + case class ClientConnected(name: String, address: Address) case class ClientDisconnected(name: String) case object GetNodes + case object GetPort - case class NodeInfo(name: String, host: String, port: Int, fsm: ActorRef) + case class NodeInfo(name: String, addr: Address, fsm: ActorRef) } class Controller extends Actor { import Controller._ - val config = context.system.settings.config - - val host = config.getString("akka.testconductor.host") - val port = config.getInt("akka.testconductor.port") - val connection = RemoteConnection(Server, host, port, + val settings = TestConductor().Settings + val connection = RemoteConnection(Server, settings.host, settings.port, new ConductorHandler(context.system, self, Logging(context.system, "ConductorHandler"))) val barrier = context.actorOf(Props[BarrierCoordinator], "barriers") var nodes = Map[String, NodeInfo]() - override def receive = { - case ClientConnected(name, host, port) ⇒ - nodes += name -> NodeInfo(name, host, port, sender) + override def receive = LoggingReceive { + case "ready?" ⇒ sender ! "yes" + case ClientConnected(name, addr) ⇒ + nodes += name -> NodeInfo(name, addr, sender) barrier forward ClientConnected case ClientConnected ⇒ barrier forward ClientConnected @@ -199,8 +202,7 @@ class Controller extends Actor { InjectFailure.newBuilder .setFailure(FailType.Throttle) .setDirection(TestConductorProtocol.Direction.valueOf(direction.toString)) - .setHost(t.host) - .setPort(t.port) + .setAddress(t.addr) .setRateMBit(rateMBit) .build nodes(node).fsm ! ServerFSM.Send(Wrapper.newBuilder.setFailure(throttle).build) @@ -209,8 +211,7 @@ class Controller extends Actor { val disconnect = InjectFailure.newBuilder .setFailure(if (abort) FailType.Abort else FailType.Disconnect) - .setHost(t.host) - .setPort(t.port) + .setAddress(t.addr) .build nodes(node).fsm ! ServerFSM.Send(Wrapper.newBuilder.setFailure(disconnect).build) case Terminate(node, exitValueOrKill) ⇒ @@ -224,6 +225,10 @@ class Controller extends Actor { // case Remove(node) => // nodes -= node case GetNodes ⇒ sender ! nodes.keys + case GetPort ⇒ + sender ! (connection.getLocalAddress match { + case inet: InetSocketAddress ⇒ inet.getPort + }) } } diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/Extension.scala b/akka-remote/src/main/scala/akka/remote/testconductor/Extension.scala new file mode 100644 index 0000000000..94847664c9 --- /dev/null +++ b/akka-remote/src/main/scala/akka/remote/testconductor/Extension.scala @@ -0,0 +1,31 @@ +package akka.remote.testconductor + +import akka.actor.ExtensionKey +import akka.actor.Extension +import akka.actor.ExtendedActorSystem +import akka.remote.RemoteActorRefProvider +import akka.actor.ActorContext +import akka.util.{ Duration, Timeout } +import java.util.concurrent.TimeUnit.MILLISECONDS + +object TestConductor extends ExtensionKey[TestConductorExt] { + def apply()(implicit ctx: ActorContext): TestConductorExt = apply(ctx.system) +} + +class TestConductorExt(val system: ExtendedActorSystem) extends Extension with Conductor with Player { + + object Settings { + val config = system.settings.config + + implicit val BarrierTimeout = Timeout(Duration(config.getMilliseconds("akka.testconductor.barrier-timeout"), MILLISECONDS)) + implicit val QueryTimeout = Timeout(Duration(config.getMilliseconds("akka.testconductor.query-timeout"), MILLISECONDS)) + + val name = config.getString("akka.testconductor.name") + val host = config.getString("akka.testconductor.host") + val port = config.getInt("akka.testconductor.port") + } + + val transport = system.provider.asInstanceOf[RemoteActorRefProvider].transport + val address = transport.address + +} \ No newline at end of file diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/Features.scala b/akka-remote/src/main/scala/akka/remote/testconductor/Features.scala index 399b58337b..930be600c2 100644 --- a/akka-remote/src/main/scala/akka/remote/testconductor/Features.scala +++ b/akka-remote/src/main/scala/akka/remote/testconductor/Features.scala @@ -49,6 +49,16 @@ trait FailureInject { trait RunControl { + /** + * Start the server port. + */ + def startController(): Unit + + /** + * Get the actual port used by the server. + */ + def port: Int + /** * Tell the remote node to shut itself down using System.exit with the given * exitValue. diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala b/akka-remote/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala index 88102b5e86..6569d81acc 100644 --- a/akka-remote/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala +++ b/akka-remote/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala @@ -25,11 +25,11 @@ import akka.actor.Props object NetworkFailureInjector { - val channels = new Index[Address, Channel](16, (c1, c2) => c1 compareTo c2) + val channels = new Index[Address, Channel](16, (c1, c2) ⇒ c1 compareTo c2) def close(remote: Address): Unit = { // channels will be cleaned up by the handler - for (chs <- channels.remove(remote); c <- chs) c.close() + for (chs ← channels.remove(remote); c ← chs) c.close() } } diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/Player.scala b/akka-remote/src/main/scala/akka/remote/testconductor/Player.scala index 029045394c..93aa6bc33d 100644 --- a/akka-remote/src/main/scala/akka/remote/testconductor/Player.scala +++ b/akka-remote/src/main/scala/akka/remote/testconductor/Player.scala @@ -21,23 +21,40 @@ import akka.event.LoggingAdapter import akka.actor.PoisonPill import akka.event.Logging -object Player extends BarrierSync { +trait Player extends BarrierSync { this: TestConductorExt ⇒ - val system = ActorSystem("Player", ConfigFactory.load().getConfig("player")) - - object Settings { - val config = system.settings.config - - implicit val BarrierTimeout = Timeout(Duration(config.getMilliseconds("barrier-timeout"), MILLISECONDS)) + private var _client: ActorRef = _ + private def client = _client match { + case null ⇒ throw new IllegalStateException("TestConductor client not yet started") + case x ⇒ x } - private val server = system.actorOf(Props[ClientFSM], "client") + def startClient(port: Int) { + import ClientFSM._ + import akka.actor.FSM._ + import Settings.BarrierTimeout + + if (_client ne null) throw new IllegalStateException("TestConductorClient already started") + _client = system.actorOf(Props(new ClientFSM(port)), "TestConductorClient") + val a = system.actorOf(Props(new Actor { + var waiting: ActorRef = _ + def receive = { + case fsm: ActorRef ⇒ waiting = sender; fsm ! SubscribeTransitionCallBack(self) + case Transition(_, Connecting, Connected) ⇒ waiting ! "okay" + case t: Transition[_] ⇒ waiting ! Status.Failure(new RuntimeException("unexpected transition: " + t)) + case CurrentState(_, Connected) ⇒ waiting ! "okay" + case _: CurrentState[_] ⇒ + } + })) + + Await.result(a ? client, Duration.Inf) + } override def enter(name: String*) { system.log.debug("entering barriers " + name.mkString("(", ", ", ")")) name foreach { b ⇒ import Settings.BarrierTimeout - Await.result(server ? EnterBarrier(b), Duration.Inf) + Await.result(client ? EnterBarrier(b), Duration.Inf) system.log.debug("passed barrier {}", b) } } @@ -48,35 +65,28 @@ object ClientFSM { case object Connecting extends State case object Connected extends State - case class Data(channel: Channel, msg: Either[List[ClientOp], (String, ActorRef)]) + case class Data(channel: Channel, barrier: Option[(String, ActorRef)]) class ConnectionFailure(msg: String) extends RuntimeException(msg) with NoStackTrace case object Disconnected } -class ClientFSM extends Actor with LoggingFSM[ClientFSM.State, ClientFSM.Data] { +class ClientFSM(port: Int) extends Actor with LoggingFSM[ClientFSM.State, ClientFSM.Data] { import ClientFSM._ - val config = context.system.settings.config + val settings = TestConductor().Settings - val name = config.getString("akka.testconductor.name") - val host = config.getString("akka.testconductor.host") - val port = config.getInt("akka.testconductor.port") val handler = new PlayerHandler(self, Logging(context.system, "PlayerHandler")) - val myself = "XXX" - val myport = 12345 - - startWith(Connecting, Data(RemoteConnection(Client, host, port, handler), Left(Nil))) + startWith(Connecting, Data(RemoteConnection(Client, settings.host, port, handler), None)) when(Connecting, stateTimeout = 10 seconds) { - case Event(msg: ClientOp, Data(channel, Left(msgs))) ⇒ - stay using Data(channel, Left(msg :: msgs)) - case Event(Connected, Data(channel, Left(msgs))) ⇒ - val hello = Hello.newBuilder.setName(name).setHost(myself).setPort(myport).build + case Event(msg: ClientOp, _) ⇒ + stay replying Status.Failure(new IllegalStateException("not connected yet")) + case Event(Connected, d @ Data(channel, _)) ⇒ + val hello = Hello.newBuilder.setName(settings.name).setAddress(TestConductor().address).build channel.write(Wrapper.newBuilder.setHello(hello).build) - msgs.reverse foreach sendMsg(channel) - goto(Connected) using Data(channel, Left(Nil)) + goto(Connected) case Event(_: ConnectionFailure, _) ⇒ // System.exit(1) stop @@ -92,8 +102,8 @@ class ClientFSM extends Actor with LoggingFSM[ClientFSM.State, ClientFSM.Data] { throw new ConnectionFailure("disconnect") case Event(msg: EnterBarrier, Data(channel, _)) ⇒ sendMsg(channel)(msg) - stay using Data(channel, Right((msg.name, sender))) - case Event(msg: Wrapper, Data(channel, Right((barrier, sender)))) if msg.getAllFields.size == 1 ⇒ + stay using Data(channel, Some(msg.name, sender)) + case Event(msg: Wrapper, Data(channel, Some((barrier, sender)))) if msg.getAllFields.size == 1 ⇒ if (msg.hasBarrier) { val b = msg.getBarrier.getName if (b != barrier) { @@ -102,7 +112,7 @@ class ClientFSM extends Actor with LoggingFSM[ClientFSM.State, ClientFSM.Data] { sender ! b } } - stay using Data(channel, Left(Nil)) + stay using Data(channel, None) } onTermination { @@ -110,6 +120,8 @@ class ClientFSM extends Actor with LoggingFSM[ClientFSM.State, ClientFSM.Data] { channel.close() } + initialize + private def sendMsg(channel: Channel)(msg: ClientOp) { msg match { case EnterBarrier(name) ⇒ diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/package.scala b/akka-remote/src/main/scala/akka/remote/testconductor/package.scala new file mode 100644 index 0000000000..8ebeea90a9 --- /dev/null +++ b/akka-remote/src/main/scala/akka/remote/testconductor/package.scala @@ -0,0 +1,19 @@ +package akka.remote + +import akka.actor.Address +import testconductor.{ TestConductorProtocol ⇒ TCP } + +package object testconductor { + + implicit def address2proto(addr: Address): TCP.Address = + TCP.Address.newBuilder + .setProtocol(addr.protocol) + .setSystem(addr.system) + .setHost(addr.host.get) + .setPort(addr.port.get) + .build + + implicit def address2scala(addr: TCP.Address): Address = + Address(addr.getProtocol, addr.getSystem, addr.getHost, addr.getPort) + +} \ No newline at end of file diff --git a/akka-remote/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala b/akka-remote/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala new file mode 100644 index 0000000000..cae2917577 --- /dev/null +++ b/akka-remote/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala @@ -0,0 +1,52 @@ +package akka.remote.testconductor + +import akka.remote.AkkaRemoteSpec +import com.typesafe.config.ConfigFactory +import akka.remote.AbstractRemoteActorMultiJvmSpec + +object TestConductorMultiJvmSpec extends AbstractRemoteActorMultiJvmSpec { + override def NrOfNodes = 2 + override def commonConfig = ConfigFactory.parseString(""" + akka.loglevel = DEBUG + akka.actor.provider = akka.remote.RemoteActorRefProvider + akka.actor.debug { + receive = on + fsm = on + } + akka.testconductor { + host = localhost + port = 4712 + } + """) + def nameConfig(n: Int) = ConfigFactory.parseString("akka.testconductor.name = node" + n).withFallback(nodeConfigs(n)) +} + +import TestConductorMultiJvmSpec._ + +class TestConductorMultiJvmNode1 extends AkkaRemoteSpec(nameConfig(0)) { + + val nodes = TestConductorMultiJvmSpec.NrOfNodes + + "running a test" in { + val tc = TestConductor(system) + tc.startController() + barrier("start") + barrier("first") + tc.enter("begin") + barrier("end") + } +} + +class TestConductorMultiJvmNode2 extends AkkaRemoteSpec(nameConfig(1)) { + + val nodes = TestConductorMultiJvmSpec.NrOfNodes + + "running a test" in { + barrier("start") + val tc = TestConductor(system) + tc.startClient(4712) + barrier("first") + tc.enter("begin") + barrier("end") + } +} From 0314b9abbbeac4a0c8f72dea6fb866eb29cb3847 Mon Sep 17 00:00:00 2001 From: Roland Date: Fri, 4 May 2012 22:30:00 +0200 Subject: [PATCH 04/36] fix bug in FSM when manually rescheduling non-recurring timer, see #2043 --- akka-actor/src/main/scala/akka/actor/FSM.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/akka-actor/src/main/scala/akka/actor/FSM.scala b/akka-actor/src/main/scala/akka/actor/FSM.scala index b277142e76..81126c4d8d 100644 --- a/akka-actor/src/main/scala/akka/actor/FSM.scala +++ b/akka-actor/src/main/scala/akka/actor/FSM.scala @@ -443,10 +443,10 @@ trait FSM[S, D] extends Listeners { timeoutFuture = None } generation += 1 - processMsg(msg, t) if (!repeat) { timers -= name } + processMsg(msg, t) } case SubscribeTransitionCallBack(actorRef) ⇒ // TODO use DeathWatch to clean up list From 9266ac451b4d938f972f943e352b9da9329ca226 Mon Sep 17 00:00:00 2001 From: Roland Date: Fri, 4 May 2012 22:33:08 +0200 Subject: [PATCH 05/36] integrate NetworkFailureInjector and add first test - rework socket pipeline to transform protobuf into case classes and back - introduce NetworkOp messages for that purpose - make API asynchronous (because it is, really) and add Done notification for all server operations; enter(...) is still synchronous, because that is its only purpose in life - factor out mkPipeline in NettyRemoteTransport, enabling the very slick TestConductorTransport (essentially a one-liner) - switch NetworkFailureInjector from Channel{Up,Down}streamHandler to subclassing SimpleChannelHandler, because otherwise deadlocks occurred, not sure why (but SCH is the recommended way from the netty docs, so there may well be a reason) --- .../testconductor/TestConductorProtocol.java | 126 +++++++++++++-- .../main/protocol/TestConductorProtocol.proto | 2 + .../main/scala/akka/remote/netty/Client.scala | 31 +--- .../remote/netty/NettyRemoteSupport.scala | 65 ++++++-- .../main/scala/akka/remote/netty/Server.scala | 26 +-- .../akka/remote/testconductor/Conductor.scala | 151 +++++++++--------- .../akka/remote/testconductor/DataTypes.scala | 77 ++++++++- .../akka/remote/testconductor/Extension.scala | 7 + .../akka/remote/testconductor/Features.scala | 33 ++-- .../NetworkFailureInjector.scala | 116 ++++++++------ .../akka/remote/testconductor/Player.scala | 69 ++++---- .../testconductor/RemoteConnection.scala | 3 +- .../akka/remote/testconductor/package.scala | 12 ++ .../AbstractRemoteActorMultiJvmSpec.scala | 2 +- .../testconductor/TestConductorSpec.scala | 87 ++++++++-- 15 files changed, 538 insertions(+), 269 deletions(-) diff --git a/akka-remote/src/main/java/akka/remote/testconductor/TestConductorProtocol.java b/akka-remote/src/main/java/akka/remote/testconductor/TestConductorProtocol.java index 0b2950018f..f112a1b0c2 100644 --- a/akka-remote/src/main/java/akka/remote/testconductor/TestConductorProtocol.java +++ b/akka-remote/src/main/java/akka/remote/testconductor/TestConductorProtocol.java @@ -87,10 +87,12 @@ public final class TestConductorProtocol { implements com.google.protobuf.ProtocolMessageEnum { Send(0, 1), Receive(1, 2), + Both(2, 3), ; public static final int Send_VALUE = 1; public static final int Receive_VALUE = 2; + public static final int Both_VALUE = 3; public final int getNumber() { return value; } @@ -99,6 +101,7 @@ public final class TestConductorProtocol { switch (value) { case 1: return Send; case 2: return Receive; + case 3: return Both; default: return null; } } @@ -129,7 +132,7 @@ public final class TestConductorProtocol { } private static final Direction[] VALUES = { - Send, Receive, + Send, Receive, Both, }; public static Direction valueOf( @@ -169,6 +172,10 @@ public final class TestConductorProtocol { boolean hasFailure(); akka.remote.testconductor.TestConductorProtocol.InjectFailure getFailure(); akka.remote.testconductor.TestConductorProtocol.InjectFailureOrBuilder getFailureOrBuilder(); + + // optional string done = 4; + boolean hasDone(); + String getDone(); } public static final class Wrapper extends com.google.protobuf.GeneratedMessage @@ -238,10 +245,43 @@ public final class TestConductorProtocol { return failure_; } + // optional string done = 4; + public static final int DONE_FIELD_NUMBER = 4; + private java.lang.Object done_; + public boolean hasDone() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public String getDone() { + java.lang.Object ref = done_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + done_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getDoneBytes() { + java.lang.Object ref = done_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + done_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + private void initFields() { hello_ = akka.remote.testconductor.TestConductorProtocol.Hello.getDefaultInstance(); barrier_ = akka.remote.testconductor.TestConductorProtocol.EnterBarrier.getDefaultInstance(); failure_ = akka.remote.testconductor.TestConductorProtocol.InjectFailure.getDefaultInstance(); + done_ = ""; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -282,6 +322,9 @@ public final class TestConductorProtocol { if (((bitField0_ & 0x00000004) == 0x00000004)) { output.writeMessage(3, failure_); } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeBytes(4, getDoneBytes()); + } getUnknownFields().writeTo(output); } @@ -303,6 +346,10 @@ public final class TestConductorProtocol { size += com.google.protobuf.CodedOutputStream .computeMessageSize(3, failure_); } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(4, getDoneBytes()); + } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -448,6 +495,8 @@ public final class TestConductorProtocol { failureBuilder_.clear(); } bitField0_ = (bitField0_ & ~0x00000004); + done_ = ""; + bitField0_ = (bitField0_ & ~0x00000008); return this; } @@ -510,6 +559,10 @@ public final class TestConductorProtocol { } else { result.failure_ = failureBuilder_.build(); } + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000008; + } + result.done_ = done_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -535,6 +588,9 @@ public final class TestConductorProtocol { if (other.hasFailure()) { mergeFailure(other.getFailure()); } + if (other.hasDone()) { + setDone(other.getDone()); + } this.mergeUnknownFields(other.getUnknownFields()); return this; } @@ -611,6 +667,11 @@ public final class TestConductorProtocol { setFailure(subBuilder.buildPartial()); break; } + case 34: { + bitField0_ |= 0x00000008; + done_ = input.readBytes(); + break; + } } } } @@ -887,6 +948,42 @@ public final class TestConductorProtocol { return failureBuilder_; } + // optional string done = 4; + private java.lang.Object done_ = ""; + public boolean hasDone() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public String getDone() { + java.lang.Object ref = done_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + done_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setDone(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000008; + done_ = value; + onChanged(); + return this; + } + public Builder clearDone() { + bitField0_ = (bitField0_ & ~0x00000008); + done_ = getDefaultInstance().getDone(); + onChanged(); + return this; + } + void setDone(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000008; + done_ = value; + onChanged(); + } + // @@protoc_insertion_point(builder_scope:Wrapper) } @@ -3199,20 +3296,21 @@ public final class TestConductorProtocol { descriptor; static { java.lang.String[] descriptorData = { - "\n\033TestConductorProtocol.proto\"a\n\007Wrapper" + + "\n\033TestConductorProtocol.proto\"o\n\007Wrapper" + "\022\025\n\005hello\030\001 \001(\0132\006.Hello\022\036\n\007barrier\030\002 \001(\013" + "2\r.EnterBarrier\022\037\n\007failure\030\003 \001(\0132\016.Injec" + - "tFailure\"0\n\005Hello\022\014\n\004name\030\001 \002(\t\022\031\n\007addre" + - "ss\030\002 \002(\0132\010.Address\"\034\n\014EnterBarrier\022\014\n\004na" + - "me\030\001 \002(\t\"G\n\007Address\022\020\n\010protocol\030\001 \002(\t\022\016\n" + - "\006system\030\002 \002(\t\022\014\n\004host\030\003 \002(\t\022\014\n\004port\030\004 \002(" + - "\005\"\212\001\n\rInjectFailure\022\032\n\007failure\030\001 \002(\0162\t.F" + - "ailType\022\035\n\tdirection\030\002 \001(\0162\n.Direction\022\031" + - "\n\007address\030\003 \001(\0132\010.Address\022\020\n\010rateMBit\030\006 ", - "\001(\002\022\021\n\texitValue\030\007 \001(\005*A\n\010FailType\022\014\n\010Th" + - "rottle\020\001\022\016\n\nDisconnect\020\002\022\t\n\005Abort\020\003\022\014\n\010S" + - "hutdown\020\004*\"\n\tDirection\022\010\n\004Send\020\001\022\013\n\007Rece" + - "ive\020\002B\035\n\031akka.remote.testconductorH\001" + "tFailure\022\014\n\004done\030\004 \001(\t\"0\n\005Hello\022\014\n\004name\030" + + "\001 \002(\t\022\031\n\007address\030\002 \002(\0132\010.Address\"\034\n\014Ente" + + "rBarrier\022\014\n\004name\030\001 \002(\t\"G\n\007Address\022\020\n\010pro" + + "tocol\030\001 \002(\t\022\016\n\006system\030\002 \002(\t\022\014\n\004host\030\003 \002(" + + "\t\022\014\n\004port\030\004 \002(\005\"\212\001\n\rInjectFailure\022\032\n\007fai" + + "lure\030\001 \002(\0162\t.FailType\022\035\n\tdirection\030\002 \001(\016" + + "2\n.Direction\022\031\n\007address\030\003 \001(\0132\010.Address\022", + "\020\n\010rateMBit\030\006 \001(\002\022\021\n\texitValue\030\007 \001(\005*A\n\010" + + "FailType\022\014\n\010Throttle\020\001\022\016\n\nDisconnect\020\002\022\t" + + "\n\005Abort\020\003\022\014\n\010Shutdown\020\004*,\n\tDirection\022\010\n\004" + + "Send\020\001\022\013\n\007Receive\020\002\022\010\n\004Both\020\003B\035\n\031akka.re" + + "mote.testconductorH\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -3224,7 +3322,7 @@ public final class TestConductorProtocol { internal_static_Wrapper_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_Wrapper_descriptor, - new java.lang.String[] { "Hello", "Barrier", "Failure", }, + new java.lang.String[] { "Hello", "Barrier", "Failure", "Done", }, akka.remote.testconductor.TestConductorProtocol.Wrapper.class, akka.remote.testconductor.TestConductorProtocol.Wrapper.Builder.class); internal_static_Hello_descriptor = diff --git a/akka-remote/src/main/protocol/TestConductorProtocol.proto b/akka-remote/src/main/protocol/TestConductorProtocol.proto index 213820e687..e483bf4f01 100644 --- a/akka-remote/src/main/protocol/TestConductorProtocol.proto +++ b/akka-remote/src/main/protocol/TestConductorProtocol.proto @@ -15,6 +15,7 @@ message Wrapper { optional Hello hello = 1; optional EnterBarrier barrier = 2; optional InjectFailure failure = 3; + optional string done = 4; } message Hello { @@ -42,6 +43,7 @@ enum FailType { enum Direction { Send = 1; Receive = 2; + Both = 3; } message InjectFailure { required FailType failure = 1; diff --git a/akka-remote/src/main/scala/akka/remote/netty/Client.scala b/akka-remote/src/main/scala/akka/remote/netty/Client.scala index a0e91398fc..cf143650bc 100644 --- a/akka-remote/src/main/scala/akka/remote/netty/Client.scala +++ b/akka-remote/src/main/scala/akka/remote/netty/Client.scala @@ -112,8 +112,6 @@ class ActiveRemoteClient private[akka] ( private var connection: ChannelFuture = _ @volatile private[remote] var openChannels: DefaultChannelGroup = _ - @volatile - private var executionHandler: ExecutionHandler = _ @volatile private var reconnectionTimeWindowStart = 0L @@ -156,9 +154,8 @@ class ActiveRemoteClient private[akka] ( runSwitch switchOn { openChannels = new DefaultDisposableChannelGroup(classOf[RemoteClient].getName) - executionHandler = new ExecutionHandler(netty.executor) val b = new ClientBootstrap(netty.clientChannelFactory) - b.setPipelineFactory(new ActiveRemoteClientPipelineFactory(name, b, executionHandler, remoteAddress, localAddress, this)) + b.setPipelineFactory(netty.mkPipeline(new ActiveRemoteClientHandler(name, b, remoteAddress, localAddress, netty.timer, this), true)) b.setOption("tcpNoDelay", true) b.setOption("keepAlive", true) b.setOption("connectTimeoutMillis", settings.ConnectionTimeout.toMillis) @@ -206,7 +203,6 @@ class ActiveRemoteClient private[akka] ( if (openChannels ne null) openChannels.close.awaitUninterruptibly() } finally { connection = null - executionHandler = null } } @@ -319,31 +315,6 @@ class ActiveRemoteClientHandler( } } -class ActiveRemoteClientPipelineFactory( - name: String, - bootstrap: ClientBootstrap, - executionHandler: ExecutionHandler, - remoteAddress: Address, - localAddress: Address, - client: ActiveRemoteClient) extends ChannelPipelineFactory { - - import client.netty.settings - - def getPipeline: ChannelPipeline = { - val timeout = new IdleStateHandler(client.netty.timer, - settings.ReadTimeout.toSeconds.toInt, - settings.WriteTimeout.toSeconds.toInt, - settings.AllTimeout.toSeconds.toInt) - val lenDec = new LengthFieldBasedFrameDecoder(settings.MessageFrameSize, 0, 4, 0, 4) - val lenPrep = new LengthFieldPrepender(4) - val messageDec = new RemoteMessageDecoder - val messageEnc = new RemoteMessageEncoder(client.netty) - val remoteClient = new ActiveRemoteClientHandler(name, bootstrap, remoteAddress, localAddress, client.netty.timer, client) - - new StaticChannelPipeline(timeout, lenDec, messageDec, lenPrep, messageEnc, executionHandler, remoteClient) - } -} - class PassiveRemoteClient(val currentChannel: Channel, netty: NettyRemoteTransport, remoteAddress: Address) diff --git a/akka-remote/src/main/scala/akka/remote/netty/NettyRemoteSupport.scala b/akka-remote/src/main/scala/akka/remote/netty/NettyRemoteSupport.scala index c3a41f8275..35ef3bf7fd 100644 --- a/akka-remote/src/main/scala/akka/remote/netty/NettyRemoteSupport.scala +++ b/akka-remote/src/main/scala/akka/remote/netty/NettyRemoteSupport.scala @@ -22,6 +22,13 @@ import akka.event.Logging import akka.remote.RemoteProtocol.AkkaRemoteProtocol import akka.remote.{ RemoteTransportException, RemoteTransport, RemoteSettings, RemoteMarshallingOps, RemoteActorRefProvider, RemoteActorRef, RemoteServerStarted } import akka.util.NonFatal +import org.jboss.netty.channel.StaticChannelPipeline +import org.jboss.netty.channel.ChannelHandler +import org.jboss.netty.handler.codec.frame.LengthFieldPrepender +import org.jboss.netty.handler.codec.frame.LengthFieldBasedFrameDecoder +import org.jboss.netty.handler.timeout.IdleStateHandler +import org.jboss.netty.channel.ChannelPipelineFactory +import org.jboss.netty.handler.execution.ExecutionHandler /** * Provides the implementation of the Netty remote support @@ -34,20 +41,54 @@ class NettyRemoteTransport(val remoteSettings: RemoteSettings, val system: Actor // TODO replace by system.scheduler val timer: HashedWheelTimer = new HashedWheelTimer(system.threadFactory) - // TODO make configurable - lazy val executor = new OrderedMemoryAwareThreadPoolExecutor( - settings.ExecutionPoolSize, - settings.MaxChannelMemorySize, - settings.MaxTotalMemorySize, - settings.ExecutionPoolKeepalive.length, - settings.ExecutionPoolKeepalive.unit, - system.threadFactory) - // TODO make configurable/shareable with server socket factory val clientChannelFactory = new NioClientSocketChannelFactory( Executors.newCachedThreadPool(system.threadFactory), Executors.newCachedThreadPool(system.threadFactory)) + object PipelineFactory { + def apply(handlers: Seq[ChannelHandler]): StaticChannelPipeline = new StaticChannelPipeline(handlers: _*) + def apply(endpoint: ⇒ Seq[ChannelHandler], withTimeout: Boolean): ChannelPipelineFactory = + new ChannelPipelineFactory { + def getPipeline = apply(defaultStack(withTimeout) ++ endpoint) + } + + def defaultStack(withTimeout: Boolean): Seq[ChannelHandler] = + (if (withTimeout) timeout :: Nil else Nil) ::: + msgFormat ::: + authenticator ::: + executionHandler :: + Nil + + def timeout = new IdleStateHandler(timer, + settings.ReadTimeout.toSeconds.toInt, + settings.WriteTimeout.toSeconds.toInt, + settings.AllTimeout.toSeconds.toInt) + + def msgFormat = new LengthFieldBasedFrameDecoder(settings.MessageFrameSize, 0, 4, 0, 4) :: + new LengthFieldPrepender(4) :: + new RemoteMessageDecoder :: + new RemoteMessageEncoder(NettyRemoteTransport.this) :: + Nil + + val executionHandler = new ExecutionHandler(new OrderedMemoryAwareThreadPoolExecutor( + settings.ExecutionPoolSize, + settings.MaxChannelMemorySize, + settings.MaxTotalMemorySize, + settings.ExecutionPoolKeepalive.length, + settings.ExecutionPoolKeepalive.unit, + system.threadFactory)) + + def authenticator = if (settings.RequireCookie) new RemoteServerAuthenticationHandler(settings.SecureCookie) :: Nil else Nil + } + + /** + * This method is factored out to provide an extension point in case the + * pipeline shall be changed. It is recommended to use + */ + def mkPipeline(endpoint: ⇒ ChannelHandler, withTimeout: Boolean): ChannelPipelineFactory = + PipelineFactory(Seq(endpoint), withTimeout) + private val remoteClients = new HashMap[Address, RemoteClient] private val clientsLock = new ReentrantReadWriteLock @@ -105,11 +146,7 @@ class NettyRemoteTransport(val remoteSettings: RemoteSettings, val system: Actor try { timer.stop() } finally { - try { - clientChannelFactory.releaseExternalResources() - } finally { - executor.shutdown() - } + clientChannelFactory.releaseExternalResources() } } } diff --git a/akka-remote/src/main/scala/akka/remote/netty/Server.scala b/akka-remote/src/main/scala/akka/remote/netty/Server.scala index ac4289e8ae..f9d4ede1d8 100644 --- a/akka-remote/src/main/scala/akka/remote/netty/Server.scala +++ b/akka-remote/src/main/scala/akka/remote/netty/Server.scala @@ -30,14 +30,12 @@ class NettyRemoteServer(val netty: NettyRemoteTransport) { Executors.newCachedThreadPool(netty.system.threadFactory), Executors.newCachedThreadPool(netty.system.threadFactory)) - private val executionHandler = new ExecutionHandler(netty.executor) - // group of open channels, used for clean-up private val openChannels: ChannelGroup = new DefaultDisposableChannelGroup("akka-remote-server") private val bootstrap = { val b = new ServerBootstrap(factory) - b.setPipelineFactory(makePipeline()) + b.setPipelineFactory(netty.mkPipeline(new RemoteServerHandler(openChannels, netty), false)) b.setOption("backlog", settings.Backlog) b.setOption("tcpNoDelay", true) b.setOption("child.keepAlive", true) @@ -45,8 +43,6 @@ class NettyRemoteServer(val netty: NettyRemoteTransport) { b } - protected def makePipeline(): ChannelPipelineFactory = new RemoteServerPipelineFactory(openChannels, executionHandler, netty) - @volatile private[akka] var channel: Channel = _ @@ -79,26 +75,6 @@ class NettyRemoteServer(val netty: NettyRemoteTransport) { } } -class RemoteServerPipelineFactory( - val openChannels: ChannelGroup, - val executionHandler: ExecutionHandler, - val netty: NettyRemoteTransport) extends ChannelPipelineFactory { - - import netty.settings - - def getPipeline: ChannelPipeline = { - val lenDec = new LengthFieldBasedFrameDecoder(settings.MessageFrameSize, 0, 4, 0, 4) - val lenPrep = new LengthFieldPrepender(4) - val messageDec = new RemoteMessageDecoder - val messageEnc = new RemoteMessageEncoder(netty) - - val authenticator = if (settings.RequireCookie) new RemoteServerAuthenticationHandler(settings.SecureCookie) :: Nil else Nil - val remoteServer = new RemoteServerHandler(openChannels, netty) - val stages: List[ChannelHandler] = lenDec :: messageDec :: lenPrep :: messageEnc :: executionHandler :: authenticator ::: remoteServer :: Nil - new StaticChannelPipeline(stages: _*) - } -} - @ChannelHandler.Sharable class RemoteServerAuthenticationHandler(secureCookie: Option[String]) extends SimpleChannelUpstreamHandler { val authenticated = new AnyRef diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala b/akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala index c46e22eb9f..c9cbeadf83 100644 --- a/akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala +++ b/akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala @@ -21,6 +21,7 @@ import scala.util.control.NoStackTrace import akka.event.LoggingReceive import akka.actor.Address import java.net.InetSocketAddress +import akka.dispatch.Future trait Conductor extends RunControl with FailureInject { this: TestConductorExt ⇒ @@ -32,55 +33,63 @@ trait Conductor extends RunControl with FailureInject { this: TestConductorExt case x ⇒ x } - override def startController() { + override def startController(): Future[Int] = { if (_controller ne null) throw new RuntimeException("TestConductorServer was already started") _controller = system.actorOf(Props[Controller], "controller") import Settings.BarrierTimeout - startClient(Await.result(controller ? GetPort mapTo, Duration.Inf)) + controller ? GetPort flatMap { case port: Int ⇒ startClient(port) map (_ ⇒ port) } } - override def port: Int = { + override def port: Future[Int] = { import Settings.QueryTimeout - Await.result(controller ? GetPort mapTo, Duration.Inf) + controller ? GetPort mapTo } - override def throttle(node: String, target: String, direction: Direction, rateMBit: Float) { - controller ! Throttle(node, target, direction, rateMBit) - } - - override def blackhole(node: String, target: String, direction: Direction) { - controller ! Throttle(node, target, direction, 0f) - } - - override def disconnect(node: String, target: String) { - controller ! Disconnect(node, target, false) - } - - override def abort(node: String, target: String) { - controller ! Disconnect(node, target, true) - } - - override def shutdown(node: String, exitValue: Int) { - controller ! Terminate(node, exitValue) - } - - override def kill(node: String) { - controller ! Terminate(node, -1) - } - - override def getNodes = { + override def throttle(node: String, target: String, direction: Direction, rateMBit: Double): Future[Done] = { import Settings.QueryTimeout - Await.result(controller ? GetNodes mapTo manifest[List[String]], Duration.Inf) + controller ? Throttle(node, target, direction, rateMBit.toFloat) mapTo } - override def removeNode(node: String) { - controller ! Remove(node) + override def blackhole(node: String, target: String, direction: Direction): Future[Done] = { + import Settings.QueryTimeout + controller ? Throttle(node, target, direction, 0f) mapTo + } + + override def disconnect(node: String, target: String): Future[Done] = { + import Settings.QueryTimeout + controller ? Disconnect(node, target, false) mapTo + } + + override def abort(node: String, target: String): Future[Done] = { + import Settings.QueryTimeout + controller ? Disconnect(node, target, true) mapTo + } + + override def shutdown(node: String, exitValue: Int): Future[Done] = { + import Settings.QueryTimeout + controller ? Terminate(node, exitValue) mapTo + } + + override def kill(node: String): Future[Done] = { + import Settings.QueryTimeout + controller ? Terminate(node, -1) mapTo + } + + override def getNodes: Future[List[String]] = { + import Settings.QueryTimeout + controller ? GetNodes mapTo + } + + override def removeNode(node: String): Future[Done] = { + import Settings.QueryTimeout + controller ? Remove(node) mapTo } } class ConductorHandler(system: ActorSystem, controller: ActorRef, log: LoggingAdapter) extends SimpleChannelUpstreamHandler { + @volatile var clients = Map[Channel, ActorRef]() override def channelConnected(ctx: ChannelHandlerContext, event: ChannelStateEvent) = { @@ -102,7 +111,7 @@ class ConductorHandler(system: ActorSystem, controller: ActorRef, log: LoggingAd val channel = event.getChannel log.debug("message from {}: {}", getAddrString(channel), event.getMessage) event.getMessage match { - case msg: Wrapper if msg.getAllFields.size == 1 ⇒ + case msg: NetworkOp ⇒ clients(channel) ! msg case msg ⇒ log.info("client {} sent garbage '{}', disconnecting", getAddrString(channel), msg) @@ -116,28 +125,26 @@ object ServerFSM { sealed trait State case object Initial extends State case object Ready extends State - - case class Send(msg: Wrapper) } -class ServerFSM(val controller: ActorRef, val channel: Channel) extends Actor with LoggingFSM[ServerFSM.State, Null] { +class ServerFSM(val controller: ActorRef, val channel: Channel) extends Actor with LoggingFSM[ServerFSM.State, Option[ActorRef]] { import ServerFSM._ import akka.actor.FSM._ import Controller._ - startWith(Initial, null) + startWith(Initial, None) when(Initial, stateTimeout = 10 seconds) { - case Event(msg: Wrapper, _) ⇒ - if (msg.hasHello) { - val hello = msg.getHello - controller ! ClientConnected(hello.getName, hello.getAddress) - goto(Ready) - } else { - log.warning("client {} sent no Hello in first message, disconnecting", getAddrString(channel)) - channel.close() - stop() - } + case Event(Hello(name, addr), _) ⇒ + controller ! ClientConnected(name, addr) + goto(Ready) + case Event(x: NetworkOp, _) ⇒ + log.warning("client {} sent no Hello in first message (instead {}), disconnecting", getAddrString(channel), x) + channel.close() + stop() + case Event(Send(msg), _) ⇒ + log.warning("cannot send {} in state Initial", msg) + stay case Event(StateTimeout, _) ⇒ log.info("closing channel to {} because of Hello timeout", getAddrString(channel)) channel.close() @@ -145,20 +152,24 @@ class ServerFSM(val controller: ActorRef, val channel: Channel) extends Actor wi } when(Ready) { - case Event(msg: Wrapper, _) ⇒ - if (msg.hasBarrier) { - val barrier = msg.getBarrier - controller ! EnterBarrier(barrier.getName) - } else { - log.warning("client {} sent unsupported message {}", getAddrString(channel), msg) - } + case Event(msg: EnterBarrier, _) ⇒ + controller ! msg stay - case Event(Send(msg), _) ⇒ + case Event(d: Done, Some(s)) ⇒ + s ! d + stay using None + case Event(msg: NetworkOp, _) ⇒ + log.warning("client {} sent unsupported message {}", getAddrString(channel), msg) + channel.close() + stop() + case Event(Send(msg: EnterBarrier), _) ⇒ channel.write(msg) stay - case Event(EnterBarrier(name), _) ⇒ - val barrier = TestConductorProtocol.EnterBarrier.newBuilder.setName(name).build - channel.write(Wrapper.newBuilder.setBarrier(barrier).build) + case Event(Send(msg), None) ⇒ + channel.write(msg) + stay using Some(sender) + case Event(Send(msg), _) ⇒ + log.warning("cannot send {} while waiting for previous ACK", msg) stay } @@ -185,7 +196,6 @@ class Controller extends Actor { var nodes = Map[String, NodeInfo]() override def receive = LoggingReceive { - case "ready?" ⇒ sender ! "yes" case ClientConnected(name, addr) ⇒ nodes += name -> NodeInfo(name, addr, sender) barrier forward ClientConnected @@ -198,28 +208,15 @@ class Controller extends Actor { barrier forward e case Throttle(node, target, direction, rateMBit) ⇒ val t = nodes(target) - val throttle = - InjectFailure.newBuilder - .setFailure(FailType.Throttle) - .setDirection(TestConductorProtocol.Direction.valueOf(direction.toString)) - .setAddress(t.addr) - .setRateMBit(rateMBit) - .build - nodes(node).fsm ! ServerFSM.Send(Wrapper.newBuilder.setFailure(throttle).build) + nodes(node).fsm forward Send(ThrottleMsg(t.addr, direction, rateMBit)) case Disconnect(node, target, abort) ⇒ val t = nodes(target) - val disconnect = - InjectFailure.newBuilder - .setFailure(if (abort) FailType.Abort else FailType.Disconnect) - .setAddress(t.addr) - .build - nodes(node).fsm ! ServerFSM.Send(Wrapper.newBuilder.setFailure(disconnect).build) + nodes(node).fsm forward Send(DisconnectMsg(t.addr, abort)) case Terminate(node, exitValueOrKill) ⇒ if (exitValueOrKill < 0) { // TODO: kill via SBT } else { - val shutdown = InjectFailure.newBuilder.setFailure(FailType.Shutdown).setExitValue(exitValueOrKill).build - nodes(node).fsm ! ServerFSM.Send(Wrapper.newBuilder.setFailure(shutdown).build) + nodes(node).fsm forward Send(TerminateMsg(exitValueOrKill)) } // TODO: properly remove node from BarrierCoordinator // case Remove(node) => @@ -269,7 +266,7 @@ class BarrierCoordinator extends Actor with LoggingFSM[BarrierCoordinator.State, if (name != barrier) throw new IllegalStateException("trying enter barrier '" + name + "' while barrier '" + barrier + "' is active") val together = sender :: arrived if (together.size == num) { - together foreach (_ ! e) + together foreach (_ ! Send(e)) goto(Idle) using Data(num, "", Nil) } else { stay using d.copy(arrived = together) @@ -280,7 +277,7 @@ class BarrierCoordinator extends Actor with LoggingFSM[BarrierCoordinator.State, val expected = num - 1 if (arrived.size == expected) { val e = EnterBarrier(barrier) - sender :: arrived foreach (_ ! e) + sender :: arrived foreach (_ ! Send(e)) goto(Idle) using Data(expected, "", Nil) } else { stay using d.copy(clients = expected) diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/DataTypes.scala b/akka-remote/src/main/scala/akka/remote/testconductor/DataTypes.scala index 2b54ea1018..90d7eeccd5 100644 --- a/akka-remote/src/main/scala/akka/remote/testconductor/DataTypes.scala +++ b/akka-remote/src/main/scala/akka/remote/testconductor/DataTypes.scala @@ -3,11 +3,82 @@ */ package akka.remote.testconductor -sealed trait ClientOp -sealed trait ServerOp +import org.jboss.netty.handler.codec.oneone.OneToOneEncoder +import org.jboss.netty.channel.ChannelHandlerContext +import org.jboss.netty.channel.Channel +import akka.remote.testconductor.{ TestConductorProtocol ⇒ TCP } +import com.google.protobuf.Message +import akka.actor.Address +import org.jboss.netty.handler.codec.oneone.OneToOneDecoder -case class EnterBarrier(name: String) extends ClientOp with ServerOp +case class Send(msg: NetworkOp) + +sealed trait ClientOp // messages sent to Player FSM +sealed trait ServerOp // messages sent to Conductor FSM +sealed trait NetworkOp // messages sent over the wire + +case class Hello(name: String, addr: Address) extends NetworkOp +case class EnterBarrier(name: String) extends ClientOp with ServerOp with NetworkOp case class Throttle(node: String, target: String, direction: Direction, rateMBit: Float) extends ServerOp +case class ThrottleMsg(target: Address, direction: Direction, rateMBit: Float) extends NetworkOp case class Disconnect(node: String, target: String, abort: Boolean) extends ServerOp +case class DisconnectMsg(target: Address, abort: Boolean) extends NetworkOp case class Terminate(node: String, exitValueOrKill: Int) extends ServerOp +case class TerminateMsg(exitValue: Int) extends NetworkOp +abstract class Done extends NetworkOp +case object Done extends Done { + def getInstance: Done = this +} + case class Remove(node: String) extends ServerOp + +class MsgEncoder extends OneToOneEncoder { + def encode(ctx: ChannelHandlerContext, ch: Channel, msg: AnyRef): AnyRef = msg match { + case x: NetworkOp ⇒ + val w = TCP.Wrapper.newBuilder + x match { + case Hello(name, addr) ⇒ + w.setHello(TCP.Hello.newBuilder.setName(name).setAddress(addr)) + case EnterBarrier(name) ⇒ + w.setBarrier(TCP.EnterBarrier.newBuilder.setName(name)) + case ThrottleMsg(target, dir, rate) ⇒ + w.setFailure(TCP.InjectFailure.newBuilder.setAddress(target) + .setFailure(TCP.FailType.Throttle).setDirection(dir).setRateMBit(rate)) + case DisconnectMsg(target, abort) ⇒ + w.setFailure(TCP.InjectFailure.newBuilder.setAddress(target) + .setFailure(if (abort) TCP.FailType.Abort else TCP.FailType.Disconnect)) + case TerminateMsg(exitValue) ⇒ + w.setFailure(TCP.InjectFailure.newBuilder.setFailure(TCP.FailType.Shutdown).setExitValue(exitValue)) + case _: Done ⇒ + w.setDone("") + } + w.build + case _ ⇒ throw new IllegalArgumentException("wrong message " + msg) + } +} + +class MsgDecoder extends OneToOneDecoder { + def decode(ctx: ChannelHandlerContext, ch: Channel, msg: AnyRef): AnyRef = msg match { + case w: TCP.Wrapper if w.getAllFields.size == 1 ⇒ + if (w.hasHello) { + val h = w.getHello + Hello(h.getName, h.getAddress) + } else if (w.hasBarrier) { + EnterBarrier(w.getBarrier.getName) + } else if (w.hasFailure) { + val f = w.getFailure + import TCP.{ FailType ⇒ FT } + f.getFailure match { + case FT.Throttle ⇒ ThrottleMsg(f.getAddress, f.getDirection, f.getRateMBit) + case FT.Abort ⇒ DisconnectMsg(f.getAddress, true) + case FT.Disconnect ⇒ DisconnectMsg(f.getAddress, false) + case FT.Shutdown ⇒ TerminateMsg(f.getExitValue) + } + } else if (w.hasDone) { + Done + } else { + throw new IllegalArgumentException("unknown message " + msg) + } + case _ ⇒ throw new IllegalArgumentException("wrong message " + msg) + } +} diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/Extension.scala b/akka-remote/src/main/scala/akka/remote/testconductor/Extension.scala index 94847664c9..bffa84847f 100644 --- a/akka-remote/src/main/scala/akka/remote/testconductor/Extension.scala +++ b/akka-remote/src/main/scala/akka/remote/testconductor/Extension.scala @@ -7,9 +7,14 @@ import akka.remote.RemoteActorRefProvider import akka.actor.ActorContext import akka.util.{ Duration, Timeout } import java.util.concurrent.TimeUnit.MILLISECONDS +import akka.actor.ActorRef +import java.util.concurrent.ConcurrentHashMap +import akka.actor.Address object TestConductor extends ExtensionKey[TestConductorExt] { + def apply()(implicit ctx: ActorContext): TestConductorExt = apply(ctx.system) + } class TestConductorExt(val system: ExtendedActorSystem) extends Extension with Conductor with Player { @@ -28,4 +33,6 @@ class TestConductorExt(val system: ExtendedActorSystem) extends Extension with C val transport = system.provider.asInstanceOf[RemoteActorRefProvider].transport val address = transport.address + val failureInjectors = new ConcurrentHashMap[Address, FailureInjector] + } \ No newline at end of file diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/Features.scala b/akka-remote/src/main/scala/akka/remote/testconductor/Features.scala index 930be600c2..b94f205726 100644 --- a/akka-remote/src/main/scala/akka/remote/testconductor/Features.scala +++ b/akka-remote/src/main/scala/akka/remote/testconductor/Features.scala @@ -3,6 +3,8 @@ */ package akka.remote.testconductor +import akka.dispatch.Future + trait BarrierSync { /** * Enter all given barriers in the order in which they were given. @@ -11,9 +13,12 @@ trait BarrierSync { } sealed trait Direction -case object Send extends Direction -case object Receive extends Direction -case object Both extends Direction + +object Direction { + case object Send extends Direction + case object Receive extends Direction + case object Both extends Direction +} trait FailureInject { @@ -21,7 +26,7 @@ trait FailureInject { * Make the remoting pipeline on the node throttle data sent to or received * from the given remote peer. */ - def throttle(node: String, target: String, direction: Direction, rateMBit: Float): Unit + def throttle(node: String, target: String, direction: Direction, rateMBit: Double): Future[Done] /** * Switch the Netty pipeline of the remote support into blackhole mode for @@ -29,56 +34,56 @@ trait FailureInject { * submitting them to the Socket or right after receiving them from the * Socket. */ - def blackhole(node: String, target: String, direction: Direction): Unit + def blackhole(node: String, target: String, direction: Direction): Future[Done] /** * Tell the remote support to shutdown the connection to the given remote * peer. It works regardless of whether the recipient was initiator or * responder. */ - def disconnect(node: String, target: String): Unit + def disconnect(node: String, target: String): Future[Done] /** * Tell the remote support to TCP_RESET the connection to the given remote * peer. It works regardless of whether the recipient was initiator or * responder. */ - def abort(node: String, target: String): Unit + def abort(node: String, target: String): Future[Done] } trait RunControl { /** - * Start the server port. + * Start the server port, returns the port number. */ - def startController(): Unit + def startController(): Future[Int] /** * Get the actual port used by the server. */ - def port: Int + def port: Future[Int] /** * Tell the remote node to shut itself down using System.exit with the given * exitValue. */ - def shutdown(node: String, exitValue: Int): Unit + def shutdown(node: String, exitValue: Int): Future[Done] /** * Tell the SBT plugin to forcibly terminate the given remote node using Process.destroy. */ - def kill(node: String): Unit + def kill(node: String): Future[Done] /** * Obtain the list of remote host names currently registered. */ - def getNodes: List[String] + def getNodes: Future[List[String]] /** * Remove a remote host from the list, so that the remaining nodes may still * pass subsequent barriers. */ - def removeNode(node: String): Unit + def removeNode(node: String): Future[Done] } diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala b/akka-remote/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala index 6569d81acc..30e5308979 100644 --- a/akka-remote/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala +++ b/akka-remote/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala @@ -1,5 +1,5 @@ /** - * Copyright (C) 2009-2011 Typesafe Inc. + * Copyright (C) 2009-2012 Typesafe Inc. */ package akka.remote.testconductor @@ -9,11 +9,9 @@ import org.jboss.netty.buffer.ChannelBuffer import org.jboss.netty.channel.ChannelState.BOUND import org.jboss.netty.channel.ChannelState.OPEN import org.jboss.netty.channel.Channel -import org.jboss.netty.channel.ChannelDownstreamHandler import org.jboss.netty.channel.ChannelEvent import org.jboss.netty.channel.ChannelHandlerContext import org.jboss.netty.channel.ChannelStateEvent -import org.jboss.netty.channel.ChannelUpstreamHandler import org.jboss.netty.channel.MessageEvent import akka.actor.FSM import akka.actor.Actor @@ -22,23 +20,26 @@ import akka.util.Index import akka.actor.Address import akka.actor.ActorSystem import akka.actor.Props +import akka.actor.ActorRef +import akka.event.Logging +import org.jboss.netty.channel.SimpleChannelHandler -object NetworkFailureInjector { - - val channels = new Index[Address, Channel](16, (c1, c2) ⇒ c1 compareTo c2) - - def close(remote: Address): Unit = { - // channels will be cleaned up by the handler - for (chs ← channels.remove(remote); c ← chs) c.close() +case class FailureInjector(sender: ActorRef, receiver: ActorRef) { + def refs(dir: Direction) = dir match { + case Direction.Send ⇒ Seq(sender) + case Direction.Receive ⇒ Seq(receiver) + case Direction.Both ⇒ Seq(sender, receiver) } } -class NetworkFailureInjector(system: ActorSystem) extends ChannelUpstreamHandler with ChannelDownstreamHandler { +object NetworkFailureInjector { + case class SetRate(rateMBit: Float) + case class Disconnect(abort: Boolean) +} - import NetworkFailureInjector._ +class NetworkFailureInjector(system: ActorSystem) extends SimpleChannelHandler { - // local cache of remote address - private var remote: Option[Address] = None + val log = Logging(system, "FailureInjector") // everything goes via these Throttle actors to enable easy steering private val sender = system.actorOf(Props(new Throttle(_.sendDownstream(_)))) @@ -54,8 +55,8 @@ class NetworkFailureInjector(system: ActorSystem) extends ChannelUpstreamHandler private case class Data(ctx: ChannelHandlerContext, rateMBit: Float, queue: Queue[MessageEvent]) - private case class SetRate(rateMBit: Float) private case class Send(ctx: ChannelHandlerContext, msg: MessageEvent) + private case class SetContext(ctx: ChannelHandlerContext) private case object Tick private class Throttle(send: (ChannelHandlerContext, MessageEvent) ⇒ Unit) extends Actor with FSM[State, Data] { @@ -65,6 +66,7 @@ class NetworkFailureInjector(system: ActorSystem) extends ChannelUpstreamHandler when(PassThrough) { case Event(Send(ctx, msg), d) ⇒ + log.debug("sending msg (PassThrough): {}", msg) send(ctx, msg) stay } @@ -77,26 +79,37 @@ class NetworkFailureInjector(system: ActorSystem) extends ChannelUpstreamHandler stay using d.copy(ctx = ctx, queue = d.queue.enqueue(msg)) case Event(Tick, d) ⇒ val (msg, queue) = d.queue.dequeue + log.debug("sending msg (Tick, {}/{} left): {}", d.queue.size, queue.size, msg) send(d.ctx, msg) - if (queue.nonEmpty) setTimer("send", Tick, (size(queue.head) / d.rateMBit) microseconds, false) + if (queue.nonEmpty) { + val time = (size(queue.head) / d.rateMBit).microseconds + log.debug("scheduling next Tick in {}", time) + setTimer("send", Tick, time, false) + } stay using d.copy(queue = queue) } onTransition { case Throttle -> PassThrough ⇒ - stateData.queue foreach (send(stateData.ctx, _)) + stateData.queue foreach { msg ⇒ + log.debug("sending msg (Transition): {}") + send(stateData.ctx, msg) + } cancelTimer("send") case Throttle -> Blackhole ⇒ cancelTimer("send") } when(Blackhole) { - case Event(Send(_, _), _) ⇒ + case Event(Send(_, msg), _) ⇒ + log.debug("dropping msg {}", msg) stay } whenUnhandled { - case Event(SetRate(rate), d) ⇒ + case Event(SetContext(ctx), d) ⇒ stay using d.copy(ctx = ctx) + case Event(NetworkFailureInjector.SetRate(rate), d) ⇒ + sender ! "ok" if (rate > 0) { goto(Throttle) using d.copy(rateMBit = rate, queue = Queue()) } else if (rate == 0) { @@ -104,6 +117,11 @@ class NetworkFailureInjector(system: ActorSystem) extends ChannelUpstreamHandler } else { goto(PassThrough) } + case Event(NetworkFailureInjector.Disconnect(abort), Data(ctx, _, _)) ⇒ + sender ! "ok" + // TODO implement abort + ctx.getChannel.disconnect() + stay } initialize @@ -114,46 +132,42 @@ class NetworkFailureInjector(system: ActorSystem) extends ChannelUpstreamHandler } } - def throttleSend(rateMBit: Float) { - sender ! SetRate(rateMBit) + private var remote: Option[Address] = None + + override def messageReceived(ctx: ChannelHandlerContext, msg: MessageEvent) { + log.debug("upstream(queued): {}", msg) + receiver ! Send(ctx, msg) } - def throttleReceive(rateMBit: Float) { - receiver ! SetRate(rateMBit) - } - - override def handleUpstream(ctx: ChannelHandlerContext, evt: ChannelEvent) { - evt match { - case msg: MessageEvent ⇒ - receiver ! Send(ctx, msg) - case state: ChannelStateEvent ⇒ - state.getState match { - case BOUND ⇒ - state.getValue match { - case null ⇒ - remote = remote flatMap { a ⇒ channels.remove(a, state.getChannel); None } - case a: InetSocketAddress ⇒ - val addr = Address("akka", "XXX", a.getHostName, a.getPort) - channels.put(addr, state.getChannel) - remote = Some(addr) - } - case OPEN if state.getValue == false ⇒ - remote = remote flatMap { a ⇒ channels.remove(a, state.getChannel); None } + override def channelConnected(ctx: ChannelHandlerContext, state: ChannelStateEvent) { + state.getValue match { + case a: InetSocketAddress ⇒ + val addr = Address("akka", "", a.getHostName, a.getPort) + log.debug("connected to {}", addr) + TestConductor(system).failureInjectors.put(addr, FailureInjector(sender, receiver)) match { + case null ⇒ // okay + case fi ⇒ system.log.error("{} already registered for address {}", fi, addr) } - ctx.sendUpstream(evt) - case _ ⇒ - ctx.sendUpstream(evt) + remote = Some(addr) + sender ! SetContext(ctx) + case x ⇒ throw new IllegalArgumentException("unknown address type: " + x) } } - override def handleDownstream(ctx: ChannelHandlerContext, evt: ChannelEvent) { - evt match { - case msg: MessageEvent ⇒ - sender ! Send(ctx, msg) - case _ ⇒ - ctx.sendUpstream(evt) + override def channelDisconnected(ctx: ChannelHandlerContext, state: ChannelStateEvent) { + log.debug("disconnected from {}", remote) + remote = remote flatMap { addr ⇒ + TestConductor(system).failureInjectors.remove(addr) + system.stop(sender) + system.stop(receiver) + None } } + override def writeRequested(ctx: ChannelHandlerContext, msg: MessageEvent) { + log.debug("downstream(queued): {}", msg) + sender ! Send(ctx, msg) + } + } diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/Player.scala b/akka-remote/src/main/scala/akka/remote/testconductor/Player.scala index 93aa6bc33d..72b15922f3 100644 --- a/akka-remote/src/main/scala/akka/remote/testconductor/Player.scala +++ b/akka-remote/src/main/scala/akka/remote/testconductor/Player.scala @@ -6,20 +6,20 @@ package akka.remote.testconductor import akka.actor.{ Actor, ActorRef, ActorSystem, LoggingFSM, Props } import RemoteConnection.getAddrString import akka.util.duration._ -import TestConductorProtocol._ import org.jboss.netty.channel.{ Channel, SimpleChannelUpstreamHandler, ChannelHandlerContext, ChannelStateEvent, MessageEvent } import com.eaio.uuid.UUID import com.typesafe.config.ConfigFactory import akka.util.Timeout import akka.util.Duration import java.util.concurrent.TimeUnit.MILLISECONDS -import akka.pattern.ask +import akka.pattern.{ ask, pipe } import akka.dispatch.Await import scala.util.control.NoStackTrace import akka.actor.Status import akka.event.LoggingAdapter import akka.actor.PoisonPill import akka.event.Logging +import akka.dispatch.Future trait Player extends BarrierSync { this: TestConductorExt ⇒ @@ -29,7 +29,7 @@ trait Player extends BarrierSync { this: TestConductorExt ⇒ case x ⇒ x } - def startClient(port: Int) { + def startClient(port: Int): Future[Done] = { import ClientFSM._ import akka.actor.FSM._ import Settings.BarrierTimeout @@ -40,21 +40,21 @@ trait Player extends BarrierSync { this: TestConductorExt ⇒ var waiting: ActorRef = _ def receive = { case fsm: ActorRef ⇒ waiting = sender; fsm ! SubscribeTransitionCallBack(self) - case Transition(_, Connecting, Connected) ⇒ waiting ! "okay" + case Transition(_, Connecting, Connected) ⇒ waiting ! Done case t: Transition[_] ⇒ waiting ! Status.Failure(new RuntimeException("unexpected transition: " + t)) - case CurrentState(_, Connected) ⇒ waiting ! "okay" + case CurrentState(_, Connected) ⇒ waiting ! Done case _: CurrentState[_] ⇒ } })) - Await.result(a ? client, Duration.Inf) + a ? client mapTo } override def enter(name: String*) { system.log.debug("entering barriers " + name.mkString("(", ", ", ")")) name foreach { b ⇒ import Settings.BarrierTimeout - Await.result(client ? EnterBarrier(b), Duration.Inf) + Await.result(client ? Send(EnterBarrier(b)), Duration.Inf) system.log.debug("passed barrier {}", b) } } @@ -84,8 +84,7 @@ class ClientFSM(port: Int) extends Actor with LoggingFSM[ClientFSM.State, Client case Event(msg: ClientOp, _) ⇒ stay replying Status.Failure(new IllegalStateException("not connected yet")) case Event(Connected, d @ Data(channel, _)) ⇒ - val hello = Hello.newBuilder.setName(settings.name).setAddress(TestConductor().address).build - channel.write(Wrapper.newBuilder.setHello(hello).build) + channel.write(Hello(settings.name, TestConductor().address)) goto(Connected) case Event(_: ConnectionFailure, _) ⇒ // System.exit(1) @@ -100,19 +99,41 @@ class ClientFSM(port: Int) extends Actor with LoggingFSM[ClientFSM.State, Client case Event(Disconnected, _) ⇒ log.info("disconnected from TestConductor") throw new ConnectionFailure("disconnect") - case Event(msg: EnterBarrier, Data(channel, _)) ⇒ - sendMsg(channel)(msg) + case Event(Send(msg: EnterBarrier), Data(channel, None)) ⇒ + channel.write(msg) stay using Data(channel, Some(msg.name, sender)) - case Event(msg: Wrapper, Data(channel, Some((barrier, sender)))) if msg.getAllFields.size == 1 ⇒ - if (msg.hasBarrier) { - val b = msg.getBarrier.getName - if (b != barrier) { - sender ! Status.Failure(new RuntimeException("wrong barrier " + b + " received while waiting for " + barrier)) - } else { - sender ! b - } + case Event(Send(d: Done), Data(channel, _)) ⇒ + channel.write(d) + stay + case Event(Send(x), _) ⇒ + log.warning("cannot send message {}", x) + stay + case Event(EnterBarrier(b), Data(channel, Some((barrier, sender)))) ⇒ + if (b != barrier) { + sender ! Status.Failure(new RuntimeException("wrong barrier " + b + " received while waiting for " + barrier)) + } else { + sender ! b } stay using Data(channel, None) + case Event(ThrottleMsg(target, dir, rate), _) ⇒ + import settings.QueryTimeout + import context.dispatcher + TestConductor().failureInjectors.get(target.copy(system = "")) match { + case null ⇒ log.warning("cannot throttle unknown address {}", target) + case inj ⇒ + Future.sequence(inj.refs(dir) map (_ ? NetworkFailureInjector.SetRate(rate))) map (_ ⇒ Send(Done)) pipeTo self + } + stay + case Event(DisconnectMsg(target, abort), _) ⇒ + import settings.QueryTimeout + TestConductor().failureInjectors.get(target.copy(system = "")) match { + case null ⇒ log.warning("cannot disconnect unknown address {}", target) + case inj ⇒ inj.sender ? NetworkFailureInjector.Disconnect(abort) map (_ ⇒ Send(Done)) pipeTo self + } + stay + case Event(TerminateMsg(exit), _) ⇒ + System.exit(exit) + stay // needed because Java doesn’t have Nothing } onTermination { @@ -122,14 +143,6 @@ class ClientFSM(port: Int) extends Actor with LoggingFSM[ClientFSM.State, Client initialize - private def sendMsg(channel: Channel)(msg: ClientOp) { - msg match { - case EnterBarrier(name) ⇒ - val enter = TestConductorProtocol.EnterBarrier.newBuilder.setName(name).build - channel.write(Wrapper.newBuilder.setBarrier(enter).build) - } - } - } class PlayerHandler(fsm: ActorRef, log: LoggingAdapter) extends SimpleChannelUpstreamHandler { @@ -152,7 +165,7 @@ class PlayerHandler(fsm: ActorRef, log: LoggingAdapter) extends SimpleChannelUps val channel = event.getChannel log.debug("message from {}: {}", getAddrString(channel), event.getMessage) event.getMessage match { - case msg: Wrapper if msg.getAllFields.size == 1 ⇒ + case msg: NetworkOp ⇒ fsm ! msg case msg ⇒ log.info("server {} sent garbage '{}', disconnecting", getAddrString(channel), msg) diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/RemoteConnection.scala b/akka-remote/src/main/scala/akka/remote/testconductor/RemoteConnection.scala index a92b6295e2..b2f4baebbb 100644 --- a/akka-remote/src/main/scala/akka/remote/testconductor/RemoteConnection.scala +++ b/akka-remote/src/main/scala/akka/remote/testconductor/RemoteConnection.scala @@ -17,7 +17,8 @@ class TestConductorPipelineFactory(handler: ChannelUpstreamHandler) extends Chan def getPipeline: ChannelPipeline = { val encap = List(new LengthFieldPrepender(4), new LengthFieldBasedFrameDecoder(10000, 0, 4, 0, 4)) val proto = List(new ProtobufEncoder, new ProtobufDecoder(TestConductorProtocol.Wrapper.getDefaultInstance)) - new StaticChannelPipeline(encap ::: proto ::: handler :: Nil: _*) + val msg = List(new MsgEncoder, new MsgDecoder) + new StaticChannelPipeline(encap ::: proto ::: msg ::: handler :: Nil: _*) } } diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/package.scala b/akka-remote/src/main/scala/akka/remote/testconductor/package.scala index 8ebeea90a9..b24279dbf6 100644 --- a/akka-remote/src/main/scala/akka/remote/testconductor/package.scala +++ b/akka-remote/src/main/scala/akka/remote/testconductor/package.scala @@ -16,4 +16,16 @@ package object testconductor { implicit def address2scala(addr: TCP.Address): Address = Address(addr.getProtocol, addr.getSystem, addr.getHost, addr.getPort) + implicit def direction2proto(dir: Direction): TCP.Direction = dir match { + case Direction.Send ⇒ TCP.Direction.Send + case Direction.Receive ⇒ TCP.Direction.Receive + case Direction.Both ⇒ TCP.Direction.Both + } + + implicit def direction2scala(dir: TCP.Direction): Direction = dir match { + case TCP.Direction.Send ⇒ Direction.Send + case TCP.Direction.Receive ⇒ Direction.Receive + case TCP.Direction.Both ⇒ Direction.Both + } + } \ No newline at end of file diff --git a/akka-remote/src/multi-jvm/scala/akka/remote/AbstractRemoteActorMultiJvmSpec.scala b/akka-remote/src/multi-jvm/scala/akka/remote/AbstractRemoteActorMultiJvmSpec.scala index ab8bdadae6..ca4313b56b 100644 --- a/akka-remote/src/multi-jvm/scala/akka/remote/AbstractRemoteActorMultiJvmSpec.scala +++ b/akka-remote/src/multi-jvm/scala/akka/remote/AbstractRemoteActorMultiJvmSpec.scala @@ -1,6 +1,7 @@ package akka.remote import com.typesafe.config.{Config, ConfigFactory} +import akka.actor.Address trait AbstractRemoteActorMultiJvmSpec { def NrOfNodes: Int @@ -8,7 +9,6 @@ trait AbstractRemoteActorMultiJvmSpec { def PortRangeStart = 1990 def NodeRange = 1 to NrOfNodes - def PortRange = PortRangeStart to NrOfNodes private[this] val remotes: IndexedSeq[String] = { val nodesOpt = Option(AkkaRemoteSpec.testNodes).map(_.split(",").toIndexedSeq) diff --git a/akka-remote/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala b/akka-remote/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala index cae2917577..096d4c5a89 100644 --- a/akka-remote/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala +++ b/akka-remote/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala @@ -3,12 +3,24 @@ package akka.remote.testconductor import akka.remote.AkkaRemoteSpec import com.typesafe.config.ConfigFactory import akka.remote.AbstractRemoteActorMultiJvmSpec +import akka.actor.Props +import akka.actor.Actor +import akka.dispatch.Await +import akka.dispatch.Await.Awaitable +import akka.util.Duration +import akka.util.duration._ +import akka.testkit.ImplicitSender object TestConductorMultiJvmSpec extends AbstractRemoteActorMultiJvmSpec { override def NrOfNodes = 2 override def commonConfig = ConfigFactory.parseString(""" akka.loglevel = DEBUG akka.actor.provider = akka.remote.RemoteActorRefProvider + akka.remote { + transport = akka.remote.testconductor.TestConductorTransport + log-received-messages = on + log-sent-messages = on + } akka.actor.debug { receive = on fsm = on @@ -19,34 +31,87 @@ object TestConductorMultiJvmSpec extends AbstractRemoteActorMultiJvmSpec { } """) def nameConfig(n: Int) = ConfigFactory.parseString("akka.testconductor.name = node" + n).withFallback(nodeConfigs(n)) + + implicit def awaitHelper[T](w: Awaitable[T]) = new AwaitHelper(w) + class AwaitHelper[T](w: Awaitable[T]) { + def await: T = Await.result(w, Duration.Inf) + } } -import TestConductorMultiJvmSpec._ +class TestConductorMultiJvmNode1 extends AkkaRemoteSpec(TestConductorMultiJvmSpec.nameConfig(0)) { -class TestConductorMultiJvmNode1 extends AkkaRemoteSpec(nameConfig(0)) { + import TestConductorMultiJvmSpec._ - val nodes = TestConductorMultiJvmSpec.NrOfNodes + val nodes = NrOfNodes - "running a test" in { - val tc = TestConductor(system) - tc.startController() + val tc = TestConductor(system) + + val echo = system.actorOf(Props(new Actor { + def receive = { + case x ⇒ testActor ! x; sender ! x + } + }), "echo") + + "running a test with barrier" in { + tc.startController().await barrier("start") barrier("first") tc.enter("begin") barrier("end") } + + "throttling" in { + expectMsg("start") + tc.throttle("node1", "node0", Direction.Send, 0.016).await + tc.enter("throttled_send") + within(1 second, 2 seconds) { + receiveN(10) must be(0 to 9) + } + tc.enter("throttled_send2") + tc.throttle("node1", "node0", Direction.Send, -1).await + + tc.throttle("node1", "node0", Direction.Receive, 0.016).await + tc.enter("throttled_recv") + receiveN(10, 500 millis) must be(10 to 19) + tc.enter("throttled_recv2") + tc.throttle("node1", "node0", Direction.Receive, -1).await + } } -class TestConductorMultiJvmNode2 extends AkkaRemoteSpec(nameConfig(1)) { +class TestConductorMultiJvmNode2 extends AkkaRemoteSpec(TestConductorMultiJvmSpec.nameConfig(1)) with ImplicitSender { - val nodes = TestConductorMultiJvmSpec.NrOfNodes + import TestConductorMultiJvmSpec._ - "running a test" in { + val nodes = NrOfNodes + + val tc = TestConductor(system) + + val echo = system.actorFor("akka://" + akkaSpec(0) + "/user/echo") + + "running a test with barrier" in { barrier("start") - val tc = TestConductor(system) - tc.startClient(4712) + tc.startClient(4712).await barrier("first") tc.enter("begin") barrier("end") } + + "throttling" in { + echo ! "start" + expectMsg("start") + tc.enter("throttled_send") + for (i <- 0 to 9) echo ! i + expectMsg(500 millis, 0) + within(1 second, 2 seconds) { + receiveN(9) must be(1 to 9) + } + tc.enter("throttled_send2", "throttled_recv") + for (i <- 10 to 19) echo ! i + expectMsg(500 millis, 10) + within(1 second, 2 seconds) { + receiveN(9) must be(11 to 19) + } + tc.enter("throttled_recv2") + } + } From c68df0635f2213d397649533db24a0a01ffe17c5 Mon Sep 17 00:00:00 2001 From: Roland Date: Sat, 5 May 2012 15:16:21 +0200 Subject: [PATCH 06/36] add previously forgotten TestConductorTransport --- .../TestConductorTransport.scala | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 akka-remote/src/main/scala/akka/remote/testconductor/TestConductorTransport.scala diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/TestConductorTransport.scala b/akka-remote/src/main/scala/akka/remote/testconductor/TestConductorTransport.scala new file mode 100644 index 0000000000..d03adebe9a --- /dev/null +++ b/akka-remote/src/main/scala/akka/remote/testconductor/TestConductorTransport.scala @@ -0,0 +1,21 @@ +/** + * Copyright (C) 2009-2012 Typesafe Inc. + */ +package akka.remote.testconductor + +import akka.remote.netty.NettyRemoteTransport +import akka.remote.RemoteSettings +import akka.actor.ActorSystemImpl +import akka.remote.RemoteActorRefProvider +import org.jboss.netty.channel.ChannelHandler +import org.jboss.netty.channel.ChannelPipelineFactory + +class TestConductorTransport(_remoteSettings: RemoteSettings, _system: ActorSystemImpl, _provider: RemoteActorRefProvider) + extends NettyRemoteTransport(_remoteSettings, _system, _provider) { + + override def mkPipeline(endpoint: ⇒ ChannelHandler, withTimeout: Boolean): ChannelPipelineFactory = + new ChannelPipelineFactory { + def getPipeline = PipelineFactory(new NetworkFailureInjector(system) +: PipelineFactory.defaultStack(withTimeout) :+ endpoint) + } + +} \ No newline at end of file From a351e6ad9fb9ca24d69a20da9f6bb2028f901a91 Mon Sep 17 00:00:00 2001 From: Roland Date: Sat, 5 May 2012 22:14:42 +0200 Subject: [PATCH 07/36] implement more precise bandwith throttling - will keep track of theoretical packet boundaries and send on timer tick or send request according to actual time - will split packets if calculated release time is >100ms into the future (configurable) to simulate proper trickling --- akka-remote/src/main/resources/reference.conf | 5 + .../akka/remote/testconductor/Extension.scala | 1 + .../NetworkFailureInjector.scala | 131 +++++++++++++----- .../testconductor/TestConductorSpec.scala | 4 +- .../src/main/scala/akka/testkit/TestKit.scala | 6 +- 5 files changed, 109 insertions(+), 38 deletions(-) diff --git a/akka-remote/src/main/resources/reference.conf b/akka-remote/src/main/resources/reference.conf index 384d00b55d..f14ee3d87c 100644 --- a/akka-remote/src/main/resources/reference.conf +++ b/akka-remote/src/main/resources/reference.conf @@ -165,6 +165,11 @@ akka { # Timeout for interrogation of TestConductor’s Controller actor query-timeout = 5s + # Threshold for packet size in time unit above which the failure injector will + # split the packet and deliver in smaller portions; do not give value smaller + # than HashedWheelTimer resolution (would not make sense) + packet-split-threshold = 100ms + # Default port to start the conductor on; 0 means port = 0 diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/Extension.scala b/akka-remote/src/main/scala/akka/remote/testconductor/Extension.scala index bffa84847f..97f5dd7295 100644 --- a/akka-remote/src/main/scala/akka/remote/testconductor/Extension.scala +++ b/akka-remote/src/main/scala/akka/remote/testconductor/Extension.scala @@ -24,6 +24,7 @@ class TestConductorExt(val system: ExtendedActorSystem) extends Extension with C implicit val BarrierTimeout = Timeout(Duration(config.getMilliseconds("akka.testconductor.barrier-timeout"), MILLISECONDS)) implicit val QueryTimeout = Timeout(Duration(config.getMilliseconds("akka.testconductor.query-timeout"), MILLISECONDS)) + val PacketSplitThreshold = Duration(config.getMilliseconds("akka.testconductor.packet-split-threshold"), MILLISECONDS) val name = config.getString("akka.testconductor.name") val host = config.getString("akka.testconductor.host") diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala b/akka-remote/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala index 30e5308979..5e101dea0c 100644 --- a/akka-remote/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala +++ b/akka-remote/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala @@ -23,6 +23,13 @@ import akka.actor.Props import akka.actor.ActorRef import akka.event.Logging import org.jboss.netty.channel.SimpleChannelHandler +import scala.annotation.tailrec +import akka.util.Duration +import akka.actor.LoggingFSM +import org.jboss.netty.channel.Channels +import org.jboss.netty.channel.ChannelFuture +import org.jboss.netty.channel.ChannelFutureListener +import org.jboss.netty.channel.ChannelFuture case class FailureInjector(sender: ActorRef, receiver: ActorRef) { def refs(dir: Direction) = dir match { @@ -42,8 +49,10 @@ class NetworkFailureInjector(system: ActorSystem) extends SimpleChannelHandler { val log = Logging(system, "FailureInjector") // everything goes via these Throttle actors to enable easy steering - private val sender = system.actorOf(Props(new Throttle(_.sendDownstream(_)))) - private val receiver = system.actorOf(Props(new Throttle(_.sendUpstream(_)))) + private val sender = system.actorOf(Props(new Throttle(Direction.Send))) + private val receiver = system.actorOf(Props(new Throttle(Direction.Receive))) + + private val packetSplitThreshold = TestConductor(system).Settings.PacketSplitThreshold /* * State, Data and Messages for the internal Throttle actor @@ -53,47 +62,40 @@ class NetworkFailureInjector(system: ActorSystem) extends SimpleChannelHandler { private case object Throttle extends State private case object Blackhole extends State - private case class Data(ctx: ChannelHandlerContext, rateMBit: Float, queue: Queue[MessageEvent]) + private case class Data(lastSent: Long, rateMBit: Float, queue: Queue[Send]) - private case class Send(ctx: ChannelHandlerContext, msg: MessageEvent) + private case class Send(ctx: ChannelHandlerContext, future: Option[ChannelFuture], msg: AnyRef) private case class SetContext(ctx: ChannelHandlerContext) private case object Tick - private class Throttle(send: (ChannelHandlerContext, MessageEvent) ⇒ Unit) extends Actor with FSM[State, Data] { + private class Throttle(dir: Direction) extends Actor with LoggingFSM[State, Data] { import FSM._ - startWith(PassThrough, Data(null, -1, Queue())) + var channelContext: ChannelHandlerContext = _ + + startWith(PassThrough, Data(0, -1, Queue())) when(PassThrough) { - case Event(Send(ctx, msg), d) ⇒ + case Event(s @ Send(_, _, msg), _) ⇒ log.debug("sending msg (PassThrough): {}", msg) - send(ctx, msg) + send(s) stay } when(Throttle) { - case Event(Send(ctx, msg), d) ⇒ - if (!timerActive_?("send")) { - setTimer("send", Tick, (size(msg) / d.rateMBit) microseconds, false) - } - stay using d.copy(ctx = ctx, queue = d.queue.enqueue(msg)) - case Event(Tick, d) ⇒ - val (msg, queue) = d.queue.dequeue - log.debug("sending msg (Tick, {}/{} left): {}", d.queue.size, queue.size, msg) - send(d.ctx, msg) - if (queue.nonEmpty) { - val time = (size(queue.head) / d.rateMBit).microseconds - log.debug("scheduling next Tick in {}", time) - setTimer("send", Tick, time, false) - } - stay using d.copy(queue = queue) + case Event(s: Send, d @ Data(_, _, Queue())) ⇒ + stay using sendThrottled(d.copy(lastSent = System.nanoTime, queue = Queue(s))) + case Event(s: Send, data) ⇒ + stay using sendThrottled(data.copy(queue = data.queue.enqueue(s))) + case Event(Tick, data) ⇒ + stay using sendThrottled(data) } onTransition { case Throttle -> PassThrough ⇒ - stateData.queue foreach { msg ⇒ - log.debug("sending msg (Transition): {}") - send(stateData.ctx, msg) + for (s ← stateData.queue) { + log.debug("sending msg (Transition): {}", s.msg) + send(s) } cancelTimer("send") case Throttle -> Blackhole ⇒ @@ -101,32 +103,95 @@ class NetworkFailureInjector(system: ActorSystem) extends SimpleChannelHandler { } when(Blackhole) { - case Event(Send(_, msg), _) ⇒ + case Event(Send(_, _, msg), _) ⇒ log.debug("dropping msg {}", msg) stay } whenUnhandled { - case Event(SetContext(ctx), d) ⇒ stay using d.copy(ctx = ctx) case Event(NetworkFailureInjector.SetRate(rate), d) ⇒ sender ! "ok" if (rate > 0) { - goto(Throttle) using d.copy(rateMBit = rate, queue = Queue()) + goto(Throttle) using d.copy(lastSent = System.nanoTime, rateMBit = rate, queue = Queue()) } else if (rate == 0) { goto(Blackhole) } else { goto(PassThrough) } + case Event(SetContext(ctx), _) ⇒ channelContext = ctx; stay case Event(NetworkFailureInjector.Disconnect(abort), Data(ctx, _, _)) ⇒ sender ! "ok" // TODO implement abort - ctx.getChannel.disconnect() + channelContext.getChannel.disconnect() stay } initialize - private def size(msg: MessageEvent) = msg.getMessage() match { + private def sendThrottled(d: Data): Data = { + val (data, toSend, toTick) = schedule(d) + for (s ← toSend) { + log.debug("sending msg (Tick): {}", s.msg) + send(s) + } + for (time ← toTick) { + log.debug("scheduling next Tick in {}", time) + setTimer("send", Tick, time, false) + } + data + } + + private def send(s: Send): Unit = dir match { + case Direction.Send ⇒ Channels.write(s.ctx, s.future getOrElse Channels.future(s.ctx.getChannel), s.msg) + case Direction.Receive ⇒ Channels.fireMessageReceived(s.ctx, s.msg) + case _ ⇒ + } + + private def schedule(d: Data): (Data, Seq[Send], Option[Duration]) = { + val now = System.nanoTime + @tailrec def rec(d: Data, toSend: Seq[Send]): (Data, Seq[Send], Option[Duration]) = { + if (d.queue.isEmpty) (d, toSend, None) + else { + val timeForPacket = d.lastSent + (1000 * size(d.queue.head.msg) / d.rateMBit).toLong + if (timeForPacket <= now) rec(Data(timeForPacket, d.rateMBit, d.queue.tail), toSend :+ d.queue.head) + else { + val deadline = now + packetSplitThreshold.toNanos + if (timeForPacket <= deadline) (d, toSend, Some((timeForPacket - now).nanos)) + else { + val micros = (deadline - d.lastSent) / 1000 + val (s1, s2) = split(d.queue.head, (micros * d.rateMBit / 8).toInt) + (d.copy(queue = s1 +: s2 +: d.queue.tail), toSend, Some(packetSplitThreshold)) + } + } + } + } + rec(d, Seq()) + } + + private def split(s: Send, bytes: Int): (Send, Send) = { + s.msg match { + case buf: ChannelBuffer ⇒ + val f = s.future map { f ⇒ + val newF = Channels.future(s.ctx.getChannel) + newF.addListener(new ChannelFutureListener { + def operationComplete(future: ChannelFuture) { + if (future.isCancelled) f.cancel() + else future.getCause match { + case null ⇒ + case thr ⇒ f.setFailure(thr) + } + } + }) + newF + } + val b = buf.slice() + b.writerIndex(b.readerIndex + bytes) + buf.readerIndex(buf.readerIndex + bytes) + (Send(s.ctx, f, b), Send(s.ctx, s.future, buf)) + } + } + + private def size(msg: AnyRef) = msg match { case b: ChannelBuffer ⇒ b.readableBytes() * 8 case _ ⇒ throw new UnsupportedOperationException("NetworkFailureInjector only supports ChannelBuffer messages") } @@ -136,7 +201,7 @@ class NetworkFailureInjector(system: ActorSystem) extends SimpleChannelHandler { override def messageReceived(ctx: ChannelHandlerContext, msg: MessageEvent) { log.debug("upstream(queued): {}", msg) - receiver ! Send(ctx, msg) + receiver ! Send(ctx, Option(msg.getFuture), msg.getMessage) } override def channelConnected(ctx: ChannelHandlerContext, state: ChannelStateEvent) { @@ -166,7 +231,7 @@ class NetworkFailureInjector(system: ActorSystem) extends SimpleChannelHandler { override def writeRequested(ctx: ChannelHandlerContext, msg: MessageEvent) { log.debug("downstream(queued): {}", msg) - sender ! Send(ctx, msg) + sender ! Send(ctx, Option(msg.getFuture), msg.getMessage) } } diff --git a/akka-remote/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala b/akka-remote/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala index 096d4c5a89..c7e848caf3 100644 --- a/akka-remote/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala +++ b/akka-remote/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala @@ -62,7 +62,7 @@ class TestConductorMultiJvmNode1 extends AkkaRemoteSpec(TestConductorMultiJvmSpe "throttling" in { expectMsg("start") - tc.throttle("node1", "node0", Direction.Send, 0.016).await + tc.throttle("node1", "node0", Direction.Send, 0.01).await tc.enter("throttled_send") within(1 second, 2 seconds) { receiveN(10) must be(0 to 9) @@ -70,7 +70,7 @@ class TestConductorMultiJvmNode1 extends AkkaRemoteSpec(TestConductorMultiJvmSpe tc.enter("throttled_send2") tc.throttle("node1", "node0", Direction.Send, -1).await - tc.throttle("node1", "node0", Direction.Receive, 0.016).await + tc.throttle("node1", "node0", Direction.Receive, 0.01).await tc.enter("throttled_recv") receiveN(10, 500 millis) must be(10 to 19) tc.enter("throttled_recv2") diff --git a/akka-testkit/src/main/scala/akka/testkit/TestKit.scala b/akka-testkit/src/main/scala/akka/testkit/TestKit.scala index bcac5c24cf..cbcfc2a77d 100644 --- a/akka-testkit/src/main/scala/akka/testkit/TestKit.scala +++ b/akka-testkit/src/main/scala/akka/testkit/TestKit.scala @@ -69,7 +69,7 @@ class TestActor(queue: BlockingDeque[TestActor.Message]) extends Actor { * 
  * class Test extends TestKit(ActorSystem()) {
  *     try {
- *     
+ *
  *       val test = system.actorOf(Props[SomeActor]
  *
  *       within (1 second) {
@@ -77,7 +77,7 @@ class TestActor(queue: BlockingDeque[TestActor.Message]) extends Actor {
  *         expectMsg(Result1) // bounded to 1 second
  *         expectMsg(Result2) // bounded to the remainder of the 1 second
  *       }
- *     
+ *
  *     } finally {
  *       system.shutdown()
  *     }
@@ -86,7 +86,7 @@ class TestActor(queue: BlockingDeque[TestActor.Message]) extends Actor {
  *
  * Beware of two points:
  *
- *  - the ActorSystem passed into the constructor needs to be shutdown, 
+ *  - the ActorSystem passed into the constructor needs to be shutdown,
  *    otherwise thread pools and memory will be leaked
  *  - this trait is not thread-safe (only one actor with one queue, one stack
  *    of `within` blocks); it is expected that the code is executed from a

From 0076bddb523a358d0c00ba6fd5725deaffd926e3 Mon Sep 17 00:00:00 2001
From: Roland 
Date: Mon, 7 May 2012 07:36:02 +0200
Subject: [PATCH 08/36] optimize partial message scheduling

- split only right before send (if necessary)
- do not reschedule Tick if that has already been done, because the head
  of the queue does not change so the old data are still correct
- make test a bit less fickle wrt. timing
---
 .../NetworkFailureInjector.scala              | 23 ++++++++++---------
 .../testconductor/TestConductorSpec.scala     |  6 ++---
 2 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala b/akka-remote/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala
index 5e101dea0c..b853523979 100644
--- a/akka-remote/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala
+++ b/akka-remote/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala
@@ -83,8 +83,8 @@ class NetworkFailureInjector(system: ActorSystem) extends SimpleChannelHandler {
     }
 
     when(Throttle) {
-      case Event(s: Send, d @ Data(_, _, Queue())) ⇒
-        stay using sendThrottled(d.copy(lastSent = System.nanoTime, queue = Queue(s)))
+      case Event(s: Send, data @ Data(_, _, Queue())) ⇒
+        stay using sendThrottled(data.copy(lastSent = System.nanoTime, queue = Queue(s)))
       case Event(s: Send, data) ⇒
         stay using sendThrottled(data.copy(queue = data.queue.enqueue(s)))
       case Event(Tick, data) ⇒
@@ -134,10 +134,11 @@ class NetworkFailureInjector(system: ActorSystem) extends SimpleChannelHandler {
         log.debug("sending msg (Tick): {}", s.msg)
         send(s)
       }
-      for (time ← toTick) {
-        log.debug("scheduling next Tick in {}", time)
-        setTimer("send", Tick, time, false)
-      }
+      if (!timerActive_?("send"))
+        for (time ← toTick) {
+          log.debug("scheduling next Tick in {}", time)
+          setTimer("send", Tick, time, false)
+        }
       data
     }
 
@@ -155,12 +156,12 @@ class NetworkFailureInjector(system: ActorSystem) extends SimpleChannelHandler {
           val timeForPacket = d.lastSent + (1000 * size(d.queue.head.msg) / d.rateMBit).toLong
           if (timeForPacket <= now) rec(Data(timeForPacket, d.rateMBit, d.queue.tail), toSend :+ d.queue.head)
           else {
-            val deadline = now + packetSplitThreshold.toNanos
-            if (timeForPacket <= deadline) (d, toSend, Some((timeForPacket - now).nanos))
+            val splitThreshold = d.lastSent + packetSplitThreshold.toNanos
+            if (now < splitThreshold) (d, toSend, Some((timeForPacket - now).nanos min (splitThreshold - now).nanos))
             else {
-              val micros = (deadline - d.lastSent) / 1000
-              val (s1, s2) = split(d.queue.head, (micros * d.rateMBit / 8).toInt)
-              (d.copy(queue = s1 +: s2 +: d.queue.tail), toSend, Some(packetSplitThreshold))
+              val microsToSend = (now - d.lastSent) / 1000
+              val (s1, s2) = split(d.queue.head, (microsToSend * d.rateMBit / 8).toInt)
+              (d.copy(queue = s2 +: d.queue.tail), toSend :+ s1, Some((timeForPacket - now).nanos min packetSplitThreshold))
             }
           }
         }
diff --git a/akka-remote/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala b/akka-remote/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
index c7e848caf3..16193f7bd3 100644
--- a/akka-remote/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
+++ b/akka-remote/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
@@ -64,7 +64,7 @@ class TestConductorMultiJvmNode1 extends AkkaRemoteSpec(TestConductorMultiJvmSpe
     expectMsg("start")
     tc.throttle("node1", "node0", Direction.Send, 0.01).await
     tc.enter("throttled_send")
-    within(1 second, 2 seconds) {
+    within(0.6 seconds, 2 seconds) {
       receiveN(10) must be(0 to 9)
     }
     tc.enter("throttled_send2")
@@ -102,13 +102,13 @@ class TestConductorMultiJvmNode2 extends AkkaRemoteSpec(TestConductorMultiJvmSpe
     tc.enter("throttled_send")
     for (i <- 0 to 9) echo ! i
     expectMsg(500 millis, 0)
-    within(1 second, 2 seconds) {
+    within(0.6 seconds, 2 seconds) {
       receiveN(9) must be(1 to 9)
     }
     tc.enter("throttled_send2", "throttled_recv")
     for (i <- 10 to 19) echo ! i
     expectMsg(500 millis, 10)
-    within(1 second, 2 seconds) {
+    within(0.6 seconds, 2 seconds) {
       receiveN(9) must be(11 to 19)
     }
     tc.enter("throttled_recv2")

From f81184236fa0748304a26ae133d791d074c95536 Mon Sep 17 00:00:00 2001
From: Roland 
Date: Mon, 7 May 2012 08:04:15 +0200
Subject: [PATCH 09/36] wait for initial crew before starting the party

- the Controller is started with the required initial number of
  participants
- if that is >0, it will hold off sending Done to the clients until that
  number has connected, then set it to zero
- if that is <=0, send Done back immediately upon connect
---
 .../akka/remote/testconductor/Conductor.scala  | 17 +++++++++++------
 .../akka/remote/testconductor/Features.scala   |  2 +-
 .../akka/remote/testconductor/Player.scala     | 18 ++++++++++++++++--
 .../testconductor/TestConductorSpec.scala      |  8 +-------
 4 files changed, 29 insertions(+), 16 deletions(-)

diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala b/akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala
index c9cbeadf83..7e3d315fea 100644
--- a/akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala
+++ b/akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala
@@ -33,9 +33,9 @@ trait Conductor extends RunControl with FailureInject { this: TestConductorExt 
     case x    ⇒ x
   }
 
-  override def startController(): Future[Int] = {
+  override def startController(participants: Int): Future[Int] = {
     if (_controller ne null) throw new RuntimeException("TestConductorServer was already started")
-    _controller = system.actorOf(Props[Controller], "controller")
+    _controller = system.actorOf(Props(new Controller(participants)), "controller")
     import Settings.BarrierTimeout
     controller ? GetPort flatMap { case port: Int ⇒ startClient(port) map (_ ⇒ port) }
   }
@@ -162,7 +162,7 @@ class ServerFSM(val controller: ActorRef, val channel: Channel) extends Actor wi
       log.warning("client {} sent unsupported message {}", getAddrString(channel), msg)
       channel.close()
       stop()
-    case Event(Send(msg: EnterBarrier), _) ⇒
+    case Event(Send(msg @ (_: EnterBarrier | _: Done)), _) ⇒
       channel.write(msg)
       stay
     case Event(Send(msg), None) ⇒
@@ -185,9 +185,11 @@ object Controller {
   case class NodeInfo(name: String, addr: Address, fsm: ActorRef)
 }
 
-class Controller extends Actor {
+class Controller(_participants: Int) extends Actor {
   import Controller._
 
+  var initialParticipants = _participants
+
   val settings = TestConductor().Settings
   val connection = RemoteConnection(Server, settings.host, settings.port,
     new ConductorHandler(context.system, self, Logging(context.system, "ConductorHandler")))
@@ -199,8 +201,11 @@ class Controller extends Actor {
     case ClientConnected(name, addr) ⇒
       nodes += name -> NodeInfo(name, addr, sender)
       barrier forward ClientConnected
-    case ClientConnected ⇒
-      barrier forward ClientConnected
+      if (initialParticipants <= 0) sender ! Done
+      else if (nodes.size == initialParticipants) {
+        for (NodeInfo(_, _, client) ← nodes.values) client ! Send(Done)
+        initialParticipants = 0
+      }
     case ClientDisconnected(name) ⇒
       nodes -= name
       barrier forward ClientDisconnected
diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/Features.scala b/akka-remote/src/main/scala/akka/remote/testconductor/Features.scala
index b94f205726..336d04c368 100644
--- a/akka-remote/src/main/scala/akka/remote/testconductor/Features.scala
+++ b/akka-remote/src/main/scala/akka/remote/testconductor/Features.scala
@@ -57,7 +57,7 @@ trait RunControl {
   /**
    * Start the server port, returns the port number.
    */
-  def startController(): Future[Int]
+  def startController(participants: Int): Future[Int]
 
   /**
    * Get the actual port used by the server.
diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/Player.scala b/akka-remote/src/main/scala/akka/remote/testconductor/Player.scala
index 72b15922f3..f7d2fbd532 100644
--- a/akka-remote/src/main/scala/akka/remote/testconductor/Player.scala
+++ b/akka-remote/src/main/scala/akka/remote/testconductor/Player.scala
@@ -40,7 +40,8 @@ trait Player extends BarrierSync { this: TestConductorExt ⇒
       var waiting: ActorRef = _
       def receive = {
         case fsm: ActorRef                        ⇒ waiting = sender; fsm ! SubscribeTransitionCallBack(self)
-        case Transition(_, Connecting, Connected) ⇒ waiting ! Done
+        case Transition(_, Connecting, AwaitDone) ⇒ // step 1, not there yet
+        case Transition(_, AwaitDone, Connected)  ⇒ waiting ! Done
         case t: Transition[_]                     ⇒ waiting ! Status.Failure(new RuntimeException("unexpected transition: " + t))
         case CurrentState(_, Connected)           ⇒ waiting ! Done
         case _: CurrentState[_]                   ⇒
@@ -63,6 +64,7 @@ trait Player extends BarrierSync { this: TestConductorExt ⇒
 object ClientFSM {
   sealed trait State
   case object Connecting extends State
+  case object AwaitDone extends State
   case object Connected extends State
 
   case class Data(channel: Channel, barrier: Option[(String, ActorRef)])
@@ -85,7 +87,7 @@ class ClientFSM(port: Int) extends Actor with LoggingFSM[ClientFSM.State, Client
       stay replying Status.Failure(new IllegalStateException("not connected yet"))
     case Event(Connected, d @ Data(channel, _)) ⇒
       channel.write(Hello(settings.name, TestConductor().address))
-      goto(Connected)
+      goto(AwaitDone)
     case Event(_: ConnectionFailure, _) ⇒
       // System.exit(1)
       stop
@@ -95,6 +97,18 @@ class ClientFSM(port: Int) extends Actor with LoggingFSM[ClientFSM.State, Client
       stop
   }
 
+  when(AwaitDone, stateTimeout = settings.BarrierTimeout.duration) {
+    case Event(Done, _) ⇒
+      log.debug("received Done: starting test")
+      goto(Connected)
+    case Event(msg: ClientOp, _) ⇒
+      stay replying Status.Failure(new IllegalStateException("not connected yet"))
+    case Event(StateTimeout, _) ⇒
+      log.error("connect timeout to TestConductor")
+      // System.exit(1)
+      stop
+  }
+
   when(Connected) {
     case Event(Disconnected, _) ⇒
       log.info("disconnected from TestConductor")
diff --git a/akka-remote/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala b/akka-remote/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
index 16193f7bd3..512757c130 100644
--- a/akka-remote/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
+++ b/akka-remote/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
@@ -53,11 +53,8 @@ class TestConductorMultiJvmNode1 extends AkkaRemoteSpec(TestConductorMultiJvmSpe
   }), "echo")
 
   "running a test with barrier" in {
-    tc.startController().await
-    barrier("start")
-    barrier("first")
+    tc.startController(2).await
     tc.enter("begin")
-    barrier("end")
   }
 
   "throttling" in {
@@ -89,11 +86,8 @@ class TestConductorMultiJvmNode2 extends AkkaRemoteSpec(TestConductorMultiJvmSpe
   val echo = system.actorFor("akka://" + akkaSpec(0) + "/user/echo")
 
   "running a test with barrier" in {
-    barrier("start")
     tc.startClient(4712).await
-    barrier("first")
     tc.enter("begin")
-    barrier("end")
   }
 
   "throttling" in {

From d8268f8e6fe93f3d7a428c3b171325774c896b8b Mon Sep 17 00:00:00 2001
From: Roland 
Date: Mon, 7 May 2012 18:12:53 +0200
Subject: [PATCH 10/36] fix BuilderParent in generated
 TestConductorProtocol.java

---
 .../remote/testconductor/TestConductorProtocol.java    | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/akka-remote/src/main/java/akka/remote/testconductor/TestConductorProtocol.java b/akka-remote/src/main/java/akka/remote/testconductor/TestConductorProtocol.java
index f112a1b0c2..4b9da03059 100644
--- a/akka-remote/src/main/java/akka/remote/testconductor/TestConductorProtocol.java
+++ b/akka-remote/src/main/java/akka/remote/testconductor/TestConductorProtocol.java
@@ -460,7 +460,7 @@ public final class TestConductorProtocol {
         maybeForceBuilderInitialization();
       }
       
-      private Builder(BuilderParent parent) {
+      private Builder(com.google.protobuf.GeneratedMessage.BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
       }
@@ -1242,7 +1242,7 @@ public final class TestConductorProtocol {
         maybeForceBuilderInitialization();
       }
       
-      private Builder(BuilderParent parent) {
+      private Builder(com.google.protobuf.GeneratedMessage.BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
       }
@@ -1750,7 +1750,7 @@ public final class TestConductorProtocol {
         maybeForceBuilderInitialization();
       }
       
-      private Builder(BuilderParent parent) {
+      private Builder(com.google.protobuf.GeneratedMessage.BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
       }
@@ -2255,7 +2255,7 @@ public final class TestConductorProtocol {
         maybeForceBuilderInitialization();
       }
       
-      private Builder(BuilderParent parent) {
+      private Builder(com.google.protobuf.GeneratedMessage.BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
       }
@@ -2861,7 +2861,7 @@ public final class TestConductorProtocol {
         maybeForceBuilderInitialization();
       }
       
-      private Builder(BuilderParent parent) {
+      private Builder(com.google.protobuf.GeneratedMessage.BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
       }

From 33cea733a315bc5c8306a81f7ddbb6853360c319 Mon Sep 17 00:00:00 2001
From: Roland 
Date: Tue, 8 May 2012 10:05:14 +0200
Subject: [PATCH 11/36] rename mkPipeline => createPipeline

---
 akka-remote/src/main/scala/akka/remote/netty/Client.scala   | 2 +-
 .../main/scala/akka/remote/netty/NettyRemoteSupport.scala   | 6 +++---
 akka-remote/src/main/scala/akka/remote/netty/Server.scala   | 2 +-
 .../akka/remote/testconductor/TestConductorTransport.scala  | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/akka-remote/src/main/scala/akka/remote/netty/Client.scala b/akka-remote/src/main/scala/akka/remote/netty/Client.scala
index cf143650bc..4735132534 100644
--- a/akka-remote/src/main/scala/akka/remote/netty/Client.scala
+++ b/akka-remote/src/main/scala/akka/remote/netty/Client.scala
@@ -155,7 +155,7 @@ class ActiveRemoteClient private[akka] (
       openChannels = new DefaultDisposableChannelGroup(classOf[RemoteClient].getName)
 
       val b = new ClientBootstrap(netty.clientChannelFactory)
-      b.setPipelineFactory(netty.mkPipeline(new ActiveRemoteClientHandler(name, b, remoteAddress, localAddress, netty.timer, this), true))
+      b.setPipelineFactory(netty.createPipeline(new ActiveRemoteClientHandler(name, b, remoteAddress, localAddress, netty.timer, this), true))
       b.setOption("tcpNoDelay", true)
       b.setOption("keepAlive", true)
       b.setOption("connectTimeoutMillis", settings.ConnectionTimeout.toMillis)
diff --git a/akka-remote/src/main/scala/akka/remote/netty/NettyRemoteSupport.scala b/akka-remote/src/main/scala/akka/remote/netty/NettyRemoteSupport.scala
index 35ef3bf7fd..60c2ac6097 100644
--- a/akka-remote/src/main/scala/akka/remote/netty/NettyRemoteSupport.scala
+++ b/akka-remote/src/main/scala/akka/remote/netty/NettyRemoteSupport.scala
@@ -86,7 +86,7 @@ class NettyRemoteTransport(val remoteSettings: RemoteSettings, val system: Actor
    * This method is factored out to provide an extension point in case the
    * pipeline shall be changed. It is recommended to use
    */
-  def mkPipeline(endpoint: ⇒ ChannelHandler, withTimeout: Boolean): ChannelPipelineFactory =
+  def createPipeline(endpoint: ⇒ ChannelHandler, withTimeout: Boolean): ChannelPipelineFactory =
     PipelineFactory(Seq(endpoint), withTimeout)
 
   private val remoteClients = new HashMap[Address, RemoteClient]
@@ -98,13 +98,13 @@ class NettyRemoteTransport(val remoteSettings: RemoteSettings, val system: Actor
 
   /**
    * Override this method to inject a subclass of NettyRemoteServer instead of
-   * the normal one, e.g. for altering the pipeline.
+   * the normal one, e.g. for inserting security hooks.
    */
   protected def createServer(): NettyRemoteServer = new NettyRemoteServer(this)
 
   /**
    * Override this method to inject a subclass of RemoteClient instead of
-   * the normal one, e.g. for altering the pipeline. Get this transport’s
+   * the normal one, e.g. for inserting security hooks. Get this transport’s
    * address from `this.address`.
    */
   protected def createClient(recipient: Address): RemoteClient = new ActiveRemoteClient(this, recipient, address)
diff --git a/akka-remote/src/main/scala/akka/remote/netty/Server.scala b/akka-remote/src/main/scala/akka/remote/netty/Server.scala
index f9d4ede1d8..87993f783d 100644
--- a/akka-remote/src/main/scala/akka/remote/netty/Server.scala
+++ b/akka-remote/src/main/scala/akka/remote/netty/Server.scala
@@ -35,7 +35,7 @@ class NettyRemoteServer(val netty: NettyRemoteTransport) {
 
   private val bootstrap = {
     val b = new ServerBootstrap(factory)
-    b.setPipelineFactory(netty.mkPipeline(new RemoteServerHandler(openChannels, netty), false))
+    b.setPipelineFactory(netty.createPipeline(new RemoteServerHandler(openChannels, netty), false))
     b.setOption("backlog", settings.Backlog)
     b.setOption("tcpNoDelay", true)
     b.setOption("child.keepAlive", true)
diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/TestConductorTransport.scala b/akka-remote/src/main/scala/akka/remote/testconductor/TestConductorTransport.scala
index d03adebe9a..2c51c2cf18 100644
--- a/akka-remote/src/main/scala/akka/remote/testconductor/TestConductorTransport.scala
+++ b/akka-remote/src/main/scala/akka/remote/testconductor/TestConductorTransport.scala
@@ -13,7 +13,7 @@ import org.jboss.netty.channel.ChannelPipelineFactory
 class TestConductorTransport(_remoteSettings: RemoteSettings, _system: ActorSystemImpl, _provider: RemoteActorRefProvider)
   extends NettyRemoteTransport(_remoteSettings, _system, _provider) {
 
-  override def mkPipeline(endpoint: ⇒ ChannelHandler, withTimeout: Boolean): ChannelPipelineFactory =
+  override def createPipeline(endpoint: ⇒ ChannelHandler, withTimeout: Boolean): ChannelPipelineFactory =
     new ChannelPipelineFactory {
       def getPipeline = PipelineFactory(new NetworkFailureInjector(system) +: PipelineFactory.defaultStack(withTimeout) :+ endpoint)
     }

From e950045015e16659b34c9901b7daa34a0e1f185e Mon Sep 17 00:00:00 2001
From: Roland 
Date: Tue, 8 May 2012 11:08:43 +0200
Subject: [PATCH 12/36] handle barrier failures better

---
 .../testconductor/TestConductorProtocol.java  | 81 ++++++++++++++++---
 .../main/protocol/TestConductorProtocol.proto |  1 +
 .../akka/remote/testconductor/Conductor.scala | 22 +++--
 .../akka/remote/testconductor/DataTypes.scala |  7 +-
 .../akka/remote/testconductor/Player.scala    |  3 +
 5 files changed, 96 insertions(+), 18 deletions(-)

diff --git a/akka-remote/src/main/java/akka/remote/testconductor/TestConductorProtocol.java b/akka-remote/src/main/java/akka/remote/testconductor/TestConductorProtocol.java
index 4b9da03059..3d6c145097 100644
--- a/akka-remote/src/main/java/akka/remote/testconductor/TestConductorProtocol.java
+++ b/akka-remote/src/main/java/akka/remote/testconductor/TestConductorProtocol.java
@@ -1543,6 +1543,10 @@ public final class TestConductorProtocol {
     // required string name = 1;
     boolean hasName();
     String getName();
+    
+    // optional bool failed = 2;
+    boolean hasFailed();
+    boolean getFailed();
   }
   public static final class EnterBarrier extends
       com.google.protobuf.GeneratedMessage
@@ -1605,8 +1609,19 @@ public final class TestConductorProtocol {
       }
     }
     
+    // optional bool failed = 2;
+    public static final int FAILED_FIELD_NUMBER = 2;
+    private boolean failed_;
+    public boolean hasFailed() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public boolean getFailed() {
+      return failed_;
+    }
+    
     private void initFields() {
       name_ = "";
+      failed_ = false;
     }
     private byte memoizedIsInitialized = -1;
     public final boolean isInitialized() {
@@ -1627,6 +1642,9 @@ public final class TestConductorProtocol {
       if (((bitField0_ & 0x00000001) == 0x00000001)) {
         output.writeBytes(1, getNameBytes());
       }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeBool(2, failed_);
+      }
       getUnknownFields().writeTo(output);
     }
     
@@ -1640,6 +1658,10 @@ public final class TestConductorProtocol {
         size += com.google.protobuf.CodedOutputStream
           .computeBytesSize(1, getNameBytes());
       }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBoolSize(2, failed_);
+      }
       size += getUnknownFields().getSerializedSize();
       memoizedSerializedSize = size;
       return size;
@@ -1766,6 +1788,8 @@ public final class TestConductorProtocol {
         super.clear();
         name_ = "";
         bitField0_ = (bitField0_ & ~0x00000001);
+        failed_ = false;
+        bitField0_ = (bitField0_ & ~0x00000002);
         return this;
       }
       
@@ -1808,6 +1832,10 @@ public final class TestConductorProtocol {
           to_bitField0_ |= 0x00000001;
         }
         result.name_ = name_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.failed_ = failed_;
         result.bitField0_ = to_bitField0_;
         onBuilt();
         return result;
@@ -1827,6 +1855,9 @@ public final class TestConductorProtocol {
         if (other.hasName()) {
           setName(other.getName());
         }
+        if (other.hasFailed()) {
+          setFailed(other.getFailed());
+        }
         this.mergeUnknownFields(other.getUnknownFields());
         return this;
       }
@@ -1867,6 +1898,11 @@ public final class TestConductorProtocol {
               name_ = input.readBytes();
               break;
             }
+            case 16: {
+              bitField0_ |= 0x00000002;
+              failed_ = input.readBool();
+              break;
+            }
           }
         }
       }
@@ -1909,6 +1945,27 @@ public final class TestConductorProtocol {
         onChanged();
       }
       
+      // optional bool failed = 2;
+      private boolean failed_ ;
+      public boolean hasFailed() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public boolean getFailed() {
+        return failed_;
+      }
+      public Builder setFailed(boolean value) {
+        bitField0_ |= 0x00000002;
+        failed_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearFailed() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        failed_ = false;
+        onChanged();
+        return this;
+      }
+      
       // @@protoc_insertion_point(builder_scope:EnterBarrier)
     }
     
@@ -3300,17 +3357,17 @@ public final class TestConductorProtocol {
       "\022\025\n\005hello\030\001 \001(\0132\006.Hello\022\036\n\007barrier\030\002 \001(\013" +
       "2\r.EnterBarrier\022\037\n\007failure\030\003 \001(\0132\016.Injec" +
       "tFailure\022\014\n\004done\030\004 \001(\t\"0\n\005Hello\022\014\n\004name\030" +
-      "\001 \002(\t\022\031\n\007address\030\002 \002(\0132\010.Address\"\034\n\014Ente" +
-      "rBarrier\022\014\n\004name\030\001 \002(\t\"G\n\007Address\022\020\n\010pro" +
-      "tocol\030\001 \002(\t\022\016\n\006system\030\002 \002(\t\022\014\n\004host\030\003 \002(" +
-      "\t\022\014\n\004port\030\004 \002(\005\"\212\001\n\rInjectFailure\022\032\n\007fai" +
-      "lure\030\001 \002(\0162\t.FailType\022\035\n\tdirection\030\002 \001(\016" +
-      "2\n.Direction\022\031\n\007address\030\003 \001(\0132\010.Address\022",
-      "\020\n\010rateMBit\030\006 \001(\002\022\021\n\texitValue\030\007 \001(\005*A\n\010" +
-      "FailType\022\014\n\010Throttle\020\001\022\016\n\nDisconnect\020\002\022\t" +
-      "\n\005Abort\020\003\022\014\n\010Shutdown\020\004*,\n\tDirection\022\010\n\004" +
-      "Send\020\001\022\013\n\007Receive\020\002\022\010\n\004Both\020\003B\035\n\031akka.re" +
-      "mote.testconductorH\001"
+      "\001 \002(\t\022\031\n\007address\030\002 \002(\0132\010.Address\",\n\014Ente" +
+      "rBarrier\022\014\n\004name\030\001 \002(\t\022\016\n\006failed\030\002 \001(\010\"G" +
+      "\n\007Address\022\020\n\010protocol\030\001 \002(\t\022\016\n\006system\030\002 " +
+      "\002(\t\022\014\n\004host\030\003 \002(\t\022\014\n\004port\030\004 \002(\005\"\212\001\n\rInje" +
+      "ctFailure\022\032\n\007failure\030\001 \002(\0162\t.FailType\022\035\n" +
+      "\tdirection\030\002 \001(\0162\n.Direction\022\031\n\007address\030",
+      "\003 \001(\0132\010.Address\022\020\n\010rateMBit\030\006 \001(\002\022\021\n\texi" +
+      "tValue\030\007 \001(\005*A\n\010FailType\022\014\n\010Throttle\020\001\022\016" +
+      "\n\nDisconnect\020\002\022\t\n\005Abort\020\003\022\014\n\010Shutdown\020\004*" +
+      ",\n\tDirection\022\010\n\004Send\020\001\022\013\n\007Receive\020\002\022\010\n\004B" +
+      "oth\020\003B\035\n\031akka.remote.testconductorH\001"
     };
     com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
       new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -3338,7 +3395,7 @@ public final class TestConductorProtocol {
           internal_static_EnterBarrier_fieldAccessorTable = new
             com.google.protobuf.GeneratedMessage.FieldAccessorTable(
               internal_static_EnterBarrier_descriptor,
-              new java.lang.String[] { "Name", },
+              new java.lang.String[] { "Name", "Failed", },
               akka.remote.testconductor.TestConductorProtocol.EnterBarrier.class,
               akka.remote.testconductor.TestConductorProtocol.EnterBarrier.Builder.class);
           internal_static_Address_descriptor =
diff --git a/akka-remote/src/main/protocol/TestConductorProtocol.proto b/akka-remote/src/main/protocol/TestConductorProtocol.proto
index e483bf4f01..007965b2e8 100644
--- a/akka-remote/src/main/protocol/TestConductorProtocol.proto
+++ b/akka-remote/src/main/protocol/TestConductorProtocol.proto
@@ -25,6 +25,7 @@ message Hello {
 
 message EnterBarrier {
   required string name = 1;
+  optional bool failed = 2;
 }
 
 message Address {
diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala b/akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala
index 7e3d315fea..2bbae6d28b 100644
--- a/akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala
+++ b/akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala
@@ -22,6 +22,8 @@ import akka.event.LoggingReceive
 import akka.actor.Address
 import java.net.InetSocketAddress
 import akka.dispatch.Future
+import akka.actor.OneForOneStrategy
+import akka.actor.SupervisorStrategy
 
 trait Conductor extends RunControl with FailureInject { this: TestConductorExt ⇒
 
@@ -194,6 +196,15 @@ class Controller(_participants: Int) extends Actor {
   val connection = RemoteConnection(Server, settings.host, settings.port,
     new ConductorHandler(context.system, self, Logging(context.system, "ConductorHandler")))
 
+  override def supervisorStrategy = OneForOneStrategy() {
+    case e: BarrierCoordinator.BarrierTimeoutException ⇒ SupervisorStrategy.Resume
+    case e: BarrierCoordinator.WrongBarrierException ⇒
+      // I think we are lacking a means of communication here: this is not correct!
+      for (i ← 1 to e.data.clients) barrier ! ClientConnected
+      for (c ← e.data.arrived) c ! BarrierFailed(e.barrier)
+      SupervisorStrategy.Restart
+  }
+
   val barrier = context.actorOf(Props[BarrierCoordinator], "barriers")
   var nodes = Map[String, NodeInfo]()
 
@@ -240,7 +251,8 @@ object BarrierCoordinator {
   case object Waiting extends State
 
   case class Data(clients: Int, barrier: String, arrived: List[ActorRef])
-  class BarrierTimeoutException(msg: String) extends RuntimeException(msg) with NoStackTrace
+  class BarrierTimeoutException(val data: Data) extends RuntimeException(data.barrier) with NoStackTrace
+  class WrongBarrierException(val barrier: String, val client: ActorRef, val data: Data) extends RuntimeException(barrier) with NoStackTrace
 }
 
 class BarrierCoordinator extends Actor with LoggingFSM[BarrierCoordinator.State, BarrierCoordinator.Data] {
@@ -262,13 +274,13 @@ class BarrierCoordinator extends Actor with LoggingFSM[BarrierCoordinator.State,
   }
 
   onTransition {
-    case Idle -> Waiting ⇒ setTimer("Timeout", StateTimeout, 30 seconds, false)
+    case Idle -> Waiting ⇒ setTimer("Timeout", StateTimeout, TestConductor().Settings.BarrierTimeout.duration, false)
     case Waiting -> Idle ⇒ cancelTimer("Timeout")
   }
 
   when(Waiting) {
     case Event(e @ EnterBarrier(name), d @ Data(num, barrier, arrived)) ⇒
-      if (name != barrier) throw new IllegalStateException("trying enter barrier '" + name + "' while barrier '" + barrier + "' is active")
+      if (name != barrier) throw new WrongBarrierException(barrier, sender, d)
       val together = sender :: arrived
       if (together.size == num) {
         together foreach (_ ! Send(e))
@@ -287,8 +299,8 @@ class BarrierCoordinator extends Actor with LoggingFSM[BarrierCoordinator.State,
       } else {
         stay using d.copy(clients = expected)
       }
-    case Event(StateTimeout, Data(num, barrier, arrived)) ⇒
-      throw new BarrierTimeoutException("only " + arrived.size + " of " + num + " arrived at barrier " + barrier)
+    case Event(StateTimeout, data) ⇒
+      throw new BarrierTimeoutException(data)
   }
 
   initialize
diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/DataTypes.scala b/akka-remote/src/main/scala/akka/remote/testconductor/DataTypes.scala
index 90d7eeccd5..cadd69f786 100644
--- a/akka-remote/src/main/scala/akka/remote/testconductor/DataTypes.scala
+++ b/akka-remote/src/main/scala/akka/remote/testconductor/DataTypes.scala
@@ -19,6 +19,7 @@ sealed trait NetworkOp // messages sent over the wire
 
 case class Hello(name: String, addr: Address) extends NetworkOp
 case class EnterBarrier(name: String) extends ClientOp with ServerOp with NetworkOp
+case class BarrierFailed(name: String) extends NetworkOp
 case class Throttle(node: String, target: String, direction: Direction, rateMBit: Float) extends ServerOp
 case class ThrottleMsg(target: Address, direction: Direction, rateMBit: Float) extends NetworkOp
 case class Disconnect(node: String, target: String, abort: Boolean) extends ServerOp
@@ -41,6 +42,8 @@ class MsgEncoder extends OneToOneEncoder {
           w.setHello(TCP.Hello.newBuilder.setName(name).setAddress(addr))
         case EnterBarrier(name) ⇒
           w.setBarrier(TCP.EnterBarrier.newBuilder.setName(name))
+        case BarrierFailed(name) ⇒
+          w.setBarrier(TCP.EnterBarrier.newBuilder.setName(name).setFailed(true))
         case ThrottleMsg(target, dir, rate) ⇒
           w.setFailure(TCP.InjectFailure.newBuilder.setAddress(target)
             .setFailure(TCP.FailType.Throttle).setDirection(dir).setRateMBit(rate))
@@ -64,7 +67,9 @@ class MsgDecoder extends OneToOneDecoder {
         val h = w.getHello
         Hello(h.getName, h.getAddress)
       } else if (w.hasBarrier) {
-        EnterBarrier(w.getBarrier.getName)
+        val barrier = w.getBarrier
+        if (barrier.hasFailed && barrier.getFailed) BarrierFailed(barrier.getName)
+        else EnterBarrier(w.getBarrier.getName)
       } else if (w.hasFailure) {
         val f = w.getFailure
         import TCP.{ FailType ⇒ FT }
diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/Player.scala b/akka-remote/src/main/scala/akka/remote/testconductor/Player.scala
index f7d2fbd532..6e78610cfb 100644
--- a/akka-remote/src/main/scala/akka/remote/testconductor/Player.scala
+++ b/akka-remote/src/main/scala/akka/remote/testconductor/Player.scala
@@ -129,6 +129,9 @@ class ClientFSM(port: Int) extends Actor with LoggingFSM[ClientFSM.State, Client
         sender ! b
       }
       stay using Data(channel, None)
+    case Event(BarrierFailed(b), Data(channel, Some((_, sender)))) ⇒
+      sender ! Status.Failure(new RuntimeException("barrier failed: " + b))
+      stay using Data(channel, None)
     case Event(ThrottleMsg(target, dir, rate), _) ⇒
       import settings.QueryTimeout
       import context.dispatcher

From 9a33f468c0082ee6735d7f4684945ca06f531fcc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Antonsson?= 
Date: Tue, 8 May 2012 13:56:24 +0200
Subject: [PATCH 13/36] Adding a doc diagram for the Test Conductor Extension

---
 akka-docs/dev/multi-jvm-testing.rst            |   7 +++++++
 akka-docs/images/akka-remote-testconductor.png | Bin 0 -> 18288 bytes
 2 files changed, 7 insertions(+)
 create mode 100644 akka-docs/images/akka-remote-testconductor.png

diff --git a/akka-docs/dev/multi-jvm-testing.rst b/akka-docs/dev/multi-jvm-testing.rst
index 33c7dc7507..d19344e751 100644
--- a/akka-docs/dev/multi-jvm-testing.rst
+++ b/akka-docs/dev/multi-jvm-testing.rst
@@ -380,3 +380,10 @@ same machine at the same time.
 The machines that are used for testing (slaves) should have ssh access to the outside world and be able to talk
 to each other with the internal addresses given. On the master machine ssh client is required. Obviosly git
 and sbt should be installed on both master and slave machines.
+
+The Test Conductor Extension
+============================
+
+The Test Conductor Extension is aimed at enhancing the multi JVM and multi node testing facilities.
+
+.. image:: ../images/akka-remote-testconductor.png
diff --git a/akka-docs/images/akka-remote-testconductor.png b/akka-docs/images/akka-remote-testconductor.png
new file mode 100644
index 0000000000000000000000000000000000000000..b21353832670a12eb8130292416df62c6f1209a7
GIT binary patch
literal 18288
zcmeAS@N?(olHy`uVBq!ia0y~yU|z?-z<8g7je&t7;CZV(1B1J{r;B4q#jUq@t5@ix
z?|lA|&0NYP%fRc41iv(|+ZPQfp*;)Qbpw^OKK4!Q2yAi@IV17$-3;fZioWd&6n$I*
z4dr@VZuVJ99k*y(oWUWr$lX=3Pf&TWkHjBk9;1{6%j@@E-&`BM^w+j;yS_aA+$vwa
z{@trpuU=KXs;ztf`$}{(BLhi5=UBm#-7+BOGcY^``3puR#DGmgrFyi%hC?YZ8xQf|
zz()%i28OVX#nHUx7gxQyHqlS$6oW)K!>9B`U(e2OcgbFG_*MWzk2b?ky{#`w}$dbB4VU}j*LU_bd!a!a|%hU;9I_!$_WK0304iGcy3mtlgvFjRrTIv0?a
zp$cH0d~#l;UaZ1}iT^;+mU_8%W~ecUfcebGde6W9`}Bv~>;A0XS#tV(U44T6nXl%5
zGk%=EUQ@W{&;JGfQ{?}A>VKR(>G5_i^&j8!{=bdc|Fy1OAL>D!`f2vY>x8E^$IPF*
z^mBai!+A1a-W*Pq|5Nb3TK?aa=WYGge}4Ra>m9fI*{;RxWlZ~@%D$I>@BXcJ{kFoN
z?(1)U?LKq!eBHg6{a+W|e{H?}Z`zM_^}ocC3_m@;-2TI*&l4xFxH{9n@>WHxc-)_D
zx$EPWEN$_F)+Q+{SpRfPB`>}%UQ^DnY@&zw0
ze0Y4lu2Pp-UwqD*|C{6gew)V<_kH_Yt>;Jc{+fDZaQp4w
zcfWZhAcp
z`1PCbg11er{~R~1^5|qHgzPGXRQuWKED>OY+Nj=BL6EjhQEIK
zTJubHrM@cP-TMXVPbHDUCoQJj^uwp-5K9-nGS-us{grRFO;#}d^73ls2WL5>IJNiB
zjn7*=zxh~L-#&`3!fy7{PygqnRPL?n7TX)R@0+cy=gqWjbH8r?`uJt*Nu&I_-@A{$
zPrn%~UwZ4u)<3m}&u-6MZF|0a&HGvJC#23Re$sloFOW59d;9WtjVFG;?fq1$QzY|R
zu0>`#OO{@m<;2fX-?Z14*Ux_U-J79ofBe)gm&o|OpNnkWw$?E1h&%oN^R3F~`*(de
zzvH>q?lD(GA*(|DH2c%@C9lWw?wM0^_t%d(Yb-sBio(vj&kf1dbgGKk^zMs+uaw&V
zGvRqM`k!B2kKc2l?)j6|=bx&qt`?a0^Um{p>&jP0&Xi|-SKjMX`B!!Qo(FaJO@D{1
zUz(nIVJBzxy#uE=o;37&u*EiZ>-()S@hcBkum9un{(JSmO8vP%{(hPn!;@_*>hY8B
zrrZ2W$yehlHm%w0F&FYtd4_XD=f1(p+Dks|)|`{lt7d
zci<`AD=(NC7>P|1YET(UZ^J
zKI*ob9$EoKMcZD#=YI3%v}C$nDePxyiYat)No1XEiv>#O*q;S~zE(T>F=tCG1OH{aiff
zkL0ZU{6p{lWt47OR-SaLPOT$bezkgbkbdw~>odX?7uU=@`D(8#v;IMY8H?Ta?S8v2
z>VH`1JA>E1-gzSFTrC)DoXrp^Cbm)3uANqfZd
zGl}!$HqK>-CIaBi6+
zQfce_@MY?W&Pl>6_fE8V|5PP&`HFLn`RT!{4|uKplaL+tYMY6Y>65FUuY40IeJ{A;
zUZ41$A5R}&zkGO$$Ahh#7j0#n@Zex$pJs(w;^J>>4zwQv0Is@#*C&b4LN
zg)8@0E`2|3<`(hAIji@@$3A(+x#vvHwA%SgUaed``{8ZrkFV<*zOj9_$}^3Ye$16H
z|F(VqVMa)1WW87R^vHVVsS0QMXC{7_Ww)!k=~_=p;J^8)x8>JX#j{19Z_8@7e7*j=
z(21{mR=hfy*=+j%Jm;kEMTgR>zwTN7&F!@-lm71j+hEIwS`V_VOM5oG-4&2ByR_)q
z<^{KMpDsCnba|T5>h}@n7p_VVz54x3)uw+d`a-||-g?l&|H0WHCVSh!$ojhV-;^fa
zxBoq_cE#V^e7pHqP9F!=#Ed6EIWhg`=XI=cOltN~zvnF6x9jbf02PnLb8gg@KL0uK
zQ=IN}$*s5U&6ir`f4167dDGi{e@}AGdAV-!n*3X9l3woh^bguLee#0cIZdkf*S}x(
z@s9h2T809@buj^(_N_5*oc#Wlt(&FV&zxzeT^H{2zQ3)AC+J+@an<9SqQB3(H>>7-
zOwX;iTkh+>FJFFb-nYL(0hRgFKK=dl^Uwb1eHZ5`m*y1ryfV+3?|%%%x${`$F4lFU
zwjdsJ!I~-1s?`V7GEn)z04^5Qej0+d4tT``Nto^ns$%|+p(#TMxi074GC(b`WmduuRQ*{
zGt7^__QqjoO>caW?aBG?p&ta-Zt%O_#*`4lcw&C%HS_wqs~*!Y&wHsKt$r-D@94aj
zNpID`t&GatlHVTx?-ow5Pp;pYynn;<%Gjb2J09Y
z7!GVIc=_O;@%?>g5*a1J85kH6B7b=H7w$a><4RUsTJS$|alPzIk#=@)lfI$(*(uIr
zrk~CPHWacp^zO5(mf>U2VK4VLTxVZW!Eg-XO2?vwc
z2VeS&j>9Yr3=9VA4mf^1G4b8(q^S309iR@-F;R<*yje*x>q{ya9FkZ-LBKbQd%47Q
zV;MPy2@;G9G4mUnpFKIak#pv*WnaVE*`OXvh}^O6_>QX@(Ly1M)**S~jP*6SCF
z{#W^T)LtUoVBL`&M(bRJ!aw?}{5!a#sZcd3Ms=Ob>sdSWet&iTGx-PmV=kzg9_`0o
zM|K?gD%qng_*kn@wNv}C)<4fb%stu?;Yl%);Yyv_kG+r$=~?|lJt^klFP>vlI|`@n
zm~kLPzZ6|#bC67q~OS1c^nn+S7*S4^kMx<~Up{xm}rc*S@=;`-?SNIoIv
z;0_^>lP2xxs#gMe9_mDNey8>k`_B3u{|wh9#B>%;-67-|6Rv0|^dIDErXFp>bs&AG
zb_f}+E2w1yxdCLwsU47*;W{=|BK+7?k4L9=2zAyQfQ*m`?*xU#zfOtp9d^x-uv00V
zD$xm!N<11o{}`_WyV`J_i_mpnk2`a`p>D`sQCR!SLtQ@Mj3=u1s_gybU)n1FebgX*
zDd+dQiR)xudcK~uLvHWWZx2J~@2>mtF?+wwpX>F%zdr
zJ74=u+njIxk0)CXpRfCR`!J|~x%c(|bGiE^*DL?`yWKDU_t*L3+qZKa4|#NIhnr=1
zvFwiz%k9kfsXh61Vf!6W7j&D%iXFG#7ysR6{eGt8`@s9(|6DTPXaDDU_CAY0U&QyD
zeXM?F74Z7RwzRFq7nfwWgCgx%X=~|UUwf&2Po|3V_*b7vJ}!QS-R{MbhsNhBzgZOC
zSzqhHym$6%P&f7QeU1BT_a*FW3<>vibarvFNzQ673Ils3~&AeW8
zuHjzg|ySk{sES{c_*LJeikUmFI5h>U<6Al@`8Ht*rk2d!0o2j+8Bi
zoOd7d+1I5_+QxOv^uOA>UsD91nK7RKw7%l5>tn%XM{?W}=hffyZm+$5N9~}r`o+oL
z7OzX~(SE#(YuVOw-%I44@97IXYx?kPe*O0ylhv+wU)xjGo4roCQdfCnb!OD)U$giX4tj4f(o|zq%sdGXIgy
zYp*-yz1jP!t#1F$Gw!(bFkDmzR5~U2_GI^DO7(tu;P+~#Z*!3C{JYg&pUS-|m)&i?
zwl`eeGbP^h{+71w@9JN^OI~Yy`~B_zu07e`G=H0A{*PM9UUA9RLOU(yzl}v$k#&sm
z-JM>Y*`<8GUWbbIee0d~q}<0cQRt4&-f7Ru1g)986q{e^od0Z%QoK+i)5NTpB{QC;i0rU
z{^?}7_QN~gg;>^KR`WM1URrPS-SzRLhr8Bqy`n9Oej1aXb@NDl
zT=w<(*Nb~VLmYka{$i%<4CjLeHH7cipPMM27acBn-ROJmREQv@fa$eVQXqm-f1wn*6Nysxy*`c^_Id
zuWeKHe}Av`TYu*q-*N2EG9Sy|SKh@>%dFo0IodsQN%l^eW>Zbkb1CjKzr1*7>GO~^
z`uYF5>}z#*jW1nQw#~jOblq!%(I$iZ%=mL}=PWPPuR7{9>15f?sC=t=uV$#+`<(BS
zR=!vAQ>u31Ol$>9lv-_Rz**evaaYuHj8J^o@IPQy)T=C#RV6#!Bs-@pM=I`ga#SnJ2ec@+SO?@6Elfkv7}#SmvEm
z6P6vfbTi}3mwIZlDYE=)$ob@$)z>Xf*0=>N?vW^+{JCcR8}Drs+YC>atM^4rJ~{K8
z>`U(E-w|b9o?I_WIAu(-Nm~nb!J;t7ySz_nvtUSJmcK;eJOg%Tw7!2zV6)F
zwY}`fratF$Pmg#CJ?^>mJ@<3w9{#O!%^KGQ&Rn-^
z@sYQCGdq9hXkTCMx%gsg%;^PXr&~{7P+KP%eysGSDD(8cen$`GNnO{Pex4&u`t;Rn
zq3!&|Cqg0*6Vt-r0Tu68JGTbk?ZH*wpZz3WRS75ZA=3D=XkJgWv&
zD|4-xSs0sQvSjb`b(5ZXwrvx-m6a03e53EnvOgJ7D+^2)voB05TS{(%Fo>#oovG8e
z`utbtanyF@FK3e{zILAa+OPZRk-*)(na4^Ep7$L3>akklMILLY)xzCxbF*Vk_iR6X
zCHrMdGPqPsi^+bHBKUaL#ePq=by9n^Y}dIhi=Gsr+L+sHbaKZw;eai|JGMOa&3zQN
zbn<73>#y$SrJKGrJC%Cq^bS9%pOKHB?`U)9@3XCX*2kRVQGEZ^Q{}lE&vj<27Fqre
zdKA4_&~o|dlw&iFm3-?>Q(??Ke|c5U43XoZxf5@ArUz8}o;I80^DS51Gc{?g?fhRp
z8LAf~T|NJln!b(uC~;xNja}|r-BNGe42gbxOILhd;Eh)&l42yUr-rXmExY7sb$W+d
zpD=gr#yPJ&t=iI#7HymOWA^bK%Om#*&)v~?t*&a6S
zyuL+ZO|^gVzH65ErZM|p-lAe8Yd-bM(ipqNnz=J`qxTw||9NG)f?iTg@%bvD^O>hp
zOqTBTE9uR?`(5nt{P$Ph**`wDW8I;@?>cu!e!M$(e$uo;U-i&O+4myu@js4|4Xo{F
zoP8zmp6R;M+%q1frTea~bp-`;!r7keyZW=-axdj9%k)?_2q?{n0viF?pe_#E@
z6XBM1>*B0T&y=ItllINH_O9oMVXdik!TgtNEpF*PjuKw_^=PuS*Bv{%#n~~pf8YBw
zSLiXQP0>~C@AGQT*%!&*Ct7aq$-d{`?swl*ZDY(;q2HM?+F`%*QeWIV7ZMsB`#5TD
zq^5Y@`}FjfgFB4YML0jc^{q$PfM=j@D$*gaYK>ZVfv*Krcpzb$+k
zYIkL>%qlOfbjj;)+-=?0&0V6FwrJVQ^=y+1Q)9k+87^sF%NbbYpFX$yr@7j?w9V^&
zteMz4XY$&WYd-JQl1r1k-nDd=$WrUW`L^MvOHbw2T=^IF{9>Nu^&?TX##XY)Z~s0q
zyeT?4*Xh3Lx=_72@2^agFj^-)bH~~@w(nNjocKB0A|u(xFiLpG*11NTd$LO{j;m~Y
zTNWB{)ig={*w(p6Yc2*oj7nb@`PMj9`1eW6UBZsvB(H0UpEJx>uSo$H0U0s3C*AB>
zw}jEtdBTz{Lbs-HZtu*NT4N%T>2Ntib>bPL=5xg=;)d%=<>UXY{ryHw_OZ!5&%D(O
zPkqwlmMM`tZhZc6!733=&YFGRKKZIA-mcJltW|i|J9WwyHD$@1vn+T1?a$UaZL^G7
z{dtUU=&DnI=tD*e#j%z)KzSf-hRcE@ppwIui
z-7UMvQNQoh9#aOTzOGAw!RP&Db8MwnJzAvC_pGun`>a&l#yQzX6SA3(7#;7HxE`Xq
z_4=9^eo$*-#nyYj@^b$?x4Im6{#Q@-RBrcNchSdO$3!E7A4l14^8Av-v0CtiM@mnO
zoc4G1^K;)~!#~|qN;7w3DuI+vc+N2aQ*6H-QWfKP$?vy$P+9bl*i
zkeSGR_GZvn54g{MEcA~sN^f`7PSDT*BS?qgI-zi-&ecEEh5kePs+jyqJDmPZ{?Yzg
z{xP_>i|HxP7}a%;<{#M)?hisc(7Ud6PW+(`>ZgOc;i)m5pni2nVe4ySiEwbh356dE
z1&uu*jZYk#iq!A#lnDRmk1`gq`iKWX4Ugu73V+aGgzBUnc}|a>#ejnf5z`qK^SUN+_xbnU*$Xa1hHea6-TOYt;ncctVhU>3emK@;xIB8;DW|Efj+IWeVo_xFt
zhHMwk`_KBZ=hVZ57{!0j4c7lZxsX}x*#!^r-V%3V@14`XCO0R4-^LWNsqJG{J9C88
zxfd#nzvq~)0Vz+Pc|AT~^`W}ulmj7IZ4Y+O{_*qjgB!Xf%Pnjgjz}EHS@*`%&6nli
z4yQl<#_Pkq4>az4F2+2R_qH$CmnrLFHJ(4;D}`n5QWnt4n%h
zz#x?DP}5Y{`XgOq!;}{aj0%QqN7N<4JO0NybE=mKtGoIZXx`eWrTXU2~Iya!tBPWN_hW027({b>0nfZ>s_ckA(o
z9shlM>)AZpnIJ|U6Ma;#8ZfD9gT_O)1&J&QcA(^<-{9Hsjel|);{^kz8K-VGCoc^T
z^m?>P=k==#bH
zdpdQhC)io#eP$ag-yc{a+3@3d-o`v(k(o2N4jeP>QMXn(aOuM)HM{l277QQx+w#xy
zS}WRI#(&Q_`g|uJyTZSX%a2!2^~yObz>~08`q*>}nFE&!uRe%)|6j2^N%^CN
z%t!ry-{+^!yK;i(AcNVy#69kZIdoLh^N(tP!X{^(r-`${1}E{p?u_$a=DY8@S$HFO
zub0GQ7K=9mpm4A-;9I40o#)^Iu1~KI{*}M&v?VL*y3xGF(VeEC^+g9i`6X?7b1d}F
zWwoRA_gy@ye@YyT=kd!qr18=7)^p|ro6EPkuOmCWVxUz~m%q?_kl3n36<%e~AoH=Q9w3WAqtcfQy6TmVO)XUY
zcoJl@P`Fb6VZr|@|0Khe-Wr2OnT0YXK=w@q4IpR@D
z4qSRjFtX)``p5G-gfiz#hJy$FrQAT3s?jisVWZ9r*qJ
zF8GlaXn-8*uq+AB!zXrFP457OHz@EyxC7Ks>RJ6m1rkEAxH)3~*lS7cz4XUbr_a~f
z+x#~b|Mn{P{jUoje$B4ivYy@k_TE2>ufI9DYk&Fg{C3bXo#f^z@;^cIBR22^4lgpXH6CY3HPuaiz*PXAA_xIl``aLi2`||z2Pk+ocS|@b?LT~)SYG4
zZ9UqLt>$Z;`!&VJV&e_H`G4%+m&V`kuDx|Wx9hy9`MSt$I~RP>O3UBRSGP^Pc-!sQ
zy)Ua@9G0_-`wv>VkrRh-0xO7a_frZYGuirkh
z_0tqH^AkU3m?f|G{VJ`u^UC|9JJv;qMz8m)JhJj}%+6`+=M?L&w#c6Ut$6mMV&9(Z
z`H^Q%oB$0l|M?f!|7mK{sj}+mm!4k9Gvv3;R#^VA=9uOAd*#jhgdfkUFRuAjwWR#>
zkM!9+tB;hLe{Jlqd9}my?Em6<;1x~#e!hJ#XZJ&qU-EU^?EK3NnIP8%XzBXZUdcS}
zo3)NLbN1)=pU+pn{Fv=J^ZGs;)w$&YtLjh?@ey8idZvL7+YkE5Qhs{Fqwj?{H1EH|F0SES)MF8VsU!eb@RoJH_z{Td#>2$^UcaW*WMgIE3vKov5&3J
zf1j6r^Os%i{nBZLcYEKT*pc_l)pYO9+ULGi71OuAYV_NYSo6z6ef`QpTcv;H%R;_B
ziKzK=OaJ@6|F5pc+tk@}zOku1Ri}FO(fq<#_QzH)|CXpc|2reX@Ys%ZYYKZCPyPli
z$lG=|f6~R1&m4Nco_TIpe>)@7i+Qt;Ti$Ay531|;B>X9l^FK6e)7?Gue*12|x96Vj
z+05yyi`sJRPnU21`g48#_aECX{g^6PeXu*;?yJR;<>sfiz4lo=>+huk
zwlaA}+@l$fwMtKI%y@slyYuwFrxsxgmL1=DCRR`6r}g^dF7Y3Zc^^Bz>aXqX$C1Co
z7oYf9xAm`;U9GDB*4|s^-;|fir2PF_cWq+n^?y(A=PW+8>y>qxHS?#LHPc_MJZCuj
zyPeyy(%Px-|6bXAZsw1=b4%?W&#LfCp7p&buGD&GQQpx`k(L1n5}r$EI4jr?TSg=68o=hxmtK`C{
zJyZAbd^g+Ms{bJmGl^6w616>Yz2@chp@ziD$G-#ou+Uq)|c?(#<~dZjyG
z&*)hdXL9MR?RUSmn+tQlPM)YI`*Px=`b}D4yXU%hKhwNh9V7f&bbe;C_O#1xTXPeS
z7CqJ3>F08_F>a?{Z1JmG8(x0jWwh>fevG#GTEXk@{w@=>?wrh{rpoUVqjIdMZp$;%
zkFncdsIKqPTA%y4@S5$qt!IxE7d1Zldoosd_Fb=2PtI5Bo=)9=D_5^*Z(>aNksW?P
zQVnGyrktshcI~@3d6)T8x0kP2-t?7CD?H12)mP{CG0pc+SU>r0-G>Oa+%
z?aqvuyn@fFI8>$X^0sYn^NtqXww#*n^Hd~PeY=s{u~Oll>|57(cvSr+T@L+ib<*Y7
zmTfx=*^F=TJuLaBvhPygw$o4lpIklpdiXp0^@V56ZZmF?y5iM$;aBUr#&c$$Z+dQL
z>N+oKw$5_qE%%2hCJRrcd|F)I>+${A%RhH@bJq%Q$gVgg^Yv@Ow%%af@b5cIyg5F{
z30*9Uep#3~ef6=NiSGA8cDlsRt>4b*yWok1?=BCOlk0vv{dUV-m9r|}rQ*`ZH&WO4
zoiCIAX>ul3c;17)mv8P#fQH+OE`FCa{x#?6>zWtFmwHwo*_x!z8y&qa?)toMVJ$19
z_S!MeIln}GYj0!T%)-0d@4YoSy{*W0HJk3~EwcKP*D-FLyD$49o7y^kxtFUZ{xg|j
zsQh%&RyWxvW&7TBJN%C4{hR-N^`TYoUoyQ-IKStZ>HOET-hP`EP@GyaJ^T5q
z$mfCKOAqy~3SVsfwuUF%O*`qxuJ60fRn>T#s%+*7w>}m4Y~_@R56W)d3j1Dn>dU@o
zUtQ<%Myvl^5_pTHeCh!|sgpVG$=Tnw?Ry+^`_=CFbz8o!oWOOp8
zbk&rXYunaZ?Jweeou_<#nc22%>$`m$d3|I4C;MG?tG_(=QT=v~>W9gDN}5WRYb?=IsiPqj_!gI47&J>B3@t0!7}Gi~*(^~tdVI_xN@W66wRNfPfloA_pX1=^&RJoV&2xK
zMuyLSl``+ko8?}9yH$_WP7X+&$&zAPGPx&Pa_5&%e#YzGt~oHduQ)ebzc*XzN!1g7
zp8CFYA7{6Puk8-amAd}x?CQe}g{d*(p*>mWPD|V{sVe333ts2BIcWiCDqz`Vwc9Ih
zo?RWBH|y1&^>1@-wfF=}9aGQs^(y%;v+Qlt+8J@?XMf%M-c$0`8y=NRK>ovFvB7T$eQ9h3fafv9z@+~Tw4uf_I0nH4)ZX4ReAxW`u7aj%PC
zE||5|_v22+V_TQ4^FMU`+@;&PUuTBi+O?u^t
zxXI@$Q|jE%B7?OdJG_K9-_x>vv)e-@a9-J*d}6D
zYn7Y+@~N+{pQt^zf0z9pw}kXk3*Kcm`!w5jtWsXJcy0B%^TuE8A6sorv?*VrpELES
z+!Nb%zyDgy+4p`{b#1=H%hc%a46k?0`tW0Q&4^v~(
zrLWIYUNoCqE$;P7<-Rc+d>E2}h)ZB4(j=*-O}Ri91tLhGW`Z*)ps&z!7s
z)g!)lnUB%Bx14jAY<+Jz$<)#NE`Rv#v*qvZor(Q^Cakx9Dsy>t|KytO!s5PG=OXL3
zO%+_SO?iRsJoT&?^}GLPf4#P*GgK*Pzi)I^C9C|_
zhuB>Cn`OG{)?R(<(_w$k8m%)t(s@`aHn%$FwcN~oHyexd(yh1JJhloyJfpKiPv+&a
zwVYjMCwKfSSkmoo<@)D*-Mz}QN9;3&_IF)>_oQudPxh_TM=J0C;SxO^Dz&GhaINHV
z$?#)S3;#yHxzgg18$GLHUCnl{@4t%uc2%i~#7uFcHVGrG?EcvsV8
zW7U&u0#BH5d~IGXxI%MXa>}Q2shK*_o7>(^E6jDzHhML?O@02_i!-l9_!+M=Y%%q%
z^Sj65*AscByjN{``F1ak-M7zvKl5FmdHMsL$ur+xO-}qY^G*Dz`qhc+s_mDT3WV$|
zkUFXI^Jd((=w+W=db7{|5L2FT{+P@YW69oC_t$I-JUBmf^IjDeL+-D&uL|9M*2JB*
z+qO3!)Ro9}nr5Q)HfN9J_Eo~G=Bli;ntUs(`$*NU#3|t=aE+NlJF%cXpJ8ZhpFK`KD;G3BoJqRfl;d+8!>s
zV0`&~r^NMJan_6>euq@%F#9v#^}wTw|&37
z<^1$nrkmy-+cnoflQClP?%jr#!F7|3R+g^MX4)mxQDRr(zW)|?pq0idktKio?iM!}
zPTlcMb7NF*^w9#_cbOZ$_H276@~zZPCG9rn(OaLhrxoU&U%fBrr%B$`%FxMHSITY$
zxg9H2l{)s(EK|NMw@Uwo3=`lB%7Z8X1i{|
zzQ;>`x0YwFDmzx{JN4p!i4&_{d59j~lCHDd@XX$Ok#?ucylaiBX1C?tG7g>0kUsNW
z&*Qt2+k?N&f2(rxt@hvZR^L8XEl*5|F<g3(OuiQ<$Hh2GTH*fW)rz>l3nH_!3CJh<{IqU%)jZcrc9<9|68V3=4>;)Q-01w+2
zszS!gg$&m{0*{^atp1S#8ZY2g&XiF62U_Unax4AVRM4;nj
zTzl1EQrkrE5=iI}3`i~9M;)M%4^_|v3uyV`QFYMx2+Um2tVBA=FdcBs_);q<~f(W=entR>12F!P7X9;ScSJ-)4fWhK%-r0~c)w
z2_+z=8|<^;YbQ8bH4$VacytN01`{4j&=D?hV8J|f{p7K!8ArEXJN8xgeTDtSQ!f|t
zS6hJBcA@gNt?REQku6-|*{{FKzlefeQtmgcKi^{S4t}eTto}UB?
zA%@ilS8tOlS`>QqsJGps%IQDUtuFkqPhPUx{a1E!M7_nO)6;DJfyR*-85kN&pTBHQ
z4z15kJ$P~AyxmH&;h@Espmo)=itc%AH%T*v%uM{=BM*`Q?|v+p$
z;940Xg0H2vC4dqrD+9xUSsU}<#xPuKDV?i00~D2v3=A6{t@#Z%vOzR8816ELH4?MU
zV8$yHvW7MI&Vy;2u!G6F2kz)j?G00AAWL-;DOCd3#qhXm=@EoGNM;|#s+Z5EkoA`V
z%nuC5M7uTR=cgH-&Hi7#q?+w`8sES9&B>cN|KIx<6|tRfTDj2bSi^OO3@)pevCCM6
z9Q*3ytINP}qxZ#?D~f?(gZ8nn(f0pDFP?h2<+x3ma?k4fv%+lb
zB*PgP!j4GX+xq19^E#a({<}ib;rmV3R@4}+1Et6wZO(=`BBJ;sJUcvq>m9*Tn
zwZ2~VrRSr`;LQWarb=8-ee!B;?Vi#N6(=`940lNK>3Kb`_*24j)x%JAorZ?%ejl26
z@SIoMQFU+#g6#j^J9i3CH^fSnV@GzpleX5B6oc63(bHLYH@uHWOdBfFP;1;K*Y12|
zhgdgU(_<||wPU6CBbTH^)CfK10%cW@P3(sz9;|`cePD-IPiLX6!85qg$3j64Y&7(T
zIT#We{8N;hU!v(MST+^rK(Jk356lcR(y{9VrMU?b!jE@}$Vh&f4pRd1gC1`!%nSyQ
zTfCj)Vb*}dC8#iE!(NzkH*j@+1g*9H5N-eO>Icvm^_kRH?{#MHPhUCh|3Ur#7XN;4
zvwlBwM{iV`T&iv0zyFKl|5vW;OaG>Myt^ei&hW2JtfHGbXO_gVZum%D$a_PotG
z*XHiuEO&ek$ca6#%|J6Jz285A_Qw{_yf)`*`Rrv;|3Isnf4m2+3>V%Wc>2uZHR`AS
zgBFYbE}GMKuju!OPyKP5-)@br`}X_c?0Z!=w_V@lZyDF~`qHIoJjUyaS3N4ewezm{
z^Rt#Frt>!EBw5y{+xA&L=3+QtCFR_I_$KS)Uk%RfZSlX3eHS-c2O7G5tWrEl%y3=h
z`rkim)J{fz|C&?3zc;(@{Ll3}3vOq(&;M}b<3Gc7+Q$Wtm8Ksvt$IIoe#D6#c3%?t
z=U2^L@rviz)IwGUP?zz!(K^GV+NXOa_her`p=%^$+`nw*fxl+!YVVtODnG8WpJ!)b
zwyH9qtt4(;T8y~)x}APDKNR_ImT4c*d3>vPq9-WHgA&QH(q)Bfmuz02QgO2vy!iUz
z?R#GrK3pAN|FObELv_#FP4fG{od$1kJ}jNLFSp{Lw7%rN*Yp3S-mWc~|MOJu<4O0!
z_q<#5FjfA~hi%m%pZ|NUm|u8qbAP`6|9PJ}f{&m2zkdHq_u%h8XT9$0+isXCAAV;C
zbSU&ef_uj1x4YeU6>a!ixwdxBnH_q0F~zkf_JdcY-xWW<1C)y!3hx#_o~2^4bbqbt
z;nO)W^Xu<=S#+ZY9fE&l6eZ`vJ6Nw%=RqwfV{NGbsk=Ek7Olc>m{{
z$@9yueQ4yj%3L$!ZjIUL(54v^m;JGS|MY&2`KkZOZNI-+-afI7Jv#Ng=a0?*-guw+
z!XWuF8V{zs2|wYw$yR&!-x%>)d^$_TPLzIdD8M^t|3Db^VY1;V9!KrUUn1>Z_0~h(aGiQu
zO!2v8#Tv2(oy#MSZCxew>q?lp!MYx8P=)*0>U+<@sWr=%d~-d(4+L5jHV&><;(6^7HmIp7$wX56dFK;Q&sLX6r(Y?x;^}{X*+A#>c2+oYLDqJc6|GM(t}iyt+8sUyDm?9GDR*uP}tbCwlMEm42GtN5{0sdt;Y}6rZ)Jg+^{FN9<>wQ*^d)4ZFW$*N
zduu(Y_7SeTw(7`pBkAjTCpT2bfGUp-w~kdk?aAJrQX$)$UHk9t``t;8I$ler+x=|m
z$xaiJ)2X!lz^9&WRlY=@r*GnOyq53}$8%L9i6L(l7f
zQrEYo>_~eZtNwV_#YH_@>BsKBdv07crF4Gzztzj7t{<1J`xEkbR%bff+~2vk1K&>H
z^Qmd$&e(^?AEqo?%0d0@J-pbQ+&^#UV}{=BchP%mmKa}CwF^JEBkbgd1?h)WVg8sWecdea
zZti0%|4F;9A2a1Fmpn0rb-G}$=i6!Dr_Gc#S-Su0irE_f-`{@Tv+LT4(p=@#6V2z8
ztFIl{(RH5JbY1RA14t9I^W!{?laIEz7%KNvp3B_r!xR5+-SdU#f`o4B$RFN)q9gm>
ze3QzcYg519PG39CC^`T4&Ug3kzW)w765>z9T#1i&mQO$auw;5S5cK(FJY?
z94Jzm2d?EITn2Eh3!W%vWMBZb&b$1TKn)mB9S+ljO%Z5z9<(F<$PATZ(8f4;tr>=e
z44_IIYN$sKXm%gr6)&)-Kp`kP-vw0KM9e$#1JPId;U)oTcG-Ccb2A*sQki%9kHwtd
zT9#^2iF5ZYyWao*`F7QhoD2*S*AG4jRgU%Ba%fjY?|OMq_YrjT323J&17u4i1ITm+
zP;my@+z8rz3E2b*;t|Aa6y(~OL8_o;ftZAF@#p`Ht$qdRXIdBLgVcJu`njxgN@xNA
DUVNTH

literal 0
HcmV?d00001


From d931a6e727ee70952136f42250132e28a8aa34b4 Mon Sep 17 00:00:00 2001
From: Roland 
Date: Thu, 10 May 2012 10:24:05 +0200
Subject: [PATCH 14/36] break out TestConductor stuff into akka-remote-tests
 project

---
 .../testconductor/TestConductorProtocol.java  |  0
 .../main/protocol/TestConductorProtocol.proto |  0
 .../src/main/resources/reference.conf         | 34 +++++++++++++++++++
 .../akka/remote/testconductor/Conductor.scala | 12 +++----
 .../akka/remote/testconductor/DataTypes.scala |  0
 .../akka/remote/testconductor/Extension.scala |  0
 .../akka/remote/testconductor/Features.scala  |  0
 .../NetworkFailureInjector.scala              |  0
 .../akka/remote/testconductor/Player.scala    |  0
 .../testconductor/RemoteConnection.scala      |  0
 .../TestConductorTransport.scala              |  0
 .../akka/remote/testconductor/package.scala   |  0
 .../testconductor/TestConductorSpec.scala     |  0
 akka-remote/src/main/resources/reference.conf | 26 --------------
 project/AkkaBuild.scala                       | 20 ++++++++++-
 15 files changed, 59 insertions(+), 33 deletions(-)
 rename {akka-remote => akka-remote-tests}/src/main/java/akka/remote/testconductor/TestConductorProtocol.java (100%)
 rename {akka-remote => akka-remote-tests}/src/main/protocol/TestConductorProtocol.proto (100%)
 create mode 100644 akka-remote-tests/src/main/resources/reference.conf
 rename {akka-remote => akka-remote-tests}/src/main/scala/akka/remote/testconductor/Conductor.scala (97%)
 rename {akka-remote => akka-remote-tests}/src/main/scala/akka/remote/testconductor/DataTypes.scala (100%)
 rename {akka-remote => akka-remote-tests}/src/main/scala/akka/remote/testconductor/Extension.scala (100%)
 rename {akka-remote => akka-remote-tests}/src/main/scala/akka/remote/testconductor/Features.scala (100%)
 rename {akka-remote => akka-remote-tests}/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala (100%)
 rename {akka-remote => akka-remote-tests}/src/main/scala/akka/remote/testconductor/Player.scala (100%)
 rename {akka-remote => akka-remote-tests}/src/main/scala/akka/remote/testconductor/RemoteConnection.scala (100%)
 rename {akka-remote => akka-remote-tests}/src/main/scala/akka/remote/testconductor/TestConductorTransport.scala (100%)
 rename {akka-remote => akka-remote-tests}/src/main/scala/akka/remote/testconductor/package.scala (100%)
 rename {akka-remote => akka-remote-tests}/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala (100%)

diff --git a/akka-remote/src/main/java/akka/remote/testconductor/TestConductorProtocol.java b/akka-remote-tests/src/main/java/akka/remote/testconductor/TestConductorProtocol.java
similarity index 100%
rename from akka-remote/src/main/java/akka/remote/testconductor/TestConductorProtocol.java
rename to akka-remote-tests/src/main/java/akka/remote/testconductor/TestConductorProtocol.java
diff --git a/akka-remote/src/main/protocol/TestConductorProtocol.proto b/akka-remote-tests/src/main/protocol/TestConductorProtocol.proto
similarity index 100%
rename from akka-remote/src/main/protocol/TestConductorProtocol.proto
rename to akka-remote-tests/src/main/protocol/TestConductorProtocol.proto
diff --git a/akka-remote-tests/src/main/resources/reference.conf b/akka-remote-tests/src/main/resources/reference.conf
new file mode 100644
index 0000000000..f0d8a9d6ae
--- /dev/null
+++ b/akka-remote-tests/src/main/resources/reference.conf
@@ -0,0 +1,34 @@
+#############################################
+# Akka Remote Testing Reference Config File #
+#############################################
+
+# This is the reference config file that contains all the default settings.
+# Make your edits/overrides in your application.conf.
+
+akka {
+  testconductor {
+
+    # Timeout for joining a barrier: this is the maximum time any participants
+    # waits for everybody else to join a named barrier.
+    barrier-timeout = 30s
+    
+    # Timeout for interrogation of TestConductor’s Controller actor
+    query-timeout = 5s
+    
+    # Threshold for packet size in time unit above which the failure injector will
+    # split the packet and deliver in smaller portions; do not give value smaller
+    # than HashedWheelTimer resolution (would not make sense)
+    packet-split-threshold = 100ms
+    
+    # Default port to start the conductor on; 0 means 
+    port = 0
+    
+    # Hostname of the TestConductor server, used by the server to bind to the IP
+    # and by the client to connect to it.
+    host = localhost
+    
+    # Name of the TestConductor client (for identification on the server e.g. for
+    # failure injection)
+    name = "noname"
+  }
+}
\ No newline at end of file
diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala
similarity index 97%
rename from akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala
rename to akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala
index 2bbae6d28b..b25bd1838c 100644
--- a/akka-remote/src/main/scala/akka/remote/testconductor/Conductor.scala
+++ b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala
@@ -24,6 +24,7 @@ import java.net.InetSocketAddress
 import akka.dispatch.Future
 import akka.actor.OneForOneStrategy
 import akka.actor.SupervisorStrategy
+import java.util.concurrent.ConcurrentHashMap
 
 trait Conductor extends RunControl with FailureInject { this: TestConductorExt ⇒
 
@@ -91,22 +92,21 @@ trait Conductor extends RunControl with FailureInject { this: TestConductorExt 
 
 class ConductorHandler(system: ActorSystem, controller: ActorRef, log: LoggingAdapter) extends SimpleChannelUpstreamHandler {
 
-  @volatile
-  var clients = Map[Channel, ActorRef]()
+  val clients = new ConcurrentHashMap[Channel, ActorRef]()
 
   override def channelConnected(ctx: ChannelHandlerContext, event: ChannelStateEvent) = {
     val channel = event.getChannel
     log.debug("connection from {}", getAddrString(channel))
     val fsm = system.actorOf(Props(new ServerFSM(controller, channel)))
-    clients += channel -> fsm
+    clients.put(channel, fsm)
   }
 
   override def channelDisconnected(ctx: ChannelHandlerContext, event: ChannelStateEvent) = {
     val channel = event.getChannel
     log.debug("disconnect from {}", getAddrString(channel))
-    val fsm = clients(channel)
+    val fsm = clients.get(channel)
     fsm ! PoisonPill
-    clients -= channel
+    clients.remove(channel)
   }
 
   override def messageReceived(ctx: ChannelHandlerContext, event: MessageEvent) = {
@@ -114,7 +114,7 @@ class ConductorHandler(system: ActorSystem, controller: ActorRef, log: LoggingAd
     log.debug("message from {}: {}", getAddrString(channel), event.getMessage)
     event.getMessage match {
       case msg: NetworkOp ⇒
-        clients(channel) ! msg
+        clients.get(channel) ! msg
       case msg ⇒
         log.info("client {} sent garbage '{}', disconnecting", getAddrString(channel), msg)
         channel.close()
diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/DataTypes.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/DataTypes.scala
similarity index 100%
rename from akka-remote/src/main/scala/akka/remote/testconductor/DataTypes.scala
rename to akka-remote-tests/src/main/scala/akka/remote/testconductor/DataTypes.scala
diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/Extension.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Extension.scala
similarity index 100%
rename from akka-remote/src/main/scala/akka/remote/testconductor/Extension.scala
rename to akka-remote-tests/src/main/scala/akka/remote/testconductor/Extension.scala
diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/Features.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Features.scala
similarity index 100%
rename from akka-remote/src/main/scala/akka/remote/testconductor/Features.scala
rename to akka-remote-tests/src/main/scala/akka/remote/testconductor/Features.scala
diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala
similarity index 100%
rename from akka-remote/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala
rename to akka-remote-tests/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala
diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/Player.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Player.scala
similarity index 100%
rename from akka-remote/src/main/scala/akka/remote/testconductor/Player.scala
rename to akka-remote-tests/src/main/scala/akka/remote/testconductor/Player.scala
diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/RemoteConnection.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/RemoteConnection.scala
similarity index 100%
rename from akka-remote/src/main/scala/akka/remote/testconductor/RemoteConnection.scala
rename to akka-remote-tests/src/main/scala/akka/remote/testconductor/RemoteConnection.scala
diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/TestConductorTransport.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/TestConductorTransport.scala
similarity index 100%
rename from akka-remote/src/main/scala/akka/remote/testconductor/TestConductorTransport.scala
rename to akka-remote-tests/src/main/scala/akka/remote/testconductor/TestConductorTransport.scala
diff --git a/akka-remote/src/main/scala/akka/remote/testconductor/package.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/package.scala
similarity index 100%
rename from akka-remote/src/main/scala/akka/remote/testconductor/package.scala
rename to akka-remote-tests/src/main/scala/akka/remote/testconductor/package.scala
diff --git a/akka-remote/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala b/akka-remote-tests/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
similarity index 100%
rename from akka-remote/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
rename to akka-remote-tests/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
diff --git a/akka-remote/src/main/resources/reference.conf b/akka-remote/src/main/resources/reference.conf
index f14ee3d87c..1438904fe2 100644
--- a/akka-remote/src/main/resources/reference.conf
+++ b/akka-remote/src/main/resources/reference.conf
@@ -155,30 +155,4 @@ akka {
       type = PinnedDispatcher
     }
   }
-  
-  testconductor {
-
-    # Timeout for joining a barrier: this is the maximum time any participants
-    # waits for everybody else to join a named barrier.
-    barrier-timeout = 30s
-    
-    # Timeout for interrogation of TestConductor’s Controller actor
-    query-timeout = 5s
-    
-    # Threshold for packet size in time unit above which the failure injector will
-    # split the packet and deliver in smaller portions; do not give value smaller
-    # than HashedWheelTimer resolution (would not make sense)
-    packet-split-threshold = 100ms
-    
-    # Default port to start the conductor on; 0 means 
-    port = 0
-    
-    # Hostname of the TestConductor server, used by the server to bind to the IP
-    # and by the client to connect to it.
-    host = localhost
-    
-    # Name of the TestConductor client (for identification on the server e.g. for
-    # failure injection)
-    name = "noname"
-  }
 }
diff --git a/project/AkkaBuild.scala b/project/AkkaBuild.scala
index f9fbfc6c4b..b899bdec45 100644
--- a/project/AkkaBuild.scala
+++ b/project/AkkaBuild.scala
@@ -87,6 +87,24 @@ object AkkaBuild extends Build {
     )
   ) configs (MultiJvm)
 
+  lazy val remoteTests = Project(
+    id = "akka-remote-tests",
+    base = file("akka-remote-tests"),
+    dependencies = Seq(remote % "compile;test->test;multi-jvm->multi-jvm", actorTests % "test->test", testkit % "test->test"),
+    settings = defaultSettings ++ multiJvmSettings ++ schoirSettings ++ Seq(
+      // disable parallel tests
+      parallelExecution in Test := false,
+      extraOptions in MultiJvm <<= (sourceDirectory in MultiJvm) { src =>
+        (name: String) => (src ** (name + ".conf")).get.headOption.map("-Dakka.config=" + _.absolutePath).toSeq
+      },
+      scalatestOptions in MultiJvm := Seq("-r", "org.scalatest.akka.QuietReporter"),
+      jvmOptions in MultiJvm := {
+        if (getBoolean("sbt.log.noformat")) Seq("-Dakka.test.nocolor=true") else Nil
+      },
+      test in Test <<= (test in Test) dependsOn (test in MultiJvm)
+    )
+  ) configs (MultiJvm)
+
   lazy val cluster = Project(
     id = "akka-cluster",
     base = file("akka-cluster"),
@@ -438,7 +456,7 @@ object Dependencies {
     Test.zookeeper, Test.log4j // needed for ZkBarrier in multi-jvm tests
   )
 
- val cluster = Seq(Test.junit, Test.scalatest)
+  val cluster = Seq(Test.junit, Test.scalatest)
 
   val slf4j = Seq(slf4jApi, Test.logback)
 

From 160aa730667a82e4acd1eb6af5c54eb85ee6adc1 Mon Sep 17 00:00:00 2001
From: Roland 
Date: Thu, 10 May 2012 21:08:06 +0200
Subject: [PATCH 15/36] scaladoc for TestConductor

---
 .../akka/remote/testconductor/Conductor.scala | 287 ++++++++++++++----
 .../akka/remote/testconductor/Extension.scala |  16 +
 .../akka/remote/testconductor/Features.scala  |  89 ------
 .../akka/remote/testconductor/Player.scala    |  34 ++-
 4 files changed, 283 insertions(+), 143 deletions(-)
 delete mode 100644 akka-remote-tests/src/main/scala/akka/remote/testconductor/Features.scala

diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala
index b25bd1838c..347973a255 100644
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala
+++ b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala
@@ -25,8 +25,26 @@ import akka.dispatch.Future
 import akka.actor.OneForOneStrategy
 import akka.actor.SupervisorStrategy
 import java.util.concurrent.ConcurrentHashMap
+import akka.actor.Status
 
-trait Conductor extends RunControl with FailureInject { this: TestConductorExt ⇒
+sealed trait Direction
+
+object Direction {
+  case object Send extends Direction
+  case object Receive extends Direction
+  case object Both extends Direction
+}
+
+/**
+ * The conductor is the one orchestrating the test: it governs the
+ * [[akka.remote.testconductor.Controller]]’s port to which all
+ * [[akka.remote.testconductor.Player]]s connect, it issues commands to their
+ * [[akka.remote.testconductor.NetworkFailureInjector]] and provides support
+ * for barriers using the [[akka.remote.testconductor.BarrierCoordinator]].
+ * All of this is bundled inside the [[akka.remote.testconductor.TestConductorExt]]
+ * extension.
+ */
+trait Conductor { this: TestConductorExt ⇒
 
   import Controller._
 
@@ -36,60 +54,154 @@ trait Conductor extends RunControl with FailureInject { this: TestConductorExt 
     case x    ⇒ x
   }
 
-  override def startController(participants: Int): Future[Int] = {
+  /**
+   * Start the [[akka.remote.testconductor.Controller]], which in turn will
+   * bind to a TCP port as specified in the `akka.testconductor.port` config
+   * property, where 0 denotes automatic allocation. Since the latter is
+   * actually preferred, a `Future[Int]` is returned which will be completed
+   * with the port number actually chosen, so that this can then be communicated
+   * to the players for their proper start-up.
+   *
+   * This method also invokes [[akka.remote.testconductor.Player]].startClient,
+   * since it is expected that the conductor participates in barriers for
+   * overall coordination. The returned Future will only be completed once the
+   * client’s start-up finishes, which in fact waits for all other players to
+   * connect.
+   *
+   * @param participants gives the number of participants which shall connect
+   * before any of their startClient() operations complete.
+   */
+  def startController(participants: Int): Future[Int] = {
     if (_controller ne null) throw new RuntimeException("TestConductorServer was already started")
     _controller = system.actorOf(Props(new Controller(participants)), "controller")
     import Settings.BarrierTimeout
     controller ? GetPort flatMap { case port: Int ⇒ startClient(port) map (_ ⇒ port) }
   }
 
-  override def port: Future[Int] = {
+  /**
+   * Obtain the port to which the controller’s socket is actually bound. This
+   * will deviate from the configuration in `akka.testconductor.port` in case
+   * that was given as zero.
+   */
+  def port: Future[Int] = {
     import Settings.QueryTimeout
     controller ? GetPort mapTo
   }
 
-  override def throttle(node: String, target: String, direction: Direction, rateMBit: Double): Future[Done] = {
+  /**
+   * Make the remoting pipeline on the node throttle data sent to or received
+   * from the given remote peer. Throttling works by delaying packet submission
+   * within the netty pipeline until the packet would have been completely sent
+   * according to the given rate, the previous packet completion and the current
+   * packet length. In case of large packets they are split up if the calculated
+   * send pause would exceed `akka.testconductor.packet-split-threshold`
+   * (roughly). All of this uses the system’s HashedWheelTimer, which is not
+   * terribly precise and will execute tasks later than they are schedule (even
+   * on average), but that is countered by using the actual execution time for
+   * determining how much to send, leading to the correct output rate, but with
+   * increased latency.
+   * 
+   * @param node is the symbolic name of the node which is to be affected
+   * @param target is the symbolic name of the other node to which connectivity shall be throttled
+   * @param direction can be either `Direction.Send`, `Direction.Receive` or `Direction.Both`
+   * @param rateMBit is the maximum data rate in MBit
+   */
+  def throttle(node: String, target: String, direction: Direction, rateMBit: Double): Future[Done] = {
     import Settings.QueryTimeout
     controller ? Throttle(node, target, direction, rateMBit.toFloat) mapTo
   }
 
-  override def blackhole(node: String, target: String, direction: Direction): Future[Done] = {
+  /**
+   * Switch the Netty pipeline of the remote support into blackhole mode for
+   * sending and/or receiving: it will just drop all messages right before
+   * submitting them to the Socket or right after receiving them from the
+   * Socket.
+   * 
+   * @param node is the symbolic name of the node which is to be affected
+   * @param target is the symbolic name of the other node to which connectivity shall be impeded
+   * @param direction can be either `Direction.Send`, `Direction.Receive` or `Direction.Both`
+   */
+  def blackhole(node: String, target: String, direction: Direction): Future[Done] = {
     import Settings.QueryTimeout
     controller ? Throttle(node, target, direction, 0f) mapTo
   }
 
-  override def disconnect(node: String, target: String): Future[Done] = {
+  /**
+   * Tell the remote support to shutdown the connection to the given remote
+   * peer. It works regardless of whether the recipient was initiator or
+   * responder.
+   * 
+   * @param node is the symbolic name of the node which is to be affected
+   * @param target is the symbolic name of the other node to which connectivity shall be impeded
+   */
+  def disconnect(node: String, target: String): Future[Done] = {
     import Settings.QueryTimeout
     controller ? Disconnect(node, target, false) mapTo
   }
 
-  override def abort(node: String, target: String): Future[Done] = {
+  /**
+   * Tell the remote support to TCP_RESET the connection to the given remote
+   * peer. It works regardless of whether the recipient was initiator or
+   * responder.
+   * 
+   * @param node is the symbolic name of the node which is to be affected
+   * @param target is the symbolic name of the other node to which connectivity shall be impeded
+   */
+  def abort(node: String, target: String): Future[Done] = {
     import Settings.QueryTimeout
     controller ? Disconnect(node, target, true) mapTo
   }
 
-  override def shutdown(node: String, exitValue: Int): Future[Done] = {
+  /**
+   * Tell the remote node to shut itself down using System.exit with the given
+   * exitValue.
+   * 
+   * @param node is the symbolic name of the node which is to be affected
+   * @param exitValue is the return code which shall be given to System.exit
+   */
+  def shutdown(node: String, exitValue: Int): Future[Done] = {
     import Settings.QueryTimeout
     controller ? Terminate(node, exitValue) mapTo
   }
 
-  override def kill(node: String): Future[Done] = {
+  /**
+   * Tell the SBT plugin to forcibly terminate the given remote node using Process.destroy.
+   * 
+   * @param node is the symbolic name of the node which is to be affected
+   */
+  def kill(node: String): Future[Done] = {
     import Settings.QueryTimeout
     controller ? Terminate(node, -1) mapTo
   }
 
-  override def getNodes: Future[List[String]] = {
+  /**
+   * Obtain the list of remote host names currently registered.
+   */
+  def getNodes: Future[List[String]] = {
     import Settings.QueryTimeout
     controller ? GetNodes mapTo
   }
 
-  override def removeNode(node: String): Future[Done] = {
+  /**
+   * Remove a remote host from the list, so that the remaining nodes may still
+   * pass subsequent barriers. This must be done before the client connection
+   * breaks down in order to affect an “orderly” removal (i.e. without failing
+   * present and future barriers).
+   * 
+   * @param node is the symbolic name of the node which is to be removed
+   */
+  def removeNode(node: String): Future[Done] = {
     import Settings.QueryTimeout
     controller ? Remove(node) mapTo
   }
 
 }
 
+/**
+ * This handler is installed at the end of the controller’s netty pipeline. Its only
+ * purpose is to dispatch incoming messages to the right ServerFSM actor. There is
+ * one shared instance of this class for all connections accepted by one Controller.
+ */
 class ConductorHandler(system: ActorSystem, controller: ActorRef, log: LoggingAdapter) extends SimpleChannelUpstreamHandler {
 
   val clients = new ConcurrentHashMap[Channel, ActorRef]()
@@ -105,7 +217,7 @@ class ConductorHandler(system: ActorSystem, controller: ActorRef, log: LoggingAd
     val channel = event.getChannel
     log.debug("disconnect from {}", getAddrString(channel))
     val fsm = clients.get(channel)
-    fsm ! PoisonPill
+    fsm ! Controller.ClientDisconnected
     clients.remove(channel)
   }
 
@@ -129,6 +241,19 @@ object ServerFSM {
   case object Ready extends State
 }
 
+/**
+ * The server part of each client connection is represented by a ServerFSM.
+ * The Initial state handles reception of the new client’s
+ * [[akka.remote.testconductor.Hello]] message (which is needed for all subsequent
+ * node name translations).
+ *
+ * In the Ready state, messages from the client are forwarded to the controller
+ * and [[akka.remote.testconductor.Send]] requests are sent, but the latter is
+ * treated specially: all client operations are to be confirmed by a
+ * [[akka.remote.testconductor.Done]] message, and there can be only one such
+ * request outstanding at a given time (i.e. a Send fails if the previous has
+ * not yet been acknowledged).
+ */
 class ServerFSM(val controller: ActorRef, val channel: Channel) extends Actor with LoggingFSM[ServerFSM.State, Option[ActorRef]] {
   import ServerFSM._
   import akka.actor.FSM._
@@ -136,9 +261,20 @@ class ServerFSM(val controller: ActorRef, val channel: Channel) extends Actor wi
 
   startWith(Initial, None)
 
+  whenUnhandled {
+    case Event(ClientDisconnected, Some(s)) ⇒
+      s ! Status.Failure(new RuntimeException("client disconnected in state " + stateName + ": " + channel))
+      stop()
+    case Event(ClientDisconnected, None) ⇒ stop()
+  }
+
+  onTermination {
+    case _ ⇒ controller ! ClientDisconnected
+  }
+
   when(Initial, stateTimeout = 10 seconds) {
     case Event(Hello(name, addr), _) ⇒
-      controller ! ClientConnected(name, addr)
+      controller ! NodeInfo(name, addr, self)
       goto(Ready)
     case Event(x: NetworkOp, _) ⇒
       log.warning("client {} sent no Hello in first message (instead {}), disconnecting", getAddrString(channel), x)
@@ -162,7 +298,6 @@ class ServerFSM(val controller: ActorRef, val channel: Channel) extends Actor wi
       stay using None
     case Event(msg: NetworkOp, _) ⇒
       log.warning("client {} sent unsupported message {}", getAddrString(channel), msg)
-      channel.close()
       stop()
     case Event(Send(msg @ (_: EnterBarrier | _: Done)), _) ⇒
       channel.write(msg)
@@ -176,10 +311,13 @@ class ServerFSM(val controller: ActorRef, val channel: Channel) extends Actor wi
   }
 
   initialize
+
+  onTermination {
+    case _ ⇒ channel.close()
+  }
 }
 
 object Controller {
-  case class ClientConnected(name: String, address: Address)
   case class ClientDisconnected(name: String)
   case object GetNodes
   case object GetPort
@@ -187,6 +325,11 @@ object Controller {
   case class NodeInfo(name: String, addr: Address, fsm: ActorRef)
 }
 
+/**
+ * This controls test execution by managing barriers (delegated to
+ * [[akka.remote.testconductor.BarrierCoordinator]], its child) and allowing
+ * network and other failures to be injected at the test nodes.
+ */
 class Controller(_participants: Int) extends Actor {
   import Controller._
 
@@ -199,8 +342,8 @@ class Controller(_participants: Int) extends Actor {
   override def supervisorStrategy = OneForOneStrategy() {
     case e: BarrierCoordinator.BarrierTimeoutException ⇒ SupervisorStrategy.Resume
     case e: BarrierCoordinator.WrongBarrierException ⇒
-      // I think we are lacking a means of communication here: this is not correct!
-      for (i ← 1 to e.data.clients) barrier ! ClientConnected
+      for (NodeInfo(c, _, _) ← e.data.clients; info ← nodes get c)
+        barrier ! NodeInfo(c, info.addr, info.fsm)
       for (c ← e.data.arrived) c ! BarrierFailed(e.barrier)
       SupervisorStrategy.Restart
   }
@@ -209,17 +352,17 @@ class Controller(_participants: Int) extends Actor {
   var nodes = Map[String, NodeInfo]()
 
   override def receive = LoggingReceive {
-    case ClientConnected(name, addr) ⇒
-      nodes += name -> NodeInfo(name, addr, sender)
-      barrier forward ClientConnected
+    case c @ NodeInfo(name, addr, fsm) ⇒
+      nodes += name -> c
+      barrier forward c
       if (initialParticipants <= 0) sender ! Done
       else if (nodes.size == initialParticipants) {
         for (NodeInfo(_, _, client) ← nodes.values) client ! Send(Done)
         initialParticipants = 0
       }
-    case ClientDisconnected(name) ⇒
+    case c @ ClientDisconnected(name) ⇒
       nodes -= name
-      barrier forward ClientDisconnected
+      barrier forward c
     case e @ EnterBarrier(name) ⇒
       barrier forward e
     case Throttle(node, target, direction, rateMBit) ⇒
@@ -234,9 +377,9 @@ class Controller(_participants: Int) extends Actor {
       } else {
         nodes(node).fsm forward Send(TerminateMsg(exitValueOrKill))
       }
-    // TODO: properly remove node from BarrierCoordinator
-    //    case Remove(node) =>
-    //      nodes -= node
+    case Remove(node) ⇒
+      nodes -= node
+      barrier ! BarrierCoordinator.RemoveClient(node)
     case GetNodes ⇒ sender ! nodes.keys
     case GetPort ⇒
       sender ! (connection.getLocalAddress match {
@@ -250,27 +393,60 @@ object BarrierCoordinator {
   case object Idle extends State
   case object Waiting extends State
 
-  case class Data(clients: Int, barrier: String, arrived: List[ActorRef])
+  case class RemoveClient(name: String)
+
+  case class Data(clients: Set[Controller.NodeInfo], barrier: String, arrived: List[ActorRef])
   class BarrierTimeoutException(val data: Data) extends RuntimeException(data.barrier) with NoStackTrace
   class WrongBarrierException(val barrier: String, val client: ActorRef, val data: Data) extends RuntimeException(barrier) with NoStackTrace
 }
 
+/**
+ * This barrier coordinator gets informed of players connecting (NodeInfo),
+ * players being deliberately removed (RemoveClient) or failing (ClientDisconnected)
+ * by the controller. It also receives EnterBarrier requests, where upon the first
+ * one received the name of the current barrier is set and all other known clients
+ * are expected to join the barrier, whereupon all of the will be sent the successful
+ * EnterBarrier return message. In case of planned removals, this may just happen
+ * earlier, in case of failures the current barrier (and all subsequent ones) will
+ * be failed by sending BarrierFailed responses.
+ */
 class BarrierCoordinator extends Actor with LoggingFSM[BarrierCoordinator.State, BarrierCoordinator.Data] {
   import BarrierCoordinator._
   import akka.actor.FSM._
   import Controller._
 
-  startWith(Idle, Data(0, "", Nil))
+  // this shall be set to false if all subsequent barriers shall fail
+  var failed = false
+  override def preRestart(reason: Throwable, message: Option[Any]) {}
+  override def postRestart(reason: Throwable) { failed = true }
+
+  // TODO what happens with the other waiting players in case of a test failure?
+
+  startWith(Idle, Data(Set(), "", Nil))
+
+  whenUnhandled {
+    case Event(n: NodeInfo, d @ Data(clients, _, _)) ⇒
+      stay using d.copy(clients = clients + n)
+  }
 
   when(Idle) {
-    case Event(EnterBarrier(name), Data(num, _, _)) ⇒
-      if (num == 0) throw new IllegalStateException("no client expected yet")
-      goto(Waiting) using Data(num, name, sender :: Nil)
-    case Event(ClientConnected, d @ Data(num, _, _)) ⇒
-      stay using d.copy(clients = num + 1)
-    case Event(ClientDisconnected, d @ Data(num, _, _)) ⇒
-      if (num == 0) throw new IllegalStateException("no client to disconnect")
-      stay using d.copy(clients = num - 1)
+    case Event(EnterBarrier(name), d @ Data(clients, _, _)) ⇒
+      if (clients.isEmpty) throw new IllegalStateException("no client expected yet")
+      if (failed)
+        stay replying BarrierFailed(name)
+      else
+        goto(Waiting) using d.copy(barrier = name, arrived = sender :: Nil)
+    case Event(ClientDisconnected(name), d @ Data(clients, _, _)) ⇒
+      if (clients.isEmpty) throw new IllegalStateException("no client to disconnect")
+      (clients filterNot (_.name == name)) match {
+        case `clients` ⇒ stay
+        case c ⇒
+          failed = true
+          stay using d.copy(clients = c)
+      }
+    case Event(RemoveClient(name), d @ Data(clients, _, _)) ⇒
+      if (clients.isEmpty) throw new IllegalStateException("no client to remove")
+      stay using d.copy(clients = clients filterNot (_.name == name))
   }
 
   onTransition {
@@ -279,30 +455,37 @@ class BarrierCoordinator extends Actor with LoggingFSM[BarrierCoordinator.State,
   }
 
   when(Waiting) {
-    case Event(e @ EnterBarrier(name), d @ Data(num, barrier, arrived)) ⇒
+    case Event(e @ EnterBarrier(name), d @ Data(clients, barrier, arrived)) ⇒
       if (name != barrier) throw new WrongBarrierException(barrier, sender, d)
       val together = sender :: arrived
-      if (together.size == num) {
-        together foreach (_ ! Send(e))
-        goto(Idle) using Data(num, "", Nil)
-      } else {
-        stay using d.copy(arrived = together)
-      }
-    case Event(ClientConnected, d @ Data(num, _, _)) ⇒
-      stay using d.copy(clients = num + 1)
-    case Event(ClientDisconnected, d @ Data(num, barrier, arrived)) ⇒
-      val expected = num - 1
-      if (arrived.size == expected) {
-        val e = EnterBarrier(barrier)
-        sender :: arrived foreach (_ ! Send(e))
-        goto(Idle) using Data(expected, "", Nil)
-      } else {
-        stay using d.copy(clients = expected)
+      handleBarrier(d.copy(arrived = together))
+    case Event(RemoveClient(name), d @ Data(clients, barrier, arrived)) ⇒
+      val newClients = clients filterNot (_.name == name)
+      val newArrived = arrived filterNot (_ == name)
+      handleBarrier(d.copy(clients = newClients, arrived = newArrived))
+    case Event(ClientDisconnected(name), d @ Data(clients, barrier, arrived)) ⇒
+      (clients filterNot (_.name == name)) match {
+        case `clients` ⇒ stay
+        case c ⇒
+          val f = BarrierFailed(barrier)
+          arrived foreach (_ ! Send(f))
+          failed = true
+          goto(Idle) using Data(c, "", Nil)
       }
     case Event(StateTimeout, data) ⇒
       throw new BarrierTimeoutException(data)
   }
 
   initialize
+
+  def handleBarrier(data: Data): State =
+    if ((data.clients.map(_.fsm) -- data.arrived).isEmpty) {
+      val e = EnterBarrier(data.barrier)
+      data.arrived foreach (_ ! Send(e))
+      goto(Idle) using data.copy(barrier = "", arrived = Nil)
+    } else {
+      stay using data
+    }
+
 }
 
diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Extension.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Extension.scala
index 97f5dd7295..ff1d77fb9d 100644
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Extension.scala
+++ b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Extension.scala
@@ -11,12 +11,28 @@ import akka.actor.ActorRef
 import java.util.concurrent.ConcurrentHashMap
 import akka.actor.Address
 
+/**
+ * Access to the [[akka.remote.testconductor.TestConductorExt]] extension:
+ * 
+ * {{{
+ * val tc = TestConductor(system)
+ * tc.startController(numPlayers)
+ * // OR
+ * tc.startClient(conductorPort)
+ * }}}
+ */
 object TestConductor extends ExtensionKey[TestConductorExt] {
 
   def apply()(implicit ctx: ActorContext): TestConductorExt = apply(ctx.system)
 
 }
 
+/**
+ * This binds together the [[akka.remote.testconductor.Conductor]] and
+ * [[akka.remote.testconductor.Player]] roles inside an Akka
+ * [[akka.actor.Extension]]. Please follow the aforementioned links for
+ * more information.
+ */
 class TestConductorExt(val system: ExtendedActorSystem) extends Extension with Conductor with Player {
 
   object Settings {
diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Features.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Features.scala
deleted file mode 100644
index 336d04c368..0000000000
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Features.scala
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- *  Copyright (C) 2009-2011 Typesafe Inc. 
- */
-package akka.remote.testconductor
-
-import akka.dispatch.Future
-
-trait BarrierSync {
-  /**
-   * Enter all given barriers in the order in which they were given.
-   */
-  def enter(name: String*): Unit
-}
-
-sealed trait Direction
-
-object Direction {
-  case object Send extends Direction
-  case object Receive extends Direction
-  case object Both extends Direction
-}
-
-trait FailureInject {
-
-  /**
-   * Make the remoting pipeline on the node throttle data sent to or received
-   * from the given remote peer.
-   */
-  def throttle(node: String, target: String, direction: Direction, rateMBit: Double): Future[Done]
-
-  /**
-   * Switch the Netty pipeline of the remote support into blackhole mode for
-   * sending and/or receiving: it will just drop all messages right before
-   * submitting them to the Socket or right after receiving them from the
-   * Socket.
-   */
-  def blackhole(node: String, target: String, direction: Direction): Future[Done]
-
-  /**
-   * Tell the remote support to shutdown the connection to the given remote
-   * peer. It works regardless of whether the recipient was initiator or
-   * responder.
-   */
-  def disconnect(node: String, target: String): Future[Done]
-
-  /**
-   * Tell the remote support to TCP_RESET the connection to the given remote
-   * peer. It works regardless of whether the recipient was initiator or
-   * responder.
-   */
-  def abort(node: String, target: String): Future[Done]
-
-}
-
-trait RunControl {
-
-  /**
-   * Start the server port, returns the port number.
-   */
-  def startController(participants: Int): Future[Int]
-
-  /**
-   * Get the actual port used by the server.
-   */
-  def port: Future[Int]
-
-  /**
-   * Tell the remote node to shut itself down using System.exit with the given
-   * exitValue.
-   */
-  def shutdown(node: String, exitValue: Int): Future[Done]
-
-  /**
-   * Tell the SBT plugin to forcibly terminate the given remote node using Process.destroy.
-   */
-  def kill(node: String): Future[Done]
-
-  /**
-   * Obtain the list of remote host names currently registered.
-   */
-  def getNodes: Future[List[String]]
-
-  /**
-   * Remove a remote host from the list, so that the remaining nodes may still
-   * pass subsequent barriers.
-   */
-  def removeNode(node: String): Future[Done]
-
-}
diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Player.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Player.scala
index 6e78610cfb..38d0f6ef34 100644
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Player.scala
+++ b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Player.scala
@@ -21,7 +21,13 @@ import akka.actor.PoisonPill
 import akka.event.Logging
 import akka.dispatch.Future
 
-trait Player extends BarrierSync { this: TestConductorExt ⇒
+/**
+ * The Player is the client component of the
+ * [[akka.remote.testconductor.TestConductorExt]] extension. It registers with
+ * the [[akka.remote.testconductor.Conductor]]’s [[akka.remote.testconductor.Controller]]
+ * in order to participate in barriers and enable network failure injection.
+ */
+trait Player { this: TestConductorExt ⇒
 
   private var _client: ActorRef = _
   private def client = _client match {
@@ -29,6 +35,14 @@ trait Player extends BarrierSync { this: TestConductorExt ⇒
     case x    ⇒ x
   }
 
+  /**
+   * Connect to the conductor on the given port (the host is taken from setting
+   * `akka.testconductor.host`). The connection is made asynchronously, but you
+   * should await completion of the returned Future because that implies that
+   * all expected participants of this test have successfully connected (i.e.
+   * this is a first barrier in itself). The number of expected participants is
+   * set in [[akka.remote.testconductor.Conductor]]`.startController()`.
+   */
   def startClient(port: Int): Future[Done] = {
     import ClientFSM._
     import akka.actor.FSM._
@@ -51,7 +65,11 @@ trait Player extends BarrierSync { this: TestConductorExt ⇒
     a ? client mapTo
   }
 
-  override def enter(name: String*) {
+  /**
+   * Enter the named barriers, one after the other, in the order given. Will
+   * throw an exception in case of timeouts or other errors.
+   */
+  def enter(name: String*) {
     system.log.debug("entering barriers " + name.mkString("(", ", ", ")"))
     name foreach { b ⇒
       import Settings.BarrierTimeout
@@ -73,6 +91,15 @@ object ClientFSM {
   case object Disconnected
 }
 
+/**
+ * This is the controlling entity on the [[akka.remote.testconductor.Player]]
+ * side: in a first step it registers itself with a symbolic name and its remote
+ * address at the [[akka.remote.testconductor.Controller]], then waits for the
+ * `Done` message which signals that all other expected test participants have
+ * done the same. After that, it will pass barrier requests to and from the
+ * coordinator and react to the [[akka.remote.testconductor.Conductor]]’s
+ * requests for failure injection.
+ */
 class ClientFSM(port: Int) extends Actor with LoggingFSM[ClientFSM.State, ClientFSM.Data] {
   import ClientFSM._
 
@@ -162,6 +189,9 @@ class ClientFSM(port: Int) extends Actor with LoggingFSM[ClientFSM.State, Client
 
 }
 
+/**
+ * This handler only forwards messages received from the conductor to the [[akka.remote.testconductor.ClientFSM]].
+ */
 class PlayerHandler(fsm: ActorRef, log: LoggingAdapter) extends SimpleChannelUpstreamHandler {
 
   import ClientFSM._

From 439f653427d4ad26d504b2b35633b2de9d421d8b Mon Sep 17 00:00:00 2001
From: Roland 
Date: Fri, 11 May 2012 11:31:44 +0200
Subject: [PATCH 16/36] add some tests for BarrierCoordinator and Controller

---
 .../akka/remote/testconductor/Conductor.scala | 129 +++--
 .../akka/remote/testconductor/Extension.scala |   2 +-
 .../akka/remote/testconductor/Player.scala    |  27 +-
 .../remote/testconductor/BarrierSpec.scala    | 465 ++++++++++++++++++
 .../remote/testconductor/ControllerSpec.scala |  38 ++
 5 files changed, 599 insertions(+), 62 deletions(-)
 create mode 100644 akka-remote-tests/src/test/scala/akka/remote/testconductor/BarrierSpec.scala
 create mode 100644 akka-remote-tests/src/test/scala/akka/remote/testconductor/ControllerSpec.scala

diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala
index 347973a255..09a6faeeb0 100644
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala
+++ b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala
@@ -1,5 +1,5 @@
 /**
- *  Copyright (C) 2009-2011 Typesafe Inc. 
+ *  Copyright (C) 2009-2012 Typesafe Inc. 
  */
 package akka.remote.testconductor
 
@@ -100,7 +100,7 @@ trait Conductor { this: TestConductorExt ⇒
    * on average), but that is countered by using the actual execution time for
    * determining how much to send, leading to the correct output rate, but with
    * increased latency.
-   * 
+   *
    * @param node is the symbolic name of the node which is to be affected
    * @param target is the symbolic name of the other node to which connectivity shall be throttled
    * @param direction can be either `Direction.Send`, `Direction.Receive` or `Direction.Both`
@@ -116,7 +116,7 @@ trait Conductor { this: TestConductorExt ⇒
    * sending and/or receiving: it will just drop all messages right before
    * submitting them to the Socket or right after receiving them from the
    * Socket.
-   * 
+   *
    * @param node is the symbolic name of the node which is to be affected
    * @param target is the symbolic name of the other node to which connectivity shall be impeded
    * @param direction can be either `Direction.Send`, `Direction.Receive` or `Direction.Both`
@@ -130,7 +130,7 @@ trait Conductor { this: TestConductorExt ⇒
    * Tell the remote support to shutdown the connection to the given remote
    * peer. It works regardless of whether the recipient was initiator or
    * responder.
-   * 
+   *
    * @param node is the symbolic name of the node which is to be affected
    * @param target is the symbolic name of the other node to which connectivity shall be impeded
    */
@@ -143,7 +143,7 @@ trait Conductor { this: TestConductorExt ⇒
    * Tell the remote support to TCP_RESET the connection to the given remote
    * peer. It works regardless of whether the recipient was initiator or
    * responder.
-   * 
+   *
    * @param node is the symbolic name of the node which is to be affected
    * @param target is the symbolic name of the other node to which connectivity shall be impeded
    */
@@ -155,7 +155,7 @@ trait Conductor { this: TestConductorExt ⇒
   /**
    * Tell the remote node to shut itself down using System.exit with the given
    * exitValue.
-   * 
+   *
    * @param node is the symbolic name of the node which is to be affected
    * @param exitValue is the return code which shall be given to System.exit
    */
@@ -166,7 +166,7 @@ trait Conductor { this: TestConductorExt ⇒
 
   /**
    * Tell the SBT plugin to forcibly terminate the given remote node using Process.destroy.
-   * 
+   *
    * @param node is the symbolic name of the node which is to be affected
    */
   def kill(node: String): Future[Done] = {
@@ -177,7 +177,7 @@ trait Conductor { this: TestConductorExt ⇒
   /**
    * Obtain the list of remote host names currently registered.
    */
-  def getNodes: Future[List[String]] = {
+  def getNodes: Future[Iterable[String]] = {
     import Settings.QueryTimeout
     controller ? GetNodes mapTo
   }
@@ -187,7 +187,7 @@ trait Conductor { this: TestConductorExt ⇒
    * pass subsequent barriers. This must be done before the client connection
    * breaks down in order to affect an “orderly” removal (i.e. without failing
    * present and future barriers).
-   * 
+   *
    * @param node is the symbolic name of the node which is to be removed
    */
   def removeNode(node: String): Future[Done] = {
@@ -330,22 +330,32 @@ object Controller {
  * [[akka.remote.testconductor.BarrierCoordinator]], its child) and allowing
  * network and other failures to be injected at the test nodes.
  */
-class Controller(_participants: Int) extends Actor {
+class Controller(private var initialParticipants: Int) extends Actor {
   import Controller._
-
-  var initialParticipants = _participants
+  import BarrierCoordinator._
 
   val settings = TestConductor().Settings
   val connection = RemoteConnection(Server, settings.host, settings.port,
     new ConductorHandler(context.system, self, Logging(context.system, "ConductorHandler")))
 
+  /*
+   * Supervision of the BarrierCoordinator means to catch all his bad emotions
+   * and sometimes console him (BarrierEmpty, BarrierTimeout), sometimes tell
+   * him to hate the world (WrongBarrier, DuplicateNode, ClientLost). The latter shall help
+   * terminate broken tests as quickly as possible (i.e. without awaiting
+   * BarrierTimeouts in the players).
+   */
   override def supervisorStrategy = OneForOneStrategy() {
-    case e: BarrierCoordinator.BarrierTimeoutException ⇒ SupervisorStrategy.Resume
-    case e: BarrierCoordinator.WrongBarrierException ⇒
-      for (NodeInfo(c, _, _) ← e.data.clients; info ← nodes get c)
-        barrier ! NodeInfo(c, info.addr, info.fsm)
-      for (c ← e.data.arrived) c ! BarrierFailed(e.barrier)
-      SupervisorStrategy.Restart
+    case BarrierTimeout(data)             ⇒ SupervisorStrategy.Resume
+    case BarrierEmpty(data, msg)          ⇒ SupervisorStrategy.Resume
+    case WrongBarrier(name, client, data) ⇒ client ! Send(BarrierFailed(name)); failBarrier(data)
+    case ClientLost(data, node)           ⇒ failBarrier(data)
+    case DuplicateNode(data, node)        ⇒ failBarrier(data)
+  }
+
+  def failBarrier(data: Data): SupervisorStrategy.Directive = {
+    for (c ← data.arrived) c ! Send(BarrierFailed(data.barrier))
+    SupervisorStrategy.Restart
   }
 
   val barrier = context.actorOf(Props[BarrierCoordinator], "barriers")
@@ -353,12 +363,20 @@ class Controller(_participants: Int) extends Actor {
 
   override def receive = LoggingReceive {
     case c @ NodeInfo(name, addr, fsm) ⇒
-      nodes += name -> c
       barrier forward c
-      if (initialParticipants <= 0) sender ! Done
-      else if (nodes.size == initialParticipants) {
-        for (NodeInfo(_, _, client) ← nodes.values) client ! Send(Done)
-        initialParticipants = 0
+      if (nodes contains name) {
+        if (initialParticipants > 0) {
+          for (NodeInfo(_, _, client) ← nodes.values) client ! Send(BarrierFailed("initial startup"))
+          initialParticipants = 0
+        }
+        fsm ! Send(BarrierFailed("initial startup"))
+      } else {
+        nodes += name -> c
+        if (initialParticipants <= 0) fsm ! Send(Done)
+        else if (nodes.size == initialParticipants) {
+          for (NodeInfo(_, _, client) ← nodes.values) client ! Send(Done)
+          initialParticipants = 0
+        }
       }
     case c @ ClientDisconnected(name) ⇒
       nodes -= name
@@ -396,8 +414,16 @@ object BarrierCoordinator {
   case class RemoveClient(name: String)
 
   case class Data(clients: Set[Controller.NodeInfo], barrier: String, arrived: List[ActorRef])
-  class BarrierTimeoutException(val data: Data) extends RuntimeException(data.barrier) with NoStackTrace
-  class WrongBarrierException(val barrier: String, val client: ActorRef, val data: Data) extends RuntimeException(barrier) with NoStackTrace
+
+  trait Printer { this: Product with Throwable with NoStackTrace ⇒
+    override def toString = productPrefix + productIterator.mkString("(", ", ", ")")
+  }
+
+  case class BarrierTimeout(data: Data) extends RuntimeException(data.barrier) with NoStackTrace with Printer
+  case class DuplicateNode(data: Data, node: Controller.NodeInfo) extends RuntimeException with NoStackTrace with Printer
+  case class WrongBarrier(barrier: String, client: ActorRef, data: Data) extends RuntimeException(barrier) with NoStackTrace with Printer
+  case class BarrierEmpty(data: Data, msg: String) extends RuntimeException(msg) with NoStackTrace with Printer
+  case class ClientLost(data: Data, client: String) extends RuntimeException with NoStackTrace with Printer
 }
 
 /**
@@ -426,26 +452,28 @@ class BarrierCoordinator extends Actor with LoggingFSM[BarrierCoordinator.State,
 
   whenUnhandled {
     case Event(n: NodeInfo, d @ Data(clients, _, _)) ⇒
+      if (clients.find(_.name == n.name).isDefined) throw new DuplicateNode(d, n)
       stay using d.copy(clients = clients + n)
+    case Event(ClientDisconnected(name), d @ Data(clients, _, arrived)) ⇒
+      if (clients.isEmpty) throw BarrierEmpty(d, "no client to disconnect")
+      (clients find (_.name == name)) match {
+        case None    ⇒ stay
+        case Some(c) ⇒ throw ClientLost(d.copy(clients = clients - c, arrived = arrived filterNot (_ == c.fsm)), name)
+      }
   }
 
   when(Idle) {
-    case Event(EnterBarrier(name), d @ Data(clients, _, _)) ⇒
-      if (clients.isEmpty) throw new IllegalStateException("no client expected yet")
+    case Event(e @ EnterBarrier(name), d @ Data(clients, _, _)) ⇒
       if (failed)
-        stay replying BarrierFailed(name)
+        stay replying Send(BarrierFailed(name))
+      else if (clients.map(_.fsm) == Set(sender))
+        stay replying Send(e)
+      else if (clients.find(_.fsm == sender).isEmpty)
+        stay replying Send(BarrierFailed(name))
       else
         goto(Waiting) using d.copy(barrier = name, arrived = sender :: Nil)
-    case Event(ClientDisconnected(name), d @ Data(clients, _, _)) ⇒
-      if (clients.isEmpty) throw new IllegalStateException("no client to disconnect")
-      (clients filterNot (_.name == name)) match {
-        case `clients` ⇒ stay
-        case c ⇒
-          failed = true
-          stay using d.copy(clients = c)
-      }
     case Event(RemoveClient(name), d @ Data(clients, _, _)) ⇒
-      if (clients.isEmpty) throw new IllegalStateException("no client to remove")
+      if (clients.isEmpty) throw BarrierEmpty(d, "no client to remove")
       stay using d.copy(clients = clients filterNot (_.name == name))
   }
 
@@ -456,36 +484,33 @@ class BarrierCoordinator extends Actor with LoggingFSM[BarrierCoordinator.State,
 
   when(Waiting) {
     case Event(e @ EnterBarrier(name), d @ Data(clients, barrier, arrived)) ⇒
-      if (name != barrier) throw new WrongBarrierException(barrier, sender, d)
+      if (name != barrier || clients.find(_.fsm == sender).isEmpty) throw WrongBarrier(name, sender, d)
       val together = sender :: arrived
       handleBarrier(d.copy(arrived = together))
     case Event(RemoveClient(name), d @ Data(clients, barrier, arrived)) ⇒
-      val newClients = clients filterNot (_.name == name)
-      val newArrived = arrived filterNot (_ == name)
-      handleBarrier(d.copy(clients = newClients, arrived = newArrived))
-    case Event(ClientDisconnected(name), d @ Data(clients, barrier, arrived)) ⇒
-      (clients filterNot (_.name == name)) match {
-        case `clients` ⇒ stay
-        case c ⇒
-          val f = BarrierFailed(barrier)
-          arrived foreach (_ ! Send(f))
-          failed = true
-          goto(Idle) using Data(c, "", Nil)
+      clients find (_.name == name) match {
+        case None ⇒ stay
+        case Some(client) ⇒
+          handleBarrier(d.copy(clients = clients - client, arrived = arrived filterNot (_ == client.fsm)))
       }
     case Event(StateTimeout, data) ⇒
-      throw new BarrierTimeoutException(data)
+      throw BarrierTimeout(data)
   }
 
   initialize
 
-  def handleBarrier(data: Data): State =
-    if ((data.clients.map(_.fsm) -- data.arrived).isEmpty) {
+  def handleBarrier(data: Data): State = {
+    log.debug("handleBarrier({})", data)
+    if (data.arrived.isEmpty) {
+      goto(Idle) using data.copy(barrier = "")
+    } else if ((data.clients.map(_.fsm) -- data.arrived).isEmpty) {
       val e = EnterBarrier(data.barrier)
       data.arrived foreach (_ ! Send(e))
       goto(Idle) using data.copy(barrier = "", arrived = Nil)
     } else {
       stay using data
     }
+  }
 
 }
 
diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Extension.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Extension.scala
index ff1d77fb9d..5d7826c60c 100644
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Extension.scala
+++ b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Extension.scala
@@ -13,7 +13,7 @@ import akka.actor.Address
 
 /**
  * Access to the [[akka.remote.testconductor.TestConductorExt]] extension:
- * 
+ *
  * {{{
  * val tc = TestConductor(system)
  * tc.startController(numPlayers)
diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Player.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Player.scala
index 38d0f6ef34..a82a090b23 100644
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Player.scala
+++ b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Player.scala
@@ -55,9 +55,9 @@ trait Player { this: TestConductorExt ⇒
       def receive = {
         case fsm: ActorRef                        ⇒ waiting = sender; fsm ! SubscribeTransitionCallBack(self)
         case Transition(_, Connecting, AwaitDone) ⇒ // step 1, not there yet
-        case Transition(_, AwaitDone, Connected)  ⇒ waiting ! Done
-        case t: Transition[_]                     ⇒ waiting ! Status.Failure(new RuntimeException("unexpected transition: " + t))
-        case CurrentState(_, Connected)           ⇒ waiting ! Done
+        case Transition(_, AwaitDone, Connected)  ⇒ waiting ! Done; context stop self
+        case t: Transition[_]                     ⇒ waiting ! Status.Failure(new RuntimeException("unexpected transition: " + t)); context stop self
+        case CurrentState(_, Connected)           ⇒ waiting ! Done; context stop self
         case _: CurrentState[_]                   ⇒
       }
     }))
@@ -84,6 +84,7 @@ object ClientFSM {
   case object Connecting extends State
   case object AwaitDone extends State
   case object Connected extends State
+  case object Failed extends State
 
   case class Data(channel: Channel, barrier: Option[(String, ActorRef)])
 
@@ -116,24 +117,24 @@ class ClientFSM(port: Int) extends Actor with LoggingFSM[ClientFSM.State, Client
       channel.write(Hello(settings.name, TestConductor().address))
       goto(AwaitDone)
     case Event(_: ConnectionFailure, _) ⇒
-      // System.exit(1)
-      stop
+      goto(Failed)
     case Event(StateTimeout, _) ⇒
       log.error("connect timeout to TestConductor")
-      // System.exit(1)
-      stop
+      goto(Failed)
   }
 
   when(AwaitDone, stateTimeout = settings.BarrierTimeout.duration) {
     case Event(Done, _) ⇒
       log.debug("received Done: starting test")
       goto(Connected)
+    case Event(msg: NetworkOp, _) ⇒
+      log.error("received {} instead of Done", msg)
+      goto(Failed)
     case Event(msg: ClientOp, _) ⇒
       stay replying Status.Failure(new IllegalStateException("not connected yet"))
     case Event(StateTimeout, _) ⇒
       log.error("connect timeout to TestConductor")
-      // System.exit(1)
-      stop
+      goto(Failed)
   }
 
   when(Connected) {
@@ -180,6 +181,14 @@ class ClientFSM(port: Int) extends Actor with LoggingFSM[ClientFSM.State, Client
       stay // needed because Java doesn’t have Nothing
   }
 
+  when(Failed) {
+    case Event(msg: ClientOp, _) ⇒
+      stay replying Status.Failure(new RuntimeException("cannot do " + msg + " while Failed"))
+    case Event(msg: NetworkOp, _) ⇒
+      log.warning("ignoring network message {} while Failed", msg)
+      stay
+  }
+
   onTermination {
     case StopEvent(_, _, Data(channel, _)) ⇒
       channel.close()
diff --git a/akka-remote-tests/src/test/scala/akka/remote/testconductor/BarrierSpec.scala b/akka-remote-tests/src/test/scala/akka/remote/testconductor/BarrierSpec.scala
new file mode 100644
index 0000000000..f0b668d1ed
--- /dev/null
+++ b/akka-remote-tests/src/test/scala/akka/remote/testconductor/BarrierSpec.scala
@@ -0,0 +1,465 @@
+/**
+ *  Copyright (C) 2009-2012 Typesafe Inc. 
+ */
+package akka.remote.testconductor
+
+import akka.testkit.AkkaSpec
+import akka.actor.Props
+import akka.actor.AddressFromURIString
+import akka.actor.ActorRef
+import akka.testkit.ImplicitSender
+import akka.actor.Actor
+import akka.actor.OneForOneStrategy
+import akka.actor.SupervisorStrategy
+import akka.testkit.EventFilter
+import akka.testkit.TestProbe
+import akka.util.duration._
+import akka.event.Logging
+import org.scalatest.BeforeAndAfterEach
+
+object BarrierSpec {
+  case class Failed(ref: ActorRef, thr: Throwable)
+  val config = """
+    akka.testconductor.barrier-timeout = 5s
+    akka.actor.provider = akka.remote.RemoteActorRefProvider
+    akka.remote.netty.port = 0
+    akka.actor.debug.fsm = on
+    akka.actor.debug.lifecycle = on
+    """
+}
+
+class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with BeforeAndAfterEach {
+
+  import BarrierSpec._
+  import Controller._
+  import BarrierCoordinator._
+
+  override def afterEach {
+    system.eventStream.setLogLevel(Logging.WarningLevel)
+  }
+
+  "A BarrierCoordinator" must {
+
+    "register clients and remove them" in {
+      val b = getBarrier()
+      b ! NodeInfo("a", AddressFromURIString("akka://sys"), system.deadLetters)
+      b ! RemoveClient("b")
+      b ! RemoveClient("a")
+      EventFilter[BarrierEmpty](occurrences = 1) intercept {
+        b ! RemoveClient("a")
+      }
+      expectMsg(Failed(b, BarrierEmpty(Data(Set(), "", Nil), "no client to remove")))
+    }
+
+    "register clients and disconnect them" in {
+      val b = getBarrier()
+      b ! NodeInfo("a", AddressFromURIString("akka://sys"), system.deadLetters)
+      b ! ClientDisconnected("b")
+      EventFilter[ClientLost](occurrences = 1) intercept {
+        b ! ClientDisconnected("a")
+      }
+      expectMsg(Failed(b, ClientLost(Data(Set(), "", Nil), "a")))
+      EventFilter[BarrierEmpty](occurrences = 1) intercept {
+        b ! ClientDisconnected("a")
+      }
+      expectMsg(Failed(b, BarrierEmpty(Data(Set(), "", Nil), "no client to disconnect")))
+    }
+
+    "fail entering barrier when nobody registered" in {
+      val b = getBarrier()
+      b ! EnterBarrier("b")
+      expectMsg(Send(BarrierFailed("b")))
+    }
+
+    "enter barrier" in {
+      val barrier = getBarrier()
+      val a, b = TestProbe()
+      barrier ! NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
+      barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      a.send(barrier, EnterBarrier("bar"))
+      noMsg(a, b)
+      within(1 second) {
+        b.send(barrier, EnterBarrier("bar"))
+        a.expectMsg(Send(EnterBarrier("bar")))
+        b.expectMsg(Send(EnterBarrier("bar")))
+      }
+    }
+
+    "enter barrier with joining node" in {
+      val barrier = getBarrier()
+      val a, b, c = TestProbe()
+      barrier ! NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
+      barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      a.send(barrier, EnterBarrier("bar"))
+      barrier ! NodeInfo("c", AddressFromURIString("akka://sys"), c.ref)
+      b.send(barrier, EnterBarrier("bar"))
+      noMsg(a, b, c)
+      within(1 second) {
+        c.send(barrier, EnterBarrier("bar"))
+        a.expectMsg(Send(EnterBarrier("bar")))
+        b.expectMsg(Send(EnterBarrier("bar")))
+        c.expectMsg(Send(EnterBarrier("bar")))
+      }
+    }
+
+    "enter barrier with leaving node" in {
+      val barrier = getBarrier()
+      val a, b, c = TestProbe()
+      barrier ! NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
+      barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      barrier ! NodeInfo("c", AddressFromURIString("akka://sys"), c.ref)
+      a.send(barrier, EnterBarrier("bar"))
+      b.send(barrier, EnterBarrier("bar"))
+      barrier ! RemoveClient("a")
+      barrier ! ClientDisconnected("a")
+      noMsg(a, b, c)
+      b.within(1 second) {
+        barrier ! RemoveClient("c")
+        b.expectMsg(Send(EnterBarrier("bar")))
+      }
+      barrier ! ClientDisconnected("c")
+      expectNoMsg(1 second)
+    }
+
+    "leave barrier when last “arrived” is removed" in {
+      val barrier = getBarrier()
+      val a, b = TestProbe()
+      barrier ! NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
+      barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      a.send(barrier, EnterBarrier("bar"))
+      barrier ! RemoveClient("a")
+      b.send(barrier, EnterBarrier("foo"))
+      b.expectMsg(Send(EnterBarrier("foo")))
+    }
+
+    "fail barrier with disconnecing node" in {
+      val barrier = getBarrier()
+      val a, b = TestProbe()
+      val nodeA = NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
+      barrier ! nodeA
+      barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      a.send(barrier, EnterBarrier("bar"))
+      EventFilter[ClientLost](occurrences = 1) intercept {
+        barrier ! ClientDisconnected("b")
+      }
+      expectMsg(Failed(barrier, ClientLost(Data(Set(nodeA), "bar", a.ref :: Nil), "b")))
+    }
+
+    "fail barrier with disconnecing node who already arrived" in {
+      val barrier = getBarrier()
+      val a, b, c = TestProbe()
+      val nodeA = NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
+      val nodeC = NodeInfo("c", AddressFromURIString("akka://sys"), c.ref)
+      barrier ! nodeA
+      barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      barrier ! nodeC
+      a.send(barrier, EnterBarrier("bar"))
+      b.send(barrier, EnterBarrier("bar"))
+      EventFilter[ClientLost](occurrences = 1) intercept {
+        barrier ! ClientDisconnected("b")
+      }
+      expectMsg(Failed(barrier, ClientLost(Data(Set(nodeA, nodeC), "bar", a.ref :: Nil), "b")))
+    }
+
+    "fail when entering wrong barrier" in {
+      val barrier = getBarrier()
+      val a, b = TestProbe()
+      val nodeA = NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
+      barrier ! nodeA
+      val nodeB = NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      barrier ! nodeB
+      a.send(barrier, EnterBarrier("bar"))
+      EventFilter[WrongBarrier](occurrences = 1) intercept {
+        b.send(barrier, EnterBarrier("foo"))
+      }
+      expectMsg(Failed(barrier, WrongBarrier("foo", b.ref, Data(Set(nodeA, nodeB), "bar", a.ref :: Nil))))
+    }
+
+    "fail barrier after first failure" in {
+      val barrier = getBarrier()
+      val a = TestProbe()
+      EventFilter[BarrierEmpty](occurrences = 1) intercept {
+        barrier ! RemoveClient("a")
+      }
+      expectMsg(Failed(barrier, BarrierEmpty(Data(Set(), "", Nil), "no client to remove")))
+      barrier ! NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
+      a.send(barrier, EnterBarrier("right"))
+      a.expectMsg(Send(BarrierFailed("right")))
+    }
+
+    "fail after barrier timeout" in {
+      val barrier = getBarrier()
+      val a, b = TestProbe()
+      val nodeA = NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
+      val nodeB = NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      barrier ! nodeA
+      barrier ! nodeB
+      a.send(barrier, EnterBarrier("right"))
+      EventFilter[BarrierTimeout](occurrences = 1) intercept {
+        expectMsg(7 seconds, Failed(barrier, BarrierTimeout(Data(Set(nodeA, nodeB), "right", a.ref :: Nil))))
+      }
+    }
+
+    "fail if a node registers twice" in {
+      val barrier = getBarrier()
+      val a, b = TestProbe()
+      val nodeA = NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
+      val nodeB = NodeInfo("a", AddressFromURIString("akka://sys"), b.ref)
+      barrier ! nodeA
+      EventFilter[DuplicateNode](occurrences = 1) intercept {
+        barrier ! nodeB
+      }
+      expectMsg(Failed(barrier, DuplicateNode(Data(Set(nodeA), "", Nil), nodeB)))
+    }
+
+    "finally have no failure messages left" in {
+      expectNoMsg(1 second)
+    }
+
+  }
+
+  "A Controller with BarrierCoordinator" must {
+
+    "register clients and remove them" in {
+      val b = getController(1)
+      b ! NodeInfo("a", AddressFromURIString("akka://sys"), testActor)
+      expectMsg(Send(Done))
+      b ! Remove("b")
+      b ! Remove("a")
+      EventFilter[BarrierEmpty](occurrences = 1) intercept {
+        b ! Remove("a")
+      }
+    }
+
+    "register clients and disconnect them" in {
+      val b = getController(1)
+      b ! NodeInfo("a", AddressFromURIString("akka://sys"), testActor)
+      expectMsg(Send(Done))
+      b ! ClientDisconnected("b")
+      EventFilter[ClientLost](occurrences = 1) intercept {
+        b ! ClientDisconnected("a")
+      }
+      EventFilter[BarrierEmpty](occurrences = 1) intercept {
+        b ! ClientDisconnected("a")
+      }
+    }
+
+    "fail entering barrier when nobody registered" in {
+      val b = getController(0)
+      b ! EnterBarrier("b")
+      expectMsg(Send(BarrierFailed("b")))
+    }
+
+    "enter barrier" in {
+      val barrier = getController(2)
+      val a, b = TestProbe()
+      barrier ! NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
+      barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      a.expectMsg(Send(Done))
+      b.expectMsg(Send(Done))
+      a.send(barrier, EnterBarrier("bar"))
+      noMsg(a, b)
+      within(1 second) {
+        b.send(barrier, EnterBarrier("bar"))
+        a.expectMsg(Send(EnterBarrier("bar")))
+        b.expectMsg(Send(EnterBarrier("bar")))
+      }
+    }
+
+    "enter barrier with joining node" in {
+      val barrier = getController(2)
+      val a, b, c = TestProbe()
+      barrier ! NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
+      barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      a.expectMsg(Send(Done))
+      b.expectMsg(Send(Done))
+      a.send(barrier, EnterBarrier("bar"))
+      barrier ! NodeInfo("c", AddressFromURIString("akka://sys"), c.ref)
+      c.expectMsg(Send(Done))
+      b.send(barrier, EnterBarrier("bar"))
+      noMsg(a, b, c)
+      within(1 second) {
+        c.send(barrier, EnterBarrier("bar"))
+        a.expectMsg(Send(EnterBarrier("bar")))
+        b.expectMsg(Send(EnterBarrier("bar")))
+        c.expectMsg(Send(EnterBarrier("bar")))
+      }
+    }
+
+    "enter barrier with leaving node" in {
+      val barrier = getController(3)
+      val a, b, c = TestProbe()
+      barrier ! NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
+      barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      barrier ! NodeInfo("c", AddressFromURIString("akka://sys"), c.ref)
+      a.expectMsg(Send(Done))
+      b.expectMsg(Send(Done))
+      c.expectMsg(Send(Done))
+      a.send(barrier, EnterBarrier("bar"))
+      b.send(barrier, EnterBarrier("bar"))
+      barrier ! Remove("a")
+      barrier ! ClientDisconnected("a")
+      noMsg(a, b, c)
+      b.within(1 second) {
+        barrier ! Remove("c")
+        b.expectMsg(Send(EnterBarrier("bar")))
+      }
+      barrier ! ClientDisconnected("c")
+      expectNoMsg(1 second)
+    }
+
+    "leave barrier when last “arrived” is removed" in {
+      val barrier = getController(2)
+      val a, b = TestProbe()
+      barrier ! NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
+      barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      a.expectMsg(Send(Done))
+      b.expectMsg(Send(Done))
+      a.send(barrier, EnterBarrier("bar"))
+      barrier ! Remove("a")
+      b.send(barrier, EnterBarrier("foo"))
+      b.expectMsg(Send(EnterBarrier("foo")))
+    }
+
+    "fail barrier with disconnecing node" in {
+      val barrier = getController(2)
+      val a, b = TestProbe()
+      val nodeA = NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
+      barrier ! nodeA
+      barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      a.expectMsg(Send(Done))
+      b.expectMsg(Send(Done))
+      a.send(barrier, EnterBarrier("bar"))
+      barrier ! ClientDisconnected("unknown")
+      noMsg(a)
+      EventFilter[ClientLost](occurrences = 1) intercept {
+        barrier ! ClientDisconnected("b")
+      }
+      a.expectMsg(Send(BarrierFailed("bar")))
+    }
+
+    "fail barrier with disconnecing node who already arrived" in {
+      val barrier = getController(3)
+      val a, b, c = TestProbe()
+      val nodeA = NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
+      val nodeC = NodeInfo("c", AddressFromURIString("akka://sys"), c.ref)
+      barrier ! nodeA
+      barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      barrier ! nodeC
+      a.expectMsg(Send(Done))
+      b.expectMsg(Send(Done))
+      c.expectMsg(Send(Done))
+      a.send(barrier, EnterBarrier("bar"))
+      b.send(barrier, EnterBarrier("bar"))
+      EventFilter[ClientLost](occurrences = 1) intercept {
+        barrier ! ClientDisconnected("b")
+      }
+      a.expectMsg(Send(BarrierFailed("bar")))
+    }
+
+    "fail when entering wrong barrier" in {
+      val barrier = getController(2)
+      val a, b = TestProbe()
+      val nodeA = NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
+      barrier ! nodeA
+      val nodeB = NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      barrier ! nodeB
+      a.expectMsg(Send(Done))
+      b.expectMsg(Send(Done))
+      a.send(barrier, EnterBarrier("bar"))
+      EventFilter[WrongBarrier](occurrences = 1) intercept {
+        b.send(barrier, EnterBarrier("foo"))
+      }
+      a.expectMsg(Send(BarrierFailed("bar")))
+      b.expectMsg(Send(BarrierFailed("foo")))
+    }
+
+    "not really fail after barrier timeout" in {
+      val barrier = getController(2)
+      val a, b = TestProbe()
+      val nodeA = NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
+      val nodeB = NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      barrier ! nodeA
+      barrier ! nodeB
+      a.expectMsg(Send(Done))
+      b.expectMsg(Send(Done))
+      a.send(barrier, EnterBarrier("right"))
+      EventFilter[BarrierTimeout](occurrences = 1) intercept {
+        Thread.sleep(5000)
+      }
+      b.send(barrier, EnterBarrier("right"))
+      a.expectMsg(Send(EnterBarrier("right")))
+      b.expectMsg(Send(EnterBarrier("right")))
+    }
+
+    "fail if a node registers twice" in {
+      val controller = getController(2)
+      val a, b = TestProbe()
+      val nodeA = NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
+      val nodeB = NodeInfo("a", AddressFromURIString("akka://sys"), b.ref)
+      controller ! nodeA
+      EventFilter[DuplicateNode](occurrences = 1) intercept {
+        controller ! nodeB
+      }
+      a.expectMsg(Send(BarrierFailed("initial startup")))
+      b.expectMsg(Send(BarrierFailed("initial startup")))
+    }
+
+    "fail subsequent barriers if a node registers twice" in {
+      val controller = getController(1)
+      val a, b = TestProbe()
+      val nodeA = NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
+      val nodeB = NodeInfo("a", AddressFromURIString("akka://sys"), b.ref)
+      controller ! nodeA
+      a.expectMsg(Send(Done))
+      EventFilter[DuplicateNode](occurrences = 1) intercept {
+        controller ! nodeB
+        b.expectMsg(Send(BarrierFailed("initial startup")))
+      }
+      a.send(controller, EnterBarrier("x"))
+      a.expectMsg(Send(BarrierFailed("x")))
+    }
+
+    "finally have no failure messages left" in {
+      expectNoMsg(1 second)
+    }
+
+  }
+
+  private def getController(participants: Int): ActorRef = {
+    system.actorOf(Props(new Actor {
+      val controller = context.actorOf(Props(new Controller(participants)))
+      controller ! GetPort
+      override def supervisorStrategy = OneForOneStrategy() {
+        case x ⇒ testActor ! Failed(controller, x); SupervisorStrategy.Restart
+      }
+      def receive = {
+        case x: Int ⇒ testActor ! controller
+      }
+    }))
+    expectMsgType[ActorRef]
+  }
+
+  /**
+   * Produce a BarrierCoordinator which is supervised with a strategy which
+   * forwards all failures to the testActor.
+   */
+  private def getBarrier(): ActorRef = {
+    system.actorOf(Props(new Actor {
+      val barrier = context.actorOf(Props[BarrierCoordinator])
+      override def supervisorStrategy = OneForOneStrategy() {
+        case x ⇒ testActor ! Failed(barrier, x); SupervisorStrategy.Restart
+      }
+      def receive = {
+        case _ ⇒ sender ! barrier
+      }
+    })) ! ""
+    expectMsgType[ActorRef]
+  }
+
+  private def noMsg(probes: TestProbe*) {
+    expectNoMsg(1 second)
+    probes foreach (_.msgAvailable must be(false))
+  }
+
+}
\ No newline at end of file
diff --git a/akka-remote-tests/src/test/scala/akka/remote/testconductor/ControllerSpec.scala b/akka-remote-tests/src/test/scala/akka/remote/testconductor/ControllerSpec.scala
new file mode 100644
index 0000000000..db0e3cfe69
--- /dev/null
+++ b/akka-remote-tests/src/test/scala/akka/remote/testconductor/ControllerSpec.scala
@@ -0,0 +1,38 @@
+/**
+ *  Copyright (C) 2009-2012 Typesafe Inc. 
+ */
+package akka.remote.testconductor
+
+import akka.testkit.AkkaSpec
+import akka.actor.Props
+import akka.testkit.ImplicitSender
+import akka.remote.testconductor.Controller.NodeInfo
+import akka.actor.AddressFromURIString
+
+object ControllerSpec {
+  val config = """
+    akka.testconductor.barrier-timeout = 5s
+    akka.actor.provider = akka.remote.RemoteActorRefProvider
+    akka.remote.netty.port = 0
+    akka.actor.debug.fsm = on
+    akka.actor.debug.lifecycle = on
+    """
+}
+
+class ControllerSpec extends AkkaSpec(ControllerSpec.config) with ImplicitSender {
+
+  "A Controller" must {
+
+    "publish its nodes" in {
+      val c = system.actorOf(Props(new Controller(1)))
+      c ! NodeInfo("a", AddressFromURIString("akka://sys"), testActor)
+      expectMsg(Send(Done))
+      c ! NodeInfo("b", AddressFromURIString("akka://sys"), testActor)
+      expectMsg(Send(Done))
+      c ! Controller.GetNodes
+      expectMsgType[Iterable[String]].toSet must be(Set("a", "b"))
+    }
+
+  }
+
+}
\ No newline at end of file

From 14dc08b75a2c08f83b5357bba285150dcf7896c1 Mon Sep 17 00:00:00 2001
From: Patrik Nordwall 
Date: Mon, 14 May 2012 14:26:32 +0200
Subject: [PATCH 17/36] Added failing DirectRoutedRemoteActorMultiJvmSpec. See
 #2069

---
 .../remote/SimpleRemoteMultiJvmSpec.scala     | 82 +++++++++++++++++
 .../DirectRoutedRemoteActorMultiJvmSpec.scala | 90 +++++++++++++++++++
 2 files changed, 172 insertions(+)
 create mode 100644 akka-remote-tests/src/multi-jvm/scala/akka/remote/SimpleRemoteMultiJvmSpec.scala
 create mode 100644 akka-remote-tests/src/multi-jvm/scala/akka/remote/router/DirectRoutedRemoteActorMultiJvmSpec.scala

diff --git a/akka-remote-tests/src/multi-jvm/scala/akka/remote/SimpleRemoteMultiJvmSpec.scala b/akka-remote-tests/src/multi-jvm/scala/akka/remote/SimpleRemoteMultiJvmSpec.scala
new file mode 100644
index 0000000000..9209deb9a5
--- /dev/null
+++ b/akka-remote-tests/src/multi-jvm/scala/akka/remote/SimpleRemoteMultiJvmSpec.scala
@@ -0,0 +1,82 @@
+/**
+ *  Copyright (C) 2009-2011 Typesafe Inc. 
+ */
+package akka.remote
+
+import akka.actor.Actor
+import akka.actor.ActorRef
+import akka.actor.Props
+import akka.dispatch.Await
+import akka.pattern.ask
+import akka.remote.testconductor.TestConductor
+import akka.testkit.DefaultTimeout
+import akka.testkit.ImplicitSender
+import akka.util.Duration
+import com.typesafe.config.ConfigFactory
+
+object SimpleRemoteMultiJvmSpec extends AbstractRemoteActorMultiJvmSpec {
+  override def NrOfNodes = 2
+
+  class SomeActor extends Actor with Serializable {
+    def receive = {
+      case "identify" ⇒ sender ! self
+    }
+  }
+
+  override def commonConfig = ConfigFactory.parseString("""
+      akka {
+        loglevel = INFO
+        actor {
+          provider = akka.remote.RemoteActorRefProvider
+          debug {
+            receive = on
+            fsm = on
+          }
+        }
+        remote {
+          transport = akka.remote.testconductor.TestConductorTransport
+          log-received-messages = on
+          log-sent-messages = on
+        }
+        testconductor {
+          host = localhost
+          port = 4712
+        }
+      }""")
+
+  def nameConfig(n: Int) = ConfigFactory.parseString("akka.testconductor.name = node" + n).withFallback(nodeConfigs(n))
+}
+
+class SimpleRemoteMultiJvmNode1 extends AkkaRemoteSpec(SimpleRemoteMultiJvmSpec.nameConfig(0)) {
+  import SimpleRemoteMultiJvmSpec._
+  val nodes = NrOfNodes
+  val tc = TestConductor(system)
+
+  "lookup remote actor" in {
+    Await.result(tc.startController(2), Duration.Inf)
+    system.actorOf(Props[SomeActor], "service-hello")
+    tc.enter("begin", "done")
+  }
+
+}
+
+class SimpleRemoteMultiJvmNode2 extends AkkaRemoteSpec(SimpleRemoteMultiJvmSpec.nameConfig(1))
+  with ImplicitSender with DefaultTimeout {
+
+  import SimpleRemoteMultiJvmSpec._
+  val nodes = NrOfNodes
+  val tc = TestConductor(system)
+
+  "lookup remote actor" in {
+    Await.result(tc.startClient(4712), Duration.Inf)
+    tc.enter("begin")
+    log.info("### begin ok")
+    val actor = system.actorFor("akka://" + akkaSpec(0) + "/user/service-hello")
+    log.info("### actor lookup " + akkaSpec(0) + "/service-hello")
+    actor.isInstanceOf[RemoteActorRef] must be(true)
+    Await.result(actor ? "identify", timeout.duration).asInstanceOf[ActorRef].path.address.hostPort must equal(akkaSpec(0))
+    log.info("### actor ok")
+    tc.enter("done")
+  }
+
+}
diff --git a/akka-remote-tests/src/multi-jvm/scala/akka/remote/router/DirectRoutedRemoteActorMultiJvmSpec.scala b/akka-remote-tests/src/multi-jvm/scala/akka/remote/router/DirectRoutedRemoteActorMultiJvmSpec.scala
new file mode 100644
index 0000000000..d44beff605
--- /dev/null
+++ b/akka-remote-tests/src/multi-jvm/scala/akka/remote/router/DirectRoutedRemoteActorMultiJvmSpec.scala
@@ -0,0 +1,90 @@
+/**
+ *  Copyright (C) 2009-2011 Typesafe Inc. 
+ */
+package akka.remote.router
+
+import akka.actor.{ Actor, ActorRef, Props }
+import akka.remote.AkkaRemoteSpec
+import akka.remote.AbstractRemoteActorMultiJvmSpec
+import akka.remote.RemoteActorRef
+import akka.remote.testconductor.TestConductor
+import akka.testkit._
+import akka.dispatch.Await
+import akka.pattern.ask
+import akka.util.Duration
+
+object DirectRoutedRemoteActorMultiJvmSpec extends AbstractRemoteActorMultiJvmSpec {
+  override def NrOfNodes = 2
+
+  class SomeActor extends Actor with Serializable {
+    def receive = {
+      case "identify" ⇒ sender ! self
+    }
+  }
+
+  import com.typesafe.config.ConfigFactory
+  override def commonConfig = ConfigFactory.parseString("""
+      akka {
+        loglevel = INFO
+        actor {
+          provider = akka.remote.RemoteActorRefProvider
+          deployment {
+            /service-hello.remote = %s
+          }
+          debug {
+            receive = on
+            fsm = on
+          }
+        }
+        remote {
+          transport = akka.remote.testconductor.TestConductorTransport
+          log-received-messages = on
+          log-sent-messages = on
+        }
+        testconductor {
+          host = localhost
+          port = 4712
+        }
+      }""" format akkaURIs(1))
+
+  def nameConfig(n: Int) = ConfigFactory.parseString("akka.testconductor.name = node" + n).withFallback(nodeConfigs(n))
+}
+
+class DirectRoutedRemoteActorMultiJvmNode1 extends AkkaRemoteSpec(DirectRoutedRemoteActorMultiJvmSpec.nameConfig(0)) {
+  import DirectRoutedRemoteActorMultiJvmSpec._
+  val nodes = NrOfNodes
+  val tc = TestConductor(system)
+
+  "A new remote actor configured with a Direct router" must {
+    "be locally instantiated on a remote node and be able to communicate through its RemoteActorRef" in {
+      Await.result(tc.startController(2), Duration.Inf)
+      tc.enter("begin", "done")
+    }
+  }
+
+}
+
+class DirectRoutedRemoteActorMultiJvmNode2 extends AkkaRemoteSpec(DirectRoutedRemoteActorMultiJvmSpec.nameConfig(1))
+  with ImplicitSender with DefaultTimeout {
+
+  import DirectRoutedRemoteActorMultiJvmSpec._
+  val nodes = NrOfNodes
+  val tc = TestConductor(system)
+
+  "A new remote actor configured with a Direct router" must {
+    "be locally instantiated on a remote node and be able to communicate through its RemoteActorRef" in {
+      Await.result(tc.startClient(4712), Duration.Inf)
+      tc.enter("begin")
+
+      val actor = system.actorOf(Props[SomeActor], "service-hello")
+      actor.isInstanceOf[RemoteActorRef] must be(true)
+
+      Await.result(actor ? "identify", timeout.duration).asInstanceOf[ActorRef].path.address.hostPort must equal(akkaSpec(0))
+
+      // shut down the actor before we let the other node(s) shut down so we don't try to send
+      // "Terminate" to a shut down node
+      system.stop(actor)
+      tc.enter("done")
+    }
+  }
+}

From c86051505b3c78629ec416f6a26b60552ac93c80 Mon Sep 17 00:00:00 2001
From: Roland 
Date: Fri, 18 May 2012 15:55:04 +0200
Subject: [PATCH 18/36] wrap up MultiNodeSpec, see #1934 and #2063
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- restructure message classes in sealed traits according to message flow
  direction and include confirmed/unconfirmed status in the type
- add GetAddress query for obtaining the remote transport address of
  another test participant
- add reconnects to Player
- add small DSL with runOn(node...), ifNode(node...)()()
  and node():ActorPath
- rewrite TestConductorSpec to use that DSL and run within a single test
  procedure instead of separate NodeX classes
- hook up that test into current multi-jvm infrastructure temporarily
  for testing (will use Björn’s new remote-multi-jvm stuff later)
---
 .../testconductor/TestConductorProtocol.java  | 809 ++++++++++++++++--
 .../main/protocol/TestConductorProtocol.proto |   8 +-
 .../src/main/resources/reference.conf         |  15 +-
 .../akka/remote/testconductor/Conductor.scala | 109 +--
 .../akka/remote/testconductor/DataTypes.scala |  55 +-
 .../akka/remote/testconductor/Extension.scala |   8 +-
 .../akka/remote/testconductor/Player.scala    | 172 ++--
 .../testconductor/RemoteConnection.scala      |   3 +-
 .../testconductor/TestConductorSpec.scala     | 140 +--
 .../remote/testconductor/BarrierSpec.scala    | 106 +--
 .../remote/testconductor/ControllerSpec.scala |   8 +-
 .../akka/remote/testkit/MultiNodeSpec.scala   | 157 ++++
 scripts/fix-protobuf.sh                       |   3 +
 13 files changed, 1291 insertions(+), 302 deletions(-)
 create mode 100644 akka-remote-tests/src/test/scala/akka/remote/testkit/MultiNodeSpec.scala
 create mode 100755 scripts/fix-protobuf.sh

diff --git a/akka-remote-tests/src/main/java/akka/remote/testconductor/TestConductorProtocol.java b/akka-remote-tests/src/main/java/akka/remote/testconductor/TestConductorProtocol.java
index 3d6c145097..4ae1aae07a 100644
--- a/akka-remote-tests/src/main/java/akka/remote/testconductor/TestConductorProtocol.java
+++ b/akka-remote-tests/src/main/java/akka/remote/testconductor/TestConductorProtocol.java
@@ -176,6 +176,11 @@ public final class TestConductorProtocol {
     // optional string done = 4;
     boolean hasDone();
     String getDone();
+    
+    // optional .AddressRequest addr = 5;
+    boolean hasAddr();
+    akka.remote.testconductor.TestConductorProtocol.AddressRequest getAddr();
+    akka.remote.testconductor.TestConductorProtocol.AddressRequestOrBuilder getAddrOrBuilder();
   }
   public static final class Wrapper extends
       com.google.protobuf.GeneratedMessage
@@ -277,11 +282,25 @@ public final class TestConductorProtocol {
       }
     }
     
+    // optional .AddressRequest addr = 5;
+    public static final int ADDR_FIELD_NUMBER = 5;
+    private akka.remote.testconductor.TestConductorProtocol.AddressRequest addr_;
+    public boolean hasAddr() {
+      return ((bitField0_ & 0x00000010) == 0x00000010);
+    }
+    public akka.remote.testconductor.TestConductorProtocol.AddressRequest getAddr() {
+      return addr_;
+    }
+    public akka.remote.testconductor.TestConductorProtocol.AddressRequestOrBuilder getAddrOrBuilder() {
+      return addr_;
+    }
+    
     private void initFields() {
       hello_ = akka.remote.testconductor.TestConductorProtocol.Hello.getDefaultInstance();
       barrier_ = akka.remote.testconductor.TestConductorProtocol.EnterBarrier.getDefaultInstance();
       failure_ = akka.remote.testconductor.TestConductorProtocol.InjectFailure.getDefaultInstance();
       done_ = "";
+      addr_ = akka.remote.testconductor.TestConductorProtocol.AddressRequest.getDefaultInstance();
     }
     private byte memoizedIsInitialized = -1;
     public final boolean isInitialized() {
@@ -306,6 +325,12 @@ public final class TestConductorProtocol {
           return false;
         }
       }
+      if (hasAddr()) {
+        if (!getAddr().isInitialized()) {
+          memoizedIsInitialized = 0;
+          return false;
+        }
+      }
       memoizedIsInitialized = 1;
       return true;
     }
@@ -325,6 +350,9 @@ public final class TestConductorProtocol {
       if (((bitField0_ & 0x00000008) == 0x00000008)) {
         output.writeBytes(4, getDoneBytes());
       }
+      if (((bitField0_ & 0x00000010) == 0x00000010)) {
+        output.writeMessage(5, addr_);
+      }
       getUnknownFields().writeTo(output);
     }
     
@@ -350,6 +378,10 @@ public final class TestConductorProtocol {
         size += com.google.protobuf.CodedOutputStream
           .computeBytesSize(4, getDoneBytes());
       }
+      if (((bitField0_ & 0x00000010) == 0x00000010)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeMessageSize(5, addr_);
+      }
       size += getUnknownFields().getSerializedSize();
       memoizedSerializedSize = size;
       return size;
@@ -460,7 +492,7 @@ public final class TestConductorProtocol {
         maybeForceBuilderInitialization();
       }
       
-      private Builder(com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      private Builder(BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
       }
@@ -469,6 +501,7 @@ public final class TestConductorProtocol {
           getHelloFieldBuilder();
           getBarrierFieldBuilder();
           getFailureFieldBuilder();
+          getAddrFieldBuilder();
         }
       }
       private static Builder create() {
@@ -497,6 +530,12 @@ public final class TestConductorProtocol {
         bitField0_ = (bitField0_ & ~0x00000004);
         done_ = "";
         bitField0_ = (bitField0_ & ~0x00000008);
+        if (addrBuilder_ == null) {
+          addr_ = akka.remote.testconductor.TestConductorProtocol.AddressRequest.getDefaultInstance();
+        } else {
+          addrBuilder_.clear();
+        }
+        bitField0_ = (bitField0_ & ~0x00000010);
         return this;
       }
       
@@ -563,6 +602,14 @@ public final class TestConductorProtocol {
           to_bitField0_ |= 0x00000008;
         }
         result.done_ = done_;
+        if (((from_bitField0_ & 0x00000010) == 0x00000010)) {
+          to_bitField0_ |= 0x00000010;
+        }
+        if (addrBuilder_ == null) {
+          result.addr_ = addr_;
+        } else {
+          result.addr_ = addrBuilder_.build();
+        }
         result.bitField0_ = to_bitField0_;
         onBuilt();
         return result;
@@ -591,6 +638,9 @@ public final class TestConductorProtocol {
         if (other.hasDone()) {
           setDone(other.getDone());
         }
+        if (other.hasAddr()) {
+          mergeAddr(other.getAddr());
+        }
         this.mergeUnknownFields(other.getUnknownFields());
         return this;
       }
@@ -614,6 +664,12 @@ public final class TestConductorProtocol {
             return false;
           }
         }
+        if (hasAddr()) {
+          if (!getAddr().isInitialized()) {
+            
+            return false;
+          }
+        }
         return true;
       }
       
@@ -672,6 +728,15 @@ public final class TestConductorProtocol {
               done_ = input.readBytes();
               break;
             }
+            case 42: {
+              akka.remote.testconductor.TestConductorProtocol.AddressRequest.Builder subBuilder = akka.remote.testconductor.TestConductorProtocol.AddressRequest.newBuilder();
+              if (hasAddr()) {
+                subBuilder.mergeFrom(getAddr());
+              }
+              input.readMessage(subBuilder, extensionRegistry);
+              setAddr(subBuilder.buildPartial());
+              break;
+            }
           }
         }
       }
@@ -984,6 +1049,96 @@ public final class TestConductorProtocol {
         onChanged();
       }
       
+      // optional .AddressRequest addr = 5;
+      private akka.remote.testconductor.TestConductorProtocol.AddressRequest addr_ = akka.remote.testconductor.TestConductorProtocol.AddressRequest.getDefaultInstance();
+      private com.google.protobuf.SingleFieldBuilder<
+          akka.remote.testconductor.TestConductorProtocol.AddressRequest, akka.remote.testconductor.TestConductorProtocol.AddressRequest.Builder, akka.remote.testconductor.TestConductorProtocol.AddressRequestOrBuilder> addrBuilder_;
+      public boolean hasAddr() {
+        return ((bitField0_ & 0x00000010) == 0x00000010);
+      }
+      public akka.remote.testconductor.TestConductorProtocol.AddressRequest getAddr() {
+        if (addrBuilder_ == null) {
+          return addr_;
+        } else {
+          return addrBuilder_.getMessage();
+        }
+      }
+      public Builder setAddr(akka.remote.testconductor.TestConductorProtocol.AddressRequest value) {
+        if (addrBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          addr_ = value;
+          onChanged();
+        } else {
+          addrBuilder_.setMessage(value);
+        }
+        bitField0_ |= 0x00000010;
+        return this;
+      }
+      public Builder setAddr(
+          akka.remote.testconductor.TestConductorProtocol.AddressRequest.Builder builderForValue) {
+        if (addrBuilder_ == null) {
+          addr_ = builderForValue.build();
+          onChanged();
+        } else {
+          addrBuilder_.setMessage(builderForValue.build());
+        }
+        bitField0_ |= 0x00000010;
+        return this;
+      }
+      public Builder mergeAddr(akka.remote.testconductor.TestConductorProtocol.AddressRequest value) {
+        if (addrBuilder_ == null) {
+          if (((bitField0_ & 0x00000010) == 0x00000010) &&
+              addr_ != akka.remote.testconductor.TestConductorProtocol.AddressRequest.getDefaultInstance()) {
+            addr_ =
+              akka.remote.testconductor.TestConductorProtocol.AddressRequest.newBuilder(addr_).mergeFrom(value).buildPartial();
+          } else {
+            addr_ = value;
+          }
+          onChanged();
+        } else {
+          addrBuilder_.mergeFrom(value);
+        }
+        bitField0_ |= 0x00000010;
+        return this;
+      }
+      public Builder clearAddr() {
+        if (addrBuilder_ == null) {
+          addr_ = akka.remote.testconductor.TestConductorProtocol.AddressRequest.getDefaultInstance();
+          onChanged();
+        } else {
+          addrBuilder_.clear();
+        }
+        bitField0_ = (bitField0_ & ~0x00000010);
+        return this;
+      }
+      public akka.remote.testconductor.TestConductorProtocol.AddressRequest.Builder getAddrBuilder() {
+        bitField0_ |= 0x00000010;
+        onChanged();
+        return getAddrFieldBuilder().getBuilder();
+      }
+      public akka.remote.testconductor.TestConductorProtocol.AddressRequestOrBuilder getAddrOrBuilder() {
+        if (addrBuilder_ != null) {
+          return addrBuilder_.getMessageOrBuilder();
+        } else {
+          return addr_;
+        }
+      }
+      private com.google.protobuf.SingleFieldBuilder<
+          akka.remote.testconductor.TestConductorProtocol.AddressRequest, akka.remote.testconductor.TestConductorProtocol.AddressRequest.Builder, akka.remote.testconductor.TestConductorProtocol.AddressRequestOrBuilder> 
+          getAddrFieldBuilder() {
+        if (addrBuilder_ == null) {
+          addrBuilder_ = new com.google.protobuf.SingleFieldBuilder<
+              akka.remote.testconductor.TestConductorProtocol.AddressRequest, akka.remote.testconductor.TestConductorProtocol.AddressRequest.Builder, akka.remote.testconductor.TestConductorProtocol.AddressRequestOrBuilder>(
+                  addr_,
+                  getParentForChildren(),
+                  isClean());
+          addr_ = null;
+        }
+        return addrBuilder_;
+      }
+      
       // @@protoc_insertion_point(builder_scope:Wrapper)
     }
     
@@ -1242,7 +1397,7 @@ public final class TestConductorProtocol {
         maybeForceBuilderInitialization();
       }
       
-      private Builder(com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      private Builder(BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
       }
@@ -1544,9 +1699,9 @@ public final class TestConductorProtocol {
     boolean hasName();
     String getName();
     
-    // optional bool failed = 2;
-    boolean hasFailed();
-    boolean getFailed();
+    // optional bool status = 2;
+    boolean hasStatus();
+    boolean getStatus();
   }
   public static final class EnterBarrier extends
       com.google.protobuf.GeneratedMessage
@@ -1609,19 +1764,19 @@ public final class TestConductorProtocol {
       }
     }
     
-    // optional bool failed = 2;
-    public static final int FAILED_FIELD_NUMBER = 2;
-    private boolean failed_;
-    public boolean hasFailed() {
+    // optional bool status = 2;
+    public static final int STATUS_FIELD_NUMBER = 2;
+    private boolean status_;
+    public boolean hasStatus() {
       return ((bitField0_ & 0x00000002) == 0x00000002);
     }
-    public boolean getFailed() {
-      return failed_;
+    public boolean getStatus() {
+      return status_;
     }
     
     private void initFields() {
       name_ = "";
-      failed_ = false;
+      status_ = false;
     }
     private byte memoizedIsInitialized = -1;
     public final boolean isInitialized() {
@@ -1643,7 +1798,7 @@ public final class TestConductorProtocol {
         output.writeBytes(1, getNameBytes());
       }
       if (((bitField0_ & 0x00000002) == 0x00000002)) {
-        output.writeBool(2, failed_);
+        output.writeBool(2, status_);
       }
       getUnknownFields().writeTo(output);
     }
@@ -1660,7 +1815,7 @@ public final class TestConductorProtocol {
       }
       if (((bitField0_ & 0x00000002) == 0x00000002)) {
         size += com.google.protobuf.CodedOutputStream
-          .computeBoolSize(2, failed_);
+          .computeBoolSize(2, status_);
       }
       size += getUnknownFields().getSerializedSize();
       memoizedSerializedSize = size;
@@ -1772,7 +1927,7 @@ public final class TestConductorProtocol {
         maybeForceBuilderInitialization();
       }
       
-      private Builder(com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      private Builder(BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
       }
@@ -1788,7 +1943,7 @@ public final class TestConductorProtocol {
         super.clear();
         name_ = "";
         bitField0_ = (bitField0_ & ~0x00000001);
-        failed_ = false;
+        status_ = false;
         bitField0_ = (bitField0_ & ~0x00000002);
         return this;
       }
@@ -1835,7 +1990,7 @@ public final class TestConductorProtocol {
         if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
           to_bitField0_ |= 0x00000002;
         }
-        result.failed_ = failed_;
+        result.status_ = status_;
         result.bitField0_ = to_bitField0_;
         onBuilt();
         return result;
@@ -1855,8 +2010,8 @@ public final class TestConductorProtocol {
         if (other.hasName()) {
           setName(other.getName());
         }
-        if (other.hasFailed()) {
-          setFailed(other.getFailed());
+        if (other.hasStatus()) {
+          setStatus(other.getStatus());
         }
         this.mergeUnknownFields(other.getUnknownFields());
         return this;
@@ -1900,7 +2055,7 @@ public final class TestConductorProtocol {
             }
             case 16: {
               bitField0_ |= 0x00000002;
-              failed_ = input.readBool();
+              status_ = input.readBool();
               break;
             }
           }
@@ -1945,23 +2100,23 @@ public final class TestConductorProtocol {
         onChanged();
       }
       
-      // optional bool failed = 2;
-      private boolean failed_ ;
-      public boolean hasFailed() {
+      // optional bool status = 2;
+      private boolean status_ ;
+      public boolean hasStatus() {
         return ((bitField0_ & 0x00000002) == 0x00000002);
       }
-      public boolean getFailed() {
-        return failed_;
+      public boolean getStatus() {
+        return status_;
       }
-      public Builder setFailed(boolean value) {
+      public Builder setStatus(boolean value) {
         bitField0_ |= 0x00000002;
-        failed_ = value;
+        status_ = value;
         onChanged();
         return this;
       }
-      public Builder clearFailed() {
+      public Builder clearStatus() {
         bitField0_ = (bitField0_ & ~0x00000002);
-        failed_ = false;
+        status_ = false;
         onChanged();
         return this;
       }
@@ -1977,6 +2132,544 @@ public final class TestConductorProtocol {
     // @@protoc_insertion_point(class_scope:EnterBarrier)
   }
   
+  public interface AddressRequestOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // required string node = 1;
+    boolean hasNode();
+    String getNode();
+    
+    // optional .Address addr = 2;
+    boolean hasAddr();
+    akka.remote.testconductor.TestConductorProtocol.Address getAddr();
+    akka.remote.testconductor.TestConductorProtocol.AddressOrBuilder getAddrOrBuilder();
+  }
+  public static final class AddressRequest extends
+      com.google.protobuf.GeneratedMessage
+      implements AddressRequestOrBuilder {
+    // Use AddressRequest.newBuilder() to construct.
+    private AddressRequest(Builder builder) {
+      super(builder);
+    }
+    private AddressRequest(boolean noInit) {}
+    
+    private static final AddressRequest defaultInstance;
+    public static AddressRequest getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public AddressRequest getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return akka.remote.testconductor.TestConductorProtocol.internal_static_AddressRequest_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return akka.remote.testconductor.TestConductorProtocol.internal_static_AddressRequest_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // required string node = 1;
+    public static final int NODE_FIELD_NUMBER = 1;
+    private java.lang.Object node_;
+    public boolean hasNode() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public String getNode() {
+      java.lang.Object ref = node_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          node_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getNodeBytes() {
+      java.lang.Object ref = node_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        node_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional .Address addr = 2;
+    public static final int ADDR_FIELD_NUMBER = 2;
+    private akka.remote.testconductor.TestConductorProtocol.Address addr_;
+    public boolean hasAddr() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public akka.remote.testconductor.TestConductorProtocol.Address getAddr() {
+      return addr_;
+    }
+    public akka.remote.testconductor.TestConductorProtocol.AddressOrBuilder getAddrOrBuilder() {
+      return addr_;
+    }
+    
+    private void initFields() {
+      node_ = "";
+      addr_ = akka.remote.testconductor.TestConductorProtocol.Address.getDefaultInstance();
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      if (!hasNode()) {
+        memoizedIsInitialized = 0;
+        return false;
+      }
+      if (hasAddr()) {
+        if (!getAddr().isInitialized()) {
+          memoizedIsInitialized = 0;
+          return false;
+        }
+      }
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeBytes(1, getNodeBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeMessage(2, addr_);
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(1, getNodeBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeMessageSize(2, addr_);
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    public static akka.remote.testconductor.TestConductorProtocol.AddressRequest parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static akka.remote.testconductor.TestConductorProtocol.AddressRequest parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static akka.remote.testconductor.TestConductorProtocol.AddressRequest parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static akka.remote.testconductor.TestConductorProtocol.AddressRequest parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static akka.remote.testconductor.TestConductorProtocol.AddressRequest parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static akka.remote.testconductor.TestConductorProtocol.AddressRequest parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static akka.remote.testconductor.TestConductorProtocol.AddressRequest parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static akka.remote.testconductor.TestConductorProtocol.AddressRequest parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static akka.remote.testconductor.TestConductorProtocol.AddressRequest parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static akka.remote.testconductor.TestConductorProtocol.AddressRequest parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(akka.remote.testconductor.TestConductorProtocol.AddressRequest prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder
+       implements akka.remote.testconductor.TestConductorProtocol.AddressRequestOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return akka.remote.testconductor.TestConductorProtocol.internal_static_AddressRequest_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return akka.remote.testconductor.TestConductorProtocol.internal_static_AddressRequest_fieldAccessorTable;
+      }
+      
+      // Construct using akka.remote.testconductor.TestConductorProtocol.AddressRequest.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+          getAddrFieldBuilder();
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        node_ = "";
+        bitField0_ = (bitField0_ & ~0x00000001);
+        if (addrBuilder_ == null) {
+          addr_ = akka.remote.testconductor.TestConductorProtocol.Address.getDefaultInstance();
+        } else {
+          addrBuilder_.clear();
+        }
+        bitField0_ = (bitField0_ & ~0x00000002);
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return akka.remote.testconductor.TestConductorProtocol.AddressRequest.getDescriptor();
+      }
+      
+      public akka.remote.testconductor.TestConductorProtocol.AddressRequest getDefaultInstanceForType() {
+        return akka.remote.testconductor.TestConductorProtocol.AddressRequest.getDefaultInstance();
+      }
+      
+      public akka.remote.testconductor.TestConductorProtocol.AddressRequest build() {
+        akka.remote.testconductor.TestConductorProtocol.AddressRequest result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private akka.remote.testconductor.TestConductorProtocol.AddressRequest buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        akka.remote.testconductor.TestConductorProtocol.AddressRequest result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public akka.remote.testconductor.TestConductorProtocol.AddressRequest buildPartial() {
+        akka.remote.testconductor.TestConductorProtocol.AddressRequest result = new akka.remote.testconductor.TestConductorProtocol.AddressRequest(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.node_ = node_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        if (addrBuilder_ == null) {
+          result.addr_ = addr_;
+        } else {
+          result.addr_ = addrBuilder_.build();
+        }
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof akka.remote.testconductor.TestConductorProtocol.AddressRequest) {
+          return mergeFrom((akka.remote.testconductor.TestConductorProtocol.AddressRequest)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(akka.remote.testconductor.TestConductorProtocol.AddressRequest other) {
+        if (other == akka.remote.testconductor.TestConductorProtocol.AddressRequest.getDefaultInstance()) return this;
+        if (other.hasNode()) {
+          setNode(other.getNode());
+        }
+        if (other.hasAddr()) {
+          mergeAddr(other.getAddr());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        if (!hasNode()) {
+          
+          return false;
+        }
+        if (hasAddr()) {
+          if (!getAddr().isInitialized()) {
+            
+            return false;
+          }
+        }
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 10: {
+              bitField0_ |= 0x00000001;
+              node_ = input.readBytes();
+              break;
+            }
+            case 18: {
+              akka.remote.testconductor.TestConductorProtocol.Address.Builder subBuilder = akka.remote.testconductor.TestConductorProtocol.Address.newBuilder();
+              if (hasAddr()) {
+                subBuilder.mergeFrom(getAddr());
+              }
+              input.readMessage(subBuilder, extensionRegistry);
+              setAddr(subBuilder.buildPartial());
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // required string node = 1;
+      private java.lang.Object node_ = "";
+      public boolean hasNode() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public String getNode() {
+        java.lang.Object ref = node_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          node_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setNode(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000001;
+        node_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearNode() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        node_ = getDefaultInstance().getNode();
+        onChanged();
+        return this;
+      }
+      void setNode(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000001;
+        node_ = value;
+        onChanged();
+      }
+      
+      // optional .Address addr = 2;
+      private akka.remote.testconductor.TestConductorProtocol.Address addr_ = akka.remote.testconductor.TestConductorProtocol.Address.getDefaultInstance();
+      private com.google.protobuf.SingleFieldBuilder<
+          akka.remote.testconductor.TestConductorProtocol.Address, akka.remote.testconductor.TestConductorProtocol.Address.Builder, akka.remote.testconductor.TestConductorProtocol.AddressOrBuilder> addrBuilder_;
+      public boolean hasAddr() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public akka.remote.testconductor.TestConductorProtocol.Address getAddr() {
+        if (addrBuilder_ == null) {
+          return addr_;
+        } else {
+          return addrBuilder_.getMessage();
+        }
+      }
+      public Builder setAddr(akka.remote.testconductor.TestConductorProtocol.Address value) {
+        if (addrBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          addr_ = value;
+          onChanged();
+        } else {
+          addrBuilder_.setMessage(value);
+        }
+        bitField0_ |= 0x00000002;
+        return this;
+      }
+      public Builder setAddr(
+          akka.remote.testconductor.TestConductorProtocol.Address.Builder builderForValue) {
+        if (addrBuilder_ == null) {
+          addr_ = builderForValue.build();
+          onChanged();
+        } else {
+          addrBuilder_.setMessage(builderForValue.build());
+        }
+        bitField0_ |= 0x00000002;
+        return this;
+      }
+      public Builder mergeAddr(akka.remote.testconductor.TestConductorProtocol.Address value) {
+        if (addrBuilder_ == null) {
+          if (((bitField0_ & 0x00000002) == 0x00000002) &&
+              addr_ != akka.remote.testconductor.TestConductorProtocol.Address.getDefaultInstance()) {
+            addr_ =
+              akka.remote.testconductor.TestConductorProtocol.Address.newBuilder(addr_).mergeFrom(value).buildPartial();
+          } else {
+            addr_ = value;
+          }
+          onChanged();
+        } else {
+          addrBuilder_.mergeFrom(value);
+        }
+        bitField0_ |= 0x00000002;
+        return this;
+      }
+      public Builder clearAddr() {
+        if (addrBuilder_ == null) {
+          addr_ = akka.remote.testconductor.TestConductorProtocol.Address.getDefaultInstance();
+          onChanged();
+        } else {
+          addrBuilder_.clear();
+        }
+        bitField0_ = (bitField0_ & ~0x00000002);
+        return this;
+      }
+      public akka.remote.testconductor.TestConductorProtocol.Address.Builder getAddrBuilder() {
+        bitField0_ |= 0x00000002;
+        onChanged();
+        return getAddrFieldBuilder().getBuilder();
+      }
+      public akka.remote.testconductor.TestConductorProtocol.AddressOrBuilder getAddrOrBuilder() {
+        if (addrBuilder_ != null) {
+          return addrBuilder_.getMessageOrBuilder();
+        } else {
+          return addr_;
+        }
+      }
+      private com.google.protobuf.SingleFieldBuilder<
+          akka.remote.testconductor.TestConductorProtocol.Address, akka.remote.testconductor.TestConductorProtocol.Address.Builder, akka.remote.testconductor.TestConductorProtocol.AddressOrBuilder> 
+          getAddrFieldBuilder() {
+        if (addrBuilder_ == null) {
+          addrBuilder_ = new com.google.protobuf.SingleFieldBuilder<
+              akka.remote.testconductor.TestConductorProtocol.Address, akka.remote.testconductor.TestConductorProtocol.Address.Builder, akka.remote.testconductor.TestConductorProtocol.AddressOrBuilder>(
+                  addr_,
+                  getParentForChildren(),
+                  isClean());
+          addr_ = null;
+        }
+        return addrBuilder_;
+      }
+      
+      // @@protoc_insertion_point(builder_scope:AddressRequest)
+    }
+    
+    static {
+      defaultInstance = new AddressRequest(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:AddressRequest)
+  }
+  
   public interface AddressOrBuilder
       extends com.google.protobuf.MessageOrBuilder {
     
@@ -2312,7 +3005,7 @@ public final class TestConductorProtocol {
         maybeForceBuilderInitialization();
       }
       
-      private Builder(com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      private Builder(BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
       }
@@ -2918,7 +3611,7 @@ public final class TestConductorProtocol {
         maybeForceBuilderInitialization();
       }
       
-      private Builder(com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      private Builder(BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
       }
@@ -3334,6 +4027,11 @@ public final class TestConductorProtocol {
   private static
     com.google.protobuf.GeneratedMessage.FieldAccessorTable
       internal_static_EnterBarrier_fieldAccessorTable;
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_AddressRequest_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_AddressRequest_fieldAccessorTable;
   private static com.google.protobuf.Descriptors.Descriptor
     internal_static_Address_descriptor;
   private static
@@ -3353,21 +4051,24 @@ public final class TestConductorProtocol {
       descriptor;
   static {
     java.lang.String[] descriptorData = {
-      "\n\033TestConductorProtocol.proto\"o\n\007Wrapper" +
-      "\022\025\n\005hello\030\001 \001(\0132\006.Hello\022\036\n\007barrier\030\002 \001(\013" +
-      "2\r.EnterBarrier\022\037\n\007failure\030\003 \001(\0132\016.Injec" +
-      "tFailure\022\014\n\004done\030\004 \001(\t\"0\n\005Hello\022\014\n\004name\030" +
-      "\001 \002(\t\022\031\n\007address\030\002 \002(\0132\010.Address\",\n\014Ente" +
-      "rBarrier\022\014\n\004name\030\001 \002(\t\022\016\n\006failed\030\002 \001(\010\"G" +
-      "\n\007Address\022\020\n\010protocol\030\001 \002(\t\022\016\n\006system\030\002 " +
-      "\002(\t\022\014\n\004host\030\003 \002(\t\022\014\n\004port\030\004 \002(\005\"\212\001\n\rInje" +
-      "ctFailure\022\032\n\007failure\030\001 \002(\0162\t.FailType\022\035\n" +
-      "\tdirection\030\002 \001(\0162\n.Direction\022\031\n\007address\030",
-      "\003 \001(\0132\010.Address\022\020\n\010rateMBit\030\006 \001(\002\022\021\n\texi" +
-      "tValue\030\007 \001(\005*A\n\010FailType\022\014\n\010Throttle\020\001\022\016" +
-      "\n\nDisconnect\020\002\022\t\n\005Abort\020\003\022\014\n\010Shutdown\020\004*" +
-      ",\n\tDirection\022\010\n\004Send\020\001\022\013\n\007Receive\020\002\022\010\n\004B" +
-      "oth\020\003B\035\n\031akka.remote.testconductorH\001"
+      "\n\033TestConductorProtocol.proto\"\216\001\n\007Wrappe" +
+      "r\022\025\n\005hello\030\001 \001(\0132\006.Hello\022\036\n\007barrier\030\002 \001(" +
+      "\0132\r.EnterBarrier\022\037\n\007failure\030\003 \001(\0132\016.Inje" +
+      "ctFailure\022\014\n\004done\030\004 \001(\t\022\035\n\004addr\030\005 \001(\0132\017." +
+      "AddressRequest\"0\n\005Hello\022\014\n\004name\030\001 \002(\t\022\031\n" +
+      "\007address\030\002 \002(\0132\010.Address\",\n\014EnterBarrier" +
+      "\022\014\n\004name\030\001 \002(\t\022\016\n\006status\030\002 \001(\010\"6\n\016Addres" +
+      "sRequest\022\014\n\004node\030\001 \002(\t\022\026\n\004addr\030\002 \001(\0132\010.A" +
+      "ddress\"G\n\007Address\022\020\n\010protocol\030\001 \002(\t\022\016\n\006s" +
+      "ystem\030\002 \002(\t\022\014\n\004host\030\003 \002(\t\022\014\n\004port\030\004 \002(\005\"",
+      "\212\001\n\rInjectFailure\022\032\n\007failure\030\001 \002(\0162\t.Fai" +
+      "lType\022\035\n\tdirection\030\002 \001(\0162\n.Direction\022\031\n\007" +
+      "address\030\003 \001(\0132\010.Address\022\020\n\010rateMBit\030\006 \001(" +
+      "\002\022\021\n\texitValue\030\007 \001(\005*A\n\010FailType\022\014\n\010Thro" +
+      "ttle\020\001\022\016\n\nDisconnect\020\002\022\t\n\005Abort\020\003\022\014\n\010Shu" +
+      "tdown\020\004*,\n\tDirection\022\010\n\004Send\020\001\022\013\n\007Receiv" +
+      "e\020\002\022\010\n\004Both\020\003B\035\n\031akka.remote.testconduct" +
+      "orH\001"
     };
     com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
       new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -3379,7 +4080,7 @@ public final class TestConductorProtocol {
           internal_static_Wrapper_fieldAccessorTable = new
             com.google.protobuf.GeneratedMessage.FieldAccessorTable(
               internal_static_Wrapper_descriptor,
-              new java.lang.String[] { "Hello", "Barrier", "Failure", "Done", },
+              new java.lang.String[] { "Hello", "Barrier", "Failure", "Done", "Addr", },
               akka.remote.testconductor.TestConductorProtocol.Wrapper.class,
               akka.remote.testconductor.TestConductorProtocol.Wrapper.Builder.class);
           internal_static_Hello_descriptor =
@@ -3395,11 +4096,19 @@ public final class TestConductorProtocol {
           internal_static_EnterBarrier_fieldAccessorTable = new
             com.google.protobuf.GeneratedMessage.FieldAccessorTable(
               internal_static_EnterBarrier_descriptor,
-              new java.lang.String[] { "Name", "Failed", },
+              new java.lang.String[] { "Name", "Status", },
               akka.remote.testconductor.TestConductorProtocol.EnterBarrier.class,
               akka.remote.testconductor.TestConductorProtocol.EnterBarrier.Builder.class);
-          internal_static_Address_descriptor =
+          internal_static_AddressRequest_descriptor =
             getDescriptor().getMessageTypes().get(3);
+          internal_static_AddressRequest_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_AddressRequest_descriptor,
+              new java.lang.String[] { "Node", "Addr", },
+              akka.remote.testconductor.TestConductorProtocol.AddressRequest.class,
+              akka.remote.testconductor.TestConductorProtocol.AddressRequest.Builder.class);
+          internal_static_Address_descriptor =
+            getDescriptor().getMessageTypes().get(4);
           internal_static_Address_fieldAccessorTable = new
             com.google.protobuf.GeneratedMessage.FieldAccessorTable(
               internal_static_Address_descriptor,
@@ -3407,7 +4116,7 @@ public final class TestConductorProtocol {
               akka.remote.testconductor.TestConductorProtocol.Address.class,
               akka.remote.testconductor.TestConductorProtocol.Address.Builder.class);
           internal_static_InjectFailure_descriptor =
-            getDescriptor().getMessageTypes().get(4);
+            getDescriptor().getMessageTypes().get(5);
           internal_static_InjectFailure_fieldAccessorTable = new
             com.google.protobuf.GeneratedMessage.FieldAccessorTable(
               internal_static_InjectFailure_descriptor,
diff --git a/akka-remote-tests/src/main/protocol/TestConductorProtocol.proto b/akka-remote-tests/src/main/protocol/TestConductorProtocol.proto
index 007965b2e8..648234614e 100644
--- a/akka-remote-tests/src/main/protocol/TestConductorProtocol.proto
+++ b/akka-remote-tests/src/main/protocol/TestConductorProtocol.proto
@@ -16,6 +16,7 @@ message Wrapper {
   optional EnterBarrier barrier = 2;
   optional InjectFailure failure = 3;
   optional string done = 4;
+  optional AddressRequest addr = 5;
 }
 
 message Hello {
@@ -25,7 +26,12 @@ message Hello {
 
 message EnterBarrier {
   required string name = 1;
-  optional bool failed = 2;
+  optional bool status = 2;
+}
+
+message AddressRequest {
+  required string node = 1;
+  optional Address addr = 2;
 }
 
 message Address {
diff --git a/akka-remote-tests/src/main/resources/reference.conf b/akka-remote-tests/src/main/resources/reference.conf
index f0d8a9d6ae..40c16c4ccd 100644
--- a/akka-remote-tests/src/main/resources/reference.conf
+++ b/akka-remote-tests/src/main/resources/reference.conf
@@ -20,15 +20,14 @@ akka {
     # than HashedWheelTimer resolution (would not make sense)
     packet-split-threshold = 100ms
     
-    # Default port to start the conductor on; 0 means 
-    port = 0
+    # amount of time for the ClientFSM to wait for the connection to the conductor
+    # to be successful
+    connect-timeout = 20s
     
-    # Hostname of the TestConductor server, used by the server to bind to the IP
-    # and by the client to connect to it.
-    host = localhost
+    # Number of connect attempts to be made to the conductor controller
+    client-reconnects = 10
     
-    # Name of the TestConductor client (for identification on the server e.g. for
-    # failure injection)
-    name = "noname"
+    # minimum time interval which is to be inserted between reconnect attempts
+    reconnect-backoff = 1s
   }
 }
\ No newline at end of file
diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala
index 09a6faeeb0..d4fa3152e6 100644
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala
+++ b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala
@@ -71,11 +71,11 @@ trait Conductor { this: TestConductorExt ⇒
    * @param participants gives the number of participants which shall connect
    * before any of their startClient() operations complete.
    */
-  def startController(participants: Int): Future[Int] = {
+  def startController(participants: Int, name: String, controllerPort: InetSocketAddress): Future[InetSocketAddress] = {
     if (_controller ne null) throw new RuntimeException("TestConductorServer was already started")
-    _controller = system.actorOf(Props(new Controller(participants)), "controller")
+    _controller = system.actorOf(Props(new Controller(participants, controllerPort)), "controller")
     import Settings.BarrierTimeout
-    controller ? GetPort flatMap { case port: Int ⇒ startClient(port) map (_ ⇒ port) }
+    controller ? GetSockAddr flatMap { case sockAddr: InetSocketAddress ⇒ startClient(name, sockAddr) map (_ ⇒ sockAddr) }
   }
 
   /**
@@ -83,9 +83,9 @@ trait Conductor { this: TestConductorExt ⇒
    * will deviate from the configuration in `akka.testconductor.port` in case
    * that was given as zero.
    */
-  def port: Future[Int] = {
+  def sockAddr: Future[InetSocketAddress] = {
     import Settings.QueryTimeout
-    controller ? GetPort mapTo
+    controller ? GetSockAddr mapTo
   }
 
   /**
@@ -280,7 +280,7 @@ class ServerFSM(val controller: ActorRef, val channel: Channel) extends Actor wi
       log.warning("client {} sent no Hello in first message (instead {}), disconnecting", getAddrString(channel), x)
       channel.close()
       stop()
-    case Event(Send(msg), _) ⇒
+    case Event(ToClient(msg), _) ⇒
       log.warning("cannot send {} in state Initial", msg)
       stay
     case Event(StateTimeout, _) ⇒
@@ -290,22 +290,22 @@ class ServerFSM(val controller: ActorRef, val channel: Channel) extends Actor wi
   }
 
   when(Ready) {
-    case Event(msg: EnterBarrier, _) ⇒
-      controller ! msg
-      stay
     case Event(d: Done, Some(s)) ⇒
       s ! d
       stay using None
+    case Event(op: ServerOp, _) ⇒
+      controller ! op
+      stay
     case Event(msg: NetworkOp, _) ⇒
       log.warning("client {} sent unsupported message {}", getAddrString(channel), msg)
       stop()
-    case Event(Send(msg @ (_: EnterBarrier | _: Done)), _) ⇒
+    case Event(ToClient(msg: UnconfirmedClientOp), _) ⇒
       channel.write(msg)
       stay
-    case Event(Send(msg), None) ⇒
+    case Event(ToClient(msg), None) ⇒
       channel.write(msg)
       stay using Some(sender)
-    case Event(Send(msg), _) ⇒
+    case Event(ToClient(msg), _) ⇒
       log.warning("cannot send {} while waiting for previous ACK", msg)
       stay
   }
@@ -320,7 +320,7 @@ class ServerFSM(val controller: ActorRef, val channel: Channel) extends Actor wi
 object Controller {
   case class ClientDisconnected(name: String)
   case object GetNodes
-  case object GetPort
+  case object GetSockAddr
 
   case class NodeInfo(name: String, addr: Address, fsm: ActorRef)
 }
@@ -330,12 +330,12 @@ object Controller {
  * [[akka.remote.testconductor.BarrierCoordinator]], its child) and allowing
  * network and other failures to be injected at the test nodes.
  */
-class Controller(private var initialParticipants: Int) extends Actor {
+class Controller(private var initialParticipants: Int, controllerPort: InetSocketAddress) extends Actor {
   import Controller._
   import BarrierCoordinator._
 
   val settings = TestConductor().Settings
-  val connection = RemoteConnection(Server, settings.host, settings.port,
+  val connection = RemoteConnection(Server, controllerPort,
     new ConductorHandler(context.system, self, Logging(context.system, "ConductorHandler")))
 
   /*
@@ -348,61 +348,73 @@ class Controller(private var initialParticipants: Int) extends Actor {
   override def supervisorStrategy = OneForOneStrategy() {
     case BarrierTimeout(data)             ⇒ SupervisorStrategy.Resume
     case BarrierEmpty(data, msg)          ⇒ SupervisorStrategy.Resume
-    case WrongBarrier(name, client, data) ⇒ client ! Send(BarrierFailed(name)); failBarrier(data)
+    case WrongBarrier(name, client, data) ⇒ client ! ToClient(BarrierResult(name, false)); failBarrier(data)
     case ClientLost(data, node)           ⇒ failBarrier(data)
     case DuplicateNode(data, node)        ⇒ failBarrier(data)
   }
 
   def failBarrier(data: Data): SupervisorStrategy.Directive = {
-    for (c ← data.arrived) c ! Send(BarrierFailed(data.barrier))
+    for (c ← data.arrived) c ! ToClient(BarrierResult(data.barrier, false))
     SupervisorStrategy.Restart
   }
 
   val barrier = context.actorOf(Props[BarrierCoordinator], "barriers")
   var nodes = Map[String, NodeInfo]()
 
+  // map keeping unanswered queries for node addresses (enqueued upon GetAddress, serviced upon NodeInfo)
+  var addrInterest = Map[String, Set[ActorRef]]()
+
   override def receive = LoggingReceive {
     case c @ NodeInfo(name, addr, fsm) ⇒
       barrier forward c
       if (nodes contains name) {
         if (initialParticipants > 0) {
-          for (NodeInfo(_, _, client) ← nodes.values) client ! Send(BarrierFailed("initial startup"))
+          for (NodeInfo(_, _, client) ← nodes.values) client ! ToClient(BarrierResult("initial startup", false))
           initialParticipants = 0
         }
-        fsm ! Send(BarrierFailed("initial startup"))
+        fsm ! ToClient(BarrierResult("initial startup", false))
       } else {
         nodes += name -> c
-        if (initialParticipants <= 0) fsm ! Send(Done)
+        if (initialParticipants <= 0) fsm ! ToClient(Done)
         else if (nodes.size == initialParticipants) {
-          for (NodeInfo(_, _, client) ← nodes.values) client ! Send(Done)
+          for (NodeInfo(_, _, client) ← nodes.values) client ! ToClient(Done)
           initialParticipants = 0
         }
+        if (addrInterest contains name) {
+          addrInterest(name) foreach (_ ! ToClient(AddressReply(name, addr)))
+          addrInterest -= name
+        }
       }
     case c @ ClientDisconnected(name) ⇒
       nodes -= name
       barrier forward c
-    case e @ EnterBarrier(name) ⇒
-      barrier forward e
-    case Throttle(node, target, direction, rateMBit) ⇒
-      val t = nodes(target)
-      nodes(node).fsm forward Send(ThrottleMsg(t.addr, direction, rateMBit))
-    case Disconnect(node, target, abort) ⇒
-      val t = nodes(target)
-      nodes(node).fsm forward Send(DisconnectMsg(t.addr, abort))
-    case Terminate(node, exitValueOrKill) ⇒
-      if (exitValueOrKill < 0) {
-        // TODO: kill via SBT
-      } else {
-        nodes(node).fsm forward Send(TerminateMsg(exitValueOrKill))
+    case op: ServerOp ⇒
+      op match {
+        case _: EnterBarrier ⇒ barrier forward op
+        case GetAddress(node) ⇒
+          if (nodes contains node) sender ! ToClient(AddressReply(node, nodes(node).addr))
+          else addrInterest += node -> ((addrInterest get node getOrElse Set()) + sender)
       }
-    case Remove(node) ⇒
-      nodes -= node
-      barrier ! BarrierCoordinator.RemoveClient(node)
-    case GetNodes ⇒ sender ! nodes.keys
-    case GetPort ⇒
-      sender ! (connection.getLocalAddress match {
-        case inet: InetSocketAddress ⇒ inet.getPort
-      })
+    case op: CommandOp ⇒
+      op match {
+        case Throttle(node, target, direction, rateMBit) ⇒
+          val t = nodes(target)
+          nodes(node).fsm forward ToClient(ThrottleMsg(t.addr, direction, rateMBit))
+        case Disconnect(node, target, abort) ⇒
+          val t = nodes(target)
+          nodes(node).fsm forward ToClient(DisconnectMsg(t.addr, abort))
+        case Terminate(node, exitValueOrKill) ⇒
+          if (exitValueOrKill < 0) {
+            // TODO: kill via SBT
+          } else {
+            nodes(node).fsm forward ToClient(TerminateMsg(exitValueOrKill))
+          }
+        case Remove(node) ⇒
+          nodes -= node
+          barrier ! BarrierCoordinator.RemoveClient(node)
+      }
+    case GetNodes    ⇒ sender ! nodes.keys
+    case GetSockAddr ⇒ sender ! connection.getLocalAddress
   }
 }
 
@@ -463,13 +475,13 @@ class BarrierCoordinator extends Actor with LoggingFSM[BarrierCoordinator.State,
   }
 
   when(Idle) {
-    case Event(e @ EnterBarrier(name), d @ Data(clients, _, _)) ⇒
+    case Event(EnterBarrier(name), d @ Data(clients, _, _)) ⇒
       if (failed)
-        stay replying Send(BarrierFailed(name))
+        stay replying ToClient(BarrierResult(name, false))
       else if (clients.map(_.fsm) == Set(sender))
-        stay replying Send(e)
+        stay replying ToClient(BarrierResult(name, true))
       else if (clients.find(_.fsm == sender).isEmpty)
-        stay replying Send(BarrierFailed(name))
+        stay replying ToClient(BarrierResult(name, false))
       else
         goto(Waiting) using d.copy(barrier = name, arrived = sender :: Nil)
     case Event(RemoveClient(name), d @ Data(clients, _, _)) ⇒
@@ -483,7 +495,7 @@ class BarrierCoordinator extends Actor with LoggingFSM[BarrierCoordinator.State,
   }
 
   when(Waiting) {
-    case Event(e @ EnterBarrier(name), d @ Data(clients, barrier, arrived)) ⇒
+    case Event(EnterBarrier(name), d @ Data(clients, barrier, arrived)) ⇒
       if (name != barrier || clients.find(_.fsm == sender).isEmpty) throw WrongBarrier(name, sender, d)
       val together = sender :: arrived
       handleBarrier(d.copy(arrived = together))
@@ -504,8 +516,7 @@ class BarrierCoordinator extends Actor with LoggingFSM[BarrierCoordinator.State,
     if (data.arrived.isEmpty) {
       goto(Idle) using data.copy(barrier = "")
     } else if ((data.clients.map(_.fsm) -- data.arrived).isEmpty) {
-      val e = EnterBarrier(data.barrier)
-      data.arrived foreach (_ ! Send(e))
+      data.arrived foreach (_ ! ToClient(BarrierResult(data.barrier, true)))
       goto(Idle) using data.copy(barrier = "", arrived = Nil)
     } else {
       stay using data
diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/DataTypes.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/DataTypes.scala
index cadd69f786..0273055469 100644
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/DataTypes.scala
+++ b/akka-remote-tests/src/main/scala/akka/remote/testconductor/DataTypes.scala
@@ -11,27 +11,42 @@ import com.google.protobuf.Message
 import akka.actor.Address
 import org.jboss.netty.handler.codec.oneone.OneToOneDecoder
 
-case class Send(msg: NetworkOp)
+case class ToClient(msg: ClientOp with NetworkOp)
+case class ToServer(msg: ServerOp with NetworkOp)
 
-sealed trait ClientOp // messages sent to Player FSM
-sealed trait ServerOp // messages sent to Conductor FSM
+sealed trait ClientOp // messages sent to from Conductor to Player
+sealed trait ServerOp // messages sent to from Player to Conductor
+sealed trait CommandOp // messages sent from TestConductorExt to Conductor
 sealed trait NetworkOp // messages sent over the wire
+sealed trait UnconfirmedClientOp extends ClientOp // unconfirmed messages going to the Player
+sealed trait ConfirmedClientOp extends ClientOp
 
+/**
+ * First message of connection sets names straight.
+ */
 case class Hello(name: String, addr: Address) extends NetworkOp
-case class EnterBarrier(name: String) extends ClientOp with ServerOp with NetworkOp
-case class BarrierFailed(name: String) extends NetworkOp
-case class Throttle(node: String, target: String, direction: Direction, rateMBit: Float) extends ServerOp
-case class ThrottleMsg(target: Address, direction: Direction, rateMBit: Float) extends NetworkOp
-case class Disconnect(node: String, target: String, abort: Boolean) extends ServerOp
-case class DisconnectMsg(target: Address, abort: Boolean) extends NetworkOp
-case class Terminate(node: String, exitValueOrKill: Int) extends ServerOp
-case class TerminateMsg(exitValue: Int) extends NetworkOp
-abstract class Done extends NetworkOp
+
+case class EnterBarrier(name: String) extends ServerOp with NetworkOp
+case class BarrierResult(name: String, success: Boolean) extends UnconfirmedClientOp with NetworkOp
+
+case class Throttle(node: String, target: String, direction: Direction, rateMBit: Float) extends CommandOp
+case class ThrottleMsg(target: Address, direction: Direction, rateMBit: Float) extends ConfirmedClientOp with NetworkOp
+
+case class Disconnect(node: String, target: String, abort: Boolean) extends CommandOp
+case class DisconnectMsg(target: Address, abort: Boolean) extends ConfirmedClientOp with NetworkOp
+
+case class Terminate(node: String, exitValueOrKill: Int) extends CommandOp
+case class TerminateMsg(exitValue: Int) extends ConfirmedClientOp with NetworkOp
+
+case class GetAddress(node: String) extends ServerOp with NetworkOp
+case class AddressReply(node: String, addr: Address) extends UnconfirmedClientOp with NetworkOp
+
+abstract class Done extends ServerOp with UnconfirmedClientOp with NetworkOp
 case object Done extends Done {
   def getInstance: Done = this
 }
 
-case class Remove(node: String) extends ServerOp
+case class Remove(node: String) extends CommandOp
 
 class MsgEncoder extends OneToOneEncoder {
   def encode(ctx: ChannelHandlerContext, ch: Channel, msg: AnyRef): AnyRef = msg match {
@@ -42,8 +57,8 @@ class MsgEncoder extends OneToOneEncoder {
           w.setHello(TCP.Hello.newBuilder.setName(name).setAddress(addr))
         case EnterBarrier(name) ⇒
           w.setBarrier(TCP.EnterBarrier.newBuilder.setName(name))
-        case BarrierFailed(name) ⇒
-          w.setBarrier(TCP.EnterBarrier.newBuilder.setName(name).setFailed(true))
+        case BarrierResult(name, success) ⇒
+          w.setBarrier(TCP.EnterBarrier.newBuilder.setName(name).setStatus(success))
         case ThrottleMsg(target, dir, rate) ⇒
           w.setFailure(TCP.InjectFailure.newBuilder.setAddress(target)
             .setFailure(TCP.FailType.Throttle).setDirection(dir).setRateMBit(rate))
@@ -52,6 +67,10 @@ class MsgEncoder extends OneToOneEncoder {
             .setFailure(if (abort) TCP.FailType.Abort else TCP.FailType.Disconnect))
         case TerminateMsg(exitValue) ⇒
           w.setFailure(TCP.InjectFailure.newBuilder.setFailure(TCP.FailType.Shutdown).setExitValue(exitValue))
+        case GetAddress(node) ⇒
+          w.setAddr(TCP.AddressRequest.newBuilder.setNode(node))
+        case AddressReply(node, addr) ⇒
+          w.setAddr(TCP.AddressRequest.newBuilder.setNode(node).setAddr(addr))
         case _: Done ⇒
           w.setDone("")
       }
@@ -68,7 +87,7 @@ class MsgDecoder extends OneToOneDecoder {
         Hello(h.getName, h.getAddress)
       } else if (w.hasBarrier) {
         val barrier = w.getBarrier
-        if (barrier.hasFailed && barrier.getFailed) BarrierFailed(barrier.getName)
+        if (barrier.hasStatus) BarrierResult(barrier.getName, barrier.getStatus)
         else EnterBarrier(w.getBarrier.getName)
       } else if (w.hasFailure) {
         val f = w.getFailure
@@ -79,6 +98,10 @@ class MsgDecoder extends OneToOneDecoder {
           case FT.Disconnect ⇒ DisconnectMsg(f.getAddress, false)
           case FT.Shutdown   ⇒ TerminateMsg(f.getExitValue)
         }
+      } else if (w.hasAddr) {
+        val a = w.getAddr
+        if (a.hasAddr) AddressReply(a.getNode, a.getAddr)
+        else GetAddress(a.getNode)
       } else if (w.hasDone) {
         Done
       } else {
diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Extension.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Extension.scala
index 5d7826c60c..7f6b576128 100644
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Extension.scala
+++ b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Extension.scala
@@ -38,13 +38,13 @@ class TestConductorExt(val system: ExtendedActorSystem) extends Extension with C
   object Settings {
     val config = system.settings.config
 
+    val ConnectTimeout = Duration(config.getMilliseconds("akka.testconductor.connect-timeout"), MILLISECONDS)
+    val ClientReconnects = config.getInt("akka.testconductor.client-reconnects")
+    val ReconnectBackoff = Duration(config.getMilliseconds("akka.testconductor.reconnect-backoff"), MILLISECONDS)
+
     implicit val BarrierTimeout = Timeout(Duration(config.getMilliseconds("akka.testconductor.barrier-timeout"), MILLISECONDS))
     implicit val QueryTimeout = Timeout(Duration(config.getMilliseconds("akka.testconductor.query-timeout"), MILLISECONDS))
     val PacketSplitThreshold = Duration(config.getMilliseconds("akka.testconductor.packet-split-threshold"), MILLISECONDS)
-
-    val name = config.getString("akka.testconductor.name")
-    val host = config.getString("akka.testconductor.host")
-    val port = config.getInt("akka.testconductor.port")
   }
 
   val transport = system.provider.asInstanceOf[RemoteActorRefProvider].transport
diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Player.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Player.scala
index a82a090b23..27a2487364 100644
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Player.scala
+++ b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Player.scala
@@ -20,6 +20,13 @@ import akka.event.LoggingAdapter
 import akka.actor.PoisonPill
 import akka.event.Logging
 import akka.dispatch.Future
+import java.net.InetSocketAddress
+import akka.actor.Address
+import org.jboss.netty.channel.ExceptionEvent
+import org.jboss.netty.channel.WriteCompletionEvent
+import java.net.ConnectException
+import akka.util.Deadline
+import akka.actor.Scheduler
 
 /**
  * The Player is the client component of the
@@ -43,13 +50,13 @@ trait Player { this: TestConductorExt ⇒
    * this is a first barrier in itself). The number of expected participants is
    * set in [[akka.remote.testconductor.Conductor]]`.startController()`.
    */
-  def startClient(port: Int): Future[Done] = {
+  def startClient(name: String, controllerAddr: InetSocketAddress): Future[Done] = {
     import ClientFSM._
     import akka.actor.FSM._
     import Settings.BarrierTimeout
 
     if (_client ne null) throw new IllegalStateException("TestConductorClient already started")
-    _client = system.actorOf(Props(new ClientFSM(port)), "TestConductorClient")
+    _client = system.actorOf(Props(new ClientFSM(name, controllerAddr)), "TestConductorClient")
     val a = system.actorOf(Props(new Actor {
       var waiting: ActorRef = _
       def receive = {
@@ -73,10 +80,18 @@ trait Player { this: TestConductorExt ⇒
     system.log.debug("entering barriers " + name.mkString("(", ", ", ")"))
     name foreach { b ⇒
       import Settings.BarrierTimeout
-      Await.result(client ? Send(EnterBarrier(b)), Duration.Inf)
+      Await.result(client ? ToServer(EnterBarrier(b)), Duration.Inf)
       system.log.debug("passed barrier {}", b)
     }
   }
+
+  /**
+   * Query remote transport address of named node.
+   */
+  def getAddressFor(name: String): Future[Address] = {
+    import Settings.BarrierTimeout
+    client ? ToServer(GetAddress(name)) mapTo
+  }
 }
 
 object ClientFSM {
@@ -86,9 +101,10 @@ object ClientFSM {
   case object Connected extends State
   case object Failed extends State
 
-  case class Data(channel: Channel, barrier: Option[(String, ActorRef)])
+  case class Data(channel: Option[Channel], runningOp: Option[(String, ActorRef)])
 
-  class ConnectionFailure(msg: String) extends RuntimeException(msg) with NoStackTrace
+  case class Connected(channel: Channel)
+  case class ConnectionFailure(msg: String) extends RuntimeException(msg) with NoStackTrace
   case object Disconnected
 }
 
@@ -101,21 +117,22 @@ object ClientFSM {
  * coordinator and react to the [[akka.remote.testconductor.Conductor]]’s
  * requests for failure injection.
  */
-class ClientFSM(port: Int) extends Actor with LoggingFSM[ClientFSM.State, ClientFSM.Data] {
+class ClientFSM(name: String, controllerAddr: InetSocketAddress) extends Actor with LoggingFSM[ClientFSM.State, ClientFSM.Data] {
   import ClientFSM._
 
   val settings = TestConductor().Settings
 
-  val handler = new PlayerHandler(self, Logging(context.system, "PlayerHandler"))
+  val handler = new PlayerHandler(controllerAddr, settings.ClientReconnects, settings.ReconnectBackoff,
+    self, Logging(context.system, "PlayerHandler"), context.system.scheduler)
 
-  startWith(Connecting, Data(RemoteConnection(Client, settings.host, port, handler), None))
+  startWith(Connecting, Data(None, None))
 
-  when(Connecting, stateTimeout = 10 seconds) {
+  when(Connecting, stateTimeout = settings.ConnectTimeout) {
     case Event(msg: ClientOp, _) ⇒
       stay replying Status.Failure(new IllegalStateException("not connected yet"))
-    case Event(Connected, d @ Data(channel, _)) ⇒
-      channel.write(Hello(settings.name, TestConductor().address))
-      goto(AwaitDone)
+    case Event(Connected(channel), _) ⇒
+      channel.write(Hello(name, TestConductor().address))
+      goto(AwaitDone) using Data(Some(channel), None)
     case Event(_: ConnectionFailure, _) ⇒
       goto(Failed)
     case Event(StateTimeout, _) ⇒
@@ -130,7 +147,7 @@ class ClientFSM(port: Int) extends Actor with LoggingFSM[ClientFSM.State, Client
     case Event(msg: NetworkOp, _) ⇒
       log.error("received {} instead of Done", msg)
       goto(Failed)
-    case Event(msg: ClientOp, _) ⇒
+    case Event(msg: ServerOp, _) ⇒
       stay replying Status.Failure(new IllegalStateException("not connected yet"))
     case Event(StateTimeout, _) ⇒
       log.error("connect timeout to TestConductor")
@@ -141,44 +158,63 @@ class ClientFSM(port: Int) extends Actor with LoggingFSM[ClientFSM.State, Client
     case Event(Disconnected, _) ⇒
       log.info("disconnected from TestConductor")
       throw new ConnectionFailure("disconnect")
-    case Event(Send(msg: EnterBarrier), Data(channel, None)) ⇒
+    case Event(ToServer(Done), Data(Some(channel), _)) ⇒
+      channel.write(Done)
+      stay
+    case Event(ToServer(msg), d @ Data(Some(channel), None)) ⇒
       channel.write(msg)
-      stay using Data(channel, Some(msg.name, sender))
-    case Event(Send(d: Done), Data(channel, _)) ⇒
-      channel.write(d)
-      stay
-    case Event(Send(x), _) ⇒
-      log.warning("cannot send message {}", x)
-      stay
-    case Event(EnterBarrier(b), Data(channel, Some((barrier, sender)))) ⇒
-      if (b != barrier) {
-        sender ! Status.Failure(new RuntimeException("wrong barrier " + b + " received while waiting for " + barrier))
-      } else {
-        sender ! b
-      }
-      stay using Data(channel, None)
-    case Event(BarrierFailed(b), Data(channel, Some((_, sender)))) ⇒
-      sender ! Status.Failure(new RuntimeException("barrier failed: " + b))
-      stay using Data(channel, None)
-    case Event(ThrottleMsg(target, dir, rate), _) ⇒
-      import settings.QueryTimeout
-      import context.dispatcher
-      TestConductor().failureInjectors.get(target.copy(system = "")) match {
-        case null ⇒ log.warning("cannot throttle unknown address {}", target)
-        case inj ⇒
-          Future.sequence(inj.refs(dir) map (_ ? NetworkFailureInjector.SetRate(rate))) map (_ ⇒ Send(Done)) pipeTo self
+      val token = msg match {
+        case EnterBarrier(barrier) ⇒ barrier
+        case GetAddress(node)      ⇒ node
       }
+      stay using d.copy(runningOp = Some(token, sender))
+    case Event(ToServer(op), Data(channel, Some((token, _)))) ⇒
+      log.error("cannot write {} while waiting for {}", op, token)
       stay
-    case Event(DisconnectMsg(target, abort), _) ⇒
-      import settings.QueryTimeout
-      TestConductor().failureInjectors.get(target.copy(system = "")) match {
-        case null ⇒ log.warning("cannot disconnect unknown address {}", target)
-        case inj  ⇒ inj.sender ? NetworkFailureInjector.Disconnect(abort) map (_ ⇒ Send(Done)) pipeTo self
+    case Event(op: ClientOp, d @ Data(Some(channel), runningOp)) ⇒
+      op match {
+        case BarrierResult(b, success) ⇒
+          runningOp match {
+            case Some((barrier, requester)) ⇒
+              if (b != barrier) {
+                requester ! Status.Failure(new RuntimeException("wrong barrier " + b + " received while waiting for " + barrier))
+              } else if (!success) {
+                requester ! Status.Failure(new RuntimeException("barrier failed: " + b))
+              } else {
+                requester ! b
+              }
+            case None ⇒
+              log.warning("did not expect {}", op)
+          }
+          stay using d.copy(runningOp = None)
+        case AddressReply(node, addr) ⇒
+          runningOp match {
+            case Some((_, requester)) ⇒
+              requester ! addr
+            case None ⇒
+              log.warning("did not expect {}", op)
+          }
+          stay using d.copy(runningOp = None)
+        case ThrottleMsg(target, dir, rate) ⇒
+          import settings.QueryTimeout
+          import context.dispatcher
+          TestConductor().failureInjectors.get(target.copy(system = "")) match {
+            case null ⇒ log.warning("cannot throttle unknown address {}", target)
+            case inj ⇒
+              Future.sequence(inj.refs(dir) map (_ ? NetworkFailureInjector.SetRate(rate))) map (_ ⇒ ToServer(Done)) pipeTo self
+          }
+          stay
+        case DisconnectMsg(target, abort) ⇒
+          import settings.QueryTimeout
+          TestConductor().failureInjectors.get(target.copy(system = "")) match {
+            case null ⇒ log.warning("cannot disconnect unknown address {}", target)
+            case inj  ⇒ inj.sender ? NetworkFailureInjector.Disconnect(abort) map (_ ⇒ ToServer(Done)) pipeTo self
+          }
+          stay
+        case TerminateMsg(exit) ⇒
+          System.exit(exit)
+          stay // needed because Java doesn’t have Nothing
       }
-      stay
-    case Event(TerminateMsg(exit), _) ⇒
-      System.exit(exit)
-      stay // needed because Java doesn’t have Nothing
   }
 
   when(Failed) {
@@ -190,7 +226,7 @@ class ClientFSM(port: Int) extends Actor with LoggingFSM[ClientFSM.State, Client
   }
 
   onTermination {
-    case StopEvent(_, _, Data(channel, _)) ⇒
+    case StopEvent(_, _, Data(Some(channel), _)) ⇒
       channel.close()
   }
 
@@ -201,14 +237,46 @@ class ClientFSM(port: Int) extends Actor with LoggingFSM[ClientFSM.State, Client
 /**
  * This handler only forwards messages received from the conductor to the [[akka.remote.testconductor.ClientFSM]].
  */
-class PlayerHandler(fsm: ActorRef, log: LoggingAdapter) extends SimpleChannelUpstreamHandler {
+class PlayerHandler(
+  server: InetSocketAddress,
+  private var reconnects: Int,
+  backoff: Duration,
+  fsm: ActorRef,
+  log: LoggingAdapter,
+  scheduler: Scheduler)
+  extends SimpleChannelUpstreamHandler {
 
   import ClientFSM._
 
+  reconnect()
+
+  var nextAttempt: Deadline = _
+
+  override def channelOpen(ctx: ChannelHandlerContext, event: ChannelStateEvent) = log.debug("channel {} open", event.getChannel)
+  override def channelClosed(ctx: ChannelHandlerContext, event: ChannelStateEvent) = log.debug("channel {} closed", event.getChannel)
+  override def channelBound(ctx: ChannelHandlerContext, event: ChannelStateEvent) = log.debug("channel {} bound", event.getChannel)
+  override def channelUnbound(ctx: ChannelHandlerContext, event: ChannelStateEvent) = log.debug("channel {} unbound", event.getChannel)
+  override def writeComplete(ctx: ChannelHandlerContext, event: WriteCompletionEvent) = log.debug("channel {} written {}", event.getChannel, event.getWrittenAmount)
+
+  override def exceptionCaught(ctx: ChannelHandlerContext, event: ExceptionEvent) = {
+    log.debug("channel {} exception {}", event.getChannel, event.getCause)
+    event.getCause match {
+      case c: ConnectException if reconnects > 0 ⇒
+        reconnects -= 1
+        scheduler.scheduleOnce(nextAttempt.timeLeft)(reconnect())
+      case e ⇒ fsm ! ConnectionFailure(e.getMessage)
+    }
+  }
+
+  private def reconnect(): Unit = {
+    nextAttempt = Deadline.now + backoff
+    RemoteConnection(Client, server, this)
+  }
+
   override def channelConnected(ctx: ChannelHandlerContext, event: ChannelStateEvent) = {
-    val channel = event.getChannel
-    log.debug("connected to {}", getAddrString(channel))
-    fsm ! Connected
+    val ch = event.getChannel
+    log.debug("connected to {}", getAddrString(ch))
+    fsm ! Connected(ch)
   }
 
   override def channelDisconnected(ctx: ChannelHandlerContext, event: ChannelStateEvent) = {
diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/RemoteConnection.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/RemoteConnection.scala
index b2f4baebbb..5b1c454b0c 100644
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/RemoteConnection.scala
+++ b/akka-remote-tests/src/main/scala/akka/remote/testconductor/RemoteConnection.scala
@@ -27,8 +27,7 @@ case object Client extends Role
 case object Server extends Role
 
 object RemoteConnection {
-  def apply(role: Role, host: String, port: Int, handler: ChannelUpstreamHandler): Channel = {
-    val sockaddr = new InetSocketAddress(host, port)
+  def apply(role: Role, sockaddr: InetSocketAddress, handler: ChannelUpstreamHandler): Channel = {
     role match {
       case Client ⇒
         val socketfactory = new NioClientSocketChannelFactory(Executors.newCachedThreadPool, Executors.newCachedThreadPool)
diff --git a/akka-remote-tests/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala b/akka-remote-tests/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
index 512757c130..39d25981aa 100644
--- a/akka-remote-tests/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
+++ b/akka-remote-tests/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
@@ -10,14 +10,15 @@ import akka.dispatch.Await.Awaitable
 import akka.util.Duration
 import akka.util.duration._
 import akka.testkit.ImplicitSender
+import java.net.InetSocketAddress
+import java.net.InetAddress
+import akka.remote.testkit.MultiNodeSpec
 
 object TestConductorMultiJvmSpec extends AbstractRemoteActorMultiJvmSpec {
   override def NrOfNodes = 2
   override def commonConfig = ConfigFactory.parseString("""
     akka.loglevel = DEBUG
-    akka.actor.provider = akka.remote.RemoteActorRefProvider
     akka.remote {
-      transport = akka.remote.testconductor.TestConductorTransport
       log-received-messages = on
       log-sent-messages = on
     }
@@ -25,87 +26,96 @@ object TestConductorMultiJvmSpec extends AbstractRemoteActorMultiJvmSpec {
       receive = on
       fsm = on
     }
-    akka.testconductor {
-      host = localhost
-      port = 4712
-    }
   """)
-  def nameConfig(n: Int) = ConfigFactory.parseString("akka.testconductor.name = node" + n).withFallback(nodeConfigs(n))
+}
 
-  implicit def awaitHelper[T](w: Awaitable[T]) = new AwaitHelper(w)
-  class AwaitHelper[T](w: Awaitable[T]) {
-    def await: T = Await.result(w, Duration.Inf)
+object H {
+  def apply(x: Int) = {
+    System.setProperty("multinode.hosts", "localhost,localhost")
+    System.setProperty("multinode.index", x.toString)
   }
 }
 
-class TestConductorMultiJvmNode1 extends AkkaRemoteSpec(TestConductorMultiJvmSpec.nameConfig(0)) {
+class TestConductorMultiJvmNode1 extends { val dummy = H(0) } with TestConductorSpec
+class TestConductorMultiJvmNode2 extends { val dummy = H(1) } with TestConductorSpec
 
-  import TestConductorMultiJvmSpec._
+class TestConductorSpec extends MultiNodeSpec(TestConductorMultiJvmSpec.commonConfig) with ImplicitSender {
 
-  val nodes = NrOfNodes
+  def initialParticipants = 2
+  lazy val roles = Seq("master", "slave")
 
-  val tc = TestConductor(system)
-
-  val echo = system.actorOf(Props(new Actor {
-    def receive = {
-      case x ⇒ testActor ! x; sender ! x
-    }
-  }), "echo")
-
-  "running a test with barrier" in {
-    tc.startController(2).await
-    tc.enter("begin")
+  runOn("master") {
+    system.actorOf(Props(new Actor {
+      def receive = {
+        case x ⇒ testActor ! x; sender ! x
+      }
+    }), "echo")
   }
 
-  "throttling" in {
-    expectMsg("start")
-    tc.throttle("node1", "node0", Direction.Send, 0.01).await
-    tc.enter("throttled_send")
-    within(0.6 seconds, 2 seconds) {
-      receiveN(10) must be(0 to 9)
+  val echo = system.actorFor(node("master") / "user" / "echo")
+
+  "A TestConductor" must {
+
+    "enter a barrier" in {
+      testConductor.enter("name")
     }
-    tc.enter("throttled_send2")
-    tc.throttle("node1", "node0", Direction.Send, -1).await
-    
-    tc.throttle("node1", "node0", Direction.Receive, 0.01).await
-    tc.enter("throttled_recv")
-    receiveN(10, 500 millis) must be(10 to 19)
-    tc.enter("throttled_recv2")
-    tc.throttle("node1", "node0", Direction.Receive, -1).await
-  }
-}
 
-class TestConductorMultiJvmNode2 extends AkkaRemoteSpec(TestConductorMultiJvmSpec.nameConfig(1)) with ImplicitSender {
+    "support throttling of network connections" in {
 
-  import TestConductorMultiJvmSpec._
+      runOn("slave") {
+        // start remote network connection so that it can be throttled
+        echo ! "start"
+      }
 
-  val nodes = NrOfNodes
+      expectMsg("start")
 
-  val tc = TestConductor(system)
-  
-  val echo = system.actorFor("akka://" + akkaSpec(0) + "/user/echo")
+      runOn("master") {
+        testConductor.throttle("slave", "master", Direction.Send, rateMBit = 0.01).await
+      }
 
-  "running a test with barrier" in {
-    tc.startClient(4712).await
-    tc.enter("begin")
-  }
+      testConductor.enter("throttled_send")
 
-  "throttling" in {
-    echo ! "start"
-    expectMsg("start")
-    tc.enter("throttled_send")
-    for (i <- 0 to 9) echo ! i
-    expectMsg(500 millis, 0)
-    within(0.6 seconds, 2 seconds) {
-      receiveN(9) must be(1 to 9)
+      runOn("slave") {
+        for (i ← 0 to 9) echo ! i
+      }
+
+      within(0.6 seconds, 2 seconds) {
+        expectMsg(500 millis, 0)
+        receiveN(9) must be(1 to 9)
+      }
+
+      testConductor.enter("throttled_send2")
+
+      runOn("master") {
+        testConductor.throttle("slave", "master", Direction.Send, -1).await
+        testConductor.throttle("slave", "master", Direction.Receive, rateMBit = 0.01).await
+      }
+
+      testConductor.enter("throttled_recv")
+
+      runOn("slave") {
+        for (i ← 10 to 19) echo ! i
+      }
+
+      val (min, max) =
+        ifNode("master") {
+          (0 seconds, 500 millis)
+        } {
+          (0.6 seconds, 2 seconds)
+        }
+
+      within(min, max) {
+        expectMsg(500 millis, 10)
+        receiveN(9) must be(11 to 19)
+      }
+
+      testConductor.enter("throttled_recv2")
+
+      runOn("master") {
+        testConductor.throttle("slave", "master", Direction.Receive, -1).await
+      }
     }
-    tc.enter("throttled_send2", "throttled_recv")
-    for (i <- 10 to 19) echo ! i
-    expectMsg(500 millis, 10)
-    within(0.6 seconds, 2 seconds) {
-      receiveN(9) must be(11 to 19)
-    }
-    tc.enter("throttled_recv2")
+
   }
 
 }
diff --git a/akka-remote-tests/src/test/scala/akka/remote/testconductor/BarrierSpec.scala b/akka-remote-tests/src/test/scala/akka/remote/testconductor/BarrierSpec.scala
index f0b668d1ed..aa14b93f9d 100644
--- a/akka-remote-tests/src/test/scala/akka/remote/testconductor/BarrierSpec.scala
+++ b/akka-remote-tests/src/test/scala/akka/remote/testconductor/BarrierSpec.scala
@@ -16,6 +16,8 @@ import akka.testkit.TestProbe
 import akka.util.duration._
 import akka.event.Logging
 import org.scalatest.BeforeAndAfterEach
+import java.net.InetSocketAddress
+import java.net.InetAddress
 
 object BarrierSpec {
   case class Failed(ref: ActorRef, thr: Throwable)
@@ -68,7 +70,7 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
     "fail entering barrier when nobody registered" in {
       val b = getBarrier()
       b ! EnterBarrier("b")
-      expectMsg(Send(BarrierFailed("b")))
+      expectMsg(ToClient(BarrierResult("b", false)))
     }
 
     "enter barrier" in {
@@ -80,8 +82,8 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
       noMsg(a, b)
       within(1 second) {
         b.send(barrier, EnterBarrier("bar"))
-        a.expectMsg(Send(EnterBarrier("bar")))
-        b.expectMsg(Send(EnterBarrier("bar")))
+        a.expectMsg(ToClient(BarrierResult("bar", true)))
+        b.expectMsg(ToClient(BarrierResult("bar", true)))
       }
     }
 
@@ -96,9 +98,9 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
       noMsg(a, b, c)
       within(1 second) {
         c.send(barrier, EnterBarrier("bar"))
-        a.expectMsg(Send(EnterBarrier("bar")))
-        b.expectMsg(Send(EnterBarrier("bar")))
-        c.expectMsg(Send(EnterBarrier("bar")))
+        a.expectMsg(ToClient(BarrierResult("bar", true)))
+        b.expectMsg(ToClient(BarrierResult("bar", true)))
+        c.expectMsg(ToClient(BarrierResult("bar", true)))
       }
     }
 
@@ -115,7 +117,7 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
       noMsg(a, b, c)
       b.within(1 second) {
         barrier ! RemoveClient("c")
-        b.expectMsg(Send(EnterBarrier("bar")))
+        b.expectMsg(ToClient(BarrierResult("bar", true)))
       }
       barrier ! ClientDisconnected("c")
       expectNoMsg(1 second)
@@ -129,7 +131,7 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
       a.send(barrier, EnterBarrier("bar"))
       barrier ! RemoveClient("a")
       b.send(barrier, EnterBarrier("foo"))
-      b.expectMsg(Send(EnterBarrier("foo")))
+      b.expectMsg(ToClient(BarrierResult("foo", true)))
     }
 
     "fail barrier with disconnecing node" in {
@@ -184,7 +186,7 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
       expectMsg(Failed(barrier, BarrierEmpty(Data(Set(), "", Nil), "no client to remove")))
       barrier ! NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
       a.send(barrier, EnterBarrier("right"))
-      a.expectMsg(Send(BarrierFailed("right")))
+      a.expectMsg(ToClient(BarrierResult("right", false)))
     }
 
     "fail after barrier timeout" in {
@@ -223,7 +225,7 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
     "register clients and remove them" in {
       val b = getController(1)
       b ! NodeInfo("a", AddressFromURIString("akka://sys"), testActor)
-      expectMsg(Send(Done))
+      expectMsg(ToClient(Done))
       b ! Remove("b")
       b ! Remove("a")
       EventFilter[BarrierEmpty](occurrences = 1) intercept {
@@ -234,7 +236,7 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
     "register clients and disconnect them" in {
       val b = getController(1)
       b ! NodeInfo("a", AddressFromURIString("akka://sys"), testActor)
-      expectMsg(Send(Done))
+      expectMsg(ToClient(Done))
       b ! ClientDisconnected("b")
       EventFilter[ClientLost](occurrences = 1) intercept {
         b ! ClientDisconnected("a")
@@ -247,7 +249,7 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
     "fail entering barrier when nobody registered" in {
       val b = getController(0)
       b ! EnterBarrier("b")
-      expectMsg(Send(BarrierFailed("b")))
+      expectMsg(ToClient(BarrierResult("b", false)))
     }
 
     "enter barrier" in {
@@ -255,14 +257,14 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
       val a, b = TestProbe()
       barrier ! NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
       barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
-      a.expectMsg(Send(Done))
-      b.expectMsg(Send(Done))
+      a.expectMsg(ToClient(Done))
+      b.expectMsg(ToClient(Done))
       a.send(barrier, EnterBarrier("bar"))
       noMsg(a, b)
       within(1 second) {
         b.send(barrier, EnterBarrier("bar"))
-        a.expectMsg(Send(EnterBarrier("bar")))
-        b.expectMsg(Send(EnterBarrier("bar")))
+        a.expectMsg(ToClient(BarrierResult("bar", true)))
+        b.expectMsg(ToClient(BarrierResult("bar", true)))
       }
     }
 
@@ -271,18 +273,18 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
       val a, b, c = TestProbe()
       barrier ! NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
       barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
-      a.expectMsg(Send(Done))
-      b.expectMsg(Send(Done))
+      a.expectMsg(ToClient(Done))
+      b.expectMsg(ToClient(Done))
       a.send(barrier, EnterBarrier("bar"))
       barrier ! NodeInfo("c", AddressFromURIString("akka://sys"), c.ref)
-      c.expectMsg(Send(Done))
+      c.expectMsg(ToClient(Done))
       b.send(barrier, EnterBarrier("bar"))
       noMsg(a, b, c)
       within(1 second) {
         c.send(barrier, EnterBarrier("bar"))
-        a.expectMsg(Send(EnterBarrier("bar")))
-        b.expectMsg(Send(EnterBarrier("bar")))
-        c.expectMsg(Send(EnterBarrier("bar")))
+        a.expectMsg(ToClient(BarrierResult("bar", true)))
+        b.expectMsg(ToClient(BarrierResult("bar", true)))
+        c.expectMsg(ToClient(BarrierResult("bar", true)))
       }
     }
 
@@ -292,9 +294,9 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
       barrier ! NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
       barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
       barrier ! NodeInfo("c", AddressFromURIString("akka://sys"), c.ref)
-      a.expectMsg(Send(Done))
-      b.expectMsg(Send(Done))
-      c.expectMsg(Send(Done))
+      a.expectMsg(ToClient(Done))
+      b.expectMsg(ToClient(Done))
+      c.expectMsg(ToClient(Done))
       a.send(barrier, EnterBarrier("bar"))
       b.send(barrier, EnterBarrier("bar"))
       barrier ! Remove("a")
@@ -302,7 +304,7 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
       noMsg(a, b, c)
       b.within(1 second) {
         barrier ! Remove("c")
-        b.expectMsg(Send(EnterBarrier("bar")))
+        b.expectMsg(ToClient(BarrierResult("bar", true)))
       }
       barrier ! ClientDisconnected("c")
       expectNoMsg(1 second)
@@ -313,12 +315,12 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
       val a, b = TestProbe()
       barrier ! NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
       barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
-      a.expectMsg(Send(Done))
-      b.expectMsg(Send(Done))
+      a.expectMsg(ToClient(Done))
+      b.expectMsg(ToClient(Done))
       a.send(barrier, EnterBarrier("bar"))
       barrier ! Remove("a")
       b.send(barrier, EnterBarrier("foo"))
-      b.expectMsg(Send(EnterBarrier("foo")))
+      b.expectMsg(ToClient(BarrierResult("foo", true)))
     }
 
     "fail barrier with disconnecing node" in {
@@ -327,15 +329,15 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
       val nodeA = NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
       barrier ! nodeA
       barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
-      a.expectMsg(Send(Done))
-      b.expectMsg(Send(Done))
+      a.expectMsg(ToClient(Done))
+      b.expectMsg(ToClient(Done))
       a.send(barrier, EnterBarrier("bar"))
       barrier ! ClientDisconnected("unknown")
       noMsg(a)
       EventFilter[ClientLost](occurrences = 1) intercept {
         barrier ! ClientDisconnected("b")
       }
-      a.expectMsg(Send(BarrierFailed("bar")))
+      a.expectMsg(ToClient(BarrierResult("bar", false)))
     }
 
     "fail barrier with disconnecing node who already arrived" in {
@@ -346,15 +348,15 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
       barrier ! nodeA
       barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
       barrier ! nodeC
-      a.expectMsg(Send(Done))
-      b.expectMsg(Send(Done))
-      c.expectMsg(Send(Done))
+      a.expectMsg(ToClient(Done))
+      b.expectMsg(ToClient(Done))
+      c.expectMsg(ToClient(Done))
       a.send(barrier, EnterBarrier("bar"))
       b.send(barrier, EnterBarrier("bar"))
       EventFilter[ClientLost](occurrences = 1) intercept {
         barrier ! ClientDisconnected("b")
       }
-      a.expectMsg(Send(BarrierFailed("bar")))
+      a.expectMsg(ToClient(BarrierResult("bar", false)))
     }
 
     "fail when entering wrong barrier" in {
@@ -364,14 +366,14 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
       barrier ! nodeA
       val nodeB = NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
       barrier ! nodeB
-      a.expectMsg(Send(Done))
-      b.expectMsg(Send(Done))
+      a.expectMsg(ToClient(Done))
+      b.expectMsg(ToClient(Done))
       a.send(barrier, EnterBarrier("bar"))
       EventFilter[WrongBarrier](occurrences = 1) intercept {
         b.send(barrier, EnterBarrier("foo"))
       }
-      a.expectMsg(Send(BarrierFailed("bar")))
-      b.expectMsg(Send(BarrierFailed("foo")))
+      a.expectMsg(ToClient(BarrierResult("bar", false)))
+      b.expectMsg(ToClient(BarrierResult("foo", false)))
     }
 
     "not really fail after barrier timeout" in {
@@ -381,15 +383,15 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
       val nodeB = NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
       barrier ! nodeA
       barrier ! nodeB
-      a.expectMsg(Send(Done))
-      b.expectMsg(Send(Done))
+      a.expectMsg(ToClient(Done))
+      b.expectMsg(ToClient(Done))
       a.send(barrier, EnterBarrier("right"))
       EventFilter[BarrierTimeout](occurrences = 1) intercept {
         Thread.sleep(5000)
       }
       b.send(barrier, EnterBarrier("right"))
-      a.expectMsg(Send(EnterBarrier("right")))
-      b.expectMsg(Send(EnterBarrier("right")))
+      a.expectMsg(ToClient(BarrierResult("right", true)))
+      b.expectMsg(ToClient(BarrierResult("right", true)))
     }
 
     "fail if a node registers twice" in {
@@ -401,8 +403,8 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
       EventFilter[DuplicateNode](occurrences = 1) intercept {
         controller ! nodeB
       }
-      a.expectMsg(Send(BarrierFailed("initial startup")))
-      b.expectMsg(Send(BarrierFailed("initial startup")))
+      a.expectMsg(ToClient(BarrierResult("initial startup", false)))
+      b.expectMsg(ToClient(BarrierResult("initial startup", false)))
     }
 
     "fail subsequent barriers if a node registers twice" in {
@@ -411,13 +413,13 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
       val nodeA = NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
       val nodeB = NodeInfo("a", AddressFromURIString("akka://sys"), b.ref)
       controller ! nodeA
-      a.expectMsg(Send(Done))
+      a.expectMsg(ToClient(Done))
       EventFilter[DuplicateNode](occurrences = 1) intercept {
         controller ! nodeB
-        b.expectMsg(Send(BarrierFailed("initial startup")))
+        b.expectMsg(ToClient(BarrierResult("initial startup", false)))
       }
       a.send(controller, EnterBarrier("x"))
-      a.expectMsg(Send(BarrierFailed("x")))
+      a.expectMsg(ToClient(BarrierResult("x", false)))
     }
 
     "finally have no failure messages left" in {
@@ -428,13 +430,13 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
 
   private def getController(participants: Int): ActorRef = {
     system.actorOf(Props(new Actor {
-      val controller = context.actorOf(Props(new Controller(participants)))
-      controller ! GetPort
+      val controller = context.actorOf(Props(new Controller(participants, new InetSocketAddress(InetAddress.getLocalHost, 0))))
+      controller ! GetSockAddr
       override def supervisorStrategy = OneForOneStrategy() {
         case x ⇒ testActor ! Failed(controller, x); SupervisorStrategy.Restart
       }
       def receive = {
-        case x: Int ⇒ testActor ! controller
+        case x: InetSocketAddress ⇒ testActor ! controller
       }
     }))
     expectMsgType[ActorRef]
diff --git a/akka-remote-tests/src/test/scala/akka/remote/testconductor/ControllerSpec.scala b/akka-remote-tests/src/test/scala/akka/remote/testconductor/ControllerSpec.scala
index db0e3cfe69..c4e0ca6cd0 100644
--- a/akka-remote-tests/src/test/scala/akka/remote/testconductor/ControllerSpec.scala
+++ b/akka-remote-tests/src/test/scala/akka/remote/testconductor/ControllerSpec.scala
@@ -8,6 +8,8 @@ import akka.actor.Props
 import akka.testkit.ImplicitSender
 import akka.remote.testconductor.Controller.NodeInfo
 import akka.actor.AddressFromURIString
+import java.net.InetSocketAddress
+import java.net.InetAddress
 
 object ControllerSpec {
   val config = """
@@ -24,11 +26,11 @@ class ControllerSpec extends AkkaSpec(ControllerSpec.config) with ImplicitSender
   "A Controller" must {
 
     "publish its nodes" in {
-      val c = system.actorOf(Props(new Controller(1)))
+      val c = system.actorOf(Props(new Controller(1, new InetSocketAddress(InetAddress.getLocalHost, 0))))
       c ! NodeInfo("a", AddressFromURIString("akka://sys"), testActor)
-      expectMsg(Send(Done))
+      expectMsg(ToClient(Done))
       c ! NodeInfo("b", AddressFromURIString("akka://sys"), testActor)
-      expectMsg(Send(Done))
+      expectMsg(ToClient(Done))
       c ! Controller.GetNodes
       expectMsgType[Iterable[String]].toSet must be(Set("a", "b"))
     }
diff --git a/akka-remote-tests/src/test/scala/akka/remote/testkit/MultiNodeSpec.scala b/akka-remote-tests/src/test/scala/akka/remote/testkit/MultiNodeSpec.scala
new file mode 100644
index 0000000000..7acde4eac9
--- /dev/null
+++ b/akka-remote-tests/src/test/scala/akka/remote/testkit/MultiNodeSpec.scala
@@ -0,0 +1,157 @@
+/**
+ *  Copyright (C) 2009-2012 Typesafe Inc. 
+ */
+package akka.remote.testkit
+
+import akka.testkit.AkkaSpec
+import akka.actor.ActorSystem
+import akka.remote.testconductor.TestConductor
+import java.net.InetAddress
+import java.net.InetSocketAddress
+import akka.remote.testconductor.TestConductorExt
+import com.typesafe.config.Config
+import com.typesafe.config.ConfigFactory
+import akka.dispatch.Await.Awaitable
+import akka.dispatch.Await
+import akka.util.Duration
+import akka.actor.ActorPath
+import akka.actor.RootActorPath
+
+object MultiNodeSpec {
+
+  /**
+   * Names (or IP addresses; must be resolvable using InetAddress.getByName)
+   * of all nodes taking part in this test, including symbolic name and host
+   * definition:
+   *
+   * {{{
+   * -D"multinode.hosts=host1@workerA.example.com,host2@workerB.example.com"
+   * }}}
+   */
+  val nodeNames: Seq[String] = Vector.empty ++ (
+    Option(System.getProperty("multinode.hosts")) getOrElse
+    (throw new IllegalStateException("need system property multinode.hosts to be set")) split ",")
+
+  require(nodeNames != List(""), "multinode.hosts must not be empty")
+
+  /**
+   * Index of this node in the nodeNames / nodeAddresses lists. The TestConductor
+   * is started in “controller” mode on selfIndex 0, i.e. there you can inject
+   * failures and shutdown other nodes etc.
+   */
+  val selfIndex = Option(Integer.getInteger("multinode.index")) getOrElse
+    (throw new IllegalStateException("need system property multinode.index to be set"))
+
+  require(selfIndex >= 0 && selfIndex < nodeNames.size, "selfIndex out of bounds: " + selfIndex)
+
+  val nodeConfig = AkkaSpec.mapToConfig(Map(
+    "akka.actor.provider" -> "akka.remote.RemoteActorRefProvider",
+    "akka.remote.transport" -> "akka.remote.testconductor.TestConductorTransport",
+    "akka.remote.netty.hostname" -> nodeNames(selfIndex),
+    "akka.remote.netty.port" -> 0))
+
+}
+
+abstract class MultiNodeSpec(_system: ActorSystem) extends AkkaSpec(_system) {
+
+  import MultiNodeSpec._
+
+  def this(config: Config) = this(ActorSystem(AkkaSpec.getCallerName,
+    MultiNodeSpec.nodeConfig.withFallback(config.withFallback(AkkaSpec.testConf))))
+
+  def this(s: String) = this(ConfigFactory.parseString(s))
+
+  def this(configMap: Map[String, _]) = this(AkkaSpec.mapToConfig(configMap))
+
+  def this() = this(AkkaSpec.testConf)
+
+  /*
+   * Test Class Interface
+   */
+
+  /**
+   * TO BE DEFINED BY USER: Defines the number of participants required for starting the test. This
+   * might not be equals to the number of nodes available to the test.
+   *
+   * Must be a `def`:
+   * {{{
+   * def initialParticipants = 5
+   * }}}
+   */
+  def initialParticipants: Int
+  require(initialParticipants > 0, "initialParticipants must be a 'def' or early initializer, and it must be greater zero")
+  require(initialParticipants <= nodeNames.size, "not enough nodes to run this test")
+
+  /**
+   * Access to the barriers, failure injection, etc. The extension will have
+   * been started either in Conductor or Player mode when the constructor of
+   * MultiNodeSpec finishes, i.e. do not call the start*() methods yourself!
+   */
+  val testConductor: TestConductorExt = TestConductor(system)
+
+  /**
+   * TO BE DEFINED BY USER: The test class must define a set of role names to
+   * be used throughout the run, e.g. in naming nodes in failure injections.
+   * These will be mapped to the available nodes such that the first name will
+   * be the Controller, i.e. on this one you can do failure injection.
+   *
+   * Should be a lazy val due to initialization order:
+   * {{{
+   * lazy val roles = Seq("master", "slave")
+   * }}}
+   */
+  def roles: Seq[String]
+
+  require(roles.size >= initialParticipants, "not enough roles for initialParticipants")
+  require(roles.size <= nodeNames.size, "not enough nodes for number of roles")
+  require(roles.distinct.size == roles.size, "role names must be distinct")
+
+  val mySelf = {
+    if (selfIndex >= roles.size) System.exit(0)
+    roles(selfIndex)
+  }
+
+  /**
+   * Execute the given block of code only on the given nodes (names according
+   * to the `roleMap`).
+   */
+  def runOn(nodes: String*)(thunk: ⇒ Unit): Unit = {
+    if (nodes exists (_ == mySelf)) {
+      thunk
+    }
+  }
+
+  def ifNode[T](nodes: String*)(yes: ⇒ T)(no: ⇒ T): T = {
+    if (nodes exists (_ == mySelf)) yes else no
+  }
+
+  /**
+   * Query the controller for the transport address of the given node (by role name) and
+   * return that as an ActorPath for easy composition:
+   *
+   * {{{
+   * val serviceA = system.actorFor(node("master") / "user" / "serviceA")
+   * }}}
+   */
+  def node(name: String): ActorPath = RootActorPath(testConductor.getAddressFor(name).await)
+
+  /**
+   * Enrich `.await()` onto all Awaitables, using BarrierTimeout.
+   */
+  implicit def awaitHelper[T](w: Awaitable[T]) = new AwaitHelper(w)
+  class AwaitHelper[T](w: Awaitable[T]) {
+    def await: T = Await.result(w, testConductor.Settings.BarrierTimeout.duration)
+  }
+
+  /*
+   * Implementation (i.e. wait for start etc.)
+   */
+
+  private val controllerAddr = new InetSocketAddress(nodeNames(0), 4711)
+  if (selfIndex == 0) {
+    testConductor.startController(initialParticipants, roles(0), controllerAddr).await
+  } else {
+    testConductor.startClient(roles(selfIndex), controllerAddr).await
+  }
+
+}
\ No newline at end of file
diff --git a/scripts/fix-protobuf.sh b/scripts/fix-protobuf.sh
new file mode 100755
index 0000000000..b0c8831091
--- /dev/null
+++ b/scripts/fix-protobuf.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+find . -name \*.proto -print0 | xargs -0 perl -pi -e 's/\Qprivate Builder(BuilderParent parent)/private Builder(com.google.protobuf.GeneratedMessage.BuilderParent parent)/'

From 4217d639f96e25d900e40d8c2e33b5130d35f32a Mon Sep 17 00:00:00 2001
From: Roland 
Date: Fri, 18 May 2012 16:00:33 +0200
Subject: [PATCH 19/36] add utility for fixing up broken PROTOC code (and apply
 it)

---
 .../remote/testconductor/TestConductorProtocol.java  | 12 ++++++------
 scripts/fix-protobuf.sh                              |  2 +-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/akka-remote-tests/src/main/java/akka/remote/testconductor/TestConductorProtocol.java b/akka-remote-tests/src/main/java/akka/remote/testconductor/TestConductorProtocol.java
index 4ae1aae07a..99c33e6728 100644
--- a/akka-remote-tests/src/main/java/akka/remote/testconductor/TestConductorProtocol.java
+++ b/akka-remote-tests/src/main/java/akka/remote/testconductor/TestConductorProtocol.java
@@ -492,7 +492,7 @@ public final class TestConductorProtocol {
         maybeForceBuilderInitialization();
       }
       
-      private Builder(BuilderParent parent) {
+      private Builder(com.google.protobuf.GeneratedMessage.BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
       }
@@ -1397,7 +1397,7 @@ public final class TestConductorProtocol {
         maybeForceBuilderInitialization();
       }
       
-      private Builder(BuilderParent parent) {
+      private Builder(com.google.protobuf.GeneratedMessage.BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
       }
@@ -1927,7 +1927,7 @@ public final class TestConductorProtocol {
         maybeForceBuilderInitialization();
       }
       
-      private Builder(BuilderParent parent) {
+      private Builder(com.google.protobuf.GeneratedMessage.BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
       }
@@ -2377,7 +2377,7 @@ public final class TestConductorProtocol {
         maybeForceBuilderInitialization();
       }
       
-      private Builder(BuilderParent parent) {
+      private Builder(com.google.protobuf.GeneratedMessage.BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
       }
@@ -3005,7 +3005,7 @@ public final class TestConductorProtocol {
         maybeForceBuilderInitialization();
       }
       
-      private Builder(BuilderParent parent) {
+      private Builder(com.google.protobuf.GeneratedMessage.BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
       }
@@ -3611,7 +3611,7 @@ public final class TestConductorProtocol {
         maybeForceBuilderInitialization();
       }
       
-      private Builder(BuilderParent parent) {
+      private Builder(com.google.protobuf.GeneratedMessage.BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
       }
diff --git a/scripts/fix-protobuf.sh b/scripts/fix-protobuf.sh
index b0c8831091..e53ce297ab 100755
--- a/scripts/fix-protobuf.sh
+++ b/scripts/fix-protobuf.sh
@@ -1,3 +1,3 @@
 #!/bin/bash
 
-find . -name \*.proto -print0 | xargs -0 perl -pi -e 's/\Qprivate Builder(BuilderParent parent)/private Builder(com.google.protobuf.GeneratedMessage.BuilderParent parent)/'
+find . -name \*.java -print0 | xargs -0 perl -pi -e 's/\Qprivate Builder(BuilderParent parent)/private Builder(com.google.protobuf.GeneratedMessage.BuilderParent parent)/'

From 483083708e0be1dcc226842c96bad2849024712e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jonas=20Bone=CC=81r?= 
Date: Fri, 18 May 2012 16:05:38 +0200
Subject: [PATCH 20/36] Added verification that a BalancingDispatcher can not
 be used with any kind of Router (impl + test).
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Also had to 'ignore' one test that violates this principle. Should be looked into later.

Signed-off-by: Jonas Bonér 
---
 .../ActorConfigurationVerificationSpec.scala  | 80 +++++++++++++++++++
 .../test/scala/akka/routing/ResizerSpec.scala |  3 +-
 .../src/main/scala/akka/actor/ActorCell.scala | 12 +++
 3 files changed, 94 insertions(+), 1 deletion(-)
 create mode 100644 akka-actor-tests/src/test/scala/akka/actor/ActorConfigurationVerificationSpec.scala

diff --git a/akka-actor-tests/src/test/scala/akka/actor/ActorConfigurationVerificationSpec.scala b/akka-actor-tests/src/test/scala/akka/actor/ActorConfigurationVerificationSpec.scala
new file mode 100644
index 0000000000..cdaa421a59
--- /dev/null
+++ b/akka-actor-tests/src/test/scala/akka/actor/ActorConfigurationVerificationSpec.scala
@@ -0,0 +1,80 @@
+/**
+ * Copyright (C) 2009-2012 Typesafe Inc. 
+ */
+package akka.actor
+
+import akka.testkit._
+import akka.testkit.DefaultTimeout
+import akka.testkit.TestEvent._
+import akka.dispatch.Await
+import akka.util.duration._
+import akka.routing._
+import akka.config.ConfigurationException
+import com.typesafe.config.{ Config, ConfigFactory }
+import org.scalatest.BeforeAndAfterEach
+import org.scalatest.junit.JUnitSuite
+
+object ActorConfigurationVerificationSpec {
+
+  class TestActor extends Actor {
+    def receive: Receive = {
+      case _ ⇒
+    }
+  }
+
+  val config = """
+    balancing-dispatcher {
+      type = BalancingDispatcher
+      throughput = 1
+    }
+    pinned-dispatcher {
+      executor = "thread-pool-executor"
+      type = PinnedDispatcher
+    }
+    """
+}
+
+@org.junit.runner.RunWith(classOf[org.scalatest.junit.JUnitRunner])
+class ActorConfigurationVerificationSpec extends AkkaSpec(ActorConfigurationVerificationSpec.config) with DefaultTimeout with BeforeAndAfterEach {
+  import ActorConfigurationVerificationSpec._
+
+  override def atStartup {
+    system.eventStream.publish(Mute(EventFilter[ConfigurationException]("")))
+  }
+
+  "An Actor configured with a BalancingDispatcher" must {
+    "fail verification with a ConfigurationException if also configured with a RoundRobinRouter" in {
+      intercept[ConfigurationException] {
+        system.actorOf(Props[TestActor].withDispatcher("balancing-dispatcher").withRouter(RoundRobinRouter(2)))
+      }
+    }
+    "fail verification with a ConfigurationException if also configured with a BroadcastRouter" in {
+      intercept[ConfigurationException] {
+        system.actorOf(Props[TestActor].withDispatcher("balancing-dispatcher").withRouter(BroadcastRouter(2)))
+      }
+    }
+    "fail verification with a ConfigurationException if also configured with a RandomRouter" in {
+      intercept[ConfigurationException] {
+        system.actorOf(Props[TestActor].withDispatcher("balancing-dispatcher").withRouter(RandomRouter(2)))
+      }
+    }
+    "fail verification with a ConfigurationException if also configured with a SmallestMailboxRouter" in {
+      intercept[ConfigurationException] {
+        system.actorOf(Props[TestActor].withDispatcher("balancing-dispatcher").withRouter(SmallestMailboxRouter(2)))
+      }
+    }
+    "fail verification with a ConfigurationException if also configured with a ScatterGatherFirstCompletedRouter" in {
+      intercept[ConfigurationException] {
+        system.actorOf(Props[TestActor].withDispatcher("balancing-dispatcher").withRouter(ScatterGatherFirstCompletedRouter(nrOfInstances = 2, within = 2 seconds)))
+      }
+    }
+    "not fail verification with a ConfigurationException also not configured with a Router" in {
+      system.actorOf(Props[TestActor].withDispatcher("balancing-dispatcher"))
+    }
+  }
+  "An Actor configured with a non-balancing dispatcher" must {
+    "not fail verification with a ConfigurationException if also configured with a Router" in {
+      system.actorOf(Props[TestActor].withDispatcher("pinned-dispatcher").withRouter(RoundRobinRouter(2)))
+    }
+  }
+}
diff --git a/akka-actor-tests/src/test/scala/akka/routing/ResizerSpec.scala b/akka-actor-tests/src/test/scala/akka/routing/ResizerSpec.scala
index 457c4ab411..111460e3ac 100644
--- a/akka-actor-tests/src/test/scala/akka/routing/ResizerSpec.scala
+++ b/akka-actor-tests/src/test/scala/akka/routing/ResizerSpec.scala
@@ -128,7 +128,8 @@ class ResizerSpec extends AkkaSpec(ResizerSpec.config) with DefaultTimeout with
       current.routees.size must be(2)
     }
 
-    "resize when busy" in {
+    // FIXME this test violates the rule that you can not use a BalancingDispatcher with any kind of Router - now throws a ConfigurationException in verification process
+    "resize when busy" ignore {
 
       val busy = new TestLatch(1)
 
diff --git a/akka-actor/src/main/scala/akka/actor/ActorCell.scala b/akka-actor/src/main/scala/akka/actor/ActorCell.scala
index 8c68ba3315..9cc993062f 100644
--- a/akka-actor/src/main/scala/akka/actor/ActorCell.scala
+++ b/akka-actor/src/main/scala/akka/actor/ActorCell.scala
@@ -357,7 +357,19 @@ private[akka] class ActorCell(
     case _ ⇒ true
   }
 
+  private def verifyActorConfiguration(system: ActorSystem, props: Props, actorName: String): Unit = {
+    import akka.config.ConfigurationException
+    import akka.routing.NoRouter
+    // verify that a BalancingDispatcher is not used with a Router
+    if (system.dispatchers.lookup(props.dispatcher).isInstanceOf[BalancingDispatcher] && props.routerConfig != NoRouter)
+      throw new ConfigurationException(
+        "Configuration for actor [" + actorName +
+          "] is invalid - you can not use a 'BalancingDispatcher' together with any type of 'Router'")
+  }
+
   private def _actorOf(props: Props, name: String): ActorRef = {
+    verifyActorConfiguration(systemImpl, props, name)
+
     if (system.settings.SerializeAllCreators && !props.creator.isInstanceOf[NoSerializationVerificationNeeded]) {
       val ser = SerializationExtension(system)
       ser.serialize(props.creator) match {

From 6d962174fedb48dc3d494f1f6ae47e9996885f8d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jonas=20Bone=CC=81r?= 
Date: Fri, 18 May 2012 16:16:41 +0200
Subject: [PATCH 21/36] Added documentation about the verification of
 BalancingDispatcher + Router (added to both Dispatcher and Routing docs)

---
 akka-docs/java/dispatchers.rst  |  6 ++++--
 akka-docs/java/routing.rst      |  3 ++-
 akka-docs/scala/dispatchers.rst | 10 ++++++----
 akka-docs/scala/routing.rst     |  4 +++-
 4 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/akka-docs/java/dispatchers.rst b/akka-docs/java/dispatchers.rst
index 6ef0d44d7e..90a0e9cb6a 100644
--- a/akka-docs/java/dispatchers.rst
+++ b/akka-docs/java/dispatchers.rst
@@ -70,7 +70,7 @@ There are 4 different types of message dispatchers:
 
 * BalancingDispatcher
 
-  - This is an executor based event driven dispatcher that will try to redistribute work from busy actors to idle actors. 
+  - This is an executor based event driven dispatcher that will try to redistribute work from busy actors to idle actors.
 
   - It is assumed that all actors using the same instance of this dispatcher can process all messages that have been sent to one of the actors; i.e. the actors belong to a pool of actors, and to the client there is no guarantee about which actor instance actually processes a given message.
 
@@ -85,9 +85,11 @@ There are 4 different types of message dispatchers:
                "thread-pool-executor" or the FQCN of
                an ``akka.dispatcher.ExecutorServiceConfigurator``
 
+  - Note that you can **not** use a ``BalancingDispatcher`` together with any kind of ``Router``, trying to do so will make your actor fail verification.
+
 * CallingThreadDispatcher
 
-  - This dispatcher runs invocations on the current thread only. This dispatcher does not create any new threads, 
+  - This dispatcher runs invocations on the current thread only. This dispatcher does not create any new threads,
     but it can be used from different threads concurrently for the same actor. See :ref:`TestCallingThreadDispatcherRef`
     for details and restrictions.
 
diff --git a/akka-docs/java/routing.rst b/akka-docs/java/routing.rst
index 4d01642a72..e006c7db63 100644
--- a/akka-docs/java/routing.rst
+++ b/akka-docs/java/routing.rst
@@ -375,7 +375,8 @@ The dispatcher for created children of the router will be taken from
 makes sense to configure the :class:`BalancingDispatcher` if the precise
 routing is not so important (i.e. no consistent hashing or round-robin is
 required); this enables newly created routees to pick up work immediately by
-stealing it from their siblings.
+stealing it from their siblings. Note that you can **not** use a ``BalancingDispatcher``
+together with any kind of ``Router``, trying to do so will make your actor fail verification.
 
 The “head” router, of course, cannot run on the same balancing dispatcher,
 because it does not process the same messages, hence this special actor does
diff --git a/akka-docs/scala/dispatchers.rst b/akka-docs/scala/dispatchers.rst
index 7d6a1f6334..a1cc431643 100644
--- a/akka-docs/scala/dispatchers.rst
+++ b/akka-docs/scala/dispatchers.rst
@@ -71,7 +71,7 @@ There are 4 different types of message dispatchers:
 
 * BalancingDispatcher
 
-  - This is an executor based event driven dispatcher that will try to redistribute work from busy actors to idle actors. 
+  - This is an executor based event driven dispatcher that will try to redistribute work from busy actors to idle actors.
 
   - It is assumed that all actors using the same instance of this dispatcher can process all messages that have been sent to one of the actors; i.e. the actors belong to a pool of actors, and to the client there is no guarantee about which actor instance actually processes a given message.
 
@@ -86,9 +86,11 @@ There are 4 different types of message dispatchers:
                "thread-pool-executor" or the FQCN of
                an ``akka.dispatcher.ExecutorServiceConfigurator``
 
+  - Note that you can **not** use a ``BalancingDispatcher`` together with any kind of ``Router``, trying to do so will make your actor fail verification.
+
 * CallingThreadDispatcher
 
-  - This dispatcher runs invocations on the current thread only. This dispatcher does not create any new threads, 
+  - This dispatcher runs invocations on the current thread only. This dispatcher does not create any new threads,
     but it can be used from different threads concurrently for the same actor. See :ref:`TestCallingThreadDispatcherRef`
     for details and restrictions.
 
@@ -112,8 +114,8 @@ And then using it:
 
 .. includecode:: ../scala/code/akka/docs/dispatcher/DispatcherDocSpec.scala#defining-pinned-dispatcher
 
-Note that ``thread-pool-executor`` configuration as per the above ``my-thread-pool-dispatcher`` exmaple is 
-NOT applicable. This is because every actor will have its own thread pool when using ``PinnedDispatcher``, 
+Note that ``thread-pool-executor`` configuration as per the above ``my-thread-pool-dispatcher`` exmaple is
+NOT applicable. This is because every actor will have its own thread pool when using ``PinnedDispatcher``,
 and that pool will have only one thread.
 
 Mailboxes
diff --git a/akka-docs/scala/routing.rst b/akka-docs/scala/routing.rst
index 737c9e31e7..0d0625be36 100644
--- a/akka-docs/scala/routing.rst
+++ b/akka-docs/scala/routing.rst
@@ -375,7 +375,9 @@ The dispatcher for created children of the router will be taken from
 makes sense to configure the :class:`BalancingDispatcher` if the precise
 routing is not so important (i.e. no consistent hashing or round-robin is
 required); this enables newly created routees to pick up work immediately by
-stealing it from their siblings.
+stealing it from their siblings. Note that you can **not** use a ``BalancingDispatcher``
+together with any kind of ``Router``, trying to do so will make your actor fail verification.
+
 
 .. note::
 

From e99c9385283256a6254b78dd4c6fe24eed464fb0 Mon Sep 17 00:00:00 2001
From: Roland 
Date: Fri, 18 May 2012 16:26:48 +0200
Subject: [PATCH 22/36] =?UTF-8?q?switch=20to=20Bj=C3=B6rn=E2=80=99s=20new?=
 =?UTF-8?q?=20multi-jvm=20setup=20(i.e.=20remove=20system=20properties)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../remote/testconductor/TestConductorSpec.scala | 16 ++++------------
 project/plugins.sbt                              |  2 +-
 2 files changed, 5 insertions(+), 13 deletions(-)

diff --git a/akka-remote-tests/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala b/akka-remote-tests/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
index 39d25981aa..7f3763fcc1 100644
--- a/akka-remote-tests/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
+++ b/akka-remote-tests/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
@@ -14,9 +14,8 @@ import java.net.InetSocketAddress
 import java.net.InetAddress
 import akka.remote.testkit.MultiNodeSpec
 
-object TestConductorMultiJvmSpec extends AbstractRemoteActorMultiJvmSpec {
-  override def NrOfNodes = 2
-  override def commonConfig = ConfigFactory.parseString("""
+object TestConductorMultiJvmSpec {
+  def commonConfig = ConfigFactory.parseString("""
     akka.loglevel = DEBUG
     akka.remote {
       log-received-messages = on
@@ -29,15 +28,8 @@ object TestConductorMultiJvmSpec extends AbstractRemoteActorMultiJvmSpec {
   """)
 }
 
-object H {
-  def apply(x: Int) = {
-    System.setProperty("multinode.hosts", "localhost,localhost")
-    System.setProperty("multinode.index", x.toString)
-  }
-}
-
-class TestConductorMultiJvmNode1 extends { val dummy = H(0) } with TestConductorSpec
-class TestConductorMultiJvmNode2 extends { val dummy = H(1) } with TestConductorSpec
+class TestConductorMultiJvmNode1 extends TestConductorSpec
+class TestConductorMultiJvmNode2 extends TestConductorSpec
 
 class TestConductorSpec extends MultiNodeSpec(TestConductorMultiJvmSpec.commonConfig) with ImplicitSender {
 
diff --git a/project/plugins.sbt b/project/plugins.sbt
index 80ff9db95a..f49cfb688d 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -1,7 +1,7 @@
 
 resolvers += Classpaths.typesafeResolver
 
-addSbtPlugin("com.typesafe.sbtmultijvm" % "sbt-multi-jvm" % "0.1.9")
+addSbtPlugin("com.typesafe.sbtmultijvm" % "sbt-multi-jvm" % "0.2.0-SNAPSHOT")
 
 addSbtPlugin("com.typesafe.schoir" % "schoir" % "0.1.2")
 

From 66600f9c52dfefb577fd1ef4bd89a0fde685d724 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jonas=20Bone=CC=81r?= 
Date: Fri, 18 May 2012 16:51:12 +0200
Subject: [PATCH 23/36] Moved the dispatcher/router verification to
 RoutedActorRef, also checks dispatcher only through the config so we don't
 trigger creation of dispatcher.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Jonas Bonér 
---
 akka-actor/src/main/scala/akka/actor/ActorCell.scala | 12 ------------
 .../src/main/scala/akka/dispatch/Dispatchers.scala   |  6 +++---
 akka-actor/src/main/scala/akka/routing/Routing.scala | 10 ++++++++--
 3 files changed, 11 insertions(+), 17 deletions(-)

diff --git a/akka-actor/src/main/scala/akka/actor/ActorCell.scala b/akka-actor/src/main/scala/akka/actor/ActorCell.scala
index 9cc993062f..8c68ba3315 100644
--- a/akka-actor/src/main/scala/akka/actor/ActorCell.scala
+++ b/akka-actor/src/main/scala/akka/actor/ActorCell.scala
@@ -357,19 +357,7 @@ private[akka] class ActorCell(
     case _ ⇒ true
   }
 
-  private def verifyActorConfiguration(system: ActorSystem, props: Props, actorName: String): Unit = {
-    import akka.config.ConfigurationException
-    import akka.routing.NoRouter
-    // verify that a BalancingDispatcher is not used with a Router
-    if (system.dispatchers.lookup(props.dispatcher).isInstanceOf[BalancingDispatcher] && props.routerConfig != NoRouter)
-      throw new ConfigurationException(
-        "Configuration for actor [" + actorName +
-          "] is invalid - you can not use a 'BalancingDispatcher' together with any type of 'Router'")
-  }
-
   private def _actorOf(props: Props, name: String): ActorRef = {
-    verifyActorConfiguration(systemImpl, props, name)
-
     if (system.settings.SerializeAllCreators && !props.creator.isInstanceOf[NoSerializationVerificationNeeded]) {
       val ser = SerializationExtension(system)
       ser.serialize(props.creator) match {
diff --git a/akka-actor/src/main/scala/akka/dispatch/Dispatchers.scala b/akka-actor/src/main/scala/akka/dispatch/Dispatchers.scala
index 93d44e007d..a81a8e6c2b 100644
--- a/akka-actor/src/main/scala/akka/dispatch/Dispatchers.scala
+++ b/akka-actor/src/main/scala/akka/dispatch/Dispatchers.scala
@@ -120,9 +120,9 @@ class Dispatchers(val settings: ActorSystem.Settings, val prerequisites: Dispatc
    * Throws: IllegalArgumentException if the value of "type" is not valid
    *         IllegalArgumentException if it cannot create the MessageDispatcherConfigurator
    */
-  private[akka] def from(cfg: Config): MessageDispatcher = {
-    configuratorFrom(cfg).dispatcher()
-  }
+  private[akka] def from(cfg: Config): MessageDispatcher = configuratorFrom(cfg).dispatcher()
+
+  private[akka] def isBalancingDispatcher(id: String): Boolean = settings.config.hasPath(id) && config(id).getString("type") == "BalancingDispatcher"
 
   /*
    * Creates a MessageDispatcherConfigurator from a Config.
diff --git a/akka-actor/src/main/scala/akka/routing/Routing.scala b/akka-actor/src/main/scala/akka/routing/Routing.scala
index fdf14a5b96..58ecbfcdc5 100644
--- a/akka-actor/src/main/scala/akka/routing/Routing.scala
+++ b/akka-actor/src/main/scala/akka/routing/Routing.scala
@@ -31,11 +31,17 @@ private[akka] class RoutedActorRef(_system: ActorSystemImpl, _props: Props, _sup
     _supervisor,
     _path) {
 
+  // verify that a BalancingDispatcher is not used with a Router
+  if (_system.dispatchers.isBalancingDispatcher(_props.dispatcher) && _props.routerConfig != NoRouter)
+    throw new ConfigurationException(
+      "Configuration for actor [" + _path.toString +
+        "] is invalid - you can not use a 'BalancingDispatcher' together with any type of 'Router'")
+
   /*
    * CAUTION: RoutedActorRef is PROBLEMATIC
    * ======================================
-   * 
-   * We are constructing/assembling the children outside of the scope of the 
+   *
+   * We are constructing/assembling the children outside of the scope of the
    * Router actor, inserting them in its childrenRef list, which is not at all
    * synchronized. This is done exactly once at start-up, all other accesses
    * are done from the Router actor. This means that the only thing which is

From e825a8ac4f0dd84297fb7d93f01f1f755e573631 Mon Sep 17 00:00:00 2001
From: Roland 
Date: Fri, 18 May 2012 18:44:53 +0200
Subject: [PATCH 24/36] switch MultiNodeSpec to use RoleName type when
 referring to participants

- also add MultiNodeConfig base class for conveniently declaring a test
  setup including roles and (node specific) config settings
---
 .../akka/remote/testconductor/Conductor.scala |  32 ++--
 .../akka/remote/testconductor/DataTypes.scala |  22 +--
 .../akka/remote/testconductor/Player.scala    |  10 +-
 .../testconductor/TestConductorSpec.scala     |  41 +++--
 .../remote/testconductor/BarrierSpec.scala    | 164 +++++++++---------
 .../remote/testconductor/ControllerSpec.scala |   9 +-
 .../akka/remote/testkit/MultiNodeSpec.scala   |  87 ++++++----
 7 files changed, 197 insertions(+), 168 deletions(-)

diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala
index d4fa3152e6..6c26fcaae2 100644
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala
+++ b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala
@@ -71,7 +71,7 @@ trait Conductor { this: TestConductorExt ⇒
    * @param participants gives the number of participants which shall connect
    * before any of their startClient() operations complete.
    */
-  def startController(participants: Int, name: String, controllerPort: InetSocketAddress): Future[InetSocketAddress] = {
+  def startController(participants: Int, name: RoleName, controllerPort: InetSocketAddress): Future[InetSocketAddress] = {
     if (_controller ne null) throw new RuntimeException("TestConductorServer was already started")
     _controller = system.actorOf(Props(new Controller(participants, controllerPort)), "controller")
     import Settings.BarrierTimeout
@@ -106,7 +106,7 @@ trait Conductor { this: TestConductorExt ⇒
    * @param direction can be either `Direction.Send`, `Direction.Receive` or `Direction.Both`
    * @param rateMBit is the maximum data rate in MBit
    */
-  def throttle(node: String, target: String, direction: Direction, rateMBit: Double): Future[Done] = {
+  def throttle(node: RoleName, target: RoleName, direction: Direction, rateMBit: Double): Future[Done] = {
     import Settings.QueryTimeout
     controller ? Throttle(node, target, direction, rateMBit.toFloat) mapTo
   }
@@ -121,7 +121,7 @@ trait Conductor { this: TestConductorExt ⇒
    * @param target is the symbolic name of the other node to which connectivity shall be impeded
    * @param direction can be either `Direction.Send`, `Direction.Receive` or `Direction.Both`
    */
-  def blackhole(node: String, target: String, direction: Direction): Future[Done] = {
+  def blackhole(node: RoleName, target: RoleName, direction: Direction): Future[Done] = {
     import Settings.QueryTimeout
     controller ? Throttle(node, target, direction, 0f) mapTo
   }
@@ -134,7 +134,7 @@ trait Conductor { this: TestConductorExt ⇒
    * @param node is the symbolic name of the node which is to be affected
    * @param target is the symbolic name of the other node to which connectivity shall be impeded
    */
-  def disconnect(node: String, target: String): Future[Done] = {
+  def disconnect(node: RoleName, target: RoleName): Future[Done] = {
     import Settings.QueryTimeout
     controller ? Disconnect(node, target, false) mapTo
   }
@@ -147,7 +147,7 @@ trait Conductor { this: TestConductorExt ⇒
    * @param node is the symbolic name of the node which is to be affected
    * @param target is the symbolic name of the other node to which connectivity shall be impeded
    */
-  def abort(node: String, target: String): Future[Done] = {
+  def abort(node: RoleName, target: RoleName): Future[Done] = {
     import Settings.QueryTimeout
     controller ? Disconnect(node, target, true) mapTo
   }
@@ -159,7 +159,7 @@ trait Conductor { this: TestConductorExt ⇒
    * @param node is the symbolic name of the node which is to be affected
    * @param exitValue is the return code which shall be given to System.exit
    */
-  def shutdown(node: String, exitValue: Int): Future[Done] = {
+  def shutdown(node: RoleName, exitValue: Int): Future[Done] = {
     import Settings.QueryTimeout
     controller ? Terminate(node, exitValue) mapTo
   }
@@ -169,7 +169,7 @@ trait Conductor { this: TestConductorExt ⇒
    *
    * @param node is the symbolic name of the node which is to be affected
    */
-  def kill(node: String): Future[Done] = {
+  def kill(node: RoleName): Future[Done] = {
     import Settings.QueryTimeout
     controller ? Terminate(node, -1) mapTo
   }
@@ -177,7 +177,7 @@ trait Conductor { this: TestConductorExt ⇒
   /**
    * Obtain the list of remote host names currently registered.
    */
-  def getNodes: Future[Iterable[String]] = {
+  def getNodes: Future[Iterable[RoleName]] = {
     import Settings.QueryTimeout
     controller ? GetNodes mapTo
   }
@@ -190,7 +190,7 @@ trait Conductor { this: TestConductorExt ⇒
    *
    * @param node is the symbolic name of the node which is to be removed
    */
-  def removeNode(node: String): Future[Done] = {
+  def removeNode(node: RoleName): Future[Done] = {
     import Settings.QueryTimeout
     controller ? Remove(node) mapTo
   }
@@ -274,7 +274,7 @@ class ServerFSM(val controller: ActorRef, val channel: Channel) extends Actor wi
 
   when(Initial, stateTimeout = 10 seconds) {
     case Event(Hello(name, addr), _) ⇒
-      controller ! NodeInfo(name, addr, self)
+      controller ! NodeInfo(RoleName(name), addr, self)
       goto(Ready)
     case Event(x: NetworkOp, _) ⇒
       log.warning("client {} sent no Hello in first message (instead {}), disconnecting", getAddrString(channel), x)
@@ -318,11 +318,11 @@ class ServerFSM(val controller: ActorRef, val channel: Channel) extends Actor wi
 }
 
 object Controller {
-  case class ClientDisconnected(name: String)
+  case class ClientDisconnected(name: RoleName)
   case object GetNodes
   case object GetSockAddr
 
-  case class NodeInfo(name: String, addr: Address, fsm: ActorRef)
+  case class NodeInfo(name: RoleName, addr: Address, fsm: ActorRef)
 }
 
 /**
@@ -359,10 +359,10 @@ class Controller(private var initialParticipants: Int, controllerPort: InetSocke
   }
 
   val barrier = context.actorOf(Props[BarrierCoordinator], "barriers")
-  var nodes = Map[String, NodeInfo]()
+  var nodes = Map[RoleName, NodeInfo]()
 
   // map keeping unanswered queries for node addresses (enqueued upon GetAddress, serviced upon NodeInfo)
-  var addrInterest = Map[String, Set[ActorRef]]()
+  var addrInterest = Map[RoleName, Set[ActorRef]]()
 
   override def receive = LoggingReceive {
     case c @ NodeInfo(name, addr, fsm) ⇒
@@ -423,7 +423,7 @@ object BarrierCoordinator {
   case object Idle extends State
   case object Waiting extends State
 
-  case class RemoveClient(name: String)
+  case class RemoveClient(name: RoleName)
 
   case class Data(clients: Set[Controller.NodeInfo], barrier: String, arrived: List[ActorRef])
 
@@ -435,7 +435,7 @@ object BarrierCoordinator {
   case class DuplicateNode(data: Data, node: Controller.NodeInfo) extends RuntimeException with NoStackTrace with Printer
   case class WrongBarrier(barrier: String, client: ActorRef, data: Data) extends RuntimeException(barrier) with NoStackTrace with Printer
   case class BarrierEmpty(data: Data, msg: String) extends RuntimeException(msg) with NoStackTrace with Printer
-  case class ClientLost(data: Data, client: String) extends RuntimeException with NoStackTrace with Printer
+  case class ClientLost(data: Data, client: RoleName) extends RuntimeException with NoStackTrace with Printer
 }
 
 /**
diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/DataTypes.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/DataTypes.scala
index 0273055469..2bb7d50c37 100644
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/DataTypes.scala
+++ b/akka-remote-tests/src/main/scala/akka/remote/testconductor/DataTypes.scala
@@ -11,6 +11,8 @@ import com.google.protobuf.Message
 import akka.actor.Address
 import org.jboss.netty.handler.codec.oneone.OneToOneDecoder
 
+case class RoleName(name: String)
+
 case class ToClient(msg: ClientOp with NetworkOp)
 case class ToServer(msg: ServerOp with NetworkOp)
 
@@ -29,24 +31,24 @@ case class Hello(name: String, addr: Address) extends NetworkOp
 case class EnterBarrier(name: String) extends ServerOp with NetworkOp
 case class BarrierResult(name: String, success: Boolean) extends UnconfirmedClientOp with NetworkOp
 
-case class Throttle(node: String, target: String, direction: Direction, rateMBit: Float) extends CommandOp
+case class Throttle(node: RoleName, target: RoleName, direction: Direction, rateMBit: Float) extends CommandOp
 case class ThrottleMsg(target: Address, direction: Direction, rateMBit: Float) extends ConfirmedClientOp with NetworkOp
 
-case class Disconnect(node: String, target: String, abort: Boolean) extends CommandOp
+case class Disconnect(node: RoleName, target: RoleName, abort: Boolean) extends CommandOp
 case class DisconnectMsg(target: Address, abort: Boolean) extends ConfirmedClientOp with NetworkOp
 
-case class Terminate(node: String, exitValueOrKill: Int) extends CommandOp
+case class Terminate(node: RoleName, exitValueOrKill: Int) extends CommandOp
 case class TerminateMsg(exitValue: Int) extends ConfirmedClientOp with NetworkOp
 
-case class GetAddress(node: String) extends ServerOp with NetworkOp
-case class AddressReply(node: String, addr: Address) extends UnconfirmedClientOp with NetworkOp
+case class GetAddress(node: RoleName) extends ServerOp with NetworkOp
+case class AddressReply(node: RoleName, addr: Address) extends UnconfirmedClientOp with NetworkOp
 
 abstract class Done extends ServerOp with UnconfirmedClientOp with NetworkOp
 case object Done extends Done {
   def getInstance: Done = this
 }
 
-case class Remove(node: String) extends CommandOp
+case class Remove(node: RoleName) extends CommandOp
 
 class MsgEncoder extends OneToOneEncoder {
   def encode(ctx: ChannelHandlerContext, ch: Channel, msg: AnyRef): AnyRef = msg match {
@@ -68,9 +70,9 @@ class MsgEncoder extends OneToOneEncoder {
         case TerminateMsg(exitValue) ⇒
           w.setFailure(TCP.InjectFailure.newBuilder.setFailure(TCP.FailType.Shutdown).setExitValue(exitValue))
         case GetAddress(node) ⇒
-          w.setAddr(TCP.AddressRequest.newBuilder.setNode(node))
+          w.setAddr(TCP.AddressRequest.newBuilder.setNode(node.name))
         case AddressReply(node, addr) ⇒
-          w.setAddr(TCP.AddressRequest.newBuilder.setNode(node).setAddr(addr))
+          w.setAddr(TCP.AddressRequest.newBuilder.setNode(node.name).setAddr(addr))
         case _: Done ⇒
           w.setDone("")
       }
@@ -100,8 +102,8 @@ class MsgDecoder extends OneToOneDecoder {
         }
       } else if (w.hasAddr) {
         val a = w.getAddr
-        if (a.hasAddr) AddressReply(a.getNode, a.getAddr)
-        else GetAddress(a.getNode)
+        if (a.hasAddr) AddressReply(RoleName(a.getNode), a.getAddr)
+        else GetAddress(RoleName(a.getNode))
       } else if (w.hasDone) {
         Done
       } else {
diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Player.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Player.scala
index 27a2487364..10434007e1 100644
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Player.scala
+++ b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Player.scala
@@ -50,7 +50,7 @@ trait Player { this: TestConductorExt ⇒
    * this is a first barrier in itself). The number of expected participants is
    * set in [[akka.remote.testconductor.Conductor]]`.startController()`.
    */
-  def startClient(name: String, controllerAddr: InetSocketAddress): Future[Done] = {
+  def startClient(name: RoleName, controllerAddr: InetSocketAddress): Future[Done] = {
     import ClientFSM._
     import akka.actor.FSM._
     import Settings.BarrierTimeout
@@ -88,7 +88,7 @@ trait Player { this: TestConductorExt ⇒
   /**
    * Query remote transport address of named node.
    */
-  def getAddressFor(name: String): Future[Address] = {
+  def getAddressFor(name: RoleName): Future[Address] = {
     import Settings.BarrierTimeout
     client ? ToServer(GetAddress(name)) mapTo
   }
@@ -117,7 +117,7 @@ object ClientFSM {
  * coordinator and react to the [[akka.remote.testconductor.Conductor]]’s
  * requests for failure injection.
  */
-class ClientFSM(name: String, controllerAddr: InetSocketAddress) extends Actor with LoggingFSM[ClientFSM.State, ClientFSM.Data] {
+class ClientFSM(name: RoleName, controllerAddr: InetSocketAddress) extends Actor with LoggingFSM[ClientFSM.State, ClientFSM.Data] {
   import ClientFSM._
 
   val settings = TestConductor().Settings
@@ -131,7 +131,7 @@ class ClientFSM(name: String, controllerAddr: InetSocketAddress) extends Actor w
     case Event(msg: ClientOp, _) ⇒
       stay replying Status.Failure(new IllegalStateException("not connected yet"))
     case Event(Connected(channel), _) ⇒
-      channel.write(Hello(name, TestConductor().address))
+      channel.write(Hello(name.name, TestConductor().address))
       goto(AwaitDone) using Data(Some(channel), None)
     case Event(_: ConnectionFailure, _) ⇒
       goto(Failed)
@@ -165,7 +165,7 @@ class ClientFSM(name: String, controllerAddr: InetSocketAddress) extends Actor w
       channel.write(msg)
       val token = msg match {
         case EnterBarrier(barrier) ⇒ barrier
-        case GetAddress(node)      ⇒ node
+        case GetAddress(node)      ⇒ node.name
       }
       stay using d.copy(runningOp = Some(token, sender))
     case Event(ToServer(op), Data(channel, Some((token, _)))) ⇒
diff --git a/akka-remote-tests/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala b/akka-remote-tests/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
index 7f3763fcc1..087aac55c7 100644
--- a/akka-remote-tests/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
+++ b/akka-remote-tests/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
@@ -13,9 +13,10 @@ import akka.testkit.ImplicitSender
 import java.net.InetSocketAddress
 import java.net.InetAddress
 import akka.remote.testkit.MultiNodeSpec
+import akka.remote.testkit.MultiNodeConfig
 
-object TestConductorMultiJvmSpec {
-  def commonConfig = ConfigFactory.parseString("""
+object TestConductorMultiJvmSpec extends MultiNodeConfig {
+  commonConfig(ConfigFactory.parseString("""
     akka.loglevel = DEBUG
     akka.remote {
       log-received-messages = on
@@ -25,18 +26,22 @@ object TestConductorMultiJvmSpec {
       receive = on
       fsm = on
     }
-  """)
+  """))
+  
+  val master = role("master")
+  val slave = role("slave")
 }
 
 class TestConductorMultiJvmNode1 extends TestConductorSpec
 class TestConductorMultiJvmNode2 extends TestConductorSpec
 
-class TestConductorSpec extends MultiNodeSpec(TestConductorMultiJvmSpec.commonConfig) with ImplicitSender {
+class TestConductorSpec extends MultiNodeSpec(TestConductorMultiJvmSpec) with ImplicitSender {
+  
+  import TestConductorMultiJvmSpec._
 
   def initialParticipants = 2
-  lazy val roles = Seq("master", "slave")
 
-  runOn("master") {
+  runOn(master) {
     system.actorOf(Props(new Actor {
       def receive = {
         case x ⇒ testActor ! x; sender ! x
@@ -44,7 +49,7 @@ class TestConductorSpec extends MultiNodeSpec(TestConductorMultiJvmSpec.commonCo
     }), "echo")
   }
 
-  val echo = system.actorFor(node("master") / "user" / "echo")
+  val echo = system.actorFor(node(master) / "user" / "echo")
 
   "A TestConductor" must {
 
@@ -54,20 +59,20 @@ class TestConductorSpec extends MultiNodeSpec(TestConductorMultiJvmSpec.commonCo
 
     "support throttling of network connections" in {
 
-      runOn("slave") {
+      runOn(slave) {
         // start remote network connection so that it can be throttled
         echo ! "start"
       }
 
       expectMsg("start")
 
-      runOn("master") {
-        testConductor.throttle("slave", "master", Direction.Send, rateMBit = 0.01).await
+      runOn(master) {
+        testConductor.throttle(slave, master, Direction.Send, rateMBit = 0.01).await
       }
 
       testConductor.enter("throttled_send")
 
-      runOn("slave") {
+      runOn(slave) {
         for (i ← 0 to 9) echo ! i
       }
 
@@ -78,19 +83,19 @@ class TestConductorSpec extends MultiNodeSpec(TestConductorMultiJvmSpec.commonCo
 
       testConductor.enter("throttled_send2")
 
-      runOn("master") {
-        testConductor.throttle("slave", "master", Direction.Send, -1).await
-        testConductor.throttle("slave", "master", Direction.Receive, rateMBit = 0.01).await
+      runOn(master) {
+        testConductor.throttle(slave, master, Direction.Send, -1).await
+        testConductor.throttle(slave, master, Direction.Receive, rateMBit = 0.01).await
       }
 
       testConductor.enter("throttled_recv")
 
-      runOn("slave") {
+      runOn(slave) {
         for (i ← 10 to 19) echo ! i
       }
 
       val (min, max) =
-        ifNode("master") {
+        ifNode(master) {
           (0 seconds, 500 millis)
         } {
           (0.6 seconds, 2 seconds)
@@ -103,8 +108,8 @@ class TestConductorSpec extends MultiNodeSpec(TestConductorMultiJvmSpec.commonCo
 
       testConductor.enter("throttled_recv2")
 
-      runOn("master") {
-        testConductor.throttle("slave", "master", Direction.Receive, -1).await
+      runOn(master) {
+        testConductor.throttle(slave, master, Direction.Receive, -1).await
       }
     }
 
diff --git a/akka-remote-tests/src/test/scala/akka/remote/testconductor/BarrierSpec.scala b/akka-remote-tests/src/test/scala/akka/remote/testconductor/BarrierSpec.scala
index aa14b93f9d..e0fd5dfb97 100644
--- a/akka-remote-tests/src/test/scala/akka/remote/testconductor/BarrierSpec.scala
+++ b/akka-remote-tests/src/test/scala/akka/remote/testconductor/BarrierSpec.scala
@@ -36,6 +36,10 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
   import Controller._
   import BarrierCoordinator._
 
+  val A = RoleName("a")
+  val B = RoleName("b")
+  val C = RoleName("c")
+
   override def afterEach {
     system.eventStream.setLogLevel(Logging.WarningLevel)
   }
@@ -44,25 +48,25 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
 
     "register clients and remove them" in {
       val b = getBarrier()
-      b ! NodeInfo("a", AddressFromURIString("akka://sys"), system.deadLetters)
-      b ! RemoveClient("b")
-      b ! RemoveClient("a")
+      b ! NodeInfo(A, AddressFromURIString("akka://sys"), system.deadLetters)
+      b ! RemoveClient(B)
+      b ! RemoveClient(A)
       EventFilter[BarrierEmpty](occurrences = 1) intercept {
-        b ! RemoveClient("a")
+        b ! RemoveClient(A)
       }
       expectMsg(Failed(b, BarrierEmpty(Data(Set(), "", Nil), "no client to remove")))
     }
 
     "register clients and disconnect them" in {
       val b = getBarrier()
-      b ! NodeInfo("a", AddressFromURIString("akka://sys"), system.deadLetters)
-      b ! ClientDisconnected("b")
+      b ! NodeInfo(A, AddressFromURIString("akka://sys"), system.deadLetters)
+      b ! ClientDisconnected(B)
       EventFilter[ClientLost](occurrences = 1) intercept {
-        b ! ClientDisconnected("a")
+        b ! ClientDisconnected(A)
       }
-      expectMsg(Failed(b, ClientLost(Data(Set(), "", Nil), "a")))
+      expectMsg(Failed(b, ClientLost(Data(Set(), "", Nil), A)))
       EventFilter[BarrierEmpty](occurrences = 1) intercept {
-        b ! ClientDisconnected("a")
+        b ! ClientDisconnected(A)
       }
       expectMsg(Failed(b, BarrierEmpty(Data(Set(), "", Nil), "no client to disconnect")))
     }
@@ -76,8 +80,8 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
     "enter barrier" in {
       val barrier = getBarrier()
       val a, b = TestProbe()
-      barrier ! NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
-      barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      barrier ! NodeInfo(A, AddressFromURIString("akka://sys"), a.ref)
+      barrier ! NodeInfo(B, AddressFromURIString("akka://sys"), b.ref)
       a.send(barrier, EnterBarrier("bar"))
       noMsg(a, b)
       within(1 second) {
@@ -90,10 +94,10 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
     "enter barrier with joining node" in {
       val barrier = getBarrier()
       val a, b, c = TestProbe()
-      barrier ! NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
-      barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      barrier ! NodeInfo(A, AddressFromURIString("akka://sys"), a.ref)
+      barrier ! NodeInfo(B, AddressFromURIString("akka://sys"), b.ref)
       a.send(barrier, EnterBarrier("bar"))
-      barrier ! NodeInfo("c", AddressFromURIString("akka://sys"), c.ref)
+      barrier ! NodeInfo(C, AddressFromURIString("akka://sys"), c.ref)
       b.send(barrier, EnterBarrier("bar"))
       noMsg(a, b, c)
       within(1 second) {
@@ -107,29 +111,29 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
     "enter barrier with leaving node" in {
       val barrier = getBarrier()
       val a, b, c = TestProbe()
-      barrier ! NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
-      barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
-      barrier ! NodeInfo("c", AddressFromURIString("akka://sys"), c.ref)
+      barrier ! NodeInfo(A, AddressFromURIString("akka://sys"), a.ref)
+      barrier ! NodeInfo(B, AddressFromURIString("akka://sys"), b.ref)
+      barrier ! NodeInfo(C, AddressFromURIString("akka://sys"), c.ref)
       a.send(barrier, EnterBarrier("bar"))
       b.send(barrier, EnterBarrier("bar"))
-      barrier ! RemoveClient("a")
-      barrier ! ClientDisconnected("a")
+      barrier ! RemoveClient(A)
+      barrier ! ClientDisconnected(A)
       noMsg(a, b, c)
       b.within(1 second) {
-        barrier ! RemoveClient("c")
+        barrier ! RemoveClient(C)
         b.expectMsg(ToClient(BarrierResult("bar", true)))
       }
-      barrier ! ClientDisconnected("c")
+      barrier ! ClientDisconnected(C)
       expectNoMsg(1 second)
     }
 
     "leave barrier when last “arrived” is removed" in {
       val barrier = getBarrier()
       val a, b = TestProbe()
-      barrier ! NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
-      barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      barrier ! NodeInfo(A, AddressFromURIString("akka://sys"), a.ref)
+      barrier ! NodeInfo(B, AddressFromURIString("akka://sys"), b.ref)
       a.send(barrier, EnterBarrier("bar"))
-      barrier ! RemoveClient("a")
+      barrier ! RemoveClient(A)
       b.send(barrier, EnterBarrier("foo"))
       b.expectMsg(ToClient(BarrierResult("foo", true)))
     }
@@ -137,38 +141,38 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
     "fail barrier with disconnecing node" in {
       val barrier = getBarrier()
       val a, b = TestProbe()
-      val nodeA = NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
+      val nodeA = NodeInfo(A, AddressFromURIString("akka://sys"), a.ref)
       barrier ! nodeA
-      barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      barrier ! NodeInfo(B, AddressFromURIString("akka://sys"), b.ref)
       a.send(barrier, EnterBarrier("bar"))
       EventFilter[ClientLost](occurrences = 1) intercept {
-        barrier ! ClientDisconnected("b")
+        barrier ! ClientDisconnected(B)
       }
-      expectMsg(Failed(barrier, ClientLost(Data(Set(nodeA), "bar", a.ref :: Nil), "b")))
+      expectMsg(Failed(barrier, ClientLost(Data(Set(nodeA), "bar", a.ref :: Nil), B)))
     }
 
     "fail barrier with disconnecing node who already arrived" in {
       val barrier = getBarrier()
       val a, b, c = TestProbe()
-      val nodeA = NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
-      val nodeC = NodeInfo("c", AddressFromURIString("akka://sys"), c.ref)
+      val nodeA = NodeInfo(A, AddressFromURIString("akka://sys"), a.ref)
+      val nodeC = NodeInfo(C, AddressFromURIString("akka://sys"), c.ref)
       barrier ! nodeA
-      barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      barrier ! NodeInfo(B, AddressFromURIString("akka://sys"), b.ref)
       barrier ! nodeC
       a.send(barrier, EnterBarrier("bar"))
       b.send(barrier, EnterBarrier("bar"))
       EventFilter[ClientLost](occurrences = 1) intercept {
-        barrier ! ClientDisconnected("b")
+        barrier ! ClientDisconnected(B)
       }
-      expectMsg(Failed(barrier, ClientLost(Data(Set(nodeA, nodeC), "bar", a.ref :: Nil), "b")))
+      expectMsg(Failed(barrier, ClientLost(Data(Set(nodeA, nodeC), "bar", a.ref :: Nil), B)))
     }
 
     "fail when entering wrong barrier" in {
       val barrier = getBarrier()
       val a, b = TestProbe()
-      val nodeA = NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
+      val nodeA = NodeInfo(A, AddressFromURIString("akka://sys"), a.ref)
       barrier ! nodeA
-      val nodeB = NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      val nodeB = NodeInfo(B, AddressFromURIString("akka://sys"), b.ref)
       barrier ! nodeB
       a.send(barrier, EnterBarrier("bar"))
       EventFilter[WrongBarrier](occurrences = 1) intercept {
@@ -181,10 +185,10 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
       val barrier = getBarrier()
       val a = TestProbe()
       EventFilter[BarrierEmpty](occurrences = 1) intercept {
-        barrier ! RemoveClient("a")
+        barrier ! RemoveClient(A)
       }
       expectMsg(Failed(barrier, BarrierEmpty(Data(Set(), "", Nil), "no client to remove")))
-      barrier ! NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
+      barrier ! NodeInfo(A, AddressFromURIString("akka://sys"), a.ref)
       a.send(barrier, EnterBarrier("right"))
       a.expectMsg(ToClient(BarrierResult("right", false)))
     }
@@ -192,8 +196,8 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
     "fail after barrier timeout" in {
       val barrier = getBarrier()
       val a, b = TestProbe()
-      val nodeA = NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
-      val nodeB = NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      val nodeA = NodeInfo(A, AddressFromURIString("akka://sys"), a.ref)
+      val nodeB = NodeInfo(B, AddressFromURIString("akka://sys"), b.ref)
       barrier ! nodeA
       barrier ! nodeB
       a.send(barrier, EnterBarrier("right"))
@@ -205,8 +209,8 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
     "fail if a node registers twice" in {
       val barrier = getBarrier()
       val a, b = TestProbe()
-      val nodeA = NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
-      val nodeB = NodeInfo("a", AddressFromURIString("akka://sys"), b.ref)
+      val nodeA = NodeInfo(A, AddressFromURIString("akka://sys"), a.ref)
+      val nodeB = NodeInfo(A, AddressFromURIString("akka://sys"), b.ref)
       barrier ! nodeA
       EventFilter[DuplicateNode](occurrences = 1) intercept {
         barrier ! nodeB
@@ -224,25 +228,25 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
 
     "register clients and remove them" in {
       val b = getController(1)
-      b ! NodeInfo("a", AddressFromURIString("akka://sys"), testActor)
+      b ! NodeInfo(A, AddressFromURIString("akka://sys"), testActor)
       expectMsg(ToClient(Done))
-      b ! Remove("b")
-      b ! Remove("a")
+      b ! Remove(B)
+      b ! Remove(A)
       EventFilter[BarrierEmpty](occurrences = 1) intercept {
-        b ! Remove("a")
+        b ! Remove(A)
       }
     }
 
     "register clients and disconnect them" in {
       val b = getController(1)
-      b ! NodeInfo("a", AddressFromURIString("akka://sys"), testActor)
+      b ! NodeInfo(A, AddressFromURIString("akka://sys"), testActor)
       expectMsg(ToClient(Done))
-      b ! ClientDisconnected("b")
+      b ! ClientDisconnected(B)
       EventFilter[ClientLost](occurrences = 1) intercept {
-        b ! ClientDisconnected("a")
+        b ! ClientDisconnected(A)
       }
       EventFilter[BarrierEmpty](occurrences = 1) intercept {
-        b ! ClientDisconnected("a")
+        b ! ClientDisconnected(A)
       }
     }
 
@@ -255,8 +259,8 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
     "enter barrier" in {
       val barrier = getController(2)
       val a, b = TestProbe()
-      barrier ! NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
-      barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      barrier ! NodeInfo(A, AddressFromURIString("akka://sys"), a.ref)
+      barrier ! NodeInfo(B, AddressFromURIString("akka://sys"), b.ref)
       a.expectMsg(ToClient(Done))
       b.expectMsg(ToClient(Done))
       a.send(barrier, EnterBarrier("bar"))
@@ -271,12 +275,12 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
     "enter barrier with joining node" in {
       val barrier = getController(2)
       val a, b, c = TestProbe()
-      barrier ! NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
-      barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      barrier ! NodeInfo(A, AddressFromURIString("akka://sys"), a.ref)
+      barrier ! NodeInfo(B, AddressFromURIString("akka://sys"), b.ref)
       a.expectMsg(ToClient(Done))
       b.expectMsg(ToClient(Done))
       a.send(barrier, EnterBarrier("bar"))
-      barrier ! NodeInfo("c", AddressFromURIString("akka://sys"), c.ref)
+      barrier ! NodeInfo(C, AddressFromURIString("akka://sys"), c.ref)
       c.expectMsg(ToClient(Done))
       b.send(barrier, EnterBarrier("bar"))
       noMsg(a, b, c)
@@ -291,34 +295,34 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
     "enter barrier with leaving node" in {
       val barrier = getController(3)
       val a, b, c = TestProbe()
-      barrier ! NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
-      barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
-      barrier ! NodeInfo("c", AddressFromURIString("akka://sys"), c.ref)
+      barrier ! NodeInfo(A, AddressFromURIString("akka://sys"), a.ref)
+      barrier ! NodeInfo(B, AddressFromURIString("akka://sys"), b.ref)
+      barrier ! NodeInfo(C, AddressFromURIString("akka://sys"), c.ref)
       a.expectMsg(ToClient(Done))
       b.expectMsg(ToClient(Done))
       c.expectMsg(ToClient(Done))
       a.send(barrier, EnterBarrier("bar"))
       b.send(barrier, EnterBarrier("bar"))
-      barrier ! Remove("a")
-      barrier ! ClientDisconnected("a")
+      barrier ! Remove(A)
+      barrier ! ClientDisconnected(A)
       noMsg(a, b, c)
       b.within(1 second) {
-        barrier ! Remove("c")
+        barrier ! Remove(C)
         b.expectMsg(ToClient(BarrierResult("bar", true)))
       }
-      barrier ! ClientDisconnected("c")
+      barrier ! ClientDisconnected(C)
       expectNoMsg(1 second)
     }
 
     "leave barrier when last “arrived” is removed" in {
       val barrier = getController(2)
       val a, b = TestProbe()
-      barrier ! NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
-      barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      barrier ! NodeInfo(A, AddressFromURIString("akka://sys"), a.ref)
+      barrier ! NodeInfo(B, AddressFromURIString("akka://sys"), b.ref)
       a.expectMsg(ToClient(Done))
       b.expectMsg(ToClient(Done))
       a.send(barrier, EnterBarrier("bar"))
-      barrier ! Remove("a")
+      barrier ! Remove(A)
       b.send(barrier, EnterBarrier("foo"))
       b.expectMsg(ToClient(BarrierResult("foo", true)))
     }
@@ -326,16 +330,16 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
     "fail barrier with disconnecing node" in {
       val barrier = getController(2)
       val a, b = TestProbe()
-      val nodeA = NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
+      val nodeA = NodeInfo(A, AddressFromURIString("akka://sys"), a.ref)
       barrier ! nodeA
-      barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      barrier ! NodeInfo(B, AddressFromURIString("akka://sys"), b.ref)
       a.expectMsg(ToClient(Done))
       b.expectMsg(ToClient(Done))
       a.send(barrier, EnterBarrier("bar"))
-      barrier ! ClientDisconnected("unknown")
+      barrier ! ClientDisconnected(RoleName("unknown"))
       noMsg(a)
       EventFilter[ClientLost](occurrences = 1) intercept {
-        barrier ! ClientDisconnected("b")
+        barrier ! ClientDisconnected(B)
       }
       a.expectMsg(ToClient(BarrierResult("bar", false)))
     }
@@ -343,10 +347,10 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
     "fail barrier with disconnecing node who already arrived" in {
       val barrier = getController(3)
       val a, b, c = TestProbe()
-      val nodeA = NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
-      val nodeC = NodeInfo("c", AddressFromURIString("akka://sys"), c.ref)
+      val nodeA = NodeInfo(A, AddressFromURIString("akka://sys"), a.ref)
+      val nodeC = NodeInfo(C, AddressFromURIString("akka://sys"), c.ref)
       barrier ! nodeA
-      barrier ! NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      barrier ! NodeInfo(B, AddressFromURIString("akka://sys"), b.ref)
       barrier ! nodeC
       a.expectMsg(ToClient(Done))
       b.expectMsg(ToClient(Done))
@@ -354,7 +358,7 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
       a.send(barrier, EnterBarrier("bar"))
       b.send(barrier, EnterBarrier("bar"))
       EventFilter[ClientLost](occurrences = 1) intercept {
-        barrier ! ClientDisconnected("b")
+        barrier ! ClientDisconnected(B)
       }
       a.expectMsg(ToClient(BarrierResult("bar", false)))
     }
@@ -362,9 +366,9 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
     "fail when entering wrong barrier" in {
       val barrier = getController(2)
       val a, b = TestProbe()
-      val nodeA = NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
+      val nodeA = NodeInfo(A, AddressFromURIString("akka://sys"), a.ref)
       barrier ! nodeA
-      val nodeB = NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      val nodeB = NodeInfo(B, AddressFromURIString("akka://sys"), b.ref)
       barrier ! nodeB
       a.expectMsg(ToClient(Done))
       b.expectMsg(ToClient(Done))
@@ -379,8 +383,8 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
     "not really fail after barrier timeout" in {
       val barrier = getController(2)
       val a, b = TestProbe()
-      val nodeA = NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
-      val nodeB = NodeInfo("b", AddressFromURIString("akka://sys"), b.ref)
+      val nodeA = NodeInfo(A, AddressFromURIString("akka://sys"), a.ref)
+      val nodeB = NodeInfo(B, AddressFromURIString("akka://sys"), b.ref)
       barrier ! nodeA
       barrier ! nodeB
       a.expectMsg(ToClient(Done))
@@ -397,8 +401,8 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
     "fail if a node registers twice" in {
       val controller = getController(2)
       val a, b = TestProbe()
-      val nodeA = NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
-      val nodeB = NodeInfo("a", AddressFromURIString("akka://sys"), b.ref)
+      val nodeA = NodeInfo(A, AddressFromURIString("akka://sys"), a.ref)
+      val nodeB = NodeInfo(A, AddressFromURIString("akka://sys"), b.ref)
       controller ! nodeA
       EventFilter[DuplicateNode](occurrences = 1) intercept {
         controller ! nodeB
@@ -410,8 +414,8 @@ class BarrierSpec extends AkkaSpec(BarrierSpec.config) with ImplicitSender with
     "fail subsequent barriers if a node registers twice" in {
       val controller = getController(1)
       val a, b = TestProbe()
-      val nodeA = NodeInfo("a", AddressFromURIString("akka://sys"), a.ref)
-      val nodeB = NodeInfo("a", AddressFromURIString("akka://sys"), b.ref)
+      val nodeA = NodeInfo(A, AddressFromURIString("akka://sys"), a.ref)
+      val nodeB = NodeInfo(A, AddressFromURIString("akka://sys"), b.ref)
       controller ! nodeA
       a.expectMsg(ToClient(Done))
       EventFilter[DuplicateNode](occurrences = 1) intercept {
diff --git a/akka-remote-tests/src/test/scala/akka/remote/testconductor/ControllerSpec.scala b/akka-remote-tests/src/test/scala/akka/remote/testconductor/ControllerSpec.scala
index c4e0ca6cd0..13140adfb5 100644
--- a/akka-remote-tests/src/test/scala/akka/remote/testconductor/ControllerSpec.scala
+++ b/akka-remote-tests/src/test/scala/akka/remote/testconductor/ControllerSpec.scala
@@ -23,16 +23,19 @@ object ControllerSpec {
 
 class ControllerSpec extends AkkaSpec(ControllerSpec.config) with ImplicitSender {
 
+  val A = RoleName("a")
+  val B = RoleName("b")
+
   "A Controller" must {
 
     "publish its nodes" in {
       val c = system.actorOf(Props(new Controller(1, new InetSocketAddress(InetAddress.getLocalHost, 0))))
-      c ! NodeInfo("a", AddressFromURIString("akka://sys"), testActor)
+      c ! NodeInfo(A, AddressFromURIString("akka://sys"), testActor)
       expectMsg(ToClient(Done))
-      c ! NodeInfo("b", AddressFromURIString("akka://sys"), testActor)
+      c ! NodeInfo(B, AddressFromURIString("akka://sys"), testActor)
       expectMsg(ToClient(Done))
       c ! Controller.GetNodes
-      expectMsgType[Iterable[String]].toSet must be(Set("a", "b"))
+      expectMsgType[Iterable[RoleName]].toSet must be(Set(A, B))
     }
 
   }
diff --git a/akka-remote-tests/src/test/scala/akka/remote/testkit/MultiNodeSpec.scala b/akka-remote-tests/src/test/scala/akka/remote/testkit/MultiNodeSpec.scala
index 7acde4eac9..92e65247fb 100644
--- a/akka-remote-tests/src/test/scala/akka/remote/testkit/MultiNodeSpec.scala
+++ b/akka-remote-tests/src/test/scala/akka/remote/testkit/MultiNodeSpec.scala
@@ -16,6 +16,50 @@ import akka.dispatch.Await
 import akka.util.Duration
 import akka.actor.ActorPath
 import akka.actor.RootActorPath
+import akka.remote.testconductor.RoleName
+
+/**
+ * Configure the role names and participants of the test, including configuration settings.
+ */
+abstract class MultiNodeConfig {
+
+  private var _commonConf: Option[Config] = None
+  private var _nodeConf = Map[RoleName, Config]()
+  private var _roles = Seq[RoleName]()
+
+  /**
+   * Register a common base config for all test participants, if so desired.
+   */
+  def commonConfig(config: Config): Unit = _commonConf = Some(config)
+
+  /**
+   * Register a config override for a specific participant.
+   */
+  def nodeConfig(role: RoleName, config: Config): Unit = _nodeConf += role -> config
+
+  /**
+   * Construct a RoleName and return it, to be used as an identifier in the
+   * test. Registration of a role name creates a role which then needs to be
+   * filled.
+   */
+  def role(name: String): RoleName = {
+    if (_roles exists (_.name == name)) throw new IllegalArgumentException("non-unique role name " + name)
+    val r = RoleName(name)
+    _roles :+= r
+    r
+  }
+
+  private[testkit] lazy val mySelf: RoleName = {
+    require(_roles.size > MultiNodeSpec.selfIndex, "not enough roles declared for this test")
+    _roles(MultiNodeSpec.selfIndex)
+  }
+
+  private[testkit] def config: Config = {
+    val configs = (_nodeConf get mySelf).toList ::: _commonConf.toList ::: MultiNodeSpec.nodeConfig :: AkkaSpec.testConf :: Nil
+    configs reduce (_ withFallback _)
+  }
+
+}
 
 object MultiNodeSpec {
 
@@ -52,18 +96,11 @@ object MultiNodeSpec {
 
 }
 
-abstract class MultiNodeSpec(_system: ActorSystem) extends AkkaSpec(_system) {
+abstract class MultiNodeSpec(val mySelf: RoleName, _system: ActorSystem) extends AkkaSpec(_system) {
 
   import MultiNodeSpec._
 
-  def this(config: Config) = this(ActorSystem(AkkaSpec.getCallerName,
-    MultiNodeSpec.nodeConfig.withFallback(config.withFallback(AkkaSpec.testConf))))
-
-  def this(s: String) = this(ConfigFactory.parseString(s))
-
-  def this(configMap: Map[String, _]) = this(AkkaSpec.mapToConfig(configMap))
-
-  def this() = this(AkkaSpec.testConf)
+  def this(config: MultiNodeConfig) = this(config.mySelf, ActorSystem(AkkaSpec.getCallerName, config.config))
 
   /*
    * Test Class Interface
@@ -89,39 +126,17 @@ abstract class MultiNodeSpec(_system: ActorSystem) extends AkkaSpec(_system) {
    */
   val testConductor: TestConductorExt = TestConductor(system)
 
-  /**
-   * TO BE DEFINED BY USER: The test class must define a set of role names to
-   * be used throughout the run, e.g. in naming nodes in failure injections.
-   * These will be mapped to the available nodes such that the first name will
-   * be the Controller, i.e. on this one you can do failure injection.
-   *
-   * Should be a lazy val due to initialization order:
-   * {{{
-   * lazy val roles = Seq("master", "slave")
-   * }}}
-   */
-  def roles: Seq[String]
-
-  require(roles.size >= initialParticipants, "not enough roles for initialParticipants")
-  require(roles.size <= nodeNames.size, "not enough nodes for number of roles")
-  require(roles.distinct.size == roles.size, "role names must be distinct")
-
-  val mySelf = {
-    if (selfIndex >= roles.size) System.exit(0)
-    roles(selfIndex)
-  }
-
   /**
    * Execute the given block of code only on the given nodes (names according
    * to the `roleMap`).
    */
-  def runOn(nodes: String*)(thunk: ⇒ Unit): Unit = {
+  def runOn(nodes: RoleName*)(thunk: ⇒ Unit): Unit = {
     if (nodes exists (_ == mySelf)) {
       thunk
     }
   }
 
-  def ifNode[T](nodes: String*)(yes: ⇒ T)(no: ⇒ T): T = {
+  def ifNode[T](nodes: RoleName*)(yes: ⇒ T)(no: ⇒ T): T = {
     if (nodes exists (_ == mySelf)) yes else no
   }
 
@@ -133,7 +148,7 @@ abstract class MultiNodeSpec(_system: ActorSystem) extends AkkaSpec(_system) {
    * val serviceA = system.actorFor(node("master") / "user" / "serviceA")
    * }}}
    */
-  def node(name: String): ActorPath = RootActorPath(testConductor.getAddressFor(name).await)
+  def node(role: RoleName): ActorPath = RootActorPath(testConductor.getAddressFor(role).await)
 
   /**
    * Enrich `.await()` onto all Awaitables, using BarrierTimeout.
@@ -149,9 +164,9 @@ abstract class MultiNodeSpec(_system: ActorSystem) extends AkkaSpec(_system) {
 
   private val controllerAddr = new InetSocketAddress(nodeNames(0), 4711)
   if (selfIndex == 0) {
-    testConductor.startController(initialParticipants, roles(0), controllerAddr).await
+    testConductor.startController(initialParticipants, mySelf, controllerAddr).await
   } else {
-    testConductor.startClient(roles(selfIndex), controllerAddr).await
+    testConductor.startClient(mySelf, controllerAddr).await
   }
 
 }
\ No newline at end of file

From 162d59db35b680594bcbdae56934807697371845 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jonas=20Bone=CC=81r?= 
Date: Mon, 21 May 2012 08:08:51 +0200
Subject: [PATCH 25/36] Removed ResizerSpec test violating routing rules

---
 .../test/scala/akka/routing/ResizerSpec.scala | 30 -------------------
 1 file changed, 30 deletions(-)

diff --git a/akka-actor-tests/src/test/scala/akka/routing/ResizerSpec.scala b/akka-actor-tests/src/test/scala/akka/routing/ResizerSpec.scala
index 111460e3ac..ede4a69d7c 100644
--- a/akka-actor-tests/src/test/scala/akka/routing/ResizerSpec.scala
+++ b/akka-actor-tests/src/test/scala/akka/routing/ResizerSpec.scala
@@ -128,36 +128,6 @@ class ResizerSpec extends AkkaSpec(ResizerSpec.config) with DefaultTimeout with
       current.routees.size must be(2)
     }
 
-    // FIXME this test violates the rule that you can not use a BalancingDispatcher with any kind of Router - now throws a ConfigurationException in verification process
-    "resize when busy" ignore {
-
-      val busy = new TestLatch(1)
-
-      val resizer = DefaultResizer(
-        lowerBound = 1,
-        upperBound = 3,
-        pressureThreshold = 0,
-        messagesPerResize = 1)
-
-      val router = system.actorOf(Props[BusyActor].withRouter(RoundRobinRouter(resizer = Some(resizer))).withDispatcher("bal-disp"))
-
-      val latch1 = new TestLatch(1)
-      router ! (latch1, busy)
-      Await.ready(latch1, 2 seconds)
-
-      val latch2 = new TestLatch(1)
-      router ! (latch2, busy)
-      Await.ready(latch2, 2 seconds)
-
-      val latch3 = new TestLatch(1)
-      router ! (latch3, busy)
-      Await.ready(latch3, 2 seconds)
-
-      Await.result(router ? CurrentRoutees, 5 seconds).asInstanceOf[RouterRoutees].routees.size must be(3)
-
-      busy.countDown()
-    }
-
     "grow as needed under pressure" in {
       // make sure the pool starts at the expected lower limit and grows to the upper as needed
       // as influenced by the backlog of blocking pooled actors

From 508d8f70a5a24e02e5462f58f747e109fef2daf7 Mon Sep 17 00:00:00 2001
From: Roland 
Date: Tue, 22 May 2012 15:19:45 +0200
Subject: [PATCH 26/36] incorporate review comments into TestConductor work

- protect all internal API using private[akka] and ScalaDoc
- remove package object which was after a previous refactoring only used
  from a single place anyway
- document all public API methods, add brief description how failure
  injector works
- include remoteTests in the top-level aggregate project
---
 .../akka/remote/testconductor/Conductor.scala | 42 ++++++++---
 .../akka/remote/testconductor/DataTypes.scala | 73 +++++++++++++------
 .../akka/remote/testconductor/Extension.scala | 18 ++++-
 .../NetworkFailureInjector.scala              | 24 +++++-
 .../akka/remote/testconductor/Player.scala    | 13 +++-
 .../testconductor/RemoteConnection.scala      | 25 +++++--
 .../TestConductorTransport.scala              |  5 +-
 .../akka/remote/testconductor/package.scala   | 31 --------
 .../remote/netty/NettyRemoteSupport.scala     | 37 +++++++++-
 project/AkkaBuild.scala                       |  2 +-
 10 files changed, 188 insertions(+), 82 deletions(-)
 delete mode 100644 akka-remote-tests/src/main/scala/akka/remote/testconductor/package.scala

diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala
index 6c26fcaae2..1ec172e9ce 100644
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala
+++ b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala
@@ -50,7 +50,7 @@ trait Conductor { this: TestConductorExt ⇒
 
   private var _controller: ActorRef = _
   private def controller: ActorRef = _controller match {
-    case null ⇒ throw new RuntimeException("TestConductorServer was not started")
+    case null ⇒ throw new IllegalStateException("TestConductorServer was not started")
     case x    ⇒ x
   }
 
@@ -169,10 +169,11 @@ trait Conductor { this: TestConductorExt ⇒
    *
    * @param node is the symbolic name of the node which is to be affected
    */
-  def kill(node: RoleName): Future[Done] = {
-    import Settings.QueryTimeout
-    controller ? Terminate(node, -1) mapTo
-  }
+  // TODO: uncomment (and implement in Controller) if really needed
+  //  def kill(node: RoleName): Future[Done] = {
+  //    import Settings.QueryTimeout
+  //    controller ? Terminate(node, -1) mapTo
+  //  }
 
   /**
    * Obtain the list of remote host names currently registered.
@@ -201,8 +202,10 @@ trait Conductor { this: TestConductorExt ⇒
  * This handler is installed at the end of the controller’s netty pipeline. Its only
  * purpose is to dispatch incoming messages to the right ServerFSM actor. There is
  * one shared instance of this class for all connections accepted by one Controller.
+ *
+ * INTERNAL API.
  */
-class ConductorHandler(system: ActorSystem, controller: ActorRef, log: LoggingAdapter) extends SimpleChannelUpstreamHandler {
+private[akka] class ConductorHandler(system: ActorSystem, controller: ActorRef, log: LoggingAdapter) extends SimpleChannelUpstreamHandler {
 
   val clients = new ConcurrentHashMap[Channel, ActorRef]()
 
@@ -235,7 +238,10 @@ class ConductorHandler(system: ActorSystem, controller: ActorRef, log: LoggingAd
 
 }
 
-object ServerFSM {
+/**
+ * INTERNAL API.
+ */
+private[akka] object ServerFSM {
   sealed trait State
   case object Initial extends State
   case object Ready extends State
@@ -253,8 +259,10 @@ object ServerFSM {
  * [[akka.remote.testconductor.Done]] message, and there can be only one such
  * request outstanding at a given time (i.e. a Send fails if the previous has
  * not yet been acknowledged).
+ *
+ * INTERNAL API.
  */
-class ServerFSM(val controller: ActorRef, val channel: Channel) extends Actor with LoggingFSM[ServerFSM.State, Option[ActorRef]] {
+private[akka] class ServerFSM(val controller: ActorRef, val channel: Channel) extends Actor with LoggingFSM[ServerFSM.State, Option[ActorRef]] {
   import ServerFSM._
   import akka.actor.FSM._
   import Controller._
@@ -317,7 +325,10 @@ class ServerFSM(val controller: ActorRef, val channel: Channel) extends Actor wi
   }
 }
 
-object Controller {
+/**
+ * INTERNAL API.
+ */
+private[akka] object Controller {
   case class ClientDisconnected(name: RoleName)
   case object GetNodes
   case object GetSockAddr
@@ -329,8 +340,10 @@ object Controller {
  * This controls test execution by managing barriers (delegated to
  * [[akka.remote.testconductor.BarrierCoordinator]], its child) and allowing
  * network and other failures to be injected at the test nodes.
+ *
+ * INTERNAL API.
  */
-class Controller(private var initialParticipants: Int, controllerPort: InetSocketAddress) extends Actor {
+private[akka] class Controller(private var initialParticipants: Int, controllerPort: InetSocketAddress) extends Actor {
   import Controller._
   import BarrierCoordinator._
 
@@ -418,7 +431,10 @@ class Controller(private var initialParticipants: Int, controllerPort: InetSocke
   }
 }
 
-object BarrierCoordinator {
+/**
+ * INTERNAL API.
+ */
+private[akka] object BarrierCoordinator {
   sealed trait State
   case object Idle extends State
   case object Waiting extends State
@@ -447,8 +463,10 @@ object BarrierCoordinator {
  * EnterBarrier return message. In case of planned removals, this may just happen
  * earlier, in case of failures the current barrier (and all subsequent ones) will
  * be failed by sending BarrierFailed responses.
+ *
+ * INTERNAL API.
  */
-class BarrierCoordinator extends Actor with LoggingFSM[BarrierCoordinator.State, BarrierCoordinator.Data] {
+private[akka] class BarrierCoordinator extends Actor with LoggingFSM[BarrierCoordinator.State, BarrierCoordinator.Data] {
   import BarrierCoordinator._
   import akka.actor.FSM._
   import Controller._
diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/DataTypes.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/DataTypes.scala
index 2bb7d50c37..022ae2d89b 100644
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/DataTypes.scala
+++ b/akka-remote-tests/src/main/scala/akka/remote/testconductor/DataTypes.scala
@@ -13,44 +13,59 @@ import org.jboss.netty.handler.codec.oneone.OneToOneDecoder
 
 case class RoleName(name: String)
 
-case class ToClient(msg: ClientOp with NetworkOp)
-case class ToServer(msg: ServerOp with NetworkOp)
+private[akka] case class ToClient(msg: ClientOp with NetworkOp)
+private[akka] case class ToServer(msg: ServerOp with NetworkOp)
 
-sealed trait ClientOp // messages sent to from Conductor to Player
-sealed trait ServerOp // messages sent to from Player to Conductor
-sealed trait CommandOp // messages sent from TestConductorExt to Conductor
-sealed trait NetworkOp // messages sent over the wire
-sealed trait UnconfirmedClientOp extends ClientOp // unconfirmed messages going to the Player
-sealed trait ConfirmedClientOp extends ClientOp
+private[akka] sealed trait ClientOp // messages sent to from Conductor to Player
+private[akka] sealed trait ServerOp // messages sent to from Player to Conductor
+private[akka] sealed trait CommandOp // messages sent from TestConductorExt to Conductor
+private[akka] sealed trait NetworkOp // messages sent over the wire
+private[akka] sealed trait UnconfirmedClientOp extends ClientOp // unconfirmed messages going to the Player
+private[akka] sealed trait ConfirmedClientOp extends ClientOp
 
 /**
  * First message of connection sets names straight.
  */
-case class Hello(name: String, addr: Address) extends NetworkOp
+private[akka] case class Hello(name: String, addr: Address) extends NetworkOp
 
-case class EnterBarrier(name: String) extends ServerOp with NetworkOp
-case class BarrierResult(name: String, success: Boolean) extends UnconfirmedClientOp with NetworkOp
+private[akka] case class EnterBarrier(name: String) extends ServerOp with NetworkOp
+private[akka] case class BarrierResult(name: String, success: Boolean) extends UnconfirmedClientOp with NetworkOp
 
-case class Throttle(node: RoleName, target: RoleName, direction: Direction, rateMBit: Float) extends CommandOp
-case class ThrottleMsg(target: Address, direction: Direction, rateMBit: Float) extends ConfirmedClientOp with NetworkOp
+private[akka] case class Throttle(node: RoleName, target: RoleName, direction: Direction, rateMBit: Float) extends CommandOp
+private[akka] case class ThrottleMsg(target: Address, direction: Direction, rateMBit: Float) extends ConfirmedClientOp with NetworkOp
 
-case class Disconnect(node: RoleName, target: RoleName, abort: Boolean) extends CommandOp
-case class DisconnectMsg(target: Address, abort: Boolean) extends ConfirmedClientOp with NetworkOp
+private[akka] case class Disconnect(node: RoleName, target: RoleName, abort: Boolean) extends CommandOp
+private[akka] case class DisconnectMsg(target: Address, abort: Boolean) extends ConfirmedClientOp with NetworkOp
 
-case class Terminate(node: RoleName, exitValueOrKill: Int) extends CommandOp
-case class TerminateMsg(exitValue: Int) extends ConfirmedClientOp with NetworkOp
+private[akka] case class Terminate(node: RoleName, exitValueOrKill: Int) extends CommandOp
+private[akka] case class TerminateMsg(exitValue: Int) extends ConfirmedClientOp with NetworkOp
 
-case class GetAddress(node: RoleName) extends ServerOp with NetworkOp
-case class AddressReply(node: RoleName, addr: Address) extends UnconfirmedClientOp with NetworkOp
+private[akka] case class GetAddress(node: RoleName) extends ServerOp with NetworkOp
+private[akka] case class AddressReply(node: RoleName, addr: Address) extends UnconfirmedClientOp with NetworkOp
 
-abstract class Done extends ServerOp with UnconfirmedClientOp with NetworkOp
-case object Done extends Done {
+private[akka] abstract class Done extends ServerOp with UnconfirmedClientOp with NetworkOp
+private[akka] case object Done extends Done {
   def getInstance: Done = this
 }
 
-case class Remove(node: RoleName) extends CommandOp
+private[akka] case class Remove(node: RoleName) extends CommandOp
+
+private[akka] class MsgEncoder extends OneToOneEncoder {
+
+  implicit def address2proto(addr: Address): TCP.Address =
+    TCP.Address.newBuilder
+      .setProtocol(addr.protocol)
+      .setSystem(addr.system)
+      .setHost(addr.host.get)
+      .setPort(addr.port.get)
+      .build
+
+  implicit def direction2proto(dir: Direction): TCP.Direction = dir match {
+    case Direction.Send    ⇒ TCP.Direction.Send
+    case Direction.Receive ⇒ TCP.Direction.Receive
+    case Direction.Both    ⇒ TCP.Direction.Both
+  }
 
-class MsgEncoder extends OneToOneEncoder {
   def encode(ctx: ChannelHandlerContext, ch: Channel, msg: AnyRef): AnyRef = msg match {
     case x: NetworkOp ⇒
       val w = TCP.Wrapper.newBuilder
@@ -81,7 +96,17 @@ class MsgEncoder extends OneToOneEncoder {
   }
 }
 
-class MsgDecoder extends OneToOneDecoder {
+private[akka] class MsgDecoder extends OneToOneDecoder {
+
+  implicit def address2scala(addr: TCP.Address): Address =
+    Address(addr.getProtocol, addr.getSystem, addr.getHost, addr.getPort)
+
+  implicit def direction2scala(dir: TCP.Direction): Direction = dir match {
+    case TCP.Direction.Send    ⇒ Direction.Send
+    case TCP.Direction.Receive ⇒ Direction.Receive
+    case TCP.Direction.Both    ⇒ Direction.Both
+  }
+
   def decode(ctx: ChannelHandlerContext, ch: Channel, msg: AnyRef): AnyRef = msg match {
     case w: TCP.Wrapper if w.getAllFields.size == 1 ⇒
       if (w.hasHello) {
diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Extension.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Extension.scala
index 7f6b576128..6800253ae0 100644
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Extension.scala
+++ b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Extension.scala
@@ -32,6 +32,9 @@ object TestConductor extends ExtensionKey[TestConductorExt] {
  * [[akka.remote.testconductor.Player]] roles inside an Akka
  * [[akka.actor.Extension]]. Please follow the aforementioned links for
  * more information.
+ *
+ * This extension requires the `akka.actor.provider`
+ * to be a [[akka.remote.RemoteActorRefProvider]].
  */
 class TestConductorExt(val system: ExtendedActorSystem) extends Extension with Conductor with Player {
 
@@ -47,9 +50,22 @@ class TestConductorExt(val system: ExtendedActorSystem) extends Extension with C
     val PacketSplitThreshold = Duration(config.getMilliseconds("akka.testconductor.packet-split-threshold"), MILLISECONDS)
   }
 
+  /**
+   * Remote transport used by the actor ref provider.
+   */
   val transport = system.provider.asInstanceOf[RemoteActorRefProvider].transport
+
+  /**
+   * Transport address of this Netty-like remote transport.
+   */
   val address = transport.address
 
-  val failureInjectors = new ConcurrentHashMap[Address, FailureInjector]
+  /**
+   * INTERNAL API.
+   *
+   * [[akka.remote.testconductor.FailureInjector]]s register themselves here so that
+   * failures can be injected.
+   */
+  private[akka] val failureInjectors = new ConcurrentHashMap[Address, FailureInjector]
 
 }
\ No newline at end of file
diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala
index b853523979..629a15d51f 100644
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala
+++ b/akka-remote-tests/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala
@@ -31,7 +31,10 @@ import org.jboss.netty.channel.ChannelFuture
 import org.jboss.netty.channel.ChannelFutureListener
 import org.jboss.netty.channel.ChannelFuture
 
-case class FailureInjector(sender: ActorRef, receiver: ActorRef) {
+/**
+ * INTERNAL API.
+ */
+private[akka] case class FailureInjector(sender: ActorRef, receiver: ActorRef) {
   def refs(dir: Direction) = dir match {
     case Direction.Send    ⇒ Seq(sender)
     case Direction.Receive ⇒ Seq(receiver)
@@ -39,12 +42,27 @@ case class FailureInjector(sender: ActorRef, receiver: ActorRef) {
   }
 }
 
-object NetworkFailureInjector {
+/**
+ * INTERNAL API.
+ */
+private[akka] object NetworkFailureInjector {
   case class SetRate(rateMBit: Float)
   case class Disconnect(abort: Boolean)
 }
 
-class NetworkFailureInjector(system: ActorSystem) extends SimpleChannelHandler {
+/**
+ * Brief overview: all network traffic passes through the `sender`/`receiver` FSMs, which can
+ * pass through requests immediately, drop them or throttle to a desired rate. The FSMs are
+ * registered in the TestConductorExt.failureInjectors so that settings can be applied from
+ * the ClientFSMs.
+ *
+ * I found that simply forwarding events using ctx.sendUpstream/sendDownstream does not work,
+ * it deadlocks and gives strange errors; in the end I just trusted the Netty docs which
+ * recommend to prefer `Channels.write()` and `Channels.fireMessageReceived()`.
+ *
+ * INTERNAL API.
+ */
+private[akka] class NetworkFailureInjector(system: ActorSystem) extends SimpleChannelHandler {
 
   val log = Logging(system, "FailureInjector")
 
diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Player.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Player.scala
index 10434007e1..2a4eeb6ad1 100644
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Player.scala
+++ b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Player.scala
@@ -94,7 +94,10 @@ trait Player { this: TestConductorExt ⇒
   }
 }
 
-object ClientFSM {
+/**
+ * INTERNAL API.
+ */
+private[akka] object ClientFSM {
   sealed trait State
   case object Connecting extends State
   case object AwaitDone extends State
@@ -116,8 +119,10 @@ object ClientFSM {
  * done the same. After that, it will pass barrier requests to and from the
  * coordinator and react to the [[akka.remote.testconductor.Conductor]]’s
  * requests for failure injection.
+ *
+ * INTERNAL API.
  */
-class ClientFSM(name: RoleName, controllerAddr: InetSocketAddress) extends Actor with LoggingFSM[ClientFSM.State, ClientFSM.Data] {
+private[akka] class ClientFSM(name: RoleName, controllerAddr: InetSocketAddress) extends Actor with LoggingFSM[ClientFSM.State, ClientFSM.Data] {
   import ClientFSM._
 
   val settings = TestConductor().Settings
@@ -236,8 +241,10 @@ class ClientFSM(name: RoleName, controllerAddr: InetSocketAddress) extends Actor
 
 /**
  * This handler only forwards messages received from the conductor to the [[akka.remote.testconductor.ClientFSM]].
+ *
+ * INTERNAL API.
  */
-class PlayerHandler(
+private[akka] class PlayerHandler(
   server: InetSocketAddress,
   private var reconnects: Int,
   backoff: Duration,
diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/RemoteConnection.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/RemoteConnection.scala
index 5b1c454b0c..5aeb484c42 100644
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/RemoteConnection.scala
+++ b/akka-remote-tests/src/main/scala/akka/remote/testconductor/RemoteConnection.scala
@@ -13,7 +13,10 @@ import org.jboss.netty.handler.timeout.{ ReadTimeoutHandler, ReadTimeoutExceptio
 import java.net.InetSocketAddress
 import java.util.concurrent.Executors
 
-class TestConductorPipelineFactory(handler: ChannelUpstreamHandler) extends ChannelPipelineFactory {
+/**
+ * INTERNAL API.
+ */
+private[akka] class TestConductorPipelineFactory(handler: ChannelUpstreamHandler) extends ChannelPipelineFactory {
   def getPipeline: ChannelPipeline = {
     val encap = List(new LengthFieldPrepender(4), new LengthFieldBasedFrameDecoder(10000, 0, 4, 0, 4))
     val proto = List(new ProtobufEncoder, new ProtobufDecoder(TestConductorProtocol.Wrapper.getDefaultInstance))
@@ -22,11 +25,23 @@ class TestConductorPipelineFactory(handler: ChannelUpstreamHandler) extends Chan
   }
 }
 
-sealed trait Role
-case object Client extends Role
-case object Server extends Role
+/**
+ * INTERNAL API.
+ */
+private[akka] sealed trait Role
+/**
+ * INTERNAL API.
+ */
+private[akka] case object Client extends Role
+/**
+ * INTERNAL API.
+ */
+private[akka] case object Server extends Role
 
-object RemoteConnection {
+/**
+ * INTERNAL API.
+ */
+private[akka] object RemoteConnection {
   def apply(role: Role, sockaddr: InetSocketAddress, handler: ChannelUpstreamHandler): Channel = {
     role match {
       case Client ⇒
diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/TestConductorTransport.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/TestConductorTransport.scala
index 2c51c2cf18..a036bcfff0 100644
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/TestConductorTransport.scala
+++ b/akka-remote-tests/src/main/scala/akka/remote/testconductor/TestConductorTransport.scala
@@ -10,7 +10,10 @@ import akka.remote.RemoteActorRefProvider
 import org.jboss.netty.channel.ChannelHandler
 import org.jboss.netty.channel.ChannelPipelineFactory
 
-class TestConductorTransport(_remoteSettings: RemoteSettings, _system: ActorSystemImpl, _provider: RemoteActorRefProvider)
+/**
+ * INTERNAL API.
+ */
+private[akka] class TestConductorTransport(_remoteSettings: RemoteSettings, _system: ActorSystemImpl, _provider: RemoteActorRefProvider)
   extends NettyRemoteTransport(_remoteSettings, _system, _provider) {
 
   override def createPipeline(endpoint: ⇒ ChannelHandler, withTimeout: Boolean): ChannelPipelineFactory =
diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/package.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/package.scala
deleted file mode 100644
index b24279dbf6..0000000000
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/package.scala
+++ /dev/null
@@ -1,31 +0,0 @@
-package akka.remote
-
-import akka.actor.Address
-import testconductor.{ TestConductorProtocol ⇒ TCP }
-
-package object testconductor {
-
-  implicit def address2proto(addr: Address): TCP.Address =
-    TCP.Address.newBuilder
-      .setProtocol(addr.protocol)
-      .setSystem(addr.system)
-      .setHost(addr.host.get)
-      .setPort(addr.port.get)
-      .build
-
-  implicit def address2scala(addr: TCP.Address): Address =
-    Address(addr.getProtocol, addr.getSystem, addr.getHost, addr.getPort)
-
-  implicit def direction2proto(dir: Direction): TCP.Direction = dir match {
-    case Direction.Send    ⇒ TCP.Direction.Send
-    case Direction.Receive ⇒ TCP.Direction.Receive
-    case Direction.Both    ⇒ TCP.Direction.Both
-  }
-
-  implicit def direction2scala(dir: TCP.Direction): Direction = dir match {
-    case TCP.Direction.Send    ⇒ Direction.Send
-    case TCP.Direction.Receive ⇒ Direction.Receive
-    case TCP.Direction.Both    ⇒ Direction.Both
-  }
-
-}
\ No newline at end of file
diff --git a/akka-remote/src/main/scala/akka/remote/netty/NettyRemoteSupport.scala b/akka-remote/src/main/scala/akka/remote/netty/NettyRemoteSupport.scala
index 60c2ac6097..f0b6cd1870 100644
--- a/akka-remote/src/main/scala/akka/remote/netty/NettyRemoteSupport.scala
+++ b/akka-remote/src/main/scala/akka/remote/netty/NettyRemoteSupport.scala
@@ -46,13 +46,31 @@ class NettyRemoteTransport(val remoteSettings: RemoteSettings, val system: Actor
     Executors.newCachedThreadPool(system.threadFactory),
     Executors.newCachedThreadPool(system.threadFactory))
 
+  /**
+   * Backing scaffolding for the default implementation of NettyRemoteSupport.createPipeline.
+   */
   object PipelineFactory {
+    /**
+     * Construct a StaticChannelPipeline from a sequence of handlers; to be used
+     * in implementations of ChannelPipelineFactory.
+     */
     def apply(handlers: Seq[ChannelHandler]): StaticChannelPipeline = new StaticChannelPipeline(handlers: _*)
+
+    /**
+     * Constructs the NettyRemoteTransport default pipeline with the give “head” handler, which
+     * is taken by-name to allow it not to be shared across pipelines.
+     *
+     * @param withTimeout determines whether an IdleStateHandler shall be included
+     */
     def apply(endpoint: ⇒ Seq[ChannelHandler], withTimeout: Boolean): ChannelPipelineFactory =
       new ChannelPipelineFactory {
         def getPipeline = apply(defaultStack(withTimeout) ++ endpoint)
       }
 
+    /**
+     * Construct a default protocol stack, excluding the “head” handler (i.e. the one which
+     * actually dispatches the received messages to the local target actors).
+     */
     def defaultStack(withTimeout: Boolean): Seq[ChannelHandler] =
       (if (withTimeout) timeout :: Nil else Nil) :::
         msgFormat :::
@@ -60,17 +78,28 @@ class NettyRemoteTransport(val remoteSettings: RemoteSettings, val system: Actor
         executionHandler ::
         Nil
 
+    /**
+     * Construct an IdleStateHandler which uses [[akka.remote.netty.NettyRemoteTransport]].timer.
+     */
     def timeout = new IdleStateHandler(timer,
       settings.ReadTimeout.toSeconds.toInt,
       settings.WriteTimeout.toSeconds.toInt,
       settings.AllTimeout.toSeconds.toInt)
 
+    /**
+     * Construct frame&protobuf encoder/decoder.
+     */
     def msgFormat = new LengthFieldBasedFrameDecoder(settings.MessageFrameSize, 0, 4, 0, 4) ::
       new LengthFieldPrepender(4) ::
       new RemoteMessageDecoder ::
       new RemoteMessageEncoder(NettyRemoteTransport.this) ::
       Nil
 
+    /**
+     * Construct an ExecutionHandler which is used to ensure that message dispatch does not
+     * happen on a netty thread (that could be bad if re-sending over the network for
+     * remote-deployed actors).
+     */
     val executionHandler = new ExecutionHandler(new OrderedMemoryAwareThreadPoolExecutor(
       settings.ExecutionPoolSize,
       settings.MaxChannelMemorySize,
@@ -79,6 +108,11 @@ class NettyRemoteTransport(val remoteSettings: RemoteSettings, val system: Actor
       settings.ExecutionPoolKeepalive.unit,
       system.threadFactory))
 
+    /**
+     * Construct and authentication handler which uses the SecureCookie to somewhat
+     * protect the TCP port from unauthorized use (don’t rely on it too much, though,
+     * as this is NOT a cryptographic feature).
+     */
     def authenticator = if (settings.RequireCookie) new RemoteServerAuthenticationHandler(settings.SecureCookie) :: Nil else Nil
   }
 
@@ -98,7 +132,8 @@ class NettyRemoteTransport(val remoteSettings: RemoteSettings, val system: Actor
 
   /**
    * Override this method to inject a subclass of NettyRemoteServer instead of
-   * the normal one, e.g. for inserting security hooks.
+   * the normal one, e.g. for inserting security hooks. If this method throws
+   * an exception, the transport will shut itself down and re-throw.
    */
   protected def createServer(): NettyRemoteServer = new NettyRemoteServer(this)
 
diff --git a/project/AkkaBuild.scala b/project/AkkaBuild.scala
index b899bdec45..26eab59037 100644
--- a/project/AkkaBuild.scala
+++ b/project/AkkaBuild.scala
@@ -32,7 +32,7 @@ object AkkaBuild extends Build {
       Unidoc.unidocExclude := Seq(samples.id, tutorials.id),
       Dist.distExclude := Seq(actorTests.id, akkaSbtPlugin.id, docs.id)
     ),
-    aggregate = Seq(actor, testkit, actorTests, remote, camel, cluster, slf4j, agent, transactor, mailboxes, zeroMQ, kernel, akkaSbtPlugin, actorMigration, samples, tutorials, docs)
+    aggregate = Seq(actor, testkit, actorTests, remote, remoteTests, camel, cluster, slf4j, agent, transactor, mailboxes, zeroMQ, kernel, akkaSbtPlugin, actorMigration, samples, tutorials, docs)
   )
 
   lazy val actor = Project(

From a211e4daf6a2e866546813a3303cd866b1fc9d63 Mon Sep 17 00:00:00 2001
From: Patrik Nordwall 
Date: Tue, 22 May 2012 16:35:27 +0200
Subject: [PATCH 27/36] Adjusted tests to latest testconductor, remote deploy
 still fails. See #2069

---
 .../{SimpleRemoteMultiJvmSpec.scala => SimpleRemoteSpec.scala}  | 0
 project/plugins.sbt                                             | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename akka-remote-tests/src/multi-jvm/scala/akka/remote/{SimpleRemoteMultiJvmSpec.scala => SimpleRemoteSpec.scala} (100%)

diff --git a/akka-remote-tests/src/multi-jvm/scala/akka/remote/SimpleRemoteMultiJvmSpec.scala b/akka-remote-tests/src/multi-jvm/scala/akka/remote/SimpleRemoteSpec.scala
similarity index 100%
rename from akka-remote-tests/src/multi-jvm/scala/akka/remote/SimpleRemoteMultiJvmSpec.scala
rename to akka-remote-tests/src/multi-jvm/scala/akka/remote/SimpleRemoteSpec.scala
diff --git a/project/plugins.sbt b/project/plugins.sbt
index f49cfb688d..0a7f9999a7 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -1,7 +1,7 @@
 
 resolvers += Classpaths.typesafeResolver
 
-addSbtPlugin("com.typesafe.sbtmultijvm" % "sbt-multi-jvm" % "0.2.0-SNAPSHOT")
+addSbtPlugin("com.typesafe.sbtmultijvm" % "sbt-multi-jvm" % "0.2.0-M1")
 
 addSbtPlugin("com.typesafe.schoir" % "schoir" % "0.1.2")
 

From 1577bffe17db53da665c843a71de5d1a2689b012 Mon Sep 17 00:00:00 2001
From: Roland 
Date: Wed, 23 May 2012 09:25:12 +0200
Subject: [PATCH 28/36] =?UTF-8?q?make=20too=20verbose=20logging=20in=20Tes?=
 =?UTF-8?q?tConductorSpec=20go=20away=20(d=E2=80=99oh)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../scala/akka/remote/testconductor/TestConductorSpec.scala     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/akka-remote-tests/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala b/akka-remote-tests/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
index 087aac55c7..e311fa0023 100644
--- a/akka-remote-tests/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
+++ b/akka-remote-tests/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
@@ -17,7 +17,7 @@ import akka.remote.testkit.MultiNodeConfig
 
 object TestConductorMultiJvmSpec extends MultiNodeConfig {
   commonConfig(ConfigFactory.parseString("""
-    akka.loglevel = DEBUG
+    # akka.loglevel = DEBUG
     akka.remote {
       log-received-messages = on
       log-sent-messages = on

From 12ff07f0251525fc7232f25570432241b938f966 Mon Sep 17 00:00:00 2001
From: Patrik Nordwall 
Date: Wed, 23 May 2012 09:26:20 +0200
Subject: [PATCH 29/36] Adjusted tests to latest testconductor, remote deploy
 still fails. See #2069

---
 .../scala/akka/remote/SimpleRemoteSpec.scala  |  97 ++++++--------
 .../DirectRoutedRemoteActorMultiJvmSpec.scala | 118 +++++++++---------
 2 files changed, 97 insertions(+), 118 deletions(-)

diff --git a/akka-remote-tests/src/multi-jvm/scala/akka/remote/SimpleRemoteSpec.scala b/akka-remote-tests/src/multi-jvm/scala/akka/remote/SimpleRemoteSpec.scala
index 9209deb9a5..70cca7c34b 100644
--- a/akka-remote-tests/src/multi-jvm/scala/akka/remote/SimpleRemoteSpec.scala
+++ b/akka-remote-tests/src/multi-jvm/scala/akka/remote/SimpleRemoteSpec.scala
@@ -1,21 +1,19 @@
 /**
- *  Copyright (C) 2009-2011 Typesafe Inc. 
+ *  Copyright (C) 2009-2012 Typesafe Inc. 
  */
 package akka.remote
 
+import com.typesafe.config.ConfigFactory
+
 import akka.actor.Actor
 import akka.actor.ActorRef
 import akka.actor.Props
-import akka.dispatch.Await
 import akka.pattern.ask
-import akka.remote.testconductor.TestConductor
-import akka.testkit.DefaultTimeout
-import akka.testkit.ImplicitSender
-import akka.util.Duration
-import com.typesafe.config.ConfigFactory
+import akka.remote.testkit.MultiNodeConfig
+import akka.remote.testkit.MultiNodeSpec
+import akka.testkit._
 
-object SimpleRemoteMultiJvmSpec extends AbstractRemoteActorMultiJvmSpec {
-  override def NrOfNodes = 2
+object SimpleRemoteMultiJvmSpec extends MultiNodeConfig {
 
   class SomeActor extends Actor with Serializable {
     def receive = {
@@ -23,60 +21,47 @@ object SimpleRemoteMultiJvmSpec extends AbstractRemoteActorMultiJvmSpec {
     }
   }
 
-  override def commonConfig = ConfigFactory.parseString("""
-      akka {
-        loglevel = INFO
-        actor {
-          provider = akka.remote.RemoteActorRefProvider
-          debug {
-            receive = on
-            fsm = on
-          }
-        }
-        remote {
-          transport = akka.remote.testconductor.TestConductorTransport
-          log-received-messages = on
-          log-sent-messages = on
-        }
-        testconductor {
-          host = localhost
-          port = 4712
-        }
-      }""")
+  commonConfig(ConfigFactory.parseString("""
+    akka.loglevel = DEBUG
+    akka.remote {
+      log-received-messages = on
+      log-sent-messages = on
+    }
+    akka.actor.debug {
+      receive = on
+      fsm = on
+    }
+  """))
 
-  def nameConfig(n: Int) = ConfigFactory.parseString("akka.testconductor.name = node" + n).withFallback(nodeConfigs(n))
-}
-
-class SimpleRemoteMultiJvmNode1 extends AkkaRemoteSpec(SimpleRemoteMultiJvmSpec.nameConfig(0)) {
-  import SimpleRemoteMultiJvmSpec._
-  val nodes = NrOfNodes
-  val tc = TestConductor(system)
-
-  "lookup remote actor" in {
-    Await.result(tc.startController(2), Duration.Inf)
-    system.actorOf(Props[SomeActor], "service-hello")
-    tc.enter("begin", "done")
-  }
+  val master = role("master")
+  val slave = role("slave")
 
 }
 
-class SimpleRemoteMultiJvmNode2 extends AkkaRemoteSpec(SimpleRemoteMultiJvmSpec.nameConfig(1))
+class SimpleRemoteMultiJvmNode1 extends SimpleRemoteSpec
+class SimpleRemoteMultiJvmNode2 extends SimpleRemoteSpec
+
+class SimpleRemoteSpec extends MultiNodeSpec(SimpleRemoteMultiJvmSpec)
   with ImplicitSender with DefaultTimeout {
-
   import SimpleRemoteMultiJvmSpec._
-  val nodes = NrOfNodes
-  val tc = TestConductor(system)
 
-  "lookup remote actor" in {
-    Await.result(tc.startClient(4712), Duration.Inf)
-    tc.enter("begin")
-    log.info("### begin ok")
-    val actor = system.actorFor("akka://" + akkaSpec(0) + "/user/service-hello")
-    log.info("### actor lookup " + akkaSpec(0) + "/service-hello")
-    actor.isInstanceOf[RemoteActorRef] must be(true)
-    Await.result(actor ? "identify", timeout.duration).asInstanceOf[ActorRef].path.address.hostPort must equal(akkaSpec(0))
-    log.info("### actor ok")
-    tc.enter("done")
+  def initialParticipants = 2
+
+  runOn(master) {
+    system.actorOf(Props[SomeActor], "service-hello")
+  }
+
+  "Remoting" must {
+    "lookup remote actor" in {
+      runOn(slave) {
+        val hello = system.actorFor(node(master) / "user" / "service-hello")
+        hello.isInstanceOf[RemoteActorRef] must be(true)
+        val masterAddress = testConductor.getAddressFor(master).await
+        (hello ? "identify").await.asInstanceOf[ActorRef].path.address must equal(masterAddress)
+      }
+      testConductor.enter("done")
+    }
   }
 
 }
+
diff --git a/akka-remote-tests/src/multi-jvm/scala/akka/remote/router/DirectRoutedRemoteActorMultiJvmSpec.scala b/akka-remote-tests/src/multi-jvm/scala/akka/remote/router/DirectRoutedRemoteActorMultiJvmSpec.scala
index d44beff605..2690378ef1 100644
--- a/akka-remote-tests/src/multi-jvm/scala/akka/remote/router/DirectRoutedRemoteActorMultiJvmSpec.scala
+++ b/akka-remote-tests/src/multi-jvm/scala/akka/remote/router/DirectRoutedRemoteActorMultiJvmSpec.scala
@@ -1,20 +1,20 @@
 /**
- *  Copyright (C) 2009-2011 Typesafe Inc. 
+ *  Copyright (C) 2009-2012 Typesafe Inc. 
  */
 package akka.remote.router
 
-import akka.actor.{ Actor, ActorRef, Props }
-import akka.remote.AkkaRemoteSpec
-import akka.remote.AbstractRemoteActorMultiJvmSpec
-import akka.remote.RemoteActorRef
-import akka.remote.testconductor.TestConductor
-import akka.testkit._
-import akka.dispatch.Await
-import akka.pattern.ask
-import akka.util.Duration
+import com.typesafe.config.ConfigFactory
 
-object DirectRoutedRemoteActorMultiJvmSpec extends AbstractRemoteActorMultiJvmSpec {
-  override def NrOfNodes = 2
+import akka.actor.Actor
+import akka.actor.ActorRef
+import akka.actor.Props
+import akka.pattern.ask
+import akka.remote.RemoteActorRef
+import akka.remote.testkit.MultiNodeConfig
+import akka.remote.testkit.MultiNodeSpec
+import akka.testkit._
+
+object DirectRoutedRemoteActorMultiJvmSpec extends MultiNodeConfig {
 
   class SomeActor extends Actor with Serializable {
     def receive = {
@@ -23,68 +23,62 @@ object DirectRoutedRemoteActorMultiJvmSpec extends AbstractRemoteActorMultiJvmSp
   }
 
   import com.typesafe.config.ConfigFactory
-  override def commonConfig = ConfigFactory.parseString("""
-      akka {
-        loglevel = INFO
-        actor {
-          provider = akka.remote.RemoteActorRefProvider
-          deployment {
-            /service-hello.remote = %s
-          }
-          debug {
-            receive = on
-            fsm = on
-          }
-        }
-        remote {
-          transport = akka.remote.testconductor.TestConductorTransport
-          log-received-messages = on
-          log-sent-messages = on
-        }
-        testconductor {
-          host = localhost
-          port = 4712
-        }
-      }""" format akkaURIs(1))
-
-  def nameConfig(n: Int) = ConfigFactory.parseString("akka.testconductor.name = node" + n).withFallback(nodeConfigs(n))
-}
-
-class DirectRoutedRemoteActorMultiJvmNode1 extends AkkaRemoteSpec(DirectRoutedRemoteActorMultiJvmSpec.nameConfig(0)) {
-  import DirectRoutedRemoteActorMultiJvmSpec._
-  val nodes = NrOfNodes
-  val tc = TestConductor(system)
-
-  "A new remote actor configured with a Direct router" must {
-    "be locally instantiated on a remote node and be able to communicate through its RemoteActorRef" in {
-      Await.result(tc.startController(2), Duration.Inf)
-      tc.enter("begin", "done")
+  commonConfig(ConfigFactory.parseString("""
+    akka.loglevel = DEBUG
+    akka.remote {
+      log-received-messages = on
+      log-sent-messages = on
     }
-  }
+    akka.actor.debug {
+      receive = on
+      fsm = on
+    }
+  """))
+
+  val master = role("master")
+  val slave = role("slave")
+
+  nodeConfig(master, ConfigFactory.parseString("""
+    akka.actor {
+      deployment {
+        /service-hello.remote = "akka://MultiNodeSpec@%s"
+      }
+    }
+    # FIXME When using NettyRemoteTransport instead of TestConductorTransport it works
+    # akka.remote.transport = "akka.remote.netty.NettyRemoteTransport"
+  """.format("localhost:2553"))) // FIXME is there a way to avoid hardcoding the host:port here?
+
+  nodeConfig(slave, ConfigFactory.parseString("""
+    akka.remote.netty.port = 2553
+  """))
 
 }
 
-class DirectRoutedRemoteActorMultiJvmNode2 extends AkkaRemoteSpec(DirectRoutedRemoteActorMultiJvmSpec.nameConfig(1))
+class DirectRoutedRemoteActorMultiJvmNode1 extends DirectRoutedRemoteActorSpec
+class DirectRoutedRemoteActorMultiJvmNode2 extends DirectRoutedRemoteActorSpec
+
+class DirectRoutedRemoteActorSpec extends MultiNodeSpec(DirectRoutedRemoteActorMultiJvmSpec)
   with ImplicitSender with DefaultTimeout {
-
   import DirectRoutedRemoteActorMultiJvmSpec._
-  val nodes = NrOfNodes
-  val tc = TestConductor(system)
+
+  def initialParticipants = 2
 
   "A new remote actor configured with a Direct router" must {
     "be locally instantiated on a remote node and be able to communicate through its RemoteActorRef" in {
-      Await.result(tc.startClient(4712), Duration.Inf)
-      tc.enter("begin")
 
-      val actor = system.actorOf(Props[SomeActor], "service-hello")
-      actor.isInstanceOf[RemoteActorRef] must be(true)
+      runOn(master) {
+        val actor = system.actorOf(Props[SomeActor], "service-hello")
+        actor.isInstanceOf[RemoteActorRef] must be(true)
 
-      Await.result(actor ? "identify", timeout.duration).asInstanceOf[ActorRef].path.address.hostPort must equal(akkaSpec(0))
+        val slaveAddress = testConductor.getAddressFor(slave).await
+        (actor ? "identify").await.asInstanceOf[ActorRef].path.address must equal(slaveAddress)
 
-      // shut down the actor before we let the other node(s) shut down so we don't try to send
-      // "Terminate" to a shut down node
-      system.stop(actor)
-      tc.enter("done")
+        // shut down the actor before we let the other node(s) shut down so we don't try to send
+        // "Terminate" to a shut down node
+        system.stop(actor)
+      }
+
+      testConductor.enter("done")
     }
   }
 }

From 3ab02e95199cb3b386e8f25a9b50138500fefc8d Mon Sep 17 00:00:00 2001
From: Patrik Nordwall 
Date: Wed, 23 May 2012 16:35:42 +0200
Subject: [PATCH 30/36] Convert NodeStartupSpec to MultiNodeSpec. See #1948

---
 .../scala/akka/cluster/NodeStartupSpec.scala  | 87 +++++++++++++++++++
 .../scala/akka/cluster/NodeStartupSpec.scala  | 84 ------------------
 .../testconductor/TestConductorSpec.scala     | 19 ++--
 .../akka/remote/testkit/MultiNodeSpec.scala   | 19 ++++
 project/AkkaBuild.scala                       |  2 +-
 5 files changed, 113 insertions(+), 98 deletions(-)
 create mode 100644 akka-cluster/src/multi-jvm/scala/akka/cluster/NodeStartupSpec.scala
 delete mode 100644 akka-cluster/src/test/scala/akka/cluster/NodeStartupSpec.scala

diff --git a/akka-cluster/src/multi-jvm/scala/akka/cluster/NodeStartupSpec.scala b/akka-cluster/src/multi-jvm/scala/akka/cluster/NodeStartupSpec.scala
new file mode 100644
index 0000000000..a0e0e19943
--- /dev/null
+++ b/akka-cluster/src/multi-jvm/scala/akka/cluster/NodeStartupSpec.scala
@@ -0,0 +1,87 @@
+/**
+ *  Copyright (C) 2009-2012 Typesafe Inc. 
+ */
+package akka.cluster
+
+import com.typesafe.config.ConfigFactory
+
+import akka.remote.testkit.MultiNodeConfig
+import akka.remote.testkit.MultiNodeSpec
+import akka.testkit._
+
+object NodeStartupMultiJvmSpec extends MultiNodeConfig {
+  val first = role("first")
+  val second = role("second")
+
+  commonConfig(debugConfig(on = false).withFallback(ConfigFactory.parseString("""
+    # FIXME get rid of this hardcoded host:port
+    akka.cluster.node-to-join = "akka://MultiNodeSpec@localhost:2601"
+    """)))
+
+  nodeConfig(first, ConfigFactory.parseString("""
+    # FIXME get rid of this hardcoded port
+    akka.remote.netty.port=2601
+    """))
+
+}
+
+class NodeStartupMultiJvmNode1 extends NodeStartupSpec
+class NodeStartupMultiJvmNode2 extends NodeStartupSpec
+
+class NodeStartupSpec extends MultiNodeSpec(NodeStartupMultiJvmSpec) with ImplicitSender {
+  import NodeStartupMultiJvmSpec._
+
+  override def initialParticipants = 2
+
+  var firstNode: Cluster = _
+
+  runOn(first) {
+    firstNode = Cluster(system)
+  }
+
+  "A first cluster node with a 'node-to-join' config set to empty string (singleton cluster)" must {
+
+    "be a singleton cluster when started up" taggedAs LongRunningTest in {
+      runOn(first) {
+        awaitCond(firstNode.isSingletonCluster)
+      }
+
+      testConductor.enter("done")
+    }
+
+    "be in 'Joining' phase when started up" taggedAs LongRunningTest in {
+      runOn(first) {
+        val members = firstNode.latestGossip.members
+        members.size must be(1)
+        val firstAddress = testConductor.getAddressFor(first).await
+        val joiningMember = members find (_.address == firstAddress)
+        joiningMember must not be (None)
+        joiningMember.get.status must be(MemberStatus.Joining)
+      }
+
+      testConductor.enter("done")
+    }
+  }
+
+  "A second cluster node with a 'node-to-join' config defined" must {
+    "join the other node cluster when sending a Join command" taggedAs LongRunningTest in {
+      runOn(second) {
+        // start cluster on second node, and join
+        Cluster(system)
+      }
+
+      runOn(first) {
+        val secondAddress = testConductor.getAddressFor(second).await
+        awaitCond {
+          firstNode.latestGossip.members.exists { member ⇒
+            member.address == secondAddress && member.status == MemberStatus.Up
+          }
+        }
+        firstNode.latestGossip.members.size must be(2)
+      }
+
+      testConductor.enter("done")
+    }
+  }
+
+}
diff --git a/akka-cluster/src/test/scala/akka/cluster/NodeStartupSpec.scala b/akka-cluster/src/test/scala/akka/cluster/NodeStartupSpec.scala
deleted file mode 100644
index 711a0552b4..0000000000
--- a/akka-cluster/src/test/scala/akka/cluster/NodeStartupSpec.scala
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- *  Copyright (C) 2009-2012 Typesafe Inc. 
- */
-package akka.cluster
-
-import java.net.InetSocketAddress
-
-import akka.testkit._
-import akka.dispatch._
-import akka.actor._
-import akka.remote._
-import akka.util.duration._
-
-import com.typesafe.config._
-
-class NodeStartupSpec extends ClusterSpec with ImplicitSender {
-  val portPrefix = 8
-
-  var node0: Cluster = _
-  var node1: Cluster = _
-  var system0: ActorSystemImpl = _
-  var system1: ActorSystemImpl = _
-
-  try {
-    "A first cluster node with a 'node-to-join' config set to empty string (singleton cluster)" must {
-      system0 = ActorSystem("system0", ConfigFactory
-        .parseString("""
-          akka {
-            actor.provider = "akka.remote.RemoteActorRefProvider"
-            remote.netty.port=%d550
-          }""".format(portPrefix))
-        .withFallback(system.settings.config))
-        .asInstanceOf[ActorSystemImpl]
-      val remote0 = system0.provider.asInstanceOf[RemoteActorRefProvider]
-      node0 = Cluster(system0)
-
-      "be a singleton cluster when started up" taggedAs LongRunningTest in {
-        Thread.sleep(1.seconds.dilated.toMillis)
-        node0.isSingletonCluster must be(true)
-      }
-
-      "be in 'Joining' phase when started up" taggedAs LongRunningTest in {
-        val members = node0.latestGossip.members
-        val joiningMember = members find (_.address.port.get == 550.withPortPrefix)
-        joiningMember must be('defined)
-        joiningMember.get.status must be(MemberStatus.Joining)
-      }
-    }
-
-    "A second cluster node with a 'node-to-join' config defined" must {
-      "join the other node cluster when sending a Join command" taggedAs LongRunningTest in {
-        system1 = ActorSystem("system1", ConfigFactory
-          .parseString("""
-            akka {
-              actor.provider = "akka.remote.RemoteActorRefProvider"
-              remote.netty.port=%d551
-              cluster.node-to-join = "akka://system0@localhost:%d550"
-            }""".format(portPrefix, portPrefix))
-          .withFallback(system.settings.config))
-          .asInstanceOf[ActorSystemImpl]
-        val remote1 = system1.provider.asInstanceOf[RemoteActorRefProvider]
-        node1 = Cluster(system1)
-
-        Thread.sleep(10.seconds.dilated.toMillis) // give enough time for node1 to JOIN node0 and leader to move him to UP
-        val members = node0.latestGossip.members
-        val joiningMember = members find (_.address.port.get == 551.withPortPrefix)
-        joiningMember must be('defined)
-        joiningMember.get.status must be(MemberStatus.Up)
-      }
-    }
-  } catch {
-    case e: Exception ⇒
-      e.printStackTrace
-      fail(e.toString)
-  }
-
-  override def atTermination() {
-    if (node0 ne null) node0.shutdown()
-    if (system0 ne null) system0.shutdown()
-
-    if (node1 ne null) node1.shutdown()
-    if (system1 ne null) system1.shutdown()
-  }
-}
diff --git a/akka-remote-tests/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala b/akka-remote-tests/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
index e311fa0023..5ff19a806b 100644
--- a/akka-remote-tests/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
+++ b/akka-remote-tests/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
@@ -1,3 +1,6 @@
+/**
+ *  Copyright (C) 2009-2012 Typesafe Inc. 
+ */
 package akka.remote.testconductor
 
 import akka.remote.AkkaRemoteSpec
@@ -16,18 +19,8 @@ import akka.remote.testkit.MultiNodeSpec
 import akka.remote.testkit.MultiNodeConfig
 
 object TestConductorMultiJvmSpec extends MultiNodeConfig {
-  commonConfig(ConfigFactory.parseString("""
-    # akka.loglevel = DEBUG
-    akka.remote {
-      log-received-messages = on
-      log-sent-messages = on
-    }
-    akka.actor.debug {
-      receive = on
-      fsm = on
-    }
-  """))
-  
+  commonConfig(debugConfig(on = true))
+
   val master = role("master")
   val slave = role("slave")
 }
@@ -36,7 +29,7 @@ class TestConductorMultiJvmNode1 extends TestConductorSpec
 class TestConductorMultiJvmNode2 extends TestConductorSpec
 
 class TestConductorSpec extends MultiNodeSpec(TestConductorMultiJvmSpec) with ImplicitSender {
-  
+
   import TestConductorMultiJvmSpec._
 
   def initialParticipants = 2
diff --git a/akka-remote-tests/src/test/scala/akka/remote/testkit/MultiNodeSpec.scala b/akka-remote-tests/src/test/scala/akka/remote/testkit/MultiNodeSpec.scala
index 92e65247fb..3822a1f529 100644
--- a/akka-remote-tests/src/test/scala/akka/remote/testkit/MultiNodeSpec.scala
+++ b/akka-remote-tests/src/test/scala/akka/remote/testkit/MultiNodeSpec.scala
@@ -37,6 +37,25 @@ abstract class MultiNodeConfig {
    */
   def nodeConfig(role: RoleName, config: Config): Unit = _nodeConf += role -> config
 
+  /**
+   * Include for verbose debug logging
+   * @param on when `true` debug Config is returned, otherwise empty Config
+   */
+  def debugConfig(on: Boolean): Config =
+    if (on)
+      ConfigFactory.parseString("""
+        akka.loglevel = DEBUG
+        akka.remote {
+          log-received-messages = on
+          log-sent-messages = on
+        }
+        akka.actor.debug {
+          receive = on
+          fsm = on
+        }
+        """)
+    else ConfigFactory.empty
+
   /**
    * Construct a RoleName and return it, to be used as an identifier in the
    * test. Registration of a role name creates a role which then needs to be
diff --git a/project/AkkaBuild.scala b/project/AkkaBuild.scala
index 206e32e1f3..f884894d52 100644
--- a/project/AkkaBuild.scala
+++ b/project/AkkaBuild.scala
@@ -114,7 +114,7 @@ object AkkaBuild extends Build {
   lazy val cluster = Project(
     id = "akka-cluster",
     base = file("akka-cluster"),
-    dependencies = Seq(remote, remote % "test->test", testkit % "test->test"),
+    dependencies = Seq(remote, remoteTests % "compile;test->test;multi-jvm->multi-jvm", testkit % "test->test"),
     settings = defaultSettings ++ multiJvmSettings ++ Seq(
       libraryDependencies ++= Dependencies.cluster,
       // disable parallel tests

From 3e416a4b2c2b9a3dcc3d8723f62b38cf95142166 Mon Sep 17 00:00:00 2001
From: Patrik Nordwall 
Date: Wed, 23 May 2012 17:39:34 +0200
Subject: [PATCH 31/36] Add convergence verification to NodeStartupSpec. See
 #1948

---
 .../src/multi-jvm/scala/akka/cluster/NodeStartupSpec.scala    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/akka-cluster/src/multi-jvm/scala/akka/cluster/NodeStartupSpec.scala b/akka-cluster/src/multi-jvm/scala/akka/cluster/NodeStartupSpec.scala
index a0e0e19943..10b5945ee5 100644
--- a/akka-cluster/src/multi-jvm/scala/akka/cluster/NodeStartupSpec.scala
+++ b/akka-cluster/src/multi-jvm/scala/akka/cluster/NodeStartupSpec.scala
@@ -67,7 +67,8 @@ class NodeStartupSpec extends MultiNodeSpec(NodeStartupMultiJvmSpec) with Implic
     "join the other node cluster when sending a Join command" taggedAs LongRunningTest in {
       runOn(second) {
         // start cluster on second node, and join
-        Cluster(system)
+        val secondNode = Cluster(system)
+        awaitCond(secondNode.convergence.isDefined)
       }
 
       runOn(first) {
@@ -78,6 +79,7 @@ class NodeStartupSpec extends MultiNodeSpec(NodeStartupMultiJvmSpec) with Implic
           }
         }
         firstNode.latestGossip.members.size must be(2)
+        awaitCond(firstNode.convergence.isDefined)
       }
 
       testConductor.enter("done")

From a934d7f29d92c2ff0268b20c1c07e0b519e213e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Antonsson?= 
Date: Wed, 23 May 2012 22:52:43 +0200
Subject: [PATCH 32/36] Make sure normal tests are run even if multi-jvm tests
 fail

---
 project/AkkaBuild.scala | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/project/AkkaBuild.scala b/project/AkkaBuild.scala
index 206e32e1f3..62c81bd3eb 100644
--- a/project/AkkaBuild.scala
+++ b/project/AkkaBuild.scala
@@ -89,7 +89,7 @@ object AkkaBuild extends Build {
       jvmOptions in MultiJvm := {
         if (getBoolean("sbt.log.noformat")) Seq("-Dakka.test.nocolor=true") else Nil
       },
-      test in Test <<= (test in Test) dependsOn (test in MultiJvm)
+      test in Test <<= ((test in Test), (test in MultiJvm)) map { case x => x }
     )
   ) configs (MultiJvm)
 
@@ -107,7 +107,7 @@ object AkkaBuild extends Build {
       jvmOptions in MultiJvm := {
         if (getBoolean("sbt.log.noformat")) Seq("-Dakka.test.nocolor=true") else Nil
       },
-      test in Test <<= (test in Test) dependsOn (test in MultiJvm)
+      test in Test <<= ((test in Test), (test in MultiJvm)) map { case x => x }
     )
   ) configs (MultiJvm)
 
@@ -126,7 +126,7 @@ object AkkaBuild extends Build {
       jvmOptions in MultiJvm := {
         if (getBoolean("sbt.log.noformat")) Seq("-Dakka.test.nocolor=true") else Nil
       },
-      test in Test <<= (test in Test) dependsOn (test in MultiJvm)
+      test in Test <<= ((test in Test), (test in MultiJvm)) map { case x => x }
     )
   ) configs (MultiJvm)
 

From d3511d25a497e7b0df7a945ef86e18fa9a18b965 Mon Sep 17 00:00:00 2001
From: Patrik Nordwall 
Date: Thu, 24 May 2012 08:32:50 +0200
Subject: [PATCH 33/36] Placed the barrier in after instead. See #1948

---
 .../scala/akka/cluster/NodeStartupSpec.scala  | 31 ++++++++++---------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/akka-cluster/src/multi-jvm/scala/akka/cluster/NodeStartupSpec.scala b/akka-cluster/src/multi-jvm/scala/akka/cluster/NodeStartupSpec.scala
index 10b5945ee5..6807d7032a 100644
--- a/akka-cluster/src/multi-jvm/scala/akka/cluster/NodeStartupSpec.scala
+++ b/akka-cluster/src/multi-jvm/scala/akka/cluster/NodeStartupSpec.scala
@@ -4,7 +4,7 @@
 package akka.cluster
 
 import com.typesafe.config.ConfigFactory
-
+import org.scalatest.BeforeAndAfter
 import akka.remote.testkit.MultiNodeConfig
 import akka.remote.testkit.MultiNodeSpec
 import akka.testkit._
@@ -13,43 +13,48 @@ object NodeStartupMultiJvmSpec extends MultiNodeConfig {
   val first = role("first")
   val second = role("second")
 
-  commonConfig(debugConfig(on = false).withFallback(ConfigFactory.parseString("""
-    # FIXME get rid of this hardcoded host:port
-    akka.cluster.node-to-join = "akka://MultiNodeSpec@localhost:2601"
-    """)))
+  commonConfig(debugConfig(on = false))
 
   nodeConfig(first, ConfigFactory.parseString("""
     # FIXME get rid of this hardcoded port
     akka.remote.netty.port=2601
     """))
 
+  nodeConfig(second, ConfigFactory.parseString("""
+    # FIXME get rid of this hardcoded host:port
+    akka.cluster.node-to-join = "akka://MultiNodeSpec@localhost:2601"
+    """))
+
 }
 
 class NodeStartupMultiJvmNode1 extends NodeStartupSpec
 class NodeStartupMultiJvmNode2 extends NodeStartupSpec
 
-class NodeStartupSpec extends MultiNodeSpec(NodeStartupMultiJvmSpec) with ImplicitSender {
+class NodeStartupSpec extends MultiNodeSpec(NodeStartupMultiJvmSpec) with ImplicitSender with BeforeAndAfter {
   import NodeStartupMultiJvmSpec._
 
   override def initialParticipants = 2
 
   var firstNode: Cluster = _
 
+  after {
+    testConductor.enter("after")
+  }
+
   runOn(first) {
     firstNode = Cluster(system)
   }
 
   "A first cluster node with a 'node-to-join' config set to empty string (singleton cluster)" must {
 
-    "be a singleton cluster when started up" taggedAs LongRunningTest in {
+    "be a singleton cluster when started up" in {
       runOn(first) {
         awaitCond(firstNode.isSingletonCluster)
+        firstNode.convergence must be(None)
       }
-
-      testConductor.enter("done")
     }
 
-    "be in 'Joining' phase when started up" taggedAs LongRunningTest in {
+    "be in 'Joining' phase when started up" in {
       runOn(first) {
         val members = firstNode.latestGossip.members
         members.size must be(1)
@@ -58,13 +63,11 @@ class NodeStartupSpec extends MultiNodeSpec(NodeStartupMultiJvmSpec) with Implic
         joiningMember must not be (None)
         joiningMember.get.status must be(MemberStatus.Joining)
       }
-
-      testConductor.enter("done")
     }
   }
 
   "A second cluster node with a 'node-to-join' config defined" must {
-    "join the other node cluster when sending a Join command" taggedAs LongRunningTest in {
+    "join the other node cluster when sending a Join command" in {
       runOn(second) {
         // start cluster on second node, and join
         val secondNode = Cluster(system)
@@ -81,8 +84,6 @@ class NodeStartupSpec extends MultiNodeSpec(NodeStartupMultiJvmSpec) with Implic
         firstNode.latestGossip.members.size must be(2)
         awaitCond(firstNode.convergence.isDefined)
       }
-
-      testConductor.enter("done")
     }
   }
 

From db4730978f77a0a22c8c2b1a5f8cd8d954bfddb8 Mon Sep 17 00:00:00 2001
From: Patrik Nordwall 
Date: Thu, 24 May 2012 08:51:36 +0200
Subject: [PATCH 34/36] FIXME singletonCluster should reach convergence. See
 #2117

---
 .../src/multi-jvm/scala/akka/cluster/NodeStartupSpec.scala     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/akka-cluster/src/multi-jvm/scala/akka/cluster/NodeStartupSpec.scala b/akka-cluster/src/multi-jvm/scala/akka/cluster/NodeStartupSpec.scala
index 6807d7032a..694d4ac57d 100644
--- a/akka-cluster/src/multi-jvm/scala/akka/cluster/NodeStartupSpec.scala
+++ b/akka-cluster/src/multi-jvm/scala/akka/cluster/NodeStartupSpec.scala
@@ -50,7 +50,8 @@ class NodeStartupSpec extends MultiNodeSpec(NodeStartupMultiJvmSpec) with Implic
     "be a singleton cluster when started up" in {
       runOn(first) {
         awaitCond(firstNode.isSingletonCluster)
-        firstNode.convergence must be(None)
+        // FIXME #2117 singletonCluster should reach convergence
+        //awaitCond(firstNode.convergence.isDefined)
       }
     }
 

From be87215fc6c9b11de736c803c9b90d9c1b8c5d23 Mon Sep 17 00:00:00 2001
From: Patrik Nordwall 
Date: Thu, 24 May 2012 10:00:35 +0200
Subject: [PATCH 35/36] Pass timefactor to multi-jvm process

---
 project/AkkaBuild.scala | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/project/AkkaBuild.scala b/project/AkkaBuild.scala
index 9e0a32ebce..fd18e931c7 100644
--- a/project/AkkaBuild.scala
+++ b/project/AkkaBuild.scala
@@ -86,9 +86,7 @@ object AkkaBuild extends Build {
         (name: String) => (src ** (name + ".conf")).get.headOption.map("-Dakka.config=" + _.absolutePath).toSeq
       },
       scalatestOptions in MultiJvm := Seq("-r", "org.scalatest.akka.QuietReporter"),
-      jvmOptions in MultiJvm := {
-        if (getBoolean("sbt.log.noformat")) Seq("-Dakka.test.nocolor=true") else Nil
-      },
+      jvmOptions in MultiJvm := defaultMultiJvmOptions,
       test in Test <<= ((test in Test), (test in MultiJvm)) map { case x => x }
     )
   ) configs (MultiJvm)
@@ -104,9 +102,7 @@ object AkkaBuild extends Build {
         (name: String) => (src ** (name + ".conf")).get.headOption.map("-Dakka.config=" + _.absolutePath).toSeq
       },
       scalatestOptions in MultiJvm := Seq("-r", "org.scalatest.akka.QuietReporter"),
-      jvmOptions in MultiJvm := {
-        if (getBoolean("sbt.log.noformat")) Seq("-Dakka.test.nocolor=true") else Nil
-      },
+      jvmOptions in MultiJvm := defaultMultiJvmOptions,
       test in Test <<= ((test in Test), (test in MultiJvm)) map { case x => x }
     )
   ) configs (MultiJvm)
@@ -123,9 +119,7 @@ object AkkaBuild extends Build {
         (name: String) => (src ** (name + ".conf")).get.headOption.map("-Dakka.config=" + _.absolutePath).toSeq
       },
       scalatestOptions in MultiJvm := Seq("-r", "org.scalatest.akka.QuietReporter"),
-      jvmOptions in MultiJvm := {
-        if (getBoolean("sbt.log.noformat")) Seq("-Dakka.test.nocolor=true") else Nil
-      },
+      jvmOptions in MultiJvm := defaultMultiJvmOptions,
       test in Test <<= ((test in Test), (test in MultiJvm)) map { case x => x }
     )
   ) configs (MultiJvm)
@@ -304,6 +298,14 @@ object AkkaBuild extends Build {
 
   val defaultExcludedTags = Seq("timing", "long-running")
 
+  val defaultMultiJvmOptions: Seq[String] = {
+    (System.getProperty("akka.test.timefactor") match {
+      case null => Nil
+      case x => List("-Dakka.test.timefactor=" + x)
+    }) :::
+    (if (getBoolean("sbt.log.noformat")) List("-Dakka.test.nocolor=true") else Nil)
+  }
+
   lazy val defaultSettings = baseSettings ++ formatSettings ++ Seq(
     resolvers += "Typesafe Repo" at "http://repo.typesafe.com/typesafe/releases/",
 

From e05481604734ea287c9185c1c499673f91a86d72 Mon Sep 17 00:00:00 2001
From: Roland 
Date: Thu, 24 May 2012 10:56:32 +0200
Subject: [PATCH 36/36] make failure injection idempotent

- instead of creating local top-level actors per pipeline, just create
  one system actor through which everything is sent
- this enables storing settings (like what to throttle how) within this
  actor and applying settings when connections come up later
- it also gets rid of the blocking actor creation from
  NetworkFailureInjector, fixing the dead-lock
- moved also the ServerFSMs to be children of the Controller
- all actors have proper names now for easier debugging
---
 .../src/main/scala/akka/event/Logging.scala   |   2 +-
 .../akka/remote/testconductor/Conductor.scala |  37 +-
 .../akka/remote/testconductor/Extension.scala |   4 +-
 .../NetworkFailureInjector.scala              | 475 ++++++++++--------
 .../akka/remote/testconductor/Player.scala    |  16 +-
 .../scala/akka/remote/SimpleRemoteSpec.scala  |   2 +-
 .../DirectRoutedRemoteActorMultiJvmSpec.scala |   2 +-
 .../testconductor/TestConductorSpec.scala     |   2 +-
 .../main/scala/akka/remote/netty/Client.scala |   5 +-
 .../remote/netty/NettyRemoteSupport.scala     |   5 +
 .../main/scala/akka/remote/netty/Server.scala |  16 +-
 11 files changed, 325 insertions(+), 241 deletions(-)

diff --git a/akka-actor/src/main/scala/akka/event/Logging.scala b/akka-actor/src/main/scala/akka/event/Logging.scala
index 93019318dd..b044fd09ab 100644
--- a/akka-actor/src/main/scala/akka/event/Logging.scala
+++ b/akka-actor/src/main/scala/akka/event/Logging.scala
@@ -601,7 +601,7 @@ object Logging {
     import java.text.SimpleDateFormat
     import java.util.Date
 
-    val dateFormat = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss.S")
+    val dateFormat = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss.SSS")
 
     def timestamp = dateFormat.format(new Date)
 
diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala
index 6c26fcaae2..5e467fde19 100644
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala
+++ b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Conductor.scala
@@ -27,12 +27,26 @@ import akka.actor.SupervisorStrategy
 import java.util.concurrent.ConcurrentHashMap
 import akka.actor.Status
 
-sealed trait Direction
+sealed trait Direction {
+  def includes(other: Direction): Boolean
+}
 
 object Direction {
-  case object Send extends Direction
-  case object Receive extends Direction
-  case object Both extends Direction
+  case object Send extends Direction {
+    override def includes(other: Direction): Boolean = other match {
+      case Send ⇒ true
+      case _    ⇒ false
+    }
+  }
+  case object Receive extends Direction {
+    override def includes(other: Direction): Boolean = other match {
+      case Receive ⇒ true
+      case _       ⇒ false
+    }
+  }
+  case object Both extends Direction {
+    override def includes(other: Direction): Boolean = true
+  }
 }
 
 /**
@@ -202,14 +216,15 @@ trait Conductor { this: TestConductorExt ⇒
  * purpose is to dispatch incoming messages to the right ServerFSM actor. There is
  * one shared instance of this class for all connections accepted by one Controller.
  */
-class ConductorHandler(system: ActorSystem, controller: ActorRef, log: LoggingAdapter) extends SimpleChannelUpstreamHandler {
+class ConductorHandler(_createTimeout: Timeout, controller: ActorRef, log: LoggingAdapter) extends SimpleChannelUpstreamHandler {
 
+  implicit val createTimeout = _createTimeout
   val clients = new ConcurrentHashMap[Channel, ActorRef]()
 
   override def channelConnected(ctx: ChannelHandlerContext, event: ChannelStateEvent) = {
     val channel = event.getChannel
     log.debug("connection from {}", getAddrString(channel))
-    val fsm = system.actorOf(Props(new ServerFSM(controller, channel)))
+    val fsm: ActorRef = Await.result(controller ? Controller.CreateServerFSM(channel) mapTo, Duration.Inf)
     clients.put(channel, fsm)
   }
 
@@ -321,6 +336,7 @@ object Controller {
   case class ClientDisconnected(name: RoleName)
   case object GetNodes
   case object GetSockAddr
+  case class CreateServerFSM(channel: Channel)
 
   case class NodeInfo(name: RoleName, addr: Address, fsm: ActorRef)
 }
@@ -336,7 +352,7 @@ class Controller(private var initialParticipants: Int, controllerPort: InetSocke
 
   val settings = TestConductor().Settings
   val connection = RemoteConnection(Server, controllerPort,
-    new ConductorHandler(context.system, self, Logging(context.system, "ConductorHandler")))
+    new ConductorHandler(settings.QueryTimeout, self, Logging(context.system, "ConductorHandler")))
 
   /*
    * Supervision of the BarrierCoordinator means to catch all his bad emotions
@@ -363,8 +379,15 @@ class Controller(private var initialParticipants: Int, controllerPort: InetSocke
 
   // map keeping unanswered queries for node addresses (enqueued upon GetAddress, serviced upon NodeInfo)
   var addrInterest = Map[RoleName, Set[ActorRef]]()
+  val generation = Iterator from 1
 
   override def receive = LoggingReceive {
+    case CreateServerFSM(channel) ⇒
+      val (ip, port) = channel.getRemoteAddress match {
+        case s: InetSocketAddress ⇒ (s.getHostString, s.getPort)
+      }
+      val name = ip + ":" + port + "-server" + generation.next
+      sender ! context.actorOf(Props(new ServerFSM(self, channel)), name)
     case c @ NodeInfo(name, addr, fsm) ⇒
       barrier forward c
       if (nodes contains name) {
diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Extension.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Extension.scala
index 7f6b576128..09ffd7319f 100644
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Extension.scala
+++ b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Extension.scala
@@ -10,6 +10,8 @@ import java.util.concurrent.TimeUnit.MILLISECONDS
 import akka.actor.ActorRef
 import java.util.concurrent.ConcurrentHashMap
 import akka.actor.Address
+import akka.actor.ActorSystemImpl
+import akka.actor.Props
 
 /**
  * Access to the [[akka.remote.testconductor.TestConductorExt]] extension:
@@ -50,6 +52,6 @@ class TestConductorExt(val system: ExtendedActorSystem) extends Extension with C
   val transport = system.provider.asInstanceOf[RemoteActorRefProvider].transport
   val address = transport.address
 
-  val failureInjectors = new ConcurrentHashMap[Address, FailureInjector]
+  val failureInjector = system.asInstanceOf[ActorSystemImpl].systemActorOf(Props[FailureInjector], "FailureInjector")
 
 }
\ No newline at end of file
diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala
index b853523979..1fcb1a7bf9 100644
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala
+++ b/akka-remote-tests/src/main/scala/akka/remote/testconductor/NetworkFailureInjector.scala
@@ -4,236 +4,303 @@
 package akka.remote.testconductor
 
 import java.net.InetSocketAddress
-import scala.collection.immutable.Queue
-import org.jboss.netty.buffer.ChannelBuffer
-import org.jboss.netty.channel.ChannelState.BOUND
-import org.jboss.netty.channel.ChannelState.OPEN
-import org.jboss.netty.channel.Channel
-import org.jboss.netty.channel.ChannelEvent
-import org.jboss.netty.channel.ChannelHandlerContext
-import org.jboss.netty.channel.ChannelStateEvent
-import org.jboss.netty.channel.MessageEvent
-import akka.actor.FSM
-import akka.actor.Actor
-import akka.util.duration.doubleToDurationDouble
-import akka.util.Index
-import akka.actor.Address
-import akka.actor.ActorSystem
-import akka.actor.Props
-import akka.actor.ActorRef
-import akka.event.Logging
-import org.jboss.netty.channel.SimpleChannelHandler
-import scala.annotation.tailrec
-import akka.util.Duration
-import akka.actor.LoggingFSM
-import org.jboss.netty.channel.Channels
-import org.jboss.netty.channel.ChannelFuture
-import org.jboss.netty.channel.ChannelFutureListener
-import org.jboss.netty.channel.ChannelFuture
 
-case class FailureInjector(sender: ActorRef, receiver: ActorRef) {
-  def refs(dir: Direction) = dir match {
-    case Direction.Send    ⇒ Seq(sender)
-    case Direction.Receive ⇒ Seq(receiver)
-    case Direction.Both    ⇒ Seq(sender, receiver)
+import scala.annotation.tailrec
+import scala.collection.immutable.Queue
+
+import org.jboss.netty.buffer.ChannelBuffer
+import org.jboss.netty.channel.{ SimpleChannelHandler, MessageEvent, Channels, ChannelStateEvent, ChannelHandlerContext, ChannelFutureListener, ChannelFuture }
+
+import akka.actor.{ Props, LoggingFSM, Address, ActorSystem, ActorRef, ActorLogging, Actor, FSM }
+import akka.event.Logging
+import akka.remote.netty.ChannelAddress
+import akka.util.Duration
+import akka.util.duration._
+
+class FailureInjector extends Actor with ActorLogging {
+  import ThrottleActor._
+  import NetworkFailureInjector._
+
+  case class ChannelSettings(
+    ctx: Option[ChannelHandlerContext] = None,
+    throttleSend: Option[SetRate] = None,
+    throttleReceive: Option[SetRate] = None)
+  case class Injectors(sender: ActorRef, receiver: ActorRef)
+
+  var channels = Map[ChannelHandlerContext, Injectors]()
+  var settings = Map[Address, ChannelSettings]()
+  var generation = Iterator from 1
+
+  /**
+   * Only for a NEW ctx, start ThrottleActors, prime them and update all maps.
+   */
+  def ingestContextAddress(ctx: ChannelHandlerContext, addr: Address): Injectors = {
+    val gen = generation.next
+    val name = addr.host.get + ":" + addr.port.get
+    val thrSend = context.actorOf(Props(new ThrottleActor(ctx)), name + "-snd" + gen)
+    val thrRecv = context.actorOf(Props(new ThrottleActor(ctx)), name + "-rcv" + gen)
+    val injectors = Injectors(thrSend, thrRecv)
+    channels += ctx -> injectors
+    settings += addr -> (settings get addr map {
+      case c @ ChannelSettings(prevCtx, ts, tr) ⇒
+        ts foreach (thrSend ! _)
+        tr foreach (thrRecv ! _)
+        prevCtx match {
+          case Some(p) ⇒ log.warning("installing context {} instead of {} for address {}", ctx, p, addr)
+          case None    ⇒ // okay
+        }
+        c.copy(ctx = Some(ctx))
+    } getOrElse ChannelSettings(Some(ctx)))
+    injectors
+  }
+
+  /**
+   * Retrieve target settings, also if they were sketchy before (i.e. no system name)
+   */
+  def retrieveTargetSettings(target: Address): Option[ChannelSettings] = {
+    settings get target orElse {
+      val host = target.host
+      val port = target.port
+      settings find {
+        case (Address("akka", "", `host`, `port`), s) ⇒ true
+        case _                                        ⇒ false
+      } map {
+        case (_, s) ⇒ settings += target -> s; s
+      }
+    }
+  }
+
+  def receive = {
+    case RemoveContext(ctx) ⇒
+      channels get ctx foreach { inj ⇒
+        context stop inj.sender
+        context stop inj.receiver
+      }
+      channels -= ctx
+      settings ++= settings collect { case (addr, c @ ChannelSettings(Some(`ctx`), _, _)) ⇒ (addr, c.copy(ctx = None)) }
+    case ThrottleMsg(target, dir, rateMBit) ⇒
+      val setting = retrieveTargetSettings(target)
+      settings += target -> ((setting getOrElse ChannelSettings() match {
+        case cs @ ChannelSettings(ctx, _, _) if dir includes Direction.Send ⇒
+          ctx foreach (c ⇒ channels get c foreach (_.sender ! SetRate(rateMBit)))
+          cs.copy(throttleSend = Some(SetRate(rateMBit)))
+        case x ⇒ x
+      }) match {
+        case cs @ ChannelSettings(ctx, _, _) if dir includes Direction.Receive ⇒
+          ctx foreach (c ⇒ channels get c foreach (_.receiver ! SetRate(rateMBit)))
+          cs.copy(throttleReceive = Some(SetRate(rateMBit)))
+        case x ⇒ x
+      })
+      sender ! "ok"
+    case DisconnectMsg(target, abort) ⇒
+      retrieveTargetSettings(target) foreach {
+        case ChannelSettings(Some(ctx), _, _) ⇒
+          val ch = ctx.getChannel
+          if (abort) {
+            ch.getConfig.setOption("soLinger", 0)
+            log.info("aborting connection {}", ch)
+          } else log.info("closing connection {}", ch)
+          ch.close
+        case _ ⇒ log.debug("no connection to {} to close or abort", target)
+      }
+      sender ! "ok"
+    case s @ Send(ctx, direction, future, msg) ⇒
+      channels get ctx match {
+        case Some(Injectors(snd, rcv)) ⇒
+          if (direction includes Direction.Send) snd ! s
+          if (direction includes Direction.Receive) rcv ! s
+        case None ⇒
+          val (ipaddr, ip, port) = ctx.getChannel.getRemoteAddress match {
+            case s: InetSocketAddress ⇒ (s.getAddress, s.getAddress.getHostAddress, s.getPort)
+          }
+          val addr = ChannelAddress.get(ctx.getChannel) orElse {
+            settings collect { case (a @ Address("akka", _, Some(`ip`), Some(`port`)), _) ⇒ a } headOption
+          } orElse {
+            val name = ipaddr.getHostName
+            if (name == ip) None
+            else settings collect { case (a @ Address("akka", _, Some(`name`), Some(`port`)), _) ⇒ a } headOption
+          } getOrElse Address("akka", "", ip, port) // this will not match later requests directly, but be picked up by retrieveTargetSettings
+          val inj = ingestContextAddress(ctx, addr)
+          if (direction includes Direction.Send) inj.sender ! s
+          if (direction includes Direction.Receive) inj.receiver ! s
+      }
   }
 }
 
 object NetworkFailureInjector {
-  case class SetRate(rateMBit: Float)
-  case class Disconnect(abort: Boolean)
+  case class RemoveContext(ctx: ChannelHandlerContext)
 }
 
 class NetworkFailureInjector(system: ActorSystem) extends SimpleChannelHandler {
+  import NetworkFailureInjector._
 
-  val log = Logging(system, "FailureInjector")
+  private val log = Logging(system, "FailureInjector")
 
-  // everything goes via these Throttle actors to enable easy steering
-  private val sender = system.actorOf(Props(new Throttle(Direction.Send)))
-  private val receiver = system.actorOf(Props(new Throttle(Direction.Receive)))
-
-  private val packetSplitThreshold = TestConductor(system).Settings.PacketSplitThreshold
-
-  /*
-   * State, Data and Messages for the internal Throttle actor
-   */
-  sealed private trait State
-  private case object PassThrough extends State
-  private case object Throttle extends State
-  private case object Blackhole extends State
-
-  private case class Data(lastSent: Long, rateMBit: Float, queue: Queue[Send])
-
-  private case class Send(ctx: ChannelHandlerContext, future: Option[ChannelFuture], msg: AnyRef)
-  private case class SetContext(ctx: ChannelHandlerContext)
-  private case object Tick
-
-  private class Throttle(dir: Direction) extends Actor with LoggingFSM[State, Data] {
-    import FSM._
-
-    var channelContext: ChannelHandlerContext = _
-
-    startWith(PassThrough, Data(0, -1, Queue()))
-
-    when(PassThrough) {
-      case Event(s @ Send(_, _, msg), _) ⇒
-        log.debug("sending msg (PassThrough): {}", msg)
-        send(s)
-        stay
-    }
-
-    when(Throttle) {
-      case Event(s: Send, data @ Data(_, _, Queue())) ⇒
-        stay using sendThrottled(data.copy(lastSent = System.nanoTime, queue = Queue(s)))
-      case Event(s: Send, data) ⇒
-        stay using sendThrottled(data.copy(queue = data.queue.enqueue(s)))
-      case Event(Tick, data) ⇒
-        stay using sendThrottled(data)
-    }
-
-    onTransition {
-      case Throttle -> PassThrough ⇒
-        for (s ← stateData.queue) {
-          log.debug("sending msg (Transition): {}", s.msg)
-          send(s)
-        }
-        cancelTimer("send")
-      case Throttle -> Blackhole ⇒
-        cancelTimer("send")
-    }
-
-    when(Blackhole) {
-      case Event(Send(_, _, msg), _) ⇒
-        log.debug("dropping msg {}", msg)
-        stay
-    }
-
-    whenUnhandled {
-      case Event(NetworkFailureInjector.SetRate(rate), d) ⇒
-        sender ! "ok"
-        if (rate > 0) {
-          goto(Throttle) using d.copy(lastSent = System.nanoTime, rateMBit = rate, queue = Queue())
-        } else if (rate == 0) {
-          goto(Blackhole)
-        } else {
-          goto(PassThrough)
-        }
-      case Event(SetContext(ctx), _) ⇒ channelContext = ctx; stay
-      case Event(NetworkFailureInjector.Disconnect(abort), Data(ctx, _, _)) ⇒
-        sender ! "ok"
-        // TODO implement abort
-        channelContext.getChannel.disconnect()
-        stay
-    }
-
-    initialize
-
-    private def sendThrottled(d: Data): Data = {
-      val (data, toSend, toTick) = schedule(d)
-      for (s ← toSend) {
-        log.debug("sending msg (Tick): {}", s.msg)
-        send(s)
-      }
-      if (!timerActive_?("send"))
-        for (time ← toTick) {
-          log.debug("scheduling next Tick in {}", time)
-          setTimer("send", Tick, time, false)
-        }
-      data
-    }
-
-    private def send(s: Send): Unit = dir match {
-      case Direction.Send    ⇒ Channels.write(s.ctx, s.future getOrElse Channels.future(s.ctx.getChannel), s.msg)
-      case Direction.Receive ⇒ Channels.fireMessageReceived(s.ctx, s.msg)
-      case _                 ⇒
-    }
-
-    private def schedule(d: Data): (Data, Seq[Send], Option[Duration]) = {
-      val now = System.nanoTime
-      @tailrec def rec(d: Data, toSend: Seq[Send]): (Data, Seq[Send], Option[Duration]) = {
-        if (d.queue.isEmpty) (d, toSend, None)
-        else {
-          val timeForPacket = d.lastSent + (1000 * size(d.queue.head.msg) / d.rateMBit).toLong
-          if (timeForPacket <= now) rec(Data(timeForPacket, d.rateMBit, d.queue.tail), toSend :+ d.queue.head)
-          else {
-            val splitThreshold = d.lastSent + packetSplitThreshold.toNanos
-            if (now < splitThreshold) (d, toSend, Some((timeForPacket - now).nanos min (splitThreshold - now).nanos))
-            else {
-              val microsToSend = (now - d.lastSent) / 1000
-              val (s1, s2) = split(d.queue.head, (microsToSend * d.rateMBit / 8).toInt)
-              (d.copy(queue = s2 +: d.queue.tail), toSend :+ s1, Some((timeForPacket - now).nanos min packetSplitThreshold))
-            }
-          }
-        }
-      }
-      rec(d, Seq())
-    }
-
-    private def split(s: Send, bytes: Int): (Send, Send) = {
-      s.msg match {
-        case buf: ChannelBuffer ⇒
-          val f = s.future map { f ⇒
-            val newF = Channels.future(s.ctx.getChannel)
-            newF.addListener(new ChannelFutureListener {
-              def operationComplete(future: ChannelFuture) {
-                if (future.isCancelled) f.cancel()
-                else future.getCause match {
-                  case null ⇒
-                  case thr  ⇒ f.setFailure(thr)
-                }
-              }
-            })
-            newF
-          }
-          val b = buf.slice()
-          b.writerIndex(b.readerIndex + bytes)
-          buf.readerIndex(buf.readerIndex + bytes)
-          (Send(s.ctx, f, b), Send(s.ctx, s.future, buf))
-      }
-    }
-
-    private def size(msg: AnyRef) = msg match {
-      case b: ChannelBuffer ⇒ b.readableBytes() * 8
-      case _                ⇒ throw new UnsupportedOperationException("NetworkFailureInjector only supports ChannelBuffer messages")
-    }
-  }
-
-  private var remote: Option[Address] = None
-
-  override def messageReceived(ctx: ChannelHandlerContext, msg: MessageEvent) {
-    log.debug("upstream(queued): {}", msg)
-    receiver ! Send(ctx, Option(msg.getFuture), msg.getMessage)
-  }
+  private val conductor = TestConductor(system)
+  private var announced = false
 
   override def channelConnected(ctx: ChannelHandlerContext, state: ChannelStateEvent) {
     state.getValue match {
       case a: InetSocketAddress ⇒
         val addr = Address("akka", "", a.getHostName, a.getPort)
         log.debug("connected to {}", addr)
-        TestConductor(system).failureInjectors.put(addr, FailureInjector(sender, receiver)) match {
-          case null ⇒ // okay
-          case fi   ⇒ system.log.error("{} already registered for address {}", fi, addr)
-        }
-        remote = Some(addr)
-        sender ! SetContext(ctx)
       case x ⇒ throw new IllegalArgumentException("unknown address type: " + x)
     }
   }
 
   override def channelDisconnected(ctx: ChannelHandlerContext, state: ChannelStateEvent) {
-    log.debug("disconnected from {}", remote)
-    remote = remote flatMap { addr ⇒
-      TestConductor(system).failureInjectors.remove(addr)
-      system.stop(sender)
-      system.stop(receiver)
-      None
-    }
+    log.debug("disconnected from {}", state.getChannel)
+    conductor.failureInjector ! RemoveContext(ctx)
+  }
+
+  override def messageReceived(ctx: ChannelHandlerContext, msg: MessageEvent) {
+    log.debug("upstream(queued): {}", msg)
+    conductor.failureInjector ! ThrottleActor.Send(ctx, Direction.Receive, Option(msg.getFuture), msg.getMessage)
   }
 
   override def writeRequested(ctx: ChannelHandlerContext, msg: MessageEvent) {
     log.debug("downstream(queued): {}", msg)
-    sender ! Send(ctx, Option(msg.getFuture), msg.getMessage)
+    conductor.failureInjector ! ThrottleActor.Send(ctx, Direction.Send, Option(msg.getFuture), msg.getMessage)
   }
 
 }
 
+private[akka] object ThrottleActor {
+  sealed trait State
+  case object PassThrough extends State
+  case object Throttle extends State
+  case object Blackhole extends State
+
+  case class Data(lastSent: Long, rateMBit: Float, queue: Queue[Send])
+
+  case class Send(ctx: ChannelHandlerContext, direction: Direction, future: Option[ChannelFuture], msg: AnyRef)
+  case class SetRate(rateMBit: Float)
+  case object Tick
+}
+
+private[akka] class ThrottleActor(channelContext: ChannelHandlerContext)
+  extends Actor with LoggingFSM[ThrottleActor.State, ThrottleActor.Data] {
+
+  import ThrottleActor._
+  import FSM._
+
+  private val packetSplitThreshold = TestConductor(context.system).Settings.PacketSplitThreshold
+
+  startWith(PassThrough, Data(0, -1, Queue()))
+
+  when(PassThrough) {
+    case Event(s @ Send(_, _, _, msg), _) ⇒
+      log.debug("sending msg (PassThrough): {}", msg)
+      send(s)
+      stay
+  }
+
+  when(Throttle) {
+    case Event(s: Send, data @ Data(_, _, Queue())) ⇒
+      stay using sendThrottled(data.copy(lastSent = System.nanoTime, queue = Queue(s)))
+    case Event(s: Send, data) ⇒
+      stay using sendThrottled(data.copy(queue = data.queue.enqueue(s)))
+    case Event(Tick, data) ⇒
+      stay using sendThrottled(data)
+  }
+
+  onTransition {
+    case Throttle -> PassThrough ⇒
+      for (s ← stateData.queue) {
+        log.debug("sending msg (Transition): {}", s.msg)
+        send(s)
+      }
+      cancelTimer("send")
+    case Throttle -> Blackhole ⇒
+      cancelTimer("send")
+  }
+
+  when(Blackhole) {
+    case Event(Send(_, _, _, msg), _) ⇒
+      log.debug("dropping msg {}", msg)
+      stay
+  }
+
+  whenUnhandled {
+    case Event(SetRate(rate), d) ⇒
+      if (rate > 0) {
+        goto(Throttle) using d.copy(lastSent = System.nanoTime, rateMBit = rate, queue = Queue())
+      } else if (rate == 0) {
+        goto(Blackhole)
+      } else {
+        goto(PassThrough)
+      }
+  }
+
+  initialize
+
+  private def sendThrottled(d: Data): Data = {
+    val (data, toSend, toTick) = schedule(d)
+    for (s ← toSend) {
+      log.debug("sending msg (Tick): {}", s.msg)
+      send(s)
+    }
+    if (!timerActive_?("send"))
+      for (time ← toTick) {
+        log.debug("scheduling next Tick in {}", time)
+        setTimer("send", Tick, time, false)
+      }
+    data
+  }
+
+  private def send(s: Send): Unit = s.direction match {
+    case Direction.Send    ⇒ Channels.write(s.ctx, s.future getOrElse Channels.future(s.ctx.getChannel), s.msg)
+    case Direction.Receive ⇒ Channels.fireMessageReceived(s.ctx, s.msg)
+    case _                 ⇒
+  }
+
+  private def schedule(d: Data): (Data, Seq[Send], Option[Duration]) = {
+    val now = System.nanoTime
+    @tailrec def rec(d: Data, toSend: Seq[Send]): (Data, Seq[Send], Option[Duration]) = {
+      if (d.queue.isEmpty) (d, toSend, None)
+      else {
+        val timeForPacket = d.lastSent + (1000 * size(d.queue.head.msg) / d.rateMBit).toLong
+        if (timeForPacket <= now) rec(Data(timeForPacket, d.rateMBit, d.queue.tail), toSend :+ d.queue.head)
+        else {
+          val splitThreshold = d.lastSent + packetSplitThreshold.toNanos
+          if (now < splitThreshold) (d, toSend, Some((timeForPacket - now).nanos min (splitThreshold - now).nanos))
+          else {
+            val microsToSend = (now - d.lastSent) / 1000
+            val (s1, s2) = split(d.queue.head, (microsToSend * d.rateMBit / 8).toInt)
+            (d.copy(queue = s2 +: d.queue.tail), toSend :+ s1, Some((timeForPacket - now).nanos min packetSplitThreshold))
+          }
+        }
+      }
+    }
+    rec(d, Seq())
+  }
+
+  private def split(s: Send, bytes: Int): (Send, Send) = {
+    s.msg match {
+      case buf: ChannelBuffer ⇒
+        val f = s.future map { f ⇒
+          val newF = Channels.future(s.ctx.getChannel)
+          newF.addListener(new ChannelFutureListener {
+            def operationComplete(future: ChannelFuture) {
+              if (future.isCancelled) f.cancel()
+              else future.getCause match {
+                case null ⇒
+                case thr  ⇒ f.setFailure(thr)
+              }
+            }
+          })
+          newF
+        }
+        val b = buf.slice()
+        b.writerIndex(b.readerIndex + bytes)
+        buf.readerIndex(buf.readerIndex + bytes)
+        (Send(s.ctx, s.direction, f, b), Send(s.ctx, s.direction, s.future, buf))
+    }
+  }
+
+  private def size(msg: AnyRef) = msg match {
+    case b: ChannelBuffer ⇒ b.readableBytes() * 8
+    case _                ⇒ throw new UnsupportedOperationException("NetworkFailureInjector only supports ChannelBuffer messages")
+  }
+}
+
diff --git a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Player.scala b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Player.scala
index 10434007e1..254b1a7d45 100644
--- a/akka-remote-tests/src/main/scala/akka/remote/testconductor/Player.scala
+++ b/akka-remote-tests/src/main/scala/akka/remote/testconductor/Player.scala
@@ -195,21 +195,13 @@ class ClientFSM(name: RoleName, controllerAddr: InetSocketAddress) extends Actor
               log.warning("did not expect {}", op)
           }
           stay using d.copy(runningOp = None)
-        case ThrottleMsg(target, dir, rate) ⇒
+        case t: ThrottleMsg ⇒
           import settings.QueryTimeout
-          import context.dispatcher
-          TestConductor().failureInjectors.get(target.copy(system = "")) match {
-            case null ⇒ log.warning("cannot throttle unknown address {}", target)
-            case inj ⇒
-              Future.sequence(inj.refs(dir) map (_ ? NetworkFailureInjector.SetRate(rate))) map (_ ⇒ ToServer(Done)) pipeTo self
-          }
+          TestConductor().failureInjector ? t map (_ ⇒ ToServer(Done)) pipeTo self
           stay
-        case DisconnectMsg(target, abort) ⇒
+        case d: DisconnectMsg ⇒
           import settings.QueryTimeout
-          TestConductor().failureInjectors.get(target.copy(system = "")) match {
-            case null ⇒ log.warning("cannot disconnect unknown address {}", target)
-            case inj  ⇒ inj.sender ? NetworkFailureInjector.Disconnect(abort) map (_ ⇒ ToServer(Done)) pipeTo self
-          }
+          TestConductor().failureInjector ? d map (_ ⇒ ToServer(Done)) pipeTo self
           stay
         case TerminateMsg(exit) ⇒
           System.exit(exit)
diff --git a/akka-remote-tests/src/multi-jvm/scala/akka/remote/SimpleRemoteSpec.scala b/akka-remote-tests/src/multi-jvm/scala/akka/remote/SimpleRemoteSpec.scala
index 70cca7c34b..9f9257c69b 100644
--- a/akka-remote-tests/src/multi-jvm/scala/akka/remote/SimpleRemoteSpec.scala
+++ b/akka-remote-tests/src/multi-jvm/scala/akka/remote/SimpleRemoteSpec.scala
@@ -22,7 +22,7 @@ object SimpleRemoteMultiJvmSpec extends MultiNodeConfig {
   }
 
   commonConfig(ConfigFactory.parseString("""
-    akka.loglevel = DEBUG
+    # akka.loglevel = DEBUG
     akka.remote {
       log-received-messages = on
       log-sent-messages = on
diff --git a/akka-remote-tests/src/multi-jvm/scala/akka/remote/router/DirectRoutedRemoteActorMultiJvmSpec.scala b/akka-remote-tests/src/multi-jvm/scala/akka/remote/router/DirectRoutedRemoteActorMultiJvmSpec.scala
index 2690378ef1..e15027cc73 100644
--- a/akka-remote-tests/src/multi-jvm/scala/akka/remote/router/DirectRoutedRemoteActorMultiJvmSpec.scala
+++ b/akka-remote-tests/src/multi-jvm/scala/akka/remote/router/DirectRoutedRemoteActorMultiJvmSpec.scala
@@ -24,7 +24,7 @@ object DirectRoutedRemoteActorMultiJvmSpec extends MultiNodeConfig {
 
   import com.typesafe.config.ConfigFactory
   commonConfig(ConfigFactory.parseString("""
-    akka.loglevel = DEBUG
+    # akka.loglevel = DEBUG
     akka.remote {
       log-received-messages = on
       log-sent-messages = on
diff --git a/akka-remote-tests/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala b/akka-remote-tests/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
index 087aac55c7..e311fa0023 100644
--- a/akka-remote-tests/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
+++ b/akka-remote-tests/src/multi-jvm/scala/akka/remote/testconductor/TestConductorSpec.scala
@@ -17,7 +17,7 @@ import akka.remote.testkit.MultiNodeConfig
 
 object TestConductorMultiJvmSpec extends MultiNodeConfig {
   commonConfig(ConfigFactory.parseString("""
-    akka.loglevel = DEBUG
+    # akka.loglevel = DEBUG
     akka.remote {
       log-received-messages = on
       log-sent-messages = on
diff --git a/akka-remote/src/main/scala/akka/remote/netty/Client.scala b/akka-remote/src/main/scala/akka/remote/netty/Client.scala
index 4735132534..9091864348 100644
--- a/akka-remote/src/main/scala/akka/remote/netty/Client.scala
+++ b/akka-remote/src/main/scala/akka/remote/netty/Client.scala
@@ -173,6 +173,7 @@ class ActiveRemoteClient private[akka] (
         notifyListeners(RemoteClientError(connection.getCause, netty, remoteAddress))
         false
       } else {
+        ChannelAddress.set(connection.getChannel, Some(remoteAddress))
         sendSecureCookie(connection)
         notifyListeners(RemoteClientStarted(netty, remoteAddress))
         true
@@ -196,8 +197,10 @@ class ActiveRemoteClient private[akka] (
 
     notifyListeners(RemoteClientShutdown(netty, remoteAddress))
     try {
-      if ((connection ne null) && (connection.getChannel ne null))
+      if ((connection ne null) && (connection.getChannel ne null)) {
+        ChannelAddress.remove(connection.getChannel)
         connection.getChannel.close()
+      }
     } finally {
       try {
         if (openChannels ne null) openChannels.close.awaitUninterruptibly()
diff --git a/akka-remote/src/main/scala/akka/remote/netty/NettyRemoteSupport.scala b/akka-remote/src/main/scala/akka/remote/netty/NettyRemoteSupport.scala
index 60c2ac6097..84ee77bbbe 100644
--- a/akka-remote/src/main/scala/akka/remote/netty/NettyRemoteSupport.scala
+++ b/akka-remote/src/main/scala/akka/remote/netty/NettyRemoteSupport.scala
@@ -29,6 +29,11 @@ import org.jboss.netty.handler.codec.frame.LengthFieldBasedFrameDecoder
 import org.jboss.netty.handler.timeout.IdleStateHandler
 import org.jboss.netty.channel.ChannelPipelineFactory
 import org.jboss.netty.handler.execution.ExecutionHandler
+import org.jboss.netty.channel.ChannelLocal
+
+object ChannelAddress extends ChannelLocal[Option[Address]] {
+  override def initialValue(ch: Channel): Option[Address] = None
+}
 
 /**
  * Provides the implementation of the Netty remote support
diff --git a/akka-remote/src/main/scala/akka/remote/netty/Server.scala b/akka-remote/src/main/scala/akka/remote/netty/Server.scala
index 87993f783d..5903dacd83 100644
--- a/akka-remote/src/main/scala/akka/remote/netty/Server.scala
+++ b/akka-remote/src/main/scala/akka/remote/netty/Server.scala
@@ -102,19 +102,11 @@ class RemoteServerAuthenticationHandler(secureCookie: Option[String]) extends Si
   }
 }
 
-object ChannelLocalSystem extends ChannelLocal[ActorSystemImpl] {
-  override def initialValue(ch: Channel): ActorSystemImpl = null
-}
-
 @ChannelHandler.Sharable
 class RemoteServerHandler(
   val openChannels: ChannelGroup,
   val netty: NettyRemoteTransport) extends SimpleChannelUpstreamHandler {
 
-  val channelAddress = new ChannelLocal[Option[Address]](false) {
-    override def initialValue(channel: Channel) = None
-  }
-
   import netty.settings
 
   private var addressToSet = true
@@ -138,16 +130,16 @@ class RemoteServerHandler(
   override def channelConnected(ctx: ChannelHandlerContext, event: ChannelStateEvent) = ()
 
   override def channelDisconnected(ctx: ChannelHandlerContext, event: ChannelStateEvent) = {
-    netty.notifyListeners(RemoteServerClientDisconnected(netty, channelAddress.get(ctx.getChannel)))
+    netty.notifyListeners(RemoteServerClientDisconnected(netty, ChannelAddress.get(ctx.getChannel)))
   }
 
   override def channelClosed(ctx: ChannelHandlerContext, event: ChannelStateEvent) = {
-    val address = channelAddress.get(ctx.getChannel)
+    val address = ChannelAddress.get(ctx.getChannel)
     if (address.isDefined && settings.UsePassiveConnections)
       netty.unbindClient(address.get)
 
     netty.notifyListeners(RemoteServerClientClosed(netty, address))
-    channelAddress.remove(ctx.getChannel)
+    ChannelAddress.remove(ctx.getChannel)
   }
 
   override def messageReceived(ctx: ChannelHandlerContext, event: MessageEvent) = try {
@@ -161,7 +153,7 @@ class RemoteServerHandler(
           case CommandType.CONNECT ⇒
             val origin = instruction.getOrigin
             val inbound = Address("akka", origin.getSystem, origin.getHostname, origin.getPort)
-            channelAddress.set(event.getChannel, Option(inbound))
+            ChannelAddress.set(event.getChannel, Option(inbound))
 
             //If we want to reuse the inbound connections as outbound we need to get busy
             if (settings.UsePassiveConnections)