htc #20379 allow registering custom media types (#20401)

htc #20379 add mima filters for custom media type

akka-docs/rst/scala/http/common/http-model.rst

@@ -348,3 +348,22 @@ provided to parse (or render to) Strings or byte arrays.
   and can override them if needed. This is useful, since both ``client`` and ``host-connection-pool`` APIs,
   such as the Client API ``Http().outgoingConnection`` or the Host Connection Pool APIs ``Http().singleRequest`` or ``Http().superPool``,
   usually need the same settings, however the ``server`` most likely has a very different set of settings.
+
+.. _registeringCustomMediaTypes:
+
+Registering Custom Media Types
+------------------------------
+
+Akka HTTP `predefines`_ most commonly encoutered media types and emits them in their well-typed form while parsing http messages.
+Sometimes you may want to define a custom media type and inform the parser infrastructure about how to handle these custom
+media types, e.g. that ``application/custom`` is to be treated as ``NonBinary`` with ``WithFixedCharset``. To achieve this you
+need to register the custom media type in the server's settings by configuring ``ParserSettings`` like this:
+
+.. includecode:: ../../../../../akka-http-tests/src/test/scala/akka/http/scaladsl/CustomMediaTypesSpec.scala
+   :include: application-custom
+
+You may also want to read about MediaType `Registration trees`_, in order to register your vendor specific media types
+in the right style / place.
+
+.. _Registration trees: https://en.wikipedia.org/wiki/Media_type#Registration_trees
+.. _predefines: https://github.com/akka/akka/blob/master/akka-http-core/src/main/scala/akka/http/scaladsl/model/MediaType.scala#L297

akka-docs/rst/scala/http/migration-from-spray.rst

@@ -8,3 +8,4 @@ Migration Guide from spray
 - ``respondWithMediaType`` was considered an anti-pattern in spray and is not ported to Akka HTTP.
   Instead users should rely on content type negotiation as Akka HTTP implements it.
   More information here: https://github.com/akka/akka/issues/18625
+- :ref:`registeringCustomMediaTypes` changed from Spray in order not to rely on global state.

akka-http-core/src/main/scala/akka/http/impl/engine/parsing/HttpHeaderParser.scala

@@ -11,7 +11,7 @@ import scala.annotation.tailrec
 import akka.parboiled2.CharUtils
 import akka.util.ByteString
 import akka.http.impl.util._
-import akka.http.scaladsl.model.{ IllegalHeaderException, StatusCodes, HttpHeader, ErrorInfo }
+import akka.http.scaladsl.model._
 import akka.http.scaladsl.model.headers.{ EmptyHeader, RawHeader }
 import akka.http.impl.model.parser.HeaderParser
 import akka.http.impl.model.parser.CharacterClasses._
@@ -412,6 +412,7 @@ private[http] object HttpHeaderParser {
     def maxHeaderNameLength: Int
     def maxHeaderValueLength: Int
     def headerValueCacheLimit(headerName: String): Int
+    def customMediaTypes: MediaTypes.FindCustom
   }
 
   private def predefinedHeaders = Seq(

akka-http-core/src/main/scala/akka/http/impl/model/parser/CommonActions.scala

@@ -5,15 +5,20 @@
 package akka.http.impl.model.parser
 
 import akka.http.impl.util._
+import akka.http.scaladsl.model.MediaType.Binary
 import akka.http.scaladsl.model._
-import MediaTypes._
+import akka.stream.impl.ConstantFun
 
+/** INTERNAL API */
 private[parser] trait CommonActions {
 
+  def customMediaTypes: MediaTypes.FindCustom
+
   type StringMapBuilder = scala.collection.mutable.Builder[(String, String), Map[String, String]]
 
   def getMediaType(mainType: String, subType: String, charsetDefined: Boolean,
                    params: Map[String, String]): MediaType = {
+    import MediaTypes._
     val subLower = subType.toRootLowerCase
     mainType.toRootLowerCase match {
       case "multipart" ⇒ subLower match {
@@ -26,20 +31,29 @@ private[parser] trait CommonActions {
         case custom        ⇒ MediaType.customMultipart(custom, params)
       }
       case mainLower ⇒
-        MediaTypes.getForKey((mainLower, subLower)) match {
+        // attempt fetching custom media type if configured
+        if (areCustomMediaTypesDefined)
+          customMediaTypes(mainLower, subType) getOrElse fallbackMediaType(subType, params, mainLower)
+        else MediaTypes.getForKey((mainLower, subLower)) match {
           case Some(registered) ⇒ if (params.isEmpty) registered else registered.withParams(params)
           case None ⇒
             if (charsetDefined)
               MediaType.customWithOpenCharset(mainLower, subType, params = params, allowArbitrarySubtypes = true)
             else
-              MediaType.customBinary(mainLower, subType, MediaType.Compressible, params = params,
+              fallbackMediaType(subType, params, mainLower)
-                allowArbitrarySubtypes = true)
         }
     }
   }
 
+  /** Provide a generic MediaType when no known-ones matched. */
+  private def fallbackMediaType(subType: String, params: Map[String, String], mainLower: String): Binary =
+    MediaType.customBinary(mainLower, subType, MediaType.Compressible, params = params, allowArbitrarySubtypes = true)
+
   def getCharset(name: String): HttpCharset =
     HttpCharsets
       .getForKeyCaseInsensitive(name)
       .getOrElse(HttpCharset.custom(name))
+
+  @inline private def areCustomMediaTypesDefined: Boolean = customMediaTypes ne ConstantFun.two2none
+
 }

akka-http-core/src/main/scala/akka/http/impl/model/parser/HeaderParser.scala

@@ -7,6 +7,7 @@ package akka.http.impl.model.parser
 import akka.http.scaladsl.settings.ParserSettings
 import akka.http.scaladsl.settings.ParserSettings.CookieParsingMode
 import akka.http.scaladsl.model.headers.HttpCookiePair
+import akka.stream.impl.ConstantFun
 import scala.util.control.NonFatal
 import akka.http.impl.util.SingletonException
 import akka.parboiled2._
@@ -16,7 +17,10 @@ import akka.http.scaladsl.model._
 /**
  * INTERNAL API.
  */
-private[http] class HeaderParser(val input: ParserInput, settings: HeaderParser.Settings = HeaderParser.DefaultSettings) extends Parser with DynamicRuleHandler[HeaderParser, HttpHeader :: HNil]
+private[http] class HeaderParser(
+  val input: ParserInput,
+  settings: HeaderParser.Settings = HeaderParser.DefaultSettings)
+  extends Parser with DynamicRuleHandler[HeaderParser, HttpHeader :: HNil]
   with CommonRules
   with AcceptCharsetHeader
   with AcceptEncodingHeader
@@ -33,6 +37,8 @@ private[http] class HeaderParser(val input: ParserInput, settings: HeaderParser.
   with WebSocketHeaders {
   import CharacterClasses._
 
+  override def customMediaTypes = settings.customMediaTypes
+
   // http://www.rfc-editor.org/errata_search.php?rfc=7230 errata id 4189
   def `header-field-value`: Rule1[String] = rule {
     FWS ~ clearSB() ~ `field-value` ~ FWS ~ EOI ~ push(sb.toString)
@@ -161,15 +167,19 @@ private[http] object HeaderParser {
   abstract class Settings {
     def uriParsingMode: Uri.ParsingMode
     def cookieParsingMode: ParserSettings.CookieParsingMode
+    def customMediaTypes: MediaTypes.FindCustom
   }
   def Settings(uriParsingMode: Uri.ParsingMode = Uri.ParsingMode.Relaxed,
-               cookieParsingMode: ParserSettings.CookieParsingMode = ParserSettings.CookieParsingMode.RFC6265): Settings = {
+               cookieParsingMode: ParserSettings.CookieParsingMode = ParserSettings.CookieParsingMode.RFC6265,
+               customMediaTypes: MediaTypes.FindCustom = ConstantFun.scalaAnyTwoToNone): Settings = {
     val _uriParsingMode = uriParsingMode
     val _cookieParsingMode = cookieParsingMode
+    val _customMediaTypes = customMediaTypes
 
     new Settings {
       def uriParsingMode: Uri.ParsingMode = _uriParsingMode
       def cookieParsingMode: CookieParsingMode = _cookieParsingMode
+      def customMediaTypes: MediaTypes.FindCustom = _customMediaTypes
     }
   }
   val DefaultSettings: Settings = Settings()

akka-http-core/src/main/scala/akka/http/impl/settings/ParserSettingsImpl.scala

@@ -6,9 +6,10 @@ package akka.http.impl.settings
 
 import akka.http.scaladsl.settings.ParserSettings
 import akka.http.scaladsl.settings.ParserSettings.{ ErrorLoggingVerbosity, CookieParsingMode }
+import akka.stream.impl.ConstantFun
 import com.typesafe.config.Config
 import scala.collection.JavaConverters._
-import akka.http.scaladsl.model.{ StatusCode, HttpMethod, Uri }
+import akka.http.scaladsl.model._
 import akka.http.impl.util._
 
 /** INTERNAL API */
@@ -29,7 +30,8 @@ private[akka] final case class ParserSettingsImpl(
   headerValueCacheLimits: Map[String, Int],
   includeTlsSessionInfoHeader: Boolean,
   customMethods: String ⇒ Option[HttpMethod],
-  customStatusCodes: Int ⇒ Option[StatusCode])
+  customStatusCodes: Int ⇒ Option[StatusCode],
+  customMediaTypes: MediaTypes.FindCustom)
   extends akka.http.scaladsl.settings.ParserSettings {
 
   require(maxUriLength > 0, "max-uri-length must be > 0")
@@ -52,9 +54,9 @@ private[akka] final case class ParserSettingsImpl(
 
 object ParserSettingsImpl extends SettingsCompanion[ParserSettingsImpl]("akka.http.parsing") {
 
-  // for equality
+  private[this] val noCustomMethods: String ⇒ Option[HttpMethod] = ConstantFun.scalaAnyToNone
-  private[this] val noCustomMethods: String ⇒ Option[HttpMethod] = _ ⇒ None
+  private[this] val noCustomStatusCodes: Int ⇒ Option[StatusCode] = ConstantFun.scalaAnyToNone
-  private[this] val noCustomStatusCodes: Int ⇒ Option[StatusCode] = _ ⇒ None
+  private[this] val noCustomMediaTypes: (String, String) ⇒ Option[MediaType] = ConstantFun.scalaAnyTwoToNone
 
   def fromSubConfig(root: Config, inner: Config) = {
     val c = inner.withFallback(root.getConfig(prefix))
@@ -77,7 +79,8 @@ object ParserSettingsImpl extends SettingsCompanion[ParserSettingsImpl]("akka.ht
       cacheConfig.entrySet.asScala.map(kvp ⇒ kvp.getKey -> cacheConfig.getInt(kvp.getKey))(collection.breakOut),
       c getBoolean "tls-session-info-header",
       noCustomMethods,
-      noCustomStatusCodes)
+      noCustomStatusCodes,
+      noCustomMediaTypes)
   }
 
 }

akka-http-core/src/main/scala/akka/http/impl/settings/ServerSettingsImpl.scala

@@ -77,7 +77,7 @@ object ServerSettingsImpl extends SettingsCompanion[ServerSettingsImpl]("akka.ht
     c getInt "backlog",
     SocketOptionSettings.fromSubConfig(root, c.getConfig("socket-options")),
     defaultHostHeader =
-      HttpHeader.parse("Host", c getString "default-host-header") match {
+      HttpHeader.parse("Host", c getString "default-host-header", ParserSettings(root)) match {
         case HttpHeader.ParsingResult.Ok(x: Host, Nil) ⇒ x
         case result ⇒
           val info = result.errors.head.withSummary("Configured `default-host-header` is illegal")

akka-http-core/src/main/scala/akka/http/javadsl/settings/ParserSettings.scala

@@ -12,7 +12,7 @@ import akka.http.impl.util.JavaMapping.Implicits._
 import scala.annotation.varargs
 import scala.collection.JavaConverters._
 
-import akka.http.javadsl.model.{ HttpMethod, StatusCode, Uri }
+import akka.http.javadsl.model.{ MediaType, HttpMethod, StatusCode, Uri }
 import com.typesafe.config.Config
 
 /**
@@ -37,6 +37,7 @@ abstract class ParserSettings private[akka] () extends BodyPartParser.Settings {
   def headerValueCacheLimits: Map[String, Int]
   def getCustomMethods: java.util.function.Function[String, Optional[HttpMethod]]
   def getCustomStatusCodes: java.util.function.Function[Int, Optional[StatusCode]]
+  def getCustomMediaTypes: akka.japi.function.Function2[String, String, Optional[MediaType]]
 
   // ---
 
@@ -68,6 +69,11 @@ abstract class ParserSettings private[akka] () extends BodyPartParser.Settings {
     val map = codes.map(c ⇒ c.intValue -> c.asScala).toMap
     self.copy(customStatusCodes = map.get)
   }
+  @varargs
+  def withCustomMediaTypes(mediaTypes: MediaType*): ParserSettings = {
+    val map = mediaTypes.map(c ⇒ (c.mainType, c.subType) -> c.asScala).toMap
+    self.copy(customMediaTypes = (main, sub) ⇒ map.get(main -> sub))
+  }
 
 }
 

akka-http-core/src/main/scala/akka/http/scaladsl/model/HttpHeader.scala

@@ -4,6 +4,8 @@
 
 package akka.http.scaladsl.model
 
+import akka.http.scaladsl.settings.ParserSettings
+
 import scala.util.{ Success, Failure }
 import akka.parboiled2.ParseError
 import akka.http.impl.util.ToStringRenderable

akka-http-core/src/main/scala/akka/http/scaladsl/model/HttpMessage.scala

@@ -84,7 +84,12 @@ sealed trait HttpMessage extends jm.HttpMessage {
   /** Returns the first header of the given type if there is one */
   def header[T <: jm.HttpHeader: ClassTag]: Option[T] = {
     val erasure = classTag[T].runtimeClass
-    headers.find(erasure.isInstance).asInstanceOf[Option[T]]
+    headers.find(erasure.isInstance).asInstanceOf[Option[T]] match {
+      case header: Some[T] => header
+      case _ if erasure == classOf[`Content-Type`] => Some(entity.contentType).asInstanceOf[Option[T]]
+      case _ => None
+    }
+
   }
 
   /**

akka-http-core/src/main/scala/akka/http/scaladsl/model/MediaType.scala

@@ -258,6 +258,8 @@ object MediaType {
 }
 
 object MediaTypes extends ObjectRegistry[(String, String), MediaType] {
+  type FindCustom = (String, String) => Option[MediaType]
+
   private[this] var extensionMap = Map.empty[String, MediaType]
 
   def forExtensionOption(ext: String): Option[MediaType] = extensionMap.get(ext.toLowerCase)

akka-http-core/src/main/scala/akka/http/scaladsl/settings/ParserSettings.scala

@@ -9,7 +9,8 @@ import java.util.function.Function
 
 import akka.http.impl.settings.ParserSettingsImpl
 import akka.http.impl.util._
-import akka.http.scaladsl.model.{ HttpMethod, StatusCode, Uri }
+import akka.http.javadsl.model
+import akka.http.scaladsl.model._
 import akka.http.scaladsl.{ settings ⇒ js }
 import com.typesafe.config.Config
 
@@ -37,6 +38,7 @@ abstract class ParserSettings private[akka] () extends akka.http.javadsl.setting
   def includeTlsSessionInfoHeader: Boolean
   def customMethods: String ⇒ Option[HttpMethod]
   def customStatusCodes: Int ⇒ Option[StatusCode]
+  def customMediaTypes: MediaTypes.FindCustom
 
   /* Java APIs */
   override def getCookieParsingMode: js.ParserSettings.CookieParsingMode = cookieParsingMode
@@ -61,6 +63,10 @@ abstract class ParserSettings private[akka] () extends akka.http.javadsl.setting
   override def getCustomStatusCodes = new Function[Int, Optional[akka.http.javadsl.model.StatusCode]] {
     override def apply(t: Int) = OptionConverters.toJava(customStatusCodes(t))
   }
+  override def getCustomMediaTypes = new akka.japi.function.Function2[String, String, Optional[akka.http.javadsl.model.MediaType]] {
+    override def apply(mainType: String, subType: String): Optional[model.MediaType] =
+      OptionConverters.toJava(customMediaTypes(mainType, subType))
+  }
 
   // ---
 
@@ -90,6 +96,10 @@ abstract class ParserSettings private[akka] () extends akka.http.javadsl.setting
     val map = codes.map(c ⇒ c.intValue -> c).toMap
     self.copy(customStatusCodes = map.get)
   }
+  def withCustomMediaTypes(types: MediaType*): ParserSettings = {
+    val map = types.map(c ⇒ (c.mainType, c.subType) -> c).toMap
+    self.copy(customMediaTypes = (main, sub) ⇒ map.get((main, sub)))
+  }
 }
 
 object ParserSettings extends SettingsCompanion[ParserSettings] {

akka-http-tests/src/test/scala/akka/http/scaladsl/CustomMediaTypesSpec.scala

@@ -0,0 +1,59 @@
+/*
+ * Copyright (C) 2009-2016 Lightbend Inc. <http://www.lightbend.com>
+ */
+
+package akka.http.scaladsl
+
+import akka.http.scaladsl.client.RequestBuilding
+import akka.http.scaladsl.model.MediaType.WithFixedCharset
+import akka.http.scaladsl.model._
+import akka.http.scaladsl.model.headers._
+import akka.http.scaladsl.server.{ Directives, RoutingSpec }
+import akka.http.scaladsl.settings.{ ParserSettings, ServerSettings }
+import akka.stream.ActorMaterializer
+import akka.testkit.AkkaSpec
+import akka.util.ByteString
+import org.scalatest.concurrent.ScalaFutures
+import scala.concurrent.duration._
+
+class CustomMediaTypesSpec extends AkkaSpec with ScalaFutures
+  with Directives with RequestBuilding {
+
+  implicit val mat = ActorMaterializer()
+
+  "Http" should {
+    "allow registering custom media type" in {
+      import system.dispatcher
+      val (_, host, port) = TestUtils.temporaryServerHostnameAndPort()
+
+      //#application-custom
+
+      // similarily in Java: `akka.http.javadsl.settings.[...]`
+      import akka.http.scaladsl.settings.ParserSettings
+      import akka.http.scaladsl.settings.ServerSettings
+
+      // define custom media type:
+      val utf8 = HttpCharsets.`UTF-8`
+      val `application/custom`: WithFixedCharset =
+        MediaType.customWithFixedCharset("application", "custom", utf8)
+
+      // add custom media type to parser settings:
+      val parserSettings = ParserSettings(system).withCustomMediaTypes(`application/custom`)
+      val serverSettings = ServerSettings(system).withParserSettings(parserSettings)
+
+      val routes = extractRequest { r ⇒
+        complete(r.entity.contentType.toString + " = " + r.entity.contentType.getClass)
+      }
+      val binding = Http().bindAndHandle(routes, host, port, settings = serverSettings)
+      //#application-custom
+
+      val request = Get(s"http://$host:$port/").withEntity(HttpEntity(`application/custom`, "~~example~=~value~~"))
+      val response = Http().singleRequest(request).futureValue
+
+      response.status should ===(StatusCodes.OK)
+      val responseBody = response.toStrict(1.second).futureValue.entity.dataBytes.runFold(ByteString.empty)(_ ++ _).futureValue.utf8String
+      responseBody should ===("application/custom = class akka.http.scaladsl.model.ContentType$WithFixedCharset")
+    }
+  }
+}
+

akka-stream/src/main/scala/akka/stream/impl/ConstantFun.scala

@@ -21,9 +21,17 @@ private[akka] object ConstantFun {
 
   def scalaIdentityFunction[T]: T ⇒ T = conforms
 
+  def scalaAnyToNone[A, B]: A ⇒ Option[B] = none
+  def scalaAnyTwoToNone[A, B, C]: (A, B) ⇒ Option[C] = two2none
+  def javaAnyToNone[A, B]: A ⇒ Option[B] = none
+
   val zeroLong = (_: Any) ⇒ 0L
 
   val oneLong = (_: Any) ⇒ 1L
 
   val oneInt = (_: Any) ⇒ 1
+
+  val none = (_: Any) ⇒ None
+
+  val two2none = (_: Any, _: Any) ⇒ None
 }

project/MiMa.scala

@@ -758,9 +758,22 @@ object MiMa extends AutoPlugin {
         ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.http.javadsl.model.HttpMessage#MessageTransformations.withEntity"),
         ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.http.scaladsl.model.HttpMessage.withEntity"),
         
+        // #20401 custom media types registering
+        ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.http.impl.model.parser.CommonActions.customMediaTypes"),
+        ProblemFilters.exclude[DirectMissingMethodProblem]("akka.http.impl.model.parser.HeaderParser.Settings"),
+        ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.http.impl.model.parser.HeaderParser#Settings.customMediaTypes"),
+        ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.http.impl.engine.parsing.HttpHeaderParser#Settings.customMediaTypes"),
+        ProblemFilters.exclude[DirectMissingMethodProblem]("akka.http.impl.settings.ParserSettingsImpl.apply"),
+        ProblemFilters.exclude[DirectMissingMethodProblem]("akka.http.impl.settings.ParserSettingsImpl.copy"),
+        ProblemFilters.exclude[DirectMissingMethodProblem]("akka.http.impl.settings.ParserSettingsImpl.this"),
+        
         // #20123
         ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.stream.scaladsl.FlowOps.recoverWithRetries"),
 
+        // #20379 Allow registering custom media types
+        ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.http.javadsl.settings.ParserSettings.getCustomMediaTypes"),
+        ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.http.scaladsl.settings.ParserSettings.customMediaTypes"),
+
         // internal api
         FilterAnyProblemStartingWith("akka.stream.impl"),
         FilterAnyProblemStartingWith("akka.http.impl"),