Skip to content

Add kotlin.serialization implementation of Serializer #124

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 17 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,175 @@
/*
* Copyright (c) 2010-2021. Axon Framework
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.axonframework.extensions.kotlin.serialization

import kotlinx.serialization.ContextualSerializer
import kotlinx.serialization.KSerializer
import kotlinx.serialization.SerializationException
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonNull
import org.axonframework.common.ObjectUtils
import org.axonframework.serialization.AnnotationRevisionResolver
import org.axonframework.serialization.ChainingConverter
import org.axonframework.serialization.Converter
import org.axonframework.serialization.RevisionResolver
import org.axonframework.serialization.SerializedObject
import org.axonframework.serialization.SerializedType
import org.axonframework.serialization.Serializer
import org.axonframework.serialization.SimpleSerializedObject
import org.axonframework.serialization.SimpleSerializedType
import org.axonframework.serialization.UnknownSerializedType
import kotlin.reflect.full.companionObject
import kotlin.reflect.full.companionObjectInstance
import org.axonframework.serialization.SerializationException as AxonSerializationException

/**
* Implementation of Axon Serializer that uses a kotlinx.serialization implementation.
* The serialized format is JSON.
*
* The DSL function kotlinSerializer can be used to easily configure the parameters
* for this serializer.
*
* @see kotlinx.serialization.Serializer
* @see org.axonframework.serialization.Serializer
* @see kotlinSerializer
*
* @since 0.2.0
* @author Hidde Wieringa
*/
class KotlinSerializer(
private val revisionResolver: RevisionResolver = AnnotationRevisionResolver(),
private val converter: Converter = ChainingConverter(),
private val json: Json = Json,
) : Serializer {

private val serializerCache: MutableMap<Class<*>, KSerializer<*>> = mutableMapOf()

override fun <T> serialize(value: Any?, expectedRepresentation: Class<T>): SerializedObject<T> {
try {
val type = ObjectUtils.nullSafeTypeOf(value)

if (expectedRepresentation.isAssignableFrom(JsonElement::class.java)) {
return SimpleSerializedObject(
(if (value == null) JsonNull else json.encodeToJsonElement(type.serializer(), value)) as T,
expectedRepresentation,
typeForClass(type)
)
}

// By default, encode to String. This can be converted to other types by the converter
val stringSerialized: SerializedObject<String> = SimpleSerializedObject(
(if (value == null) "null" else json.encodeToString(type.serializer(), value)),
String::class.java,
typeForClass(type)
)

return converter.convert(stringSerialized, expectedRepresentation)
} catch (ex: SerializationException) {
throw AxonSerializationException("Cannot serialize type ${value?.javaClass?.name} to representation $expectedRepresentation.", ex)
}
}

override fun <T> canSerializeTo(expectedRepresentation: Class<T>): Boolean =
expectedRepresentation.isAssignableFrom(JsonElement::class.java) ||
converter.canConvert(String::class.java, expectedRepresentation)

override fun <S, T> deserialize(serializedObject: SerializedObject<S>?): T? {
try {
if (serializedObject == null) {
return null
}

if (serializedObject.type == SerializedType.emptyType()) {
return null
}

val foundType = classForType(serializedObject.type)
if (UnknownSerializedType::class.java.isAssignableFrom(foundType)) {
return UnknownSerializedType(this, serializedObject) as T
}

val serializer: KSerializer<T> = foundType.serializer() as KSerializer<T>

if (serializedObject.contentType.isAssignableFrom(JsonElement::class.java)) {
return json.decodeFromJsonElement(serializer, serializedObject.data as JsonElement)
}

val stringSerialized: SerializedObject<String> = converter.convert(serializedObject, String::class.java)
return json.decodeFromString(serializer, stringSerialized.data)
} catch (ex: SerializationException) {
throw AxonSerializationException(
"Could not deserialize from content type ${serializedObject?.contentType} to type ${serializedObject?.type}",
ex
)
}
}

private fun <S, T> SerializedObject<S>.serializer(): KSerializer<T> =
classForType(type).serializer() as KSerializer<T>

/**
* When a type is compiled by the Kotlin compiler extension, a companion object
* is created which contains a method `serializer()`. This method should be called
* to get the serializer of the class.
*
* In a 'normal' serialization environment, you would call the MyClass.serializer()
* method directly. Here we are in a generic setting, and need reflection to call
* the method.
*
* If there is no `serializer` method, a ContextualSerializer will be created. This
* serializer requires manual configuration of the SerializersModule containing a
* KSerializer which will be used when this class is serialized.
*
* This method caches the reflection mapping from class to serializer for efficiency.
*/
private fun <T> Class<T>.serializer(): KSerializer<T> =
serializerCache.computeIfAbsent(this) {
// Class<T>: T must be non-null
val kClass = (this as Class<Any>).kotlin

val companion = kClass.companionObject
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assume that the hard requirement on the @Serializable annotation makes it pretty tough to use this serializer for Axon objects, like the TrackingToken.
I've skimmed the Kotlin Serializer documentation somewhat but didn't spot a solution for this just yet.
Do you, perchance, have a solution in mind for this, @hiddewie?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the toughest question :)

As far as I can see, there are two solutions:

  1. Annotate the serializable types in Axon core with @Serializable. This adds the compile time requirement of having the Kotlin compiler plugin dependency in the build process. That may or may not be OK. Although, there are also Jackson annotations in the source code already. In runtime there is no Kotlin requirement.
  2. Create handwritten serializers for every class that needs to be serializable. This is quite some (cumbersome, error prone) work, depending on the number of serializable classes in the Axon core. The serializer code can be maintained in this repository.

To illustrate option 2, I added a small example and custom serializer with test.

The third option is abandoning this approach because the Kotlin serialization is not wanted in the core framework, and it is not worth the effort to maintain the custom serializers.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(After thinking about option 1 a bit more: I have not tested adding @Serializable on a Java class and then running the Kotlin compiler plugin on it. Theoretically it should work but in practice I don't know)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the insights here.
If you'd be at leisure to validate whether it is even possible to add the @Serializable annotation to a Java class, that would be amazing (and save me some personal investigation. 😅).

In the meantime, I'll start an internal discussion about whether we want to add the @Serializable annotation to the core of Axon Framework.

Copy link
Contributor Author

@hiddewie hiddewie Sep 20, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I tried a few things to test how the interoperability of Java classes with Kotlin annotations, and the Kotlin annotation processors work together.

What I did:

  • Complie Axon framewok locally (4.6.0-SNAPHSOT)
  • Add the Maven Kotlin compiler plugin which would compile the Kotlin annotations into serializer functions, see instructions here https://github.com/Kotlin/kotlinx.serialization#maven
  • Add @kotlinx.serialization.Serializable to ConfigToken

What I saw:

  • The annotation gets added (just like any other annotation) in the classfile
  • No serializer static method is generated.

What I did:

  • Copy the Java class and copy it to this repository, with the .java extension.
  • Let the Kotlin compiler, with the Kotlin serialization compiler plugin comple the .java class as well

What I saw:

  • The annotation gets added in the classfile
  • Still no serializer static method is generated.

What I did:

  • Let IntelliJ convert the .java file to a .kt Kotlin file (very ugly)

What I saw:

  • The annotation gets added in the classfile
  • A serializer method is generated.

So my conclusion is that the Kotlin serialization compiler plugin only works for Kotlin source files, and that the .serializer() static method cannot be generated for Java source files.

That is very unfortunate, I might open an issue with the Kotlin Serialization team to ask if that is supposed to work (ref Kotlin/kotlinx.serialization#1687).

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Either way, we can not assume that every class that comes here will have a serializer attached to it (nor be a Kotlin class)

If there is no serializer defined, or just based on types, we can fall back on a registry that we can keep as a map of Type (Class, Class name, etc) and a Serializer. That map could be configurable so someone might add support for classes without @Serializable like Java Dates, and we can pre-populate it with AF serializers configuration. Or keep the AF serializers in a separate config, but that's a finer detail.

This and
this is what I mean, as we need to pass the serializer explicitly in the very broad generic setting anyway, we might as well keep it somewhere in a config map, similar as to how this cache is doing but is just caching all serializers.

?: return@computeIfAbsent ContextualSerializer(kClass)

val serializerMethod = companion.java.getMethod("serializer")
?: return@computeIfAbsent ContextualSerializer(kClass)

serializerMethod.invoke(kClass.companionObjectInstance) as KSerializer<*>
} as KSerializer<T>

override fun classForType(type: SerializedType): Class<*> =
if (SerializedType.emptyType() == type) {
Void.TYPE
} else {
try {
Class.forName(type.name)
} catch (e: ClassNotFoundException) {
UnknownSerializedType::class.java
}
}

override fun typeForClass(type: Class<*>?): SerializedType =
if (type == null || Void.TYPE == type || Void::class.java == type) {
SimpleSerializedType.emptyType()
} else {
SimpleSerializedType(type.name, revisionResolver.revisionOf(type))
}

override fun getConverter(): Converter =
converter

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package org.axonframework.extensions.kotlin.serialization.serializer

import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.KSerializer
import kotlinx.serialization.Serializable
import kotlinx.serialization.Serializer
import kotlinx.serialization.descriptors.SerialDescriptor
import kotlinx.serialization.encoding.Decoder
import kotlinx.serialization.encoding.Encoder
import org.axonframework.eventhandling.tokenstore.ConfigToken

@Serializable
data class ConfigTokenSurrogate(
private val config: Map<String, String>
) {
fun toToken() = ConfigToken(config)
}

fun ConfigToken.toSurrogate() = ConfigTokenSurrogate(config)

@OptIn(ExperimentalSerializationApi::class)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would you be able to elaborate a little on the ExperimentalSerializationApi you're "opting in" here?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The @Serializer annotation below is marked as experimental. It might change its implementation slightly between versions of Kotlin Serialization. (https://kotlin.github.io/kotlinx.serialization/kotlinx-serialization-core/kotlinx-serialization-core/kotlinx.serialization/-experimental-serialization-api/index.html)

We might wait to merge this feature until the used serialization API is marked fully stable, although the Axon extension-kotlin library is also marked as experimental itself.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So to clarify, only @Serializer which is used to hook custom serializer objects is marked as Experimental, the rest of Json serialization is not experimental anymore and is stable. Is this a correct assumption @hiddewie ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes exactly. The following is stable:

@Serializable
class XXX

But the following needs experimental opt in:

@Serializer(forClass = QQQ::class)
class XXX

In particular doing this for Java classes gives compiler errors, and it might simply not be supported (see Kotlin/kotlinx.serialization#1687)

@Serializer(forClass = ConfigToken::class)
class ConfigTokenSerializer : KSerializer<ConfigToken> {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the custom serializer implemented manually for the internal Axon type. These type of implementations are needed if the @Serializable annotation is not used in combination with the compiler plugin.

Copy link
Member

@smcvb smcvb Sep 8, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd assume this should be constructed for every concrete implementation of serializable Axon objects in that case, correct?

This would at least enable moving further with this serializer, so that's good.

If we go this route, it will require some KDoc, of course ;-)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I just saw https://github.com/Kotlin/kotlinx.serialization/blob/f673bc2/docs/serializers.md#deriving-external-serializer-for-another-kotlin-class-experimental for the first time. This would allow us to skip the 'manual' implementation of the serializer.

Unfortunately I can't yet get it to work, I run into this bug in the Kotlin complier plugin (Kotlin/kotlinx.serialization#1680).

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Have you tried Surrogate serializers and Delegating serializers?

They look much simpler and might just get the job done. Hopefully, they might just work and there would be no hidden issues like you encountered previously.

Copy link
Contributor Author

@hiddewie hiddewie Apr 16, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(#124 (comment)) I changed the fully-custom example ConfigTokenSerializer into a serializer using a ConfigTokenSurrogate. Its slightly better maintainable, but still a LOT of work to do (and maintain) for every Axon class.

The work means, for every Axon class:

  • Create a Kotlin class with @Serializable
  • Put all the fields of the class into the surrogate, and add converter methods to convert from the class to the surrogate, and back.
  • Manually implement a serializer for that class, using the SerialDescriptor generated by the surrogate.
  • During serialization, convert from the real value to a surrogate and serialize it.
  • During deserialization, deserialize the surrogate and convert from the surrogate to the value of the real class.

The serializers for surrogates are very simple and similar (only a few lines of code). However the surrogates themselves need to contain exactly the public data fields of the class they are a surrogate for. This will cause bugs if a field is forgotten in the surrogate.

override val descriptor: SerialDescriptor = ConfigTokenSurrogate.serializer().descriptor

override fun serialize(encoder: Encoder, value: ConfigToken) =
encoder.encodeSerializableValue(ConfigTokenSurrogate.serializer(), value.toSurrogate())

override fun deserialize(decoder: Decoder): ConfigToken =
decoder.decodeSerializableValue(ConfigTokenSurrogate.serializer()).toToken()
}

// The following does not work, emits compiler errors.
// Also with the `-Xuse-ir` Kotlin compiler argument (version 1.6.20)

//@OptIn(ExperimentalSerializationApi::class)
//@Serializer(forClass = ConfigToken::class)
//object ConfigTokenSerializer
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
package org.axonframework.extensions.kotlin.serialization

import kotlinx.serialization.Serializable
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.modules.SerializersModule
import kotlinx.serialization.modules.contextual
import org.axonframework.eventhandling.tokenstore.ConfigToken
import org.axonframework.extensions.kotlin.serialization.serializer.ConfigTokenSerializer
import org.axonframework.serialization.AnnotationRevisionResolver
import org.axonframework.serialization.ChainingConverter
import org.axonframework.serialization.SerializedType
import org.axonframework.serialization.SimpleSerializedObject
import org.axonframework.serialization.UnknownSerializedType
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Test
import java.io.InputStream
import kotlin.test.assertNotNull
import kotlin.test.assertNull

class KotlinSerializerTest {

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am missing test cases that validate if the KotlinSerializer can be used for Axon objects, like the Message, TrackingToken, and SagaEntry.

Assuming that's because those objects aren't annotated with the @Serializable annotation.
However, the Serializer in Axon must de-/serialize those objects for it to be usable.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Indeed. See https://github.com/AxonFramework/extension-kotlin/pull/124/files#r645048334 for the comment about serializing internal classes.

I added one test case for the ConfigToken, and every internal class that should be serializable can be added in that way, if there is a serializer.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Gotcha. To make this Serializer implementation workable for Axon Framework entirely, these tests will be a requirement too.

Making these can wait until we've concluded the conversation on the Serializer implementation though.
On the subject of using the @Serializable annotation on Axon components, and if that even works for Java classes, that is.

/**
* This class will automatically become serializable through the Kotlin serialization compiler plugin.
*/
@Serializable
data class TestData(
val name: String,
val value: Float?
)

@Test
fun canSerializeTo() {
val serializer = KotlinSerializer()

assertTrue(serializer.canSerializeTo(String::class.java))
assertTrue(serializer.canSerializeTo(ByteArray::class.java))
assertTrue(serializer.canSerializeTo(JsonElement::class.java))
assertTrue(serializer.canSerializeTo(InputStream::class.java))
}

@Test
fun `configuration options`() {
val serializer = KotlinSerializer(
revisionResolver = AnnotationRevisionResolver(),
converter = ChainingConverter(),
json = Json,
)

assertNotNull(serializer)
}

@Test
fun serialize() {
val serializer = KotlinSerializer()

val emptySerialized = serializer.serialize(TestData("", null), String::class.java)
assertEquals("SimpleSerializedType[org.axonframework.extensions.kotlin.serialization.KotlinSerializerTest\$TestData] (revision null)", emptySerialized.type.toString())
assertEquals("""{"name":"","value":null}""", emptySerialized.data)
assertEquals(String::class.java, emptySerialized.contentType)

val filledSerialized = serializer.serialize(TestData("name", 1.23f), String::class.java)
assertEquals("SimpleSerializedType[org.axonframework.extensions.kotlin.serialization.KotlinSerializerTest\$TestData] (revision null)", filledSerialized.type.toString())
assertEquals("""{"name":"name","value":1.23}""", filledSerialized.data)
assertEquals(String::class.java, filledSerialized.contentType)

val nullSerialized = serializer.serialize(null, String::class.java)
assertEquals("null", nullSerialized.data)
assertEquals(String::class.java, nullSerialized.contentType)
}

@Test
fun deserialize() {
val serializer = KotlinSerializer()

val nullDeserialized: Any? = serializer.deserialize(
SimpleSerializedObject(
"",
String::class.java,
SerializedType.emptyType()
)
)
assertNull(nullDeserialized)

val emptyDeserialized: Any? = serializer.deserialize(
SimpleSerializedObject(
"""{"name":"","value":null}""",
String::class.java,
TestData::class.java.name,
null
)
)
assertNotNull(emptyDeserialized as TestData)
assertEquals(emptyDeserialized.name, "")
assertEquals(emptyDeserialized.value, null)

val filledDeserialized: Any? = serializer.deserialize(
SimpleSerializedObject(
"""{"name":"name","value":1.23}""",
String::class.java,
TestData::class.java.name,
null
)
)
assertNotNull(filledDeserialized as TestData)
assertEquals(filledDeserialized.name, "name")
assertEquals(filledDeserialized.value, 1.23f)

val unknownDeserializedType: Any? = serializer.deserialize(
SimpleSerializedObject(
"""anything""",
String::class.java,
UnknownSerializedType::class.java.name,
null
)
)
assertNotNull(unknownDeserializedType as UnknownSerializedType)
}

@Test
fun `byte arrays`() {
val serializer = KotlinSerializer()

assertNotNull(serializer.deserialize(serializer.serialize(TestData("name", null), ByteArray::class.java)))
}

@Test
fun `JSON elements`() {
val serializer = KotlinSerializer()

assertNotNull(serializer.deserialize(serializer.serialize(TestData("name", null), JsonElement::class.java)))
}

@Test
fun `input stream`() {
val serializer = KotlinSerializer()

assertNotNull(serializer.deserialize(serializer.serialize(TestData("name", null), InputStream::class.java)))
}

@Test
fun `example of custom serializer for ConfigToken`() {
val serializer = KotlinSerializer(
json = Json {
serializersModule = SerializersModule {
contextual(ConfigTokenSerializer())
}
}
)

val tokenBefore = ConfigToken(mapOf("test" to "value"))
val serialized = serializer.serialize(tokenBefore, String::class.java)
assertEquals("""{"config":{"test":"value"}}""", serialized.data)
val token: ConfigToken? = serializer.deserialize(serialized)
assertNotNull(token as ConfigToken)
assertEquals("value", token.get("test"))
}
}
Loading