Skip to content

Commit f3e9dc5

Browse files
sugmanuesugmanue
authored
Consolidate unmarshalling and parsing in a single pass (#5643)
* Consolidate unmarshalling and parsing in a single pass * Address PR comments * Address PR comments * Add a test to assert that a non-object literal throws * Add more testing * Use Jackson's built-in fast float parsing * Add customizaion option to enable the fast unmarshalling path * Fix checkstyle issues * Account for tests with unset clientConfiguration * Fix codegen checkstyle
1 parent 440cc9e commit f3e9dc5

File tree

77 files changed

+5182
-1568
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

77 files changed

+5182
-1568
lines changed

‎.changes/next-release/feature-AWSSDKforJavav2-70d110c.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"category": "AWS SDK for Java v2",
3+
"contributor": "",
4+
"type": "feature",
5+
"description": "Improve unmarshalling performance of all JSON protocols by unifying parsing with unmarshalling instead of doing one after the other."
6+
}

‎codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -347,6 +347,11 @@ public class CustomizationConfig {
347347
*/
348348
private boolean batchManagerSupported;
349349

350+
/**
351+
* A boolean flag to indicate if the fast unmarshaller code path is enabled.
352+
*/
353+
private boolean enableFastUnmarshaller;
354+
350355
private CustomizationConfig() {
351356
}
352357

@@ -914,4 +919,11 @@ public void setBatchManagerSupported(boolean batchManagerSupported) {
914919
this.batchManagerSupported = batchManagerSupported;
915920
}
916921

922+
public boolean getEnableFastUnmarshaller() {
923+
return enableFastUnmarshaller;
924+
}
925+
926+
public void setEnableFastUnmarshaller(boolean enableFastUnmarshaller) {
927+
this.enableFastUnmarshaller = enableFastUnmarshaller;
928+
}
917929
}

‎codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/BaseClientBuilderClass.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@
7373
import software.amazon.awssdk.identity.spi.IdentityProvider;
7474
import software.amazon.awssdk.identity.spi.IdentityProviders;
7575
import software.amazon.awssdk.identity.spi.TokenIdentity;
76+
import software.amazon.awssdk.protocols.json.internal.unmarshall.SdkClientJsonProtocolAdvancedOption;
7677
import software.amazon.awssdk.regions.ServiceMetadataAdvancedOption;
7778
import software.amazon.awssdk.utils.AttributeMap;
7879
import software.amazon.awssdk.utils.CollectionUtils;
@@ -479,6 +480,13 @@ private MethodSpec finalizeServiceConfigurationMethod() {
479480
.addCode(" .fipsEnabled(c.get($T.FIPS_ENDPOINT_ENABLED))", AwsClientOption.class)
480481
.addCode(" .build());");
481482

483+
if (model.getMetadata().isJsonProtocol()) {
484+
if (model.getCustomizationConfig().getEnableFastUnmarshaller()) {
485+
builder.addStatement("builder.option($1T.ENABLE_FAST_UNMARSHALLER, true)",
486+
SdkClientJsonProtocolAdvancedOption.class);
487+
}
488+
}
489+
482490
builder.addStatement("return builder.build()");
483491
return builder.build();
484492
}

‎codegen/src/main/java/software/amazon/awssdk/codegen/poet/model/AwsServiceModel.java‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import java.util.Collection;
3939
import java.util.Collections;
4040
import java.util.List;
41+
import java.util.Map;
4142
import java.util.Optional;
4243
import java.util.function.BiConsumer;
4344
import java.util.function.Consumer;
@@ -98,7 +99,6 @@ public TypeSpec poetSpec() {
9899
if (shapeModel.isEventStream()) {
99100
return eventStreamInterfaceSpec();
100101
}
101-
102102
List<FieldSpec> fields = shapeModelSpec.fields();
103103

104104
TypeSpec.Builder specBuilder = TypeSpec.classBuilder(className())
@@ -110,9 +110,9 @@ public TypeSpec poetSpec() {
110110
.addFields(fields)
111111
.addFields(shapeModelSpec.staticFields())
112112
.addMethod(addModifier(sdkFieldsMethod(), FINAL))
113+
.addMethod(addModifier(sdkFieldNameToFieldMethod(), FINAL))
113114
.addTypes(nestedModelClassTypes());
114115

115-
116116
if (shapeModel.isUnion()) {
117117
specBuilder.addField(unionTypeField());
118118
}
@@ -316,6 +316,17 @@ private MethodSpec sdkFieldsMethod() {
316316
.build();
317317
}
318318

319+
private MethodSpec sdkFieldNameToFieldMethod() {
320+
ParameterizedTypeName sdkFieldType = ParameterizedTypeName.get(ClassName.get(SdkField.class),
321+
WildcardTypeName.subtypeOf(ClassName.get(Object.class)));
322+
return MethodSpec.methodBuilder("sdkFieldNameToField")
323+
.addModifiers(PUBLIC)
324+
.addAnnotation(Override.class)
325+
.returns(ParameterizedTypeName.get(ClassName.get(Map.class), ClassName.get(String.class), sdkFieldType))
326+
.addCode("return SDK_NAME_TO_FIELD;")
327+
.build();
328+
}
329+
319330
private MethodSpec getterCreator() {
320331
TypeVariableName t = TypeVariableName.get("T");
321332
return MethodSpec.methodBuilder("getter")

‎codegen/src/main/java/software/amazon/awssdk/codegen/poet/model/ModelBuilderSpecs.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import java.util.ArrayList;
3232
import java.util.EnumSet;
3333
import java.util.List;
34+
import java.util.Map;
3435
import java.util.Set;
3536
import java.util.function.Consumer;
3637
import javax.lang.model.element.Modifier;
@@ -141,6 +142,7 @@ public TypeSpec beanStyleBuilder() {
141142
builderClassBuilder.addMethods(accessors());
142143
builderClassBuilder.addMethod(buildMethod());
143144
builderClassBuilder.addMethod(sdkFieldsMethod());
145+
builderClassBuilder.addMethod(sdkFieldNameToFieldMethod());
144146

145147
if (shapeModel.isUnion()) {
146148
builderClassBuilder.addMethod(handleUnionValueChangeMethod());
@@ -169,6 +171,17 @@ private MethodSpec sdkFieldsMethod() {
169171
.build();
170172
}
171173

174+
private MethodSpec sdkFieldNameToFieldMethod() {
175+
ParameterizedTypeName sdkFieldType = ParameterizedTypeName.get(ClassName.get(SdkField.class),
176+
WildcardTypeName.subtypeOf(ClassName.get(Object.class)));
177+
return MethodSpec.methodBuilder("sdkFieldNameToField")
178+
.addModifiers(PUBLIC)
179+
.addAnnotation(Override.class)
180+
.returns(ParameterizedTypeName.get(ClassName.get(Map.class), ClassName.get(String.class), sdkFieldType))
181+
.addCode("return SDK_NAME_TO_FIELD;")
182+
.build();
183+
}
184+
172185
private TypeName builderImplSuperClass() {
173186
if (isRequest()) {
174187
return new AwsServiceBaseRequestSpec(intermediateModel).className().nestedClass("BuilderImpl");

‎codegen/src/main/java/software/amazon/awssdk/codegen/poet/model/ShapeModelSpec.java‎

Lines changed: 43 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,10 @@
2323
import java.util.ArrayList;
2424
import java.util.Arrays;
2525
import java.util.Collections;
26+
import java.util.HashMap;
27+
import java.util.LinkedHashMap;
2628
import java.util.List;
29+
import java.util.Map;
2730
import java.util.Optional;
2831
import java.util.stream.Collectors;
2932
import javax.lang.model.element.Modifier;
@@ -95,6 +98,7 @@ public List<FieldSpec> fields(Modifier... modifiers) {
9598

9699
public Iterable<FieldSpec> staticFields(Modifier... modifiers) {
97100
List<FieldSpec> fields = new ArrayList<>();
101+
Map<String, String> nameToField = new LinkedHashMap<>();
98102
shapeModel.getNonStreamingMembers().stream()
99103
// Exceptions can be members of event stream shapes, need to filter those out of the models
100104
.filter(m -> m.getShape() == null || m.getShape().getShapeType() != ShapeType.Exception)
@@ -107,6 +111,8 @@ public Iterable<FieldSpec> staticFields(Modifier... modifiers) {
107111
Modifier.PRIVATE, Modifier.STATIC, Modifier.FINAL)
108112
.initializer(sdkFieldInitializer(m))
109113
.build());
114+
String name = m.getHttp().getMarshallLocationName();
115+
nameToField.put(name, namingStrategy.getSdkFieldFieldName(m));
110116
});
111117

112118
ParameterizedTypeName sdkFieldType = ParameterizedTypeName.get(ClassName.get(SdkField.class),
@@ -115,13 +121,15 @@ public Iterable<FieldSpec> staticFields(Modifier... modifiers) {
115121
fields.add(FieldSpec.builder(ParameterizedTypeName.get(ClassName.get(List.class),
116122
sdkFieldType), "SDK_FIELDS",
117123
Modifier.PRIVATE, Modifier.STATIC, Modifier.FINAL)
118-
.initializer("$T.unmodifiableList($T.asList($L))",
119-
ClassName.get(Collections.class),
120-
ClassName.get(Arrays.class),
121-
fields.stream()
122-
.map(f -> f.name)
123-
.collect(Collectors.joining(",")))
124+
.initializer(sdkFieldsInitializer(fields))
124125
.build());
126+
fields.add(FieldSpec.builder(ParameterizedTypeName.get(ClassName.get(Map.class),
127+
ClassName.get(String.class),
128+
sdkFieldType),
129+
"SDK_NAME_TO_FIELD",
130+
Modifier.PRIVATE, Modifier.STATIC, Modifier.FINAL)
131+
.initializer(memberNameToFieldInitializer(nameToField))
132+
.build());
125133
return fields;
126134
}
127135

@@ -369,4 +377,33 @@ private CodeBlock constructor(MemberModel m) {
369377
}
370378
}
371379

380+
private CodeBlock sdkFieldsInitializer(List<FieldSpec> fields) {
381+
CodeBlock.Builder builder = CodeBlock.builder();
382+
if (fields.isEmpty()) {
383+
builder.add("$T.emptyList()", Collections.class);
384+
return builder.build();
385+
}
386+
builder.add("$T.unmodifiableList($T.asList($L))",
387+
ClassName.get(Collections.class),
388+
ClassName.get(Arrays.class),
389+
fields.stream()
390+
.map(f -> f.name)
391+
.collect(Collectors.joining(",")));
392+
return builder.build();
393+
}
394+
395+
private CodeBlock memberNameToFieldInitializer(Map<String, String> nameToField) {
396+
CodeBlock.Builder builder = CodeBlock.builder();
397+
if (nameToField.isEmpty()) {
398+
builder.add("$T.emptyMap()", Collections.class);
399+
return builder.build();
400+
}
401+
builder.add("$T.unmodifiableMap(", Collections.class);
402+
builder.add("new $T<$T, $T<?>>() {{\n", HashMap.class, String.class, SdkField.class);
403+
nameToField.forEach((name, field) -> builder.add("put($S, $L);\n", name, field));
404+
builder.add("}}");
405+
builder.add(")");
406+
return builder.build();
407+
}
408+
372409
}

‎codegen/src/test/resources/software/amazon/awssdk/codegen/poet/model/alltypesrequest.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import java.util.Arrays;
66
import java.util.Collection;
77
import java.util.Collections;
8+
import java.util.HashMap;
89
import java.util.List;
910
import java.util.Map;
1011
import java.util.Objects;
@@ -450,6 +451,48 @@ SdkField.<SdkBytes> builder(MarshallingType.SDK_BYTES)
450451
POLYMORPHIC_TYPE_WITHOUT_SUB_TYPES_FIELD, ENUM_TYPE_FIELD, UNDERSCORE_NAME_TYPE_FIELD, MY_DOCUMENT_FIELD,
451452
ALL_TYPES_UNION_STRUCTURE_FIELD));
452453

454+
private static final Map<String, SdkField<?>> SDK_NAME_TO_FIELD = Collections
455+
.unmodifiableMap(new HashMap<String, SdkField<?>>() {
456+
{
457+
put("StringMember", STRING_MEMBER_FIELD);
458+
put("IntegerMember", INTEGER_MEMBER_FIELD);
459+
put("BooleanMember", BOOLEAN_MEMBER_FIELD);
460+
put("FloatMember", FLOAT_MEMBER_FIELD);
461+
put("DoubleMember", DOUBLE_MEMBER_FIELD);
462+
put("LongMember", LONG_MEMBER_FIELD);
463+
put("ShortMember", SHORT_MEMBER_FIELD);
464+
put("ByteMember", BYTE_MEMBER_FIELD);
465+
put("SimpleList", SIMPLE_LIST_FIELD);
466+
put("ListOfEnums", LIST_OF_ENUMS_FIELD);
467+
put("ListOfMaps", LIST_OF_MAPS_FIELD);
468+
put("ListOfStructs", LIST_OF_STRUCTS_FIELD);
469+
put("ListOfMapOfEnumToString", LIST_OF_MAP_OF_ENUM_TO_STRING_FIELD);
470+
put("ListOfMapOfStringToStruct", LIST_OF_MAP_OF_STRING_TO_STRUCT_FIELD);
471+
put("MapOfStringToIntegerList", MAP_OF_STRING_TO_INTEGER_LIST_FIELD);
472+
put("MapOfStringToString", MAP_OF_STRING_TO_STRING_FIELD);
473+
put("MapOfStringToSimpleStruct", MAP_OF_STRING_TO_SIMPLE_STRUCT_FIELD);
474+
put("MapOfEnumToEnum", MAP_OF_ENUM_TO_ENUM_FIELD);
475+
put("MapOfEnumToString", MAP_OF_ENUM_TO_STRING_FIELD);
476+
put("MapOfStringToEnum", MAP_OF_STRING_TO_ENUM_FIELD);
477+
put("MapOfEnumToSimpleStruct", MAP_OF_ENUM_TO_SIMPLE_STRUCT_FIELD);
478+
put("MapOfEnumToListOfEnums", MAP_OF_ENUM_TO_LIST_OF_ENUMS_FIELD);
479+
put("MapOfEnumToMapOfStringToEnum", MAP_OF_ENUM_TO_MAP_OF_STRING_TO_ENUM_FIELD);
480+
put("TimestampMember", TIMESTAMP_MEMBER_FIELD);
481+
put("StructWithNestedTimestampMember", STRUCT_WITH_NESTED_TIMESTAMP_MEMBER_FIELD);
482+
put("BlobArg", BLOB_ARG_FIELD);
483+
put("StructWithNestedBlob", STRUCT_WITH_NESTED_BLOB_FIELD);
484+
put("BlobMap", BLOB_MAP_FIELD);
485+
put("ListOfBlobs", LIST_OF_BLOBS_FIELD);
486+
put("RecursiveStruct", RECURSIVE_STRUCT_FIELD);
487+
put("PolymorphicTypeWithSubTypes", POLYMORPHIC_TYPE_WITH_SUB_TYPES_FIELD);
488+
put("PolymorphicTypeWithoutSubTypes", POLYMORPHIC_TYPE_WITHOUT_SUB_TYPES_FIELD);
489+
put("EnumType", ENUM_TYPE_FIELD);
490+
put("Underscore_Name_Type", UNDERSCORE_NAME_TYPE_FIELD);
491+
put("MyDocument", MY_DOCUMENT_FIELD);
492+
put("AllTypesUnionStructure", ALL_TYPES_UNION_STRUCTURE_FIELD);
493+
}
494+
});
495+
453496
private final String stringMember;
454497

455498
private final Integer integerMember;
@@ -1604,6 +1647,11 @@ public final List<SdkField<?>> sdkFields() {
16041647
return SDK_FIELDS;
16051648
}
16061649

1650+
@Override
1651+
public final Map<String, SdkField<?>> sdkFieldNameToField() {
1652+
return SDK_NAME_TO_FIELD;
1653+
}
1654+
16071655
private static <T> Function<Object, T> getter(Function<AllTypesRequest, T> g) {
16081656
return obj -> g.apply((AllTypesRequest) obj);
16091657
}
@@ -3065,5 +3113,10 @@ public AllTypesRequest build() {
30653113
public List<SdkField<?>> sdkFields() {
30663114
return SDK_FIELDS;
30673115
}
3116+
3117+
@Override
3118+
public Map<String, SdkField<?>> sdkFieldNameToField() {
3119+
return SDK_NAME_TO_FIELD;
3120+
}
30683121
}
30693122
}

‎codegen/src/test/resources/software/amazon/awssdk/codegen/poet/model/alltypesresponse.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import java.util.Arrays;
66
import java.util.Collection;
77
import java.util.Collections;
8+
import java.util.HashMap;
89
import java.util.List;
910
import java.util.Map;
1011
import java.util.Objects;
@@ -449,6 +450,48 @@ SdkField.<SdkBytes> builder(MarshallingType.SDK_BYTES)
449450
POLYMORPHIC_TYPE_WITHOUT_SUB_TYPES_FIELD, ENUM_TYPE_FIELD, UNDERSCORE_NAME_TYPE_FIELD, MY_DOCUMENT_FIELD,
450451
ALL_TYPES_UNION_STRUCTURE_FIELD));
451452

453+
private static final Map<String, SdkField<?>> SDK_NAME_TO_FIELD = Collections
454+
.unmodifiableMap(new HashMap<String, SdkField<?>>() {
455+
{
456+
put("StringMember", STRING_MEMBER_FIELD);
457+
put("IntegerMember", INTEGER_MEMBER_FIELD);
458+
put("BooleanMember", BOOLEAN_MEMBER_FIELD);
459+
put("FloatMember", FLOAT_MEMBER_FIELD);
460+
put("DoubleMember", DOUBLE_MEMBER_FIELD);
461+
put("LongMember", LONG_MEMBER_FIELD);
462+
put("ShortMember", SHORT_MEMBER_FIELD);
463+
put("ByteMember", BYTE_MEMBER_FIELD);
464+
put("SimpleList", SIMPLE_LIST_FIELD);
465+
put("ListOfEnums", LIST_OF_ENUMS_FIELD);
466+
put("ListOfMaps", LIST_OF_MAPS_FIELD);
467+
put("ListOfStructs", LIST_OF_STRUCTS_FIELD);
468+
put("ListOfMapOfEnumToString", LIST_OF_MAP_OF_ENUM_TO_STRING_FIELD);
469+
put("ListOfMapOfStringToStruct", LIST_OF_MAP_OF_STRING_TO_STRUCT_FIELD);
470+
put("MapOfStringToIntegerList", MAP_OF_STRING_TO_INTEGER_LIST_FIELD);
471+
put("MapOfStringToString", MAP_OF_STRING_TO_STRING_FIELD);
472+
put("MapOfStringToSimpleStruct", MAP_OF_STRING_TO_SIMPLE_STRUCT_FIELD);
473+
put("MapOfEnumToEnum", MAP_OF_ENUM_TO_ENUM_FIELD);
474+
put("MapOfEnumToString", MAP_OF_ENUM_TO_STRING_FIELD);
475+
put("MapOfStringToEnum", MAP_OF_STRING_TO_ENUM_FIELD);
476+
put("MapOfEnumToSimpleStruct", MAP_OF_ENUM_TO_SIMPLE_STRUCT_FIELD);
477+
put("MapOfEnumToListOfEnums", MAP_OF_ENUM_TO_LIST_OF_ENUMS_FIELD);
478+
put("MapOfEnumToMapOfStringToEnum", MAP_OF_ENUM_TO_MAP_OF_STRING_TO_ENUM_FIELD);
479+
put("TimestampMember", TIMESTAMP_MEMBER_FIELD);
480+
put("StructWithNestedTimestampMember", STRUCT_WITH_NESTED_TIMESTAMP_MEMBER_FIELD);
481+
put("BlobArg", BLOB_ARG_FIELD);
482+
put("StructWithNestedBlob", STRUCT_WITH_NESTED_BLOB_FIELD);
483+
put("BlobMap", BLOB_MAP_FIELD);
484+
put("ListOfBlobs", LIST_OF_BLOBS_FIELD);
485+
put("RecursiveStruct", RECURSIVE_STRUCT_FIELD);
486+
put("PolymorphicTypeWithSubTypes", POLYMORPHIC_TYPE_WITH_SUB_TYPES_FIELD);
487+
put("PolymorphicTypeWithoutSubTypes", POLYMORPHIC_TYPE_WITHOUT_SUB_TYPES_FIELD);
488+
put("EnumType", ENUM_TYPE_FIELD);
489+
put("Underscore_Name_Type", UNDERSCORE_NAME_TYPE_FIELD);
490+
put("MyDocument", MY_DOCUMENT_FIELD);
491+
put("AllTypesUnionStructure", ALL_TYPES_UNION_STRUCTURE_FIELD);
492+
}
493+
});
494+
452495
private final String stringMember;
453496

454497
private final Integer integerMember;
@@ -1603,6 +1646,11 @@ public final List<SdkField<?>> sdkFields() {
16031646
return SDK_FIELDS;
16041647
}
16051648

1649+
@Override
1650+
public final Map<String, SdkField<?>> sdkFieldNameToField() {
1651+
return SDK_NAME_TO_FIELD;
1652+
}
1653+
16061654
private static <T> Function<Object, T> getter(Function<AllTypesResponse, T> g) {
16071655
return obj -> g.apply((AllTypesResponse) obj);
16081656
}
@@ -3046,5 +3094,10 @@ public AllTypesResponse build() {
30463094
public List<SdkField<?>> sdkFields() {
30473095
return SDK_FIELDS;
30483096
}
3097+
3098+
@Override
3099+
public Map<String, SdkField<?>> sdkFieldNameToField() {
3100+
return SDK_NAME_TO_FIELD;
3101+
}
30493102
}
30503103
}

0 commit comments

Comments
 (0)