Skip to content

Commit c3c9a7c

Browse files
author
Grace Calianese
committed
Merge branch 'RLPNC-7502-partial-record-failingv2' into RLPNC-7547-error-list
# Conflicts: # json/src/main/java/com/basistech/rosette/apimodel/jackson/recordsimilaritydeserializers/RecordSimilarityDeserializerUtilities.java # json/src/test/java/com/basistech/rosette/apimodel/recordsimilarity/RecordSimilarityResponseTest.java # model/src/main/java/com/basistech/rosette/apimodel/recordsimilarity/RecordSimilarityResult.java
2 parents a9d8be7 + 8787659 commit c3c9a7c

11 files changed

Lines changed: 194 additions & 26 deletions

File tree

‎examples/src/main/java/com/basistech/rosette/examples/RecordSimilarityExample.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ private void run() throws IOException {
9090
primaryNameField, NameField.UnfieldedName.builder().text("Ivan R").build(),
9191
dobField, DateField.FieldedDate.builder().date(dobHyphen).build(),
9292
dob2Field, DateField.FieldedDate.builder().date("1993/04/16").build(),
93-
addrField, AddressField.FieldedAddress.builder().address("123 Roadlane Ave").build()
93+
addrField, AddressField.FieldedAddress.builder().houseNumber("123").road("Roadlane Ave").build()
9494
)
9595
)
9696
).build()

‎json/src/main/java/com/basistech/rosette/apimodel/jackson/recordsimilaritydeserializers/RecordSimilarityDeserializerUtilities.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@
2222
import java.util.Iterator;
2323
import java.util.List;
2424
import java.util.Map;
25+
import java.util.Optional;
26+
import java.util.stream.Collectors;
27+
import java.util.stream.StreamSupport;
2528

2629
import com.basistech.rosette.apimodel.recordsimilarity.RecordSimilarityExplainInfo;
2730
import com.basistech.rosette.apimodel.recordsimilarity.RecordSimilarityResult;
@@ -69,12 +72,18 @@ public static RecordSimilarityResult parseResult(
6972
errorList.add(element.asText());
7073
}
7174
}
75+
List<String> info = Optional.ofNullable(node.get("info"))
76+
.map(jsonNode -> StreamSupport.stream(jsonNode.spliterator(), false)
77+
.map(JsonNode::asText)
78+
.collect(Collectors.toList()))
79+
.orElse(null);
7280
return RecordSimilarityResult.builder()
7381
.score(score)
7482
.left(left)
7583
.right(right)
7684
.explainInfo(explainInfo)
7785
.error(errorList)
86+
.info(info)
7887
.build();
7988
}
8089

‎json/src/main/java/com/basistech/rosette/apimodel/jackson/recordsimilaritydeserializers/RecordSimilarityResponseDeserializer.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@
3030
import java.util.List;
3131
import java.util.Map;
3232
import java.util.Optional;
33+
import java.util.stream.Collectors;
34+
import java.util.stream.StreamSupport;
3335

3436
public class RecordSimilarityResponseDeserializer extends StdDeserializer<RecordSimilarityResponse> {
3537

@@ -47,6 +49,11 @@ public RecordSimilarityResponse deserialize(JsonParser jsonParser, Deserializati
4749
JsonNode fieldsNode = node.get("fields");
4850

4951
Map<String, RecordSimilarityFieldInfo> fields = fieldsNode != null ? node.get("fields").traverse(jsonParser.getCodec()).readValueAs(FIELDS_TYPE_REFERENCE) : null;
52+
List<String> info = Optional.ofNullable(node.get("info"))
53+
.map(jsonNode -> StreamSupport.stream(jsonNode.spliterator(), false)
54+
.map(JsonNode::asText)
55+
.collect(Collectors.toList()))
56+
.orElse(null);
5057
String errorMessage = Optional.ofNullable(node.get("errorMessage")).map(JsonNode::asText).orElse(null);
5158

5259
JsonNode resultsNode = node.get("results");
@@ -60,6 +67,7 @@ public RecordSimilarityResponse deserialize(JsonParser jsonParser, Deserializati
6067
return RecordSimilarityResponse.builder()
6168
.fields(fields)
6269
.results(results)
70+
.info(info)
6371
.errorMessage(errorMessage)
6472
.build();
6573
}

‎json/src/test/java/com/basistech/rosette/apimodel/RecordSimilarityRequestTest.java‎

Lines changed: 120 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,8 @@
2929
import com.basistech.util.LanguageCode;
3030
import com.fasterxml.jackson.core.JsonProcessingException;
3131
import com.fasterxml.jackson.core.type.TypeReference;
32-
import com.fasterxml.jackson.databind.MapperFeature;
32+
import com.fasterxml.jackson.databind.JsonNode;
3333
import com.fasterxml.jackson.databind.ObjectMapper;
34-
import com.fasterxml.jackson.databind.SerializationFeature;
3534
import org.junit.jupiter.api.Test;
3635

3736
import java.util.List;
@@ -42,15 +41,114 @@
4241
class RecordSimilarityRequestTest {
4342

4443
private static final ObjectMapper MAPPER = ApiModelMixinModule.setupObjectMapper(new ObjectMapper());
44+
private static final String EXPECTED_JSON = "{\"fields\":{\"dob2\":{\"type\":\"rni_date\",\"weight\":0.1},\"primaryName\":{\"type\":\"rni_name\",\"weight\":0.5},\"dob\":{\"type\":\"rni_date\",\"weight\":0.2},\"addr\":{\"type\":\"rni_address\",\"weight\":0.5,\"scoreIfNull\":0.8}},\"properties\":{\"threshold\":0.7,\"includeExplainInfo\":true},\"records\":{\"left\":[{\"dob2\":{\"date\":\"1993/04/16\"},\"primaryName\":{\"text\":\"Ethan R\",\"entityType\":\"PERSON\",\"language\":\"eng\",\"languageOfOrigin\":\"eng\",\"script\":\"Latn\"},\"dob\":\"1993-04-16\",\"addr\":\"123 Roadlane Ave\"},{\"primaryName\":{\"text\":\"Evan R\"},\"dob\":{\"date\":\"1993-04-16\"}}],\"right\":[{\"primaryName\":{\"text\":\"Seth R\",\"language\":\"eng\"},\"dob\":{\"date\":\"1993-04-16\"}},{\"dob2\":{\"date\":\"1993/04/16\"},\"primaryName\":\"Ivan R\",\"dob\":{\"date\":\"1993-04-16\"},\"addr\":{\"houseNumber\":\"123\",\"road\":\"Roadlane Ave\"}}]}}";
45+
private static final String EXPECTED_JSON_WITH_PARAMS = "{\"fields\":{\"dob2\":{\"type\":\"rni_date\",\"weight\":0.1},\"primaryName\":{\"type\":\"rni_name\",\"weight\":0.5},\"dob\":{\"type\":\"rni_date\",\"weight\":0.2},\"addr\":{\"type\":\"rni_address\",\"weight\":0.5}},\"properties\":{\"threshold\":0.7,\"includeExplainInfo\":true,\"parameters\":{\"timeDistanceWeight\":\"0.8\",\"stringDistanceWeight\":\"0.1\"}},\"records\":{\"left\":[{\"dob2\":{\"date\":\"1993/04/16\"},\"primaryName\":{\"text\":\"Ethan R\",\"entityType\":\"PERSON\",\"language\":\"eng\",\"languageOfOrigin\":\"eng\",\"script\":\"Latn\"},\"dob\":\"1993-04-16\",\"addr\":\"123 Roadlane Ave\"},{\"primaryName\":{\"text\":\"Evan R\"},\"dob\":{\"date\":\"1993-04-16\"}}],\"right\":[{\"primaryName\":{\"text\":\"Seth R\",\"language\":\"eng\"},\"dob\":{\"date\":\"1993-04-16\"}},{\"dob2\":{\"date\":\"1993/04/16\"},\"primaryName\":\"Ivan R\",\"dob\":{\"date\":\"1993-04-16\"},\"addr\":{\"houseNumber\":\"123\",\"road\":\"Roadlane Ave\"}}]}}";
46+
private static final String EXPECTED_JSON_WITH_UNIVERSE = "{\"fields\":{\"dob2\":{\"type\":\"rni_date\",\"weight\":0.1},\"primaryName\":{\"type\":\"rni_name\",\"weight\":0.5},\"dob\":{\"type\":\"rni_date\",\"weight\":0.2},\"addr\":{\"type\":\"rni_address\",\"weight\":0.5}},\"properties\":{\"threshold\":0.7,\"includeExplainInfo\":true,\"parameterUniverse\":\"myParameterUniverse\"},\"records\":{\"left\":[{\"dob2\":{\"date\":\"1993/04/16\"},\"primaryName\":{\"text\":\"Ethan R\",\"entityType\":\"PERSON\",\"language\":\"eng\",\"languageOfOrigin\":\"eng\",\"script\":\"Latn\"},\"dob\":\"1993-04-16\",\"addr\":\"123 Roadlane Ave\"},{\"primaryName\":{\"text\":\"Evan R\"},\"dob\":{\"date\":\"1993-04-16\"}}],\"right\":[{\"primaryName\":{\"text\":\"Seth R\",\"language\":\"eng\"},\"dob\":{\"date\":\"1993-04-16\"}},{\"dob2\":{\"date\":\"1993/04/16\"},\"primaryName\":\"Ivan R\",\"dob\":{\"date\":\"1993-04-16\"},\"addr\":{\"houseNumber\":\"123\",\"road\":\"Roadlane Ave\"}}]}}";
4547

46-
private static final String EXPECTED_JSON = "{\"fields\":{\"addr\":{\"type\":\"rni_address\",\"weight\":0.5},\"dob\":{\"type\":\"rni_date\",\"weight\":0.2},\"dob2\":{\"type\":\"rni_date\",\"weight\":0.1},\"primaryName\":{\"type\":\"rni_name\",\"weight\":0.5}},\"properties\":{\"includeExplainInfo\":true,\"threshold\":0.7},\"records\":{\"left\":[{\"addr\":\"123 Roadlane Ave\",\"dob\":\"1993-04-16\",\"dob2\":{\"date\":\"1993/04/16\"},\"primaryName\":{\"entityType\":\"PERSON\",\"language\":\"eng\",\"languageOfOrigin\":\"eng\",\"script\":\"Latn\",\"text\":\"Ethan R\"}},{\"dob\":{\"date\":\"1993-04-16\"},\"primaryName\":{\"text\":\"Evan R\"}}],\"right\":[{\"dob\":{\"date\":\"1993-04-16\"},\"primaryName\":{\"language\":\"eng\",\"text\":\"Seth R\"}},{\"addr\":{\"address\":\"123 Roadlane Ave\"},\"dob\":{\"date\":\"1993-04-16\"},\"dob2\":{\"date\":\"1993/04/16\"},\"primaryName\":\"Ivan R\"}]}}";
4748
private static final RecordSimilarityRequest EXPECTED_REQUEST = RecordSimilarityRequest.builder()
4849
.fields(Map.of(
50+
"addr", RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_ADDRESS).weight(0.5).scoreIfNull(0.8).build(),
51+
"dob2", RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_DATE).weight(0.1).scoreIfNull(null).build(),
52+
"primaryName", RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_NAME).weight(0.5).build(),
53+
"dob", RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_DATE).weight(0.2).build()))
54+
.properties(RecordSimilarityProperties.builder().threshold(0.7).includeExplainInfo(true).build())
55+
.records(RecordSimilarityRecords.builder()
56+
.left(
57+
List.of(
58+
Map.of(
59+
"primaryName", NameField.FieldedName.builder()
60+
.text("Ethan R").entityType("PERSON")
61+
.language(LanguageCode.ENGLISH)
62+
.languageOfOrigin(LanguageCode.ENGLISH)
63+
.script(ISO15924.Latn)
64+
.build(),
65+
"dob", DateField.UnfieldedDate.builder().date("1993-04-16").build(),
66+
"dob2", DateField.FieldedDate.builder().date("1993/04/16").build(),
67+
"addr", AddressField.UnfieldedAddress.builder().address("123 Roadlane Ave").build()
68+
),
69+
Map.of(
70+
"primaryName", NameField.FieldedName.builder().text("Evan R").build(),
71+
"dob", DateField.FieldedDate.builder().date("1993-04-16").build()
72+
)
73+
)
74+
).right(
75+
List.of(
76+
Map.of(
77+
"primaryName", NameField.FieldedName.builder().text("Seth R").language(LanguageCode.ENGLISH).build(),
78+
"dob", DateField.FieldedDate.builder().date("1993-04-16").build()
79+
),
80+
Map.of(
81+
"primaryName", NameField.UnfieldedName.builder().text("Ivan R").build(),
82+
"dob", DateField.FieldedDate.builder().date("1993-04-16").build(),
83+
"dob2", DateField.FieldedDate.builder().date("1993/04/16").build(),
84+
"addr", AddressField.FieldedAddress.builder().houseNumber("123").road("Roadlane Ave").build()
85+
)
86+
)
87+
).build()
88+
).build();
89+
90+
private static final RecordSimilarityRequest EXPECTED_REQUEST_WITH_PARAMS = RecordSimilarityRequest.builder()
91+
.fields(Map.of(
92+
"dob2", RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_DATE).weight(0.1).build(),
4993
"primaryName", RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_NAME).weight(0.5).build(),
5094
"dob", RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_DATE).weight(0.2).build(),
95+
"addr", RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_ADDRESS).weight(0.5).build()))
96+
.properties(RecordSimilarityProperties.builder()
97+
.threshold(0.7)
98+
.includeExplainInfo(true)
99+
.parameters(
100+
Map.of(
101+
"timeDistanceWeight", "0.8",
102+
"stringDistanceWeight", "0.1"
103+
)
104+
)
105+
.build())
106+
.records(RecordSimilarityRecords.builder()
107+
.left(
108+
List.of(
109+
Map.of(
110+
"primaryName", NameField.FieldedName.builder()
111+
.text("Ethan R").entityType("PERSON")
112+
.language(LanguageCode.ENGLISH)
113+
.languageOfOrigin(LanguageCode.ENGLISH)
114+
.script(ISO15924.Latn)
115+
.build(),
116+
"dob", DateField.UnfieldedDate.builder().date("1993-04-16").build(),
117+
"dob2", DateField.FieldedDate.builder().date("1993/04/16").build(),
118+
"addr", AddressField.UnfieldedAddress.builder().address("123 Roadlane Ave").build()
119+
),
120+
Map.of(
121+
"primaryName", NameField.FieldedName.builder().text("Evan R").build(),
122+
"dob", DateField.FieldedDate.builder().date("1993-04-16").build()
123+
)
124+
)
125+
).right(
126+
List.of(
127+
Map.of(
128+
"primaryName", NameField.FieldedName.builder().text("Seth R").language(LanguageCode.ENGLISH).build(),
129+
"dob", DateField.FieldedDate.builder().date("1993-04-16").build()
130+
),
131+
Map.of(
132+
"primaryName", NameField.UnfieldedName.builder().text("Ivan R").build(),
133+
"dob", DateField.FieldedDate.builder().date("1993-04-16").build(),
134+
"dob2", DateField.FieldedDate.builder().date("1993/04/16").build(),
135+
"addr", AddressField.FieldedAddress.builder().houseNumber("123").road("Roadlane Ave").build()
136+
)
137+
)
138+
).build()
139+
).build();
140+
141+
private static final RecordSimilarityRequest EXPECTED_REQUEST_WITH_UNIVERSE = RecordSimilarityRequest.builder()
142+
.fields(Map.of(
143+
"dob", RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_DATE).weight(0.2).build(),
144+
"primaryName", RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_NAME).weight(0.5).build(),
51145
"dob2", RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_DATE).weight(0.1).build(),
52146
"addr", RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_ADDRESS).weight(0.5).build()))
53-
.properties(RecordSimilarityProperties.builder().threshold(0.7).includeExplainInfo(true).build())
147+
.properties(RecordSimilarityProperties.builder()
148+
.threshold(0.7)
149+
.includeExplainInfo(true)
150+
.parameterUniverse("myParameterUniverse")
151+
.build())
54152
.records(RecordSimilarityRecords.builder()
55153
.left(
56154
List.of(
@@ -80,26 +178,37 @@ class RecordSimilarityRequestTest {
80178
"primaryName", NameField.UnfieldedName.builder().text("Ivan R").build(),
81179
"dob", DateField.FieldedDate.builder().date("1993-04-16").build(),
82180
"dob2", DateField.FieldedDate.builder().date("1993/04/16").build(),
83-
"addr", AddressField.FieldedAddress.builder().address("123 Roadlane Ave").build()
181+
"addr", AddressField.FieldedAddress.builder().houseNumber("123").road("Roadlane Ave").build()
84182
)
85183
)
86184
).build()
87185
).build();
88186

89187
@Test
90188
void testDeserialization() throws JsonProcessingException {
91-
MAPPER.enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY);
92-
MAPPER.enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS);
93189
final RecordSimilarityRequest request = MAPPER.readValue(EXPECTED_JSON, new TypeReference<>() { });
94190
assertEquals(EXPECTED_REQUEST, request);
191+
192+
final RecordSimilarityRequest requestParams = MAPPER.readValue(EXPECTED_JSON_WITH_PARAMS, new TypeReference<>() { });
193+
assertEquals(EXPECTED_REQUEST_WITH_PARAMS, requestParams);
194+
195+
final RecordSimilarityRequest requestUniverse = MAPPER.readValue(EXPECTED_JSON_WITH_UNIVERSE, new TypeReference<>() { });
196+
assertEquals(EXPECTED_REQUEST_WITH_UNIVERSE, requestUniverse);
95197
}
96198

97199
@Test
98200
void testSerialization() throws JsonProcessingException {
99-
// For testing, force ordering
100-
MAPPER.enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY);
101-
MAPPER.enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS);
102-
assertEquals(EXPECTED_JSON, MAPPER.writeValueAsString(EXPECTED_REQUEST));
201+
final JsonNode expectedJson = MAPPER.readTree(EXPECTED_JSON);
202+
final JsonNode actualJson = MAPPER.valueToTree(EXPECTED_REQUEST);
203+
assertEquals(expectedJson, actualJson);
204+
205+
final JsonNode expectedParamJson = MAPPER.readTree(EXPECTED_JSON_WITH_PARAMS);
206+
final JsonNode actualParamJson = MAPPER.valueToTree(EXPECTED_REQUEST_WITH_PARAMS);
207+
assertEquals(expectedParamJson, actualParamJson);
208+
209+
final JsonNode expectedUniverseJson = MAPPER.readTree(EXPECTED_JSON_WITH_UNIVERSE);
210+
final JsonNode actualUniverseJson = MAPPER.valueToTree(EXPECTED_REQUEST_WITH_UNIVERSE);
211+
assertEquals(expectedUniverseJson, actualUniverseJson);
103212
}
104213

105214
}

0 commit comments

Comments
 (0)