Skip to content

Commit e8b29d2

Browse files
author
Grace Calianese
committed
Merge branch 'master' into RLPNC-7502-partial-record-failingv2
# Conflicts: # json/src/test/java/com/basistech/rosette/apimodel/RecordSimilarityRequestTest.java # model/src/main/java/com/basistech/rosette/apimodel/recordsimilarity/RecordSimilarityFieldInfo.java
2 parents 2b197b7 + 4f5181b commit e8b29d2

11 files changed

Lines changed: 189 additions & 17 deletions

File tree

‎examples/src/main/java/com/basistech/rosette/examples/RecordSimilarityExample.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ private void run() throws IOException {
9090
primaryNameField, NameField.UnfieldedName.builder().text("Ivan R").build(),
9191
dobField, DateField.FieldedDate.builder().date(dobHyphen).build(),
9292
dob2Field, DateField.FieldedDate.builder().date("1993/04/16").build(),
93-
addrField, AddressField.FieldedAddress.builder().address("123 Roadlane Ave").build()
93+
addrField, AddressField.FieldedAddress.builder().houseNumber("123").road("Roadlane Ave").build()
9494
)
9595
)
9696
).build()

‎json/src/main/java/com/basistech/rosette/apimodel/jackson/recordsimilaritydeserializers/RecordSimilarityDeserializerUtilities.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,11 @@
1919
import java.io.IOException;
2020
import java.util.HashMap;
2121
import java.util.Iterator;
22+
import java.util.List;
2223
import java.util.Map;
2324
import java.util.Optional;
25+
import java.util.stream.Collectors;
26+
import java.util.stream.StreamSupport;
2427

2528
import com.basistech.rosette.apimodel.recordsimilarity.RecordSimilarityExplainInfo;
2629
import com.basistech.rosette.apimodel.recordsimilarity.RecordSimilarityResult;
@@ -61,12 +64,18 @@ public static RecordSimilarityResult parseResult(
6164
? parseRecord(node.get("right"), jsonParser, fields)
6265
: null;
6366
final String error = Optional.ofNullable(node.get("error")).map(JsonNode::asText).orElse(null);
67+
List<String> info = Optional.ofNullable(node.get("info"))
68+
.map(jsonNode -> StreamSupport.stream(jsonNode.spliterator(), false)
69+
.map(JsonNode::asText)
70+
.collect(Collectors.toList()))
71+
.orElse(null);
6472
return RecordSimilarityResult.builder()
6573
.score(score)
6674
.left(left)
6775
.right(right)
6876
.explainInfo(explainInfo)
6977
.error(error)
78+
.info(info)
7079
.build();
7180
}
7281

‎json/src/main/java/com/basistech/rosette/apimodel/jackson/recordsimilaritydeserializers/RecordSimilarityResponseDeserializer.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@
3030
import java.util.List;
3131
import java.util.Map;
3232
import java.util.Optional;
33+
import java.util.stream.Collectors;
34+
import java.util.stream.StreamSupport;
3335

3436
public class RecordSimilarityResponseDeserializer extends StdDeserializer<RecordSimilarityResponse> {
3537

@@ -47,6 +49,11 @@ public RecordSimilarityResponse deserialize(JsonParser jsonParser, Deserializati
4749
JsonNode fieldsNode = node.get("fields");
4850

4951
Map<String, RecordSimilarityFieldInfo> fields = fieldsNode != null ? node.get("fields").traverse(jsonParser.getCodec()).readValueAs(FIELDS_TYPE_REFERENCE) : null;
52+
List<String> info = Optional.ofNullable(node.get("info"))
53+
.map(jsonNode -> StreamSupport.stream(jsonNode.spliterator(), false)
54+
.map(JsonNode::asText)
55+
.collect(Collectors.toList()))
56+
.orElse(null);
5057
String errorMessage = Optional.ofNullable(node.get("errorMessage")).map(JsonNode::asText).orElse(null);
5158

5259
JsonNode resultsNode = node.get("results");
@@ -60,6 +67,7 @@ public RecordSimilarityResponse deserialize(JsonParser jsonParser, Deserializati
6067
return RecordSimilarityResponse.builder()
6168
.fields(fields)
6269
.results(results)
70+
.info(info)
6371
.errorMessage(errorMessage)
6472
.build();
6573
}

‎json/src/test/java/com/basistech/rosette/apimodel/RecordSimilarityRequestTest.java‎

Lines changed: 120 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,7 @@
2929
import com.basistech.util.LanguageCode;
3030
import com.fasterxml.jackson.core.JsonProcessingException;
3131
import com.fasterxml.jackson.core.type.TypeReference;
32-
import com.fasterxml.jackson.databind.MapperFeature;
3332
import com.fasterxml.jackson.databind.ObjectMapper;
34-
import com.fasterxml.jackson.databind.SerializationFeature;
3533
import org.junit.jupiter.api.Test;
3634

3735
import java.util.List;
@@ -42,15 +40,115 @@
4240
class RecordSimilarityRequestTest {
4341

4442
private static final ObjectMapper MAPPER = ApiModelMixinModule.setupObjectMapper(new ObjectMapper());
43+
private static final String EXPECTED_JSON = "{\"fields\":{\"dob2\":{\"type\":\"rni_date\",\"weight\":0.1},\"primaryName\":{\"type\":\"rni_name\",\"weight\":0.5},\"dob\":{\"type\":\"rni_date\",\"weight\":0.2},\"addr\":{\"type\":\"rni_address\",\"weight\":0.5,\"scoreIfNull\":0.8}},\"properties\":{\"threshold\":0.7,\"includeExplainInfo\":true},\"records\":{\"left\":[{\"dob2\":{\"date\":\"1993/04/16\"},\"primaryName\":{\"text\":\"Ethan R\",\"entityType\":\"PERSON\",\"language\":\"eng\",\"languageOfOrigin\":\"eng\",\"script\":\"Latn\"},\"dob\":\"1993-04-16\",\"addr\":\"123 Roadlane Ave\"},{\"primaryName\":{\"text\":\"Evan R\"},\"dob\":{\"date\":\"1993-04-16\"}}],\"right\":[{\"primaryName\":{\"text\":\"Seth R\",\"language\":\"eng\"},\"dob\":{\"date\":\"1993-04-16\"}},{\"dob2\":{\"date\":\"1993/04/16\"},\"primaryName\":\"Ivan R\",\"dob\":{\"date\":\"1993-04-16\"},\"addr\":{\"houseNumber\":\"123\",\"road\":\"Roadlane Ave\"}}]}}";
44+
private static final String EXPECTED_JSON_WITH_PARAMS = "{\"fields\":{\"dob2\":{\"type\":\"rni_date\",\"weight\":0.1},\"primaryName\":{\"type\":\"rni_name\",\"weight\":0.5},\"dob\":{\"type\":\"rni_date\",\"weight\":0.2},\"addr\":{\"type\":\"rni_address\",\"weight\":0.5}},\"properties\":{\"threshold\":0.7,\"includeExplainInfo\":true,\"parameters\":{\"timeDistanceWeight\":\"0.8\",\"stringDistanceWeight\":\"0.1\"}},\"records\":{\"left\":[{\"dob2\":{\"date\":\"1993/04/16\"},\"primaryName\":{\"text\":\"Ethan R\",\"entityType\":\"PERSON\",\"language\":\"eng\",\"languageOfOrigin\":\"eng\",\"script\":\"Latn\"},\"dob\":\"1993-04-16\",\"addr\":\"123 Roadlane Ave\"},{\"primaryName\":{\"text\":\"Evan R\"},\"dob\":{\"date\":\"1993-04-16\"}}],\"right\":[{\"primaryName\":{\"text\":\"Seth R\",\"language\":\"eng\"},\"dob\":{\"date\":\"1993-04-16\"}},{\"dob2\":{\"date\":\"1993/04/16\"},\"primaryName\":\"Ivan R\",\"dob\":{\"date\":\"1993-04-16\"},\"addr\":{\"houseNumber\":\"123\",\"road\":\"Roadlane Ave\"}}]}}";
45+
private static final String EXPECTED_JSON_WITH_UNIVERSE = "{\"fields\":{\"dob2\":{\"type\":\"rni_date\",\"weight\":0.1},\"primaryName\":{\"type\":\"rni_name\",\"weight\":0.5},\"dob\":{\"type\":\"rni_date\",\"weight\":0.2},\"addr\":{\"type\":\"rni_address\",\"weight\":0.5}},\"properties\":{\"threshold\":0.7,\"includeExplainInfo\":true,\"parameterUniverse\":\"myParameterUniverse\"},\"records\":{\"left\":[{\"dob2\":{\"date\":\"1993/04/16\"},\"primaryName\":{\"text\":\"Ethan R\",\"entityType\":\"PERSON\",\"language\":\"eng\",\"languageOfOrigin\":\"eng\",\"script\":\"Latn\"},\"dob\":\"1993-04-16\",\"addr\":\"123 Roadlane Ave\"},{\"primaryName\":{\"text\":\"Evan R\"},\"dob\":{\"date\":\"1993-04-16\"}}],\"right\":[{\"primaryName\":{\"text\":\"Seth R\",\"language\":\"eng\"},\"dob\":{\"date\":\"1993-04-16\"}},{\"dob2\":{\"date\":\"1993/04/16\"},\"primaryName\":\"Ivan R\",\"dob\":{\"date\":\"1993-04-16\"},\"addr\":{\"houseNumber\":\"123\",\"road\":\"Roadlane Ave\"}}]}}";
4546

46-
private static final String EXPECTED_JSON = "{\"fields\":{\"addr\":{\"type\":\"rni_address\",\"weight\":0.5},\"dob\":{\"type\":\"rni_date\",\"weight\":0.2},\"dob2\":{\"type\":\"rni_date\",\"weight\":0.1},\"primaryName\":{\"type\":\"rni_name\",\"weight\":0.5}},\"properties\":{\"includeExplainInfo\":true,\"threshold\":0.7},\"records\":{\"left\":[{\"addr\":\"123 Roadlane Ave\",\"dob\":\"1993-04-16\",\"dob2\":{\"date\":\"1993/04/16\"},\"primaryName\":{\"entityType\":\"PERSON\",\"language\":\"eng\",\"languageOfOrigin\":\"eng\",\"script\":\"Latn\",\"text\":\"Ethan R\"}},{\"dob\":{\"date\":\"1993-04-16\"},\"primaryName\":{\"text\":\"Evan R\"}}],\"right\":[{\"dob\":{\"date\":\"1993-04-16\"},\"primaryName\":{\"language\":\"eng\",\"text\":\"Seth R\"}},{\"addr\":{\"address\":\"123 Roadlane Ave\"},\"dob\":{\"date\":\"1993-04-16\"},\"dob2\":{\"date\":\"1993/04/16\"},\"primaryName\":\"Ivan R\"}]}}";
47+
private static final String EXPECTED_JSON = "{\"fields\":{\"dob2\":{\"type\":\"rni_date\",\"weight\":0.1},\"primaryName\":{\"type\":\"rni_name\",\"weight\":0.5},\"dob\":{\"type\":\"rni_date\",\"weight\":0.2},\"addr\":{\"type\":\"rni_address\",\"weight\":0.5}},\"properties\":{\"threshold\":0.7,\"includeExplainInfo\":true},\"records\":{\"left\":[{\"dob2\":{\"date\":\"1993/04/16\"},\"primaryName\":{\"text\":\"Ethan R\",\"entityType\":\"PERSON\",\"language\":\"eng\",\"languageOfOrigin\":\"eng\",\"script\":\"Latn\"},\"dob\":\"1993-04-16\",\"addr\":\"123 Roadlane Ave\"},{\"primaryName\":{\"text\":\"Evan R\"},\"dob\":{\"date\":\"1993-04-16\"}}],\"right\":[{\"primaryName\":{\"text\":\"Seth R\",\"language\":\"eng\"},\"dob\":{\"date\":\"1993-04-16\"}},{\"dob2\":{\"date\":\"1993/04/16\"},\"primaryName\":\"Ivan R\",\"dob\":{\"date\":\"1993-04-16\"},\"addr\":{\"address\":\"123 Roadlane Ave\"}}]}}";
4748
private static final RecordSimilarityRequest EXPECTED_REQUEST = RecordSimilarityRequest.builder()
4849
.fields(Map.of(
50+
"addr", RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_ADDRESS).weight(0.5).scoreIfNull(0.8).build(),
51+
"dob2", RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_DATE).weight(0.1).scoreIfNull(null).build(),
4952
"primaryName", RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_NAME).weight(0.5).build(),
53+
"dob", RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_DATE).weight(0.2).build()))
54+
.properties(RecordSimilarityProperties.builder().threshold(0.7).includeExplainInfo(true).build())
55+
.records(RecordSimilarityRecords.builder()
56+
.left(
57+
List.of(
58+
Map.of(
59+
"primaryName", NameField.FieldedName.builder()
60+
.text("Ethan R").entityType("PERSON")
61+
.language(LanguageCode.ENGLISH)
62+
.languageOfOrigin(LanguageCode.ENGLISH)
63+
.script(ISO15924.Latn)
64+
.build(),
65+
"dob", DateField.UnfieldedDate.builder().date("1993-04-16").build(),
66+
"dob2", DateField.FieldedDate.builder().date("1993/04/16").build(),
67+
"addr", AddressField.UnfieldedAddress.builder().address("123 Roadlane Ave").build()
68+
),
69+
Map.of(
70+
"primaryName", NameField.FieldedName.builder().text("Evan R").build(),
71+
"dob", DateField.FieldedDate.builder().date("1993-04-16").build()
72+
)
73+
)
74+
).right(
75+
List.of(
76+
Map.of(
77+
"primaryName", NameField.FieldedName.builder().text("Seth R").language(LanguageCode.ENGLISH).build(),
78+
"dob", DateField.FieldedDate.builder().date("1993-04-16").build()
79+
),
80+
Map.of(
81+
"primaryName", NameField.UnfieldedName.builder().text("Ivan R").build(),
82+
"dob", DateField.FieldedDate.builder().date("1993-04-16").build(),
83+
"dob2", DateField.FieldedDate.builder().date("1993/04/16").build(),
84+
"addr", AddressField.FieldedAddress.builder().houseNumber("123").road("Roadlane Ave").build()
85+
)
86+
)
87+
).build()
88+
).build();
89+
90+
private static final RecordSimilarityRequest EXPECTED_REQUEST_WITH_PARAMS = RecordSimilarityRequest.builder()
91+
.fields(Map.of(
92+
"dob2", RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_DATE).weight(0.1).build(),
93+
"primaryName", RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_NAME).weight(0.5).build(),
94+
"dob", RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_DATE).weight(0.2).build(),
95+
"addr", RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_ADDRESS).weight(0.5).build()))
96+
.properties(RecordSimilarityProperties.builder()
97+
.threshold(0.7)
98+
.includeExplainInfo(true)
99+
.parameters(
100+
Map.of(
101+
"timeDistanceWeight", "0.8",
102+
"stringDistanceWeight", "0.1"
103+
)
104+
)
105+
.build())
106+
.records(RecordSimilarityRecords.builder()
107+
.left(
108+
List.of(
109+
Map.of(
110+
"primaryName", NameField.FieldedName.builder()
111+
.text("Ethan R").entityType("PERSON")
112+
.language(LanguageCode.ENGLISH)
113+
.languageOfOrigin(LanguageCode.ENGLISH)
114+
.script(ISO15924.Latn)
115+
.build(),
116+
"dob", DateField.UnfieldedDate.builder().date("1993-04-16").build(),
117+
"dob2", DateField.FieldedDate.builder().date("1993/04/16").build(),
118+
"addr", AddressField.UnfieldedAddress.builder().address("123 Roadlane Ave").build()
119+
),
120+
Map.of(
121+
"primaryName", NameField.FieldedName.builder().text("Evan R").build(),
122+
"dob", DateField.FieldedDate.builder().date("1993-04-16").build()
123+
)
124+
)
125+
).right(
126+
List.of(
127+
Map.of(
128+
"primaryName", NameField.FieldedName.builder().text("Seth R").language(LanguageCode.ENGLISH).build(),
129+
"dob", DateField.FieldedDate.builder().date("1993-04-16").build()
130+
),
131+
Map.of(
132+
"primaryName", NameField.UnfieldedName.builder().text("Ivan R").build(),
133+
"dob", DateField.FieldedDate.builder().date("1993-04-16").build(),
134+
"dob2", DateField.FieldedDate.builder().date("1993/04/16").build(),
135+
"addr", AddressField.FieldedAddress.builder().houseNumber("123").road("Roadlane Ave").build()
136+
)
137+
)
138+
).build()
139+
).build();
140+
141+
private static final RecordSimilarityRequest EXPECTED_REQUEST_WITH_UNIVERSE = RecordSimilarityRequest.builder()
142+
.fields(Map.of(
50143
"dob", RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_DATE).weight(0.2).build(),
144+
"primaryName", RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_NAME).weight(0.5).build(),
51145
"dob2", RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_DATE).weight(0.1).build(),
52146
"addr", RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_ADDRESS).weight(0.5).build()))
53-
.properties(RecordSimilarityProperties.builder().threshold(0.7).includeExplainInfo(true).build())
147+
.properties(RecordSimilarityProperties.builder()
148+
.threshold(0.7)
149+
.includeExplainInfo(true)
150+
.parameterUniverse("myParameterUniverse")
151+
.build())
54152
.records(RecordSimilarityRecords.builder()
55153
.left(
56154
List.of(
@@ -80,7 +178,7 @@ class RecordSimilarityRequestTest {
80178
"primaryName", NameField.UnfieldedName.builder().text("Ivan R").build(),
81179
"dob", DateField.FieldedDate.builder().date("1993-04-16").build(),
82180
"dob2", DateField.FieldedDate.builder().date("1993/04/16").build(),
83-
"addr", AddressField.FieldedAddress.builder().address("123 Roadlane Ave").build()
181+
"addr", AddressField.FieldedAddress.builder().houseNumber("123").road("Roadlane Ave").build()
84182
)
85183
)
86184
).build()
@@ -92,14 +190,27 @@ void testDeserialization() throws JsonProcessingException {
92190
MAPPER.enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS);
93191
final RecordSimilarityRequest request = MAPPER.readValue(EXPECTED_JSON, new TypeReference<>() { });
94192
assertEquals(EXPECTED_REQUEST, request);
193+
194+
final RecordSimilarityRequest requestParams = MAPPER.readValue(EXPECTED_JSON_WITH_PARAMS, new TypeReference<>() { });
195+
assertEquals(EXPECTED_REQUEST_WITH_PARAMS, requestParams);
196+
197+
final RecordSimilarityRequest requestUniverse = MAPPER.readValue(EXPECTED_JSON_WITH_UNIVERSE, new TypeReference<>() { });
198+
assertEquals(EXPECTED_REQUEST_WITH_UNIVERSE, requestUniverse);
95199
}
96200

97201
@Test
98202
void testSerialization() throws JsonProcessingException {
99-
// For testing, force ordering
100-
MAPPER.enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY);
101-
MAPPER.enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS);
102-
assertEquals(EXPECTED_JSON, MAPPER.writeValueAsString(EXPECTED_REQUEST));
203+
final JsonNode expectedJson = MAPPER.readTree(EXPECTED_JSON);
204+
final JsonNode actualJson = MAPPER.valueToTree(EXPECTED_REQUEST);
205+
assertEquals(expectedJson, actualJson);
206+
207+
final JsonNode expectedParamJson = MAPPER.readTree(EXPECTED_JSON_WITH_PARAMS);
208+
final JsonNode actualParamJson = MAPPER.valueToTree(EXPECTED_REQUEST_WITH_PARAMS);
209+
assertEquals(expectedParamJson, actualParamJson);
210+
211+
final JsonNode expectedUniverseJson = MAPPER.readTree(EXPECTED_JSON_WITH_UNIVERSE);
212+
final JsonNode actualUniverseJson = MAPPER.valueToTree(EXPECTED_REQUEST_WITH_UNIVERSE);
213+
assertEquals(expectedUniverseJson, actualUniverseJson);
103214
}
104215

105216
}

0 commit comments

Comments
 (0)