Skip to content

Commit bcaf134

Browse files
bsimon-btbsimon-bt
committed
Merge branch 'refs/heads/master' into RLPNC-7509-record-similarity-properties-validation-from-fork
# Conflicts: # json/src/main/java/com/basistech/rosette/apimodel/jackson/recordsimilaritydeserializers/RecordSimilarityDeserializerUtilities.java # model/src/main/java/com/basistech/rosette/apimodel/recordsimilarity/RecordSimilarityResult.java
2 parents 1a318ed + fd83c5a commit bcaf134

16 files changed

Lines changed: 237 additions & 137 deletions

File tree

‎api/src/test/java/com/basistech/rosette/api/RosetteAPITest.java‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -241,13 +241,15 @@ private static Stream<Arguments> testMatchRecordMissingFieldParameters() throws
241241
@MethodSource("testMatchRecordMissingFieldParameters")
242242
void testMatchRecordMissingField(String testFilename, String responseStr, int statusCode) throws IOException {
243243
setStatusCodeResponse(responseStr, statusCode);
244-
245-
try {
246-
readValueRecordMatcher(testFilename);
247-
fail("Did not throw exception for a field type in request but not in mapping");
248-
} catch (IllegalArgumentException e) {
249-
assertEquals("Unsupported field name: primaryName not found in field mapping", e.getMessage());
250-
}
244+
readValueRecordMatcher(testFilename);
245+
assertEquals("{\"results\":[{\"score\":0.0,\"left\":{\"dob2\":\"1993/04/16\","
246+
+ "\"dob\":\"1993-04-16\",\"primaryName\":{\"data\":\"Ethan R\",\"language\":\"eng\","
247+
+ "\"entityType\":\"PERSON\"},\"addr\":\"123 Roadlane Ave\"},\"right\":{\"dob\":\"1993-04-16\","
248+
+ "\"primaryName\":\"Seth R\"},\"error\":\"Field 'primaryName' not found in field mapping\"},"
249+
+ "{\"score\":0.0,\"left\":{\"dob\":\"1993-04-16\",\"primaryName\":\"Evan R\"},"
250+
+ "\"right\":{\"dob2\":\"1993/04/16\",\"dob\":\"1993-04-16\",\"primaryName\":\"Ivan R\","
251+
+ "\"addr\":\"123 Roadlane Ave\"},\"error\":\"Field 'primaryName' not found in field mapping\"}]}",
252+
responseStr);
251253
}
252254

253255
private static Stream<Arguments> testMatchRecordNullFieldParameters() throws IOException {

‎api/src/test/mock-data/response/rni-1-record-similarity-missing-field.json‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1 @@
1-
{
2-
"code": "badRequestFormat",
3-
"message": "Unsupported field name: primaryName not found in field mapping",
4-
"stack": null
5-
}
1+
{"results":[{"score":0.0,"left":{"dob2":"1993/04/16","dob":"1993-04-16","primaryName":{"data":"Ethan R","language":"eng","entityType":"PERSON"},"addr":"123 Roadlane Ave"},"right":{"dob":"1993-04-16","primaryName":"Seth R"},"error":"Field 'primaryName' not found in field mapping"},{"score":0.0,"left":{"dob":"1993-04-16","primaryName":"Evan R"},"right":{"dob2":"1993/04/16","dob":"1993-04-16","primaryName":"Ivan R","addr":"123 Roadlane Ave"},"error":"Field 'primaryName' not found in field mapping"}]}

‎examples/src/main/java/com/basistech/rosette/examples/RecordSimilarityExample.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -56,10 +56,10 @@ private void run() throws IOException {
5656
String dobHyphen = "1993-04-16";
5757
RecordSimilarityRequest request = RecordSimilarityRequest.builder()
5858
.fields(Map.of(
59-
primaryNameField, RecordSimilarityFieldInfo.builder().type(RecordFieldType.NAME).weight(0.5).build(),
60-
dobField, RecordSimilarityFieldInfo.builder().type(RecordFieldType.DATE).weight(0.2).build(),
61-
dob2Field, RecordSimilarityFieldInfo.builder().type(RecordFieldType.DATE).weight(0.1).build(),
62-
addrField, RecordSimilarityFieldInfo.builder().type(RecordFieldType.ADDRESS).weight(0.5).build()))
59+
primaryNameField, RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_NAME).weight(0.5).build(),
60+
dobField, RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_DATE).weight(0.2).build(),
61+
dob2Field, RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_DATE).weight(0.1).build(),
62+
addrField, RecordSimilarityFieldInfo.builder().type(RecordFieldType.RNI_ADDRESS).weight(0.5).build()))
6363
.properties(RecordSimilarityProperties.builder().threshold(0.7).includeExplainInfo(true).build())
6464
.records(RecordSimilarityRecords.builder()
6565
.left(
@@ -90,7 +90,7 @@ private void run() throws IOException {
9090
primaryNameField, NameField.UnfieldedName.builder().text("Ivan R").build(),
9191
dobField, DateField.FieldedDate.builder().date(dobHyphen).build(),
9292
dob2Field, DateField.FieldedDate.builder().date("1993/04/16").build(),
93-
addrField, AddressField.FieldedAddress.builder().address("123 Roadlane Ave").build()
93+
addrField, AddressField.FieldedAddress.builder().houseNumber("123").road("Roadlane Ave").build()
9494
)
9595
)
9696
).build()

‎json/src/main/java/com/basistech/rosette/apimodel/jackson/ApiModelMixinModule.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ public void setupModule(Module.SetupContext context) {
8484
context.setMixInAnnotations(UnfieldedAddress.class, UnfieldedAddressMixin.class);
8585
context.setMixInAnnotations(UnfieldedAddress.UnfieldedAddressBuilder.class,
8686
UnfieldedAddressMixin.UnfieldedAddressBuilderMixin.class);
87+
8788
context.setMixInAnnotations(AddressSimilarityRequest.class, AddressSimilarityRequestMixin.class);
8889
context.setMixInAnnotations(AddressSimilarityRequest.AddressSimilarityRequestBuilder.class,
8990
AddressSimilarityRequestMixin.AddressSimilarityRequestBuilderMixin.class);

‎json/src/main/java/com/basistech/rosette/apimodel/jackson/recordsimilaritydeserializers/RecordSimilarityDeserializerUtilities.java‎

Lines changed: 38 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -37,31 +37,34 @@
3737
import com.basistech.rosette.apimodel.recordsimilarity.records.AddressField;
3838
import com.basistech.rosette.apimodel.recordsimilarity.records.DateField;
3939
import com.basistech.rosette.apimodel.recordsimilarity.records.NameField;
40+
import com.basistech.rosette.apimodel.recordsimilarity.records.RecordFieldType;
4041
import com.basistech.rosette.apimodel.recordsimilarity.records.RecordSimilarityField;
42+
import com.basistech.rosette.apimodel.recordsimilarity.records.UnknownField;
4143

4244
final class RecordSimilarityDeserializerUtilities {
4345

4446
private RecordSimilarityDeserializerUtilities() {
4547
}
4648

47-
public static RecordSimilarityResult parseResult(
48-
JsonNode node,
49-
JsonParser jsonParser,
50-
@Valid Map<String, RecordSimilarityFieldInfo> fields
51-
) throws IOException {
49+
public static RecordSimilarityResult parseResult(JsonNode node, JsonParser jsonParser) throws IOException {
5250
final Double score = node.get("score") != null
5351
? node.get("score").traverse(jsonParser.getCodec()).readValueAs(Double.class)
5452
: null;
5553
final RecordSimilarityExplainInfo explainInfo = node.get("explainInfo") != null
5654
? node.get("explainInfo").traverse(jsonParser.getCodec()).readValueAs(RecordSimilarityExplainInfo.class)
5755
: null;
58-
final Map<String, RecordSimilarityField> left = node.get("left") != null && fields != null
59-
? parseRecord(node.get("left"), jsonParser, fields)
56+
final Map<String, RecordSimilarityField> left = node.get("left") != null
57+
? parseRecordForResponse(node.get("left"), jsonParser)
6058
: null;
61-
final Map<String, RecordSimilarityField> right = node.get("right") != null && fields != null
62-
? parseRecord(node.get("right"), jsonParser, fields)
59+
final Map<String, RecordSimilarityField> right = node.get("right") != null
60+
? parseRecordForResponse(node.get("right"), jsonParser)
6361
: null;
64-
final String error = Optional.ofNullable(node.get("error")).map(JsonNode::asText).orElse(null);
62+
63+
List<String> errorList = Optional.ofNullable(node.get("error"))
64+
.map(jsonNode -> StreamSupport.stream(jsonNode.spliterator(), false)
65+
.map(JsonNode::asText)
66+
.collect(Collectors.toList()))
67+
.orElse(null);
6568
final List<String> info = Optional.ofNullable(node.get("info"))
6669
.map(jsonNode -> StreamSupport.stream(jsonNode.spliterator(), false)
6770
.map(JsonNode::asText)
@@ -72,11 +75,25 @@ public static RecordSimilarityResult parseResult(
7275
.left(left)
7376
.right(right)
7477
.explainInfo(explainInfo)
75-
.error(error)
78+
.error(errorList)
7679
.info(info)
7780
.build();
7881
}
7982

83+
static Map<String, RecordSimilarityField> parseRecordForResponse(JsonNode jsonNode, JsonParser jsonParser) {
84+
final Map<String, RecordSimilarityField> recordMap = new HashMap<>();
85+
jsonNode.fields().forEachRemaining(entry -> {
86+
String fieldName = entry.getKey();
87+
try {
88+
recordMap.put(fieldName, jsonNode.get(fieldName).traverse(jsonParser.getCodec())
89+
.readValueAs(UnknownField.class));
90+
} catch (IOException e) {
91+
throw new RuntimeException(e);
92+
}
93+
});
94+
return recordMap;
95+
}
96+
8097
static Map<String, RecordSimilarityField> parseRecord(
8198
JsonNode jsonNode,
8299
JsonParser jsonParser,
@@ -89,29 +106,32 @@ static Map<String, RecordSimilarityField> parseRecord(
89106
final String fieldName = recordEntry.getKey();
90107
final JsonNode fieldValue = recordEntry.getValue();
91108

109+
final RecordSimilarityField fieldData;
110+
92111
if (fields.containsKey(fieldName)) {
93112
final RecordSimilarityFieldInfo fieldInfo = fields.get(fieldName);
94-
final RecordSimilarityField fieldData;
95113
if (fieldInfo.getType() == null) {
96114
throw new IllegalArgumentException("Unspecified field type for: " + fieldName);
97115
}
98116
switch (fieldInfo.getType()) {
99-
case DATE:
117+
case RecordFieldType.RNI_DATE:
100118
fieldData = fieldValue.traverse(jsonParser.getCodec()).readValueAs(DateField.class);
101119
break;
102-
case NAME:
120+
case RecordFieldType.RNI_NAME:
103121
fieldData = fieldValue.traverse(jsonParser.getCodec()).readValueAs(NameField.class);
104122
break;
105-
case ADDRESS:
123+
case RecordFieldType.RNI_ADDRESS:
106124
fieldData = fieldValue.traverse(jsonParser.getCodec()).readValueAs(AddressField.class);
107125
break;
108126
default:
109-
throw new IllegalArgumentException("Unsupported field type: " + fieldInfo.getType());
127+
fieldData = fieldValue.traverse(jsonParser.getCodec()).readValueAs(UnknownField.class);
110128
}
111-
recordMap.put(fieldName, fieldData);
112129
} else {
113-
throw new IllegalArgumentException("Unsupported field name: " + fieldName + " not found in field mapping");
130+
//treat unmapped field as UnknownField so we can get to scoring,
131+
//it won't be counted toward the score anyway
132+
fieldData = fieldValue.traverse(jsonParser.getCodec()).readValueAs(UnknownField.class);
114133
}
134+
recordMap.put(fieldName, fieldData);
115135
}
116136
return recordMap;
117137
}

‎json/src/main/java/com/basistech/rosette/apimodel/jackson/recordsimilaritydeserializers/RecordSimilarityResponseDeserializer.java‎

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -16,28 +16,22 @@
1616

1717
package com.basistech.rosette.apimodel.jackson.recordsimilaritydeserializers;
1818

19-
import com.basistech.rosette.apimodel.recordsimilarity.RecordSimilarityFieldInfo;
2019
import com.basistech.rosette.apimodel.recordsimilarity.RecordSimilarityResponse;
2120
import com.basistech.rosette.apimodel.recordsimilarity.RecordSimilarityResult;
2221
import com.fasterxml.jackson.core.JsonParser;
23-
import com.fasterxml.jackson.core.type.TypeReference;
2422
import com.fasterxml.jackson.databind.DeserializationContext;
2523
import com.fasterxml.jackson.databind.JsonNode;
2624
import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
2725

2826
import java.io.IOException;
2927
import java.util.ArrayList;
3028
import java.util.List;
31-
import java.util.Map;
3229
import java.util.Optional;
3330
import java.util.stream.Collectors;
3431
import java.util.stream.StreamSupport;
3532

3633
public class RecordSimilarityResponseDeserializer extends StdDeserializer<RecordSimilarityResponse> {
3734

38-
private static final TypeReference<Map<String, RecordSimilarityFieldInfo>> FIELDS_TYPE_REFERENCE = new TypeReference<>() {
39-
};
40-
4135
public RecordSimilarityResponseDeserializer() {
4236
super(RecordSimilarityResponse.class);
4337
}
@@ -46,9 +40,6 @@ public RecordSimilarityResponseDeserializer() {
4640
public RecordSimilarityResponse deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException {
4741
final JsonNode node = jsonParser.getCodec().readTree(jsonParser);
4842

49-
JsonNode fieldsNode = node.get("fields");
50-
51-
Map<String, RecordSimilarityFieldInfo> fields = fieldsNode != null ? node.get("fields").traverse(jsonParser.getCodec()).readValueAs(FIELDS_TYPE_REFERENCE) : null;
5243
List<String> info = Optional.ofNullable(node.get("info"))
5344
.map(jsonNode -> StreamSupport.stream(jsonNode.spliterator(), false)
5445
.map(JsonNode::asText)
@@ -60,12 +51,10 @@ public RecordSimilarityResponse deserialize(JsonParser jsonParser, Deserializati
6051
List<RecordSimilarityResult> results = new ArrayList<>();
6152
if (resultsNode != null) {
6253
for (JsonNode resultNode : resultsNode) {
63-
results.add(RecordSimilarityDeserializerUtilities.parseResult(resultNode, jsonParser, fields));
54+
results.add(RecordSimilarityDeserializerUtilities.parseResult(resultNode, jsonParser));
6455
}
6556
}
66-
6757
return RecordSimilarityResponse.builder()
68-
.fields(fields)
6958
.results(results)
7059
.info(info)
7160
.errorMessage(errorMessage)

‎json/src/test/java/com/basistech/rosette/apimodel/ModelTest.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,9 @@ void packageTest(boolean inputStreams) throws ClassNotFoundException, IOExceptio
102102
if (className.contains("RecordSimilarityRequest")) {
103103
continue;
104104
}
105+
if (className.contains("UnknownField")) {
106+
continue;
107+
}
105108

106109
Class c = Class.forName(className);
107110
if (Modifier.isAbstract(c.getModifiers())) {

0 commit comments

Comments
 (0)