Fix: [AEA-5173] - handle large batches (#1721)

## Summary

- Routine Change

### Details

- handle updates of more than 100 items

Author: anthony-nhs

packages/updatePrescriptionStatus/src/updatePrescriptionStatus.ts

@@ -191,17 +191,17 @@ export function validateEntries(requestEntries: Array<BundleEntry>, responseEntr
   let valid = true
   for (const entry of requestEntries) {
     const fullUrl = entry.fullUrl!
-    logger.info("Validating entry.", {entry: entry, id: entry.fullUrl})
+    logger.debug("Validating entry.", {entry: entry, id: entry.fullUrl})
 
     const validationOutcome = validateEntry(entry)
 
     let responseEntry: BundleEntry
     if (validationOutcome.valid) {
-      logger.info("Entry validated successfully.", {entry: entry, id: entry.fullUrl})
+      logger.debug("Entry validated successfully.", {entry: entry, id: entry.fullUrl})
       responseEntry = accepted(fullUrl)
     } else {
       const errorMessage = validationOutcome.issues!
-      logger.info(`Entry failed validation. ${errorMessage}`, {entry: entry, id: entry.fullUrl})
+      logger.warn(`Entry failed validation. ${errorMessage}`, {entry: entry, id: entry.fullUrl})
       valid = false
       responseEntry = badRequest(errorMessage, fullUrl)
     }
@@ -252,7 +252,7 @@ export function buildDataItems(
 
   for (const requestEntry of requestEntries) {
     const task = requestEntry.resource as Task
-    logger.info("Building data item for task.", {task: task, id: task.id})
+    logger.debug("Building data item for task.", {task: task, id: task.id})
 
     const repeatNo = task.input?.[0]?.valueInteger
 

packages/updatePrescriptionStatus/src/utils/databaseClient.ts

@@ -33,19 +33,41 @@ function createTransactionCommand(dataItems: Array<PSUDataItem>, logger: Logger)
 }
 
 export async function persistDataItems(dataItems: Array<PSUDataItem>, logger: Logger): Promise<boolean | Timeout> {
-  const transactionCommand = createTransactionCommand(dataItems, logger)
-  try {
-    logger.info("Sending TransactWriteItemsCommand to DynamoDB.", {command: transactionCommand})
-    await client.send(transactionCommand)
-    logger.info("TransactWriteItemsCommand sent to DynamoDB successfully.", {command: transactionCommand})
-    return true
-  } catch (e) {
-    if (e instanceof TransactionCanceledException) {
-      logger.error("DynamoDB transaction cancelled due to conditional check failure.", {reasons: e.CancellationReasons})
-      throw e
+  // break the array of data items into batches less than 100
+  // to prevent dynamodb error with too many items
+  const chunkSize = 99
+  const transactionCommands = []
+  for (let i = 0; i < dataItems.length; i += chunkSize) {
+    const chunk = dataItems.slice(i, i + chunkSize)
+    const transactionCommand = createTransactionCommand(chunk, logger)
+    transactionCommands.push(transactionCommand)
+  }
+  const results = await Promise.all(transactionCommands.map(async transactionCommand => {
+    try {
+      logger.info("Sending TransactWriteItemsCommand to DynamoDB.", {command: transactionCommand})
+      await client.send(transactionCommand)
+      logger.info("TransactWriteItemsCommand sent to DynamoDB successfully.", {command: transactionCommand})
+      return {success: true}
+    } catch (e) {
+      if (e instanceof TransactionCanceledException) {
+        logger.error(
+          "DynamoDB transaction cancelled due to conditional check failure.", {reasons: e.CancellationReasons})
+        return {success: false, errorMessage: "conditional check failure", error: e}
+      } else {
+        logger.error("Error sending TransactWriteItemsCommand to DynamoDB.", {error: e})
+      }
+      return {success: false, errorMessage: "other error", error: e}
+    }
+  }))
+  const failed = results.filter(r => !r.success)
+  if (failed.length > 0) {
+    const conditionalCheckFailures = failed.filter(r=> r.errorMessage === "conditional check failure")
+    if (conditionalCheckFailures.length > 0) {
+      throw(conditionalCheckFailures[0].error)
     }
-    logger.error("Error sending TransactWriteItemsCommand to DynamoDB.", {error: e})
     return false
+  } else {
+    return true
   }
 }
 

packages/updatePrescriptionStatus/tests/testDatabaseClient.test.ts

@@ -18,7 +18,9 @@ describe("Unit test persistDataItems", () => {
   beforeEach(() => {
     jest.resetModules()
     jest.clearAllMocks()
+    jest.resetAllMocks()
   })
+
   it("when the conditional check fails, an error is thrown", async () => {
     const dataItems = [
       {
@@ -70,4 +72,105 @@ describe("Unit test persistDataItems", () => {
       reasons: [{Code: "ConditionalCheckFailedException"}]
     })
   })
+
+  it("returns success when everything works", async () => {
+    const dataItems = [
+      {
+        LastModified: "2023-01-01T00:00:00Z",
+        LineItemID: "LineItemID_1",
+        PatientNHSNumber: "PatientNHSNumber_1",
+        PharmacyODSCode: "PharmacyODSCode_1",
+        PrescriptionID: "PrescriptionID_1",
+        RequestID: "RequestID_1",
+        Status: "Status_1",
+        TaskID: "TaskID_1",
+        TerminalStatus: "TerminalStatus_1",
+        ApplicationName: "name",
+        ExpiryTime: 10
+      },
+      {
+        LastModified: "2023-01-02T00:00:00Z",
+        LineItemID: "LineItemID_2",
+        PatientNHSNumber: "PatientNHSNumber_2",
+        PharmacyODSCode: "PharmacyODSCode_2",
+        PrescriptionID: "PrescriptionID_1",
+        RequestID: "RequestID_2",
+        Status: "Status_2",
+        TaskID: "TaskID_1",
+        TerminalStatus: "TerminalStatus_2",
+        ApplicationName: "name",
+        ExpiryTime: 10
+      }
+    ]
+
+    const loggerSpy = jest.spyOn(logger, "error")
+
+    const result = await persistDataItems(dataItems, logger)
+    expect(result).toBe(true)
+    expect(loggerSpy).not.toBeCalled()
+  })
+
+  it("call dynamo update twice when there is a large batch", async () => {
+    const dataItem = {
+      LastModified: "2023-01-01T00:00:00Z",
+      LineItemID: "LineItemID_1",
+      PatientNHSNumber: "PatientNHSNumber_1",
+      PharmacyODSCode: "PharmacyODSCode_1",
+      PrescriptionID: "PrescriptionID_1",
+      RequestID: "RequestID_1",
+      Status: "Status_1",
+      TaskID: "TaskID_1",
+      TerminalStatus: "TerminalStatus_1",
+      ApplicationName: "name",
+      ExpiryTime: 10
+    }
+    const dataItems = Array(150).fill(dataItem)
+
+    const loggerSpy = jest.spyOn(logger, "error")
+
+    const result = await persistDataItems(dataItems, logger)
+    expect(result).toBe(true)
+    expect(loggerSpy).not.toBeCalled()
+    expect(mockSend).toBeCalledTimes(2)
+  })
+
+  it("returns false when there is a general error", async () => {
+    const dataItems = [
+      {
+        LastModified: "2023-01-01T00:00:00Z",
+        LineItemID: "LineItemID_1",
+        PatientNHSNumber: "PatientNHSNumber_1",
+        PharmacyODSCode: "PharmacyODSCode_1",
+        PrescriptionID: "PrescriptionID_1",
+        RequestID: "RequestID_1",
+        Status: "Status_1",
+        TaskID: "TaskID_1",
+        TerminalStatus: "TerminalStatus_1",
+        ApplicationName: "name",
+        ExpiryTime: 10
+      },
+      {
+        LastModified: "2023-01-02T00:00:00Z",
+        LineItemID: "LineItemID_2",
+        PatientNHSNumber: "PatientNHSNumber_2",
+        PharmacyODSCode: "PharmacyODSCode_2",
+        PrescriptionID: "PrescriptionID_1",
+        RequestID: "RequestID_2",
+        Status: "Status_2",
+        TaskID: "TaskID_1",
+        TerminalStatus: "TerminalStatus_2",
+        ApplicationName: "name",
+        ExpiryTime: 10
+      }
+    ]
+
+    const loggerSpy = jest.spyOn(logger, "error")
+    mockSend.mockRejectedValue(
+      new Error("General error") as never
+    )
+
+    const result = await persistDataItems(dataItems, logger)
+    expect(result).toBe(false)
+    expect(loggerSpy).toBeCalled()
+  })
 })