fix: resolve payment on RemoveTlc(Fulfill) for force-closed channels (#1222)

.github/workflows/e2e.yml

@@ -35,6 +35,7 @@ jobs:
           - watchtower/force-close-with-pending-tlcs
           - watchtower/force-close-with-pending-tlcs-and-stop-watchtower
           - watchtower/force-close-with-pending-tlcs-and-udt
+          - watchtower/force-close-preimage-settled-by-recipient
           - watchtower/revocation
         release:
           - "0.202.0"

crates/fiber-lib/src/fiber/network.rs

@@ -100,11 +100,11 @@ use fiber_types::SessionRoute;
 use fiber_types::{
     blake2b_hash_with_salt, AddTlcCommand, AwaitingTxSignaturesFlags, ChannelOpenRecord,
     ChannelOpeningStatus, ChannelState, ChannelTlcInfo, CloseFlags, EcdsaSignature, EntityHex,
-    FeatureVector, Hash256, NodeAnnouncement, PaymentCustomRecords, PaymentStatus,
-    PeeledPaymentOnionPacket, PersistentNetworkActorState, PrevTlcInfo, Privkey, Pubkey,
-    PublicChannelInfo, RemoveTlcFulfill, RemoveTlcReason, RetryableTlcOperation, RevocationData,
-    RouterHop, SettlementData, ShuttingDownFlags, TLCId, TlcErr, TlcErrPacket, TlcErrorCode,
-    TrampolineContext, UdtCfgInfos,
+    FeatureVector, Hash256, NodeAnnouncement, OutboundTlcStatus, PaymentCustomRecords,
+    PaymentStatus, PeeledPaymentOnionPacket, PersistentNetworkActorState, PrevTlcInfo, Privkey,
+    Pubkey, PublicChannelInfo, RemoveTlcFulfill, RemoveTlcReason, RetryableTlcOperation,
+    RevocationData, RouterHop, SettlementData, ShuttingDownFlags, TLCId, TlcErr, TlcErrPacket,
+    TlcErrorCode, TrampolineContext, UdtCfgInfos,
 };
 
 pub const FIBER_PROTOCOL_ID: ProtocolId = ProtocolId::new(42);
@@ -1111,6 +1111,109 @@ where
                 }
 
                 if !found {
+                    // When a RemoveTlc(Fulfill) arrives for a force-closed channel
+                    // the channel actor is gone.  Look up the TLC from persisted
+                    // state and either relay the fulfillment upstream (intermediate
+                    // node) or notify the payment actor (sender).
+                    if let FiberChannelMessage::RemoveTlc(ref remove_tlc) = msg {
+                        if let RemoveTlcReason::RemoveTlcFulfill(ref fulfill) = &remove_tlc.reason {
+                            if let Some(mut actor_state) =
+                                state.store.get_channel_actor_state(&channel_id)
+                            {
+                                if !actor_state.is_closed() {
+                                    warn!(
+                                        "Ignoring RemoveTlc(Fulfill) fallback for non-closed channel {:?}",
+                                        channel_id
+                                    );
+                                } else if actor_state.get_remote_pubkey() != peer_pubkey {
+                                    warn!(
+                                        "Ignoring RemoveTlc(Fulfill) fallback for channel {:?}: \
+                                         persisted remote pubkey {:?} does not match peer {:?}",
+                                        channel_id,
+                                        actor_state.get_remote_pubkey(),
+                                        peer_pubkey
+                                    );
+                                } else if let Some(tlc) =
+                                    actor_state.get_offered_tlc(TLCId::Offered(remove_tlc.tlc_id))
+                                {
+                                    let payment_hash = tlc.payment_hash;
+                                    let forwarding_tlc = tlc.forwarding_tlc;
+                                    let attempt_id = tlc.attempt_id;
+                                    let hash_algorithm = tlc.hash_algorithm;
+                                    let outbound_status = tlc.outbound_status();
+
+                                    let computed_hash: Hash256 =
+                                        hash_algorithm.hash(fulfill.payment_preimage).into();
+                                    if computed_hash != payment_hash {
+                                        warn!(
+                                            "Ignoring RemoveTlc(Fulfill) fallback for channel {:?} \
+                                             tlc {:?}: preimage does not match payment hash",
+                                            channel_id, remove_tlc.tlc_id,
+                                        );
+                                        return Err(Error::ChannelNotFound(channel_id));
+                                    }
+
+                                    if outbound_status != OutboundTlcStatus::Committed {
+                                        warn!(
+                                            "Ignoring RemoveTlc(Fulfill) fallback for channel {:?} \
+                                             tlc {:?}: outbound status is {:?}, expected Committed",
+                                            channel_id, remove_tlc.tlc_id, outbound_status,
+                                        );
+                                        return Err(Error::ChannelNotFound(channel_id));
+                                    }
+
+                                    state
+                                        .store
+                                        .insert_preimage(payment_hash, fulfill.payment_preimage);
+                                    // Update TLC status to RemoteRemoved in
+                                    // persisted channel state.
+                                    actor_state.tlc_state.set_offered_tlc_removed(
+                                        remove_tlc.tlc_id,
+                                        remove_tlc.reason.clone(),
+                                    );
+                                    state.store.insert_channel_actor_state(actor_state);
+
+                                    if let Some((prev_channel_id, prev_tlc_id)) = forwarding_tlc {
+                                        info!(
+                                            "RemoveTlc(Fulfill) on closed channel {:?}, \
+                                             relaying upstream to channel {:?}",
+                                            channel_id, prev_channel_id,
+                                        );
+                                        let (send, _recv) = oneshot::channel();
+                                        let rpc_reply = RpcReplyPort::from(send);
+                                        let _ = state
+                                            .send_command_to_channel(
+                                                prev_channel_id,
+                                                ChannelCommand::RemoveTlc(
+                                                    RemoveTlcCommand {
+                                                        id: prev_tlc_id,
+                                                        reason: remove_tlc.reason.clone(),
+                                                    },
+                                                    rpc_reply,
+                                                ),
+                                            )
+                                            .await;
+                                    } else {
+                                        info!(
+                                            "RemoveTlc(Fulfill) on closed channel {:?}, \
+                                             notifying payment actor for {:?}",
+                                            channel_id, payment_hash,
+                                        );
+                                        myself
+                                            .send_message(NetworkActorMessage::new_event(
+                                                NetworkActorEvent::TlcRemoveReceived(
+                                                    payment_hash,
+                                                    attempt_id,
+                                                    remove_tlc.reason.clone(),
+                                                ),
+                                            ))
+                                            .expect(ASSUME_NETWORK_MYSELF_ALIVE);
+                                    }
+                                    return Ok(());
+                                }
+                            }
+                        }
+                    }
                     error!(
                             "Received a channel message for a channel that is not created with peer: {:?}",
                             channel_id
@@ -1596,6 +1699,54 @@ where
         Ok(())
     }
 
+    /// For each fulfilled received TLC on the given channel, mark the
+    /// corresponding invoice as `Paid` if enough TLCs have been fulfilled to
+    /// cover the invoice amount. Each invoice is processed at most once.
+    fn settle_invoices_for_fulfilled_tlcs(
+        &self,
+        actor_state: &ChannelActorState,
+        channel_id: &Hash256,
+        log_context: &str,
+    ) {
+        let mut processed_invoice_hashes = HashSet::new();
+        for tlc in actor_state.tlc_state.received_tlcs.tlcs.iter() {
+            if !matches!(
+                tlc.removed_reason,
+                Some(RemoveTlcReason::RemoveTlcFulfill(_))
+            ) {
+                continue;
+            }
+            if !processed_invoice_hashes.insert(tlc.payment_hash) {
+                // process each hash once
+                continue;
+            }
+            if !self
+                .store
+                .get_invoice_status(&tlc.payment_hash)
+                .is_some_and(|s| matches!(s, CkbInvoiceStatus::Open | CkbInvoiceStatus::Received))
+            {
+                continue;
+            }
+            let Some(invoice) = self.store.get_invoice(&tlc.payment_hash) else {
+                continue;
+            };
+            let fulfilled_tlcs = actor_state.tlc_state.received_tlcs.tlcs.iter().filter(|t| {
+                t.payment_hash == tlc.payment_hash
+                    && matches!(t.removed_reason, Some(RemoveTlcReason::RemoveTlcFulfill(_)))
+            });
+            if !is_invoice_fulfilled(&invoice, fulfilled_tlcs) {
+                continue;
+            }
+            self.store
+                .update_invoice_status(&tlc.payment_hash, CkbInvoiceStatus::Paid)
+                .expect("update invoice status failed");
+            info!(
+                "Updated invoice to Paid for fulfilled TLC {:?} on {} {:?}",
+                tlc.payment_hash, log_context, channel_id
+            );
+        }
+    }
+
     pub async fn handle_command(
         &self,
         myself: ActorRef<NetworkActorMessage>,
@@ -1936,6 +2087,16 @@ where
                                 }
                             }
 
+                            // For received TLCs that have been locally removed
+                            // with a Fulfill reason but whose commitment
+                            // round-trip never completed (e.g. the remote peer
+                            // force-closed), update the invoice to Paid.
+                            self.settle_invoices_for_fulfilled_tlcs(
+                                &actor_state,
+                                &channel_id,
+                                "check channel",
+                            );
+
                             let delay_epoch = EpochNumberWithFraction::from_full_value(
                                 actor_state.commitment_delay_epoch,
                             );
@@ -2093,6 +2254,14 @@ where
                                     }
                                 }
                             }
+
+                            // For received TLCs on force-closed channels that
+                            // were fulfilled, update invoice to Paid.
+                            self.settle_invoices_for_fulfilled_tlcs(
+                                &actor_state,
+                                &channel_id,
+                                "closed channel",
+                            );
                         }
                     }
                 }