Rework payments database (#670)

The payments tables have been merged in two tables, outgoing and incoming, and a view joining the two has been added. Payment data are now serialised with a custom binary format defined in lightning-kmp. All this makes working with the payments db much easier and lets us remove code. Payments are now all using an UUID for identifier. We also use the new payments model from lightning-kmp which has strict types for incoming payments. Liquidity are now stored inside payments, whenever possible. Local and cloud payments data are migrated when starting the new app. Several tests with old payments databases have been added. --------- Co-authored-by: pm47 <[email protected]> Co-authored-by: Robbie Hanson <[email protected]>
ACINQ · Feb 6, 2025 · 414c7a6 · 414c7a6
1 parent a07c153
commit 414c7a6
Show file tree

Hide file tree

Showing 241 changed files with 8,546 additions and 9,319 deletions.
diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt
@@ -1,12 +1,13 @@
 object Versions {
-    const val lightningKmp = "1.8.4"
+    const val lightningKmp = "1.8.5-SNAPSHOT"
     const val secp256k1 = "0.14.0"
     const val torMobile = "0.2.0"
 
     const val kotlin = "1.9.22"
 
     const val ktor = "2.3.7"
     const val sqlDelight = "2.0.1"
+    const val okio = "3.8.0"
 
     const val slf4j = "1.7.30"
     const val junit = "4.13"

diff --git a/gradle.properties b/gradle.properties
@@ -33,6 +33,5 @@ xcodeproj=phoenix-ios/phoenix-ios.xcodeproj
 
 # the chain that we use
 chain="testnet"
-android.defaults.buildfeatures.buildconfig=true
 android.nonTransitiveRClass=false
 android.nonFinalResIds=false
diff --git a/phoenix-android/build.gradle.kts b/phoenix-android/build.gradle.kts
@@ -69,6 +69,7 @@ android {
         compose = true
         viewBinding = true
         dataBinding = true
+        buildConfig = true
     }
 
     composeOptions {

diff --git a/phoenix-android/src/main/kotlin/fr/acinq/phoenix/android/AppView.kt b/phoenix-android/src/main/kotlin/fr/acinq/phoenix/android/AppView.kt
@@ -40,6 +40,7 @@ import androidx.compose.runtime.LaunchedEffect
 import androidx.compose.runtime.collectAsState
 import androidx.compose.runtime.getValue
 import androidx.compose.runtime.livedata.observeAsState
+import androidx.compose.runtime.remember
 import androidx.compose.ui.Alignment
 import androidx.compose.ui.Modifier
 import androidx.compose.ui.platform.LocalContext
@@ -59,7 +60,10 @@ import com.google.accompanist.permissions.ExperimentalPermissionsApi
 import com.google.accompanist.permissions.isGranted
 import com.google.accompanist.permissions.rememberPermissionState
 import com.google.firebase.messaging.FirebaseMessaging
+import fr.acinq.lightning.db.Bolt12IncomingPayment
+import fr.acinq.lightning.db.ChannelCloseOutgoingPayment
 import fr.acinq.lightning.db.IncomingPayment
+import fr.acinq.lightning.utils.UUID
 import fr.acinq.lightning.utils.currentTimestampMillis
 import fr.acinq.phoenix.PhoenixBusiness
 import fr.acinq.phoenix.android.components.Button
@@ -74,7 +78,7 @@ import fr.acinq.phoenix.android.intro.IntroView
 import fr.acinq.phoenix.android.payments.details.PaymentDetailsView
 import fr.acinq.phoenix.android.payments.history.CsvExportView
 import fr.acinq.phoenix.android.payments.history.PaymentsHistoryView
-import fr.acinq.phoenix.android.payments.liquidity.RequestLiquidityView
+import fr.acinq.phoenix.android.payments.send.liquidity.RequestLiquidityView
 import fr.acinq.phoenix.android.payments.receive.ReceiveView
 import fr.acinq.phoenix.android.payments.send.SendView
 import fr.acinq.phoenix.android.services.NodeServiceState
@@ -114,8 +118,6 @@ import fr.acinq.phoenix.android.utils.extensions.findActivitySafe
 import fr.acinq.phoenix.android.utils.logger
 import fr.acinq.phoenix.data.BitcoinUnit
 import fr.acinq.phoenix.data.FiatCurrency
-import fr.acinq.phoenix.data.WalletPaymentId
-import fr.acinq.phoenix.data.walletPaymentId
 import fr.acinq.phoenix.legacy.utils.LegacyAppStatus
 import fr.acinq.phoenix.legacy.utils.LegacyPrefsDatastore
 import io.ktor.http.decodeURLPart
@@ -305,24 +307,26 @@ fun AppView(
                         }
                     }
                     composable(
-                        route = "${Screen.PaymentDetails.route}?direction={direction}&id={id}&fromEvent={fromEvent}",
+                        route = "${Screen.PaymentDetails.route}?id={id}&fromEvent={fromEvent}",
                         arguments = listOf(
-                            navArgument("direction") { type = NavType.LongType },
                             navArgument("id") { type = NavType.StringType },
                             navArgument("fromEvent") {
                                 type = NavType.BoolType
                                 defaultValue = false
                             }
                         ),
-                        deepLinks = listOf(navDeepLink { uriPattern = "phoenix:payments/{direction}/{id}" })
+                        deepLinks = listOf(navDeepLink { uriPattern = "phoenix:payments/{id}" })
                     ) {
-                        val direction = it.arguments?.getLong("direction")
-                        val id = it.arguments?.getString("id")
-
-                        val paymentId = if (id != null && direction != null) WalletPaymentId.create(direction, id) else null
+                        val paymentId = remember {
+                            try {
+                                UUID.fromString(it.arguments!!.getString("id")!!)
+                            } catch (e: Exception) {
+                                null
+                            }
+                        }
                         if (paymentId != null) {
-                            RequireStarted(walletState, nextUri = "phoenix:payments/${direction}/${id}") {
-                                log.debug("navigating to payment-details id=$id")
+                            RequireStarted(walletState, nextUri = "phoenix:payments/${id}") {
+                                log.debug("navigating to payment=$id")
                                 val fromEvent = it.arguments?.getBoolean("fromEvent") ?: false
                                 PaymentDetailsView(
                                     paymentId = paymentId,
@@ -533,25 +537,10 @@ fun AppView(
         }
     }
 
-    val isDataMigrationExpected by LegacyPrefsDatastore.getDataMigrationExpected(context).collectAsState(initial = null)
     val lastCompletedPayment by business.paymentsManager.lastCompletedPayment.collectAsState()
-    val userPrefs = userPrefs
-    val exchangeRates = fiatRates
     lastCompletedPayment?.let { payment ->
-        LaunchedEffect(key1 = payment.walletPaymentId()) {
-            try {
-                if (isDataMigrationExpected == false) {
-                    if (payment is IncomingPayment && payment.origin is IncomingPayment.Origin.Offer) {
-                        SystemNotificationHelper.notifyPaymentsReceived(
-                            context, userPrefs, paymentHash = payment.paymentHash, amount = payment.amount, rates = exchangeRates, isHeadless = false
-                        )
-                    } else {
-                        navigateToPaymentDetails(navController, id = payment.walletPaymentId(), isFromEvent = true)
-                    }
-                }
-            } catch (e: Exception) {
-                log.warn("failed to notify UI of completed payment: {}", e.localizedMessage)
-            }
+        LaunchedEffect(key1 = payment.id) {
+            navigateToPaymentDetails(navController, id = payment.id, isFromEvent = true)
         }
     }
 
@@ -561,9 +550,9 @@ fun AppView(
     }
 }
 
-fun navigateToPaymentDetails(navController: NavController, id: WalletPaymentId, isFromEvent: Boolean) {
+fun navigateToPaymentDetails(navController: NavController, id: UUID, isFromEvent: Boolean) {
     try {
-        navController.navigate("${Screen.PaymentDetails.route}?direction=${id.dbType.value}&id=${id.dbId}&fromEvent=${isFromEvent}")
+        navController.navigate("${Screen.PaymentDetails.route}?id=${id}&fromEvent=${isFromEvent}")
     } catch (_: Exception) { }
 }
 

diff --git a/phoenix-android/src/main/kotlin/fr/acinq/phoenix/android/PaymentsViewModel.kt b/phoenix-android/src/main/kotlin/fr/acinq/phoenix/android/PaymentsViewModel.kt
@@ -19,135 +19,59 @@ package fr.acinq.phoenix.android
 import androidx.lifecycle.ViewModel
 import androidx.lifecycle.ViewModelProvider
 import androidx.lifecycle.viewModelScope
-import fr.acinq.lightning.db.IncomingPayment
-import fr.acinq.lightning.db.LightningOutgoingPayment
-import fr.acinq.lightning.payment.OfferPaymentMetadata
-import fr.acinq.phoenix.data.ContactInfo
-import fr.acinq.phoenix.data.WalletPaymentFetchOptions
-import fr.acinq.phoenix.data.WalletPaymentId
+import fr.acinq.lightning.utils.UUID
 import fr.acinq.phoenix.data.WalletPaymentInfo
-import fr.acinq.phoenix.data.walletPaymentId
-import fr.acinq.phoenix.db.WalletPaymentOrderRow
-import fr.acinq.phoenix.managers.Connections
 import fr.acinq.phoenix.managers.ContactsManager
 import fr.acinq.phoenix.managers.PaymentsManager
 import fr.acinq.phoenix.managers.PaymentsPageFetcher
 import kotlinx.coroutines.CoroutineExceptionHandler
-import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.ExperimentalCoroutinesApi
-import kotlinx.coroutines.flow.*
+import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.SharingStarted
+import kotlinx.coroutines.flow.StateFlow
+import kotlinx.coroutines.flow.asStateFlow
+import kotlinx.coroutines.flow.mapLatest
+import kotlinx.coroutines.flow.stateIn
 import kotlinx.coroutines.launch
 import org.slf4j.LoggerFactory
 
-data class PaymentRowState(
-    val orderRow: WalletPaymentOrderRow,
-    val paymentInfo: WalletPaymentInfo?,
-    val contactInfo: ContactInfo?,
-)
-
 @OptIn(ExperimentalCoroutinesApi::class)
 class PaymentsViewModel(
     private val paymentsManager: PaymentsManager,
     private val contactsManager: ContactsManager,
 ) : ViewModel() {
 
     companion object {
-        /** How many payments should be fetched by the initial subscription. */
-        private const val initialPaymentsCount = 15
-
-        /** How many payments should be visible in the home view. */
-        const val latestPaymentsCount = 15
+        const val pageSize = 40
+        const val paymentsCountInHome = 10
     }
 
     private val log = LoggerFactory.getLogger(this::class.java)
 
-    private val _paymentsFlow = MutableStateFlow<Map<String, PaymentRowState>>(HashMap())
-    /**
-     * A flow of known payments. The key is a [WalletPaymentId], the value is the payments details
-     * which are basic at first, and then updated asynchronously (see [fetchPaymentDetails]).
-     *
-     * This flow is initialized by the view model, and then updated by [subscribeToPayments] which is
-     * called by the UI when needed (paging with scrolling, see the payments history view).
-     */
-    val paymentsFlow: StateFlow<Map<String, PaymentRowState>> = _paymentsFlow.asStateFlow()
+    private val _paymentsFlow = MutableStateFlow<Map<UUID, WalletPaymentInfo>>(HashMap())
+    val paymentsFlow: StateFlow<Map<UUID, WalletPaymentInfo>> = _paymentsFlow.asStateFlow()
 
-    /** A subset of [paymentsFlow] used in the Home view. */
-    val latestPaymentsFlow: StateFlow<List<PaymentRowState>> = paymentsFlow.mapLatest {
-        it.values.take(latestPaymentsCount.coerceAtMost(initialPaymentsCount)).toList()
-    }.stateIn(
+    private val homePageFetcher: PaymentsPageFetcher = paymentsManager.makePageFetcher()
+    val homePaymentsFlow = homePageFetcher.paymentsPage.mapLatest { it.rows }.stateIn(
         scope = viewModelScope,
-        started = SharingStarted.Lazily,
-        initialValue = emptyList()
+        started = SharingStarted.Eagerly,
+        initialValue = emptyList(),
     )
 
     private val paymentsPageFetcher: PaymentsPageFetcher = paymentsManager.makePageFetcher()
+    val paymentsPage = paymentsPageFetcher.paymentsPage
 
-    init {
-        paymentsPageFetcher.subscribeToAll(offset = 0, count = initialPaymentsCount)
 
-        // get details when a payment completes
-        viewModelScope.launch(CoroutineExceptionHandler { _, e ->
-            log.error("failed to collect last completed payment: ", e)
-        }) {
-            paymentsManager.lastCompletedPayment.filterNotNull().collect {
-                // a new row object must be built to get a fresh cache key for the payment fetcher
-                val row = WalletPaymentOrderRow(
-                    id = it.walletPaymentId(),
-                    createdAt = it.createdAt,
-                    completedAt = it.completedAt,
-                    metadataModifiedAt = null
-                )
-                fetchPaymentDetails(row)
-            }
-        }
+    init {
+        paymentsPageFetcher.subscribeToAll(offset = 0, count = pageSize)
+        homePageFetcher.subscribeToAll(offset = 0, count = paymentsCountInHome)
 
         // collect changes on the payments page that we subscribed to
         viewModelScope.launch(CoroutineExceptionHandler { _, e ->
             log.error("error when collecting payments-page items: ", e)
         }) {
             paymentsPageFetcher.paymentsPage.collect { page ->
-                viewModelScope.launch(Dispatchers.Default) {
-                    // We must rewrite the whole payments flow map to keep payments ordering.
-                    // Adding the diff would only push new elements to the bottom of the map.
-                    _paymentsFlow.value = page.rows.associate { newRow ->
-                        val paymentId = newRow.id.identifier
-                        val existingData = paymentsFlow.value[paymentId]
-                        // We look at the row to check if the payment has changed (the row contains timestamps)
-                        if (existingData?.orderRow != newRow) {
-                            paymentId to PaymentRowState(newRow, paymentInfo = null, contactInfo = null)
-                        } else {
-                            paymentId to existingData
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-    /** Fetches the details for a given payment and updates [paymentsFlow]. */
-    fun fetchPaymentDetails(row: WalletPaymentOrderRow) {
-        viewModelScope.launch(Dispatchers.Main) {
-            val paymentInfo = paymentsManager.fetcher.getPayment(row, WalletPaymentFetchOptions.Descriptions)
-            val contactInfo = when (val payment = paymentInfo?.payment) {
-                is IncomingPayment -> {
-                    val origin = payment.origin
-                    if (origin is IncomingPayment.Origin.Offer) {
-                        val metadata = origin.metadata
-                        if (metadata is OfferPaymentMetadata.V1) {
-                            contactsManager.getContactForPayerPubkey(metadata.payerKey)
-                        } else null
-                    } else null
-                }
-                is LightningOutgoingPayment -> {
-                    val details = payment.details
-                    if (details is LightningOutgoingPayment.Details.Blinded) {
-                        contactsManager.getContactForOffer(details.paymentRequest.invoiceRequest.offer)
-                    } else null
-                }
-                else -> null
-            }
-            if (paymentInfo != null) {
-                _paymentsFlow.value += (row.id.identifier to PaymentRowState(row, paymentInfo, contactInfo))
+                _paymentsFlow.value += page.rows.associateBy { it.payment.id }
             }
         }
     }

diff --git a/phoenix-android/src/main/kotlin/fr/acinq/phoenix/android/home/HomeBalance.kt b/phoenix-android/src/main/kotlin/fr/acinq/phoenix/android/home/HomeBalance.kt
@@ -74,7 +74,6 @@ fun HomeBalance(
     balance: MilliSatoshi?,
     swapInBalance: WalletBalance,
     finalWalletBalance: Satoshi,
-    unconfirmedChannelsBalance: MilliSatoshi,
     onNavigateToSwapInWallet: () -> Unit,
     onNavigateToFinalWallet: () -> Unit,
     balanceDisplayMode: HomeAmountDisplayMode,
@@ -106,22 +105,21 @@ fun HomeBalance(
                     }
                 }
             )
-            OnChainBalance(swapInBalance, unconfirmedChannelsBalance, finalWalletBalance, onNavigateToSwapInWallet, onNavigateToFinalWallet, balanceDisplayMode)
+            OnChainBalance(swapInBalance, finalWalletBalance, onNavigateToSwapInWallet, onNavigateToFinalWallet, balanceDisplayMode)
         }
     }
 }
 
 @Composable
 private fun OnChainBalance(
     swapInBalance: WalletBalance,
-    pendingChannelsBalance: MilliSatoshi,
     finalWalletBalance: Satoshi,
     onNavigateToSwapInWallet: () -> Unit,
     onNavigateToFinalWallet: () -> Unit,
     balanceDisplayMode: HomeAmountDisplayMode,
 ) {
     var showOnchainDialog by remember { mutableStateOf(false) }
-    val availableOnchainBalance = swapInBalance.total.toMilliSatoshi() + pendingChannelsBalance + finalWalletBalance.toMilliSatoshi()
+    val availableOnchainBalance = swapInBalance.total.toMilliSatoshi() + finalWalletBalance.toMilliSatoshi()
 
     if (availableOnchainBalance > 0.msat) {
         val nextSwapTimeout by business.peerManager.swapInNextTimeout.collectAsState(initial = null)
@@ -167,7 +165,7 @@ private fun OnChainBalance(
                             verticalArrangement = Arrangement.spacedBy(12.dp),
                         ) {
                             // 1) funds being confirmed (swap deposits or LN channels waiting for confirmation
-                            val fundsBeingConfirmed = (swapInBalance.unconfirmed + swapInBalance.weaklyConfirmed).toMilliSatoshi() + pendingChannelsBalance
+                            val fundsBeingConfirmed = (swapInBalance.unconfirmed + swapInBalance.weaklyConfirmed).toMilliSatoshi()
                             if (fundsBeingConfirmed > 0.msat) {
                                 OnChainBalanceEntry(
                                     label = stringResource(id = R.string.home_swapin_confirming_title),