feat: Cache transparent UTXOs

Whenever an `Announcement` is parsed as transparent transaction info, we obtain
information about the UTXOs involved in that transaction. It would be nice to
display this information when we look up the UTXO, but there is no way to query
it from neptune-core.

This commit caches that information in order to display it.

Specifically, it extends `AppState` with a (smart pointer to) vector of
`TransparentUtxoInfo` (a new type). The vector, or its existing entries, are
extended with new information when it becomes available.

At this point the vector is rather silly because it is not persisted, and the
app reboots more often than not, resulting in a clear cache. A future commit
will persist this data.

src/html/page/announcement.rs

@@ -4,6 +4,7 @@ use crate::http_util::rpc_method_err;
 use crate::model::announcement_selector::AnnouncementSelector;
 use crate::model::announcement_type::AnnouncementType;
 use crate::model::app_state::AppState;
+use crate::model::transparent_utxo_tuple::TransparentUtxoTuple;
 use axum::extract::rejection::PathRejection;
 use axum::extract::Path;
 use axum::extract::State;
@@ -15,6 +16,8 @@ use neptune_cash::api::export::BlockHeight;
 use neptune_cash::prelude::tasm_lib::prelude::Digest;
 use neptune_cash::prelude::triton_vm::prelude::BFieldCodec;
 use neptune_cash::prelude::twenty_first::tip5::Tip5;
+use neptune_cash::util_types::mutator_set::addition_record::AdditionRecord;
+use std::collections::HashMap;
 use std::sync::Arc;
 use tarpc::context;
 
@@ -32,6 +35,7 @@ pub async fn announcement_page(
         block_hash: Digest,
         block_height: BlockHeight,
         announcement_type: AnnouncementType,
+        addition_record_indices: HashMap<AdditionRecord, Option<u64>>,
     }
 
     let state = &state_rw.load();
@@ -73,6 +77,58 @@ pub async fn announcement_page(
         .clone();
     let announcement_type = AnnouncementType::parse(announcement);
 
+    let mut addition_record_indices = HashMap::<AdditionRecord, Option<u64>>::new();
+    if let AnnouncementType::TransparentTxInfo(tx_info) = announcement_type.clone() {
+        let addition_records = tx_info
+            .outputs
+            .iter()
+            .map(|output| output.addition_record())
+            .collect::<Vec<_>>();
+        addition_record_indices = state.rpc_client.addition_record_indices_for_block(context::current(), state.token(), block_selector, &addition_records).await
+                .map_err(|e| not_found_html_response(state, Some(e.to_string())))?
+                .map_err(rpc_method_err)?
+                .expect(
+                    "block guaranteed to exist because we got here; getting its announcements should work",
+                );
+
+        let mut transparent_utxos_cache = state.transparent_utxos_cache.lock().await;
+
+        for input in &tx_info.inputs {
+            let addition_record = input.addition_record();
+            if let Some(existing_entry) = transparent_utxos_cache
+                .iter_mut()
+                .find(|tu| tu.addition_record() == addition_record)
+            {
+                existing_entry.upgrade_with_transparent_input(input, block_hash);
+            } else {
+                tracing::info!("Adding transparent UTXO (input side) to cache.");
+                transparent_utxos_cache.push(TransparentUtxoTuple::new_from_transparent_input(
+                    input, block_hash,
+                ));
+            }
+        }
+
+        for output in &tx_info.outputs {
+            let addition_record = output.addition_record();
+            if let Some(existing_entry) = transparent_utxos_cache
+                .iter_mut()
+                .find(|tu| tu.addition_record() == addition_record)
+            {
+                existing_entry.upgrade_with_transparent_output(block_hash);
+            } else {
+                tracing::info!("Adding transparent UTXO (output side) to cache.");
+                transparent_utxos_cache.push(TransparentUtxoTuple::new_from_transparent_output(
+                    output,
+                    addition_record_indices
+                        .get(&addition_record)
+                        .cloned()
+                        .unwrap_or(None),
+                    block_hash,
+                ));
+            }
+        }
+    }
+
     let header = HeaderHtml { state };
 
     let utxo_page = AnnouncementHtmlPage {
@@ -82,6 +138,7 @@ pub async fn announcement_page(
         block_height,
         num_announcements,
         announcement_type,
+        addition_record_indices,
     };
     Ok(Html(utxo_page.to_string()))
 }

src/html/page/utxo.rs

@@ -2,6 +2,7 @@ use crate::html::component::header::HeaderHtml;
 use crate::html::page::not_found::not_found_html_response;
 use crate::http_util::rpc_method_err;
 use crate::model::app_state::AppState;
+use crate::model::transparent_utxo_tuple::TransparentUtxoTuple;
 use axum::extract::rejection::PathRejection;
 use axum::extract::Path;
 use axum::extract::State;
@@ -9,6 +10,7 @@ use axum::response::Html;
 use axum::response::Response;
 use html_escaper::Escape;
 use html_escaper::Trusted;
+use neptune_cash::api::export::Tip5;
 use neptune_cash::prelude::tasm_lib::prelude::Digest;
 use std::sync::Arc;
 use tarpc::context;
@@ -24,16 +26,18 @@ pub async fn utxo_page(
         header: HeaderHtml<'a>,
         index: u64,
         digest: Digest,
+        transparent_utxo_info: Option<TransparentUtxoTuple>,
     }
 
     let state = &state_rw.load();
+    let cache = state.transparent_utxos_cache.clone();
 
     let Path(index) =
         index_maybe.map_err(|e| not_found_html_response(state, Some(e.to_string())))?;
 
     let digest = match state
         .rpc_client
-        .utxo_digest(context::current(), state.token(), index)
+        .utxo_digest(context::current(), state.token(), index, cache)
         .await
         .map_err(|e| not_found_html_response(state, Some(e.to_string())))?
         .map_err(rpc_method_err)?
@@ -49,10 +53,19 @@ pub async fn utxo_page(
 
     let header = HeaderHtml { state };
 
+    let transparent_utxo_info = state
+        .transparent_utxos_cache
+        .lock()
+        .await
+        .iter()
+        .find(|tu| tu.aocl_leaf_index().is_some_and(|li| li == index))
+        .cloned();
+
     let utxo_page = UtxoHtmlPage {
         index,
         header,
         digest,
+        transparent_utxo_info,
     };
     Ok(Html(utxo_page.to_string()))
 }

src/model/app_state.rs

@@ -1,4 +1,5 @@
 use crate::model::config::Config;
+use crate::model::transparent_utxo_tuple::TransparentUtxoTuple;
 use crate::neptune_rpc;
 use anyhow::Context;
 use arc_swap::ArcSwap;
@@ -8,6 +9,7 @@ use neptune_cash::models::blockchain::block::block_selector::BlockSelector;
 use neptune_cash::prelude::twenty_first::tip5::Digest;
 use neptune_cash::rpc_auth;
 use std::sync::Arc;
+use tokio::sync::Mutex;
 
 #[derive(Debug, Clone)]
 pub struct AppStateInner {
@@ -15,6 +17,12 @@ pub struct AppStateInner {
     pub config: Config,
     pub rpc_client: neptune_rpc::AuthenticatedClient,
     pub genesis_digest: Digest,
+
+    /// Whenever an announcement of type transparent transaction info is fetched
+    /// from the RPC endpoint, we learn information about UTXOs. Since we expect
+    /// transparent transactions to be rare, it is okay to cache this in RAM
+    /// instead of storing it on disk.
+    pub transparent_utxos_cache: Arc<Mutex<Vec<TransparentUtxoTuple>>>,
 }
 
 impl AppStateInner {
@@ -26,6 +34,12 @@ impl AppStateInner {
 #[derive(Clone)]
 pub struct AppState(Arc<ArcSwap<AppStateInner>>);
 
+impl AppState {
+    fn new(app_state_inner: AppStateInner) -> Self {
+        Self(Arc::new(ArcSwap::from_pointee(app_state_inner)))
+    }
+}
+
 impl std::ops::Deref for AppState {
     type Target = Arc<ArcSwap<AppStateInner>>;
 
@@ -34,24 +48,6 @@ impl std::ops::Deref for AppState {
     }
 }
 
-impl From<(Network, Config, neptune_rpc::AuthenticatedClient, Digest)> for AppState {
-    fn from(
-        (network, config, rpc_client, genesis_digest): (
-            Network,
-            Config,
-            neptune_rpc::AuthenticatedClient,
-            Digest,
-        ),
-    ) -> Self {
-        Self(Arc::new(ArcSwap::from_pointee(AppStateInner {
-            network,
-            config,
-            rpc_client,
-            genesis_digest,
-        })))
-    }
-}
-
 impl AppState {
     pub async fn init() -> Result<Self, anyhow::Error> {
         let rpc_client = neptune_rpc::gen_authenticated_rpc_client()
@@ -68,12 +64,13 @@ impl AppState {
             .with_context(|| "Failed calling neptune-core api method: block_digest")?
             .with_context(|| "neptune-core failed to provide a genesis block")?;
 
-        Ok(Self::from((
+        Ok(AppState::new(AppStateInner {
-            rpc_client.network,
+            network: rpc_client.network,
-            Config::parse(),
+            config: Config::parse(),
             rpc_client,
             genesis_digest,
-        )))
+            transparent_utxos_cache: Arc::new(Mutex::new(vec![])),
+        }))
     }
 
     /// Sets the rpc_client
@@ -95,6 +92,7 @@ impl AppState {
             rpc_client,
             config: inner.config.clone(),
             genesis_digest: inner.genesis_digest,
+            transparent_utxos_cache: Arc::new(Mutex::new(vec![])),
         };
         self.0.store(Arc::new(new_inner));
     }

src/model/mod.rs

@@ -4,3 +4,4 @@ pub mod app_state;
 pub mod block_selector_extended;
 pub mod config;
 pub mod height_or_digest;
+pub mod transparent_utxo_tuple;

src/neptune_rpc.rs

@@ -1,6 +1,7 @@
 use crate::alert_email;
 use crate::model::app_state::AppState;
 use crate::model::config::Config;
+use crate::model::transparent_utxo_tuple::TransparentUtxoTuple;
 use anyhow::Context;
 use chrono::DateTime;
 use chrono::TimeDelta;
@@ -17,11 +18,15 @@ use neptune_cash::rpc_auth;
 use neptune_cash::rpc_server::error::RpcError;
 use neptune_cash::rpc_server::RPCClient;
 use neptune_cash::rpc_server::RpcResult;
+use neptune_cash::util_types::mutator_set::addition_record::AdditionRecord;
+use std::collections::HashMap;
 use std::net::Ipv4Addr;
 use std::net::SocketAddr;
+use std::sync::Arc;
 use tarpc::client;
 use tarpc::context;
 use tarpc::tokio_serde::formats::Json as RpcJson;
+use tokio::sync::Mutex;
 use tracing::{debug, info, warn};
 
 #[cfg(feature = "mock")]
@@ -92,8 +97,32 @@ impl AuthenticatedClient {
         ctx: ::tarpc::context::Context,
         token: rpc_auth::Token,
         leaf_index: u64,
+        _transparent_utxos_cache: Arc<Mutex<Vec<TransparentUtxoTuple>>>,
     ) -> ::core::result::Result<RpcResult<Option<Digest>>, ::tarpc::client::RpcError> {
-        self.client.utxo_digest(ctx, token, leaf_index).await
+        let rpc_result = self.client.utxo_digest(ctx, token, leaf_index).await;
+
+        if let Ok(Ok(Some(_))) = rpc_result {
+            return rpc_result;
+        }
+
+        // if MOCK environment variable is set and feature is enabled,
+        // imagine some mock UTXO info
+        #[cfg(feature = "mock")]
+        if std::env::var(MOCK_KEY).is_ok() {
+            let cache = _transparent_utxos_cache.lock().await;
+            tracing::warn!(
+                "RPC query failed and MOCK flag set, so seeing if we can return a cached utxo; cache has {} objects", cache.len()
+            );
+            if let Some(entry) = cache
+                .iter()
+                .find(|tu| tu.aocl_leaf_index().is_some_and(|li| li == leaf_index))
+            {
+                tracing::warn!("returning a cached utxo");
+                return Ok(Ok(Some(entry.addition_record().canonical_commitment)));
+            }
+        }
+
+        rpc_result
     }
 
     /// Intercept and relay call to [`RPCClient::announcements_in_block`]
@@ -123,7 +152,9 @@ impl AuthenticatedClient {
             use rand::rngs::StdRng;
             use rand::Rng;
             use rand::SeedableRng;
-            tracing::warn!("RPC query failed and MOCK flag set, so returning an imagined block");
+            tracing::warn!(
+                "RPC query failed and MOCK flag set, so returning an imagined announcement"
+            );
             let mut hasher = Hasher::new();
             hasher.update(&block_selector.to_string().bytes().collect::<Vec<_>>());
             let mut rng = StdRng::from_seed(*hasher.finalize().as_bytes());
@@ -156,6 +187,67 @@ impl AuthenticatedClient {
         // otherwise, return the original error
         rpc_result
     }
+
+    /// Intercept and relay call to
+    /// [`RPCClient::addition_record_indices_for_block`].
+    ///
+    /// Also take an extra argument for mocking purposes.
+    pub async fn addition_record_indices_for_block(
+        &self,
+        ctx: ::tarpc::context::Context,
+        token: rpc_auth::Token,
+        block_selector: BlockSelector,
+        _addition_records: &[AdditionRecord],
+    ) -> ::core::result::Result<
+        RpcResult<Option<HashMap<AdditionRecord, Option<u64>>>>,
+        ::tarpc::client::RpcError,
+    > {
+        let rpc_result = self
+            .client
+            .addition_record_indices_for_block(ctx, token, block_selector)
+            .await;
+
+        // if the RPC call was successful, return that
+        if let Ok(Ok(Some(_))) = rpc_result {
+            return rpc_result;
+        }
+
+        // if MOCK environment variable is set and feature is enabled,
+        // imagine some mock hash map
+        #[cfg(feature = "mock")]
+        if std::env::var(MOCK_KEY).is_ok() {
+            use blake3::Hasher;
+            use rand::rngs::StdRng;
+            use rand::Rng;
+            use rand::SeedableRng;
+            tracing::warn!(
+                "RPC query failed and MOCK flag set, so returning an imagined addition records"
+            );
+            let mut hasher = Hasher::new();
+            hasher.update(&block_selector.to_string().bytes().collect::<Vec<_>>());
+            let mut rng = StdRng::from_seed(*hasher.finalize().as_bytes());
+
+            let aocl_offset = rng.random::<u64>() >> 1;
+            let addition_record_indices = _addition_records
+                .iter()
+                .enumerate()
+                .map(|(i, ar)| {
+                    (
+                        *ar,
+                        if rng.random_bool(0.5_f64) {
+                            Some(i as u64 + aocl_offset)
+                        } else {
+                            None
+                        },
+                    )
+                })
+                .collect::<HashMap<_, _>>();
+            return Ok(Ok(Some(addition_record_indices)));
+        }
+
+        // otherwise, return the original error
+        rpc_result
+    }
 }
 
 /// generates RPCClient, for querying neptune-core RPC server.

src/rpc/utxo_digest.rs

@@ -18,9 +18,10 @@ pub async fn utxo_digest(
     State(state): State<Arc<AppState>>,
 ) -> Result<Json<Digest>, impl IntoResponse> {
     let s = state.load();
+    let cache = s.transparent_utxos_cache.clone();
     match s
         .rpc_client
-        .utxo_digest(context::current(), s.token(), index)
+        .utxo_digest(context::current(), s.token(), index, cache)
         .await
         .map_err(rpc_err)?
         .map_err(rpc_method_err)?

templates/web/html/page/announcement.html

@@ -87,7 +87,13 @@
                         <td>
                             <details>
                                 <summary>
+                                    {% if let Some(Some(aocl_leaf_index)) =
+                                    self.addition_record_indices.get(&output.addition_record()) { %}
+                                    <a
+                                        href='/utxo/{{aocl_leaf_index}}'>{{output.addition_record().canonical_commitment.to_hex()}}</a>
+                                    {% } else { %}
                                     {{output.addition_record().canonical_commitment.to_hex()}}
+                                    {% } %}
                                 </summary>
                                 <table>
                                     <tr>

templates/web/html/page/utxo.html

@@ -2,43 +2,92 @@
 
 <head>
     <title>{{self.header.state.config.site_name}}: Utxo {{self.index}}</title>
-{{html_escaper::Trusted(include_str!( concat!(env!("CARGO_MANIFEST_DIR"), "/templates/web/html/components/head.html")))}}
+    {{html_escaper::Trusted(include_str!( concat!(env!("CARGO_MANIFEST_DIR"),
+    "/templates/web/html/components/head.html")))}}
 </head>
 
 <body>
-{{Trusted(self.header.to_string())}}
+    {{Trusted(self.header.to_string())}}
 
-<main class="container">
+    <main class="container">
 
-<article>
+        <article>
-<summary>
+            <summary>
                 <span class="tooltip">ⓘ
                     <span class="tooltiptext">
-            UTXO = Unspent Transaction Output.  It represents an output of transaction A which can also be an input to transaction B.
+                        UTXO = Unspent Transaction Output. It represents an output of transaction A which can also be an
+                        input to transaction B.
                     </span>
                 </span>
                 UTXO Information
-</summary>
+            </summary>
             <table class="striped">
                 <tr>
-            <td>Index</td>
+                    <td>AOCL Leaf Index</td>
                     <td>{{self.index}}</td>
                 </tr>
                 <tr>
-            <td>Digest</td>
+                    <td>Addition Record</td>
                     <td>{{self.digest.to_hex()}}</td>
                 </tr>
             </table>
-</article>
+        </article>
 
-<article>
+        {% if let Some(utxo_info) = &self.transparent_utxo_info { %}
+        <article>
+            <summary>
+                <span class="tooltip">ⓘ
+                    <span class="tooltiptext">
+                        UTXOs consumed or produced by a transparent transaction disclose the information they otherwise
+                        hide.
+                    </span>
+                </span>
+                Transparent UTXO Information
+            </summary>
+            <table class="striped">
+                <tr>
+                    <td>UTXO Digest:</td>
+                    <td><span class="mono">{{Tip5::hash(&utxo_info.utxo()).to_hex()}}</span></td>
+                </tr>
+                <tr>
+                    <td>Sender Randomness:</td>
+                    <td><span class="mono">{{utxo_info.sender_randomness().to_hex()}}</span></td>
+                </tr>
+                <tr>
+                    <td>Receiver Digest</td>
+                    <td><span class="mono">{{utxo_info.receiver_digest().to_hex()}}</span></td>
+                </tr>
+                {% if let Some(receiver_preimage) = utxo_info.receiver_preimage() { %}
+                <tr>
+                    <td>Receiver Preimage</td>
+                    <td><span class="mono">{{receiver_preimage.to_hex()}}</span></td>
+                </tr>
+                {% } %}
+                {% if utxo_info.utxo().has_native_currency() { %}
+                <tr>
+                    <td>Amount:</td>
+                    <td>{{utxo_info.utxo().get_native_currency_amount().display_n_decimals(5)}} NPT</td>
+                </tr>
+                {% } %}
+                {% if let Some(release_date) = utxo_info.utxo().release_date() { %}
+                <tr>
+                    <td>Time-Locked Until</td>
+                    <td><span class="mono">{{release_date.standard_format()}}</span></td>
+                </tr>
+                {% } %}
+            </table>
+        </article>
+        {% } %}
+
+        <article>
             <p>
                 <a href="/">Home</a>
                 | <a href='/block/genesis'>Genesis</a>
                 | <a href='/block/tip'>Tip</a>
             </p>
-</article>
+        </article>
 
-</main>
+    </main>
 </body>
+
 </html>