tempo_commonware_node/
storage.rs

1//! This module defines consensus archive formats
2
3use std::time::Instant;
4
5use commonware_consensus::simplex::{scheme::bls12381_threshold::vrf::Scheme, types::Finalization};
6use commonware_cryptography::{
7    bls12381::primitives::variant::MinSig, certificate::Scheme as _, ed25519::PublicKey,
8};
9use commonware_runtime::{BufferPooler, Clock, Metrics, Spawner, Storage, buffer::paged::CacheRef};
10use commonware_storage::archive::immutable;
11use commonware_utils::{NZU16, NZU64, NZUsize};
12use tracing::info;
13
14use crate::{
15    config::BLOCKS_FREEZER_TABLE_INITIAL_SIZE_BYTES,
16    consensus::{Digest, block::Block},
17};
18
19const FINALIZATIONS_BY_HEIGHT: &str = "finalizations-by-height";
20const FINALIZED_BLOCKS: &str = "finalized_blocks";
21
22const IMMUTABLE_ITEMS_PER_SECTION: std::num::NonZeroU64 = NZU64!(262_144);
23const FREEZER_TABLE_RESIZE_FREQUENCY: u8 = 4;
24const FREEZER_TABLE_RESIZE_CHUNK_SIZE: u32 = 2u32.pow(16); // 64KB chunks
25const FREEZER_VALUE_TARGET_SIZE: u64 = 1024 * 1024 * 1024; // 1GB
26const FREEZER_VALUE_COMPRESSION: Option<u8> = Some(3);
27
28pub(crate) const REPLAY_BUFFER: std::num::NonZeroUsize = NZUsize!(8 * 1024 * 1024); // 8MB
29pub(crate) const WRITE_BUFFER: std::num::NonZeroUsize = NZUsize!(1024 * 1024); // 1MB
30pub(crate) const PRUNABLE_ITEMS_PER_SECTION: std::num::NonZeroU64 = NZU64!(4_096);
31pub(crate) const MAX_REPAIR: std::num::NonZeroUsize = NZUsize!(20);
32pub(crate) const BUFFER_POOL_PAGE_SIZE: std::num::NonZeroU16 = NZU16!(4_096); // 4KB
33pub(crate) const BUFFER_POOL_CAPACITY: std::num::NonZeroUsize = NZUsize!(8_192); // 32MB (8k page slots)
34
35pub(crate) async fn init_finalizations_archive<TContext>(
36    context: &TContext,
37    partition_prefix: &str,
38    page_cache: CacheRef,
39) -> Result<
40    immutable::Archive<TContext, Digest, Finalization<Scheme<PublicKey, MinSig>, Digest>>,
41    commonware_storage::archive::Error,
42>
43where
44    TContext: Clock + Metrics + Spawner + Storage + BufferPooler + Clone + Send + 'static,
45{
46    let start = Instant::now();
47    let archive = immutable::Archive::init(
48        context.with_label("finalizations_by_height"),
49        immutable::Config {
50            metadata_partition: format!("{partition_prefix}-{FINALIZATIONS_BY_HEIGHT}-metadata"),
51            freezer_table_partition: format!(
52                "{partition_prefix}-{FINALIZATIONS_BY_HEIGHT}-freezer-table"
53            ),
54            freezer_table_initial_size: BLOCKS_FREEZER_TABLE_INITIAL_SIZE_BYTES,
55            freezer_table_resize_frequency: FREEZER_TABLE_RESIZE_FREQUENCY,
56            freezer_table_resize_chunk_size: FREEZER_TABLE_RESIZE_CHUNK_SIZE,
57            freezer_key_partition: format!(
58                "{partition_prefix}-{FINALIZATIONS_BY_HEIGHT}-freezer-key"
59            ),
60            freezer_key_page_cache: page_cache.clone(),
61            freezer_value_partition: format!(
62                "{partition_prefix}-{FINALIZATIONS_BY_HEIGHT}-freezer-value"
63            ),
64            freezer_value_target_size: FREEZER_VALUE_TARGET_SIZE,
65            freezer_value_compression: FREEZER_VALUE_COMPRESSION,
66            ordinal_partition: format!("{partition_prefix}-{FINALIZATIONS_BY_HEIGHT}-ordinal"),
67            items_per_section: IMMUTABLE_ITEMS_PER_SECTION,
68            codec_config: Scheme::<PublicKey, MinSig>::certificate_codec_config_unbounded(),
69            replay_buffer: REPLAY_BUFFER,
70            freezer_key_write_buffer: WRITE_BUFFER,
71            freezer_value_write_buffer: WRITE_BUFFER,
72            ordinal_write_buffer: WRITE_BUFFER,
73        },
74    )
75    .await;
76
77    info!(
78        elapsed = %tempo_telemetry_util::display_duration(start.elapsed()),
79        "restored finalizations by height archive"
80    );
81
82    archive
83}
84
85/// Initialize the finalized blocks archive with the standard format.
86pub(crate) async fn init_finalized_blocks_archive<TContext>(
87    context: &TContext,
88    partition_prefix: &str,
89    page_cache: CacheRef,
90) -> Result<immutable::Archive<TContext, Digest, Block>, commonware_storage::archive::Error>
91where
92    TContext: Clock + Metrics + Spawner + Storage + BufferPooler + Clone + Send + 'static,
93{
94    let start = Instant::now();
95    let archive = immutable::Archive::init(
96        context.with_label("finalized_blocks"),
97        immutable::Config {
98            metadata_partition: format!("{partition_prefix}-{FINALIZED_BLOCKS}-metadata"),
99            freezer_table_partition: format!("{partition_prefix}-{FINALIZED_BLOCKS}-freezer-table"),
100            freezer_table_initial_size: BLOCKS_FREEZER_TABLE_INITIAL_SIZE_BYTES,
101            freezer_table_resize_frequency: FREEZER_TABLE_RESIZE_FREQUENCY,
102            freezer_table_resize_chunk_size: FREEZER_TABLE_RESIZE_CHUNK_SIZE,
103            freezer_key_partition: format!("{partition_prefix}-{FINALIZED_BLOCKS}-freezer-key"),
104            freezer_key_page_cache: page_cache.clone(),
105            freezer_value_partition: format!("{partition_prefix}-{FINALIZED_BLOCKS}-freezer-value"),
106            freezer_value_target_size: FREEZER_VALUE_TARGET_SIZE,
107            freezer_value_compression: FREEZER_VALUE_COMPRESSION,
108            ordinal_partition: format!("{partition_prefix}-{FINALIZED_BLOCKS}-ordinal"),
109            items_per_section: IMMUTABLE_ITEMS_PER_SECTION,
110            codec_config: (),
111            replay_buffer: REPLAY_BUFFER,
112            freezer_key_write_buffer: WRITE_BUFFER,
113            freezer_value_write_buffer: WRITE_BUFFER,
114            ordinal_write_buffer: WRITE_BUFFER,
115        },
116    )
117    .await;
118
119    info!(elapsed = %tempo_telemetry_util::display_duration(start.elapsed()), "restored finalized blocks archive");
120
121    archive
122}
tempo_commonware_node/storage.rs

tempo_commonware_node/
storage.rs
