Skip to content

Commit bdce84b

Browse files
committed
WIP: Trust Quorum: Start implementing Node
This code builds upon #7859, which itself builds upon #7891. Those need to be merged in order first. A `Node` is the sans-io entity driving the trust quorum protocol. This PR starts the work of creating the `Node` type and coordinating an initial configuration. There's a property based test that generates an initial `ReconfigureMsg` that is then used to call `Node::coordinate_reconfiguration`, which will internally setup a `CoordinatorState` and create a bunch of messages to be sent. We verify that a new `PersistentState` is returned and that messages are sent. This needs a bit more documentation and testing, so it's still WIP.
1 parent bc07606 commit bdce84b

File tree

10 files changed

+941
-20
lines changed

10 files changed

+941
-20
lines changed

Diff for: Cargo.lock

+2
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Diff for: bootstore/src/schemes/v0/mod.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ pub use messages::{
2626
pub use peer::{Config, Node, NodeHandle, NodeRequestError, Status};
2727
pub use request_manager::{RequestManager, TrackableRequest};
2828
pub use share_pkg::{LearnedSharePkg, SharePkg, SharePkgCommon, create_pkgs};
29-
pub use storage::NetworkConfig;
29+
pub use storage::{NetworkConfig, PersistentFsmState};
3030

3131
/// The current version of supported messages within the v0 scheme
3232
///

Diff for: trust-quorum/Cargo.toml

+2
Original file line numberDiff line numberDiff line change
@@ -32,5 +32,7 @@ zeroize.workspace = true
3232
omicron-workspace-hack.workspace = true
3333

3434
[dev-dependencies]
35+
assert_matches.workspace = true
36+
omicron-test-utils.workspace = true
3537
proptest.workspace = true
3638
test-strategy.workspace = true

Diff for: trust-quorum/src/configuration.rs

+16-12
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,9 @@
55
//! A configuration of a trust quroum at a given epoch
66
77
use crate::crypto::{EncryptedRackSecret, RackSecret, Salt, Sha3_256Digest};
8-
use crate::{Epoch, PlatformId, ReconfigureMsg, Threshold};
9-
use gfss::shamir::SplitError;
8+
use crate::validators::ValidatedReconfigureMsg;
9+
use crate::{Epoch, PlatformId, Threshold};
10+
use gfss::shamir::{SecretShares, SplitError};
1011
use omicron_uuid_kinds::RackUuid;
1112
use secrecy::ExposeSecret;
1213
use serde::{Deserialize, Serialize};
@@ -57,8 +58,8 @@ impl Configuration {
5758
/// the last committed epoch.
5859
pub fn new(
5960
coordinator: PlatformId,
60-
reconfigure_msg: &ReconfigureMsg,
61-
) -> Result<Configuration, ConfigurationError> {
61+
reconfigure_msg: &ValidatedReconfigureMsg,
62+
) -> Result<(Configuration, SecretShares), ConfigurationError> {
6263
let rack_secret = RackSecret::new();
6364
let shares = rack_secret.split(
6465
reconfigure_msg.threshold,
@@ -82,14 +83,17 @@ impl Configuration {
8283
.zip(share_digests)
8384
.collect();
8485

85-
Ok(Configuration {
86-
rack_id: reconfigure_msg.rack_id,
87-
epoch: reconfigure_msg.epoch,
88-
coordinator,
89-
members,
90-
threshold: reconfigure_msg.threshold,
91-
previous_configuration: None,
92-
})
86+
Ok((
87+
Configuration {
88+
rack_id: reconfigure_msg.rack_id,
89+
epoch: reconfigure_msg.epoch,
90+
coordinator,
91+
members,
92+
threshold: reconfigure_msg.threshold,
93+
previous_configuration: None,
94+
},
95+
shares,
96+
))
9397
}
9498
}
9599

Diff for: trust-quorum/src/coordinator_state.rs

+195
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
// This Source Code Form is subject to the terms of the Mozilla Public
2+
// License, v. 2.0. If a copy of the MPL was not distributed with this
3+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
4+
5+
//! State of a reconfiguration coordinator inside a [`crate::Node`]
6+
7+
use crate::crypto::{LrtqShare, Sha3_256Digest, ShareDigestLrtq};
8+
use crate::messages::{PeerMsg, PrepareMsg};
9+
use crate::validators::{ReconfigurationError, ValidatedReconfigureMsg};
10+
use crate::{Configuration, Envelope, Epoch, PlatformId};
11+
use gfss::shamir::{SecretShares, Share};
12+
use secrecy::ExposeSecret;
13+
use std::collections::{BTreeMap, BTreeSet};
14+
use std::time::Instant;
15+
16+
/// The state of a reconfiguration coordinator.
17+
///
18+
/// A coordinator can be any trust quorum node that is a member of both the old
19+
/// and new group. The coordinator is chosen by Nexus for a given epoch when a
20+
/// trust quorum reconfiguration is triggered. Reconfiguration is only performed
21+
/// when the control plane is up, as we use Nexus to persist prepares and ensure
22+
/// commitment happens, even if the system crashes while committing. If a
23+
/// rack crash (such as a power outage) occurs before nexus is informed of the
24+
/// prepares, nexus will skip the epoch and start a new reconfiguration. This
25+
/// allows progress to always be made with a full linearization of epochs.
26+
///
27+
/// We allow some unused fields before we complete the coordination code
28+
#[allow(unused)]
29+
pub struct CoordinatorState {
30+
/// A copy of the platform_id from [`Node`] purely for ergonomics
31+
platform_id: PlatformId,
32+
33+
/// When the reconfiguration started
34+
pub start_time: Instant,
35+
36+
/// A copy of the message used to start this reconfiguration
37+
pub reconfigure_msg: ValidatedReconfigureMsg,
38+
39+
/// Configuration that will get persisted inside a `Prepare` message in a
40+
/// `Node`s `PersistentState`, once it is possible to create the Prepare.
41+
pub configuration: Configuration,
42+
43+
/// What is the coordinator currently doing
44+
pub op: CoordinatorOperation,
45+
46+
/// When to resend prepare messages next
47+
pub retry_deadline: Instant,
48+
}
49+
50+
impl CoordinatorState {
51+
/// Start coordinating a reconfiguration for a brand new trust quorum
52+
///
53+
/// Return the newly constructed `CoordinatorState` along with this node's
54+
/// `PrepareMsg` so that it can be persisted.
55+
///
56+
/// Precondition: This node must be a member of the new configuration
57+
/// or this method will panic. This is ensured as part of passing in a
58+
/// `ValidatedReconfigureMsg`.
59+
pub fn new_uninitialized(
60+
my_platform_id: PlatformId,
61+
now: Instant,
62+
msg: ValidatedReconfigureMsg,
63+
) -> Result<(CoordinatorState, PrepareMsg), ReconfigurationError> {
64+
// Create a configuration for this epoch
65+
let (config, shares) =
66+
Configuration::new(my_platform_id.clone(), &msg)?;
67+
68+
let shares_by_member: BTreeMap<PlatformId, Share> = config
69+
.members
70+
.keys()
71+
.cloned()
72+
.zip(shares.shares.expose_secret().iter().cloned())
73+
.collect();
74+
75+
let mut prepares = BTreeMap::new();
76+
let mut my_prepare_msg: Option<PrepareMsg> = None;
77+
for (platform_id, share) in shares_by_member.into_iter() {
78+
let prepare_msg = PrepareMsg { config: config.clone(), share };
79+
if platform_id == my_platform_id {
80+
// The prepare message to add to our `PersistentState`
81+
my_prepare_msg = Some(prepare_msg);
82+
} else {
83+
// Create a message that requires sending
84+
prepares.insert(platform_id, prepare_msg);
85+
}
86+
}
87+
let op = CoordinatorOperation::Prepare {
88+
prepares,
89+
prepare_acks: BTreeSet::new(),
90+
};
91+
92+
let state = CoordinatorState::new(my_platform_id, now, msg, config, op);
93+
Ok((state, my_prepare_msg.unwrap()))
94+
}
95+
96+
/// A reconfiguration from one group to another
97+
pub fn new_reconfiguration(
98+
my_platform_id: PlatformId,
99+
now: Instant,
100+
msg: ValidatedReconfigureMsg,
101+
last_committed_config: &Configuration,
102+
) -> Result<CoordinatorState, ReconfigurationError> {
103+
let (config, new_shares) =
104+
Configuration::new(my_platform_id.clone(), &msg)?;
105+
106+
// We must collect shares from the last configuration
107+
// so we can recompute the old rack secret.
108+
let op = CoordinatorOperation::CollectShares {
109+
epoch: last_committed_config.epoch,
110+
members: last_committed_config.members.clone(),
111+
collected_shares: BTreeMap::new(),
112+
new_shares,
113+
};
114+
115+
Ok(CoordinatorState::new(my_platform_id, now, msg, config, op))
116+
}
117+
118+
// Intentionallly private!
119+
fn new(
120+
platform_id: PlatformId,
121+
now: Instant,
122+
reconfigure_msg: ValidatedReconfigureMsg,
123+
configuration: Configuration,
124+
op: CoordinatorOperation,
125+
) -> CoordinatorState {
126+
// We always set the retry deadline to `now` so that we will send
127+
// messages upon new construction. This field gets updated after
128+
// prepares are sent.
129+
let retry_deadline = now;
130+
CoordinatorState {
131+
platform_id,
132+
start_time: now,
133+
reconfigure_msg,
134+
configuration,
135+
op,
136+
retry_deadline,
137+
}
138+
}
139+
140+
// Send any required messages as a reconfiguration coordinator
141+
//
142+
// This varies depending upon the current `CoordinatorState`.
143+
//
144+
// In some cases a `PrepareMsg` will be added locally to the
145+
// `PersistentState`, requiring persistence from the caller. In this case we
146+
// will return a copy of it.
147+
//
148+
// This method is "in progress" - allow unused parameters for now
149+
#[allow(unused)]
150+
pub fn send_msgs(&mut self, now: Instant, outbox: &mut Vec<Envelope>) {
151+
match &self.op {
152+
CoordinatorOperation::CollectShares {
153+
epoch,
154+
members,
155+
collected_shares,
156+
..
157+
} => {}
158+
CoordinatorOperation::CollectLrtqShares { members, shares } => {}
159+
CoordinatorOperation::Prepare { prepares, prepare_acks } => {
160+
for (platform_id, prepare) in prepares.clone().into_iter() {
161+
outbox.push(Envelope {
162+
to: platform_id,
163+
from: self.platform_id.clone(),
164+
msg: PeerMsg::Prepare(prepare),
165+
});
166+
}
167+
}
168+
}
169+
}
170+
}
171+
172+
/// What should the coordinator be doing?
173+
///
174+
/// We haven't started implementing upgrade from LRTQ yet
175+
#[allow(unused)]
176+
pub enum CoordinatorOperation {
177+
CollectShares {
178+
epoch: Epoch,
179+
members: BTreeMap<PlatformId, Sha3_256Digest>,
180+
collected_shares: BTreeMap<PlatformId, Share>,
181+
new_shares: SecretShares,
182+
},
183+
// Epoch is always 0
184+
CollectLrtqShares {
185+
members: BTreeMap<PlatformId, ShareDigestLrtq>,
186+
shares: BTreeMap<PlatformId, LrtqShare>,
187+
},
188+
Prepare {
189+
/// The set of Prepares to send to each node
190+
prepares: BTreeMap<PlatformId, PrepareMsg>,
191+
192+
/// Acknowledgements that the prepare has been received
193+
prepare_acks: BTreeSet<PlatformId>,
194+
},
195+
}

Diff for: trust-quorum/src/lib.rs

+23-2
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,18 @@ use derive_more::Display;
1313
use serde::{Deserialize, Serialize};
1414

1515
mod configuration;
16+
mod coordinator_state;
1617
pub(crate) mod crypto;
1718
mod messages;
19+
mod node;
20+
mod persistent_state;
21+
mod validators;
1822
pub use configuration::Configuration;
23+
pub(crate) use coordinator_state::CoordinatorState;
1924
pub use crypto::RackSecret;
2025
pub use messages::*;
26+
pub use node::Node;
27+
pub use persistent_state::PersistentState;
2128

2229
#[derive(
2330
Debug,
@@ -61,8 +68,22 @@ pub struct Threshold(pub u8);
6168
Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize,
6269
)]
6370
pub struct PlatformId {
64-
pub part_number: String,
65-
pub serial_number: String,
71+
part_number: String,
72+
serial_number: String,
73+
}
74+
75+
impl PlatformId {
76+
pub fn new(part_number: String, serial_number: String) -> PlatformId {
77+
PlatformId { part_number, serial_number }
78+
}
79+
80+
pub fn part_number(&self) -> &str {
81+
&self.part_number
82+
}
83+
84+
pub fn serial_number(&self) -> &str {
85+
&self.serial_number
86+
}
6687
}
6788

6889
/// A container to make messages between trust quorum nodes routable

Diff for: trust-quorum/src/messages.rs

+15-5
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ use std::{collections::BTreeSet, time::Duration};
1313

1414
/// A request from nexus informing a node to start coordinating a
1515
/// reconfiguration.
16-
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
16+
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1717
pub struct ReconfigureMsg {
1818
pub rack_id: RackUuid,
1919
pub epoch: Epoch,
@@ -32,8 +32,20 @@ pub enum PeerMsg {
3232
PrepareAck(Epoch),
3333
Commit(CommitMsg),
3434

35+
/// When a node learns about a commit for a given epoch
36+
/// but does not have a `PrepareMsg`, it must ask for it
37+
/// from another node.
38+
GetPrepare(Epoch),
39+
40+
/// Nodes reply with `PrepareAndCommit` to `GetPrepare` requests when they
41+
/// are able to.
42+
PrepareAndCommit,
43+
3544
GetShare(Epoch),
36-
Share { epoch: Epoch, share: Share },
45+
Share {
46+
epoch: Epoch,
47+
share: Share,
48+
},
3749

3850
// LRTQ shares are always at epoch 0
3951
GetLrtqShare,
@@ -47,9 +59,7 @@ pub struct PrepareMsg {
4759
pub share: Share,
4860
}
4961

50-
#[derive(
51-
Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize,
52-
)]
62+
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
5363
pub struct CommitMsg {
5464
epoch: Epoch,
5565
}

0 commit comments

Comments
 (0)