Skip to content

Commit 81df54c

Browse files
fraidevfraidev
authored
feat: support producing by mirror (#4243)
1 parent f8eeda5 commit 81df54c

File tree

3 files changed

+81
-7
lines changed

3 files changed

+81
-7
lines changed

‎crates/fluvio-cli/src/client/produce/mod.rs

+48-2
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,14 @@ mod cmd {
1212
use std::path::PathBuf;
1313

1414
use async_trait::async_trait;
15+
use fluvio_sc_schema::partition::PartitionMirrorConfig;
16+
use fluvio_sc_schema::topic::{MirrorConfig, PartitionMap, ReplicaSpec, TopicSpec};
1517
#[cfg(feature = "producer-file-io")]
1618
use futures::future::join_all;
1719
use clap::Parser;
1820
use tracing::{error, warn};
1921
use humantime::parse_duration;
20-
use anyhow::Result;
22+
use anyhow::{bail, Result};
2123

2224
use fluvio::{
2325
Compression, Fluvio, FluvioError, TopicProducerPool, TopicProducerConfigBuilder, RecordKey,
@@ -173,8 +175,12 @@ mod cmd {
173175
pub transforms_line: Vec<String>,
174176

175177
/// Partition id
176-
#[arg(short = 'p', long, value_name = "integer")]
178+
#[arg(short = 'p', long, value_name = "integer", conflicts_with = "mirror")]
177179
pub partition: Option<PartitionId>,
180+
181+
/// Remote cluster to consume from
182+
#[arg(short = 'm', long, conflicts_with = "partition")]
183+
pub mirror: Option<String>,
178184
}
179185

180186
fn validate_key_separator(separator: &str) -> std::result::Result<String, String> {
@@ -247,6 +253,46 @@ mod cmd {
247253
let config_builder =
248254
config_builder.smartmodules(self.smartmodule_invocations(initial_param)?);
249255

256+
let config_builder = if let Some(mirror) = &self.mirror {
257+
let admin = fluvio.admin().await;
258+
let topics = admin.all::<TopicSpec>().await?;
259+
let partition = topics.into_iter().find_map(|t| match t.spec.replicas() {
260+
ReplicaSpec::Mirror(MirrorConfig::Home(home_mirror_config)) => {
261+
let partitions_maps =
262+
Vec::<PartitionMap>::from(home_mirror_config.as_partition_maps());
263+
partitions_maps.iter().find_map(|p| {
264+
if let Some(PartitionMirrorConfig::Home(remote)) = &p.mirror {
265+
if remote.remote_cluster == *mirror && remote.source {
266+
return Some(p.id);
267+
}
268+
}
269+
None
270+
})
271+
}
272+
ReplicaSpec::Mirror(MirrorConfig::Remote(remote_mirror_config)) => {
273+
let partitions_maps =
274+
Vec::<PartitionMap>::from(remote_mirror_config.as_partition_maps());
275+
partitions_maps.iter().find_map(|p| {
276+
if let Some(PartitionMirrorConfig::Remote(remote)) = &p.mirror {
277+
if remote.home_cluster == *mirror && remote.target {
278+
return Some(p.id);
279+
}
280+
}
281+
None
282+
})
283+
}
284+
_ => None,
285+
});
286+
287+
if let Some(partition) = partition {
288+
config_builder.set_specific_partitioner(partition)
289+
} else {
290+
bail!("No partition found for mirror '{}'", mirror);
291+
}
292+
} else {
293+
config_builder
294+
};
295+
250296
let config_builder = if let Some(partition) = self.partition {
251297
config_builder.set_specific_partitioner(partition)
252298
} else {

‎crates/fluvio/src/fluvio.rs

+12
Original file line numberDiff line numberDiff line change
@@ -358,6 +358,18 @@ impl Fluvio {
358358
None
359359
})
360360
}
361+
ReplicaSpec::Mirror(MirrorConfig::Remote(remote_mirror_config)) => {
362+
let partitions_maps =
363+
Vec::<PartitionMap>::from(remote_mirror_config.as_partition_maps());
364+
partitions_maps.iter().find_map(|p| {
365+
if let Some(PartitionMirrorConfig::Remote(remote)) = &p.mirror {
366+
if remote.home_cluster == *mirror {
367+
return Some(p.id);
368+
}
369+
}
370+
None
371+
})
372+
}
361373
_ => None,
362374
}
363375
} else {

‎tests/cli/mirroring_smoke_tests/e2e/fluvio-core.bats

+21-5
Original file line numberDiff line numberDiff line change
@@ -122,11 +122,11 @@ setup_file() {
122122
@test "Can produce message to reverse mirror topic from home" {
123123
run bash -c 'echo 3 | timeout 15s "$FLUVIO_BIN" produce "$REVERSE_TOPIC_NAME"'
124124
assert_success
125-
run bash -c 'echo c | timeout 15s "$FLUVIO_BIN" produce "$REVERSE_TOPIC_NAME"'
125+
run bash -c 'echo c | timeout 15s "$FLUVIO_BIN" produce -p 0 "$REVERSE_TOPIC_NAME"'
126126
assert_success
127-
run bash -c 'echo 4 | timeout 15s "$FLUVIO_BIN" produce "$REVERSE_TOPIC_NAME"'
127+
run bash -c 'echo 4 | timeout 15s "$FLUVIO_BIN" produce -m "$REMOTE_NAME" "$REVERSE_TOPIC_NAME"'
128128
assert_success
129-
run bash -c 'echo d | timeout 15s "$FLUVIO_BIN" produce "$REVERSE_TOPIC_NAME"'
129+
run bash -c 'echo d | timeout 15s "$FLUVIO_BIN" produce -m "$REMOTE_NAME" "$REVERSE_TOPIC_NAME"'
130130
assert_success
131131
}
132132

@@ -186,6 +186,10 @@ setup_file() {
186186
run timeout 15s "$FLUVIO_BIN" consume "$REVERSE_TOPIC_NAME" -p 0 -B -d
187187
assert_output 3$'\n'c$'\n'4$'\n'd
188188
assert_success
189+
190+
run timeout 15s "$FLUVIO_BIN" consume "$REVERSE_TOPIC_NAME" --mirror "$REMOTE_NAME" -B -d
191+
assert_output 3$'\n'c$'\n'4$'\n'd
192+
assert_success
189193
}
190194

191195
@test "Can switch back to home cluster" {
@@ -200,9 +204,9 @@ setup_file() {
200204
assert_success
201205
run bash -c 'echo e | timeout 15s "$FLUVIO_BIN" produce -p 1 "$REVERSE_TOPIC_NAME"'
202206
assert_success
203-
run bash -c 'echo 6 | timeout 15s "$FLUVIO_BIN" produce -p 1 "$REVERSE_TOPIC_NAME"'
207+
run bash -c 'echo 6 | timeout 15s "$FLUVIO_BIN" produce -m "$REMOTE_NAME_2" "$REVERSE_TOPIC_NAME"'
204208
assert_success
205-
run bash -c 'echo f | timeout 15s "$FLUVIO_BIN" produce -p 1 "$REVERSE_TOPIC_NAME"'
209+
run bash -c 'echo f | timeout 15s "$FLUVIO_BIN" produce -m "$REMOTE_NAME_2" "$REVERSE_TOPIC_NAME"'
206210
assert_success
207211
}
208212

@@ -253,6 +257,10 @@ setup_file() {
253257
run timeout 15s "$FLUVIO_BIN" consume "$REVERSE_TOPIC_NAME" -p 0 -B -d
254258
assert_output 5$'\n'e$'\n'6$'\n'f
255259
assert_success
260+
261+
run timeout 15s "$FLUVIO_BIN" consume "$REVERSE_TOPIC_NAME" --mirror "$REMOTE_NAME_2" -B -d
262+
assert_output 5$'\n'e$'\n'6$'\n'f
263+
assert_success
256264
}
257265

258266
@test "Can't delete mirror topic from remote 2" {
@@ -283,6 +291,14 @@ setup_file() {
283291
run timeout 15s "$FLUVIO_BIN" consume "$REVERSE_TOPIC_NAME" -p 0 -B -d
284292
assert_output 3$'\n'c$'\n'4$'\n'd
285293
assert_success
294+
295+
run timeout 15s "$FLUVIO_BIN" consume "$REVERSE_TOPIC_NAME" -p 1 -B -d
296+
assert_output 5$'\n'e$'\n'6$'\n'f
297+
assert_success
298+
299+
run timeout 15s "$FLUVIO_BIN" consume "$REVERSE_TOPIC_NAME" --mirror "$HOME" -B -d
300+
assert_output 3$'\n'c$'\n'4$'\n'd$'\n'5$'\n'e$'\n'6$'\n'f
301+
assert_success
286302
}
287303

288304
@test "Can consume message from mirror topic produced from remote 1 by partition or remote" {

0 commit comments

Comments
 (0)