diff --git a/src/cli.rs b/src/cli.rs index ffb3a80eb..9a0e1bca2 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -270,6 +270,14 @@ pub struct Options { )] pub row_group_size: usize, + #[arg( + long, + env = "P_EXECUTION_BATCH_SIZE", + default_value = "20000", + help = "batch size for query execution" + )] + pub execution_batch_size: usize, + #[arg( long = "compression-algo", env = "P_PARQUET_COMPRESSION_ALGO", diff --git a/src/query/mod.rs b/src/query/mod.rs index df230ef07..6486b4836 100644 --- a/src/query/mod.rs +++ b/src/query/mod.rs @@ -111,7 +111,9 @@ impl Query { let mut config = SessionConfig::default() .with_parquet_pruning(true) .with_prefer_existing_sort(true) - .with_batch_size(1000000); + //batch size has been made configurable via environment variable + //default value is 20000 + .with_batch_size(PARSEABLE.options.execution_batch_size); // Pushdown filters allows DF to push the filters as far down in the plan as possible // and thus, reducing the number of rows decoded