From 1a3f3af255f5737dc595ddf16d0869fe388130a2 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Thu, 6 Feb 2025 01:40:46 +0100 Subject: [PATCH] add `direct` indices, except for in datastore & C# (#2205) --- .../Internal/Autogen/RawIndexAlgorithm.g.cs | 3 +- crates/bindings-macro/src/lib.rs | 2 + crates/bindings-macro/src/table.rs | 268 ++++++++++++------ crates/bindings/src/lib.rs | 2 +- crates/bindings/src/rt.rs | 28 +- crates/bindings/src/table.rs | 57 ++-- .../locking_tx_datastore/committed_state.rs | 3 +- .../datastore/locking_tx_datastore/mut_tx.rs | 3 +- crates/core/src/db/datastore/system_tables.rs | 54 ++-- crates/core/src/db/relational_db.rs | 40 ++- crates/lib/src/db/raw_def/v9.rs | 29 +- crates/primitives/src/col_list.rs | 71 +++++ crates/primitives/src/lib.rs | 2 +- crates/schema/src/auto_migrate.rs | 68 ++--- crates/schema/src/def.rs | 83 ++---- crates/schema/src/def/validate/v8.rs | 7 +- crates/schema/src/def/validate/v9.rs | 223 ++++++++++----- crates/schema/src/error.rs | 14 +- crates/schema/src/schema.rs | 9 +- 19 files changed, 617 insertions(+), 349 deletions(-) diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/RawIndexAlgorithm.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/RawIndexAlgorithm.g.cs index bf6563c58e7..6e93c908b55 100644 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/RawIndexAlgorithm.g.cs +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/RawIndexAlgorithm.g.cs @@ -10,6 +10,7 @@ namespace SpacetimeDB.Internal [SpacetimeDB.Type] public partial record RawIndexAlgorithm : SpacetimeDB.TaggedEnum<( System.Collections.Generic.List BTree, - System.Collections.Generic.List Hash + System.Collections.Generic.List Hash, + ushort Direct )>; } diff --git a/crates/bindings-macro/src/lib.rs b/crates/bindings-macro/src/lib.rs index d388f7a439b..ba4fdbf2df7 100644 --- a/crates/bindings-macro/src/lib.rs +++ b/crates/bindings-macro/src/lib.rs @@ -35,8 +35,10 @@ mod sym { symbol!(btree); symbol!(client_connected); symbol!(client_disconnected); + symbol!(column); symbol!(columns); symbol!(crate_, crate); + symbol!(direct); symbol!(index); symbol!(init); symbol!(name); diff --git a/crates/bindings-macro/src/table.rs b/crates/bindings-macro/src/table.rs index d3fa472c907..c74eea4fe4c 100644 --- a/crates/bindings-macro/src/table.rs +++ b/crates/bindings-macro/src/table.rs @@ -1,12 +1,14 @@ use crate::sats; use crate::sym; use crate::util::{check_duplicate, check_duplicate_msg, ident_to_litstr, match_meta}; +use core::slice; use heck::ToSnakeCase; use proc_macro2::{Span, TokenStream}; use quote::{format_ident, quote, quote_spanned, ToTokens}; use std::borrow::Cow; use syn::ext::IdentExt; use syn::meta::ParseNestedMeta; +use syn::parse::Parse; use syn::parse::Parser as _; use syn::punctuated::Punctuated; use syn::spanned::Spanned; @@ -44,12 +46,22 @@ struct ScheduledArg { struct IndexArg { name: Ident, + is_unique: bool, kind: IndexType, } +impl IndexArg { + fn new(name: Ident, kind: IndexType) -> Self { + // We don't know if its unique yet. + // We'll discover this once we have collected constraints. + let is_unique = false; + Self { name, is_unique, kind } + } +} + enum IndexType { BTree { columns: Vec }, - UniqueBTree { column: Ident }, + Direct { column: Ident }, } impl TableArgs { @@ -142,13 +154,19 @@ impl IndexArg { check_duplicate_msg(&algo, &meta, "index algorithm specified twice")?; algo = Some(Self::parse_btree(meta)?); } + sym::direct => { + check_duplicate_msg(&algo, &meta, "index algorithm specified twice")?; + algo = Some(Self::parse_direct(meta)?); + } }); Ok(()) })?; let name = name.ok_or_else(|| meta.error("missing index name, e.g. name = my_index"))?; - let kind = algo.ok_or_else(|| meta.error("missing index algorithm, e.g., `btree(columns = [col1, col2])`"))?; + let kind = algo.ok_or_else(|| { + meta.error("missing index algorithm, e.g., `btree(columns = [col1, col2])` or `direct(column = col1)`") + })?; - Ok(IndexArg { name, kind }) + Ok(IndexArg::new(name, kind)) } fn parse_btree(meta: ParseNestedMeta) -> syn::Result { @@ -174,7 +192,26 @@ impl IndexArg { Ok(IndexType::BTree { columns }) } - /// Parses an inline `#[index(btree)]` attribute on a field. + fn parse_direct(meta: ParseNestedMeta) -> syn::Result { + let mut column = None; + meta.parse_nested_meta(|meta| { + match_meta!(match meta { + sym::column => { + check_duplicate(&column, &meta)?; + let value = meta.value()?; + let inner; + syn::bracketed!(inner in value); + column = Some(Ident::parse(&inner)?); + } + }); + Ok(()) + })?; + let column = column + .ok_or_else(|| meta.error("must specify the column for direct index, e.g. `direct(column = col1)`"))?; + Ok(IndexType::Direct { column }) + } + + /// Parses an inline `#[index(btree)]` or `#[index(direct)]` attribute on a field. fn parse_index_attr(field: &Ident, attr: &syn::Attribute) -> syn::Result { let mut kind = None; attr.parse_nested_meta(|meta| { @@ -185,12 +222,17 @@ impl IndexArg { columns: vec![field.clone()], }); } + sym::direct => { + check_duplicate_msg(&kind, &meta, "index type specified twice")?; + kind = Some(IndexType::Direct { column: field.clone() }) + } }); Ok(()) })?; - let kind = kind.ok_or_else(|| syn::Error::new_spanned(&attr.meta, "must specify kind of index (`btree`)"))?; + let kind = kind + .ok_or_else(|| syn::Error::new_spanned(&attr.meta, "must specify kind of index (`btree` or `direct`)"))?; let name = field.clone(); - Ok(IndexArg { kind, name }) + Ok(IndexArg::new(name, kind)) } fn validate<'a>(&'a self, table_name: &str, cols: &'a [Column<'a>]) -> syn::Result> { @@ -200,29 +242,32 @@ impl IndexArg { let cols = columns.iter().map(find_column).collect::>>()?; ValidatedIndexType::BTree { cols } } - IndexType::UniqueBTree { column } => { + IndexType::Direct { column } => { let col = find_column(column)?; - ValidatedIndexType::UniqueBTree { col } + + if !self.is_unique { + return Err(syn::Error::new( + column.span(), + "a direct index must be paired with a `#[unique] constraint", + )); + } + + ValidatedIndexType::Direct { col } } }; // See crates/schema/src/validate/v9.rs for the format of index names. // It's slightly unnerving that we just trust that component to generate this format correctly, // but what can you do. - let index_name = match &kind { - ValidatedIndexType::BTree { cols } => { - let cols = cols - .iter() - .map(|col| col.field.ident.unwrap().to_string()) - .collect::>(); - let cols = cols.join("_"); - format!("{table_name}_{cols}_idx_btree") - } - ValidatedIndexType::UniqueBTree { col } => { - let col = col.field.ident.unwrap().to_string(); - format!("{table_name}_{col}_idx_btree") - } + let (cols, kind_str) = match &kind { + ValidatedIndexType::BTree { cols } => (&**cols, "btree"), + ValidatedIndexType::Direct { col } => (&[*col] as &[_], "direct"), }; + let cols = cols.iter().map(|col| col.ident.to_string()).collect::>(); + let cols = cols.join("_"); + let index_name = format!("{table_name}_{cols}_idx_{kind_str}"); + Ok(ValidatedIndex { + is_unique: self.is_unique, index_name, accessor_name: &self.name, kind, @@ -233,12 +278,13 @@ impl IndexArg { struct ValidatedIndex<'a> { index_name: String, accessor_name: &'a Ident, + is_unique: bool, kind: ValidatedIndexType<'a>, } enum ValidatedIndexType<'a> { BTree { cols: Vec<&'a Column<'a>> }, - UniqueBTree { col: &'a Column<'a> }, + Direct { col: &'a Column<'a> }, } impl ValidatedIndex<'_> { @@ -250,10 +296,10 @@ impl ValidatedIndex<'_> { columns: &[#(#col_ids),*] }) } - ValidatedIndexType::UniqueBTree { col } => { + ValidatedIndexType::Direct { col } => { let col_id = col.index; - quote!(spacetimedb::table::IndexAlgo::BTree { - columns: &[#col_id] + quote!(spacetimedb::table::IndexAlgo::Direct { + column: #col_id }) } }; @@ -267,48 +313,60 @@ impl ValidatedIndex<'_> { } fn accessor(&self, vis: &syn::Visibility, row_type_ident: &Ident) -> TokenStream { + let cols = match &self.kind { + ValidatedIndexType::BTree { cols } => &**cols, + ValidatedIndexType::Direct { col } => slice::from_ref(col), + }; + if self.is_unique { + assert_eq!(cols.len(), 1); + let col = cols[0]; + self.accessor_unique(col, row_type_ident) + } else { + self.accessor_general(vis, row_type_ident, cols) + } + } + + fn accessor_unique(&self, col: &Column<'_>, row_type_ident: &Ident) -> TokenStream { let index_ident = self.accessor_name; - match &self.kind { - ValidatedIndexType::BTree { cols } => { - let col_tys = cols.iter().map(|col| col.ty); - let mut doc = format!( - "Gets the `{index_ident}` [`BTreeIndex`][spacetimedb::BTreeIndex] as defined \ - on this table. \n\ - \n\ - This B-tree index is defined on the following columns, in order:\n" - ); - for col in cols { - use std::fmt::Write; - writeln!( - doc, - "- [`{ident}`][{row_type_ident}#structfield.{ident}]: [`{ty}`]", - ident = col.field.ident.unwrap(), - ty = col.ty.to_token_stream() - ) - .unwrap(); - } - quote! { - #[doc = #doc] - #vis fn #index_ident(&self) -> spacetimedb::BTreeIndex { - spacetimedb::BTreeIndex::__NEW - } - } + let vis = col.vis; + let col_ty = col.ty; + let column_ident = col.ident; + + let doc = format!( + "Gets the [`UniqueColumn`][spacetimedb::UniqueColumn] for the \ + [`{column_ident}`][{row_type_ident}::{column_ident}] column." + ); + quote! { + #[doc = #doc] + #vis fn #column_ident(&self) -> spacetimedb::UniqueColumn { + spacetimedb::UniqueColumn::__NEW } - ValidatedIndexType::UniqueBTree { col } => { - let vis = col.field.vis; - let col_ty = col.field.ty; - let column_ident = col.field.ident.unwrap(); - - let doc = format!( - "Gets the [`UniqueColumn`][spacetimedb::UniqueColumn] for the \ - [`{column_ident}`][{row_type_ident}::{column_ident}] column." - ); - quote! { - #[doc = #doc] - #vis fn #column_ident(&self) -> spacetimedb::UniqueColumn { - spacetimedb::UniqueColumn::__NEW - } - } + } + } + + fn accessor_general(&self, vis: &syn::Visibility, row_type_ident: &Ident, cols: &[&Column<'_>]) -> TokenStream { + let index_ident = self.accessor_name; + let col_tys = cols.iter().map(|col| col.ty); + let mut doc = format!( + "Gets the `{index_ident}` [`RangedIndex`][spacetimedb::RangedIndex] as defined \ + on this table. \n\ + \n\ + This B-tree index is defined on the following columns, in order:\n" + ); + for col in cols { + use std::fmt::Write; + writeln!( + doc, + "- [`{ident}`][{row_type_ident}#structfield.{ident}]: [`{ty}`]", + ident = col.ident, + ty = col.ty.to_token_stream() + ) + .unwrap(); + } + quote! { + #[doc = #doc] + #vis fn #index_ident(&self) -> spacetimedb::RangedIndex { + spacetimedb::RangedIndex::__NEW } } } @@ -316,13 +374,30 @@ impl ValidatedIndex<'_> { fn marker_type(&self, vis: &syn::Visibility, tablehandle_ident: &Ident) -> TokenStream { let index_ident = self.accessor_name; let index_name = &self.index_name; - let vis = if let ValidatedIndexType::UniqueBTree { col } = self.kind { - col.field.vis + + let (cols, typeck_direct_index) = match &self.kind { + ValidatedIndexType::BTree { cols } => (&**cols, None), + ValidatedIndexType::Direct { col } => { + let col_ty = col.ty; + let typeck = quote_spanned!(col_ty.span()=> + const _: () { + spacetimedb::rt::assert_column_type_valid_for_direct_index::<#col_ty>(); + }; + ); + (slice::from_ref(col), Some(typeck)) + } + }; + let vis = if self.is_unique { + assert_eq!(cols.len(), 1); + cols[0].vis } else { vis }; let vis = superize_vis(vis); + let mut decl = quote! { + #typeck_direct_index + #vis struct #index_ident; impl spacetimedb::table::Index for #index_ident { fn index_id() -> spacetimedb::table::IndexId { @@ -333,10 +408,11 @@ impl ValidatedIndex<'_> { } } }; - if let ValidatedIndexType::UniqueBTree { col } = self.kind { + if self.is_unique { + let col = cols[0]; let col_ty = col.ty; - let col_name = col.field.name.as_deref().unwrap(); - let field_ident = col.field.ident.unwrap(); + let col_name = col.ident.to_string(); + let field_ident = col.ident; decl.extend(quote! { impl spacetimedb::table::Column for #index_ident { type Table = #tablehandle_ident; @@ -378,7 +454,8 @@ fn superize_vis(vis: &syn::Visibility) -> Cow<'_, syn::Visibility> { #[derive(Copy, Clone)] struct Column<'a> { index: u16, - field: &'a sats::SatsField<'a>, + vis: &'a syn::Visibility, + ident: &'a syn::Ident, ty: &'a syn::Type, } @@ -386,8 +463,7 @@ fn try_find_column<'a, 'b, T: ?Sized>(cols: &'a [Column<'b>], name: &T) -> Optio where Ident: PartialEq, { - cols.iter() - .find(|col| col.field.ident.is_some_and(|ident| ident == name)) + cols.iter().find(|col| col.ident == name) } fn find_column<'a, 'b>(cols: &'a [Column<'b>], name: &Ident) -> syn::Result<&'a Column<'b>> { @@ -495,18 +571,13 @@ pub(crate) fn table_impl(mut args: TableArgs, item: &syn::DeriveInput) -> syn::R let column = Column { index: col_num, - field, + ident: field_ident, + vis: field.vis, ty: field.ty, }; if unique.is_some() || primary_key.is_some() { unique_columns.push(column); - args.indices.push(IndexArg { - name: field_ident.clone(), - kind: IndexType::UniqueBTree { - column: field_ident.clone(), - }, - }); } if auto_inc.is_some() { sequenced_columns.push(column); @@ -521,19 +592,38 @@ pub(crate) fn table_impl(mut args: TableArgs, item: &syn::DeriveInput) -> syn::R let row_type = quote!(#original_struct_ident); + // Mark all indices with a single column matching a unique constraint as unique. + // For all the unpaired unique columns, create a unique index. + for unique_col in &unique_columns { + if args.indices.iter_mut().any(|index| { + let covered_by_index = match &index.kind { + IndexType::BTree { columns } => &**columns == slice::from_ref(unique_col.ident), + IndexType::Direct { column } => column == unique_col.ident, + }; + index.is_unique |= covered_by_index; + covered_by_index + }) { + continue; + } + // NOTE(centril): We pick `btree` here if the user does not specify otherwise, + // as it's the safest choice of index for the general case, + // even if isn't optimal in specific cases. + let name = unique_col.ident.clone(); + let columns = vec![name.clone()]; + args.indices.push(IndexArg { + name, + is_unique: true, + kind: IndexType::BTree { columns }, + }) + } + let mut indices = args .indices .iter() .map(|index| index.validate(&table_name, &columns)) .collect::>>()?; - - // order unique accessors before index accessors - indices.sort_by(|a, b| match (&a.kind, &b.kind) { - (ValidatedIndexType::UniqueBTree { .. }, ValidatedIndexType::UniqueBTree { .. }) => std::cmp::Ordering::Equal, - (_, ValidatedIndexType::UniqueBTree { .. }) => std::cmp::Ordering::Greater, - (ValidatedIndexType::UniqueBTree { .. }, _) => std::cmp::Ordering::Less, - _ => std::cmp::Ordering::Equal, - }); + // Order unique accessors before index accessors. + indices.sort_by_key(|index| !index.is_unique); let tablehandle_ident = format_ident!("{}__TableHandle", table_ident); @@ -544,7 +634,7 @@ pub(crate) fn table_impl(mut args: TableArgs, item: &syn::DeriveInput) -> syn::R // Generate `integrate_generated_columns` // which will integrate all generated auto-inc col values into `_row`. let integrate_gen_col = sequenced_columns.iter().map(|col| { - let field = col.field.ident.unwrap(); + let field = col.ident; quote_spanned!(field.span()=> spacetimedb::table::SequenceTrigger::maybe_decode_into(&mut __row.#field, &mut __generated_cols); ) diff --git a/crates/bindings/src/lib.rs b/crates/bindings/src/lib.rs index b404b2329bf..ce81767214e 100644 --- a/crates/bindings/src/lib.rs +++ b/crates/bindings/src/lib.rs @@ -40,7 +40,7 @@ pub use spacetimedb_lib::Identity; pub use spacetimedb_lib::ScheduleAt; pub use spacetimedb_primitives::TableId; pub use sys::Errno; -pub use table::{AutoIncOverflow, BTreeIndex, Table, TryInsertError, UniqueColumn, UniqueConstraintViolation}; +pub use table::{AutoIncOverflow, RangedIndex, Table, TryInsertError, UniqueColumn, UniqueConstraintViolation}; pub use timestamp::Timestamp; pub type ReducerResult = core::result::Result<(), Box>; diff --git a/crates/bindings/src/rt.rs b/crates/bindings/src/rt.rs index a14ee93c706..dfc37fd27ef 100644 --- a/crates/bindings/src/rt.rs +++ b/crates/bindings/src/rt.rs @@ -1,5 +1,6 @@ #![deny(unsafe_op_in_unsafe_fn)] +use crate::table::IndexAlgo; use crate::timestamp::with_timestamp_set; use crate::{sys, IterBuf, ReducerContext, ReducerResult, SpacetimeType, Table, Timestamp}; pub use spacetimedb_lib::db::raw_def::v9::Lifecycle as LifecycleReducer; @@ -364,12 +365,33 @@ pub fn register_table() { }) } -impl From> for RawIndexAlgorithm { - fn from(algo: crate::table::IndexAlgo<'_>) -> RawIndexAlgorithm { +mod sealed_direct_index { + pub trait Sealed {} +} +#[diagnostic::on_unimplemented( + message = "column type must be a one of: `u8`, `u16`, `u32`, or `u64`", + label = "should be `u8`, `u16`, `u32`, or `u64`, not `{Self}`" +)] +pub trait DirectIndexKey: sealed_direct_index::Sealed {} +impl sealed_direct_index::Sealed for u8 {} +impl DirectIndexKey for u8 {} +impl sealed_direct_index::Sealed for u16 {} +impl DirectIndexKey for u16 {} +impl sealed_direct_index::Sealed for u32 {} +impl DirectIndexKey for u32 {} +impl sealed_direct_index::Sealed for u64 {} +impl DirectIndexKey for u64 {} + +/// Assert that `T` is a valid column to use direct index on. +pub const fn assert_column_type_valid_for_direct_index() {} + +impl From> for RawIndexAlgorithm { + fn from(algo: IndexAlgo<'_>) -> RawIndexAlgorithm { match algo { - crate::table::IndexAlgo::BTree { columns } => RawIndexAlgorithm::BTree { + IndexAlgo::BTree { columns } => RawIndexAlgorithm::BTree { columns: columns.iter().copied().collect(), }, + IndexAlgo::Direct { column } => RawIndexAlgorithm::Direct { column: column.into() }, } } } diff --git a/crates/bindings/src/table.rs b/crates/bindings/src/table.rs index 58320f17805..4da778ce2fb 100644 --- a/crates/bindings/src/table.rs +++ b/crates/bindings/src/table.rs @@ -142,6 +142,7 @@ pub struct IndexDesc<'a> { #[derive(Clone, Copy)] pub enum IndexAlgo<'a> { BTree { columns: &'a [u16] }, + Direct { column: u16 }, } pub struct ScheduleDesc<'a> { @@ -272,8 +273,8 @@ impl> UniqueColumn BTreeScanArgs { - BTreeScanArgs { + fn get_args(&self, col_val: &Col::ColType) -> IndexScanRangeArgs { + IndexScanRangeArgs { data: IterBuf::serialize(&std::ops::Bound::Included(col_val)).unwrap(), prefix_elems: 0, rstart_idx: 0, @@ -358,11 +359,11 @@ pub trait Index { fn index_id() -> IndexId; } -pub struct BTreeIndex { +pub struct RangedIndex { _marker: PhantomData<(Tbl, IndexType, Idx)>, } -impl BTreeIndex { +impl RangedIndex { #[doc(hidden)] pub const __NEW: Self = Self { _marker: PhantomData }; @@ -374,7 +375,7 @@ impl BTreeIndex { /// - A tuple of values for any prefix of the indexed columns, optionally terminated by a range for the next. pub fn filter(&self, b: B) -> impl Iterator where - B: BTreeIndexBounds, + B: IndexScanRangeBounds, { let index_id = Idx::index_id(); let args = b.get_args(); @@ -395,7 +396,7 @@ impl BTreeIndex { /// though as of proposing no such constraints exist. pub fn delete(&self, b: B) -> u64 where - B: BTreeIndexBounds, + B: IndexScanRangeBounds, { let index_id = Idx::index_id(); let args = b.get_args(); @@ -410,7 +411,7 @@ impl BTreeIndex { /// for a column of type `Column`. /// /// Types which can appear specifically as a terminating bound in a BTree index, -/// which may be a range, instead use [`BTreeIndexBoundsTerminator`]. +/// which may be a range, instead use [`IndexRangeScanBoundsTerminator`]. /// /// General rules for implementors of this type: /// - It should only be implemented for types that have @@ -475,17 +476,17 @@ impl_filterable_value! { // &[u8] => Vec, } -pub trait BTreeIndexBounds { +pub trait IndexScanRangeBounds { #[doc(hidden)] - fn get_args(&self) -> BTreeScanArgs; + fn get_args(&self) -> IndexScanRangeArgs; } #[doc(hidden)] -/// Arguments to one of the BTree-related host-/sys-calls. +/// Arguments to one of the ranged-index-scan-related host-/sys-calls. /// /// All pointers passed into the syscall are packed into a single buffer, `data`, /// with slices taken at the appropriate offsets, to save allocatons in WASM. -pub struct BTreeScanArgs { +pub struct IndexScanRangeArgs { data: IterBuf, prefix_elems: usize, rstart_idx: usize, @@ -493,7 +494,7 @@ pub struct BTreeScanArgs { rend_idx: Option, } -impl BTreeScanArgs { +impl IndexScanRangeArgs { /// Get slices into `self.data` for the prefix, range start and range end. pub(crate) fn args_for_syscall(&self) -> (&[u8], ColId, &[u8], &[u8]) { let prefix = &self.data[..self.rstart_idx]; @@ -507,7 +508,7 @@ impl BTreeScanArgs { } } -// Implement `BTreeIndexBounds` for all the different index column types +// Implement `IndexScanRangeBounds` for all the different index column types // and filter argument types we support. macro_rules! impl_btree_index_bounds { // In the first pattern, we accept two Prolog-style lists of type variables, @@ -559,24 +560,24 @@ macro_rules! impl_btree_index_bounds { impl< $($ColUnused,)* $ColTerminator, - Term: BTreeIndexBoundsTerminator, + Term: IndexScanRangeBoundsTerminator, $ArgTerminator: FilterableValue, - > BTreeIndexBounds<($ColTerminator, $($ColUnused,)*)> for (Term,) { - fn get_args(&self) -> BTreeScanArgs { - BTreeIndexBounds::<($ColTerminator, $($ColUnused,)*), SingleBound>::get_args(&self.0) + > IndexScanRangeBounds<($ColTerminator, $($ColUnused,)*)> for (Term,) { + fn get_args(&self) -> IndexScanRangeArgs { + IndexScanRangeBounds::<($ColTerminator, $($ColUnused,)*), SingleBound>::get_args(&self.0) } } // Implementation for bare values: serialize the value as the terminating bounds. impl< $($ColUnused,)* $ColTerminator, - Term: BTreeIndexBoundsTerminator, + Term: IndexScanRangeBoundsTerminator, $ArgTerminator: FilterableValue, - > BTreeIndexBounds<($ColTerminator, $($ColUnused,)*), SingleBound> for Term { - fn get_args(&self) -> BTreeScanArgs { + > IndexScanRangeBounds<($ColTerminator, $($ColUnused,)*), SingleBound> for Term { + fn get_args(&self) -> IndexScanRangeArgs { let mut data = IterBuf::take(); let rend_idx = self.bounds().serialize_into(&mut data); - BTreeScanArgs { data, prefix_elems: 0, rstart_idx: 0, rend_idx } + IndexScanRangeArgs { data, prefix_elems: 0, rstart_idx: 0, rend_idx } } } }; @@ -596,15 +597,15 @@ macro_rules! impl_btree_index_bounds { $($ColUnused,)* $ColTerminator, $($ColPrefix,)* - Term: BTreeIndexBoundsTerminator, + Term: IndexScanRangeBoundsTerminator, $ArgTerminator: FilterableValue, $($ArgPrefix: FilterableValue,)+ - > BTreeIndexBounds< + > IndexScanRangeBounds< ($($ColPrefix,)+ $ColTerminator, $($ColUnused,)*) > for ($($ArgPrefix,)+ Term,) { - fn get_args(&self) -> BTreeScanArgs { + fn get_args(&self) -> IndexScanRangeArgs { let mut data = IterBuf::take(); // Get the number of prefix elements. @@ -627,7 +628,7 @@ macro_rules! impl_btree_index_bounds { // and get the info required to separately slice the lower and upper bounds of that range // since the host call takes those as separate slices. let rend_idx = term.bounds().serialize_into(&mut data); - BTreeScanArgs { data, prefix_elems, rstart_idx, rend_idx } + IndexScanRangeArgs { data, prefix_elems, rstart_idx, rend_idx } } } }; @@ -667,12 +668,12 @@ impl TermBound<&Bound> { }) } } -pub trait BTreeIndexBoundsTerminator { +pub trait IndexScanRangeBoundsTerminator { type Arg; fn bounds(&self) -> TermBound<&Self::Arg>; } -impl> BTreeIndexBoundsTerminator for Arg { +impl> IndexScanRangeBoundsTerminator for Arg { type Arg = Arg; fn bounds(&self) -> TermBound<&Arg> { TermBound::Single(ops::Bound::Included(self)) @@ -681,7 +682,7 @@ impl> BTreeIndexBoundsTerminator for Arg macro_rules! impl_terminator { ($($range:ty),* $(,)?) => { - $(impl BTreeIndexBoundsTerminator for $range { + $(impl IndexScanRangeBoundsTerminator for $range { type Arg = T; fn bounds(&self) -> TermBound<&T> { TermBound::Range( diff --git a/crates/core/src/db/datastore/locking_tx_datastore/committed_state.rs b/crates/core/src/db/datastore/locking_tx_datastore/committed_state.rs index dc60779c269..9c52080166e 100644 --- a/crates/core/src/db/datastore/locking_tx_datastore/committed_state.rs +++ b/crates/core/src/db/datastore/locking_tx_datastore/committed_state.rs @@ -389,7 +389,8 @@ impl CommittedState { }; let columns = match index_row.index_algorithm { StIndexAlgorithm::BTree { columns } => columns, - _ => unimplemented!("Only BTree indexes are supported"), + StIndexAlgorithm::Direct { column: _ } => todo!("todo_direct_index"), + _ => unimplemented!("Only btree and direct indexes are supported"), }; let is_unique = unique_constraints.contains(&(table_id, (&columns).into())); let index = table.new_index(columns.clone(), is_unique)?; diff --git a/crates/core/src/db/datastore/locking_tx_datastore/mut_tx.rs b/crates/core/src/db/datastore/locking_tx_datastore/mut_tx.rs index c1d19f9114e..1ab5916b6fe 100644 --- a/crates/core/src/db/datastore/locking_tx_datastore/mut_tx.rs +++ b/crates/core/src/db/datastore/locking_tx_datastore/mut_tx.rs @@ -47,7 +47,7 @@ use spacetimedb_sats::{ AlgebraicType, AlgebraicValue, ProductType, ProductValue, WithTypespace, }; use spacetimedb_schema::{ - def::{BTreeAlgorithm, IndexAlgorithm}, + def::{BTreeAlgorithm, DirectAlgorithm, IndexAlgorithm}, schema::{ConstraintSchema, IndexSchema, RowLevelSecuritySchema, SequenceSchema, TableSchema}, }; use spacetimedb_table::{ @@ -442,6 +442,7 @@ impl MutTxId { let columns = match &index.index_algorithm { IndexAlgorithm::BTree(BTreeAlgorithm { columns }) => columns.clone(), + IndexAlgorithm::Direct(DirectAlgorithm { column: _ }) => todo!("todo_direct_index"), _ => unimplemented!(), }; // Create and build the index. diff --git a/crates/core/src/db/datastore/system_tables.rs b/crates/core/src/db/datastore/system_tables.rs index 705a6e1e490..5d083d043ef 100644 --- a/crates/core/src/db/datastore/system_tables.rs +++ b/crates/core/src/db/datastore/system_tables.rs @@ -14,7 +14,7 @@ use crate::db::relational_db::RelationalDB; use crate::error::DBError; use spacetimedb_lib::db::auth::{StAccess, StTableType}; -use spacetimedb_lib::db::raw_def::v9::{RawIndexAlgorithm, RawSql}; +use spacetimedb_lib::db::raw_def::v9::{btree, RawSql}; use spacetimedb_lib::db::raw_def::*; use spacetimedb_lib::de::{Deserialize, DeserializeOwned, Error}; use spacetimedb_lib::ser::Serialize; @@ -26,7 +26,9 @@ use spacetimedb_sats::algebraic_value::ser::value_serialize; use spacetimedb_sats::hash::Hash; use spacetimedb_sats::product_value::InvalidFieldError; use spacetimedb_sats::{impl_deserialize, impl_serialize, impl_st, u256, AlgebraicType, AlgebraicValue, ArrayValue}; -use spacetimedb_schema::def::{BTreeAlgorithm, ConstraintData, IndexAlgorithm, ModuleDef, UniqueConstraintData}; +use spacetimedb_schema::def::{ + BTreeAlgorithm, ConstraintData, DirectAlgorithm, IndexAlgorithm, ModuleDef, UniqueConstraintData, +}; use spacetimedb_schema::schema::{ ColumnSchema, ConstraintSchema, IndexSchema, RowLevelSecuritySchema, ScheduleSchema, Schema, SequenceSchema, TableSchema, @@ -291,36 +293,40 @@ fn system_module_def() -> ModuleDef { .build_table(ST_TABLE_NAME, *st_table_type.as_ref().expect("should be ref")) .with_type(TableType::System) .with_auto_inc_primary_key(StTableFields::TableId) - .with_unique_constraint(StTableFields::TableName); + .with_index_no_accessor_name(btree(StTableFields::TableId)) + .with_unique_constraint(StTableFields::TableName) + .with_index_no_accessor_name(btree(StTableFields::TableName)); let st_raw_column_type = builder.add_type::(); + let st_col_row_unique_cols = [StColumnFields::TableId.col_id(), StColumnFields::ColPos.col_id()]; builder .build_table(ST_COLUMN_NAME, *st_raw_column_type.as_ref().expect("should be ref")) .with_type(TableType::System) - .with_unique_constraint(col_list![ - StColumnFields::TableId.col_id(), - StColumnFields::ColPos.col_id() - ]); + .with_unique_constraint(st_col_row_unique_cols) + .with_index_no_accessor_name(btree(st_col_row_unique_cols)); let st_index_type = builder.add_type::(); builder .build_table(ST_INDEX_NAME, *st_index_type.as_ref().expect("should be ref")) .with_type(TableType::System) - .with_auto_inc_primary_key(StIndexFields::IndexId); + .with_auto_inc_primary_key(StIndexFields::IndexId) + .with_index_no_accessor_name(btree(StIndexFields::IndexId)); // TODO(1.0): unique constraint on name? let st_sequence_type = builder.add_type::(); builder .build_table(ST_SEQUENCE_NAME, *st_sequence_type.as_ref().expect("should be ref")) .with_type(TableType::System) - .with_auto_inc_primary_key(StSequenceFields::SequenceId); + .with_auto_inc_primary_key(StSequenceFields::SequenceId) + .with_index_no_accessor_name(btree(StSequenceFields::SequenceId)); // TODO(1.0): unique constraint on name? let st_constraint_type = builder.add_type::(); builder .build_table(ST_CONSTRAINT_NAME, *st_constraint_type.as_ref().expect("should be ref")) .with_type(TableType::System) - .with_auto_inc_primary_key(StConstraintFields::ConstraintId); + .with_auto_inc_primary_key(StConstraintFields::ConstraintId) + .with_index_no_accessor_name(btree(StConstraintFields::ConstraintId)); // TODO(1.0): unique constraint on name? let st_row_level_security_type = builder.add_type::(); @@ -332,12 +338,8 @@ fn system_module_def() -> ModuleDef { .with_type(TableType::System) .with_primary_key(StRowLevelSecurityFields::Sql) .with_unique_constraint(StRowLevelSecurityFields::Sql) - .with_index( - RawIndexAlgorithm::BTree { - columns: StRowLevelSecurityFields::TableId.into(), - }, - "accessor_name_doesnt_matter", - ); + .with_index_no_accessor_name(btree(StRowLevelSecurityFields::Sql)) + .with_index_no_accessor_name(btree(StRowLevelSecurityFields::TableId)); let st_module_type = builder.add_type::(); builder @@ -346,24 +348,29 @@ fn system_module_def() -> ModuleDef { // TODO: add empty unique constraint here, once we've implemented those. let st_client_type = builder.add_type::(); + let st_client_unique_cols = [StClientFields::Identity, StClientFields::Address]; builder .build_table(ST_CLIENT_NAME, *st_client_type.as_ref().expect("should be ref")) .with_type(TableType::System) - .with_unique_constraint(col_list![StClientFields::Identity, StClientFields::Address]); // FIXME: this is a noop? + .with_unique_constraint(st_client_unique_cols) // FIXME: this is a noop? + .with_index_no_accessor_name(btree(st_client_unique_cols)); let st_schedule_type = builder.add_type::(); builder .build_table(ST_SCHEDULED_NAME, *st_schedule_type.as_ref().expect("should be ref")) .with_type(TableType::System) - .with_unique_constraint(StScheduledFields::TableId) - .with_auto_inc_primary_key(StScheduledFields::ScheduleId); + .with_unique_constraint(StScheduledFields::TableId) // FIXME: this is a noop? + .with_index_no_accessor_name(btree(StScheduledFields::TableId)) + .with_auto_inc_primary_key(StScheduledFields::ScheduleId) // FIXME: this is a noop? + .with_index_no_accessor_name(btree(StScheduledFields::ScheduleId)); // TODO(1.0): unique constraint on name? let st_var_type = builder.add_type::(); builder .build_table(ST_VAR_NAME, *st_var_type.as_ref().expect("should be ref")) .with_type(TableType::System) - .with_unique_constraint(StVarFields::Name) + .with_unique_constraint(StVarFields::Name) // FIXME: this is a noop? + .with_index_no_accessor_name(btree(StVarFields::Name)) .with_primary_key(StVarFields::Name); let result = builder @@ -596,12 +603,16 @@ pub enum StIndexAlgorithm { /// A BTree index. BTree { columns: ColList }, + + /// A Direct index. + Direct { column: ColId }, } impl From for StIndexAlgorithm { fn from(algorithm: IndexAlgorithm) -> Self { match algorithm { - IndexAlgorithm::BTree(BTreeAlgorithm { columns }) => StIndexAlgorithm::BTree { columns }, + IndexAlgorithm::BTree(BTreeAlgorithm { columns }) => Self::BTree { columns }, + IndexAlgorithm::Direct(DirectAlgorithm { column }) => Self::Direct { column }, _ => unimplemented!(), } } @@ -628,6 +639,7 @@ impl From for IndexSchema { index_name: x.index_name, index_algorithm: match x.index_algorithm { StIndexAlgorithm::BTree { columns } => BTreeAlgorithm { columns }.into(), + StIndexAlgorithm::Direct { column } => DirectAlgorithm { column }.into(), StIndexAlgorithm::Unused(_) => panic!("Someone put a forbidden variant in the system table!"), }, } diff --git a/crates/core/src/db/relational_db.rs b/crates/core/src/db/relational_db.rs index 437f3d449cd..662f9f9f75f 100644 --- a/crates/core/src/db/relational_db.rs +++ b/crates/core/src/db/relational_db.rs @@ -30,7 +30,7 @@ pub use spacetimedb_durability::Durability; use spacetimedb_durability::{self as durability, TxOffset}; use spacetimedb_lib::address::Address; use spacetimedb_lib::db::auth::StAccess; -use spacetimedb_lib::db::raw_def::v9::{RawIndexAlgorithm, RawModuleDefV9Builder, RawSql}; +use spacetimedb_lib::db::raw_def::v9::{btree, RawModuleDefV9Builder, RawSql}; use spacetimedb_lib::Identity; use spacetimedb_paths::server::{CommitLogDir, ReplicaDir, SnapshotsPath}; use spacetimedb_primitives::*; @@ -854,12 +854,7 @@ impl RelationalDB { .with_access(access.into()); for columns in indexes { - table_builder = table_builder.with_index( - RawIndexAlgorithm::BTree { - columns: columns.clone(), - }, - "accessor_name_doesnt_matter", - ); + table_builder = table_builder.with_index(btree(columns.clone()), "accessor_name_doesnt_matter"); } table_builder.finish(); let module_def: ModuleDef = module_def_builder.finish().try_into()?; @@ -989,7 +984,7 @@ impl RelationalDB { ) -> Result { let table = self.inner.schema_for_table_mut_tx(tx, table_id)?; - let index = table.indexes.iter().find(|i| i.index_algorithm.columns() == cols); + let index = table.indexes.iter().find(|i| i.index_algorithm.columns() == *cols); let cols_set = ColSet::from(cols); let unique_constraint = table .constraints @@ -1630,7 +1625,7 @@ mod tests { use pretty_assertions::assert_eq; use spacetimedb_client_api_messages::timestamp::Timestamp; use spacetimedb_data_structures::map::IntMap; - use spacetimedb_lib::db::raw_def::v9::RawTableDefBuilder; + use spacetimedb_lib::db::raw_def::v9::{btree, RawTableDefBuilder}; use spacetimedb_lib::error::ResultTest; use spacetimedb_lib::Identity; use spacetimedb_sats::buffer::BufReader; @@ -1666,6 +1661,7 @@ mod tests { .with_primary_key(0) .with_column_sequence(0) .with_unique_constraint(0) + .with_index_no_accessor_name(btree(0)) }, ) } @@ -1675,10 +1671,7 @@ mod tests { "MyTable", ProductType::from([("my_col", AlgebraicType::I64), ("other_col", AlgebraicType::I64)]), |builder| { - let builder = builder.with_index( - RawIndexAlgorithm::BTree { columns: 0.into() }, - "accessor_name_doesnt_matter", - ); + let builder = builder.with_index_no_accessor_name(btree(0)); if is_unique { builder.with_unique_constraint(col_list![0]) @@ -2068,7 +2061,12 @@ mod tests { let schema = table( "MyTable", ProductType::from([("my_col", AlgebraicType::I64)]), - |builder| builder.with_column_sequence(0).with_unique_constraint(0), + |builder| { + builder + .with_column_sequence(0) + .with_unique_constraint(0) + .with_index_no_accessor_name(btree(0)) + }, ); let table_id = stdb.create_table(&mut tx, schema)?; @@ -2104,9 +2102,10 @@ mod tests { ]), |builder| { builder - .with_index(RawIndexAlgorithm::BTree { columns: col_list![0] }, "MyTable_col1_idx") - .with_index(RawIndexAlgorithm::BTree { columns: col_list![2] }, "MyTable_col3_idx") - .with_index(RawIndexAlgorithm::BTree { columns: col_list![3] }, "MyTable_col4_idx") + .with_index_no_accessor_name(btree(0)) + .with_index_no_accessor_name(btree(1)) + .with_index_no_accessor_name(btree(2)) + .with_index_no_accessor_name(btree(3)) .with_unique_constraint(0) .with_unique_constraint(1) .with_unique_constraint(3) @@ -2198,12 +2197,7 @@ mod tests { ]); let schema = table("t", columns, |builder| { - builder.with_index( - RawIndexAlgorithm::BTree { - columns: col_list![0, 1], - }, - "accessor_name_doesnt_matter", - ) + builder.with_index(btree([0, 1]), "accessor_name_doesnt_matter") }); let mut tx = stdb.begin_mut_tx(IsolationLevel::Serializable, Workload::ForTests); diff --git a/crates/lib/src/db/raw_def/v9.rs b/crates/lib/src/db/raw_def/v9.rs index 6d632531177..4cae222c108 100644 --- a/crates/lib/src/db/raw_def/v9.rs +++ b/crates/lib/src/db/raw_def/v9.rs @@ -274,6 +274,23 @@ pub enum RawIndexAlgorithm { /// The columns to index on. These are ordered. columns: ColList, }, + /// Implemented using direct indexing in list(s) of `RowPointer`s. + /// The column this is placed on must also have a unique constraint. + Direct { + /// The column to index on. + /// Only one is allowed, as direct indexing with more is nonsensical. + column: ColId, + }, +} + +/// Returns a btree index algorithm for the columns `cols`. +pub fn btree(cols: impl Into) -> RawIndexAlgorithm { + RawIndexAlgorithm::BTree { columns: cols.into() } +} + +/// Returns a direct index algorithm for the column `col`. +pub fn direct(col: impl Into) -> RawIndexAlgorithm { + RawIndexAlgorithm::Direct { column: col.into() } } /// Marks a table as a timer table for a scheduled reducer. @@ -709,7 +726,7 @@ impl RawTableDefBuilder<'_> { } /// Adds a primary key to the table, with corresponding unique constraint and sequence definitions. - /// This will also result in an index being created for the unique constraint. + /// You will also need to call [`Self::with_index`] to create an index on `column`. pub fn with_auto_inc_primary_key(self, column: impl Into) -> Self { let column = column.into(); self.with_primary_key(column) @@ -729,6 +746,16 @@ impl RawTableDefBuilder<'_> { self } + /// Generates a [RawIndexDef] using the supplied `columns` but with no `accessor_name`. + pub fn with_index_no_accessor_name(mut self, algorithm: RawIndexAlgorithm) -> Self { + self.table.indexes.push(RawIndexDefV9 { + name: None, + accessor_name: None, + algorithm, + }); + self + } + /// Adds a [RawSequenceDef] on the supplied `column`. pub fn with_column_sequence(mut self, column: impl Into) -> Self { let column = column.into(); diff --git a/crates/primitives/src/col_list.rs b/crates/primitives/src/col_list.rs index 322e6ea1e6c..f4155cb4867 100644 --- a/crates/primitives/src/col_list.rs +++ b/crates/primitives/src/col_list.rs @@ -347,6 +347,68 @@ impl From for ColList { } } +/// A borrowed list of columns or a single one. +pub enum ColOrCols<'a> { + /// A single column. + Col(ColId), + /// A list of columns. + ColList(&'a ColList), +} + +impl ColOrCols<'_> { + /// Returns `Some(col)` iff `self` is singleton. + pub fn as_singleton(&self) -> Option { + match self { + Self::Col(col) => Some(*col), + Self::ColList(cols) => cols.as_singleton(), + } + } + + /// Returns an iterator over all the columns in this list. + pub fn iter(&self) -> impl '_ + Iterator { + match self { + Self::Col(col) => Either::Left(iter::once(*col)), + Self::ColList(cols) => Either::Right(cols.iter()), + } + } + + /// Returns the length of this list. + pub fn len(&self) -> u16 { + match self { + Self::Col(_) => 1, + Self::ColList(cols) => cols.len(), + } + } + + /// Returns whether the list is empty. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +impl PartialEq for ColOrCols<'_> { + fn eq(&self, other: &ColList) -> bool { + self.iter().eq(other.iter()) + } +} +impl PartialEq for ColOrCols<'_> { + fn eq(&self, other: &Self) -> bool { + self.iter().eq(other.iter()) + } +} + +impl Eq for ColOrCols<'_> {} +impl Ord for ColOrCols<'_> { + fn cmp(&self, other: &Self) -> Ordering { + self.iter().cmp(other.iter()) + } +} +impl PartialOrd for ColOrCols<'_> { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + /// A compressed set of columns. Like a `ColList`, but guaranteed to be sorted and to contain no duplicate entries. /// Dereferences to a `ColList` for convenience. #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -387,6 +449,15 @@ impl From<&ColList> for ColSet { } } +impl From> for ColSet { + fn from(value: ColOrCols<'_>) -> Self { + match value { + ColOrCols::Col(col) => ColSet(col.into()), + ColOrCols::ColList(cols) => cols.into(), + } + } +} + impl> From for ColSet { fn from(value: C) -> Self { Self::from(ColList::new(value.into())) diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index c129f25d933..7ae37765514 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -6,7 +6,7 @@ pub mod errno; mod ids; pub use attr::{AttributeKind, ColumnAttribute, ConstraintKind, Constraints}; -pub use col_list::{ColList, ColSet}; +pub use col_list::{ColList, ColOrCols, ColSet}; pub use ids::{ColId, ConstraintId, IndexId, ReducerId, ScheduleId, SequenceId, TableId}; /// The minimum size of a chunk yielded by a wasm abi RowIter. diff --git a/crates/schema/src/auto_migrate.rs b/crates/schema/src/auto_migrate.rs index 61e8e525fc4..6a016ffe7b1 100644 --- a/crates/schema/src/auto_migrate.rs +++ b/crates/schema/src/auto_migrate.rs @@ -458,9 +458,12 @@ fn auto_migrate_row_level_security(plan: &mut AutoMigratePlan) -> Result<()> { mod tests { use super::*; use spacetimedb_data_structures::expect_error_matching; - use spacetimedb_lib::{db::raw_def::*, AlgebraicType, ProductType, ScheduleAt}; - use spacetimedb_primitives::{ColId, ColList}; - use v9::{RawIndexAlgorithm, RawModuleDefV9Builder, TableAccess}; + use spacetimedb_lib::{ + db::raw_def::{v9::btree, *}, + AlgebraicType, ProductType, ScheduleAt, + }; + use spacetimedb_primitives::ColId; + use v9::{RawModuleDefV9Builder, TableAccess}; use validate::tests::expect_identifier; #[test] @@ -479,18 +482,8 @@ mod tests { ) .with_column_sequence(0) .with_unique_constraint(ColId(0)) - .with_index( - RawIndexAlgorithm::BTree { - columns: ColList::from([0]), - }, - "id_index", - ) - .with_index( - RawIndexAlgorithm::BTree { - columns: ColList::from([0, 1]), - }, - "id_name_index", - ) + .with_index(btree(0), "id_index") + .with_index(btree([0, 1]), "id_name_index") .finish(); old_builder @@ -516,6 +509,7 @@ mod tests { true, ) .with_auto_inc_primary_key(0) + .with_index_no_accessor_name(btree(0)) .with_schedule("check_deliveries", 1) .finish(); old_builder.add_reducer( @@ -534,6 +528,7 @@ mod tests { true, ) .with_auto_inc_primary_key(0) + .with_index_no_accessor_name(btree(0)) .finish(); old_builder.add_row_level_security("SELECT * FROM Apples"); @@ -558,20 +553,10 @@ mod tests { ) // remove sequence // remove unique constraint - .with_index( - RawIndexAlgorithm::BTree { - columns: ColList::from([0]), - }, - "id_index", - ) + .with_index(btree(0), "id_index") // remove ["id", "name"] index // add ["id", "count"] index - .with_index( - RawIndexAlgorithm::BTree { - columns: ColList::from([0, 2]), - }, - "id_count_index", - ) + .with_index(btree([0, 2]), "id_count_index") .finish(); new_builder @@ -600,6 +585,7 @@ mod tests { true, ) .with_auto_inc_primary_key(0) + .with_index_no_accessor_name(btree(0)) // remove schedule def .finish(); @@ -619,6 +605,7 @@ mod tests { true, ) .with_auto_inc_primary_key(0) + .with_index_no_accessor_name(btree(0)) // add schedule def .with_schedule("perform_inspection", 1) .finish(); @@ -633,12 +620,7 @@ mod tests { // Add new table new_builder .build_table_with_new_type("Oranges", ProductType::from([("id", AlgebraicType::U32)]), true) - .with_index( - RawIndexAlgorithm::BTree { - columns: ColList::from([0]), - }, - "id_index", - ) + .with_index(btree(0), "id_index") .with_column_sequence(0) .with_unique_constraint(0) .with_primary_key(0) @@ -741,13 +723,9 @@ mod tests { ]), true, ) - .with_index( - RawIndexAlgorithm::BTree { - columns: ColList::from([0]), - }, - "id_index", - ) - .with_unique_constraint(ColList::from_iter([1, 2])) + .with_index(btree(0), "id_index") + .with_unique_constraint([1, 2]) + .with_index_no_accessor_name(btree([1, 2])) .with_type(TableType::User) .finish(); @@ -782,13 +760,13 @@ mod tests { true, ) .with_index( - RawIndexAlgorithm::BTree { - columns: ColList::from([1]), - }, + btree(1), "id_index_new_accessor", // change accessor name ) - .with_unique_constraint(ColList::from_iter([1, 0])) - .with_unique_constraint(ColId(0)) // add unique constraint + .with_unique_constraint([1, 0]) + .with_index_no_accessor_name(btree([1, 0])) + .with_unique_constraint(0) + .with_index_no_accessor_name(btree(0)) // add unique constraint .with_type(TableType::System) // change type .finish(); diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index 72795446aaa..e949590d62b 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -37,10 +37,9 @@ use spacetimedb_lib::db::raw_def::v9::{ RawSql, RawTableDefV9, RawTypeDefV9, RawUniqueConstraintDataV9, TableAccess, TableType, }; use spacetimedb_lib::{ProductType, RawModuleDef}; -use spacetimedb_primitives::{ColId, ColList, ColSet, ReducerId, TableId}; +use spacetimedb_primitives::{ColId, ColList, ColOrCols, ColSet, ReducerId, TableId}; use spacetimedb_sats::AlgebraicType; use spacetimedb_sats::{AlgebraicTypeRef, Typespace}; -use validate::v9::generate_index_name; pub mod deserialize; pub mod validate; @@ -280,58 +279,6 @@ impl ModuleDef { Some(TableSchema::from_module_def(self, table_def, (), table_id)) } - /// Generate indexes for the module definition. - /// We guarantee that all `unique` constraints have an index generated for them. - /// This will be removed once another enforcement mechanism is implemented. - /// This is a noop if there are already usable indexes present. - fn generate_indexes(&mut self) { - for table in self.tables.values_mut() { - for constraint in table.constraints.values() { - let ConstraintData::Unique(UniqueConstraintData { columns }) = &constraint.data; - - // if we have a constraint for the index, we're fine. - if table.indexes.values().any(|index| { - let IndexDef { - algorithm: IndexAlgorithm::BTree(BTreeAlgorithm { columns: index_columns }), - .. - } = index; - - index_columns == &**columns - }) { - continue; - } - - let index_name = generate_index_name( - &table.name, - self.typespace - .get(table.product_type_ref) - .unwrap() - .as_product() - .unwrap(), - &RawIndexAlgorithm::BTree { - columns: columns.clone().into(), - }, - ); - - let was_present = table.indexes.insert( - index_name.clone(), - IndexDef { - name: index_name.clone(), - algorithm: IndexAlgorithm::BTree(BTreeAlgorithm { - columns: columns.clone().into(), - }), - accessor_name: None, // this is a generated index. - }, - ); - assert!( - was_present.is_none(), - "generated index already present, which should be impossible" - ); - self.stored_in_table_def.insert(index_name, table.name.clone()); - } - } - } - /// Lookup a definition by its key in `self`, panicking if it is not found. pub fn expect_lookup(&self, key: T::Key<'_>) -> &T { if let Some(result) = T::lookup(self, key) { @@ -379,9 +326,7 @@ impl TryFrom for ModuleDef { type Error = ValidationErrors; fn try_from(v9_mod: raw_def::v9::RawModuleDefV9) -> Result { - let mut result = validate::v9::validate(v9_mod)?; - result.generate_indexes(); - Ok(result) + validate::v9::validate(v9_mod) } } impl From for RawModuleDefV9 { @@ -604,6 +549,7 @@ impl From for RawIndexDefV9 { name: Some(val.name), algorithm: match val.algorithm { IndexAlgorithm::BTree(BTreeAlgorithm { columns }) => RawIndexAlgorithm::BTree { columns }, + IndexAlgorithm::Direct(DirectAlgorithm { column }) => RawIndexAlgorithm::Direct { column }, }, accessor_name: val.accessor_name.map(Into::into), } @@ -616,13 +562,16 @@ impl From for RawIndexDefV9 { pub enum IndexAlgorithm { /// Implemented using a rust `std::collections::BTreeMap`. BTree(BTreeAlgorithm), + /// Implemented using `DirectUniqueIndex`. + Direct(DirectAlgorithm), } impl IndexAlgorithm { /// Get the columns of the index. - pub fn columns(&self) -> &ColList { + pub fn columns(&self) -> ColOrCols<'_> { match self { - IndexAlgorithm::BTree(btree) => &btree.columns, + Self::BTree(btree) => ColOrCols::ColList(&btree.columns), + Self::Direct(direct) => ColOrCols::Col(direct.column), } } } @@ -630,7 +579,8 @@ impl IndexAlgorithm { impl From for RawIndexAlgorithm { fn from(val: IndexAlgorithm) -> Self { match val { - IndexAlgorithm::BTree(BTreeAlgorithm { columns }) => RawIndexAlgorithm::BTree { columns }, + IndexAlgorithm::BTree(BTreeAlgorithm { columns }) => Self::BTree { columns }, + IndexAlgorithm::Direct(DirectAlgorithm { column }) => Self::Direct { column }, } } } @@ -648,6 +598,19 @@ impl From for IndexAlgorithm { } } +/// Data specifying a Direct index. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct DirectAlgorithm { + /// The column to index. + pub column: ColId, +} + +impl From for IndexAlgorithm { + fn from(val: DirectAlgorithm) -> Self { + IndexAlgorithm::Direct(val) + } +} + /// A struct representing the validated definition of a database column. /// /// Cannot be created directly. Construct a [`ModuleDef`] by validating a [`RawModuleDef`] instead, diff --git a/crates/schema/src/def/validate/v8.rs b/crates/schema/src/def/validate/v8.rs index 0f407d1993a..9e184cb6a17 100644 --- a/crates/schema/src/def/validate/v8.rs +++ b/crates/schema/src/def/validate/v8.rs @@ -72,6 +72,7 @@ fn upgrade_table( // This is the hairiest part of v8. let generated_constraints = table.schema.generated_constraints().collect::>(); let generated_sequences = table.schema.generated_sequences().collect::>(); + let generated_indexes = table.schema.generated_indexes().collect::>(); let RawTableDescV8 { schema: @@ -96,7 +97,7 @@ fn upgrade_table( check_all_column_defs(product_type_ref, columns, &table_name, typespace, extra_errors); // Now we're ready to go through the various definitions and upgrade them. - let indexes = convert_all(indexes, upgrade_index); + let indexes = convert_all(indexes.into_iter().chain(generated_indexes), upgrade_index); let sequences = convert_all(sequences.into_iter().chain(generated_sequences), upgrade_sequence); let schedule = upgrade_schedule(scheduled, scheduled_at_col); @@ -390,7 +391,7 @@ mod tests { ]), ) .with_column_constraint(Constraints::primary_key_auto(), 0) - .with_column_index(ColList::from_iter([0, 1, 2]), false), + .with_column_index([0, 1, 2], false), ); let deliveries_product_type = builder.add_table_for_tests( @@ -743,7 +744,7 @@ mod tests { ); let result: Result = builder.finish().try_into(); - expect_error_matching!(result, ValidationError::OnlyBtree { index } => { + expect_error_matching!(result, ValidationError::HashIndexUnsupported { index } => { &index[..] == "Bananas_index" }); } diff --git a/crates/schema/src/def/validate/v9.rs b/crates/schema/src/def/validate/v9.rs index 533199d5ca6..712b181eddf 100644 --- a/crates/schema/src/def/validate/v9.rs +++ b/crates/schema/src/def/validate/v9.rs @@ -6,6 +6,7 @@ use spacetimedb_data_structures::error_stream::{CollectAllErrors, CombineErrors} use spacetimedb_data_structures::map::HashSet; use spacetimedb_lib::db::default_element_ordering::{product_type_has_default_ordering, sum_type_has_default_ordering}; use spacetimedb_lib::ProductType; +use spacetimedb_primitives::col_list; /// Validate a `RawModuleDefV9` and convert it into a `ModuleDef`, /// or return a stream of errors if the definition is invalid. @@ -99,7 +100,7 @@ pub fn validate(def: RawModuleDefV9) -> Result { let typespace_for_generate = typespace_for_generate.finish(); - let mut result = ModuleDef { + Ok(ModuleDef { tables, reducers, types, @@ -109,11 +110,7 @@ pub fn validate(def: RawModuleDefV9) -> Result { refmap, row_level_security_raw, lifecycle_reducers, - }; - - result.generate_indexes(); - - Ok(result) + }) } /// Collects state used during validation. @@ -186,7 +183,7 @@ impl ModuleValidator<'_> { .validate_index_def(index) .map(|index| (index.name.clone(), index)) }) - .collect_all_errors(); + .collect_all_errors::>(); // We can't validate the primary key without validating the unique constraints first. let primary_key_head = primary_key.head(); @@ -202,6 +199,47 @@ impl ModuleValidator<'_> { table_in_progress.validate_primary_key(constraints, primary_key) }); + // Now that we've validated indices and constraints separately, + // we can validate their interactions. + // More specifically, a direct index requires a unique constraint. + let constraints_backed_by_indices = + if let (Ok((constraints, _)), Ok(indexes)) = (&constraints_primary_key, &indexes) { + constraints + .values() + .filter_map(|c| c.data.unique_columns().map(|cols| (c, cols))) + // TODO(centril): this check is actually too strict + // and ends up unnecessarily inducing extra indices. + // + // It is sufficient for `unique_cols` to: + // a) be a permutation of `index`'s columns, + // as a permutation of a set is still the same set, + // so when we use the index to check the constraint, + // the order in the index does not matter for the purposes of the constraint. + // + // b) for `unique_cols` to form a prefix of `index`'s columns, + // if the index provides efficient prefix scans. + // + // Currently, b) is unsupported, + // as we cannot decouple unique constraints from indices in the datastore today, + // and we cannot mark the entire index unique, + // as that would not be a sound representation of what the user wanted. + // If we wanted to, we could make the constraints merely use indices, + // rather than be indices. + .filter(|(_, unique_cols)| { + !indexes + .values() + .any(|i| ColSet::from(i.algorithm.columns()) == **unique_cols) + }) + .map(|(c, cols)| { + let constraint = c.name.clone(); + let columns = cols.clone(); + Err(ValidationError::UniqueConstraintWithoutIndex { constraint, columns }.into()) + }) + .collect_all_errors() + } else { + Ok(()) + }; + let sequences = sequences .into_iter() .map(|sequence| { @@ -226,8 +264,16 @@ impl ModuleValidator<'_> { } }); - let (name, columns, indexes, (constraints, primary_key), sequences, schedule) = - (name, columns, indexes, constraints_primary_key, sequences, schedule).combine_errors()?; + let (name, columns, indexes, (constraints, primary_key), (), sequences, schedule) = ( + name, + columns, + indexes, + constraints_primary_key, + constraints_backed_by_indices, + sequences, + schedule, + ) + .combine_errors()?; Ok(TableDef { name, @@ -575,7 +621,22 @@ impl TableValidator<'_, '_> { RawIndexAlgorithm::BTree { columns } => self .validate_col_ids(&name, columns) .map(|columns| BTreeAlgorithm { columns }.into()), - _ => Err(ValidationError::OnlyBtree { index: name.clone() }.into()), + RawIndexAlgorithm::Direct { column } => self.validate_col_id(&name, column).and_then(|column| { + let field = &self.product_type.elements[column.idx()]; + let ty = &field.algebraic_type; + use AlgebraicType::*; + if let U8 | U16 | U32 | U64 = ty { + } else { + return Err(ValidationError::DirectIndexOnNonUnsignedInt { + index: name.clone(), + column: field.name.clone().unwrap_or_else(|| column.idx().to_string().into()), + ty: ty.clone().into(), + } + .into()); + } + Ok(DirectAlgorithm { column }.into()) + }), + _ => Err(ValidationError::HashIndexUnsupported { index: name.clone() }.into()), }; let name = self.add_to_global_namespace(name); let accessor_name = accessor_name.map(identifier).transpose(); @@ -759,6 +820,7 @@ fn concat_column_names(table_type: &ProductType, selected: &ColList) -> String { pub fn generate_index_name(table_name: &str, table_type: &ProductType, algorithm: &RawIndexAlgorithm) -> RawIdentifier { let (label, columns) = match algorithm { RawIndexAlgorithm::BTree { columns } => ("btree", columns), + RawIndexAlgorithm::Direct { column } => ("direct", &col_list![*column]), RawIndexAlgorithm::Hash { columns } => ("hash", columns), _ => unimplemented!("Unknown index algorithm {:?}", algorithm), }; @@ -835,14 +897,18 @@ mod tests { check_product_type, expect_identifier, expect_raw_type_name, expect_resolve, expect_type_name, }; use crate::def::{validate::Result, ModuleDef}; - use crate::def::{BTreeAlgorithm, ConstraintData, ConstraintDef, IndexDef, SequenceDef, UniqueConstraintData}; + use crate::def::{ + BTreeAlgorithm, ConstraintData, ConstraintDef, DirectAlgorithm, IndexDef, SequenceDef, UniqueConstraintData, + }; use crate::error::*; use crate::type_for_generate::ClientCodegenError; + use itertools::Itertools; use spacetimedb_data_structures::expect_error_matching; + use spacetimedb_lib::db::raw_def::v9::{btree, direct}; use spacetimedb_lib::db::raw_def::*; use spacetimedb_lib::ScheduleAt; - use spacetimedb_primitives::{col_list, ColId, ColList}; + use spacetimedb_primitives::{ColId, ColList, ColSet}; use spacetimedb_sats::{AlgebraicType, AlgebraicTypeRef, ProductType}; use v9::{Lifecycle, RawIndexAlgorithm, RawModuleDefV9Builder, TableAccess, TableType}; @@ -875,12 +941,10 @@ mod tests { ]), true, ) - .with_index( - RawIndexAlgorithm::BTree { - columns: ColList::from_iter([1, 2]), - }, - "apples_id", - ) + .with_index(btree([1, 2]), "apples_id") + .with_index(direct(2), "Apples_count_direct") + .with_unique_constraint(2) + .with_index(btree(3), "Apples_type_btree") .with_unique_constraint(3) .finish(); @@ -902,13 +966,8 @@ mod tests { .with_unique_constraint(ColId(0)) .with_primary_key(0) .with_access(TableAccess::Private) - .with_index(RawIndexAlgorithm::BTree { columns: 0.into() }, "bananas_count") - .with_index( - RawIndexAlgorithm::BTree { - columns: ColList::from_iter([0, 1, 2]), - }, - "bananas_count_id_name", - ) + .with_index(btree(0), "bananas_count") + .with_index(btree([0, 1, 2]), "bananas_count_id_name") .finish(); let deliveries_product_type = builder @@ -922,6 +981,7 @@ mod tests { true, ) .with_auto_inc_primary_key(2) + .with_index(btree(2), "scheduled_id_index") .with_schedule("check_deliveries", 1) .with_type(TableType::System) .finish(); @@ -963,7 +1023,7 @@ mod tests { assert_eq!(apples_def.primary_key, None); - assert_eq!(apples_def.constraints.len(), 1); + assert_eq!(apples_def.constraints.len(), 2); let apples_unique_constraint = "Apples_type_key"; assert_eq!( apples_def.constraints[apples_unique_constraint].data, @@ -976,27 +1036,31 @@ mod tests { apples_unique_constraint ); - assert_eq!(apples_def.indexes.len(), 2); - for index in apples_def.indexes.values() { - match &index.name[..] { - // manually added - "Apples_name_count_idx_btree" => { - assert_eq!( - index.algorithm, - BTreeAlgorithm { - columns: ColList::from_iter([1, 2]) - } - .into() - ); - assert_eq!(index.accessor_name, Some(expect_identifier("apples_id"))); - } - // auto-generated for the unique constraint - _ => { - assert_eq!(index.algorithm, BTreeAlgorithm { columns: 3.into() }.into()); - assert_eq!(index.accessor_name, None); + assert_eq!(apples_def.indexes.len(), 3); + assert_eq!( + apples_def + .indexes + .values() + .sorted_by_key(|id| &id.name) + .collect::>(), + [ + &IndexDef { + name: "Apples_count_idx_direct".into(), + accessor_name: Some(expect_identifier("Apples_count_direct")), + algorithm: DirectAlgorithm { column: 2.into() }.into(), + }, + &IndexDef { + name: "Apples_name_count_idx_btree".into(), + accessor_name: Some(expect_identifier("apples_id")), + algorithm: BTreeAlgorithm { columns: [1, 2].into() }.into(), + }, + &IndexDef { + name: "Apples_type_idx_btree".into(), + accessor_name: Some(expect_identifier("Apples_type_btree")), + algorithm: BTreeAlgorithm { columns: 3.into() }.into(), } - } - } + ] + ); let bananas_def = &def.tables[&bananas]; @@ -1172,12 +1236,7 @@ mod tests { ProductType::from([("b", AlgebraicType::U16), ("a", AlgebraicType::U64)]), false, ) - .with_index( - RawIndexAlgorithm::BTree { - columns: ColList::from_iter([0, 55]), - }, - "bananas_a_b", - ) + .with_index(btree([0, 55]), "bananas_a_b") .finish(); let result: Result = builder.finish().try_into(); @@ -1256,12 +1315,7 @@ mod tests { ProductType::from([("b", AlgebraicType::U16), ("a", AlgebraicType::U64)]), false, ) - .with_index( - RawIndexAlgorithm::BTree { - columns: ColList::from_iter([0, 0]), - }, - "bananas_b_b", - ) + .with_index(btree([0, 0]), "bananas_b_b") .finish(); let result: Result = builder.finish().try_into(); @@ -1335,7 +1389,7 @@ mod tests { } #[test] - fn only_btree_indexes() { + fn hash_index_unsupported() { let mut builder = RawModuleDefV9Builder::new(); builder .build_table_with_new_type( @@ -1347,11 +1401,50 @@ mod tests { .finish(); let result: Result = builder.finish().try_into(); - expect_error_matching!(result, ValidationError::OnlyBtree { index } => { + expect_error_matching!(result, ValidationError::HashIndexUnsupported { index } => { &index[..] == "Bananas_b_idx_hash" }); } + #[test] + fn unique_constrain_without_index() { + let mut builder = RawModuleDefV9Builder::new(); + builder + .build_table_with_new_type( + "Bananas", + ProductType::from([("b", AlgebraicType::U16), ("a", AlgebraicType::U64)]), + false, + ) + .with_unique_constraint(1) + .finish(); + let result: Result = builder.finish().try_into(); + + expect_error_matching!( + result, + ValidationError::UniqueConstraintWithoutIndex { constraint, columns } => { + &**constraint == "Bananas_a_key" && *columns == ColSet::from(1) + } + ); + } + + #[test] + fn direct_index_only_u8_to_u64() { + let mut builder = RawModuleDefV9Builder::new(); + builder + .build_table_with_new_type( + "Bananas", + ProductType::from([("b", AlgebraicType::I32), ("a", AlgebraicType::U64)]), + false, + ) + .with_index(direct(0), "bananas_b") + .finish(); + let result: Result = builder.finish().try_into(); + + expect_error_matching!(result, ValidationError::DirectIndexOnNonUnsignedInt { index, .. } => { + &index[..] == "Bananas_b_idx_direct" + }); + } + #[test] fn one_auto_inc() { let mut builder = RawModuleDefV9Builder::new(); @@ -1458,6 +1551,7 @@ mod tests { true, ) .with_auto_inc_primary_key(2) + .with_index(btree(2), "scheduled_id_index") .with_schedule("check_deliveries", 1) .with_type(TableType::System) .finish(); @@ -1484,6 +1578,7 @@ mod tests { true, ) .with_auto_inc_primary_key(2) + .with_index(direct(2), "scheduled_id_idx") .with_schedule("check_deliveries", 1) .with_type(TableType::System) .finish(); @@ -1514,12 +1609,8 @@ mod tests { true, ) .with_auto_inc_primary_key(2) - .with_index( - RawIndexAlgorithm::BTree { - columns: col_list![0, 2], - }, - "nice_index_name", - ) + .with_index(direct(2), "scheduled_id_index") + .with_index(btree([0, 2]), "nice_index_name") .with_schedule("check_deliveries", 1) .with_type(TableType::System) .finish(); diff --git a/crates/schema/src/error.rs b/crates/schema/src/error.rs index 6101c54c8f9..fda129affbb 100644 --- a/crates/schema/src/error.rs +++ b/crates/schema/src/error.rs @@ -1,7 +1,7 @@ use spacetimedb_data_structures::error_stream::ErrorStream; use spacetimedb_lib::db::raw_def::v9::{Lifecycle, RawIdentifier, RawScopedTypeNameV9}; use spacetimedb_lib::{ProductType, SumType}; -use spacetimedb_primitives::{ColId, ColList}; +use spacetimedb_primitives::{ColId, ColList, ColSet}; use spacetimedb_sats::algebraic_type::fmt::fmt_algebraic_type; use spacetimedb_sats::{AlgebraicType, AlgebraicTypeRef}; use std::borrow::Cow; @@ -57,8 +57,16 @@ pub enum ValidationError { RepeatedPrimaryKey { table: RawIdentifier }, #[error("Attempt to define {column} with more than 1 auto_inc sequence")] OneAutoInc { column: RawColumnName }, - #[error("Only Btree Indexes are supported: index `{index}` is not a btree")] - OnlyBtree { index: RawIdentifier }, + #[error("Hash indexes are not supported: `{index}` is a hash index")] + HashIndexUnsupported { index: RawIdentifier }, + #[error("No index found to support unique constraint `{constraint}` for columns `{columns:?}`")] + UniqueConstraintWithoutIndex { constraint: Box, columns: ColSet }, + #[error("Direct index does not support type `{ty}` in column `{column}` in index `{index}`")] + DirectIndexOnNonUnsignedInt { + index: RawIdentifier, + column: RawIdentifier, + ty: PrettyAlgebraicType, + }, #[error("def `{def}` has duplicate columns: {columns:?}")] DuplicateColumns { def: RawIdentifier, columns: ColList }, #[error("invalid sequence column type: {column} with type `{column_type:?}` in sequence `{sequence}`")] diff --git a/crates/schema/src/schema.rs b/crates/schema/src/schema.rs index 5ecacbdb92e..cf14c675c72 100644 --- a/crates/schema/src/schema.rs +++ b/crates/schema/src/schema.rs @@ -332,6 +332,7 @@ impl TableSchema { }) .chain(self.indexes.iter().map(|x| match &x.index_algorithm { IndexAlgorithm::BTree(btree) => (btree.columns.clone(), Constraints::indexed()), + IndexAlgorithm::Direct(direct) => (direct.column.into(), Constraints::indexed()), })) .chain( self.sequences @@ -390,8 +391,12 @@ impl TableSchema { .sequences .iter() .map(|x| (DefType::Sequence, x.sequence_name.clone(), ColList::new(x.col_pos))) - .chain(self.indexes.iter().map(|x| match &x.index_algorithm { - IndexAlgorithm::BTree(btree) => (DefType::Index, x.index_name.clone(), btree.columns.clone()), + .chain(self.indexes.iter().map(|x| { + let cols = match &x.index_algorithm { + IndexAlgorithm::BTree(btree) => btree.columns.clone(), + IndexAlgorithm::Direct(direct) => direct.column.into(), + }; + (DefType::Index, x.index_name.clone(), cols) })) .chain(self.constraints.iter().map(|x| { (