From 9fea2853c02e4628940b4d03ddc743a1795b48e1 Mon Sep 17 00:00:00 2001 From: Gurleen Dhody Date: Fri, 12 Feb 2021 11:34:29 -0800 Subject: [PATCH 1/4] code --- .../microsoft/hyperspace/index/Config.scala | 129 ++++++++++++++ .../hyperspace/index/ConfigBase.scala | 25 +++ .../hyperspace/index/IndexConfig.scala | 166 ------------------ .../index/configs/covering/IndexConfig.scala | 92 ++++++++++ .../noncovering/BloomFilterIndexConfig.scala | 24 +++ 5 files changed, 270 insertions(+), 166 deletions(-) create mode 100644 src/main/scala/com/microsoft/hyperspace/index/Config.scala create mode 100644 src/main/scala/com/microsoft/hyperspace/index/ConfigBase.scala delete mode 100644 src/main/scala/com/microsoft/hyperspace/index/IndexConfig.scala create mode 100644 src/main/scala/com/microsoft/hyperspace/index/configs/covering/IndexConfig.scala create mode 100644 src/main/scala/com/microsoft/hyperspace/index/configs/noncovering/BloomFilterIndexConfig.scala diff --git a/src/main/scala/com/microsoft/hyperspace/index/Config.scala b/src/main/scala/com/microsoft/hyperspace/index/Config.scala new file mode 100644 index 000000000..264dbb4e2 --- /dev/null +++ b/src/main/scala/com/microsoft/hyperspace/index/Config.scala @@ -0,0 +1,129 @@ +package com.microsoft.hyperspace.index + +import java.util.Locale + +/** + * IndexConfig specifies the configuration of an index. + * Associated Builder [[com.microsoft.hyperspace.index.configs.covering.IndexConfig.builder()]] + * + * @param indexName Index name. + * @param indexedColumns Columns from which an index is created. + * @param includedColumns Columns to be included in the index. + */ +case class IndexConfig( + indexName: String, + indexedColumns: Seq[String], + includedColumns: Seq[String] = Seq()) + extends CoveringIndexConfigBase { + if (indexName.isEmpty || indexedColumns.isEmpty) { + throw new IllegalArgumentException("Empty index name or indexed columns are not allowed.") + } + + val lowerCaseIndexedColumns: Seq[String] = toLowerCase(indexedColumns) + val lowerCaseIncludedColumns: Seq[String] = toLowerCase(includedColumns) + val lowerCaseIncludedColumnsSet: Set[String] = lowerCaseIncludedColumns.toSet + + if (lowerCaseIndexedColumns.toSet.size < lowerCaseIndexedColumns.size) { + throw new IllegalArgumentException("Duplicate indexed column names are not allowed.") + } + + if (lowerCaseIncludedColumnsSet.size < lowerCaseIncludedColumns.size) { + throw new IllegalArgumentException("Duplicate included column names are not allowed.") + } + + for (indexedColumn <- lowerCaseIndexedColumns) { + if (lowerCaseIncludedColumns.contains(indexedColumn)) { + throw new IllegalArgumentException( + "Duplicate column names in indexed/included columns are not allowed.") + } + } + + override def equals(that: Any): Boolean = { + that match { + case IndexConfig(thatIndexName, thatIndexedColumns, thatIncludedColumns) => + indexName.equalsIgnoreCase(thatIndexName) && + lowerCaseIndexedColumns.equals(toLowerCase(thatIndexedColumns)) && + lowerCaseIncludedColumnsSet.equals(toLowerCase(thatIncludedColumns).toSet) + case _ => false + } + } + + override def hashCode(): Int = { + lowerCaseIndexedColumns.hashCode + lowerCaseIncludedColumnsSet.hashCode + } + + override def toString: String = { + val indexedColumnNames = lowerCaseIndexedColumns.mkString(", ") + val includedColumnNames = lowerCaseIncludedColumns.mkString(", ") + s"[indexName: $indexName; indexedColumns: $indexedColumnNames; " + + s"includedColumns: $includedColumnNames]" + } + + private def toLowerCase(seq: Seq[String]): Seq[String] = seq.map(_.toLowerCase(Locale.ROOT)) +} + +/** + * TODO + * @param indexName + * @param indexedColumn + * @param expectedNumItems + * @param fpp + * @param numBits + */ +case class BloomFilterIndexConfig( + indexName: String, + indexedColumn: String, + expectedNumItems: Long, + fpp: Double = -1, + numBits: Long = -1) + extends NonCoveringIndexConfigBase { + if (indexName.isEmpty || indexedColumn.isEmpty || expectedNumItems < 1) { + throw new IllegalArgumentException( + "Empty index name or indexed column or expected items less than 1 are not allowed.") + } + + if (fpp != -1 && fpp < 0) { + throw new IllegalArgumentException("False positive probability cannot be negative.") + } + + if (numBits != -1 && numBits < 0) { + throw new IllegalArgumentException("Bits given for ") + } + + val lowerCaseIndexedColumn: String = indexedColumn.toLowerCase(Locale.ROOT) + + override def equals(that: Any): Boolean = { + that match { + case BloomFilterIndexConfig( + thatIndexName, + thatIndexedColumn, + thatExpectedItems, + thatFpp, + thatBits) => + indexName.equalsIgnoreCase(thatIndexName) && + lowerCaseIndexedColumn.equals(thatIndexedColumn.toLowerCase(Locale.ROOT)) && + expectedNumItems == thatExpectedItems && + ((fpp == -1 && thatFpp == -1) || fpp == thatFpp) && + ((numBits == -1 && thatBits == -1) || numBits == thatBits) + case _ => false + } + } + + override def hashCode(): Int = { + (indexName.hashCode * (indexedColumn.hashCode + expectedNumItems + .hashCode())) % scala.Int.MaxValue + } + + override def toString: String = { + s"[indexName: $indexName; indexedColumn: $indexedColumn; " + + s"ExpectedItems: $expectedNumItems; FPP: $fpp; NumBitsUsed: $numBits;]" + } +} + +object Config { + + // TODO - prints info table about all types of index supported by hyperspace + def printAllIndexConfigInfo(): String = { + s"" + } +} diff --git a/src/main/scala/com/microsoft/hyperspace/index/ConfigBase.scala b/src/main/scala/com/microsoft/hyperspace/index/ConfigBase.scala new file mode 100644 index 000000000..f0a597116 --- /dev/null +++ b/src/main/scala/com/microsoft/hyperspace/index/ConfigBase.scala @@ -0,0 +1,25 @@ +package com.microsoft.hyperspace.index + +sealed trait IndexConfigBase { + val indexName: String + + def equals(obj: Any): Boolean + + def hashCode(): Int + + def toString: String +} + +trait CoveringIndexConfigBase extends IndexConfigBase { + /* + * Columns from which index are created. + */ + val indexedColumns: Seq[String] + + /* + * Columns to be included with the indexed columns. + */ + val includedColumns: Seq[String] +} + +trait NonCoveringIndexConfigBase extends IndexConfigBase {} diff --git a/src/main/scala/com/microsoft/hyperspace/index/IndexConfig.scala b/src/main/scala/com/microsoft/hyperspace/index/IndexConfig.scala deleted file mode 100644 index 212606e66..000000000 --- a/src/main/scala/com/microsoft/hyperspace/index/IndexConfig.scala +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright (2020) The Hyperspace Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.microsoft.hyperspace.index - -import java.util.Locale - -/** - * IndexConfig specifies the configuration of an index. - * - * @param indexName Index name. - * @param indexedColumns Columns from which an index is created. - * @param includedColumns Columns to be included in the index. - */ -case class IndexConfig( - indexName: String, - indexedColumns: Seq[String], - includedColumns: Seq[String] = Seq()) { - if (indexName.isEmpty || indexedColumns.isEmpty) { - throw new IllegalArgumentException("Empty index name or indexed columns are not allowed.") - } - - val lowerCaseIndexedColumns = toLowerCase(indexedColumns) - val lowerCaseIncludedColumns = toLowerCase(includedColumns) - val lowerCaseIncludedColumnsSet = lowerCaseIncludedColumns.toSet - - if (lowerCaseIndexedColumns.toSet.size < lowerCaseIndexedColumns.size) { - throw new IllegalArgumentException("Duplicate indexed column names are not allowed.") - } - - if (lowerCaseIncludedColumnsSet.size < lowerCaseIncludedColumns.size) { - throw new IllegalArgumentException("Duplicate included column names are not allowed.") - } - - for (indexedColumn <- lowerCaseIndexedColumns) { - if (lowerCaseIncludedColumns.contains(indexedColumn)) { - throw new IllegalArgumentException( - "Duplicate column names in indexed/included columns are not allowed.") - } - } - - override def equals(that: Any): Boolean = { - that match { - case IndexConfig(thatIndexName, thatIndexedColumns, thatIncludedColumns) => - indexName.equalsIgnoreCase(thatIndexName) && - lowerCaseIndexedColumns.equals(toLowerCase(thatIndexedColumns)) && - lowerCaseIncludedColumnsSet.equals(toLowerCase(thatIncludedColumns).toSet) - case _ => false - } - } - - override def hashCode(): Int = { - lowerCaseIndexedColumns.hashCode + lowerCaseIncludedColumnsSet.hashCode - } - - override def toString: String = { - val indexedColumnNames = lowerCaseIndexedColumns.mkString(", ") - val includedColumnNames = lowerCaseIncludedColumns.mkString(", ") - s"[indexName: $indexName; indexedColumns: $indexedColumnNames; " + - s"includedColumns: $includedColumnNames]" - } - - private def toLowerCase(seq: Seq[String]): Seq[String] = seq.map(_.toLowerCase(Locale.ROOT)) -} - -/** - * Defines [[IndexConfig.Builder]] and relevant helper methods for enabling builder pattern for - * [[IndexConfig]]. - */ -object IndexConfig { - - /** - * Builder for [[IndexConfig]]. - */ - class Builder { - - private[this] var indexedColumns: Seq[String] = Seq() - private[this] var includedColumns: Seq[String] = Seq() - private[this] var indexName: String = "" - - /** - * Updates index name for [[IndexConfig]]. - * - * @param indexName index name for the [[IndexConfig]]. - * @return an [[IndexConfig.Builder]] object with updated index name. - */ - def indexName(indexName: String): Builder = { - if (!this.indexName.isEmpty) { - throw new UnsupportedOperationException("Index name is already set.") - } - - if (indexName.isEmpty) { - throw new IllegalArgumentException("Empty index name is not allowed.") - } - - this.indexName = indexName - this - } - - /** - * Updates column names for [[IndexConfig]]. - * - * Note: API signature supports passing one or more argument. - * - * @param indexedColumn indexed column for the [[IndexConfig]]. - * @param indexedColumns indexed columns for the [[IndexConfig]]. - * @return an [[IndexConfig.Builder]] object with updated indexed columns. - */ - def indexBy(indexedColumn: String, indexedColumns: String*): Builder = { - if (this.indexedColumns.nonEmpty) { - throw new UnsupportedOperationException("Indexed columns are already set.") - } - - this.indexedColumns = indexedColumn +: indexedColumns - this - } - - /** - * Updates included columns for [[IndexConfig]]. - * - * Note: API signature supports passing one or more argument. - * - * @param includedColumn included column for [[IndexConfig]]. - * @param includedColumns included columns for [[IndexConfig]]. - * @return an [[IndexConfig.Builder]] object with updated included columns. - */ - def include(includedColumn: String, includedColumns: String*): Builder = { - if (this.includedColumns.nonEmpty) { - throw new UnsupportedOperationException("Included columns are already set.") - } - - this.includedColumns = includedColumn +: includedColumns - this - } - - /** - * Creates IndexConfig from supplied index name, indexed columns and included columns - * to [[IndexConfig.Builder]]. - * - * @return an [[IndexConfig]] object. - */ - def create(): IndexConfig = { - IndexConfig(indexName, indexedColumns, includedColumns) - } - } - - /** - * Creates new [[IndexConfig.Builder]] for constructing an [[IndexConfig]]. - * - * @return an [[IndexConfig.Builder]] object. - */ - def builder(): Builder = new Builder -} diff --git a/src/main/scala/com/microsoft/hyperspace/index/configs/covering/IndexConfig.scala b/src/main/scala/com/microsoft/hyperspace/index/configs/covering/IndexConfig.scala new file mode 100644 index 000000000..f9871338d --- /dev/null +++ b/src/main/scala/com/microsoft/hyperspace/index/configs/covering/IndexConfig.scala @@ -0,0 +1,92 @@ +package com.microsoft.hyperspace.index.configs.covering + +import com.microsoft.hyperspace.index.IndexConfig + +/** + * Defines [[IndexConfig.Builder]] and relevant helper methods for enabling builder pattern for + * [[IndexConfig]]. + */ +object IndexConfig { + + /** + * Builder for [[IndexConfig]]. + */ + private[index] class Builder { + + private[this] var indexedColumns: Seq[String] = Seq() + private[this] var includedColumns: Seq[String] = Seq() + private[this] var indexName: String = "" + + /** + * Updates index name for [[IndexConfig]]. + * + * @param indexName index name for the [[IndexConfig]]. + * @return an [[IndexConfig.Builder]] object with updated index name. + */ + def indexName(indexName: String): Builder = { + if (this.indexName.nonEmpty) { + throw new UnsupportedOperationException("Index name is already set.") + } + + if (indexName.isEmpty) { + throw new IllegalArgumentException("Empty index name is not allowed.") + } + + this.indexName = indexName + this + } + + /** + * Updates column names for [[IndexConfig]]. + * + * Note: API signature supports passing one or more argument. + * + * @param indexedColumn indexed column for the [[IndexConfig]]. + * @param indexedColumns indexed columns for the [[IndexConfig]]. + * @return an [[IndexConfig.Builder]] object with updated indexed columns. + */ + def indexBy(indexedColumn: String, indexedColumns: String*): Builder = { + if (this.indexedColumns.nonEmpty) { + throw new UnsupportedOperationException("Indexed columns are already set.") + } + + this.indexedColumns = indexedColumn +: indexedColumns + this + } + + /** + * Updates included columns for [[IndexConfig]]. + * + * Note: API signature supports passing one or more argument. + * + * @param includedColumn included column for [[IndexConfig]]. + * @param includedColumns included columns for [[IndexConfig]]. + * @return an [[IndexConfig.Builder]] object with updated included columns. + */ + def include(includedColumn: String, includedColumns: String*): Builder = { + if (this.includedColumns.nonEmpty) { + throw new UnsupportedOperationException("Included columns are already set.") + } + + this.includedColumns = includedColumn +: includedColumns + this + } + + /** + * Creates IndexConfig from supplied index name, indexed columns and included columns + * to [[IndexConfig.Builder]]. + * + * @return an [[IndexConfig]] object. + */ + def build(): IndexConfig = { + new IndexConfig(indexName, indexedColumns, includedColumns) + } + } + + /** + * Creates new [[IndexConfig.Builder]] for constructing an [[IndexConfig]]. + * + * @return an [[IndexConfig.Builder]] object. + */ + def builder(): Builder = new Builder +} diff --git a/src/main/scala/com/microsoft/hyperspace/index/configs/noncovering/BloomFilterIndexConfig.scala b/src/main/scala/com/microsoft/hyperspace/index/configs/noncovering/BloomFilterIndexConfig.scala new file mode 100644 index 000000000..7e39fe489 --- /dev/null +++ b/src/main/scala/com/microsoft/hyperspace/index/configs/noncovering/BloomFilterIndexConfig.scala @@ -0,0 +1,24 @@ +package com.microsoft.hyperspace.index.configs.noncovering + +/** + * TODO Defines [[BloomFilterIndexConfig.Builder]] and relevant helper methods for enabling builder pattern for + * [[BloomFilterIndexConfig]]. + */ +object BloomFilterIndexConfig { + + /** + * Builder for [[BloomFilterIndexConfig]]. + */ + private[index] class Builder { + + private[this] var indexedColumn: String = "" + private[this] var indexName: String = "" + } + + /** + * Creates new [[BloomFilterIndexConfig.Builder]] for constructing an [[BloomFilterIndexConfig]]. + * + * @return an [[BloomFilterIndexConfig.Builder]] object. + */ + def builder(): Builder = new Builder +} From 5d6b6cb434dd66127d2ae97294c453123b47efc7 Mon Sep 17 00:00:00 2001 From: Gurleen Dhody Date: Fri, 12 Feb 2021 12:32:11 -0800 Subject: [PATCH 2/4] test fixed for build --- .../microsoft/hyperspace/index/Config.scala | 16 +++++----- .../hyperspace/index/ConfigBase.scala | 10 ++++-- .../index/configs/covering/IndexConfig.scala | 2 +- .../noncovering/BloomFilterIndexConfig.scala | 4 +-- .../hyperspace/index/IndexConfigTest.scala | 32 +++++++++++-------- 5 files changed, 37 insertions(+), 27 deletions(-) diff --git a/src/main/scala/com/microsoft/hyperspace/index/Config.scala b/src/main/scala/com/microsoft/hyperspace/index/Config.scala index 264dbb4e2..8bf3fe273 100644 --- a/src/main/scala/com/microsoft/hyperspace/index/Config.scala +++ b/src/main/scala/com/microsoft/hyperspace/index/Config.scala @@ -2,6 +2,14 @@ package com.microsoft.hyperspace.index import java.util.Locale +object Config { + + // TODO - prints info table about all types of index supported by hyperspace + def printAllIndexConfigInfo(): String = { + s"" + } +} + /** * IndexConfig specifies the configuration of an index. * Associated Builder [[com.microsoft.hyperspace.index.configs.covering.IndexConfig.builder()]] @@ -119,11 +127,3 @@ case class BloomFilterIndexConfig( s"ExpectedItems: $expectedNumItems; FPP: $fpp; NumBitsUsed: $numBits;]" } } - -object Config { - - // TODO - prints info table about all types of index supported by hyperspace - def printAllIndexConfigInfo(): String = { - s"" - } -} diff --git a/src/main/scala/com/microsoft/hyperspace/index/ConfigBase.scala b/src/main/scala/com/microsoft/hyperspace/index/ConfigBase.scala index f0a597116..fe92c9ed1 100644 --- a/src/main/scala/com/microsoft/hyperspace/index/ConfigBase.scala +++ b/src/main/scala/com/microsoft/hyperspace/index/ConfigBase.scala @@ -1,6 +1,10 @@ package com.microsoft.hyperspace.index -sealed trait IndexConfigBase { +/** + * All index supported in Hyperspace whose user facing config needs to be defined needs + * to extend [[ConfigBase]] trait. + */ +sealed trait ConfigBase { val indexName: String def equals(obj: Any): Boolean @@ -10,7 +14,7 @@ sealed trait IndexConfigBase { def toString: String } -trait CoveringIndexConfigBase extends IndexConfigBase { +trait CoveringIndexConfigBase extends ConfigBase { /* * Columns from which index are created. */ @@ -22,4 +26,4 @@ trait CoveringIndexConfigBase extends IndexConfigBase { val includedColumns: Seq[String] } -trait NonCoveringIndexConfigBase extends IndexConfigBase {} +trait NonCoveringIndexConfigBase extends ConfigBase {} diff --git a/src/main/scala/com/microsoft/hyperspace/index/configs/covering/IndexConfig.scala b/src/main/scala/com/microsoft/hyperspace/index/configs/covering/IndexConfig.scala index f9871338d..0882e2d28 100644 --- a/src/main/scala/com/microsoft/hyperspace/index/configs/covering/IndexConfig.scala +++ b/src/main/scala/com/microsoft/hyperspace/index/configs/covering/IndexConfig.scala @@ -11,7 +11,7 @@ object IndexConfig { /** * Builder for [[IndexConfig]]. */ - private[index] class Builder { + class Builder { private[this] var indexedColumns: Seq[String] = Seq() private[this] var includedColumns: Seq[String] = Seq() diff --git a/src/main/scala/com/microsoft/hyperspace/index/configs/noncovering/BloomFilterIndexConfig.scala b/src/main/scala/com/microsoft/hyperspace/index/configs/noncovering/BloomFilterIndexConfig.scala index 7e39fe489..492426c4f 100644 --- a/src/main/scala/com/microsoft/hyperspace/index/configs/noncovering/BloomFilterIndexConfig.scala +++ b/src/main/scala/com/microsoft/hyperspace/index/configs/noncovering/BloomFilterIndexConfig.scala @@ -1,8 +1,8 @@ package com.microsoft.hyperspace.index.configs.noncovering /** - * TODO Defines [[BloomFilterIndexConfig.Builder]] and relevant helper methods for enabling builder pattern for - * [[BloomFilterIndexConfig]]. + * TODO Defines [[BloomFilterIndexConfig.Builder]] and relevant helper methods for enabling + * builder pattern for [[BloomFilterIndexConfig]]. */ object BloomFilterIndexConfig { diff --git a/src/test/scala/com/microsoft/hyperspace/index/IndexConfigTest.scala b/src/test/scala/com/microsoft/hyperspace/index/IndexConfigTest.scala index cd848b86e..66169756f 100644 --- a/src/test/scala/com/microsoft/hyperspace/index/IndexConfigTest.scala +++ b/src/test/scala/com/microsoft/hyperspace/index/IndexConfigTest.scala @@ -18,29 +18,35 @@ package com.microsoft.hyperspace.index import org.apache.spark.SparkFunSuite +import com.microsoft.hyperspace.index.configs.covering + class IndexConfigTest extends SparkFunSuite { + val CoveringIndexBuilder: covering.IndexConfig.type = covering.IndexConfig + test("Empty index name is not allowed.") { + intercept[IllegalArgumentException](IndexConfig("", Seq("c1"), Seq("c2"))) - intercept[IllegalArgumentException](IndexConfig.builder.indexBy("c1").include("c2").create) - intercept[IllegalArgumentException](IndexConfig.builder.indexName("")) + intercept[IllegalArgumentException]( + CoveringIndexBuilder.builder().indexBy("c1").include("c2").build()) + intercept[IllegalArgumentException](CoveringIndexBuilder.builder().indexName("")) } test("Empty indexed columns are not allowed.") { intercept[IllegalArgumentException](IndexConfig("name", Seq(), Seq("c1"))) intercept[IllegalArgumentException]( - IndexConfig.builder.indexName("name").include("c1").create) + CoveringIndexBuilder.builder().indexName("name").include("c1").build()) } test("Same indexed column names (case-insensitive) are not allowed.") { intercept[IllegalArgumentException](IndexConfig("name", Seq("c1", "C1"), Seq("c2"))) intercept[IllegalArgumentException]( - IndexConfig.builder.indexName("name").indexBy("c1", "C1").include("c2").create) + CoveringIndexBuilder.builder().indexName("name").indexBy("c1", "C1").include("c2").build()) } test("Same column names (case-insensitive) in indexed/included columns are not allowed.") { intercept[IllegalArgumentException](IndexConfig("name", Seq("c1"), Seq("C1", "c2"))) intercept[IllegalArgumentException]( - IndexConfig.builder.indexName("name").indexBy("c1").include("C1", "c2").create) + CoveringIndexBuilder.builder().indexName("name").indexBy("c1").include("C1", "c2").build()) } test("Test equals() function.") { @@ -98,11 +104,11 @@ class IndexConfigTest extends SparkFunSuite { val indexedColumns = Seq("C1", "c2", "C3") val includedColumns = Seq("C4", "c5", "C6") - val indexConfig = IndexConfig.builder + val indexConfig = CoveringIndexBuilder.builder() .indexName(indexName) .indexBy(indexedColumns.head, indexedColumns.tail: _*) .include(includedColumns.head, includedColumns.tail: _*) - .create + .build() assert(indexConfig.indexName.equals(indexName)) assert(indexConfig.indexedColumns.equals(indexedColumns)) @@ -111,25 +117,25 @@ class IndexConfigTest extends SparkFunSuite { test("Test exception on multiple indexBy, include and index name on IndexConfig builder.") { intercept[UnsupportedOperationException]( - IndexConfig.builder + CoveringIndexBuilder.builder() .indexName("name1") .indexName("name2") .indexBy("c1", "c2") .include("c3", "c4") - .create) + .build()) intercept[UnsupportedOperationException]( - IndexConfig.builder + CoveringIndexBuilder.builder() .indexName("name") .indexBy("c1") .indexBy("c2") .include("c3", "c4") - .create) + .build()) intercept[UnsupportedOperationException]( - IndexConfig.builder + CoveringIndexBuilder.builder() .indexName("name") .indexBy("c1") .include("c2") .include("c3") - .create) + .build()) } } From ed55d418a5aea3932235e2e07b7f555f7a702d80 Mon Sep 17 00:00:00 2001 From: Gurleen Dhody Date: Fri, 12 Feb 2021 13:20:17 -0800 Subject: [PATCH 3/4] header --- .../microsoft/hyperspace/index/Config.scala | 40 +++++++++++-------- .../hyperspace/index/ConfigBase.scala | 16 ++++++++ .../index/configs/covering/IndexConfig.scala | 16 ++++++++ .../noncovering/BloomFilterIndexConfig.scala | 16 ++++++++ 4 files changed, 72 insertions(+), 16 deletions(-) diff --git a/src/main/scala/com/microsoft/hyperspace/index/Config.scala b/src/main/scala/com/microsoft/hyperspace/index/Config.scala index 8bf3fe273..3ff5ec807 100644 --- a/src/main/scala/com/microsoft/hyperspace/index/Config.scala +++ b/src/main/scala/com/microsoft/hyperspace/index/Config.scala @@ -1,15 +1,23 @@ +/* + * Copyright (2020) The Hyperspace Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.microsoft.hyperspace.index import java.util.Locale -object Config { - - // TODO - prints info table about all types of index supported by hyperspace - def printAllIndexConfigInfo(): String = { - s"" - } -} - /** * IndexConfig specifies the configuration of an index. * Associated Builder [[com.microsoft.hyperspace.index.configs.covering.IndexConfig.builder()]] @@ -70,14 +78,6 @@ case class IndexConfig( private def toLowerCase(seq: Seq[String]): Seq[String] = seq.map(_.toLowerCase(Locale.ROOT)) } -/** - * TODO - * @param indexName - * @param indexedColumn - * @param expectedNumItems - * @param fpp - * @param numBits - */ case class BloomFilterIndexConfig( indexName: String, indexedColumn: String, @@ -127,3 +127,11 @@ case class BloomFilterIndexConfig( s"ExpectedItems: $expectedNumItems; FPP: $fpp; NumBitsUsed: $numBits;]" } } + +object Config { + + // TODO - prints info table about all types of index supported by hyperspace + def printAllIndexConfigInfo(): String = { + s"" + } +} diff --git a/src/main/scala/com/microsoft/hyperspace/index/ConfigBase.scala b/src/main/scala/com/microsoft/hyperspace/index/ConfigBase.scala index fe92c9ed1..4e1e8018a 100644 --- a/src/main/scala/com/microsoft/hyperspace/index/ConfigBase.scala +++ b/src/main/scala/com/microsoft/hyperspace/index/ConfigBase.scala @@ -1,3 +1,19 @@ +/* + * Copyright (2020) The Hyperspace Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.microsoft.hyperspace.index /** diff --git a/src/main/scala/com/microsoft/hyperspace/index/configs/covering/IndexConfig.scala b/src/main/scala/com/microsoft/hyperspace/index/configs/covering/IndexConfig.scala index 0882e2d28..838966562 100644 --- a/src/main/scala/com/microsoft/hyperspace/index/configs/covering/IndexConfig.scala +++ b/src/main/scala/com/microsoft/hyperspace/index/configs/covering/IndexConfig.scala @@ -1,3 +1,19 @@ +/* + * Copyright (2020) The Hyperspace Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.microsoft.hyperspace.index.configs.covering import com.microsoft.hyperspace.index.IndexConfig diff --git a/src/main/scala/com/microsoft/hyperspace/index/configs/noncovering/BloomFilterIndexConfig.scala b/src/main/scala/com/microsoft/hyperspace/index/configs/noncovering/BloomFilterIndexConfig.scala index 492426c4f..ceed6f68c 100644 --- a/src/main/scala/com/microsoft/hyperspace/index/configs/noncovering/BloomFilterIndexConfig.scala +++ b/src/main/scala/com/microsoft/hyperspace/index/configs/noncovering/BloomFilterIndexConfig.scala @@ -1,3 +1,19 @@ +/* + * Copyright (2020) The Hyperspace Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.microsoft.hyperspace.index.configs.noncovering /** From 4ee368c36bb89b9183a2d0ce5504fda08009650c Mon Sep 17 00:00:00 2001 From: Gurleen Dhody Date: Wed, 17 Feb 2021 17:25:48 -0800 Subject: [PATCH 4/4] code --- .../noncovering/BloomFilterIndexConfig.scala | 75 +++++++++++++++++++ .../hyperspace/index/IndexConfigTest.scala | 34 ++++++--- 2 files changed, 99 insertions(+), 10 deletions(-) diff --git a/src/main/scala/com/microsoft/hyperspace/index/configs/noncovering/BloomFilterIndexConfig.scala b/src/main/scala/com/microsoft/hyperspace/index/configs/noncovering/BloomFilterIndexConfig.scala index ceed6f68c..8ab24a9ed 100644 --- a/src/main/scala/com/microsoft/hyperspace/index/configs/noncovering/BloomFilterIndexConfig.scala +++ b/src/main/scala/com/microsoft/hyperspace/index/configs/noncovering/BloomFilterIndexConfig.scala @@ -16,6 +16,9 @@ package com.microsoft.hyperspace.index.configs.noncovering +import com.microsoft.hyperspace.index.BloomFilterIndexConfig +import com.microsoft.hyperspace.index.configs.covering.IndexConfig + /** * TODO Defines [[BloomFilterIndexConfig.Builder]] and relevant helper methods for enabling * builder pattern for [[BloomFilterIndexConfig]]. @@ -29,6 +32,78 @@ object BloomFilterIndexConfig { private[this] var indexedColumn: String = "" private[this] var indexName: String = "" + private[this] var fpp: Double = -1 + private[this] var expectedItems: Long = -1 + private[this] var numBits: Long = -1 + + /** + * Updates index name for [[IndexConfig]]. + * + * @param indexName index name for the [[BloomFilterIndexConfig]]. + * @return an [[BloomFilterIndexConfig.Builder]] object with updated index name. + */ + def init(indexName: String, indexedColumn: String): Builder = { + if (this.indexName.nonEmpty || this.indexedColumn.nonEmpty) { + // TODO: Prevent creating index config if index already exists. + throw new UnsupportedOperationException( + "Bloom Filter index metadata already set can't override, " + + "maybe try creating a new config.") + } + + if (indexName.isEmpty || indexedColumn.isEmpty) { + throw new IllegalArgumentException("Empty metadata names is not allowed.") + } + + this.indexName = indexName + this.indexedColumn = indexedColumn + this + } + + /** + * + * @param items + * @return + */ + def expectedNumItems(items: Long): Builder = { + if (items < 1) { + throw new IllegalArgumentException("Can't support the items value provided.") + } + + this.expectedItems = items + this + } + + /** + * + * @param fpp + * @return + */ + def fppToSupport(fpp: Double): Builder = { + if (fpp <= 0) { + throw new IllegalArgumentException("Can't support the fpp value.") + } + + this.fpp = fpp + this + } + + /** + * + * @param bits + * @return + */ + def numBitsToDefineBloomFilter(bits: Long): Builder = { + if (bits < 1) { + throw new IllegalArgumentException("Can't allow bits for storage be less than 1") + } + + this.numBits = bits + this + } + + def build(): BloomFilterIndexConfig = { + new BloomFilterIndexConfig(indexName, indexedColumn, expectedItems, fpp, numBits) + } } /** diff --git a/src/test/scala/com/microsoft/hyperspace/index/IndexConfigTest.scala b/src/test/scala/com/microsoft/hyperspace/index/IndexConfigTest.scala index 66169756f..5cf769f81 100644 --- a/src/test/scala/com/microsoft/hyperspace/index/IndexConfigTest.scala +++ b/src/test/scala/com/microsoft/hyperspace/index/IndexConfigTest.scala @@ -21,32 +21,42 @@ import org.apache.spark.SparkFunSuite import com.microsoft.hyperspace.index.configs.covering class IndexConfigTest extends SparkFunSuite { - val CoveringIndexBuilder: covering.IndexConfig.type = covering.IndexConfig + val CoveringIndexConfigBuilder: covering.IndexConfig.type = covering.IndexConfig test("Empty index name is not allowed.") { intercept[IllegalArgumentException](IndexConfig("", Seq("c1"), Seq("c2"))) intercept[IllegalArgumentException]( - CoveringIndexBuilder.builder().indexBy("c1").include("c2").build()) - intercept[IllegalArgumentException](CoveringIndexBuilder.builder().indexName("")) + CoveringIndexConfigBuilder.builder().indexBy("c1").include("c2").build()) + intercept[IllegalArgumentException](CoveringIndexConfigBuilder.builder().indexName("")) } test("Empty indexed columns are not allowed.") { intercept[IllegalArgumentException](IndexConfig("name", Seq(), Seq("c1"))) intercept[IllegalArgumentException]( - CoveringIndexBuilder.builder().indexName("name").include("c1").build()) + CoveringIndexConfigBuilder.builder().indexName("name").include("c1").build()) } test("Same indexed column names (case-insensitive) are not allowed.") { intercept[IllegalArgumentException](IndexConfig("name", Seq("c1", "C1"), Seq("c2"))) intercept[IllegalArgumentException]( - CoveringIndexBuilder.builder().indexName("name").indexBy("c1", "C1").include("c2").build()) + CoveringIndexConfigBuilder + .builder() + .indexName("name") + .indexBy("c1", "C1") + .include("c2") + .build()) } test("Same column names (case-insensitive) in indexed/included columns are not allowed.") { intercept[IllegalArgumentException](IndexConfig("name", Seq("c1"), Seq("C1", "c2"))) intercept[IllegalArgumentException]( - CoveringIndexBuilder.builder().indexName("name").indexBy("c1").include("C1", "c2").build()) + CoveringIndexConfigBuilder + .builder() + .indexName("name") + .indexBy("c1") + .include("C1", "c2") + .build()) } test("Test equals() function.") { @@ -104,7 +114,8 @@ class IndexConfigTest extends SparkFunSuite { val indexedColumns = Seq("C1", "c2", "C3") val includedColumns = Seq("C4", "c5", "C6") - val indexConfig = CoveringIndexBuilder.builder() + val indexConfig = CoveringIndexConfigBuilder + .builder() .indexName(indexName) .indexBy(indexedColumns.head, indexedColumns.tail: _*) .include(includedColumns.head, includedColumns.tail: _*) @@ -117,21 +128,24 @@ class IndexConfigTest extends SparkFunSuite { test("Test exception on multiple indexBy, include and index name on IndexConfig builder.") { intercept[UnsupportedOperationException]( - CoveringIndexBuilder.builder() + CoveringIndexConfigBuilder + .builder() .indexName("name1") .indexName("name2") .indexBy("c1", "c2") .include("c3", "c4") .build()) intercept[UnsupportedOperationException]( - CoveringIndexBuilder.builder() + CoveringIndexConfigBuilder + .builder() .indexName("name") .indexBy("c1") .indexBy("c2") .include("c3", "c4") .build()) intercept[UnsupportedOperationException]( - CoveringIndexBuilder.builder() + CoveringIndexConfigBuilder + .builder() .indexName("name") .indexBy("c1") .include("c2")