Add native bitstring support to Postgres(#577)

Author: Gigitsu

Earthfile

@@ -1,7 +1,7 @@
 VERSION 0.6
 
 all:
-    ARG ELIXIR_BASE=1.15.6-erlang-25.3.2.6-alpine-3.17.4
+    ARG ELIXIR_BASE=1.15.6-erlang-25.3.2.6-alpine-3.18.4
     BUILD \
         --build-arg POSTGRES=15.0 \
         --build-arg POSTGRES=11.11 \
@@ -20,7 +20,7 @@ all:
         +integration-test-mssql
 
 setup-base:
-    ARG ELIXIR_BASE=1.15.6-erlang-25.3.2.6-alpine-3.17.4
+    ARG ELIXIR_BASE=1.15.6-erlang-25.3.2.6-alpine-3.18.4
     FROM hexpm/elixir:$ELIXIR_BASE
     RUN apk add --no-progress --update git build-base
     ENV ELIXIR_ASSERT_TIMEOUT=10000
@@ -62,7 +62,7 @@ integration-test-postgres:
 
     # then run the tests
     WITH DOCKER \
-        --pull "postgres:$POSTGRES"
+        --pull "postgres:$POSTGRES" --platform linux/amd64
         RUN set -e; \
             timeout=$(expr $(date +%s) + 30); \
             docker run --name pg --network=host -d -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres "postgres:$POSTGRES"; \
@@ -84,7 +84,7 @@ integration-test-mysql:
 
     ARG MYSQL="5.7"
     WITH DOCKER \
-        --pull "mysql:$MYSQL"
+        --pull "mysql:$MYSQL" --platform linux/amd64
         RUN set -e; \
             timeout=$(expr $(date +%s) + 30); \
             docker run --name mysql --network=host -d -e MYSQL_ROOT_PASSWORD=root "mysql:$MYSQL" \
@@ -103,25 +103,26 @@ integration-test-mysql:
 
 
 integration-test-mssql:
+    ARG TARGETARCH 
     FROM +setup-base
 
     RUN apk add --no-cache curl gnupg --virtual .build-dependencies -- && \
-        curl -O https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_17.5.2.1-1_amd64.apk && \
-        curl -O https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/mssql-tools_17.5.2.1-1_amd64.apk && \
-        echo y | apk add --allow-untrusted msodbcsql17_17.5.2.1-1_amd64.apk mssql-tools_17.5.2.1-1_amd64.apk && \
+        curl -O https://download.microsoft.com/download/3/5/5/355d7943-a338-41a7-858d-53b259ea33f5/msodbcsql18_18.3.2.1-1_${TARGETARCH}.apk && \
+        curl -O https://download.microsoft.com/download/3/5/5/355d7943-a338-41a7-858d-53b259ea33f5/mssql-tools18_18.3.1.1-1_${TARGETARCH}.apk && \
+        echo y | apk add --allow-untrusted msodbcsql18_18.3.2.1-1_${TARGETARCH}.apk mssql-tools18_18.3.1.1-1_${TARGETARCH}.apk && \
         apk del .build-dependencies && rm -f msodbcsql*.sig mssql-tools*.apk
-    ENV PATH="/opt/mssql-tools/bin:${PATH}"
+    ENV PATH="/opt/mssql-tools18/bin:${PATH}"
 
     DO +COMMON_SETUP_AND_MIX
 
     ARG MSSQL="2017"
     WITH DOCKER \
-        --pull "mcr.microsoft.com/mssql/server:$MSSQL-latest"
+        --pull "mcr.microsoft.com/mssql/server:$MSSQL-latest" --platform linux/amd64
         RUN set -e; \
             timeout=$(expr $(date +%s) + 30); \
             docker run -d -p 1433:1433 --name mssql -e 'ACCEPT_EULA=Y' -e 'MSSQL_SA_PASSWORD=some!Password' "mcr.microsoft.com/mssql/server:$MSSQL-latest"; \
             # wait for mssql to start
-            while ! sqlcmd -S tcp:127.0.0.1,1433 -U sa -P 'some!Password' -Q "SELECT 1" >/dev/null 2>&1; do \
+            while ! sqlcmd -C -S tcp:127.0.0.1,1433 -U sa -P 'some!Password' -Q "SELECT 1" >/dev/null 2>&1; do \
                 test "$(date +%s)" -le "$timeout" || (echo "timed out waiting for mssql"; exit 1); \
                 echo "waiting for mssql"; \
                 sleep 1; \

integration_test/myxql/test_helper.exs

@@ -93,6 +93,8 @@ version =
   end
 
 excludes = [
+  # not sure how to support this yet
+  :bitstring_type,
   # MySQL does not have an array type
   :array_type,
   # The next two features rely on RETURNING, which MySQL does not support

integration_test/support/migration.exs

@@ -108,6 +108,14 @@ defmodule Ecto.Integration.Migration do
       end
     end
 
+    unless :bitstring_type in ExUnit.configuration()[:exclude] do
+      create table(:bitstrings) do
+        add :bs,  :bitstring
+        add :bs_with_default, :bitstring, default: <<42::6>>
+        add :bs_with_size, :bitstring, size: 10
+      end
+    end
+
     create table(:composite_pk, primary_key: false) do
       add :a, :integer, primary_key: true
       add :b, :integer, primary_key: true

integration_test/tds/test_helper.exs

@@ -4,6 +4,7 @@ ExUnit.start(
   exclude: [
     # not sure how to support this yet
     :aggregate_filters,
+    :bitstring_type,
     # subquery contains ORDER BY and that is not supported
     :subquery_aggregates,
     # sql don't have array type

lib/ecto/adapters/postgres/connection.ex

@@ -1015,6 +1015,11 @@ if Code.ensure_loaded?(Postgrex) do
       ["'\\x", Base.encode16(binary, case: :lower) | "'::bytea"]
     end
 
+    defp expr(%Ecto.Query.Tagged{value: bitstring, type: :bitstring}, _sources, _query)
+         when is_bitstring(bitstring) do
+      bitstring_literal(bitstring)
+    end
+
     defp expr(%Ecto.Query.Tagged{value: other, type: type}, sources, query) do
       [maybe_paren(other, sources, query), ?:, ?: | tagged_to_db(type)]
     end
@@ -1580,6 +1585,10 @@ if Code.ensure_loaded?(Postgrex) do
       end
     end
 
+    defp default_type(literal, _type) when is_bitstring(literal) do
+      bitstring_literal(literal)
+    end
+
     defp default_type(literal, _type) when is_number(literal), do: to_string(literal)
     defp default_type(literal, _type) when is_boolean(literal), do: to_string(literal)
 
@@ -1824,6 +1833,13 @@ if Code.ensure_loaded?(Postgrex) do
 
     defp single_quote(value), do: [?', escape_string(value), ?']
 
+    defp bitstring_literal(value) do
+      size = bit_size(value)
+      <<val::size(size)>> = value
+
+      [?b, ?', val |> Integer.to_string(2) |> String.pad_leading(size, ["0"]), ?']
+    end
+
     defp intersperse_reduce(list, separator, user_acc, reducer, acc \\ [])
 
     defp intersperse_reduce([], _separator, user_acc, _reducer, acc),
@@ -1870,6 +1886,7 @@ if Code.ensure_loaded?(Postgrex) do
     defp ecto_to_db(:bigserial), do: "bigserial"
     defp ecto_to_db(:binary_id), do: "uuid"
     defp ecto_to_db(:string), do: "varchar"
+    defp ecto_to_db(:bitstring), do: "varbit"
     defp ecto_to_db(:binary), do: "bytea"
     defp ecto_to_db(:map), do: Application.fetch_env!(:ecto_sql, :postgres_map_type)
     defp ecto_to_db({:map, _}), do: Application.fetch_env!(:ecto_sql, :postgres_map_type)

mix.exs

@@ -76,7 +76,7 @@ defmodule EctoSQL.MixProject do
     if path = System.get_env("ECTO_PATH") do
       {:ecto, path: path}
     else
-      {:ecto, "~> 3.11.0"}
+      {:ecto, github: "elixir-ecto/ecto"}
     end
   end
 

mix.lock

@@ -4,7 +4,7 @@
   "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"},
   "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"},
   "earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"},
-  "ecto": {:hex, :ecto, "3.11.0", "ff8614b4e70a774f9d39af809c426def80852048440e8785d93a6e91f48fec00", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7769dad267ef967310d6e988e92d772659b11b09a0c015f101ce0fff81ce1f81"},
+  "ecto": {:git, "https://github.com/elixir-ecto/ecto.git", "bed81b9a69f3425147fb57df1b3dd5fb4c95792c", []},
   "ex_doc": {:hex, :ex_doc, "0.30.9", "d691453495c47434c0f2052b08dd91cc32bc4e1a218f86884563448ee2502dd2", [:mix], [{:earmark_parser, "~> 1.4.31", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "d7aaaf21e95dc5cddabf89063327e96867d00013963eadf2c6ad135506a8bc10"},
   "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"},
   "makeup": {:hex, :makeup, "1.1.1", "fa0bc768698053b2b3869fa8a62616501ff9d11a562f3ce39580d60860c3a55e", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5dc62fbdd0de44de194898b6710692490be74baa02d9d108bc29f007783b0b48"},

test/ecto/adapters/postgres_test.exs

@@ -1990,6 +1990,9 @@ defmodule Ecto.Adapters.PostgresTest do
          {:add, :on_hand, :integer, [default: 0, null: true]},
          {:add, :published_at, :"time without time zone", [null: true]},
          {:add, :is_active, :boolean, [default: true]},
+         {:add, :flags, :bitstring, [null: false]},
+         {:add, :flags_with_default, :bitstring, [default: <<42::10>>]},
+         {:add, :flags_with_size, :bitstring, [size: 10]},
          {:add, :tags, {:array, :string}, [default: []]},
          {:add, :languages, {:array, :string}, [default: ["pt", "es"]]},
          {:add, :limits, {:array, :integer}, [default: [100, 30_000]]}
@@ -2002,6 +2005,9 @@ defmodule Ecto.Adapters.PostgresTest do
              "on_hand" integer DEFAULT 0 NULL,
              "published_at" time without time zone NULL,
              "is_active" boolean DEFAULT true,
+             "flags" varbit NOT NULL,
+             "flags_with_default" varbit DEFAULT b'0000101010',
+             "flags_with_size" varbit(10),
              "tags" varchar(255)[] DEFAULT ARRAY[]::varchar[],
              "languages" varchar(255)[] DEFAULT ARRAY['pt','es']::varchar[],
              "limits" integer[] DEFAULT ARRAY[100,30000]::integer[])