Skip to content

Commit f6b05c9

Browse files
devinjdangelodevinjdangelo
committed
Embed Diesel Migrations (#33)
1 parent 72006d8 commit f6b05c9

25 files changed

+263
-224
lines changed

‎.dockerignore

-1
Original file line numberDiff line numberDiff line change
@@ -1 +0,0 @@
1-
target/debug/*

‎Cargo.lock

+85
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -53,4 +53,4 @@ lto = false
5353
opt-level = 3
5454
overflow-checks = false
5555
panic = 'unwind'
56-
rpath = false
56+
rpath = false

‎core/Cargo.toml

+1-2
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ arrow-flight = { workspace = true }
1919
async-trait = "0.1.74"
2020
datafusion = { workspace = true }
2121
diesel = { version = "2.1.3", features = ["postgres", "serde_json", "uuid"] }
22+
diesel_migrations="2.0.0"
2223
diesel-async = { version="0.4.1", features = ["postgres", "bb8"] }
2324
diesel-derive-enum = { version = "2.1.0", features = ["postgres"] }
2425
diesel_as_jsonb = "1.0.0"
@@ -56,5 +57,3 @@ rabbitmq = ["dep:amqprs"]
5657
os-aws = ["object_store/aws"]
5758
os-azure = ["object_store/azure"]
5859
os-gcp = ["object_store/gcp"]
59-
60-

‎core/src/crud/mod.rs

+31
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,16 @@
1+
use std::time::Duration;
2+
13
use crate::error::MeshError;
24
use crate::error::Result;
35

6+
use diesel::Connection;
7+
use diesel::PgConnection;
48
use diesel_async::{
59
pooled_connection::bb8::{Pool, PooledConnection},
610
AsyncPgConnection,
711
};
12+
use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness};
13+
use tracing::error;
814

915
mod data;
1016
mod entity;
@@ -14,6 +20,31 @@ mod relay;
1420
mod user;
1521
mod utils;
1622

23+
pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!();
24+
25+
/// Bootstraps database with diesel migrations uses embedded code. Continues to retry on error, logging
26+
/// the error cause, and sleeping for a time. Intermittent connectivity errors on startup are expected while
27+
/// Postgres is initializing. Panics if the migrations fail for any reason other inability to connect.
28+
pub fn run_migrations(db_url: &str) {
29+
let mut con;
30+
loop {
31+
let maybe_connected = PgConnection::establish(db_url);
32+
match maybe_connected {
33+
Ok(c) => {
34+
con = c;
35+
break;
36+
}
37+
Err(e) => {
38+
error!("Failed connecting to postgres with error: {e}... retrying migrations in 5 seconds");
39+
std::thread::sleep(Duration::from_secs(5))
40+
}
41+
}
42+
}
43+
44+
con.run_pending_migrations(MIGRATIONS)
45+
.unwrap_or_else(|e| panic!("Error running migrations for {} with error {e}", db_url));
46+
}
47+
1748
/// Holds a [AsyncPgConnection] borrowed from a [Pool] and implements CRUD operations
1849
pub struct PgDb<'a> {
1950
con: PooledConnection<'a, AsyncPgConnection>,

‎deploy/Dockerfile

+27-41
Original file line numberDiff line numberDiff line change
@@ -1,57 +1,43 @@
11
# Build Stage
2-
FROM rust:1.76.0 AS debug_builder
3-
WORKDIR /usr/src/
4-
5-
RUN apt-get update
6-
RUN apt-get install musl musl-tools python3-pip -y
7-
RUN pip3 install cargo-zigbuild --break-system-packages
8-
RUN rustup target add x86_64-unknown-linux-musl
9-
2+
FROM ubuntu:22.04 AS debug_builder
103

4+
RUN apt-get update && apt-get install curl build-essential libpq-dev -y
5+
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash -s -- -y
6+
ENV PATH="/root/.cargo/bin:${PATH}"
7+
WORKDIR /usr/src/
118
COPY ./ ./
12-
RUN cargo zigbuild --target x86_64-unknown-linux-musl
9+
RUN cargo build
1310

1411
# copy out the binaries
1512
FROM scratch as debug_build
16-
COPY --from=debug_builder /usr/src/target/x86_64-unknown-linux-musl/debug/flight_server /
17-
COPY --from=debug_builder /usr/src/target/x86_64-unknown-linux-musl/debug/rest_server /
18-
COPY --from=debug_builder /usr/src/target/x86_64-unknown-linux-musl/debug/query_runner /
19-
COPY --from=debug_builder /usr/src/target/x86_64-unknown-linux-musl/debug/single_binary_deployment /
20-
COPY --from=debug_builder /usr/src/target/x86_64-unknown-linux-musl/debug/relayctl /
21-
COPY --from=debug_builder /usr/src/target/x86_64-unknown-linux-musl/debug/data_web_engine /
13+
COPY --from=debug_builder /usr/src/target/debug/flight_server /
14+
COPY --from=debug_builder /usr/src/target/debug/rest_server /
15+
COPY --from=debug_builder /usr/src/target/debug/query_runner /
16+
COPY --from=debug_builder /usr/src/target/debug/single_binary_deployment /
17+
COPY --from=debug_builder /usr/src/target/debug/relayctl /
18+
COPY --from=debug_builder /usr/src/target/debug/data_web_engine /
2219

2320
#copy binaries from a local build
2421
FROM scratch as debug_local_build
25-
COPY ./target/x86_64-unknown-linux-musl/debug/flight_server /
26-
COPY ./target/x86_64-unknown-linux-musl/debug/rest_server /
27-
COPY ./target/x86_64-unknown-linux-musl/debug/query_runner /
28-
COPY ./target/x86_64-unknown-linux-musl/debug/single_binary_deployment /
29-
COPY ./target/x86_64-unknown-linux-musl/debug/relayctl /
30-
COPY ./target/x86_64-unknown-linux-musl/debug/data_web_engine /
22+
COPY ./target/debug/flight_server /
23+
COPY ./target/debug/rest_server /
24+
COPY ./target/debug/query_runner /
25+
COPY ./target/debug/single_binary_deployment /
26+
COPY ./target/debug/relayctl /
27+
COPY ./target/debug/data_web_engine /
3128

3229
# Build Stage
33-
FROM rust:1.76.0 AS release_builder
30+
FROM rust:1.76.0-bullseye AS release_builder
3431
WORKDIR /usr/src/
3532

36-
RUN apt-get update
37-
RUN apt-get install musl musl-tools python3-pip -y
38-
RUN pip3 install cargo-zigbuild --break-system-packages
39-
RUN rustup target add x86_64-unknown-linux-musl
40-
4133
COPY ./ ./
42-
RUN cargo zigbuild --target x86_64-unknown-linux-musl --release
34+
RUN cargo build --release
4335

4436
# copy out the binaries
45-
FROM scratch as release_build
46-
COPY --from=release_builder /usr/src/target/x86_64-unknown-linux-musl/release/flight_server /
47-
COPY --from=release_builder /usr/src/target/x86_64-unknown-linux-musl/release/rest_server /
48-
COPY --from=release_builder /usr/src/target/x86_64-unknown-linux-musl/release/query_runner /
49-
COPY --from=release_builder /usr/src/target/x86_64-unknown-linux-musl/release/single_binary_deployment /
50-
COPY --from=release_builder /usr/src/target/x86_64-unknown-linux-musl/release/relayctl /
51-
COPY --from=release_builder /usr/src/target/x86_64-unknown-linux-musl/release/data_web_engine /
52-
53-
FROM rust:1.76.0 AS diesel_build
54-
RUN apt-get update
55-
RUN apt-get install libpq-dev -y
56-
RUN cargo install diesel_cli --no-default-features --features postgres
57-
ENTRYPOINT [ "diesel" ]
37+
FROM debian:11 as release_build
38+
COPY --from=release_builder /usr/src/target/release/flight_server /
39+
COPY --from=release_builder /usr/src/target/release/rest_server /
40+
COPY --from=release_builder /usr/src/target/release/query_runner /
41+
COPY --from=release_builder /usr/src/target/release/single_binary_deployment /
42+
COPY --from=release_builder /usr/src/target/release/relayctl /
43+
COPY --from=release_builder /usr/src/target/release/data_web_engine /

‎deploy/Dockerfile.Ballista

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
FROM rust:1.76.0 as scheduler
1+
FROM rust:1.76.0-bullseye as scheduler
22
RUN apt-get update && apt-get install -y protobuf-compiler
33
RUN cargo install ballista-scheduler
44
ENTRYPOINT ["ballista-scheduler"]
55

6-
FROM rust:1.76.0 as executor
6+
FROM rust:1.76.0-bullseye as executor
77
RUN apt-get update && apt-get install -y protobuf-compiler
88
RUN cargo install ballista-executor
99
ENTRYPOINT ["ballista-executor"]

‎deploy/build_all_debug.sh

-2
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,6 @@ docker build -f relayctl/Dockerfile -t relayctl --target debug_prebuild .
1616

1717
docker build -f webengine/Dockerfile -t webengine --target debug_prebuild .
1818

19-
docker build -f deploy/Dockerfile -t diesel --target diesel_build .
20-
2119
docker build -f deploy/Dockerfile.Ballista -t ballista-scheduler --target scheduler .
2220

2321
docker build -f deploy/Dockerfile.Ballista -t ballista-executor --target executor .

‎deploy/build_all_local.sh

+1-3
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/bin/bash
22

33
# builds everything in the workspace
4-
cargo zigbuild --target x86_64-unknown-linux-musl
4+
cargo build
55
docker build -f deploy/Dockerfile -t debug_build --target debug_local_build .
66

77
# subsequent builds can pull the binaries from the build image using --target prebuild
@@ -17,8 +17,6 @@ docker build -f relayctl/Dockerfile -t relayctl --target debug_prebuild .
1717

1818
docker build -f webengine/Dockerfile -t webengine --target debug_prebuild .
1919

20-
docker build -f deploy/Dockerfile -t diesel --target diesel_build .
21-
2220
docker build -f deploy/Dockerfile.Ballista -t ballista-scheduler --target scheduler .
2321

2422
docker build -f deploy/Dockerfile.Ballista -t ballista-executor --target executor .

‎deploy/build_all_release.sh

-2
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,6 @@ docker build -f relayctl/Dockerfile -t relayctl --target release_prebuild .
1616

1717
docker build -f webengine/Dockerfile -t webengine --target release_prebuild .
1818

19-
docker build -f deploy/Dockerfile -t diesel --target diesel_build .
20-
2119
docker build -f deploy/Dockerfile.Ballista -t ballista-scheduler --target scheduler .
2220

2321
docker build -f deploy/Dockerfile.Ballista -t ballista-executor --target executor .

0 commit comments

Comments
 (0)