Skip to content

Commit d1bdec3

Browse files
hippalusde-shnitisht
authored
Feat: Add Kafka integration for Parseable server #936 . (#1047)
This pull request implements the Kafka connector for Parseable by introducing better stream management, modularizing the code, and integrating Prometheus metrics. It also adds groundwork for dynamic configurations. Partition Management: The connector dynamically manages streams for each partition using StreamState. Each Kafka message is routed to the corresponding PartitionStreamReceiver for processing. Rebalance Handling: - On partition assignment: New streams are created for untracked partitions. - On partition revocation: Streams for revoked partitions are terminated and cleaned up. Stream Processing: The StreamWorker processes messages in batches, ensuring backpressure handling with configurable buffer_size and buffer_timeout. Also exposed metrics via Prometheus include: Partition lag, consumer throughput, etc provided by rd-kafka statistics. Part of #936 --------- Signed-off-by: Nitish Tiwari <[email protected]> Signed-off-by: Nikhil Sinha <[email protected]> Co-authored-by: Devdutt Shenoi <[email protected]> Co-authored-by: Nitish Tiwari <[email protected]>
1 parent bb6c352 commit d1bdec3

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+5923
-1337
lines changed

‎.github/workflows/build-push-edge-kafka.yaml

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
name: Build and push edge kafka tag
2+
3+
on:
4+
push:
5+
branches:
6+
- 'main'
7+
paths-ignore:
8+
- 'docs/**'
9+
- 'helm/**'
10+
- 'assets/**'
11+
- '**.md'
12+
13+
jobs:
14+
docker:
15+
runs-on: ubuntu-latest
16+
steps:
17+
- name: Checkout
18+
uses: actions/checkout@v4
19+
20+
- name: Set up QEMU
21+
uses: docker/setup-qemu-action@v3
22+
23+
- name: Set up Docker Buildx
24+
uses: docker/setup-buildx-action@v3
25+
26+
- name: Login to Docker Hub
27+
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
28+
with:
29+
username: ${{ secrets.DOCKERHUB_USERNAME }}
30+
password: ${{ secrets.DOCKERHUB_TOKEN }}
31+
32+
- name: Extract metadata (tags, labels) for Docker
33+
id: meta
34+
uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
35+
with:
36+
images: parseable/parseable
37+
38+
- name: Build and push
39+
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
40+
with:
41+
context: .
42+
file: ./Dockerfile.kafka
43+
push: true
44+
tags: parseable/parseable:edge-kafka
45+
platforms: linux/amd64,linux/arm64

‎.github/workflows/build.yaml

+120-45
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
name: Ensure parseable builds on all release targets
2+
13
on:
24
pull_request:
35
paths-ignore:
@@ -6,68 +8,141 @@ on:
68
- "assets/**"
79
- "**.md"
810

9-
name: Ensure parseable builds on all release targets
1011
jobs:
11-
build-linux:
12-
name: Build for ${{matrix.target}}
13-
runs-on: ubuntu-latest
12+
# Default build without Kafka
13+
build-default:
14+
name: Build Default ${{matrix.target}}
15+
runs-on: ${{ matrix.os }}
1416
strategy:
1517
fail-fast: false
1618
matrix:
17-
target:
18-
- aarch64-unknown-linux-gnu # linux(arm)
19-
- x86_64-unknown-linux-gnu # linux(64 bit)
19+
include:
20+
# Linux builds
21+
- os: ubuntu-latest
22+
target: x86_64-unknown-linux-gnu
23+
- os: ubuntu-latest
24+
target: aarch64-unknown-linux-gnu
25+
# macOS builds
26+
- os: macos-latest
27+
target: x86_64-apple-darwin
28+
- os: macos-latest
29+
target: aarch64-apple-darwin
30+
# Windows build
31+
- os: windows-latest
32+
target: x86_64-pc-windows-msvc
2033

2134
steps:
22-
- uses: actions/checkout@v2
35+
- uses: actions/checkout@v4
36+
37+
- name: Setup Rust toolchain
38+
uses: dtolnay/rust-toolchain@stable
39+
with:
40+
targets: ${{ matrix.target }}
2341

24-
- uses: actions-rs/toolchain@v1
42+
- name: Cache dependencies
43+
uses: actions/cache@v4
2544
with:
26-
toolchain: stable
27-
profile: minimal # minimal component installation (ie, no documentation)
28-
target: ${{ matrix.target }}
29-
override: true
45+
path: |
46+
~/.cargo/registry
47+
~/.cargo/git
48+
target
49+
key: ${{ runner.os }}-cargo-${{ matrix.target }}-default-${{ hashFiles('**/Cargo.lock') }}
3050

31-
- uses: actions-rs/cargo@v1
51+
- name: Build
52+
uses: actions-rs/cargo@v1
3253
with:
33-
use-cross: true
54+
use-cross: ${{ runner.os == 'Linux' }}
3455
command: build
35-
args: --target ${{matrix.target}}
56+
args: --target ${{ matrix.target }} --release
3657

37-
build-windows:
38-
name: Build for windows
39-
runs-on: windows-latest
58+
# Kafka build for supported platforms
59+
build-kafka:
60+
name: Build Kafka ${{matrix.target}}
61+
runs-on: ${{ matrix.os }}
62+
strategy:
63+
fail-fast: false
64+
matrix:
65+
include:
66+
# Linux builds
67+
- os: ubuntu-latest
68+
target: x86_64-unknown-linux-gnu
69+
- os: macos-latest
70+
target: aarch64-apple-darwin
4071

4172
steps:
42-
- uses: actions/checkout@v2
73+
- uses: actions/checkout@v4
4374

44-
- uses: actions-rs/toolchain@v1
45-
with:
46-
toolchain: stable
47-
profile: minimal # minimal component installation (ie, no documentation)
48-
default: true
49-
override: true
75+
# Linux-specific dependencies
76+
- name: Install Linux dependencies
77+
if: runner.os == 'Linux'
78+
run: |
79+
sudo apt-get update
80+
sudo apt-get install -y \
81+
build-essential \
82+
pkg-config \
83+
cmake \
84+
clang \
85+
zlib1g-dev \
86+
libzstd-dev \
87+
liblz4-dev \
88+
libssl-dev \
89+
libsasl2-dev \
90+
python3 \
91+
gcc-aarch64-linux-gnu \
92+
g++-aarch64-linux-gnu
93+
94+
# Install cross-compilation specific packages
95+
if [ "${{ matrix.target }}" = "aarch64-unknown-linux-gnu" ]; then
96+
sudo apt-get install -y \
97+
gcc-aarch64-linux-gnu \
98+
g++-aarch64-linux-gnu \
99+
libc6-dev-arm64-cross \
100+
libsasl2-dev:arm64 \
101+
libssl-dev:arm64 \
102+
pkg-config-aarch64-linux-gnu
103+
fi
50104
51-
- name: Build on windows
52-
run: cargo build --target x86_64-pc-windows-msvc
53105
54-
build-macos:
55-
name: Build for ${{matrix.target}}
56-
runs-on: macos-latest
57-
strategy:
58-
matrix:
59-
target:
60-
- aarch64-apple-darwin # macos(arm)
61-
- x86_64-apple-darwin # macos(intel 64 bit)
106+
# macOS-specific dependencies
107+
- name: Install macOS dependencies
108+
if: runner.os == 'macOS'
109+
run: |
110+
brew install \
111+
cmake \
112+
llvm \
113+
pkg-config \
114+
zstd \
115+
lz4 \
116+
[email protected] \
117+
cyrus-sasl \
118+
python3
62119
63-
steps:
64-
- uses: actions/checkout@v2
65-
- uses: actions-rs/toolchain@v1
120+
- name: Setup Rust toolchain
121+
uses: dtolnay/rust-toolchain@stable
66122
with:
67-
toolchain: stable
68-
profile: minimal
69-
target: ${{ matrix.target }}
70-
override: true
123+
targets: ${{ matrix.target }}
71124

72-
- name: Build on ${{ matrix.target }}
73-
run: cargo build --target ${{ matrix.target }}
125+
- name: Cache dependencies
126+
uses: actions/cache@v4
127+
with:
128+
path: |
129+
~/.cargo/registry
130+
~/.cargo/git
131+
target
132+
key: ${{ runner.os }}-cargo-${{ matrix.target }}-kafka-${{ hashFiles('**/Cargo.lock') }}
133+
134+
- name: Build with Kafka
135+
uses: actions-rs/cargo@v1
136+
with:
137+
use-cross: ${{ runner.os == 'Linux' }}
138+
command: build
139+
args: --target ${{ matrix.target }} --features kafka --release
140+
env:
141+
LIBRDKAFKA_SSL_VENDORED: 1
142+
PKG_CONFIG_ALLOW_CROSS: "1"
143+
PKG_CONFIG_PATH: "/usr/lib/aarch64-linux-gnu/pkgconfig"
144+
SASL2_DIR: "/usr/lib/aarch64-linux-gnu"
145+
OPENSSL_DIR: "/usr/lib/aarch64-linux-gnu"
146+
OPENSSL_ROOT_DIR: "/usr/lib/aarch64-linux-gnu"
147+
OPENSSL_STATIC: "1"
148+
SASL2_STATIC: "0"

‎.github/workflows/coverage.yaml

+12-2
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ jobs:
1111
coverage:
1212
runs-on: ubuntu-latest
1313
steps:
14-
- uses: actions/checkout@v2
14+
- uses: actions/checkout@v4
1515
- uses: dtolnay/rust-toolchain@stable
1616
with:
1717
components: clippy
@@ -24,13 +24,23 @@ jobs:
2424
with:
2525
tool: cargo-hack, cargo-llvm-cov, nextest
2626

27+
- name: Install System Dependencies
28+
run: |
29+
sudo apt-get update
30+
sudo apt-get install -y \
31+
libsasl2-dev \
32+
libssl-dev \
33+
pkg-config \
34+
build-essential
35+
if: runner.os == 'Linux'
36+
2737
- name: Check with clippy
2838
run: cargo hack clippy --verbose --each-feature --no-dev-deps -- -D warnings
2939

3040
- name: Tests
3141
run: cargo hack --each-feature llvm-cov --no-report nextest
3242

33-
- name: Genrate coverage report
43+
- name: Generate coverage report
3444
run: cargo llvm-cov report --lcov --output-path coverage.lcov
3545

3646
- name: Upload Coverage Report

‎.github/workflows/integration-test.yaml

+28-4
Original file line numberDiff line numberDiff line change
@@ -14,21 +14,45 @@ jobs:
1414
runs-on: ubuntu-latest
1515
steps:
1616
- name: Checkout
17-
uses: actions/checkout@v3
17+
uses: actions/checkout@v4
1818
- name: Start compose
1919
run: docker compose -f docker-compose-test.yaml up --build --exit-code-from quest
2020
- name: Stop compose
2121
if: always()
22-
run: docker compose -f docker-compose-test.yaml down
22+
run: docker compose -f docker-compose-test.yaml down -v
2323

2424
docker-compose-distributed-test:
2525
name: Quest Smoke and Load Tests for Distributed deployments
2626
runs-on: ubuntu-latest
2727
steps:
2828
- name: Checkout
29-
uses: actions/checkout@v3
29+
uses: actions/checkout@v4
3030
- name: Start compose
3131
run: docker compose -f docker-compose-distributed-test.yaml up --build --exit-code-from quest
3232
- name: Stop compose
3333
if: always()
34-
run: docker compose -f docker-compose-distributed-test.yaml down
34+
run: docker compose -f docker-compose-distributed-test.yaml down -v
35+
36+
docker-compose-test-with-kafka:
37+
name: Quest Smoke and Load Tests for Standalone deployments with Kafka
38+
runs-on: ubuntu-latest
39+
steps:
40+
- name: Checkout
41+
uses: actions/checkout@v4
42+
- name: Start compose
43+
run: docker compose -f docker-compose-test-with-kafka.yaml up --build --exit-code-from quest
44+
- name: Stop compose
45+
if: always()
46+
run: docker compose -f docker-compose-test-with-kafka.yaml down -v
47+
48+
docker-compose-distributed-test-with-kafka:
49+
name: Quest Smoke and Load Tests for Distributed deployments with Kafka
50+
runs-on: ubuntu-latest
51+
steps:
52+
- name: Checkout
53+
uses: actions/checkout@v4
54+
- name: Start compose
55+
run: docker compose -f docker-compose-distributed-test-with-kafka.yaml up --build --exit-code-from quest
56+
- name: Stop compose
57+
if: always()
58+
run: docker compose -f docker-compose-distributed-test-with-kafka.yaml down -v

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,4 @@ parseable
1414
parseable_*
1515
parseable-env-secret
1616
cache
17+
.idea

0 commit comments

Comments
 (0)