Skip to content

Commit d2fb5e3

Browse files
Merge branch 'main' into release/v0.1
2 parents a0a5611 + 7d74b6b commit d2fb5e3

File tree

215 files changed

+16335
-7353
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

215 files changed

+16335
-7353
lines changed

.github/workflows/build-and-push.yml

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
name: "Build GARM images"
2+
on:
3+
workflow_dispatch:
4+
inputs:
5+
push_to_project:
6+
description: "Project to build images for"
7+
required: true
8+
default: "ghcr.io/cloudbase"
9+
ref:
10+
description: "Ref to build"
11+
required: true
12+
default: "main"
13+
14+
permissions:
15+
contents: read
16+
17+
jobs:
18+
images:
19+
permissions:
20+
packages: write
21+
name: "Build GARM images"
22+
runs-on: ubuntu-22.04
23+
steps:
24+
- name: "Checkout"
25+
uses: actions/checkout@v3
26+
with:
27+
path: src/github.com/cloudbase/garm
28+
fetch-depth: 0
29+
30+
- name: Set up Docker Buildx
31+
uses: docker/setup-buildx-action@v2
32+
33+
- name: Login to GitHub Container Registry
34+
uses: docker/login-action@v2
35+
with:
36+
registry: ghcr.io
37+
username: ${{ github.actor }}
38+
password: ${{ secrets.GITHUB_TOKEN }}
39+
40+
- name: Build and push
41+
run: |
42+
cd src/github.com/cloudbase/garm
43+
VERSION=$(git describe --tags --match='v[0-9]*' --always ${{ github.event.inputs.ref }})
44+
docker buildx build \
45+
--provenance=false \
46+
--platform linux/amd64,linux/arm64 \
47+
--build-arg="GARM_REF=${{ github.event.inputs.ref }}" \
48+
-t ${{ github.event.inputs.push_to_project }}/garm:"${VERSION}" \
49+
--push .

Dockerfile

+25-7
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,29 @@
1-
FROM docker.io/golang:alpine
1+
FROM docker.io/golang:alpine AS builder
2+
ARG GARM_REF
23

3-
WORKDIR /root
4-
USER root
4+
LABEL stage=builder
55

6-
RUN apk add musl-dev gcc libtool m4 autoconf g++ make libblkid util-linux-dev git linux-headers mingw-w64-gcc
6+
RUN apk add musl-dev gcc libtool m4 autoconf g++ make libblkid util-linux-dev git linux-headers
7+
RUN git config --global --add safe.directory /build
78

8-
ADD ./scripts/build-static.sh /build-static.sh
9-
RUN chmod +x /build-static.sh
9+
ADD . /build/garm
10+
RUN cd /build/garm && git checkout ${GARM_REF}
11+
RUN git clone https://github.com/cloudbase/garm-provider-azure /build/garm-provider-azure
12+
RUN git clone https://github.com/cloudbase/garm-provider-openstack /build/garm-provider-openstack
1013

11-
CMD ["/bin/sh"]
14+
RUN cd /build/garm && go build -o /bin/garm \
15+
-tags osusergo,netgo,sqlite_omit_load_extension \
16+
-ldflags "-linkmode external -extldflags '-static' -s -w -X main.Version=$(git describe --tags --match='v[0-9]*' --dirty --always)" \
17+
/build/garm/cmd/garm
18+
RUN mkdir -p /opt/garm/providers.d
19+
RUN cd /build/garm-provider-azure && go build -ldflags="-linkmode external -extldflags '-static' -s -w" -o /opt/garm/providers.d/garm-provider-azure .
20+
RUN cd /build/garm-provider-openstack && go build -ldflags="-linkmode external -extldflags '-static' -s -w" -o /opt/garm/providers.d/garm-provider-openstack .
21+
22+
FROM scratch
23+
24+
COPY --from=builder /bin/garm /bin/garm
25+
COPY --from=builder /opt/garm/providers.d/garm-provider-openstack /opt/garm/providers.d/garm-provider-openstack
26+
COPY --from=builder /opt/garm/providers.d/garm-provider-azure /opt/garm/providers.d/garm-provider-azure
27+
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
28+
29+
ENTRYPOINT ["/bin/garm", "-config", "/etc/garm/config.toml"]

Dockerfile.build-static

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
FROM docker.io/golang:alpine
2+
3+
WORKDIR /root
4+
USER root
5+
6+
RUN apk add musl-dev gcc libtool m4 autoconf g++ make libblkid util-linux-dev git linux-headers mingw-w64-gcc
7+
8+
RUN wget http://musl.cc/aarch64-linux-musl-cross.tgz -O /tmp/aarch64-linux-musl-cross.tgz && \
9+
tar --strip-components=1 -C /usr/local -xzf /tmp/aarch64-linux-musl-cross.tgz && \
10+
rm /tmp/aarch64-linux-musl-cross.tgz
11+
12+
ADD ./scripts/build-static.sh /build-static.sh
13+
RUN chmod +x /build-static.sh
14+
15+
CMD ["/bin/sh"]

Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ default: build
1515
.PHONY : build-static test install-lint-deps lint go-test fmt fmtcheck verify-vendor verify
1616
build-static:
1717
@echo Building garm
18-
docker build --tag $(IMAGE_TAG) .
18+
docker build --tag $(IMAGE_TAG) -f Dockerfile.build-static .
1919
docker run --rm -e USER_ID=$(USER_ID) -e USER_GROUP=$(USER_GROUP) -v $(PWD):/build/garm:z $(IMAGE_TAG) /build-static.sh
2020
@echo Binaries are available in $(PWD)/bin
2121

README.md

+21-106
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
1-
# GitHub Actions Runner Manager (garm)
1+
# GitHub Actions Runner Manager (GARM)
22

33
[![Go Tests](https://github.com/cloudbase/garm/actions/workflows/go-tests.yml/badge.svg)](https://github.com/cloudbase/garm/actions/workflows/go-tests.yml)
44

5-
Welcome to garm!
5+
Welcome to GARM!
66

77
Garm enables you to create and automatically maintain pools of [self-hosted GitHub runners](https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners), with autoscaling that can be used inside your github workflow runs.
88

9-
The goal of ```garm``` is to be simple to set up, simple to configure and simple to use. It is a single binary that can run on any GNU/Linux machine without any other requirements other than the providers it creates the runners in. It is intended to be easy to deploy in any environment and can create runners in any system you can write a provider for. There is no complicated setup process and no extremely complex concepts to understand. Once set up, it's meant to stay out of your way.
9+
The goal of ```GARM``` is to be simple to set up, simple to configure and simple to use. It is a single binary that can run on any GNU/Linux machine without any other requirements other than the providers it creates the runners in. It is intended to be easy to deploy in any environment and can create runners in any system you can write a provider for. There is no complicated setup process and no extremely complex concepts to understand. Once set up, it's meant to stay out of your way.
1010

11-
Garm supports creating pools on either GitHub itself or on your own deployment of [GitHub Enterprise Server](https://docs.github.com/en/[email protected]/admin/overview/about-github-enterprise-server). For instructions on how to use ```garm``` with GHE, see the [credentials](/doc/github_credentials.md) section of the documentation.
11+
Garm supports creating pools on either GitHub itself or on your own deployment of [GitHub Enterprise Server](https://docs.github.com/en/[email protected]/admin/overview/about-github-enterprise-server). For instructions on how to use ```GARM``` with GHE, see the [credentials](/doc/github_credentials.md) section of the documentation.
1212

1313
## Join us on slack
1414

@@ -18,121 +18,36 @@ Whether you're running into issues or just want to drop by and say "hi", feel fr
1818

1919
## Installing
2020

21-
## Build from source
21+
Check out the [quickstart](/doc/quickstart.md) document for instructions on how to install ```GARM```. If you'd like to build from source, check out the [building from source](/doc/building_from_source.md) document.
2222

23-
You need to have Go installed, then run:
23+
## Installing external providers
2424

25-
```bash
26-
git clone https://github.com/cloudbase/garm
27-
cd garm
28-
go install ./...
29-
```
25+
External providers are binaries that GARM calls into to create runners in a particular IaaS. There are currently two external providers available:
3026

31-
You should now have both ```garm``` and ```garm-cli``` in your ```$GOPATH/bin``` folder.
27+
* [OpenStack](https://github.com/cloudbase/garm-provider-openstack)
28+
* [Azure](https://github.com/cloudbase/garm-provider-azure)
3229

33-
If you have docker/podman installed, you can also build statically linked binaries by running:
34-
35-
```bash
36-
make build-static
37-
```
38-
39-
The ```garm``` and ```garm-cli``` binaries will be built and copied to the ```bin/``` folder in your current working directory.
40-
41-
## Install the service
42-
43-
Add a new system user:
44-
45-
```bash
46-
useradd --shell /usr/bin/false \
47-
--system \
48-
--groups lxd \
49-
--no-create-home garm
50-
```
51-
52-
The ```lxd``` group is only needed if you have a local LXD install and want to connect to the unix socket to use it. If you're connecting to a remote LXD server over TCP, you can skip adding the ```garm``` user to the ```lxd``` group.
53-
54-
Copy the binary to somewhere in the system ```$PATH```:
55-
56-
```bash
57-
sudo cp $(go env GOPATH)/bin/garm /usr/local/bin/garm
58-
```
59-
60-
Or if you built garm using ```make```:
61-
62-
```bash
63-
sudo cp ./bin/garm /usr/local/bin/garm
64-
```
65-
66-
Create the config folder:
67-
68-
```bash
69-
sudo mkdir -p /etc/garm
70-
```
71-
72-
Copy the config template:
73-
74-
```bash
75-
sudo cp ./testdata/config.toml /etc/garm/
76-
```
77-
78-
Copy the external provider (optional):
79-
80-
```bash
81-
sudo cp -a ./contrib/providers.d /etc/garm/
82-
```
83-
84-
Copy the systemd service file:
85-
86-
```bash
87-
sudo cp ./contrib/garm.service /etc/systemd/system/
88-
```
89-
90-
Change permissions on config folder:
91-
92-
```bash
93-
sudo chown -R garm:garm /etc/garm
94-
sudo chmod 750 -R /etc/garm
95-
```
96-
97-
Enable the service:
98-
99-
```bash
100-
sudo systemctl enable garm
101-
```
102-
103-
Customize the config in ```/etc/garm/config.toml```, and start the service:
104-
105-
```bash
106-
sudo systemctl start garm
107-
```
30+
Follow the instructions in the README of each provider to install them.
10831

10932
## Configuration
11033

111-
The ```garm``` configuration is a simple ```toml```. A sample of the config file can be found in [the testdata folder](/testdata/config.toml).
112-
113-
There are 3 major sections of the config that require your attention:
114-
115-
* [Github credentials section](/doc/github_credentials.md)
116-
* [Providers section](/doc/providers.md)
117-
* [The database section](/doc/database.md)
118-
119-
Once you've configured your database, providers and github credentials, you'll need to configure your [webhooks and the callback_url](/doc/webhooks_and_callbacks.md).
120-
121-
At this point, you should be done. Have a look at the [running garm document](/doc/running_garm.md) for usage instructions and available features.
122-
123-
If you would like to use ```garm``` with a different IaaS than the ones already available, have a look at the [writing an external provider](/doc/external_provider.md) page.
124-
125-
If you like to optimize the startup time of new instance, take a look at the [performance considerations](/doc/performance_considerations.md) page.
34+
The ```GARM``` configuration is a simple ```toml```. The sample config file in [the testdata folder](/testdata/config.toml) is fairly well commented and should be enough to get you started. The configuration file is split into several sections, each of which is documented in its own page. The sections are:
12635

127-
## Security considerations
36+
* [The default section](/doc/config_default.md)
37+
* [Database](/doc/database.md)
38+
* [Github credentials](/doc/github_credentials.md)
39+
* [Providers](/doc/providers.md)
40+
* [Metrics](/doc/config_metrics.md)
41+
* [JWT authentication](/doc/config_jwt_auth.md)
42+
* [API server](/doc/config_api_server.md)
12843

129-
Garm does not apply any ACLs of any kind to the instances it creates. That task remains in the responsibility of the user. [Here is a guide for creating ACLs in LXD](https://linuxcontainers.org/lxd/docs/master/howto/network_acls/). You can of course use ```iptables``` or ```nftables``` to create any rules you wish. I recommend you create a separate isolated lxd bridge for runners, and secure it using ACLs/iptables/nftables.
44+
## Optimizing your runners
13045

131-
You must make sure that the code that runs as part of the workflows is trusted, and if that cannot be done, you must make sure that any malicious code that will be pulled in by the actions and run as part of a workload, is as contained as possible. There is a nice article about [securing your workflow runs here](https://blog.gitguardian.com/github-actions-security-cheat-sheet/).
46+
If you would like to optimize the startup time of new instance, take a look at the [performance considerations](/doc/performance_considerations.md) page.
13247

13348
## Write your own provider
13449

135-
The providers are interfaces between ```garm``` and a particular IaaS in which we spin up GitHub Runners. These providers can be either **native** or **external**. The **native** providers are written in ```Go```, and must implement [the interface defined here](https://github.com/cloudbase/garm/blob/main/runner/common/provider.go#L22-L39). **External** providers can be written in any language, as they are in the form of an external executable that ```garm``` calls into.
50+
The providers are interfaces between ```GARM``` and a particular IaaS in which we spin up GitHub Runners. These providers can be either **native** or **external**. The **native** providers are written in ```Go```, and must implement [the interface defined here](https://github.com/cloudbase/garm/blob/main/runner/common/provider.go#L22-L39). **External** providers can be written in any language, as they are in the form of an external executable that ```GARM``` calls into.
13651

13752
There is currently one **native** provider for [LXD](https://linuxcontainers.org/lxd/) and two **external** providers for [Openstack and Azure](/contrib/providers.d/).
13853

apiserver/controllers/controllers.go

+59-2
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,13 @@ import (
2121
"net/http"
2222
"strings"
2323

24+
gErrors "github.com/cloudbase/garm-provider-common/errors"
25+
"github.com/cloudbase/garm-provider-common/util"
2426
"github.com/cloudbase/garm/apiserver/params"
2527
"github.com/cloudbase/garm/auth"
26-
gErrors "github.com/cloudbase/garm/errors"
2728
"github.com/cloudbase/garm/metrics"
2829
runnerParams "github.com/cloudbase/garm/params"
2930
"github.com/cloudbase/garm/runner"
30-
"github.com/cloudbase/garm/util"
3131
wsWriter "github.com/cloudbase/garm/websocket"
3232

3333
"github.com/gorilla/websocket"
@@ -202,6 +202,13 @@ func (a *APIController) NotFoundHandler(w http.ResponseWriter, r *http.Request)
202202
}
203203
}
204204

205+
// swagger:route GET /metrics-token metrics-token GetMetricsToken
206+
//
207+
// Returns a JWT token that can be used to access the metrics endpoint.
208+
//
209+
// Responses:
210+
// 200: JWTResponse
211+
// 401: APIErrorResponse
205212
func (a *APIController) MetricsTokenHandler(w http.ResponseWriter, r *http.Request) {
206213
ctx := r.Context()
207214

@@ -222,6 +229,21 @@ func (a *APIController) MetricsTokenHandler(w http.ResponseWriter, r *http.Reque
222229
}
223230
}
224231

232+
// swagger:route POST /auth/login login Login
233+
//
234+
// Logs in a user and returns a JWT token.
235+
//
236+
// Parameters:
237+
// + name: Body
238+
// description: Login information.
239+
// type: PasswordLoginParams
240+
// in: body
241+
// required: true
242+
//
243+
// Responses:
244+
// 200: JWTResponse
245+
// 400: APIErrorResponse
246+
//
225247
// LoginHandler returns a jwt token
226248
func (a *APIController) LoginHandler(w http.ResponseWriter, r *http.Request) {
227249
var loginInfo runnerParams.PasswordLoginParams
@@ -253,6 +275,20 @@ func (a *APIController) LoginHandler(w http.ResponseWriter, r *http.Request) {
253275
}
254276
}
255277

278+
// swagger:route POST /first-run first-run FirstRun
279+
//
280+
// Initialize the first run of the controller.
281+
//
282+
// Parameters:
283+
// + name: Body
284+
// description: Create a new user.
285+
// type: NewUserParams
286+
// in: body
287+
// required: true
288+
//
289+
// Responses:
290+
// 200: User
291+
// 400: APIErrorResponse
256292
func (a *APIController) FirstRunHandler(w http.ResponseWriter, r *http.Request) {
257293
if a.auth.IsInitialized() {
258294
err := gErrors.NewConflictError("already initialized")
@@ -279,6 +315,13 @@ func (a *APIController) FirstRunHandler(w http.ResponseWriter, r *http.Request)
279315
}
280316
}
281317

318+
// swagger:route GET /credentials credentials ListCredentials
319+
//
320+
// List all credentials.
321+
//
322+
// Responses:
323+
// 200: Credentials
324+
// 400: APIErrorResponse
282325
func (a *APIController) ListCredentials(w http.ResponseWriter, r *http.Request) {
283326
ctx := r.Context()
284327
creds, err := a.r.ListCredentials(ctx)
@@ -293,6 +336,13 @@ func (a *APIController) ListCredentials(w http.ResponseWriter, r *http.Request)
293336
}
294337
}
295338

339+
// swagger:route GET /providers providers ListProviders
340+
//
341+
// List all providers.
342+
//
343+
// Responses:
344+
// 200: Providers
345+
// 400: APIErrorResponse
296346
func (a *APIController) ListProviders(w http.ResponseWriter, r *http.Request) {
297347
ctx := r.Context()
298348
providers, err := a.r.ListProviders(ctx)
@@ -307,6 +357,13 @@ func (a *APIController) ListProviders(w http.ResponseWriter, r *http.Request) {
307357
}
308358
}
309359

360+
// swagger:route GET /jobs jobs ListJobs
361+
//
362+
// List all jobs.
363+
//
364+
// Responses:
365+
// 200: Jobs
366+
// 400: APIErrorResponse
310367
func (a *APIController) ListAllJobs(w http.ResponseWriter, r *http.Request) {
311368
ctx := r.Context()
312369
jobs, err := a.r.ListAllJobs(ctx)

0 commit comments

Comments
 (0)