Merge branch 'main' into dependabot/go_modules/test_integration/githu…

…b.com/compose-spec/compose-go/v2-2.4.1
bacalhau-project · Feb 12, 2025 · 3587f95 · 3587f95
2 parents 3978519 + f0d4955
commit 3587f95
Show file tree

Hide file tree

Showing 53 changed files with 1,890 additions and 258 deletions.
diff --git a/.cspell/custom-dictionary.txt b/.cspell/custom-dictionary.txt
@@ -444,4 +444,5 @@ traefik
 bprotocolcompute
 bprotocolorchestrator
 nclprotocolcompute
-ncltest
+ncltest
+dind
diff --git a/Makefile b/Makefile
@@ -233,45 +233,85 @@ HTTP_GATEWAY_IMAGE ?= "ghcr.io/bacalhau-project/http-gateway"
 HTTP_GATEWAY_TAG ?= ${TAG}
 .PHONY: build-http-gateway-image
 build-http-gateway-image:
-	docker buildx build \
+	docker buildx build --load \
+		--platform linux/amd64,linux/arm64 \
+		-t ${HTTP_GATEWAY_IMAGE}:${HTTP_GATEWAY_TAG} \
+		pkg/executor/docker/gateway
+
+.PHONY: push-http-gateway-image
+push-http-gateway-image:
+	docker buildx build --push \
 		--platform linux/amd64,linux/arm64 \
 		-t ${HTTP_GATEWAY_IMAGE}:${HTTP_GATEWAY_TAG} \
 		pkg/executor/docker/gateway
 
 BACALHAU_IMAGE ?= ghcr.io/bacalhau-project/bacalhau
 BACALHAU_TAG ?= ${TAG}
 
-# Only tag images with :latest if the release tag is a semver tag (e.g. v0.3.12)
+# Only add latest tags if the release tag is a semver tag (e.g. v0.3.12)
 # and not a commit hash or a release candidate (e.g. v0.3.12-rc1)
-LATEST_TAG :=
 ifeq ($(shell echo ${BACALHAU_TAG} | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$$'), ${BACALHAU_TAG})
-	LATEST_TAG := --tag ${BACALHAU_IMAGE}:latest
+	BASE_TAGS := --tag ${BACALHAU_IMAGE}:${BACALHAU_TAG} \
+		--tag ${BACALHAU_IMAGE}:latest
+	DIND_TAGS := --tag ${BACALHAU_IMAGE}:${BACALHAU_TAG}-dind \
+		--tag ${BACALHAU_IMAGE}:latest-dind
+else
+	BASE_TAGS := --tag ${BACALHAU_IMAGE}:${BACALHAU_TAG}
+	DIND_TAGS := --tag ${BACALHAU_IMAGE}:${BACALHAU_TAG}-dind
 endif
 
 BACALHAU_IMAGE_FLAGS := \
 	--progress=plain \
-	--platform linux/amd64,linux/arm64 \
-	--tag ${BACALHAU_IMAGE}:${BACALHAU_TAG} \
-	${LATEST_TAG} \
 	--label org.opencontainers.artifact.created=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ") \
-	--label org.opencontainers.image.version=${BACALHAU_TAG} \
-	--cache-from=type=registry,ref=${BACALHAU_IMAGE}:latest \
-	--file docker/bacalhau-image/Dockerfile \
-	.
+	--label org.opencontainers.image.version=${BACALHAU_TAG}
+
+.PHONY: build-bacalhau-base-image
+build-bacalhau-base-image:
+	docker buildx build --load ${BACALHAU_IMAGE_FLAGS} \
+		${BASE_TAGS} \
+		--cache-from=type=registry,ref=${BACALHAU_IMAGE}:latest \
+		--file docker/bacalhau-base/Dockerfile \
+		.
+
+.PHONY: build-bacalhau-dind-image
+build-bacalhau-dind-image:
+	docker buildx build --load ${BACALHAU_IMAGE_FLAGS} \
+		${DIND_TAGS} \
+		--cache-from=type=registry,ref=${BACALHAU_IMAGE}:latest-dind \
+		--file docker/bacalhau-dind/Dockerfile \
+		.
+
+# Push targets (multi-platform)
+.PHONY: push-bacalhau-base-image
+push-bacalhau-base-image:
+	docker buildx build --push ${BACALHAU_IMAGE_FLAGS} \
+		--platform linux/amd64,linux/arm64 \
+		${BASE_TAGS} \
+		--cache-from=type=registry,ref=${BACALHAU_IMAGE}:latest \
+		--file docker/bacalhau-base/Dockerfile \
+		.
+
+.PHONY: push-bacalhau-dind-image
+push-bacalhau-dind-image:
+	docker buildx build --push ${BACALHAU_IMAGE_FLAGS} \
+		--platform linux/amd64,linux/arm64 \
+		${DIND_TAGS} \
+		--cache-from=type=registry,ref=${BACALHAU_IMAGE}:latest-dind \
+		--file docker/bacalhau-dind/Dockerfile \
+		.
 
-.PHONY: build-bacalhau-image
-build-bacalhau-image:
-	docker buildx build ${BACALHAU_IMAGE_FLAGS}
+# Combined targets for building and pushing all images
+.PHONY: build-bacalhau-images
+build-bacalhau-images: build-bacalhau-base-image build-bacalhau-dind-image
 
-.PHONY: push-bacalhau-image
-push-bacalhau-image:
-	docker buildx build --push ${BACALHAU_IMAGE_FLAGS}
+.PHONY: push-bacalhau-images
+push-bacalhau-images: push-bacalhau-base-image push-bacalhau-dind-image
 
 .PHONY: build-docker-images
 build-docker-images: build-http-gateway-image
 
 .PHONY: push-docker-images
-push-docker-images: build-http-gateway-image
+push-docker-images: push-http-gateway-image
 
 # Release tarballs suitable for upload to GitHub release pages
 ################################################################################

diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 <p align="center">
   <a href="https://github.com/bacalhau-project/bacalhau">
-    <img src="./docs/static/img/logo/Bacalhau-horizontal.svg" alt="Bacalhau" width="300"/>
+    <img src="./docs/logo/Bacalhau-horizontal.svg" alt="Bacalhau" width="300"/>
   </a>
 </p>
 

diff --git a/buildkite/scripts/bacalhau_image.sh b/buildkite/scripts/bacalhau_image.sh
@@ -11,7 +11,7 @@ docker_login() {
     echo $GHCR_PAT | docker login ghcr.io -u bacalhau-infra-bot --password-stdin
 }
 
-docker_context_create() {
+setup_buildx() {
     docker context create buildx-build
     docker buildx create --use buildx-build
 }
@@ -30,26 +30,26 @@ download_and_extract_artifact() {
 }
 
 download_artifacts() {
+    echo "--- Downloading build artifacts"
     if ! buildkite-agent artifact download "*.*" . --build "$BUILDKITE_BUILD_ID"; then
         echo "Error: Failed to download artifacts from build pipeline" >&2
         exit 1
     fi
-    echo "Downloaded artifacts from build pipeline"
 
     download_and_extract_artifact "amd64"
     download_and_extract_artifact "arm64"
 }
 
 main() {
     if [ -n "${BUILDKITE_TAG:-}" ]; then
+        echo "=== Building and pushing images for tag: ${BUILDKITE_TAG}"
         set_environment_variables
-        docker_context_create
+        setup_buildx
         download_artifacts
-        make build-bacalhau-image
         docker_login
-        make push-bacalhau-image
+        make push-bacalhau-images
     else
-        echo "Skipping artifact download: BUILDKITE_TAG is not present"
+        echo "Skipping image build: BUILDKITE_TAG is not present"
     fi
 }
 

diff --git a/cmd/util/opts/publisher_specconfig.go b/cmd/util/opts/publisher_specconfig.go
@@ -11,7 +11,6 @@ import (
 	"github.com/bacalhau-project/bacalhau/pkg/models"
 	publisher_ipfs "github.com/bacalhau-project/bacalhau/pkg/publisher/ipfs"
 	publisher_local "github.com/bacalhau-project/bacalhau/pkg/publisher/local"
-	publisher_s3 "github.com/bacalhau-project/bacalhau/pkg/s3"
 )
 
 // compile-time check to ensure type implements the flag.Value interface
@@ -101,30 +100,23 @@ func publisherStringToSpecConfig(destinationURI string, options map[string]strin
 	case "ipfs":
 		res = publisher_ipfs.NewSpecConfig()
 	case "s3":
-		var bucket, key string
-		var opts []publisher_s3.PublisherOption
-		if _, ok := options["bucket"]; !ok {
-			bucket = parsedURI.Host
-		} else {
-			bucket = options["bucket"]
+		// copy all options to params
+		params := map[string]interface{}{}
+		for k, v := range options {
+			params[k] = v
 		}
-		if _, ok := options["key"]; !ok {
-			key = strings.TrimLeft(parsedURI.Path, "/")
-		} else {
-			key = options["key"]
-		}
-		region, ok := options["region"]
-		if ok {
-			opts = append(opts, publisher_s3.WithPublisherRegion(region))
-		}
-		endpoint, ok := options["endpoint"]
-		if ok {
-			opts = append(opts, publisher_s3.WithPublisherEndpoint(endpoint))
+
+		// parse bucket and key from URI if not provided in options
+		if _, ok := params["bucket"]; !ok {
+			params["bucket"] = parsedURI.Host
 		}
-		res, err = publisher_s3.NewPublisherSpec(bucket, key, opts...)
-		if err != nil {
-			return nil, err
+		if _, ok := params["key"]; !ok {
+			params["key"] = strings.TrimLeft(parsedURI.Path, "/")
 		}
+		return &models.SpecConfig{
+			Type:   models.PublisherS3,
+			Params: params,
+		}, nil
 	case "local":
 		res = publisher_local.NewSpecConfig()
 	default:

diff --git a/cmd/util/opts/publisher_specconfig_test.go b/cmd/util/opts/publisher_specconfig_test.go
@@ -32,10 +32,8 @@ func TestParsePublisherSpecConfig(t *testing.T) {
 			expected: &models.SpecConfig{
 				Type: models.PublisherS3,
 				Params: map[string]interface{}{
-					"Bucket":   "myBucket",
-					"Key":      "dir/file-001.txt",
-					"Endpoint": "",
-					"Region":   "",
+					"bucket": "myBucket",
+					"key":    "dir/file-001.txt",
 				},
 			},
 		},
@@ -45,10 +43,22 @@ func TestParsePublisherSpecConfig(t *testing.T) {
 			expected: &models.SpecConfig{
 				Type: models.PublisherS3,
 				Params: map[string]interface{}{
-					"Bucket":   "myBucket",
-					"Key":      "dir/file-001.txt",
-					"Endpoint": "http://127.0.0.1:9000",
-					"Region":   "us-east-1",
+					"bucket":   "myBucket",
+					"key":      "dir/file-001.txt",
+					"endpoint": "http://127.0.0.1:9000",
+					"region":   "us-east-1",
+				},
+			},
+		},
+		{
+			name:  "s3 with encoding",
+			input: "s3://myBucket/dir/file-001.txt,opt=encoding=plain",
+			expected: &models.SpecConfig{
+				Type: models.PublisherS3,
+				Params: map[string]interface{}{
+					"bucket":   "myBucket",
+					"key":      "dir/file-001.txt",
+					"encoding": "plain",
 				},
 			},
 		},
@@ -58,10 +68,9 @@ func TestParsePublisherSpecConfig(t *testing.T) {
 			expected: &models.SpecConfig{
 				Type: models.PublisherS3,
 				Params: map[string]interface{}{
-					"Bucket":   "myBucket",
-					"Key":      "dir/file-001.txt",
-					"Region":   "us-east-1",
-					"Endpoint": "",
+					"bucket": "myBucket",
+					"key":    "dir/file-001.txt",
+					"region": "us-east-1",
 				},
 			},
 		},

diff --git a/docker/bacalhau-image/Dockerfile → docker/bacalhau-base/Dockerfile b/docker/bacalhau-image/Dockerfile → docker/bacalhau-base/Dockerfile
@@ -6,10 +6,13 @@ ARG TARGETPLATFORM
 # Take advantage of the format for $TARGETPLATFORM being "OS/ARCH"
 # which matches our output directory structure in ./bin
 ADD bin/$TARGETPLATFORM/bacalhau /usr/local/bin/bacalhau
+
 ENV PATH="/usr/local/bin:/usr/bin"
+
 ENTRYPOINT ["bacalhau"]
-LABEL org.opencontainers.image.source https://github.com/bacalhau-project/bacalhau
-LABEL org.opencontainers.image.title "Bacalhau"
-LABEL org.opencontainers.image.description "The Bacalhau network provides decentralized compute for compute over data. See https://bacalhau.org for more info."
-LABEL org.opencontainers.image.licenses Apache-2.0
-LABEL org.opencontainers.image.url https://bacalhau.org
+
+LABEL org.opencontainers.image.source="https://github.com/bacalhau-project/bacalhau"
+LABEL org.opencontainers.image.title="Bacalhau"
+LABEL org.opencontainers.image.description="The Bacalhau network provides decentralized compute for compute over data. See https://bacalhau.org for more info."
+LABEL org.opencontainers.image.licenses="Apache-2.0"
+LABEL org.opencontainers.image.url="https://bacalhau.org"
diff --git a/docker/bacalhau-base/README.md b/docker/bacalhau-base/README.md
@@ -0,0 +1,71 @@
+# Bacalhau Base Image
+
+This is the standard Bacalhau container image, suitable for running orchestrator nodes, clients, and compute nodes with non-Docker execution engines (like WASM).
+
+## Image Information
+
+- Base Image: `ubuntu:24.04`
+- Registry: `ghcr.io/bacalhau-project/bacalhau`
+- Tags:
+    - `latest`: Most recent stable release
+    - `vX.Y.Z`: Specific version (e.g., `v1.6.0`)
+
+## Use Cases
+
+This image is ideal for:
+- Running orchestrator nodes
+- Running the Bacalhau client for job submission
+- Running compute nodes that don't require Docker execution capabilities
+
+## Usage Examples
+
+### Running an Orchestrator Node
+
+```bash
+docker run ghcr.io/bacalhau-project/bacalhau:latest serve --orchestrator
+```
+
+### Using as a Client
+
+```bash
+docker run ghcr.io/bacalhau-project/bacalhau:latest list
+```
+
+### Running a WASM Compute Node
+
+```bash
+docker run ghcr.io/bacalhau-project/bacalhau:latest serve --compute 
+```
+
+### Running a Specific Version
+
+```bash
+docker run ghcr.io/bacalhau-project/bacalhau:v1.6.0 serve
+```
+
+## Features
+
+- Minimal image size
+- Standard Ubuntu-based environment
+- Support for orchestrator nodes
+- Support for client operations
+- Support for WASM compute nodes
+- Multi-architecture support (amd64/arm64)
+
+## When to Use This Image
+
+Use this image when:
+- Running orchestrator nodes[README.md](../bacalhau-dind/README.md)
+[README.md](README.md)
+- Using Bacalhau as a client
+- Running compute nodes with WASM execution
+- Running in environments where Docker-in-Docker is not needed or desired
+- Minimal container footprint is desired
+
+For compute nodes requiring Docker execution capabilities, use the DinD variant instead (`bacalhau:latest-dind`).
+
+## Additional Resources
+
+- [Bacalhau Documentation](https://docs.bacalhau.org/)
+- [GitHub Repository](https://github.com/bacalhau-project/bacalhau)
+- [Getting Started Guide](https://docs.bacalhau.org/getting-started)
diff --git a/docker/bacalhau-dind/Dockerfile b/docker/bacalhau-dind/Dockerfile
@@ -0,0 +1,26 @@
+FROM docker:dind
+
+# Install necessary packages
+RUN apk update && apk add --no-cache \
+    curl \
+    bash \
+    coreutils
+
+# Automatically set by Docker to be the --platform flag
+ARG TARGETPLATFORM
+
+# Take advantage of the format for $TARGETPLATFORM being "OS/ARCH"
+# which matches our output directory structure in ./bin
+ADD bin/$TARGETPLATFORM/bacalhau /usr/local/bin/bacalhau
+
+# Add our custom entrypoint script
+COPY docker/bacalhau-dind/entrypoint.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/entrypoint.sh
+
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh", "bacalhau"]
+
+LABEL org.opencontainers.image.source="https://github.com/bacalhau-project/bacalhau"
+LABEL org.opencontainers.image.title="Bacalhau"
+LABEL org.opencontainers.image.description="The Bacalhau network provides distributed compute over data. See https://bacalhau.org for more info."
+LABEL org.opencontainers.image.licenses="Apache-2.0"
+LABEL org.opencontainers.image.url="https://bacalhau.org"