initial commit

Author: Raeveen Pasupathy

.github/workflows/deploy.yaml

@@ -0,0 +1,62 @@
+name: Deploy Terraform Module
+on:
+  workflow_dispatch:
+jobs:
+  generate-terraform-plan:
+    name: Generate Terraform Plan
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Terraform Init
+        uses: hashicorp/setup-terraform@v2
+        with:
+          terraform_version: 1.1.7
+      - name: Terraform Plan
+        run: |
+          terraform init
+          terraform plan \
+            -out="plan.out"
+      - name: Upload Terraform Plan
+        uses: actions/upload-artifact@v4
+        with:
+          name: terraform-plan
+          path: plan.out
+  deploy:
+    name: Deploy to environment
+    needs: generate-terraform-plan
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: 'arm64,arm'
+      - uses: docker/setup-buildx-action@v2
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Terraform Init
+        uses: hashicorp/setup-terraform@v2
+        with:
+          terraform_version: 1.1.7
+      - name: Download Plan
+        uses: actions/download-artifact@v4
+        with:
+          name: terraform-plan
+      - name: Install Python 3
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.9"
+      - name: Terraform Apply
+        run: |
+          terraform init
+          terraform apply plan.out

.gitignore

@@ -0,0 +1,3 @@
+.DS_Store
+.terraform
+.terraform.lock.hcl

Dockerfile

@@ -0,0 +1,35 @@
+FROM ollama/ollama:latest
+COPY --from=inspectorgadget12/lambda-runtime-adapter /lambda-runtime-adapter /opt/extensions/lambda-adapter
+
+ENV OLLAMA_HOST=0.0.0.0:8080
+ENV HOME=/mnt/ollama
+
+# Pre-pull the model during build
+RUN ollama serve & sleep 2 && ollama pull llama3 && pkill ollama
+
+EXPOSE 8080
+
+# Add a script to run the model
+COPY <<EOF /run.sh
+#!/bin/bash
+
+echo "Starting Ollama server..."
+ollama serve &
+SERVE_PID=$!
+
+echo "Waiting for Ollama server to be ready..."
+while ! curl -s localhost:8080/api/tags >/dev/null 2>&1; do
+    sleep 1
+done
+
+echo "Running llama3 model..."
+ollama run llama3 &
+LLAMA_PID=$!
+
+# Keep the container running and wait for both processes
+wait $SERVE_PID $LLAMA_PID
+EOF
+
+RUN chmod +x /run.sh
+
+ENTRYPOINT ["/run.sh"]

README.md

@@ -0,0 +1,21 @@
+# Ollama on AWS Lambda
+
+# Technologies
+1. Terraform
+2. AWS Lambda
+3. Docker
+4. Bash Scripting
+
+# Description
+This project exerts the simple way on running Ollama on AWS Lambda. This leverages the power of serverless computing to run Ollama on AWS Lambda while staying relevant to the AWS Free Tier. This project is a simple demonstration of how to run Ollama on AWS Lambda.
+
+Please do not use this method for Production use-case as the inteferencing is still slow on AWS Lambda. But this is a great way of running Ollama on AWS Lambda for testing purposes or learning purposes. 
+
+# Setting up the environment
+1. `git clone https://github.com/InspectorGadget/ollama-on-lambda.git`
+2. Update `backend.tf` with your S3 Bucket name.
+3. Run `terraform init`
+4. Verify the changes that would be performed on your environment.
+5. Run `terraform apply`
+6. Wait for the changes to be applied.
+7. Perform inferencing with your newly deployed Ollama instance.

backend.tf

@@ -0,0 +1,7 @@
+terraform {
+  backend "s3" {
+    bucket = "ollama-on-aws-lambda-state"
+    key    = "terraform.tfstate"
+    region = "ap-southeast-1"
+  }
+}

data.tf

@@ -0,0 +1,37 @@
+# Current account
+data "aws_caller_identity" "current" {}
+
+# Current region
+data "aws_region" "current" {}
+
+# Get VPC by ID
+data "aws_vpc" "main" {
+  id = var.vpc_id
+}
+
+# Get Private Subnets
+data "aws_subnets" "private" {
+  filter {
+    name   = "vpc-id"
+    values = [var.vpc_id]
+  }
+
+  filter {
+    name   = "tag:Name"
+    values = ["*Private*"]
+  }
+}
+
+# Get Public Subnets
+data "aws_subnets" "public" {
+  filter {
+    name   = "vpc-id"
+    values = [var.vpc_id]
+  }
+
+  filter {
+    name   = "tag:Name"
+    values = ["*Public*"]
+  }
+}
+

docker-compose.yml

@@ -0,0 +1,15 @@
+services:
+  ollama:
+    container_name: ollama
+    build: 
+      context: .
+      dockerfile: Dockerfile
+      platforms:
+        - linux/arm64
+    ports:
+      - "8080:8080"
+    volumes:
+      - ollama:/tmp/.ollama
+
+volumes:
+  ollama:

ecr.tf

@@ -0,0 +1,33 @@
+# ECR Repository
+resource "aws_ecr_repository" "ecr" {
+  name         = "${var.project_name}-runtime"
+}
+
+# Build Image
+resource "null_resource" "build" {
+  depends_on = [
+    aws_ecr_repository.ecr
+  ]
+
+  triggers = {
+    build_number = timestamp()
+  }
+
+  provisioner "local-exec" {
+    interpreter = [
+      "/bin/sh",
+      "-c"
+    ]
+
+    command = <<-COMMAND
+      # Login to ECR (AWS STS AssumeRole)
+      aws ecr get-login-password --region ${data.aws_region.current.name} | docker login --username AWS --password-stdin ${data.aws_caller_identity.current.account_id}.dkr.ecr.${data.aws_region.current.name}.amazonaws.com
+
+      # Build Image
+      docker build -t ${aws_ecr_repository.ecr.repository_url}:latest . -f Dockerfile --platform linux/arm64 --provenance=false
+
+      # Push Image
+      docker push ${aws_ecr_repository.ecr.repository_url}:latest
+    COMMAND
+  }
+}

function_url.tf

@@ -0,0 +1,19 @@
+resource "aws_lambda_function_url" "ollama" {
+  depends_on = [
+    aws_lambda_function.ollama,
+    null_resource.post_deployment
+  ]
+
+  function_name      = aws_lambda_function.ollama.function_name
+  authorization_type = "NONE"
+  invoke_mode        = "RESPONSE_STREAM"
+
+  cors {
+    allow_credentials = true
+    allow_origins     = ["*"]
+    allow_methods     = ["*"]
+    allow_headers     = ["date", "keep-alive"]
+    expose_headers    = ["keep-alive", "date"]
+    max_age           = 86400
+  }
+}

iam.tf

@@ -0,0 +1,42 @@
+resource "aws_iam_role" "lambda" {
+  name               = "${var.project_name}-role"
+  assume_role_policy = <<-EOF
+    {
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Action": "sts:AssumeRole",
+                "Principal": {
+                    "Service": [
+                      "lambda.amazonaws.com",
+                      "apigateway.amazonaws.com",
+                      "events.amazonaws.com"
+                    ]
+                },
+                "Effect": "Allow",
+                "Sid": ""
+            }
+        ]
+    }
+    EOF
+}
+
+# Attach Basic Execution Policy to Lambda Role
+resource "aws_iam_role_policy_attachment" "lambda" {
+  depends_on = [
+    aws_iam_role.lambda
+  ]
+
+  role       = aws_iam_role.lambda.name
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
+}
+
+# Attach AWSLambdaVPCAccessExecutionRole
+resource "aws_iam_role_policy_attachment" "vpc" {
+  depends_on = [
+    aws_iam_role.lambda
+  ]
+
+  role       = aws_iam_role.lambda.name
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole"
+}

lambda.tf

@@ -0,0 +1,55 @@
+# Create AWS Lambda Function from ECR Image
+resource "aws_lambda_function" "ollama" {
+  depends_on = [
+    aws_ecr_repository.ecr,
+    null_resource.build,
+    aws_iam_role.lambda
+  ]
+
+  function_name = var.project_name
+  architectures = [
+    "arm64",
+  ]
+  package_type = "Image"
+  image_uri    = "${aws_ecr_repository.ecr.repository_url}:latest"
+  role         = aws_iam_role.lambda.arn
+  memory_size  = 10240
+  timeout      = 900
+
+  ephemeral_storage {
+    size = 10240
+  }
+
+  environment {
+    variables = {
+      RUST_LOG            = "debug"
+      PORT                = "8080"
+      HOME                = "/mnt/ollama"
+      AWS_LWA_INVOKE_MODE = "response_stream"
+    }
+  }
+}
+
+# Update Lambda Image to latest with Lambda API
+resource "null_resource" "post_deployment" {
+  depends_on = [
+    aws_lambda_function.ollama,
+    null_resource.build
+  ]
+
+  triggers = {
+    build_number = timestamp()
+  }
+
+  provisioner "local-exec" {
+    interpreter = [
+      "/bin/sh",
+      "-c"
+    ]
+
+    command = <<-COMMAND
+        # Update Lambda Image to latest with Lambda API
+        aws lambda update-function-code --function-name ${aws_lambda_function.ollama.function_name} --image-uri ${aws_ecr_repository.ecr.repository_url}:latest
+    COMMAND
+  }
+}

main.tf

@@ -0,0 +1,16 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5.0"
+    }
+    null = {
+      source  = "hashicorp/null"
+      version = "> 3.0"
+    }
+  }
+}
+
+provider "aws" {
+  region = "ap-southeast-1"
+}

output.tf

@@ -0,0 +1,3 @@
+output "lambda_function_url" {
+  value = aws_lambda_function_url.ollama.function_url
+}

variable.tf

@@ -0,0 +1,7 @@
+variable "project_name" {
+  default = "ollama-on-aws-lambda"
+}
+
+variable "vpc_id" {
+  default = "vpc-08f4f76fa7689a36b"
+}