Merge pull request #31 from ajcwebdev/next

`setup.sh` script and Fastify server integration

Author: ajcwebdev

README.md

@@ -72,14 +72,6 @@ git clone https://github.com/ggerganov/whisper.cpp.git && \
 
 > Replace `base` with `large-v2` for the largest model, `medium` for a middle sized model, or `tiny` for the smallest model.
 
-### Clone Llama Repo
-
-```bash
-git clone https://github.com/ggerganov/llama.cpp && \
-  make -C llama.cpp && \
-  cp .github/llama.Dockerfile llama.cpp/Dockerfile
-```
-
 ## Run Autoshow Node Scripts
 
 Run on a single YouTube video.
@@ -123,7 +115,7 @@ Use 3rd party LLM providers.
 ```bash
 npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --chatgpt GPT_4o_MINI
 npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --claude CLAUDE_3_5_SONNET
-npm run as -- --video "https://www.youtube.com/watch?v=h41DF9GUqx4" --gemini GEMINI_1_5_PRO
+npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --gemini GEMINI_1_5_PRO
 npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --cohere COMMAND_R_PLUS
 npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --mistral MISTRAL_LARGE
 npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --octo LLAMA_3_1_405B

docker-compose.yml

@@ -13,7 +13,7 @@ services:
       - whisper
       - ollama
     environment:
-      - OLLAMA_HOST=localhost
+      - OLLAMA_HOST=ollama
       - OLLAMA_PORT=11434
   whisper:
     build:
@@ -29,5 +29,10 @@ services:
     image: ollama/ollama
     ports:
       - "11434:11434"
-volumes:
-  whisper:
+    volumes:
+      - ./ollama:/root/.ollama
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:11434/healthz"]
+      interval: 10s
+      timeout: 5s
+      retries: 5

docs/examples.md

@@ -48,6 +48,12 @@ Run on multiple YouTube videos in a playlist.
 npm run as -- --playlist "https://www.youtube.com/playlist?list=PLCVnrVv4KhXPz0SoAVu8Rc1emAdGPbSbr"
 ```
 
+Run on playlist URL and generate JSON info file with markdown metadata of each video in the playlist:
+
+```bash
+npm run as -- --playlist "https://www.youtube.com/playlist?list=PLCVnrVv4KhXPz0SoAVu8Rc1emAdGPbSbr" --info
+```
+
 ### Process Multiple Videos Specified in a URLs File
 
 Run on an arbitrary list of URLs in `example-urls.md`.
@@ -56,6 +62,12 @@ Run on an arbitrary list of URLs in `example-urls.md`.
 npm run as -- --urls "content/example-urls.md"
 ```
 
+Run on URLs file and generate JSON info file with markdown metadata of each video:
+
+```bash
+npm run as -- --urls "content/example-urls.md" --info
+```
+
 ### Process Single Audio or Video File
 
 Run on `audio.mp3` on the `content` directory:

docs/server.md

@@ -37,8 +37,6 @@ curl --json '{
 }' http://localhost:3000/video
 ```
 
-
-
 Use LLM.
 
 ```bash
@@ -54,15 +52,15 @@ curl --json '{
 ```bash
 curl --json '{
   "playlistUrl": "https://www.youtube.com/playlist?list=PLCVnrVv4KhXPz0SoAVu8Rc1emAdGPbSbr"
-}' http://localhost:3000/playlist 
+}' http://localhost:3000/playlist
 ```
 
 ```bash
 curl --json '{
   "playlistUrl": "https://www.youtube.com/playlist?list=PLCVnrVv4KhXPz0SoAVu8Rc1emAdGPbSbr",
   "whisperModel": "tiny",
   "llm": "llama"
-}' http://localhost:3000/playlist 
+}' http://localhost:3000/playlist
 ```
 
 ### URLs Endpoint
@@ -271,14 +269,14 @@ curl --json '{
 ```bash
 curl --json '{
   "youtubeUrl": "https://www.youtube.com/watch?v=MORMZXEaONk",
-  "transcriptionService": "deepgram"
+  "transcriptService": "deepgram"
 }' http://localhost:3000/video
 ```
 
 ```bash
 curl --json '{
   "youtubeUrl": "https://www.youtube.com/watch?v=MORMZXEaONk",
-  "transcriptionService": "deepgram",
+  "transcriptService": "deepgram",
   "llm": "llama"
 }' http://localhost:3000/video
 ```
@@ -288,30 +286,30 @@ curl --json '{
 ```bash
 curl --json '{
   "youtubeUrl": "https://www.youtube.com/watch?v=MORMZXEaONk",
-  "transcriptionService": "assembly"
+  "transcriptService": "assembly"
 }' http://localhost:3000/video
 ```
 
 ```bash
 curl --json '{
   "youtubeUrl": "https://www.youtube.com/watch?v=MORMZXEaONk",
-  "transcriptionService": "assembly",
+  "transcriptService": "assembly",
   "llm": "llama"
 }' http://localhost:3000/video
 ```
 
 ```bash
 curl --json '{
   "youtubeUrl": "https://ajc.pics/audio/fsjam-short.mp3",
-  "transcriptionService": "assembly",
+  "transcriptService": "assembly",
   "speakerLabels": true
 }' http://localhost:3000/video
 ```
 
 ```bash
 curl --json '{
   "youtubeUrl": "https://ajc.pics/audio/fsjam-short.mp3",
-  "transcriptionService": "assembly",
+  "transcriptService": "assembly",
   "speakerLabels": true,
   "llm": "llama"
 }' http://localhost:3000/video

package.json

@@ -18,6 +18,7 @@
     "autoshow": "./autoshow.js"
   },
   "scripts": {
+    "setup": "bash ./setup.sh",
     "autoshow": "node --env-file=.env --no-warnings src/autoshow.js",
     "as": "node --env-file=.env --no-warnings src/autoshow.js",
     "bun-as": "bun --env-file=.env --no-warnings src/autoshow.js",
@@ -34,13 +35,15 @@
   "dependencies": {
     "@anthropic-ai/sdk": "^0.26.0",
     "@deepgram/sdk": "^3.5.1",
+    "@fastify/cors": "^10.0.1",
     "@google/generative-ai": "^0.17.1",
     "@mistralai/mistralai": "^1.0.2",
     "@octoai/sdk": "^1.5.1",
     "assemblyai": "^4.6.1",
     "cohere-ai": "^7.12.0",
     "commander": "^12.1.0",
     "fast-xml-parser": "^4.4.1",
+    "fastify": "^5.0.0",
     "ffmpeg-static": "^5.2.0",
     "file-type": "^19.4.1",
     "inquirer": "^10.2.2",

server/index.js

@@ -1,65 +1,49 @@
 // server/index.js
 
-import http from 'node:http'
+import Fastify from 'fastify'
+import cors from '@fastify/cors'
 import { handleVideoRequest } from './routes/video.js'
 import { handlePlaylistRequest } from './routes/playlist.js'
 import { handleURLsRequest } from './routes/urls.js'
 import { handleFileRequest } from './routes/file.js'
 import { handleRSSRequest } from './routes/rss.js'
 import { env } from 'node:process'
 
+// Set the port from environment variable or default to 3000
 const port = env.PORT || 3000
 
-const server = http.createServer(async (req, res) => {
-  console.log(`[${new Date().toISOString()}] Received ${req.method} request for ${req.url}`)
-  res.setHeader('Access-Control-Allow-Origin', '*')
-  res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS')
-  res.setHeader('Access-Control-Allow-Headers', 'Content-Type')
-  console.log('CORS headers set')
+async function start() {
+  // Create a Fastify instance with logging enabled
+  const fastify = Fastify({ logger: true })
 
-  if (req.method === 'OPTIONS') {
-    console.log('Handling OPTIONS preflight request')
-    res.writeHead(204)
-    res.end()
-    return
-  }
+  // Register CORS plugin to handle CORS headers and preflight requests
+  await fastify.register(cors, {
+    origin: '*',
+    methods: ['GET', 'POST', 'OPTIONS'],
+    allowedHeaders: ['Content-Type'],
+  })
+
+  // Log each incoming request
+  fastify.addHook('onRequest', async (request, reply) => {
+    console.log(
+      `[${new Date().toISOString()}] Received ${request.method} request for ${request.url}`
+    )
+  })
+
+  // Define route handlers
+  fastify.post('/video', handleVideoRequest)
+  fastify.post('/playlist', handlePlaylistRequest)
+  fastify.post('/urls', handleURLsRequest)
+  fastify.post('/file', handleFileRequest)
+  fastify.post('/rss', handleRSSRequest)
 
-  if (req.method === 'POST') {
-    switch (req.url) {
-      case '/video':
-        console.log('Routing to handleVideoRequest')
-        await handleVideoRequest(req, res)
-        break
-      case '/playlist':
-        console.log('Routing to handlePlaylistRequest')
-        await handlePlaylistRequest(req, res)
-        break
-      case '/urls':
-        console.log('Routing to handleURLsRequest')
-        await handleURLsRequest(req, res)
-        break
-      case '/file':
-        console.log('Routing to handleFileRequest')
-        await handleFileRequest(req, res)
-        break
-      case '/rss':
-        console.log('Routing to handleRSSRequest')
-        await handleRSSRequest(req, res)
-        break
-      default:
-        console.log('Unknown route, sending 404')
-        res.statusCode = 404
-        res.setHeader('Content-Type', 'application/json')
-        res.end(JSON.stringify({ error: 'Not Found' }))
-    }
-  } else {
-    console.log(`Method ${req.method} not allowed, sending 405`)
-    res.statusCode = 405
-    res.setHeader('Content-Type', 'application/json')
-    res.end(JSON.stringify({ error: 'Method Not Allowed' }))
+  try {
+    await fastify.listen({ port })
+    console.log(`Server running at http://localhost:${port}`)
+  } catch (err) {
+    fastify.log.error(err)
+    process.exit(1)
   }
-})
+}
 
-server.listen(port, () => {
-  console.log(`Server running at http://localhost:${port}`)
-})
+start()

server/routes/file.js

@@ -1,41 +1,38 @@
 // server/routes/file.js
 
 import { processFile } from '../../src/commands/processFile.js'
+import { reqToOpts } from '../utils/reqToOpts.js'
 
-const handleFileRequest = async (req, res) => {
+// Handler for /file route
+const handleFileRequest = async (request, reply) => {
   console.log('Entered handleFileRequest')
-  let body = ''
-  req.on('data', chunk => {
-    body += chunk.toString()
-    console.log('Received chunk:', chunk.toString())
-  })
-  req.on('end', async () => {
-    console.log('Request body complete:', body)
-    try {
-      const { filePath, model = 'base', llm, options = {} } = JSON.parse(body)
-      console.log('Parsed request body:', { filePath, model, llm, options })
-      if (!filePath) {
-        console.log('File path not provided, sending 400')
-        res.statusCode = 400
-        res.setHeader('Content-Type', 'application/json')
-        res.end(JSON.stringify({ error: 'File path is required' }))
-        return
-      }
-      const llmOpt = llm || null
-      await processFile(filePath, llmOpt, model, options)
-      console.log('processFile completed successfully')
-      res.statusCode = 200
-      res.setHeader('Content-Type', 'application/json')
-      res.end(JSON.stringify({ 
-        message: 'File processed successfully.'
-      }))
-    } catch (error) {
-      console.error('Error processing file:', error)
-      res.statusCode = 500
-      res.setHeader('Content-Type', 'application/json')
-      res.end(JSON.stringify({ error: 'An error occurred while processing the file' }))
+
+  try {
+    // Access parsed request body
+    const requestData = request.body
+    console.log('Parsed request body:', requestData)
+
+    // Extract file path
+    const { filePath } = requestData
+
+    if (!filePath) {
+      console.log('File path not provided, sending 400')
+      reply.status(400).send({ error: 'File path is required' })
+      return
     }
-  })
+
+    // Map request data to processing options
+    const { options, llmOpt, transcriptOpt } = reqToOpts(requestData)
+    console.log('Calling processFile with params:', { filePath, llmOpt, transcriptOpt, options })
+
+    await processFile(filePath, llmOpt, transcriptOpt, options)
+
+    console.log('processFile completed successfully')
+    reply.send({ message: 'File processed successfully.' })
+  } catch (error) {
+    console.error('Error processing file:', error)
+    reply.status(500).send({ error: 'An error occurred while processing the file' })
+  }
 }
 
 export { handleFileRequest }