Skip to content

Commit 729d3f7

Browse files
ajcwebdevajcwebdev
committed
refactor whisper and ollama
1 parent f0faece commit 729d3f7

21 files changed

+279
-347
lines changed

‎package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@
5252
"test-services": "tsx --test test/services.test.ts",
5353
"test-all": "tsx --test test/all.test.ts",
5454
"ta": "tsx --test test/all.test.ts",
55-
"clean": "tsx scripts/cleanContent.ts",
55+
"clean": "npm run tsx:base scripts/cleanContent.ts",
5656
"docker-cli": "docker run --rm --env-file .env -v $PWD/content:/usr/src/app/content autoshow",
5757
"docker-serve": "docker run -d -p 3000:3000 -v $PWD/content:/usr/src/app/content autoshow serve",
5858
"prune": "docker system prune -af --volumes && docker image prune -af && docker container prune -f && docker volume prune -af",

‎src/llms/chatgpt.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
import { env } from 'node:process'
44
import { OpenAI } from 'openai'
5-
import { GPT_MODELS } from '../utils/llm-models'
5+
import { GPT_MODELS } from '../utils/llm-globals'
66
import { err, logAPIResults } from '../utils/logging'
77
import type { LLMFunction, ChatGPTModelType } from '../types/llms'
88

‎src/llms/claude.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
import { env } from 'node:process'
44
import { Anthropic } from '@anthropic-ai/sdk'
5-
import { CLAUDE_MODELS } from '../utils/llm-models'
5+
import { CLAUDE_MODELS } from '../utils/llm-globals'
66
import { err, logAPIResults } from '../utils/logging'
77
import type { LLMFunction, ClaudeModelType } from '../types/llms'
88

‎src/llms/cohere.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
import { env } from 'node:process'
44
import { CohereClient } from 'cohere-ai'
5-
import { COHERE_MODELS } from '../utils/llm-models'
5+
import { COHERE_MODELS } from '../utils/llm-globals'
66
import { err, logAPIResults } from '../utils/logging'
77
import type { LLMFunction, CohereModelType } from '../types/llms'
88

‎src/llms/fireworks.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
// src/llms/fireworks.ts
22

33
import { env } from 'node:process'
4-
import { FIREWORKS_MODELS } from '../utils/llm-models'
4+
import { FIREWORKS_MODELS } from '../utils/llm-globals'
55
import { err, logAPIResults } from '../utils/logging'
66
import type { LLMFunction, FireworksModelType, FireworksResponse } from '../types/llms'
77

‎src/llms/gemini.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
import { env } from 'node:process'
44
import { GoogleGenerativeAI } from "@google/generative-ai"
5-
import { GEMINI_MODELS } from '../utils/llm-models'
5+
import { GEMINI_MODELS } from '../utils/llm-globals'
66
import { err, logAPIResults } from '../utils/logging'
77
import type { LLMFunction, GeminiModelType } from '../types/llms'
88

‎src/llms/groq.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
// src/llms/groq.ts
22

33
import { env } from 'node:process'
4-
import { GROQ_MODELS } from '../utils/llm-models'
4+
import { GROQ_MODELS } from '../utils/llm-globals'
55
import { err, logAPIResults } from '../utils/logging'
66
import type { LLMFunction, GroqModelType, GroqChatCompletionResponse } from '../types/llms'
77

‎src/llms/mistral.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
import { env } from 'node:process'
44
import { Mistral } from '@mistralai/mistralai'
5-
import { MISTRAL_MODELS } from '../utils/llm-models'
5+
import { MISTRAL_MODELS } from '../utils/llm-globals'
66
import { err, logAPIResults } from '../utils/logging'
77
import type { LLMFunction, MistralModelType } from '../types/llms'
88

‎src/llms/ollama.ts

+18-144
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
// src/llms/ollama.ts
22

33
import { env } from 'node:process'
4-
import { spawn } from 'node:child_process'
5-
import { OLLAMA_MODELS } from '../utils/llm-models'
4+
import { OLLAMA_MODELS } from '../utils/llm-globals'
65
import { l, err, logAPIResults } from '../utils/logging'
7-
import type { LLMFunction, OllamaModelType, OllamaResponse, OllamaTagsResponse } from '../types/llms'
6+
import { checkServerAndModel } from '../utils/validate-option'
7+
import type { LLMFunction, OllamaModelType, OllamaResponse } from '../types/llms'
88

99
/**
1010
* callOllama()
@@ -38,94 +38,7 @@ export const callOllama: LLMFunction = async (
3838

3939
const combinedPrompt = `${prompt}\n${transcript}`
4040

41-
async function checkServer(): Promise<boolean> {
42-
try {
43-
const serverResponse = await fetch(`http://${ollamaHost}:${ollamaPort}`)
44-
return serverResponse.ok
45-
} catch (error) {
46-
return false
47-
}
48-
}
49-
50-
if (await checkServer()) {
51-
l.wait('\n Ollama server is already running...')
52-
} else {
53-
if (ollamaHost === 'ollama') {
54-
throw new Error('Ollama server is not running. Please ensure the Ollama server is running and accessible.')
55-
} else {
56-
l.wait('\n Ollama server is not running. Attempting to start...')
57-
const ollamaProcess = spawn('ollama', ['serve'], {
58-
detached: true,
59-
stdio: 'ignore',
60-
})
61-
ollamaProcess.unref()
62-
63-
let attempts = 0
64-
while (attempts < 30) {
65-
if (await checkServer()) {
66-
l.wait(' - Ollama server is now ready.')
67-
break
68-
}
69-
await new Promise((resolve) => setTimeout(resolve, 1000))
70-
attempts++
71-
}
72-
if (attempts === 30) {
73-
throw new Error('Ollama server failed to become ready in time.')
74-
}
75-
}
76-
}
77-
78-
l.wait(`\n Checking if model is available: ${ollamaModelName}`)
79-
try {
80-
const tagsResponse = await fetch(`http://${ollamaHost}:${ollamaPort}/api/tags`)
81-
if (!tagsResponse.ok) {
82-
throw new Error(`HTTP error! status: ${tagsResponse.status}`)
83-
}
84-
const tagsData = (await tagsResponse.json()) as OllamaTagsResponse
85-
const isModelAvailable = tagsData.models.some((m) => m.name === ollamaModelName)
86-
87-
if (!isModelAvailable) {
88-
l.wait(`\n Model ${ollamaModelName} is not available, pulling...`)
89-
const pullResponse = await fetch(`http://${ollamaHost}:${ollamaPort}/api/pull`, {
90-
method: 'POST',
91-
headers: { 'Content-Type': 'application/json' },
92-
body: JSON.stringify({ name: ollamaModelName }),
93-
})
94-
if (!pullResponse.ok) {
95-
throw new Error(`Failed to initiate pull for model ${ollamaModelName}`)
96-
}
97-
if (!pullResponse.body) {
98-
throw new Error('Response body is null')
99-
}
100-
101-
const reader = pullResponse.body.getReader()
102-
const decoder = new TextDecoder()
103-
while (true) {
104-
const { done, value } = await reader.read()
105-
if (done) break
106-
107-
const chunk = decoder.decode(value)
108-
const lines = chunk.split('\n')
109-
for (const line of lines) {
110-
if (line.trim() === '') continue
111-
try {
112-
const parsedLine = JSON.parse(line)
113-
if (parsedLine.status === 'success') {
114-
l.wait(` - Model ${ollamaModelName} pulled successfully.\n`)
115-
break
116-
}
117-
} catch (parseError) {
118-
err(`Error parsing JSON while pulling model: ${parseError}`)
119-
}
120-
}
121-
}
122-
} else {
123-
l.wait(`\n Model ${ollamaModelName} is already available.\n`)
124-
}
125-
} catch (error) {
126-
err(`Error checking/pulling model: ${(error as Error).message}`)
127-
throw error
128-
}
41+
await checkServerAndModel(ollamaHost, ollamaPort, ollamaModelName)
12942

13043
l.wait(` - Sending chat request to http://${ollamaHost}:${ollamaPort} using model '${ollamaModelName}'`)
13144

@@ -135,68 +48,29 @@ export const callOllama: LLMFunction = async (
13548
body: JSON.stringify({
13649
model: ollamaModelName,
13750
messages: [{ role: 'user', content: combinedPrompt }],
138-
stream: true,
51+
stream: false,
13952
}),
14053
})
14154

14255
if (!response.ok) {
14356
throw new Error(`HTTP error! status: ${response.status}`)
14457
}
145-
if (!response.body) {
146-
throw new Error('Response body is null')
147-
}
148-
149-
l.wait('\n Successfully connected to Ollama /api/chat streaming endpoint.')
150-
const reader = response.body.getReader()
151-
const decoder = new TextDecoder()
152-
let fullContent = ''
153-
let isFirstChunk = true
154-
let totalPromptTokens = 0
155-
let totalCompletionTokens = 0
156-
157-
while (true) {
158-
const { done, value } = await reader.read()
159-
if (done) break
16058

161-
const chunk = decoder.decode(value)
162-
const lines = chunk.split('\n')
59+
const data = await response.json() as OllamaResponse
60+
const fullContent = data?.message?.content || ''
16361

164-
for (const line of lines) {
165-
if (line.trim() === '') continue
62+
const totalPromptTokens = data.prompt_eval_count ?? 0
63+
const totalCompletionTokens = data.eval_count ?? 0
16664

167-
try {
168-
const parsedResponse = JSON.parse(line) as OllamaResponse
169-
if (parsedResponse.message?.content) {
170-
if (isFirstChunk) {
171-
l.wait(` - Streaming response from Ollama (first chunk received)`)
172-
isFirstChunk = false
173-
}
174-
fullContent += parsedResponse.message.content
175-
}
176-
177-
if (parsedResponse.prompt_eval_count) {
178-
totalPromptTokens = parsedResponse.prompt_eval_count
179-
}
180-
if (parsedResponse.eval_count) {
181-
totalCompletionTokens = parsedResponse.eval_count
182-
}
183-
184-
if (parsedResponse.done) {
185-
logAPIResults({
186-
modelName: modelKey,
187-
stopReason: 'stop',
188-
tokenUsage: {
189-
input: totalPromptTokens || undefined,
190-
output: totalCompletionTokens || undefined,
191-
total: totalPromptTokens + totalCompletionTokens || undefined,
192-
},
193-
})
194-
}
195-
} catch (parseError) {
196-
err(`Error parsing JSON from Ollama response: ${parseError}`)
197-
}
198-
}
199-
}
65+
logAPIResults({
66+
modelName: modelKey,
67+
stopReason: 'stop',
68+
tokenUsage: {
69+
input: totalPromptTokens || undefined,
70+
output: totalCompletionTokens || undefined,
71+
total: totalPromptTokens + totalCompletionTokens || undefined,
72+
},
73+
})
20074

20175
return fullContent
20276
} catch (error) {

‎src/llms/together.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
// src/llms/together.ts
22

33
import { env } from 'node:process'
4-
import { TOGETHER_MODELS } from '../utils/llm-models'
4+
import { TOGETHER_MODELS } from '../utils/llm-globals'
55
import { err, logAPIResults } from '../utils/logging'
66
import type { LLMFunction, TogetherModelType, TogetherResponse } from '../types/llms'
77

‎src/process-commands/file.ts

+3-19
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ import { runLLM } from '../process-steps/05-run-llm'
1212
import { cleanUpFiles } from '../process-steps/06-clean-up-files'
1313
import { l, err } from '../utils/logging'
1414
import { readFile } from 'fs/promises'
15-
import { insertShowNote } from '../server/db'
1615
import type { ProcessingOptions } from '../types/process'
1716
import type { TranscriptServices } from '../types/transcription'
1817
import type { LLMServices } from '../types/llms'
@@ -23,8 +22,7 @@ import type { LLMServices } from '../types/llms'
2322
* 2. Converts the file to the required audio format
2423
* 3. Transcribes the audio content
2524
* 4. Processes the transcript with a language model (if specified)
26-
* 5. Saves the show notes into the database
27-
* 6. Cleans up temporary files (unless disabled)
25+
* 5. Cleans up temporary files (unless disabled)
2826
*
2927
* Unlike processVideo, this function handles local files and doesn't need
3028
* to check for external dependencies like yt-dlp.
@@ -87,24 +85,10 @@ export async function processFile(
8785
options,
8886
finalPath,
8987
frontMatter,
90-
llmServices,
91-
generatedPrompt,
92-
transcript
93-
)
94-
95-
// Insert into DB
96-
insertShowNote(
97-
metadata.showLink ?? '',
98-
metadata.channel ?? '',
99-
metadata.channelURL ?? '',
100-
metadata.title,
101-
metadata.description ?? '',
102-
metadata.publishDate,
103-
metadata.coverImage ?? '',
104-
frontMatter,
10588
generatedPrompt,
10689
transcript,
107-
llmOutput
90+
metadata,
91+
llmServices
10892
)
10993

11094
// Step 6 - Cleanup

‎src/process-commands/rss.ts

+3-17
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ import { cleanUpFiles } from '../process-steps/06-clean-up-files'
1414
import { validateRSSOptions } from '../utils/validate-option'
1515
import { l, err, logRSSProcessingAction, logRSSProcessingStatus, logRSSSeparator } from '../utils/logging'
1616
import { parser } from '../utils/globals'
17-
import { insertShowNote } from '../server/db'
1817
import type { ProcessingOptions, RSSItem } from '../types/process'
1918
import type { TranscriptServices } from '../types/transcription'
2019
import type { LLMServices } from '../types/llms'
@@ -198,27 +197,14 @@ async function processItem(
198197
generatedPrompt = promptText
199198
}
200199

201-
const llmOutput = await runLLM(
200+
await runLLM(
202201
options,
203202
finalPath,
204203
frontMatter,
205-
llmServices,
206-
generatedPrompt,
207-
transcript
208-
)
209-
210-
insertShowNote(
211-
metadata.showLink ?? '',
212-
metadata.channel ?? '',
213-
metadata.channelURL ?? '',
214-
metadata.title,
215-
metadata.description ?? '',
216-
metadata.publishDate,
217-
metadata.coverImage ?? '',
218-
frontMatter,
219204
generatedPrompt,
220205
transcript,
221-
llmOutput
206+
metadata,
207+
llmServices
222208
)
223209

224210
if (!options.noCleanUp) {

0 commit comments

Comments
 (0)