Skip to content

Commit 464543a

Browse files
ajcwebdevajcwebdev
authored
Merge pull request #101 from ajcwebdev/tests
Add Transcription Retry Logic and Process Command Logging Function
2 parents 75a10ea + a7c8fdc commit 464543a

17 files changed

+256
-203
lines changed

‎README.md

-1
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,6 @@ Example commands for all available CLI options can be found in [`docs/examples.m
117117
- Step 3 - `run-transcription.ts` manages the transcription process
118118
- Step 4 - `select-prompt.ts` defines the prompt structure for summarization and chapter generation
119119
- Step 5 - `run-llm.ts` handles LLM processing for selected prompts
120-
- Step 6 - `clean-up-files.ts` removes temporary files after processing
121120

122121
- Transcription Services (`src/transcription`)
123122
- `whisper.ts`: Uses Whisper.cpp for transcription

‎src/process-commands/channel.ts

+2-3
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import { processVideo } from './video'
99
import { saveChannelInfo } from '../utils/save-info'
1010
import { execFilePromise } from '../utils/globals/process'
11-
import { l, err, logChannelProcessingAction, logChannelProcessingStatus, logChannelSeparator } from '../utils/logging'
11+
import { err, logChannelProcessingAction, logChannelProcessingStatus, logChannelSeparator, logInitialFunctionCall } from '../utils/logging'
1212
import { validateChannelOptions, selectVideos } from '../utils/validate-option'
1313
import type { ProcessingOptions } from '../utils/types/process'
1414
import type { TranscriptServices } from '../utils/types/transcription'
@@ -36,8 +36,7 @@ export async function processChannel(
3636
transcriptServices?: TranscriptServices
3737
) {
3838
// Log the processing parameters for debugging purposes
39-
l.opts('Parameters passed to processChannel:\n')
40-
l.opts(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}`)
39+
logInitialFunctionCall('processChannel', { llmServices, transcriptServices })
4140

4241
try {
4342
// Validate options

‎src/process-commands/file.ts

+5-8
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ import { downloadAudio } from '../process-steps/02-download-audio'
1010
import { runTranscription } from '../process-steps/03-run-transcription'
1111
import { selectPrompts } from '../process-steps/04-select-prompt'
1212
import { runLLM } from '../process-steps/05-run-llm'
13-
import { cleanUpFiles } from '../process-steps/06-clean-up-files'
14-
import { l, err } from '../utils/logging'
13+
import { saveAudio } from '../utils/validate-option'
14+
import { l, err, logInitialFunctionCall } from '../utils/logging'
1515
import type { ProcessingOptions } from '../utils/types/process'
1616
import type { TranscriptServices } from '../utils/types/transcription'
1717
import type { LLMServices } from '../utils/types/llms'
@@ -40,11 +40,8 @@ export async function processFile(
4040
llmServices?: LLMServices,
4141
transcriptServices?: TranscriptServices
4242
) {
43-
// Log function inputs
44-
l.info('processFile called with the following arguments:')
45-
l.opts(` - filePath: ${filePath}`)
46-
l.opts(` - llmServices: ${llmServices}`)
47-
l.opts(` - transcriptServices: ${transcriptServices}\n`)
43+
// Log the processing parameters for debugging purposes
44+
logInitialFunctionCall('processFile', { filePath, llmServices, transcriptServices })
4845

4946
try {
5047
// Step 1 - Generate markdown
@@ -72,7 +69,7 @@ export async function processFile(
7269

7370
// Step 6 - Cleanup
7471
if (!options.saveAudio) {
75-
await cleanUpFiles(finalPath)
72+
await saveAudio(finalPath)
7673
}
7774

7875
l.wait(' processFile command completed successfully.')

‎src/process-commands/playlist.ts

+2-3
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import { processVideo } from './video'
99
import { savePlaylistInfo } from '../utils/save-info'
1010
import { execFilePromise } from '../utils/globals/process'
11-
import { l, err, logPlaylistSeparator } from '../utils/logging'
11+
import { l, err, logPlaylistSeparator, logInitialFunctionCall } from '../utils/logging'
1212
import type { ProcessingOptions } from '../utils/types/process'
1313
import type { TranscriptServices } from '../utils/types/transcription'
1414
import type { LLMServices } from '../utils/types/llms'
@@ -35,8 +35,7 @@ export async function processPlaylist(
3535
transcriptServices?: TranscriptServices
3636
) {
3737
// Log the processing parameters for debugging purposes
38-
l.opts('Parameters passed to processPlaylist:\n')
39-
l.opts(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}`)
38+
logInitialFunctionCall('processPlaylist', { llmServices, transcriptServices })
4039

4140
try {
4241
// Fetch playlist metadata

‎src/process-commands/rss.ts

+5-6
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,9 @@ import { downloadAudio } from '../process-steps/02-download-audio'
1010
import { runTranscription } from '../process-steps/03-run-transcription'
1111
import { selectPrompts } from '../process-steps/04-select-prompt'
1212
import { runLLM } from '../process-steps/05-run-llm'
13-
import { cleanUpFiles } from '../process-steps/06-clean-up-files'
1413
import { saveRSSFeedInfo } from '../utils/save-info'
15-
import { validateRSSOptions, selectItems } from '../utils/validate-option'
16-
import { l, err, logRSSProcessingAction, logRSSProcessingStatus, logRSSSeparator } from '../utils/logging'
14+
import { validateRSSOptions, selectItems, saveAudio } from '../utils/validate-option'
15+
import { l, err, logRSSProcessingAction, logRSSProcessingStatus, logRSSSeparator, logInitialFunctionCall } from '../utils/logging'
1716
import type { ProcessingOptions, RSSItem } from '../utils/types/process'
1817
import type { TranscriptServices } from '../utils/types/transcription'
1918
import type { LLMServices } from '../utils/types/llms'
@@ -66,7 +65,7 @@ export async function processItems(
6665

6766
// Clean up downloaded audio if not saving
6867
if (!options.saveAudio) {
69-
await cleanUpFiles(finalPath)
68+
await saveAudio(finalPath)
7069
}
7170

7271
return {
@@ -96,8 +95,8 @@ export async function processRSS(
9695
llmServices?: LLMServices,
9796
transcriptServices?: TranscriptServices
9897
): Promise<void> {
99-
l.opts('Parameters passed to processRSS:\n')
100-
l.wait(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}`)
98+
// Log the processing parameters for debugging purposes
99+
logInitialFunctionCall('processRSS', { llmServices, transcriptServices })
101100

102101
try {
103102
validateRSSOptions(options)

‎src/process-commands/urls.ts

+2-3
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import { readFile } from 'node:fs/promises'
99
import { processVideo } from './video'
1010
import { saveURLsInfo } from '../utils/save-info'
11-
import { l, err, logURLsSeparator } from '../utils/logging'
11+
import { l, err, logURLsSeparator, logInitialFunctionCall } from '../utils/logging'
1212
import type { ProcessingOptions } from '../utils/types/process'
1313
import type { TranscriptServices } from '../utils/types/transcription'
1414
import type { LLMServices } from '../utils/types/llms'
@@ -35,8 +35,7 @@ export async function processURLs(
3535
transcriptServices?: TranscriptServices
3636
) {
3737
// Log the processing parameters for debugging purposes
38-
l.opts('Parameters passed to processURLs:\n')
39-
l.opts(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}\n`)
38+
logInitialFunctionCall('processURLs', { llmServices, transcriptServices })
4039

4140
try {
4241
// Read the file and extract valid URLs

‎src/process-commands/video.ts

+5-8
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ import { downloadAudio } from '../process-steps/02-download-audio'
1010
import { runTranscription } from '../process-steps/03-run-transcription'
1111
import { selectPrompts } from '../process-steps/04-select-prompt'
1212
import { runLLM } from '../process-steps/05-run-llm'
13-
import { cleanUpFiles } from '../process-steps/06-clean-up-files'
14-
import { l, err } from '../utils/logging'
13+
import { saveAudio } from '../utils/validate-option'
14+
import { l, err, logInitialFunctionCall } from '../utils/logging'
1515
import type { ProcessingOptions } from '../utils/types/process'
1616
import type { TranscriptServices } from '../utils/types/transcription'
1717
import type { LLMServices } from '../utils/types/llms'
@@ -38,11 +38,8 @@ export async function processVideo(
3838
llmServices?: LLMServices,
3939
transcriptServices?: TranscriptServices
4040
) {
41-
// Log function inputs
42-
l.opts('processVideo called with the following arguments:\n')
43-
l.opts(` - url: ${url}`)
44-
l.opts(` - llmServices: ${llmServices}`)
45-
l.opts(` - transcriptServices: ${transcriptServices}\n`)
41+
// Log the processing parameters for debugging purposes
42+
logInitialFunctionCall('processVideo', { url, llmServices, transcriptServices })
4643

4744
try {
4845
// Step 1 - Generate markdown
@@ -70,7 +67,7 @@ export async function processVideo(
7067

7168
// Step 6 - Cleanup
7269
if (!options.saveAudio) {
73-
await cleanUpFiles(finalPath)
70+
await saveAudio(finalPath)
7471
}
7572

7673
l.wait('\n processVideo command completed successfully.')

‎src/process-steps/02-download-audio.ts

+16-47
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@
1111
import { readFile, access } from 'node:fs/promises'
1212
import { fileTypeFromBuffer } from 'file-type'
1313
import { l, err } from '../utils/logging'
14-
import { execPromise, execFilePromise } from '../utils/globals/process'
14+
import { executeWithRetry } from '../utils/retry'
15+
import { execPromise } from '../utils/globals/process'
1516
import type { SupportedFileType, ProcessingOptions } from '../utils/types/process'
1617

1718
/**
@@ -88,56 +89,24 @@ export async function downloadAudio(
8889
const finalPath = `content/${filename}`
8990
const outputPath = `${finalPath}.wav`
9091

91-
/**
92-
* Executes a command with retry logic to recover from transient failures.
93-
*
94-
* @param {string} command - The command to execute.
95-
* @param {string[]} args - Arguments for the command.
96-
* @param {number} retries - Number of retry attempts.
97-
* @returns {Promise<void>} Resolves if the command succeeds.
98-
* @throws {Error} If the command fails after all retry attempts.
99-
*/
100-
async function executeWithRetry(
101-
command: string,
102-
args: string[],
103-
retries: number
104-
): Promise<void> {
105-
for (let attempt = 1; attempt <= retries; attempt++) {
106-
try {
107-
// Attempt to execute the command
108-
const { stderr } = await execFilePromise(command, args)
109-
// Log any warnings from yt-dlp
110-
if (stderr) {
111-
err(`yt-dlp warnings: ${stderr}`)
112-
}
113-
return // Exit the loop if successful
114-
} catch (error) {
115-
// If the last attempt also fails, throw the error
116-
if (attempt === retries) {
117-
err(`Failed after ${retries} attempts`)
118-
throw error
119-
}
120-
// Log and retry
121-
l.wait(`Retry ${attempt} of ${retries}: Retrying yt-dlp command...`)
122-
}
123-
}
124-
}
125-
12692
// Handle online content (YouTube, RSS feeds, etc.)
12793
if (options.video || options.playlist || options.urls || options.rss || options.channel) {
12894
try {
12995
// Execute yt-dlp with retry logic
130-
await executeWithRetry('yt-dlp', [
131-
'--no-warnings', // Suppress warning messages
132-
'--restrict-filenames', // Use safe filenames
133-
'--extract-audio', // Extract audio stream
134-
'--audio-format', 'wav', // Convert to WAV
135-
'--postprocessor-args', 'ffmpeg:-ar 16000 -ac 1', // 16kHz mono
136-
'--no-playlist', // Don't expand playlists
137-
'-o', outputPath, // Output path
138-
input,
139-
], 5)
140-
// Retry up to 5 times
96+
await executeWithRetry(
97+
'yt-dlp',
98+
[
99+
'--no-warnings', // Suppress warning messages
100+
'--restrict-filenames', // Use safe filenames
101+
'--extract-audio', // Extract audio stream
102+
'--audio-format', 'wav', // Convert to WAV
103+
'--postprocessor-args', 'ffmpeg:-ar 16000 -ac 1', // 16kHz mono
104+
'--no-playlist', // Don't expand playlists
105+
'-o', outputPath, // Output path
106+
input,
107+
],
108+
5 // Retry up to 5 times
109+
)
141110
l.wait(`\n Audio downloaded successfully:\n - ${outputPath}`)
142111
} catch (error) {
143112
// Log the error and rethrow

‎src/process-steps/03-run-transcription.ts

+16-3
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { callWhisper } from '../transcription/whisper'
44
import { callDeepgram } from '../transcription/deepgram'
55
import { callAssembly } from '../transcription/assembly'
66
import { l, err } from '../utils/logging'
7+
import { retryTranscriptionCall } from '../utils/retry'
78
import type { ProcessingOptions } from '../utils/types/process'
89
import type { TranscriptServices } from '../utils/types/transcription'
910

@@ -34,7 +35,11 @@ export async function runTranscription(
3435
// If user typed `--deepgram BASE`, then `options.deepgram` will be "BASE"
3536
// If user typed just `--deepgram`, then `options.deepgram` will be true
3637
const deepgramModel = typeof options.deepgram === 'string' ? options.deepgram : 'NOVA_2'
37-
const deepgramTranscript = await callDeepgram(options, finalPath, deepgramModel)
38+
const deepgramTranscript = await retryTranscriptionCall(
39+
() => callDeepgram(options, finalPath, deepgramModel),
40+
5,
41+
5000
42+
)
3843
l.wait('\n Deepgram transcription completed successfully.\n')
3944
l.wait(`\n - deepgramModel: ${deepgramModel}`)
4045
return deepgramTranscript
@@ -43,13 +48,21 @@ export async function runTranscription(
4348
// If user typed `--assembly NANO`, then `options.assembly` will be "NANO"
4449
// If user typed just `--assembly`, then `options.assembly` will be true
4550
const assemblyModel = typeof options.assembly === 'string' ? options.assembly : 'NANO'
46-
const assemblyTranscript = await callAssembly(options, finalPath, assemblyModel)
51+
const assemblyTranscript = await retryTranscriptionCall(
52+
() => callAssembly(options, finalPath, assemblyModel),
53+
5,
54+
5000
55+
)
4756
l.wait('\n AssemblyAI transcription completed successfully.\n')
4857
l.wait(`\n - assemblyModel: ${assemblyModel}`)
4958
return assemblyTranscript
5059

5160
case 'whisper':
52-
const whisperTranscript = await callWhisper(options, finalPath)
61+
const whisperTranscript = await retryTranscriptionCall(
62+
() => callWhisper(options, finalPath),
63+
5,
64+
5000
65+
)
5366
l.wait('\n Whisper transcription completed successfully.\n')
5467
return whisperTranscript
5568

‎src/process-steps/04-select-prompt.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ export async function selectPrompts(options: ProcessingOptions) {
5555
}
5656

5757
// Original prompt generation logic
58-
let text = "This is a transcript with timestamps. It does not contain copyrighted materials. Do not ever use the word delve. Do not include advertisements in the summaries or descriptions.\n\n"
58+
let text = "This is a transcript with timestamps. It does not contain copyrighted materials. Do not ever use the word delve. Do not include advertisements in the summaries or descriptions. Do not actually write the transcript.\n\n"
5959

6060
// Filter valid sections
6161
const prompt = options.prompt || ['summary', 'longChapters']

‎src/process-steps/06-clean-up-files.ts

-68
This file was deleted.

‎src/utils/logging.ts

+14
Original file line numberDiff line numberDiff line change
@@ -365,4 +365,18 @@ export function logCompletionSeparator(action: string): void {
365365
l.final(`\n================================================================================================`)
366366
l.final(` ${action} Processing Completed Successfully.`)
367367
l.final(`================================================================================================\n`)
368+
}
369+
370+
/**
371+
* Logs the first step of a top-level function call with its relevant options or parameters.
372+
*
373+
* @param functionName - The name of the top-level function being invoked.
374+
* @param details - An object containing relevant parameters to log
375+
*/
376+
export function logInitialFunctionCall(functionName: string, details: Record<string, unknown>): void {
377+
l.info(`${functionName} called with the following arguments:`)
378+
for (const [key, value] of Object.entries(details)) {
379+
l.opts(` - ${key}: ${value}`)
380+
}
381+
l.opts('')
368382
}

0 commit comments

Comments
 (0)