Script to parse mail.log and produce portable/usable xml log files (1 per send attempt)

[cloudcogsio]

Monitors mail.log for changes using inotifywait and produces an xml log file per postfix send attempt.
Parses only log lines with a 'status=' string.
Output xml files can then be further consumed by other systems.

Disclaimer: Written over a few iterations with AI assistance as I'm not a bash programmer.
Not extensively tested, but works thus far.

Sample output.

<log>
    <timestamp>2024-09-09T21:30:50.439850+00:00</timestamp>
    <service>postfix/smtp</service>
    <process_id>10908</process_id>
    <message_id>5A9103713</message_id>
    <recipient>[email protected]</recipient>
    <relay>gmail-smtp-in.l.google.com[xxx.xxx.xxx.xxx]:25</relay>
    <delay>1.1</delay>
    <sub_delays>
        <connection_delay>0.32</connection_delay>
        <send_delay>0</send_delay>
        <response_delay>0.28</response_delay>
        <queue_delay>0.53</queue_delay>
    </sub_delays>
    <dsn>2.0.0</dsn>
    <status>sent</status>
    <smtp_message><![CDATA[250 2.0.0 OK  1725917450 ada2fe7eead31-49c16c9b863si100787137.151 - gsmtp]]></smtp_message>
</log>

Requires: inotify-tools

apt-get install inotify-tools

You can set to run at boot using crontab

@reboot /YOUR_BASE_PATH/parse.sh > /YOUR_BASE_PATH/parse.log 2>&1 &

parse.sh

#!/usr/bin/env bash

# Define paths and log file locations
BASE_PATH="/YOUR_BASE_PATH"
LOG_FILE="/PATH_TO/mail.log"
STATE_FILE="$BASE_PATH/parse.state"
OUTPUT_DIR="$BASE_PATH/xml"
LOG_PROCESSING_LOG="$BASE_PATH/parse.log"

# Function to log messages
log_message() {
    echo "$(date +'%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_PROCESSING_LOG"
}

# Function to process a log entry and convert it to XML
process_log_entry() {
    local log_line="$1"

    # Parse fields
    timestamp=$(echo "$log_line" | awk '{print $1}')
    service=$(echo "$log_line" | awk -F'[][]' '{print $1}' | awk '{print $3}' | sed 's/://')
    process_id=$(echo "$log_line" | awk -F'[][]' '{print $2}') 
    message_id=$(echo "$log_line" | grep -oP 'postfix/\w+\[\d+\]: \K[0-9A-F]+') 
    recipient=$(echo "$log_line" | sed -n 's/.*to=<\([^>]*\)>.*/\1/p')
    relay=$(echo "$log_line" | sed -n 's/.*relay=\([^,]*\).*/\1/p')
    delay=$(echo "$log_line" | sed -n 's/.*delay=\([^,]*\).*/\1/p') 
    delays=$(echo "$log_line" | sed -n 's/.*delays=\([^,]*\).*/\1/p') 
    dsn=$(echo "$log_line" | sed -n 's/.*dsn=\([^,]*\).*/\1/p')
    status=$(echo "$log_line" | sed -n 's/.*status=\([^ ]*\).*/\1/p') 
    smtp_message=$(echo "$log_line" | sed -n 's/.*(\(.*\))/\1/p') 

    # Split delay values
    connection_delay=$(echo "$delays" | cut -d '/' -f 1)
    send_delay=$(echo "$delays" | cut -d '/' -f 2)
    response_delay=$(echo "$delays" | cut -d '/' -f 3)
    queue_delay=$(echo "$delays" | cut -d '/' -f 4)

    # Generate XML content
    xml_content=$(cat <<EOF
<log>
    <timestamp>$timestamp</timestamp>
    <service>$service</service>
    <process_id>$process_id</process_id>
    <message_id>$message_id</message_id>
    <recipient>$recipient</recipient>
    <relay>$relay</relay>
    <delay>$delay</delay>
    <sub_delays>
        <connection_delay>$connection_delay</connection_delay>
        <send_delay>$send_delay</send_delay>
        <response_delay>$response_delay</response_delay>
        <queue_delay>$queue_delay</queue_delay>
    </sub_delays>
    <dsn>$dsn</dsn>
    <status>$status</status>
    <smtp_message><![CDATA[$smtp_message]]></smtp_message>
</log>
EOF
)

    # Save XML to a file
    xml_file="$OUTPUT_DIR/${message_id}.$delay.xml"
    echo "$xml_content" > "$xml_file"
    log_message "Processed log entry for $message_id"
}

# Main function to process new log entries
process_new_entries() {
    # Get the last processed byte offset, default to 0 if the state file doesn't exist
    if [ -f "$STATE_FILE" ]; then
        last_offset=$(cat "$STATE_FILE")
    else
        last_offset=0
    fi

    # Get the current size of the log file
    current_offset=$(stat --format="%s" "$LOG_FILE")

    # If the file has grown, process new entries
    if [ "$current_offset" -gt "$last_offset" ]; then
        # Read new lines from the log file starting from the last offset and buffer them for batch processing
        tail --bytes=+$((last_offset + 1)) "$LOG_FILE" | while read -r log_line; do
            # Filter lines that contain "status=" and exclude certain postfix services
            if echo "$log_line" | grep -q "status=" && echo "$log_line" | egrep -ve 'postfix/(cleanup|pickup|master|qmgr|smtpd|local|pipe)'; then
                # Process the log entry if it meets the condition
                process_log_entry "$log_line"
            fi
        done

        # Update the state file with the new offset
        echo "$current_offset" > "$STATE_FILE"
    fi
}

# Trap signals to cleanly handle script termination
trap "log_message 'Script interrupted.'; exit" SIGINT SIGTERM

# Monitor the log file for changes using inotifywait
log_message "Starting log processing"
inotifywait -m -e modify "$LOG_FILE" |
while read -r path action file; do
    process_new_entries
done