add_last_commit.sh

#!/bin/bash
last_commit_hash=""
last_commit_time=""

# Generate a list of commit hashes and associated file names
git log --full-history -n 1 --reverse --name-only --pretty=format:"Commit: %H" -- "coinbase/24h_stats/*.json" | \
awk '/^Commit:/ {commit=$2} !/^Commit:/ && NF {print commit, $0}' > commit_file_list.txt

# Define a function to process each commit and file
process_commit_file() {
    read -r commit file <<<"$1"

    # Fetch the commit time (no need to check for last commit as each job is independent)
    commit_time=$(git show -s --date=format:'%Y-%m-%dT%H:%M:%SZ' --format='%ad')

    # Use git show to display the content of each file at the point of the commit
    if git ls-tree -r --name-only "$commit" | grep -q "^$file$"; then
        git show "${commit}:${file}" | jq -c --arg hash "$commit" --arg commit_time "${commit_time}" '. + {commit_time: $commit_time, h: $hash}' >> "coinbase/jsonl/$(basename "$file")l"
    fi
}

export -f process_commit_file

# Use GNU Parallel to process each commit and file in parallel
cat commit_file_list.txt | parallel -j 30 process_commit_file