First working version

.github/workflows/deploy_book.yml

@@ -0,0 +1,33 @@
+name: Deploy Documentation
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+jobs:
+  deploy:
+    name: Build and Deploy Documentation
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - name: Build and Deploy to Server
+        uses: appleboy/ssh-action@master
+        with:
+          host: ${{ secrets.NEW_SERVER_HOST }}
+          username: ${{ secrets.DOCS_USER }}
+          key: ${{ secrets.DOCS_PRIVATE_KEY }}
+          script: |
+            set -e
+            echo "Cleaning up previous build..."
+            rm -rf /home/${{ secrets.DOCS_USER }}/build
+            mkdir -p /home/${{ secrets.DOCS_USER }}/build/book
+
+            echo "Cloning repository and building documentation..."
+            cd /home/${{ secrets.DOCS_USER }}/build
+            git clone [email protected]:TrueBlocks/trueblocks-khedra
+            cd trueblocks-khedra/book
+            mdbook build
+
+            echo "Deploying to the book server..."
+            rsync -avz --delete book/ /var/www/khedra.trueblocks.io/html/

.github/workflows/test_and_coverage.yml

@@ -0,0 +1,55 @@
+name: Go Test and Coverage
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+jobs:
+  test:
+    name: Run Tests and Generate Coverage
+    runs-on: ubuntu-latest
+    steps:
+      # Checkout the repository
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Install Go 1.23
+        run: |
+          ARCH=$(uname -m)
+          if [ "$ARCH" = "x86_64" ]; then
+            ARCH="amd64"
+          elif [ "$ARCH" = "aarch64" ]; then
+            ARCH="arm64"
+          else
+            echo "Unsupported architecture: $ARCH"
+            exit 1
+          fi
+          curl -LO https://golang.org/dl/go1.23.1.linux-$ARCH.tar.gz
+          sudo tar -C /usr/local -xzf go1.23.1.linux-$ARCH.tar.gz
+          export PATH=$PATH:/usr/local/go/bin
+          go version
+      # Run go mod tidy
+      - name: Ensure Dependencies are Updated
+        run: go mod tidy
+      # Run tests and generate coverage
+      - name: Run Tests with Coverage
+        run: |
+          mkdir -p coverage
+          go test -coverprofile=coverage/coverage.out ./...
+          go tool cover -html=coverage/coverage.out -o coverage/coverage.html
+      # Upload coverage HTML as an artifact
+      - name: Upload Coverage Report
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-html
+          path: coverage/coverage.html
+    # Optionally, fail the job if coverage is too low (replace 75 with your minimum coverage %)
+    # - name: Check Coverage Percentage
+    #   run: |
+    #     COVERAGE=$(go tool cover -func=coverage/coverage.out | grep total | awk '{print $3}' | sed 's/%//')
+    #     echo "Coverage: $COVERAGE%"
+    #     if (( $(echo "$COVERAGE < 75" | bc -l) )); then
+    #       echo "Coverage below threshold. Failing job."
+    #       exit 1
+    #     fi

.gitignore

@@ -1,25 +1,16 @@
-# If you prefer the allow list template instead of the deny list, see community template:
-# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
-#
-# Binaries for programs and plugins
+coverage
+x
 *.exe
 *.exe~
 *.dll
 *.so
 *.dylib
-
-# Test binary, built with `go test -c`
 *.test
-
-# Output of the go coverage tool, specifically when used with LiteIDE
 *.out
-
-# Dependency directories (remove the comment below to include it)
 # vendor/
-
-# Go workspace file
 go.work
 go.work.sum
-
-# env file
 .env
+khedra
+khedra.log
+config.yaml

book/.gitignore

@@ -0,0 +1 @@
+book

book/book.toml

@@ -0,0 +1,10 @@
+[book]
+authors = ["TrueBlocks, LLC"]
+language = "en"
+multilingual = false
+src = "src"
+title = "The Khedra Book"
+
+[output.html]
+numbered = false
+default-theme = "rust"

book/makefile

@@ -0,0 +1,16 @@
+SRC_MD := $(shell find src -name '*.md')
+
+book: $(SRC_MD)
+	@echo building...
+	mdbook build
+
+serve:
+	@make clean
+	@mdbook serve --open --hostname 127.0.0.1
+
+clean:
+	@rm -fR book
+
+deploy:
+	@make clean && touch ${SRC_MD} && make book && cd book && scp -pr * "[email protected]:/var/www/khedra.trueblocks.io/html/"
+

book/src/SUMMARY.md

@@ -0,0 +1,23 @@
+# Summary
+
+- [Introduction](index.md)
+- [About the Name](about_the_name.md)
+- [User Manual](user_man/README.md)
+  - [Introduction](user_man/introduction.md)
+  - [Getting Started](user_man/getting_started.md)
+  - [Understanding Khedra](user_man/understanding_khedra.md)
+  - [Using Khedra](user_man/using_khedra.md)
+  - [Advanced Operations](user_man/advanced_operations.md)
+  - [Maintenance and Troubleshooting](user_man/maintenance_and_troubleshooting.md)
+  - [Appendices](user_man/appendices.md)
+- [Technical Spec](tech_spec/README.md)
+  - [Introduction](tech_spec/introduction.md)
+  - [System Architecture](tech_spec/system_architecture.md)
+  - [Core Functionalities](tech_spec/core_functionalities.md)
+  - [Technical Design](tech_spec/technical_design.md)
+  - [Supported Chains](tech_spec/supported_chains.md)
+  - [Command-Line Interface](tech_spec/command_line_interface.md)
+  - [Performance and Scalability](tech_spec/performance_and_scalability.md)
+  - [Integration Points](tech_spec/integration_points.md)
+  - [Testing and Validation](tech_spec/testing_and_validation.md)
+  - [Appendices](tech_spec/appendices.md)

book/src/about_the_name.md

@@ -0,0 +1,17 @@
+# About the Name
+
+The name **khedra** (pronounced *key-drah*) is inspired by the Persian word <span style="font-size: 1.5em;"> خدمت </span> (*khedmat*), meaning "service."
+
+In ancient Persian culture, service was considered a noble pursuit, emphasizing dedication, reliability, and humility in action. Drawing from this tradition, the name **khedra** embodies the essence of a system designed to serve—efficiently, continuously, and with purpose.
+
+Like its counterpart, [chifra](https://github.com/TrueBlocks/trueblocks-core) (derived from the Persian word for "cipher"), the name **khedra** connects deeply to our roots while adapting to a modern technological context. The streamlined spelling symbolizes long-running, dependable processes that tirelessly "serves" the needs of its users.
+
+More technically, **khedra** literally is a collection of services including:
+
+- an address appearance indexer, 
+- a publisher (to both IPFS and a smart contract) of the index, 
+- an address monitoring and notification tool, 
+- a RESTful API exposing `chifra`'s command line options,
+- and an IPFS daemon.
+
+By choosing the name **khedra**, we honor a legacy of service while committing to building tools that are as resilient, adaptive, and reliable as the meaning behind its name.

book/src/index.md

@@ -0,0 +1,14 @@
+# The Khedra Book
+
+**Khedra** (pronounced *key-drah*) is an all-in-one tool for indexing and monitoring EVM-compatible blockchains.
+
+It creates and shares the [Unchained Index](https://trueblocks.io/papers/2023/specification-for-the-unchained-index-v2.0.0-release.pdf)
+which is a permissionless index of "address appearances" shared via IPFS and published to a smart contract.
+
+Address appearances record every location in a blockchain's history including appearing in an event log, appearing in an execution trace, winning a staking or block reward, appearing as a prefund address or any other location.
+
+This detailed indexing allows for near-perfect monitoring and notifications of addresses, native and ERC-20 account balance auditing and accounting, and even your own custom indexing needs. It works for any address on any chain (if you have access to the chain's RPC).
+
+Enjoy.
+
+Please help us improve this software by providing any feedback or suggestions. Contact information is available [at our website](https://trueblocks.io).

book/src/tech_spec/README.md

@@ -0,0 +1,22 @@
+# Technical Specification
+
+## Purpose of this Document
+
+This document defines the technical architecture, design, and functionalities of Khedra, enabling developers and engineers to understand its internal workings and design principles. For a less technical overview of the application, refer to the [User Manual](../user_man/).
+
+## Intended Audience
+
+This specification is for:
+
+- Developers working on Khedra or integrating it into applications.
+- System architects designing systems that use Khedra.
+- Technical professionals looking for a detailed understanding of the system.
+
+## Scope and Objectives
+
+The specification covers:
+
+- High-level architecture.
+- Core functionalities such as blockchain indexing, REST API, and address monitoring.
+- Design principles, including scalability, error handling, and integration with IPFS.
+- Supported chains, RPC requirements, and testing methodologies.

book/src/tech_spec/appendices.md

@@ -0,0 +1,33 @@
+# Appendices
+
+## Glossary of Technical Terms
+
+- **EVM**: Ethereum Virtual Machine, the runtime environment for smart contracts.
+- **RPC**: Remote Procedure Call, a protocol for interacting with blockchain nodes.
+- **IPFS**: InterPlanetary File System, a decentralized storage solution.
+
+## References and Resources
+
+- [TrueBlocks GitHub Repository](https://github.com/TrueBlocks/trueblocks-khedra)
+- [TrueBlocks Official Website](https://trueblocks.io)
+- [Ethereum Developer Documentation](https://ethereum.org/en/developers/)
+- [IPFS Documentation](https://docs.ipfs.io)
+
+## Index
+
+- **Address Monitoring**: Section 3, Core Functionalities
+- **API Access**: Section 3, Core Functionalities
+- **Architecture Overview**: Section 2, System Architecture
+- **Blockchain Indexing**: Section 3, Core Functionalities
+- **Configuration Files**: Section 4, Technical Design
+- **Data Flow**: Section 4, Technical Design
+- **Error Handling**: Section 4, Technical Design
+- **Integration Points**: Section 8, Integration Points
+- **IPFS Integration**: Section 3, Core Functionalities; Section 8, Integration Points
+- **Logging**: Section 4, Technical Design
+- **Performance Benchmarks**: Section 7, Performance and Scalability
+- **REST API**: Section 3, Core Functionalities; Section 8, Integration Points
+- **RPC Requirements**: Section 5, Supported Chains and RPCs
+- **Scalability Strategies**: Section 7, Performance and Scalability
+- **System Components**: Section 2, System Architecture
+- **Testing Guidelines**: Section 9, Testing and Validation

book/src/tech_spec/command_line_interface.md

@@ -0,0 +1,41 @@
+# Command-Line Interface
+
+## Available Commands and Options
+
+### Initialization
+
+```bash
+./khedra --init all
+```
+
+- Options: `all`, `blooms`, `none`
+
+### Scraper
+
+```bash
+./khedra --scrape on
+```
+
+- Enables or disables the blockchain scraper.
+
+### REST API
+
+```bash
+./khedra --api on
+```
+
+- Starts the API server.
+
+### Sleep Duration
+
+```bash
+./khedra --sleep 60
+```
+
+- Sets the duration (in seconds) between updates.
+
+## Detailed Behavior for Each Command
+
+1. **`--init`**: Controls how the blockchain index is initialized.
+2. **`--scrape`**: Toggles the blockchain scraper.
+3. **`--api`**: Starts or stops the API server.

book/src/tech_spec/core_functionalities.md

@@ -0,0 +1,19 @@
+# Core Functionalities
+
+## Blockchain Indexing
+
+Indexes blockchain data for fast and efficient retrieval. Supports multiple chains and tracks transactions.
+
+## REST API
+
+Exposes indexed data through a REST API. Includes endpoints for:
+- Retrieving transactions and blocks.
+- Accessing monitored address data.
+
+## Address Monitoring
+
+Allows tracking of specific blockchain addresses. Captures transactions and updates in real-time.
+
+## IPFS Integration
+
+Pins portions of the Unchained Index to IPFS for decentralized and tamper-proof storage.

book/src/tech_spec/integration_points.md

@@ -0,0 +1,23 @@
+# Integration Points
+
+## Integration with External APIs
+
+Khedra exposes data through a REST API, making it compatible with external applications. Example use cases:
+
+- Fetching transaction details for a given address.
+- Retrieving block information for analysis.
+
+## Interfacing with IPFS
+
+Data indexed by Khedra can be pinned to IPFS for decentralized storage:
+
+```bash
+./khedra --ipfs on
+```
+
+## Customizing for Specific Use Cases
+
+Users can tailor the configuration by:
+
+- Adjusting `.env` variables to include specific chains and RPC endpoints.
+- Writing custom scripts to query the REST API and process the data.

book/src/tech_spec/introduction.md

@@ -0,0 +1 @@
+# Introduction

book/src/tech_spec/performance_and_scalability.md

@@ -0,0 +1,19 @@
+# Performance and Scalability
+
+## Performance Benchmarks
+
+Khedra is designed to handle high-throughput blockchain data. Typical performance benchmarks include:
+
+- Processing speed: ~500 blocks per second (depending on RPC response time).
+- REST API response time: <50ms for standard queries.
+
+## Strategies for Handling Large-Scale Data
+
+1. Use high-performance RPC endpoints with low latency.
+2. Increase local storage capacity to handle large blockchain data.
+3. Scale horizontally by running multiple instances of Khedra for different chains.
+
+## Resource Optimization Guidelines
+
+- Limit the number of chains processed simultaneously to reduce system load.
+- Configure `--sleep` duration to balance processing speed with system resource usage.

book/src/tech_spec/supported_chains.md

@@ -0,0 +1,26 @@
+# Supported Chains
+
+## List of Supported Blockchains
+
+Khedra supports Ethereum mainnet and other EVM-compatible chains like:
+
+- Sepolia
+- Gnosis
+- Optimism
+
+## Requirements for RPC Endpoints
+
+Each chain requires a valid RPC endpoint. For example:
+
+- `TB_NODE_MAINNETRPC`: Mainnet RPC URL.
+- `TB_NODE_SEPOLIARPC`: Sepolia RPC URL.
+
+## Handling Multiple Chains
+
+To enable multiple chains, set `TB_NODE_CHAINS` in the `.env` file:
+
+```env
+TB_NODE_CHAINS="mainnet,sepolia,gnosis"
+```
+
+Ensure each chain has a corresponding RPC endpoint.