feat(docs): automate collection and organization of docs (#560)

* init

* fixes

* more fixes

* add docs for docs

* more

* more

* fix ci

* move g-ignore to docs/

* update

.circleci/config.yml

@@ -26,16 +26,16 @@ jobs:
       - run:
           name: Download Zig
           command: |
-            apt update && apt install -y wget unzip xz-utils 
+            apt update && apt install -y wget unzip xz-utils
             wget https://ziglang.org/download/0.13.0/zig-<< parameters.target >>-0.13.0.tar.xz
             tar xf zig-<< parameters.target >>-0.13.0.tar.xz
             mkdir -p workspace
             mv zig-<< parameters.target >>-0.13.0 workspace/zig
       - persist_to_workspace:
           root: workspace
           paths:
-            - '*'
-          
+            - "*"
+
   lint:
     executor: linux-executor
     steps:
@@ -45,7 +45,7 @@ jobs:
       - run:
           name: Lint
           command: workspace/zig/zig fmt --check src/ build.zig
-  
+
   check_style:
     executor: python-executor
     steps:
@@ -56,7 +56,18 @@ jobs:
       - run:
           name: Check style
           command: python scripts/style.py --check src
-
+
+  check_docs:
+    executor: python-executor
+    steps:
+      - checkout
+      - run:
+          name: Install dependencies
+          command: pip install --upgrade pip
+      - run:
+          name: Check docs
+          command: python docs/check.py ./
+
   build_linux:
     executor: linux-executor
     steps:
@@ -128,7 +139,7 @@ jobs:
           paths:
             - .zig-cache
             - ~/.cache/zig
-  
+
   build_and_test_hashmap:
     executor: linux-executor
     steps:
@@ -186,7 +197,7 @@ jobs:
       - run:
           name: Run Gossip
           command: bash scripts/gossip_test.sh 120 workspace/zig-out-release/bin/sig
-    
+
   gossip_service_fuzz:
     executor: linux-executor
     steps:
@@ -196,7 +207,7 @@ jobs:
       - run:
           name: Run Gossip Service Fuzzer
           command: workspace/zig-out-release/bin/fuzz gossip_service 19 10000
-  
+
   gossip_table_fuzz:
     executor: linux-executor
     steps:
@@ -206,7 +217,7 @@ jobs:
       - run:
           name: Run Gossip Service Fuzzer
           command: workspace/zig-out-release/bin/fuzz gossip_table 19 100000
-    
+
   allocators_fuzz:
     executor: linux-executor
     steps:
@@ -227,16 +238,16 @@ jobs:
           name: Run Gossip Service Fuzzer
           command: workspace/zig-out-release/bin/fuzz ledger 19 10000
 
-
 workflows:
   check_linux:
     jobs:
       - check_style
+      - check_docs
       - setup_zig:
           name: setup_zig_linux
           target: "linux-x86_64"
       - lint:
-          requires: 
+          requires:
             - setup_zig_linux
       - build_linux_release:
           requires:
@@ -277,4 +288,3 @@ workflows:
       - build_and_test_macos:
           requires:
             - setup_zig_macos
-

.gitignore

@@ -17,22 +17,4 @@
 .DS_Store
 .vscode/
 
-# Dependencies
-docs/node_modules
-
-# Production
-docs/build
-
-# Generated files
-docs/.docusaurus
-docs/.cache-loader
-
-# Misc
-docs/.env.local
-docs/.env.development.local
-docs/.env.test.local
-docs/.env.production.local
-
-docs/npm-debug.log*
-docs/yarn-debug.log*
-docs/yarn-error.log*
+# docs

docs/.gitignore

@@ -0,0 +1,21 @@
+# dependencies
+docusaurus/node_modules
+
+# production
+docusaurus/build
+
+# generated files
+docusaurus/.docusaurus
+docusaurus/.cache-loader
+
+# misc
+docusaurus/.env.local
+docusaurus/.env.development.local
+docusaurus/.env.test.local
+docusaurus/.env.production.local
+docusaurus/npm-debug.log*
+docusaurus/yarn-debug.log*
+docusaurus/yarn-error.log*
+
+# python scripts
+__pycache__

docs/check.py

@@ -0,0 +1,40 @@
+import argparse
+import os
+
+import generate as g
+
+# checks if the docs folder is up to date with the source readme.md files
+if __name__ == "__main__":
+    arg_parser = argparse.ArgumentParser()
+    arg_parser.add_argument("src_dir")
+    args = arg_parser.parse_args()
+
+    exclude_dirs = [
+        args.src_dir + "docs", # dont search yourself
+        args.src_dir + "data", # this should only include data
+    ]
+
+    code_path = os.path.join(args.src_dir, "docs/docusaurus/docs/code")
+    for name, src_path in g.get_markdown_files(args.src_dir, exclude_dirs):
+        docs_path = os.path.join(code_path, name + ".md")
+
+        # check to see if the files are the same !
+        with open(src_path, "r") as src_f:
+            with open(docs_path, "r") as docs_f:
+                src_lines = src_f.readlines()
+                # fix image paths for docusaurus
+                for i in range(0, len(src_lines)):
+                    if "/docs/docusaurus/static/img" in src_lines[i]:
+                        src_lines[i] = src_lines[i].replace("/docs/docusaurus/static/img", "/img")
+
+                docs_lines = docs_f.readlines()
+                if src_lines != docs_lines:
+                    print("Docs folder is not up to date, run generate.py")
+                    # print the the first difference
+                    for i in range(len(src_lines)):
+                        if src_lines[i] != docs_lines[i]:
+                            print("Difference found at line", i)
+                            print("Source:", src_lines[i])
+                            print("Docs:", docs_lines[i])
+                            break
+                    exit(1)