Upgrade dependencies. Update github actions workflows. Code updates

Author: Gal Ben David

.github/workflows/build.yml

@@ -1,16 +1,19 @@
 name: Build
-on: [push, pull_request]
+on:
+  - push
+  - pull_request
 jobs:
   lint:
     if: github.event_name == 'push' && !startsWith(github.event.ref, 'refs/tags')
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Install latest rust
         uses: actions-rs/toolchain@v1
         with:
           toolchain: stable
+          profile: minimal
           override: true
           components: clippy
       - name: Lint with clippy
@@ -24,17 +27,24 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10']
-        os: [ubuntu-latest , macos-latest, windows-latest]
+        python-version:
+          - '3.7'
+          - '3.8'
+          - '3.9'
+          - '3.10'
+        os:
+          - ubuntu-latest
+          - macos-latest
+          - windows-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Run image
-        uses: abatilo/actions-poetry@v2.0.0
+      - name: Install Poetry
+        uses: abatilo/actions-poetry@v2.1.3
       - name: Install Rust
         uses: actions-rs/toolchain@v1
         with:

.github/workflows/deploy.yml

@@ -1,20 +1,28 @@
 name: Deploy
 on:
   release:
-    types: [released]
+    types:
+      - released
 jobs:
   deploy:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10']
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version:
+          - '3.7'
+          - '3.8'
+          - '3.9'
+          - '3.10'
+        os:
+          - ubuntu-latest
+          - macos-latest
+          - windows-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install Rust
@@ -24,22 +32,9 @@ jobs:
           toolchain: stable
           override: true
       - name: Publish Package
-        if: matrix.os != 'windows-latest'
         uses: messense/maturin-action@v1
         with:
-          maturin-version: latest
           command: publish
-          manylinux: 2014
-          args: --username=__token__ --no-sdist --interpreter=python${{ matrix.python-version }}
-        env:
-          MATURIN_PASSWORD: ${{ secrets.pypi_password }}
-      - name: Publish Package
-        if: matrix.os == 'windows-latest'
-        uses: messense/maturin-action@v1
-        with:
-          maturin-version: latest
-          command: publish
-          manylinux: 2014
-          args: --username=__token__ --no-sdist --interpreter=python
+          args: --username=__token__ --no-sdist --interpreter=python${{ !startsWith(matrix.os, 'windows') && matrix.python-version || '' }}
         env:
           MATURIN_PASSWORD: ${{ secrets.pypi_password }}

.gitignore

@@ -1,3 +1,8 @@
+
+# Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode,rust,python
+# Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode,rust,python
+
+### Python ###
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -94,6 +99,13 @@ ipython_config.py
 #   install all needed dependencies.
 #Pipfile.lock
 
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
 # PEP 582; used by e.g. github.com/David-OConnor/pyflow
 __pypackages__/
 
@@ -136,17 +148,48 @@ dmypy.json
 
 # Cython debug symbols
 cython_debug/
-.gitignore
-.gitignore
 
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+### Rust ###
 # Generated by Cargo
 # will have compiled files and executables
 debug/
-target/
 
 # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
 # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
 Cargo.lock
 
 # These are backup files generated by rustfmt
 **/*.rs.bk
+
+# MSVC Windows builds of rustc generate these, which store debugging information
+*.pdb
+
+### VisualStudioCode ###
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+!.vscode/*.code-snippets
+
+# Local History for Visual Studio Code
+.history/
+
+# Built Visual Studio Code Extensions
+*.vsix
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide
+
+# Support for Project snippet scope
+
+# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode,rust,python

Cargo.toml

@@ -35,23 +35,23 @@ name = "pyrepscan"
 crate-type = ["cdylib"]
 
 [dependencies]
-regex = "1"
-chrono = "0.4"
-num_cpus = "1"
-parking_lot = "0.12"
-crossbeam = "0.8"
-crossbeam-utils = "0.8"
+regex = "1.6.0"
+chrono = "0.4.19"
+num_cpus = "1.13.1"
+parking_lot = "0.12.1"
+crossbeam = "0.8.1"
+crossbeam-utils = "0.8.10"
 
 [dependencies.libgit2-sys]
-version = "0.12"
+version = "0.13.4"
 features = ["https"]
 
 [dependencies.git2]
-version = "0.13"
+version = "0.14.4"
 features = ["vendored-openssl"]
 
 [dependencies.pyo3]
-version = "0.15.1"
+version = "0.16.5"
 features = ["extension-module"]
 
 [profile.release]

benchmarks/gitleaks.sh

@@ -1 +1,2 @@
-~/go/bin/gitleaks --repo-path=$REPO_PATH --config=gitleaks.toml
+docker pull zricethezav/gitleaks:latest
+docker run -v ${FOLDER_TO_SCAN}:/path -v ${PWD}/benchmarks/gitleaks.toml:/gitleaks.toml zricethezav/gitleaks:latest detect --source="/path" --config=/gitleaks.toml

benchmarks/pyrepscan_bench.py

@@ -12,3 +12,4 @@
     repository_path='/path/to/repository',
     branch_glob_pattern='*',
 )
+print(len(results))

pyproject.toml

@@ -1,13 +1,14 @@
 [build-system]
-requires = ["maturin>=0.11,<0.12"]
+requires = ["maturin>=0.12,<0.13"]
 build-backend = "maturin"
 
 [tool.maturin]
 sdist-include = [
-    "src/*",
     "Cargo.toml",
+    "pyproject.toml",
     "pyrepscan/*.py",
-    "pyrepscan/*.pyi"
+    "pyrepscan/*.pyi",
+    "src/*",
 ]
 
 [tool.poetry]

src/git_repository_scanner.rs

@@ -3,9 +3,10 @@ use crate::rules_manager;
 use chrono::prelude::*;
 use crossbeam_utils::atomic::AtomicCell;
 use crossbeam_utils::thread as crossbeam_thread;
-use crossbeam::queue::SegQueue;
+use crossbeam::queue::ArrayQueue;
 use git2::{Oid, Repository, Delta};
 use parking_lot::Mutex;
+use pyo3::exceptions::PyRuntimeError;
 use pyo3::prelude::*;
 use std::collections::HashMap;
 use std::path::Path;
@@ -124,18 +125,7 @@ fn scan_commit_oid(
     Ok(())
 }
 
-pub fn get_file_content(
-    repository_path: &str,
-    file_oid: &str,
-) -> Result<Vec<u8>, git2::Error> {
-    let git_repo = Repository::open(repository_path)?;
-    let oid = Oid::from_str(file_oid)?;
-    let blob = git_repo.find_blob(oid)?;
-
-    Ok(blob.content().to_vec())
-}
-
-fn get_oids(
+fn get_commit_oids(
     repository_path: &str,
     branch_glob_pattern: &str,
     from_timestamp: i64,
@@ -166,40 +156,46 @@ pub fn scan_repository(
     from_timestamp: i64,
     rules_manager: &rules_manager::RulesManager,
     output_matches: Arc<Mutex<Vec<HashMap<&str, String>>>>,
-) -> Result<(), PyErr> {
-    let oids_queue = Arc::new(SegQueue::new());
-    match get_oids(
+) -> PyResult<()> {
+    let commit_oids_queue;
+
+    match get_commit_oids(
         repository_path,
         branch_glob_pattern,
         from_timestamp
     ) {
-        Ok(oids) => {
-            for oid in oids {
-                oids_queue.push(oid);
+        Ok(commit_oids) => {
+            if commit_oids.is_empty() {
+               return Ok(());
+            }
+
+            commit_oids_queue = ArrayQueue::new(commit_oids.len());
+            for commit_oid in commit_oids {
+                commit_oids_queue.push(commit_oid).unwrap();
             }
         },
         Err(error) => {
-            return Err(pyo3::exceptions::PyRuntimeError::new_err(error.to_string()))
+            return Err(PyRuntimeError::new_err(error.to_string()))
         },
     }
 
-    py.check_signals()?;
-
     let mut py_signal_error: PyResult<()> = Ok(());
 
     let should_stop  = AtomicCell::new(false);
+    let number_of_cores = std::thread::available_parallelism().unwrap().get();
+
     crossbeam_thread::scope(
         |scope| {
-            for _ in 0..num_cpus::get() {
+            for _ in 0..number_of_cores {
                 scope.spawn(
                     |_| {
                         if let Ok(git_repo) = Repository::open(repository_path) {
                             while !should_stop.load() {
-                                if let Some(oid) = oids_queue.pop() {
+                                if let Some(commit_oid) = commit_oids_queue.pop() {
                                     scan_commit_oid(
                                         &should_stop,
                                         &git_repo,
-                                        &oid,
+                                        &commit_oid,
                                         rules_manager,
                                         output_matches.clone(),
                                     ).unwrap_or(());
@@ -212,7 +208,7 @@ pub fn scan_repository(
                 );
             }
 
-            while !oids_queue.is_empty() {
+            while !commit_oids_queue.is_empty() {
                 py_signal_error = py.check_signals();
                 if py_signal_error.is_err() {
                     should_stop.store(true);
@@ -223,7 +219,7 @@ pub fn scan_repository(
                 thread::sleep(time::Duration::from_millis(100));
             }
         }
-    ).unwrap_or(());
+    ).unwrap_or_default();
 
     py_signal_error?;