New API for parsing + Python bindings

Author: Keats

.github/workflows/tests.yml

@@ -109,4 +109,48 @@ jobs:
         run: |
           cargo +nightly install cargo-fuzz
           cargo +nightly fuzz run parse_fasta -- -max_total_time=180
-          cargo +nightly fuzz run parse_fastq -- -max_total_time=180
+          cargo +nightly fuzz run parse_fastq -- -max_total_time=180
+
+  python-bindings:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@master
+
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly
+          override: true
+
+      - name: version info
+        run:  rustc --version; cargo --version;
+
+      - name: Run all tests
+        run: cargo test --features=python_test
+
+      - name: Install python headers
+        run: sudo apt-get update && sudo apt-get install python3-dev python3-pip python3-venv
+
+      - name: Install maturin
+        run: |
+          python3 -m venv venv
+          . venv/bin/activate
+          pip3 install maturin
+          pip3 show maturin
+
+
+      - name: add library to venv
+        run: |
+          . venv/bin/activate
+          maturin develop --cargo-extra-args="--features=python"
+
+      - name: Run the python tests
+        run: |
+          . venv/bin/activate
+          python test_python.py
+
+      - name: compile taxonomy with python bindings
+        run: |
+          . venv/bin/activate
+          maturin build --cargo-extra-args="--features=python" --release --strip --manylinux=off

.gitignore

@@ -3,3 +3,4 @@ Cargo.lock
 venv/
 .DS_Store
 .idea/
+test.py

CHANGELOG.md

@@ -6,6 +6,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.4.0] - TODO
+
+## Changed
+- Added `parse_fastx_file` which replaces `parse_sequence_reader` and offers an iterator like usage and is faster
+than 0.3
+- `SequenceRecord` now offers more information about the file such as line ending, which allows writing a file identical
+to the input one.
+
+## Removed
+- Gzip files piped in through `stdin` is not supported anymore since we do require `Seek`
+
+
 ## [0.3.0] - 2019-09-12
 ### Added
 - Improved error reporting (i.e., a parse failure now gives the record it failed on).

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "needletail"
-version = "0.3.2"
+version = "0.4.0"
 authors = ["Roderick Bovee <roderick@onecodex.com>"]
 description = "FASTX parsing and k-mer methods"
 keywords = ["FASTA", "FASTQ", "kmer", "bioinformatics"]
@@ -11,16 +11,24 @@ readme = "./README.md"
 edition = "2018"
 include = ["src/**/*", "LICENSE", "README.md", "CHANGELOG.md"]
 
+[lib]
+crate-type=["cdylib", "rlib"]
+bench = false
+
 [features]
 default = ["compression"]
 compression = ["bzip2", "flate2", "xz2"]
+python = ["pyo3/extension-module"]
+python_test = ["pyo3"]
 
 [dependencies]
 flate2 = { version="1.0.6", optional=true }
 bzip2 = { version="0.3.3", optional=true }
 xz2 = { version="0.1.6", optional=true }
+pyo3 = { version = "0.10", optional = true }
 memchr = "2.2.1"
-safemem = "0.3.2"
+bytecount = {version = "0.6", features = ["runtime-dispatch-simd"]}
+buf_redux = { version = "0.8", default_features = false }
 
 [dev-dependencies]
 criterion = "0.3"
@@ -34,6 +42,9 @@ toml = "0.5"
 serde = "1.0"
 serde_derive = "1.0"
 
+[profile.release]
+lto = true
+
 [[bench]]
 name = "benchmark"
 harness = false

README.md

@@ -8,61 +8,53 @@ Needletail is a MIT-licensed, minimal-copying FASTA/FASTQ parser and _k_-mer pro
 The goal is to write a fast *and* well-tested set of functions that more specialized bioinformatics programs can use.
 Needletail's goal is to be as fast as the [readfq](https://github.com/lh3/readfq) C library at parsing FASTX files and much (i.e. 25 times) faster than equivalent Python implementations at _k_-mer counting.
 
-# Example
+## Example
 
 ```rust
-extern crate needletail;
-use needletail::{parse_sequence_path, Sequence};
-use std::env;
-
-fn main() {
-    let filename: String = env::args().nth(1).unwrap();
-
-    let mut n_bases = 0;
-    let mut n_valid_kmers = 0;
-    parse_sequence_path(
-	filename,
-        |_| {},
-        |seq| {
-            // seq.id is the name of the record
-            // seq.seq is the base sequence
-            // seq.qual is an optional quality score
-
-            // keep track of the total number of bases
-            n_bases += seq.seq.len();
-
-            // normalize to make sure all the bases are consistantly capitalized
-            let norm_seq = seq.normalize(false);
-            // we make a reverse complemented copy of the sequence first for
-            // `canonical_kmers` to draw the complemented sequences from.
-            let rc = norm_seq.reverse_complement();
-            // now we keep track of the number of AAAAs (or TTTTs via
-            // canonicalization) in the file; note we also get the postion (i.0;
-            // in the event there were `N`-containing kmers that were skipped)
-            // and whether the sequence was complemented (i.2) in addition to
-            // the canonical kmer (i.1)
-            for (_, kmer, _) in norm_seq.canonical_kmers(4, &rc) {
-                if kmer == b"AAAA" {
-                    n_valid_kmers += 1;
-                }
-            }
-        },
-    )
-    .expect("parsing failed");
-    println!("There are {} bases in your file.", n_bases);
-    println!("There are {} AAAAs in your file.", n_valid_kmers);
-}
+//! extern crate needletail;
+//! use needletail::{parse_fastx_file, Sequence, FastxReader};
+//!
+//! fn main() {
+//!     let filename = "tests/data/28S.fasta";
+//!
+//!     let mut n_bases = 0;
+//!     let mut n_valid_kmers = 0;
+//!     let mut reader = parse_fastx_file(&filename).expect("valid path/file");
+//!     while let Some(record) = reader.next() {
+//!         let seqrec = record.expect("invalid record");
+//!         // keep track of the total number of bases
+//!         n_bases += seqrec.num_bases();
+//!         // normalize to make sure all the bases are consistently capitalized and
+//!         // that we remove the newlines since this is FASTA
+//!         let norm_seq = seqrec.normalize(false);
+//!         // we make a reverse complemented copy of the sequence first for
+//!         // `canonical_kmers` to draw the complemented sequences from.
+//!         let rc = norm_seq.reverse_complement();
+//!         // now we keep track of the number of AAAAs (or TTTTs via
+//!         // canonicalization) in the file; note we also get the position (i.0;
+//!         // in the event there were `N`-containing kmers that were skipped)
+//!         // and whether the sequence was complemented (i.2) in addition to
+//!         // the canonical kmer (i.1)
+//!         for (_, kmer, _) in norm_seq.canonical_kmers(4, &rc) {
+//!             if kmer == b"AAAA" {
+//!                 n_valid_kmers += 1;
+//!             }
+//!         }
+//!     }
+//!     println!("There are {} bases in your file.", n_bases);
+//!     println!("There are {} AAAAs in your file.", n_valid_kmers);
+//! }
 ```
 
-# Installation
+## Installation
 
 Needletail requires `rust` and `cargo` to be installed.
 Please use either your local package manager (`homebrew`, `apt-get`, `pacman`, etc) or install these via [rustup](https://www.rustup.rs/).
 
 Once you have Rust set up, you can include needletail in your `Cargo.toml` file like:
 ```shell
 [dependencies]
-needletail = "^0.3.1"
+needletail = "0.4"
 ```
 
 To install needletail itself for development:
@@ -71,10 +63,25 @@ git clone https://github.com/onecodex/needletail
 cargo test  # to run tests
 ```
 
-# Getting Help
+### Python
+To work on the Python library on a Mac OS X/Unix system (requires Python 3):
+```bash
+# you need the nightly version of Rust installed
+curl https://sh.rustup.rs -sSf | sh
+rustup default nightly
+
+# finally, install the library in the local virtualenv
+maturin develop --cargo-extra-args="--features=python"
+```
+
+## Getting Help
 
 Questions are best directed as GitHub issues. We plan to add more documentation soon, but in the meantime "doc" comments are included in the source.
 
-# Contributing
+## Contributing
 
 Please do! We're happy to discuss possible additions and/or accept pull requests.
+
+## Acknowledgements
+Starting from 0.4, the parsers algorithms is taken from [seq_io](https://github.com/markschl/seq_io). While it has been slightly modified, it is mainly
+coming that library. Links to the original files are available in `src/parser/fast{a,q}.rs`.

benches/benchmark.rs

@@ -3,8 +3,8 @@ extern crate criterion;
 extern crate needletail;
 
 use criterion::Criterion;
-use needletail::parse_sequence_reader;
-use needletail::sequence::Sequence;
+use needletail::parser::FastxReader;
+use needletail::Sequence;
 use std::fs::File;
 use std::io::{Cursor, Read};
 
@@ -22,48 +22,47 @@ fn bench_kmer_speed(c: &mut Criterion) {
     group.sample_size(10);
 
     group.bench_function("Kmer", |b| {
+        use needletail::parser::FastaReader;
         b.iter(|| {
             let mut n_total = 0;
             let mut n_canonical = 0;
             let fasta_data = Cursor::new(data.clone());
-            parse_sequence_reader(
-                fasta_data,
-                |_| {},
-                |rec| {
-                    let seq = rec.seq.normalize(true);
-                    let rc = seq.reverse_complement();
-                    for (_, _kmer, was_rc) in seq.canonical_kmers(ksize, &rc) {
-                        if !was_rc {
-                            n_canonical += 1;
-                        }
-                        n_total += 1;
+            let mut reader = FastaReader::new(fasta_data);
+
+            while let Some(record) = reader.next() {
+                let rec = record.unwrap();
+                let seq = rec.normalize(true);
+                let rc = seq.reverse_complement();
+                for (_, _kmer, was_rc) in seq.canonical_kmers(ksize, &rc) {
+                    if !was_rc {
+                        n_canonical += 1;
                     }
-                },
-            )
-            .unwrap();
+                    n_total += 1;
+                }
+            }
             assert_eq!(718_007, n_total);
             assert_eq!(350_983, n_canonical);
         });
     });
 
     group.bench_function("Bitkmer", |bench| {
+        use needletail::parser::FastaReader;
         bench.iter(|| {
             let mut n_total = 0;
             let mut n_canonical = 0;
             let fasta_data = Cursor::new(data.clone());
-            parse_sequence_reader(
-                fasta_data,
-                |_| {},
-                |seq| {
-                    for (_, _kmer, was_rc) in seq.bit_kmers(ksize, true) {
-                        if !was_rc {
-                            n_canonical += 1;
-                        }
-                        n_total += 1;
+            let mut reader = FastaReader::new(fasta_data);
+            while let Some(record) = reader.next() {
+                let rec = record.unwrap();
+                let seq = rec.strip_returns();
+                for (_, _kmer, was_rc) in seq.bit_kmers(ksize, true) {
+                    if !was_rc {
+                        n_canonical += 1;
                     }
-                },
-            )
-            .unwrap();
+                    n_total += 1;
+                }
+            }
+
             assert_eq!(718_007, n_total);
             assert_eq!(350_983, n_canonical);
         });
@@ -111,28 +110,14 @@ fn bench_fastq_file(c: &mut Criterion) {
     });
 
     group.bench_function("Needletail", |bench| {
+        use needletail::parser::FastqReader;
         bench.iter(|| {
             let fastq_data = Cursor::new(data.clone());
             let mut n_bases = 0;
-            parse_sequence_reader(
-                fastq_data,
-                |_| {},
-                |seq| {
-                    n_bases += seq.seq.len();
-                },
-            )
-            .unwrap();
-            assert_eq!(250_000, n_bases);
-        });
-    });
-
-    group.bench_function("Needletail (No Buffer)", |bench| {
-        use needletail::formats::{FastqParser, RecParser};
-        bench.iter(|| {
-            let mut reader = FastqParser::from_buffer(&data, true);
-            let mut n_bases = 0;
-            for seq in reader.by_ref() {
-                n_bases += seq.unwrap().seq.len();
+            let mut reader = FastqReader::new(fastq_data);
+            while let Some(record) = reader.next() {
+                let rec = record.unwrap();
+                n_bases += rec.seq().len();
             }
             assert_eq!(250_000, n_bases);
         });
@@ -178,28 +163,14 @@ fn bench_fasta_file(c: &mut Criterion) {
     });
 
     group.bench_function("Needletail", |bench| {
+        use needletail::parser::FastaReader;
         bench.iter(|| {
             let fasta_data = Cursor::new(data.clone());
+            let mut reader = FastaReader::new(fasta_data);
             let mut n_bases = 0;
-            parse_sequence_reader(
-                fasta_data,
-                |_| {},
-                |seq| {
-                    n_bases += seq.seq.len();
-                },
-            )
-            .unwrap();
-            assert_eq!(738_580, n_bases);
-        });
-    });
-
-    group.bench_function("Needletail (No Buffer)", |bench| {
-        use needletail::formats::{FastaParser, RecParser};
-        bench.iter(|| {
-            let mut reader = FastaParser::from_buffer(&data, true);
-            let mut n_bases = 0;
-            for rec in reader.by_ref() {
-                n_bases += rec.unwrap().seq.strip_returns().len();
+            while let Some(result) = reader.next() {
+                let record = result.unwrap();
+                n_bases += record.num_bases();
             }
             assert_eq!(738_580, n_bases);
         });