Skip to content

Commit

Permalink
update tokenizations version
Browse files Browse the repository at this point in the history
  • Loading branch information
tamuhey committed Apr 1, 2021
1 parent c597280 commit 122d766
Show file tree
Hide file tree
Showing 2 changed files with 28 additions and 29 deletions.
9 changes: 5 additions & 4 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,10 @@ documentation = "https://docs.rs/textspan"

[dependencies]

tokenizations = "0.4.1"
tokenizations = "0.4.2"

[dev-dependencies]
quickcheck = "0.9"
quickcheck_macros = "0.9"
proptest = "0.10"
quickcheck = "1.0"
quickcheck_macros = "1.0"
proptest = "1.0"
rstest = "0.7.0"
48 changes: 23 additions & 25 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,7 @@ mod tests {
use proptest::collection as pc;
use proptest::prelude::*;
use proptest::strategy::Strategy;
use rstest::*;
#[quickcheck]
fn test_lift_spans_index(spans: Vec<Span>, target_spans: Vec<Span>) {
let sanitize = |spans: Vec<Span>| {
Expand Down Expand Up @@ -478,46 +479,43 @@ mod tests {
spans == output
}

#[test]
fn get_original_spans_handmade() {
let testcases = vec![
(
#[rstest(input, expected,
case(
(vec!["fあo①が", "bar"], "fあo1かbar"),
(vec![vec![(0, 5)], vec![(5, 8)]]),
vec![vec![(0, 5)], vec![(5, 8)]],
),
((vec!["New York"], "NewYork"), (vec![vec![(0, 7)]])),
(
case((vec!["New York"], "NewYork"), vec![vec![(0, 7)]]),
case(
(vec!["A'B", "", ""], "A B"),
(vec![vec![(0, 1), (2, 3)], vec![], vec![]]),
vec![vec![(0, 1), (2, 3)], vec![], vec![]],
),
(
case(
(vec!["A'b", ""], "a b"),
(vec![vec![(0, 1), (2, 3)], vec![]]),
vec![vec![(0, 1), (2, 3)], vec![]],
),
((vec!["", "", ""], ""), (vec![vec![], vec![], vec![]])),
(
case((vec!["", "", ""], ""), vec![vec![], vec![], vec![]]),
case(
(vec!["hello", "``world``"], "Hello \"world\""),
vec![vec![(0, 5)], vec![(7, 12)]],
),
(
case(
(vec!["à", " ", "", "la", "gorge", ""], "a lagorge"),
(vec![
vec![
vec![(0, 1)],
vec![(1, 2)],
vec![(5, 6)],
vec![],
vec![(6, 8)],
vec![(8, 13)],
vec![],
]),
],
),
];
for (input, expected) in testcases.into_iter() {
assert_eq!(
get_original_spans(&input.0, input.1),
expected,
"{:?}",
input
);
}
)]
fn hm_get_original_spans(input: (Vec<&str>, &str), expected: Vec<Vec<(usize, usize)>>) {
assert_eq!(
get_original_spans(&input.0, input.1),
expected,
"{:?}",
input
);
}
}

0 comments on commit 122d766

Please sign in to comment.