Add support for unicode 16.0.0.

CHANGES.md

@@ -1,4 +1,5 @@
 # dev
+- Add support for unicode `16.0.0`
 - Add API for retrieving start and stop positions separately (#155)
 
 # 3.2 (2023-06-28):

README.md

@@ -241,3 +241,6 @@ The `examples/` subdirectory contains several samples of sedlex in use.
   - improvements to the build system
   - switched parts of ppx_sedlex to using concrete syntax (with ppx_metaquot)
 - Steffen Smolka: port to dune
+- Romain Beauxis:
+  - Implementation of the unicode table extractors
+  - General maintenance

examples/dune

@@ -1,6 +1,6 @@
 (executables
  (names tokenizer regressions complement subtraction repeat performance)
- (libraries sedlex sedlex_ppx)
+ (libraries sedlex sedlex_ppx sedlex_utils)
  (preprocess
   (pps sedlex.ppx))
  (flags :standard -w +39))

examples/regressions.ml

@@ -3,7 +3,7 @@
 module CSet = Sedlex_ppx.Sedlex_cset
 module Unicode = Sedlex_ppx.Unicode
 
-let test_versions = ("14.0.0", "15.0.0")
+let test_versions = ("15.0.0", "16.0.0")
 
 let regressions =
   [ (* Example *)
@@ -38,6 +38,7 @@ let compare name (old_ : CSet.t) (new_ : CSet.t) =
 let test new_l (name, old_l) =
   (* Cn is for unassigned code points, which are allowed to be
    * used in future version. *)
+  let old_l = CSet.to_list old_l in
   if name <> "cn" then (
     let old_l =
       List.fold_left

src/common/cset.ml

@@ -39,6 +39,7 @@ let of_list l =
   check_invariant l;
   l
 
+let to_list l = l
 let max_code = 0x10ffff (* must be < max_int *)
 let min_code = -1
 let empty = []

src/common/cset.mli

@@ -10,6 +10,7 @@
 type t = private (int * int) list
 
 val of_list : (int * int) list -> t
+val to_list : t -> (int * int) list
 val min_code : int
 val max_code : int
 val empty : t

src/generator/data/base_url

@@ -1 +1 @@
-https://www.unicode.org/Public/15.0.0
+https://www.unicode.org/Public/16.0.0

src/generator/gen_unicode.ml

@@ -14,6 +14,7 @@ let parse_line l =
     match String.index_opt l '#' with None -> l | Some i -> String.sub l 0 i
   in
   String.split_on_char ';' l
+  |> List.map String.trim
 
 let parse_code s =
   try int_of_string (Printf.sprintf "0x%s" s)
@@ -46,7 +47,7 @@ let print_elements ch hashtbl cats =
           (fun (b, e) -> Printf.sprintf "0x%x, 0x%x" b e)
           (Cset.union_list (Hashtbl.find_all hashtbl c) :> (int * int) list)
       in
-      Printf.fprintf ch "  let %s = Sedlex_cset.of_list\n    [" c;
+      Printf.fprintf ch "  let %s = Sedlex_utils.Cset.of_list\n    [" c;
       List.iteri
         (fun i x ->
           if i > 0 then
@@ -91,7 +92,11 @@ let files =
               let interval = parse_interval interval in
               let prop = parse_prop prop in
               Hashtbl.add labels prop interval
-          | _ -> assert false );
+          | [_interval; ("InCB"); ("Extend"|"Consonant"|"Linker")] ->
+             (* TODO: support non-binary properties? *)
+             ()
+          | _ ->
+             assert false);
     ( "DerivedGeneralCategory.txt",
       fun s ->
         match parse_line s with