From f6bb92fdb6f7a5a59cc770b10b841bd6b6dc8695 Mon Sep 17 00:00:00 2001 From: Simon Cruanes Date: Wed, 10 Jun 2026 15:41:31 -0400 Subject: [PATCH 01/13] check for empty needle in CCString.Find --- src/core/CCString.ml | 2 ++ src/core/CCString.mli | 13 ++++++++++++- tests/core/t_string.ml | 4 ++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/core/CCString.ml b/src/core/CCString.ml index 048e06f1c..76b22e3b1 100644 --- a/src/core/CCString.ml +++ b/src/core/CCString.ml @@ -181,12 +181,14 @@ module Find = struct | P_KMP p -> kmp_pattern_length p let compile sub : [ `Direct ] pattern = + if length sub = 0 then invalid_arg "CCString.Find.compile: empty pattern"; if length sub = 1 then P_char sub.[0] else P_KMP (kmp_compile sub) let rcompile sub : [ `Reverse ] pattern = + if length sub = 0 then invalid_arg "CCString.Find.rcompile: empty pattern"; if length sub = 1 then P_char sub.[0] else diff --git a/src/core/CCString.mli b/src/core/CCString.mli index daffcbdf5..a1e7fd1d2 100644 --- a/src/core/CCString.mli +++ b/src/core/CCString.mli @@ -104,27 +104,32 @@ val to_array : string -> char array val find : ?start:int -> sub:string -> string -> int (** [find ~start ~sub s] returns the starting index of the first occurrence of [sub] within [s] or [-1]. - @param start starting position in [s]. *) + @param start starting position in [s]. + @raise Invalid_argument if [sub = ""]. *) val find_all : ?start:int -> sub:string -> string -> int gen (** [find_all ~start ~sub s] finds all occurrences of [sub] in [s], even overlapping instances and returns them in a generator [gen]. @param start starting position in [s]. + @raise Invalid_argument if [sub = ""]. @since 0.17 *) val find_all_l : ?start:int -> sub:string -> string -> int list (** [find_all_l ~sub s] finds all occurrences of [sub] in [s] and returns them in a list. @param start starting position in [s]. + @raise Invalid_argument if [sub = ""]. @since 0.17 *) val mem : ?start:int -> sub:string -> string -> bool (** [mem ~start ~sub s] is [true] iff [sub] is a substring of [s]. + @raise Invalid_argument if [sub = ""]. @since 0.12 *) val rfind : sub:string -> string -> int (** [rfind ~sub s] finds [sub] in string [s] from the right, returns its first index or [-1]. Should only be used with very small [sub]. + @raise Invalid_argument if [sub = ""]. @since 0.12 *) val replace : @@ -353,7 +358,12 @@ module Find : sig type _ pattern val compile : string -> [ `Direct ] pattern + (** [compile sub] compiles [sub] into a pattern usable with {!find}. + @raise Invalid_argument if [sub = ""]. *) + val rcompile : string -> [ `Reverse ] pattern + (** [rcompile sub] compiles [sub] into a pattern usable with {!rfind}. + @raise Invalid_argument if [sub = ""]. *) val find : ?start:int -> pattern:[ `Direct ] pattern -> string -> int (** [find ~start ~pattern s] searches for [pattern] in the string [s], left-to-right. @@ -465,6 +475,7 @@ end val split : by:string -> string -> string list (** [split ~by s] splits the string [s] along the given string [by]. Alias to {!Split.list_cpy}. + @raise Invalid_argument if [by = ""]. @since 1.2 *) (** {2 Utils} *) diff --git a/tests/core/t_string.ml b/tests/core/t_string.ml index 9cf08e270..b850dbc6f 100644 --- a/tests/core/t_string.ml +++ b/tests/core/t_string.ml @@ -20,6 +20,8 @@ eq' 6 (find ~start:5 ~sub:"a" "a1a234a");; q ~count:10_000 Q.(pair string_printable string_printable) (fun (s1, s2) -> + s2 = "" + || let i = find ~sub:s2 s1 in i < 0 || String.sub s1 i (length s2) = s2) @@ -47,6 +49,8 @@ eq' 6 (rfind ~sub:"a" "a1a234a");; q ~count:10_000 Q.(pair string_printable string_printable) (fun (s1, s2) -> + s2 = "" + || let i = rfind ~sub:s2 s1 in i < 0 || String.sub s1 i (length s2) = s2) ;; From cc3a7d729a1b41f7ccd2aa5e1c305d38cc8a5ece Mon Sep 17 00:00:00 2001 From: Simon Cruanes Date: Fri, 12 Jun 2026 15:04:25 -0400 Subject: [PATCH 02/13] more tests for strings --- tests/core/t_string.ml | 68 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/tests/core/t_string.ml b/tests/core/t_string.ml index b850dbc6f..92207e896 100644 --- a/tests/core/t_string.ml +++ b/tests/core/t_string.ml @@ -53,6 +53,74 @@ q ~count:10_000 || let i = rfind ~sub:s2 s1 in i < 0 || String.sub s1 i (length s2) = s2) + +(* Compare the KMP-based search against a naive reference, over a 2-char + alphabet so that needles actually occur in the haystack. This is a + two-sided check: it catches false negatives, wrong (non-leftmost/rightmost) + positions, and missed overlapping matches — none of which the properties + above (one-sided, large alphabet) can see. Needles can be longer than the + haystack (exercising the short-circuit) and repetitive (exercising the + failure table). *) +let gen_ab ~maxlen = + Q.Gen.( + string_size (int_range 0 maxlen) + ~gen:(fun st -> if Random.State.bool st then 'a' else 'b')) + +let arb_haystack = Q.make ~print:Q.Print.string (gen_ab ~maxlen:20) +let arb_needle = Q.make ~print:Q.Print.string (gen_ab ~maxlen:6) + +module Find_ = struct + let naive_find ~sub s = + let n = String.length sub and ls = String.length s in + let rec loop i = + if i + n > ls then + -1 + else if String.equal (String.sub s i n) sub then + i + else + loop (i + 1) + in + loop 0 + + let naive_rfind ~sub s = + let n = String.length sub and ls = String.length s in + let rec loop i = + if i < 0 then + -1 + else if String.equal (String.sub s i n) sub then + i + else + loop (i - 1) + in + loop (ls - n) + + let naive_find_all ~sub s = + let n = String.length sub and ls = String.length s in + let rec loop i acc = + if i + n > ls then + List.rev acc + else if String.equal (String.sub s i n) sub then + loop (i + 1) (i :: acc) + else + loop (i + 1) acc + in + loop 0 [] +end +;; + +q ~count:10_000 ~name:"naive1" + Q.(pair arb_haystack arb_needle) + (fun (s, sub) -> Q.assume (sub <> ""); find ~sub s = Find_.naive_find ~sub s) +;; + +q ~count:10_000 ~name:"naive2" + Q.(pair arb_haystack arb_needle) + (fun (s, sub) -> Q.assume (sub <> ""); rfind ~sub s = Find_.naive_rfind ~sub s) +;; + +q ~count:10_000 ~name:"naive3" + Q.(pair arb_haystack arb_needle) + (fun (s, sub) -> Q.assume (sub <> ""); find_all_l ~sub s = Find_.naive_find_all ~sub s) ;; eq ~printer:CCFun.id From b147fe9b0a00d8e973b71bf8e4b7d95ecae171f2 Mon Sep 17 00:00:00 2001 From: Simon Cruanes Date: Fri, 12 Jun 2026 15:04:33 -0400 Subject: [PATCH 03/13] KMP benchs --- benchs/run_benchs.ml | 166 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) diff --git a/benchs/run_benchs.ml b/benchs/run_benchs.ml index dddf908ed..b20fa08ec 100644 --- a/benchs/run_benchs.ml +++ b/benchs/run_benchs.ml @@ -1636,6 +1636,159 @@ module Str = struct assert (CCList.equal CCInt.equal (mk_naive ()) (mk_current ())); B.throughputN 3 ~repeat [ "naive", mk_naive, (); "current", mk_current, () ] + (* Inlined reference implementations to have a fixpoint to compae + CCString with. + + - [Classic_kmp] is a copy of CCString's old KMP: the + basic ("weak") failure table and the window-shift search loop. + - [New_kmp] is a bit optimized. *) + module Classic_kmp = struct + type t = { + failure: int array; + pat: string; + } + + let compile pat = + let len = String.length pat in + match len with + | 0 -> { failure = [||]; pat } + | 1 -> { failure = [| -1 |]; pat } + | _ -> + let failure = Array.make len 0 in + failure.(0) <- -1; + let i = ref 2 and j = ref 0 in + while !i < len do + if CCChar.equal pat.[!i - 1] pat.[!j] then ( + incr j; + failure.(!i) <- !j; + incr i + ) else if !j = 0 then ( + failure.(!i) <- 0; + incr i + ) else + j := failure.(!j) + done; + { failure; pat } + + let find { failure; pat } s = + let len = String.length s in + let i = ref 0 and j = ref 0 in + let pat_len = String.length pat in + while !j < pat_len && !i + !j < len do + if CCChar.equal s.[!i + !j] pat.[!j] then + incr j + else ( + let fail_offset = failure.(!j) in + if fail_offset >= 0 then ( + assert (fail_offset < !j); + i := !i + !j - fail_offset; + j := fail_offset + ) else ( + j := 0; + incr i + ) + ) + done; + if !j = pat_len then + !i + else + -1 + end + + module New_kmp = struct + type t = { + failure: int array; + pat: string; + } + + let compile pat = + let len = String.length pat in + match len with + | 0 -> { failure = [||]; pat } + | 1 -> { failure = [| -1 |]; pat } + | _ -> + let failure = Array.make len 0 in + failure.(0) <- -1; + let i = ref 2 and j = ref 0 in + while !i < len do + if CCChar.equal pat.[!i - 1] pat.[!j] then ( + incr j; + failure.(!i) <- !j; + incr i + ) else if !j = 0 then ( + failure.(!i) <- 0; + incr i + ) else + j := failure.(!j) + done; + (* strengthen: if the char after the border equals the one we'd be + retrying, skip straight to that border's (already-strengthened) + target instead of re-comparing it and failing again *) + for k = 1 to len - 1 do + let b = failure.(k) in + if b >= 0 && CCChar.equal pat.[k] pat.[b] then + failure.(k) <- failure.(b) + done; + { failure; pat } + + let find { failure; pat } s = + let len = String.length s in + let i = ref 0 and j = ref 0 in + let pat_len = String.length pat in + while !j < pat_len && !i + !j < len do + if CCChar.equal s.[!i + !j] pat.[!j] then + incr j + else ( + let fail_offset = failure.(!j) in + if fail_offset >= 0 then ( + assert (fail_offset < !j); + i := !i + !j - fail_offset; + j := fail_offset + ) else ( + (* no usable border: advance past the mismatch (NOT [incr i]) *) + i := !i + !j + 1; + j := 0 + ) + ) + done; + if !j = pat_len then + !i + else + -1 + end + + (* a worst case test to show the difference between classic and new KMP, + with a needle [a^size] searched in a haystack made of [a^(size-1)b] + blocks. *) + let bench_find_special ~size n = + let needle = String.make size 'a' in + let block = String.make (size - 1) 'a' ^ "b" in + let haystack = CCString.repeat block (n / size) in + pp_pb needle haystack; + let classic = Classic_kmp.compile needle in + let strong = New_kmp.compile needle in + let cur = CCString.find ~start:0 ~sub:needle in + let mk_naive () = find ~sub:needle haystack + and mk_classic () = Classic_kmp.find classic haystack + and mk_new () = New_kmp.find strong haystack + and mk_cur () = cur haystack in + assert (mk_naive () = -1); + assert (mk_classic () = -1); + assert (mk_new () = -1); + assert (mk_cur () = -1); + (* positive control: a haystack that does contain the needle *) + let with_match = haystack ^ needle in + let expected = CCString.find ~sub:needle with_match in + assert (Classic_kmp.find classic with_match = expected); + assert (New_kmp.find strong with_match = expected); + B.throughputN 2 ~repeat + [ + "naive", mk_naive, (); + "classic", mk_classic, (); + "new", mk_new, (); + "cur", mk_cur, (); + ] + let bench_find = bench_find_ ~dir:`Direct let bench_rfind = bench_find_ ~dir:`Reverse @@ -1757,6 +1910,19 @@ module Str = struct @>> app_ints (bench_find ~size:50) [ 100; 100_000; 500_000 ]; "500" @>> app_ints (bench_find ~size:500) [ 100_000; 500_000 ]; + (* short haystack, long needle: needle (500) longer than + haystack, so the lazy failure table is never built *) + "500_short" + @>> app_ints (bench_find ~size:500) [ 50; 100 ]; + ]; + "find_special" + @>>> [ + "100" + @>> app_ints (bench_find_special ~size:100) + [ 200_000; 1_000_000 ]; + "1000" + @>> app_ints (bench_find_special ~size:1000) + [ 200_000; 1_000_000 ]; ]; "find_all" @>>> [ From ff51885abb585b47a5ad4efd3f293a5470f5c79f Mon Sep 17 00:00:00 2001 From: Simon Cruanes Date: Fri, 12 Jun 2026 15:17:10 -0400 Subject: [PATCH 04/13] perf ccstring: stronger KMP with delayed compilation, and strenghten failure condition --- src/core/CCString.ml | 68 +++++++++++++++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 17 deletions(-) diff --git a/src/core/CCString.ml b/src/core/CCString.ml index 76b22e3b1..26e6e54a7 100644 --- a/src/core/CCString.ml +++ b/src/core/CCString.ml @@ -45,11 +45,12 @@ type _ direction = (* we follow https://en.wikipedia.org/wiki/Knuth–Morris–Pratt_algorithm *) module Find = struct type 'a kmp_pattern = { - failure: int array; + mutable failure: int array; (* empty=not computed yet *) str: string; + dir: 'a direction; } - (* invariant: [length failure = length str]. - We use a phantom type to avoid mixing the directions. *) + (* invariant: [failure=[||] || length failure = length str]. + The failure table is built lazily, we bypass it if len(haystack) String.get | Reverse -> fun s i -> s.[String.length s - i - 1] - let kmp_compile_ : type a. dir:a direction -> string -> a kmp_pattern = + (* build the KMP failure table for [str], whose elements are read + according to [dir] *) + let kmp_failure_ : type a. dir:a direction -> string -> int array = fun ~dir str -> let len = length str in let get = get_ ~dir in (* how to read elements of the string *) match len with - | 0 -> { failure = [||]; str } - | 1 -> { failure = [| -1 |]; str } + | 0 -> [||] + | 1 -> [| -1 |] | _ -> (* at least 2 elements, the algorithm can work *) let failure = Array.make len 0 in @@ -93,7 +96,26 @@ module Find = struct j := failure.(!j) done; (* Format.printf "{@[failure:%a, str:%s@]}@." CCFormat.(array int) failure str; *) - { failure; str } + + (* strengthen the failure function: if falling back to a border would + re-compare a character equal to the one that just mismatched, skip + straight to that border's (already-strengthened) target instead. + Uses the direction-aware [get] so it works for both directions. *) + for k = 1 to len - 1 do + let b = failure.(k) in + if b >= 0 && CCChar.equal (get str k) (get str b) then + failure.(k) <- failure.(b) + done; + failure + + let kmp_compile_ : type a. dir:a direction -> string -> a kmp_pattern = + fun ~dir str -> { str; failure = [||]; dir } + + let[@inline] kmp_get_ (type a) (f: a kmp_pattern) : int array = + if Array.length f.failure = 0 then ( + f.failure <- kmp_failure_ ~dir:f.dir f.str; + ); + f.failure let kmp_compile s = kmp_compile_ ~dir:Direct s let kmp_rcompile s = kmp_compile_ ~dir:Reverse s @@ -107,6 +129,7 @@ module Find = struct let i = ref idx in let j = ref 0 in let pat_len = kmp_pattern_length pattern in + let failure = kmp_get_ pattern in while !j < pat_len && !i + !j < len do let c = String.get s (!i + !j) in let expected = String.get pattern.str !j in @@ -114,16 +137,16 @@ module Find = struct (* char matches *) incr j else ( - let fail_offset = pattern.failure.(!j) in + let fail_offset = failure.(!j) in if fail_offset >= 0 then ( assert (fail_offset < !j); (* follow the failure link *) i := !i + !j - fail_offset; j := fail_offset ) else ( - (* beginning of pattern *) - j := 0; - incr i + (* no usable border: advance past the mismatch *) + i := !i + !j + 1; + j := 0 ) ) done; @@ -141,6 +164,7 @@ module Find = struct let i = ref (len - idx - 1) in let j = ref 0 in let pat_len = kmp_pattern_length pattern in + let failure = kmp_get_ pattern in while !j < pat_len && !i + !j < len do let c = String.get s (len - !i - !j - 1) in let expected = @@ -150,16 +174,16 @@ module Find = struct (* char matches *) incr j else ( - let fail_offset = pattern.failure.(!j) in + let fail_offset = failure.(!j) in if fail_offset >= 0 then ( assert (fail_offset < !j); (* follow the failure link *) i := !i + !j - fail_offset; j := fail_offset ) else ( - (* beginning of pattern *) - j := 0; - incr i + (* no usable border: advance past the mismatch *) + i := !i + !j + 1; + j := 0 ) ) done; @@ -197,7 +221,12 @@ module Find = struct let find ?(start = 0) ~(pattern : [ `Direct ] pattern) s = match pattern with | P_char c -> (try String.index_from s start c with Not_found -> -1) - | P_KMP pattern -> kmp_find ~pattern s start + | P_KMP pattern -> + (* haystack too short, bail out without forcing kmp state *) + if String.length s - start < kmp_pattern_length pattern then + -1 + else + kmp_find ~pattern s start let rfind ?start ~(pattern : [ `Reverse ] pattern) s = let start = @@ -207,7 +236,12 @@ module Find = struct in match pattern with | P_char c -> (try String.rindex_from s start c with Not_found -> -1) - | P_KMP pattern -> kmp_rfind ~pattern s start + | P_KMP pattern -> + (* haystack too short, bail out without forcing kmp state *) + if start + 1 < kmp_pattern_length pattern then + -1 + else + kmp_rfind ~pattern s start end let find ?(start = 0) ~sub = From 43b9306cec08ecaa0ca640173f031a70f4c27860 Mon Sep 17 00:00:00 2001 From: Simon Cruanes Date: Fri, 12 Jun 2026 19:17:53 +0000 Subject: [PATCH 05/13] format --- benchs/run_benchs.ml | 9 +++++---- src/core/CCString.ml | 8 ++++---- tests/core/t_string.ml | 19 ++++++++++++++----- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/benchs/run_benchs.ml b/benchs/run_benchs.ml index b20fa08ec..e9c8084d9 100644 --- a/benchs/run_benchs.ml +++ b/benchs/run_benchs.ml @@ -1912,16 +1912,17 @@ module Str = struct @>> app_ints (bench_find ~size:500) [ 100_000; 500_000 ]; (* short haystack, long needle: needle (500) longer than haystack, so the lazy failure table is never built *) - "500_short" - @>> app_ints (bench_find ~size:500) [ 50; 100 ]; + "500_short" @>> app_ints (bench_find ~size:500) [ 50; 100 ]; ]; "find_special" @>>> [ "100" - @>> app_ints (bench_find_special ~size:100) + @>> app_ints + (bench_find_special ~size:100) [ 200_000; 1_000_000 ]; "1000" - @>> app_ints (bench_find_special ~size:1000) + @>> app_ints + (bench_find_special ~size:1000) [ 200_000; 1_000_000 ]; ]; "find_all" diff --git a/src/core/CCString.ml b/src/core/CCString.ml index 26e6e54a7..7ceb95408 100644 --- a/src/core/CCString.ml +++ b/src/core/CCString.ml @@ -95,6 +95,7 @@ module Find = struct assert (!j > 0); j := failure.(!j) done; + (* Format.printf "{@[failure:%a, str:%s@]}@." CCFormat.(array int) failure str; *) (* strengthen the failure function: if falling back to a border would @@ -109,12 +110,11 @@ module Find = struct failure let kmp_compile_ : type a. dir:a direction -> string -> a kmp_pattern = - fun ~dir str -> { str; failure = [||]; dir } + fun ~dir str -> { str; failure = [||]; dir } - let[@inline] kmp_get_ (type a) (f: a kmp_pattern) : int array = - if Array.length f.failure = 0 then ( + let[@inline] kmp_get_ (type a) (f : a kmp_pattern) : int array = + if Array.length f.failure = 0 then f.failure <- kmp_failure_ ~dir:f.dir f.str; - ); f.failure let kmp_compile s = kmp_compile_ ~dir:Direct s diff --git a/tests/core/t_string.ml b/tests/core/t_string.ml index 92207e896..4f13f8310 100644 --- a/tests/core/t_string.ml +++ b/tests/core/t_string.ml @@ -63,8 +63,11 @@ q ~count:10_000 failure table). *) let gen_ab ~maxlen = Q.Gen.( - string_size (int_range 0 maxlen) - ~gen:(fun st -> if Random.State.bool st then 'a' else 'b')) + string_size (int_range 0 maxlen) ~gen:(fun st -> + if Random.State.bool st then + 'a' + else + 'b')) let arb_haystack = Q.make ~print:Q.Print.string (gen_ab ~maxlen:20) let arb_needle = Q.make ~print:Q.Print.string (gen_ab ~maxlen:6) @@ -110,17 +113,23 @@ end q ~count:10_000 ~name:"naive1" Q.(pair arb_haystack arb_needle) - (fun (s, sub) -> Q.assume (sub <> ""); find ~sub s = Find_.naive_find ~sub s) + (fun (s, sub) -> + Q.assume (sub <> ""); + find ~sub s = Find_.naive_find ~sub s) ;; q ~count:10_000 ~name:"naive2" Q.(pair arb_haystack arb_needle) - (fun (s, sub) -> Q.assume (sub <> ""); rfind ~sub s = Find_.naive_rfind ~sub s) + (fun (s, sub) -> + Q.assume (sub <> ""); + rfind ~sub s = Find_.naive_rfind ~sub s) ;; q ~count:10_000 ~name:"naive3" Q.(pair arb_haystack arb_needle) - (fun (s, sub) -> Q.assume (sub <> ""); find_all_l ~sub s = Find_.naive_find_all ~sub s) + (fun (s, sub) -> + Q.assume (sub <> ""); + find_all_l ~sub s = Find_.naive_find_all ~sub s) ;; eq ~printer:CCFun.id From 80f7b6e14f448c857d97a5efd70c983ebd03e047 Mon Sep 17 00:00:00 2001 From: Simon Cruanes Date: Tue, 18 Aug 2026 16:22:59 -0400 Subject: [PATCH 06/13] cleanup --- benchs/run_benchs.ml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/benchs/run_benchs.ml b/benchs/run_benchs.ml index e9c8084d9..50494701d 100644 --- a/benchs/run_benchs.ml +++ b/benchs/run_benchs.ml @@ -1636,7 +1636,7 @@ module Str = struct assert (CCList.equal CCInt.equal (mk_naive ()) (mk_current ())); B.throughputN 3 ~repeat [ "naive", mk_naive, (); "current", mk_current, () ] - (* Inlined reference implementations to have a fixpoint to compae + (* Inlined reference implementations to have a fixpoint to compare CCString with. - [Classic_kmp] is a copy of CCString's old KMP: the @@ -1721,9 +1721,6 @@ module Str = struct ) else j := failure.(!j) done; - (* strengthen: if the char after the border equals the one we'd be - retrying, skip straight to that border's (already-strengthened) - target instead of re-comparing it and failing again *) for k = 1 to len - 1 do let b = failure.(k) in if b >= 0 && CCChar.equal pat.[k] pat.[b] then @@ -1745,7 +1742,7 @@ module Str = struct i := !i + !j - fail_offset; j := fail_offset ) else ( - (* no usable border: advance past the mismatch (NOT [incr i]) *) + (* no usable border: advance past the mismatch *) i := !i + !j + 1; j := 0 ) From 27e9fd0272417a471eaf1804697710d81c6e47e3 Mon Sep 17 00:00:00 2001 From: Simon Cruanes Date: Tue, 18 Aug 2026 16:23:28 -0400 Subject: [PATCH 07/13] CCString: handle "" separator in finding/splitting functions try to get closer to the behavior of the new String functions in OCaml 5.6 --- src/core/CCString.ml | 153 ++++++++++++++++++++++++------------ src/core/CCString.mli | 18 ++--- src/core/CCStringLabels.mli | 9 ++- 3 files changed, 115 insertions(+), 65 deletions(-) diff --git a/src/core/CCString.ml b/src/core/CCString.ml index 7ceb95408..90ad4a6c5 100644 --- a/src/core/CCString.ml +++ b/src/core/CCString.ml @@ -197,29 +197,36 @@ module Find = struct -1 type 'a pattern = + | P_empty | P_char of char | P_KMP of 'a kmp_pattern let pattern_length = function + | P_empty -> 0 | P_char _ -> 1 | P_KMP p -> kmp_pattern_length p let compile sub : [ `Direct ] pattern = - if length sub = 0 then invalid_arg "CCString.Find.compile: empty pattern"; - if length sub = 1 then - P_char sub.[0] - else - P_KMP (kmp_compile sub) + match length sub with + | 0 -> P_empty + | 1 -> P_char sub.[0] + | _ -> P_KMP (kmp_compile sub) let rcompile sub : [ `Reverse ] pattern = - if length sub = 0 then invalid_arg "CCString.Find.rcompile: empty pattern"; - if length sub = 1 then - P_char sub.[0] - else - P_KMP (kmp_rcompile sub) + match length sub with + | 0 -> P_empty + | 1 -> P_char sub.[0] + | _ -> P_KMP (kmp_rcompile sub) let find ?(start = 0) ~(pattern : [ `Direct ] pattern) s = match pattern with + | P_empty -> + if start < 0 then + invalid_arg "CCString.Find.find: invalid offset" + else if start > length s then + -1 + else + start | P_char c -> (try String.index_from s start c with Not_found -> -1) | P_KMP pattern -> (* haystack too short, bail out without forcing kmp state *) @@ -230,11 +237,19 @@ module Find = struct let rfind ?start ~(pattern : [ `Reverse ] pattern) s = let start = - match start with - | Some n -> n - | None -> String.length s - 1 + match pattern, start with + | _, Some n -> n + | P_empty, None -> String.length s + | _, None -> String.length s - 1 in match pattern with + | P_empty -> + if start > String.length s then + invalid_arg "CCString.Find.rfind: invalid offset" + else if start < 0 then + -1 + else + start | P_char c -> (try String.rindex_from s start c with Not_found -> -1) | P_KMP pattern -> (* haystack too short, bail out without forcing kmp state *) @@ -274,7 +289,7 @@ let mem ?start ~sub s = find ?start ~sub s >= 0 let rfind ~sub = let pattern = Find.rcompile sub in - fun s -> Find.rfind ~start:(String.length s - 1) ~pattern s + fun s -> Find.rfind ~pattern s (* Replace substring [s.[pos] … s.[pos+len-1]] by [by] in [s] *) let replace_at_ ~pos ~len ~by s = @@ -304,13 +319,16 @@ let replace ?(which = `All) ~sub ~by s = let pattern = Find.compile sub in let b = Buffer.create (String.length s) in let start = ref 0 in + + let incr_by = max 1 (String.length sub) in + while !start < String.length s do let i = Find.find ~start:!start ~pattern s in if i >= 0 then ( (* between last and cur occurrences *) Buffer.add_substring b s !start (i - !start); Buffer.add_string b by; - start := i + String.length sub + start := i + incr_by ) else ( (* add remainder *) Buffer.add_substring b s !start (String.length s - !start); @@ -330,33 +348,63 @@ module Split = struct type split_state = | SplitStop - | SplitAt of int (* previous *) - - let rec _split ~by s state = + | SplitAt of { + search_start: int; + token_start: int; + } + + type split_next = + | SplitNone + | SplitSome of { + st: split_state; + off: int; + len: int; + } + + let rec _split ~by s state : split_next = match state with - | SplitStop -> None - | SplitAt prev -> _split_search ~by s prev + | SplitStop -> SplitNone + | SplitAt { search_start; token_start } -> + _split_search ~by s ~token_start ~search_start - and _split_search ~by s prev = - let j = Find.find ~start:prev ~pattern:by s in + and _split_search ~by s ~token_start ~search_start : split_next = + let j = Find.find ~start:search_start ~pattern:by s in if j < 0 then - Some (SplitStop, prev, String.length s - prev) - else - Some (SplitAt (j + Find.pattern_length by), prev, j - prev) + SplitSome + { + st = SplitStop; + off = token_start; + len = String.length s - token_start; + } + else ( + let pat_len = Find.pattern_length by in + SplitSome + { + st = + SplitAt + { + token_start = j + pat_len; + (* empty pattern: must still move +1 *) + search_start = j + max 1 pat_len; + }; + off = token_start; + len = j - token_start; + } + ) - let _tuple3 x y z = x, y, z + let[@inline] _tuple3 x y z = x, y, z - let _mkgen ~drop ~by s k = - let state = ref (SplitAt 0) in + let _mkgen ~drop ~by s k : _ gen = + let state = ref (SplitAt { search_start = 0; token_start = 0 }) in let by = Find.compile by in let rec next () = match _split ~by s !state with - | None -> None - | Some (state', 0, 0) when drop.first -> + | SplitNone -> None + | SplitSome { st = state'; off = 0; len = 0 } when drop.first -> state := state'; next () - | Some (_, i, 0) when drop.last && i = length s -> None - | Some (state', i, len) -> + | SplitSome { off = i; len = 0; _ } when drop.last && i = length s -> None + | SplitSome { st = state'; off = i; len } -> state := state'; Some (k s i len) in @@ -365,30 +413,36 @@ module Split = struct let gen ?(drop = default_drop) ~by s = _mkgen ~drop ~by s _tuple3 let gen_cpy ?(drop = default_drop) ~by s = _mkgen ~drop ~by s String.sub - let _mklist ~drop ~by s k = + let _mklist ~drop ~by s k : _ list = let by = Find.compile by in let rec build acc state = match _split ~by s state with - | None -> List.rev acc - | Some (state', 0, 0) when drop.first -> build acc state' - | Some (_, i, 0) when drop.last && i = length s -> List.rev acc - | Some (state', i, len) -> build (k s i len :: acc) state' + | SplitNone -> List.rev acc + | SplitSome { st = state'; off = 0; len = 0 } when drop.first -> + build acc state' + | SplitSome { off = i; len = 0; _ } when drop.last && i = length s -> + List.rev acc + | SplitSome { st = state'; off = i; len } -> + build (k s i len :: acc) state' in - build [] (SplitAt 0) + build [] (SplitAt { search_start = 0; token_start = 0 }) let list_ ?(drop = default_drop) ~by s = _mklist ~drop ~by s _tuple3 let list_cpy ?(drop = default_drop) ~by s = _mklist ~drop ~by s String.sub - let _mkseq ~drop ~by s k = + let _mkseq ~drop ~by s k : _ Seq.t = let by = Find.compile by in let rec make state () = match _split ~by s state with - | None -> Seq.Nil - | Some (state', 0, 0) when drop.first -> make state' () - | Some (_, i, 0) when drop.last && i = length s -> Seq.Nil - | Some (state', i, len) -> Seq.Cons (k s i len, make state') + | SplitNone -> Seq.Nil + | SplitSome { st = state'; off = 0; len = 0 } when drop.first -> + make state' () + | SplitSome { off = i; len = 0; _ } when drop.last && i = length s -> + Seq.Nil + | SplitSome { st = state'; off = i; len } -> + Seq.Cons (k s i len, make state') in - make (SplitAt 0) + make (SplitAt { search_start = 0; token_start = 0 }) let seq ?(drop = default_drop) ~by s = _mkseq ~drop ~by s _tuple3 let seq_cpy ?(drop = default_drop) ~by s = _mkseq ~drop ~by s String.sub @@ -397,14 +451,15 @@ module Split = struct let by = Find.compile by in let rec aux state = match _split ~by s state with - | None -> () - | Some (state', 0, 0) when drop.first -> aux state' - | Some (_, i, 0) when drop.last && i = length s -> () - | Some (state', i, len) -> + | SplitNone -> () + | SplitSome { st = state'; off = 0; len = 0 } when drop.first -> + aux state' + | SplitSome { off = i; len = 0; _ } when drop.last && i = length s -> () + | SplitSome { st = state'; off = i; len } -> k (f s i len); aux state' in - aux (SplitAt 0) + aux (SplitAt { search_start = 0; token_start = 0 }) let iter ?(drop = default_drop) ~by s = _mk_iter ~drop ~by s _tuple3 let iter_cpy ?(drop = default_drop) ~by s = _mk_iter ~drop ~by s String.sub diff --git a/src/core/CCString.mli b/src/core/CCString.mli index a1e7fd1d2..0118c3987 100644 --- a/src/core/CCString.mli +++ b/src/core/CCString.mli @@ -104,32 +104,28 @@ val to_array : string -> char array val find : ?start:int -> sub:string -> string -> int (** [find ~start ~sub s] returns the starting index of the first occurrence of [sub] within [s] or [-1]. - @param start starting position in [s]. - @raise Invalid_argument if [sub = ""]. *) + Returns [start] if [sub=""]. *) val find_all : ?start:int -> sub:string -> string -> int gen (** [find_all ~start ~sub s] finds all occurrences of [sub] in [s], even overlapping instances and returns them in a generator [gen]. @param start starting position in [s]. - @raise Invalid_argument if [sub = ""]. + Returns all positions between start and the end of [s] if [sub=""] @since 0.17 *) val find_all_l : ?start:int -> sub:string -> string -> int list (** [find_all_l ~sub s] finds all occurrences of [sub] in [s] and returns them in a list. @param start starting position in [s]. - @raise Invalid_argument if [sub = ""]. @since 0.17 *) val mem : ?start:int -> sub:string -> string -> bool (** [mem ~start ~sub s] is [true] iff [sub] is a substring of [s]. - @raise Invalid_argument if [sub = ""]. @since 0.12 *) val rfind : sub:string -> string -> int (** [rfind ~sub s] finds [sub] in string [s] from the right, returns its first index or [-1]. Should only be used with very small [sub]. - @raise Invalid_argument if [sub = ""]. @since 0.12 *) val replace : @@ -358,12 +354,10 @@ module Find : sig type _ pattern val compile : string -> [ `Direct ] pattern - (** [compile sub] compiles [sub] into a pattern usable with {!find}. - @raise Invalid_argument if [sub = ""]. *) + (** [compile sub] compiles [sub] into a pattern usable with {!find}. *) val rcompile : string -> [ `Reverse ] pattern - (** [rcompile sub] compiles [sub] into a pattern usable with {!rfind}. - @raise Invalid_argument if [sub = ""]. *) + (** [rcompile sub] compiles [sub] into a pattern usable with {!rfind}. *) val find : ?start:int -> pattern:[ `Direct ] pattern -> string -> int (** [find ~start ~pattern s] searches for [pattern] in the string [s], left-to-right. @@ -404,8 +398,7 @@ module Split : sig Should only be used with very small separators. @return a [list] of slices [(s,index,length)] that are separated by [by]. {!String.sub} can then be used to actually extract - a string from the slice. - @raise Failure if [by = ""]. *) + a string from the slice. *) val gen : ?drop:drop_if_empty -> by:string -> string -> (string * int * int) gen @@ -475,7 +468,6 @@ end val split : by:string -> string -> string list (** [split ~by s] splits the string [s] along the given string [by]. Alias to {!Split.list_cpy}. - @raise Invalid_argument if [by = ""]. @since 1.2 *) (** {2 Utils} *) diff --git a/src/core/CCStringLabels.mli b/src/core/CCStringLabels.mli index d46a32225..706ee9f7a 100644 --- a/src/core/CCStringLabels.mli +++ b/src/core/CCStringLabels.mli @@ -109,12 +109,13 @@ val to_array : string -> char array val find : ?start:int -> sub:(string[@keep_label]) -> string -> int (** [find ?start ~sub s] returns the starting index of the first occurrence of [sub] within [s] or [-1]. - @param start starting position in [s]. *) + Returns [start] if [sub=""]. *) val find_all : ?start:int -> sub:(string[@keep_label]) -> string -> int gen (** [find_all ?start ~sub s] finds all occurrences of [sub] in [s], even overlapping instances and returns them in a generator [gen]. @param start starting position in [s]. + Returns all positions between start and the end of [s] if [sub=""] @since 0.17 *) val find_all_l : ?start:int -> sub:(string[@keep_label]) -> string -> int list @@ -388,7 +389,10 @@ module Find : sig type _ pattern val compile : string -> [ `Direct ] pattern + (** [compile sub] compiles [sub] into a pattern usable with {!find}. *) + val rcompile : string -> [ `Reverse ] pattern + (** [rcompile sub] compiles [sub] into a pattern usable with {!rfind}. *) val find : ?start:int -> pattern:([ `Direct ] pattern[@keep_label]) -> string -> int @@ -434,8 +438,7 @@ module Split : sig Should only be used with very small separators. @return a [list] of slices [(s,index,length)] that are separated by [by]. {!String.sub} can then be used to actually extract - a string from the slice. - @raise Failure if [by = ""]. *) + a string from the slice. *) val gen : ?drop:drop_if_empty -> by:string -> string -> (string * int * int) gen From 55a26baf5623e3e51c737add67cc3a543bf628d5 Mon Sep 17 00:00:00 2001 From: Simon Cruanes Date: Tue, 18 Aug 2026 16:23:35 -0400 Subject: [PATCH 08/13] update tests for CCString --- tests/core/t_string.ml | 64 ++++++++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 15 deletions(-) diff --git a/tests/core/t_string.ml b/tests/core/t_string.ml index 4f13f8310..c6fa26b37 100644 --- a/tests/core/t_string.ml +++ b/tests/core/t_string.ml @@ -54,13 +54,7 @@ q ~count:10_000 let i = rfind ~sub:s2 s1 in i < 0 || String.sub s1 i (length s2) = s2) -(* Compare the KMP-based search against a naive reference, over a 2-char - alphabet so that needles actually occur in the haystack. This is a - two-sided check: it catches false negatives, wrong (non-leftmost/rightmost) - positions, and missed overlapping matches — none of which the properties - above (one-sided, large alphabet) can see. Needles can be longer than the - haystack (exercising the short-circuit) and repetitive (exercising the - failure table). *) +(* compare to a naive reference. *) let gen_ab ~maxlen = Q.Gen.( string_size (int_range 0 maxlen) ~gen:(fun st -> @@ -69,7 +63,7 @@ let gen_ab ~maxlen = else 'b')) -let arb_haystack = Q.make ~print:Q.Print.string (gen_ab ~maxlen:20) +let arb_haystack = Q.make ~print:Q.Print.string (gen_ab ~maxlen:30) let arb_needle = Q.make ~print:Q.Print.string (gen_ab ~maxlen:6) module Find_ = struct @@ -114,24 +108,32 @@ end q ~count:10_000 ~name:"naive1" Q.(pair arb_haystack arb_needle) (fun (s, sub) -> - Q.assume (sub <> ""); - find ~sub s = Find_.naive_find ~sub s) + let res_find = find ~sub s in + let res_naive = Find_.naive_find ~sub s in + (* Printf.printf "same: %B: %S in %S: find: %d, naive find: %d\n%!" + (res_naive = res_find) sub s res_find res_naive; *) + res_find = res_naive) ;; q ~count:10_000 ~name:"naive2" Q.(pair arb_haystack arb_needle) (fun (s, sub) -> - Q.assume (sub <> ""); - rfind ~sub s = Find_.naive_rfind ~sub s) + let res_find = rfind ~sub s in + let res_naive = Find_.naive_rfind ~sub s in + (* Printf.printf "same: %B: %S in %S: rfind: %d, naive rfind: %d\n%!" + (res_naive = res_find) sub s res_find res_naive; *) + res_find = res_naive) ;; q ~count:10_000 ~name:"naive3" Q.(pair arb_haystack arb_needle) - (fun (s, sub) -> - Q.assume (sub <> ""); - find_all_l ~sub s = Find_.naive_find_all ~sub s) + (fun (s, sub) -> find_all_l ~sub s = Find_.naive_find_all ~sub s) ;; +eq ~printer:string_of_int 0 (find ~sub:"" "");; +eq ~printer:string_of_int 2 (rfind ~sub:"" "ab");; +eq ~printer:Q.Print.(list int) [ 0; 1; 2 ] (find_all_l ~sub:"" "ab");; + eq ~printer:CCFun.id (replace ~which:`All ~sub:"a" ~by:"b" "abcdabcd") "bbcdbbcd" @@ -166,6 +168,38 @@ t @@ fun () -> Split.list_cpy ~by:" " "hello world aie" = [ "hello"; ""; "world"; "aie" ] ;; +eq ~name:__LOC__ ~printer:Q.Print.(list string) [ ""; "" ] (split ~by:"" "");; + +eq ~name:__LOC__ + ~printer:Q.Print.(list string) + [ "" ] + (Split.list_cpy ~drop:{ Split.no_drop with first = true } ~by:"" "") +;; + +eq ~name:__LOC__ + ~printer:Q.Print.(list string) + [ "" ] + (Split.list_cpy ~drop:{ Split.no_drop with last = true } ~by:"" "") +;; + +eq ~name:__LOC__ + ~printer:Q.Print.(list string) + [ ""; "a"; "b"; "" ] (split ~by:"" "ab") +;; + +t ~name:__LOC__ @@ fun () -> +let g = Split.gen_cpy ~by:"" "ab" in +let assert_next = assert_equal ~printer:Q.Print.(option string) in +assert_next (Some "") (g ()); +assert_next (Some "a") (g ()); +assert_next (Some "b") (g ()); +assert_next (Some "") (g ()); +assert_next None (g ()); +true +;; + +t ~name:__LOC__ @@ fun () -> Split.left ~by:"" "ab" = Some ("", "ab");; +t ~name:__LOC__ @@ fun () -> Split.right ~by:"" "ab" = Some ("ab", "");; t @@ fun () -> Split.left ~by:" " "ab cde f g " = Some ("ab", "cde f g ");; t @@ fun () -> Split.left ~by:"__" "a__c__e_f" = Some ("a", "c__e_f");; t @@ fun () -> Split.left ~by:"_" "abcde" = None;; From 360a1af608cab9941240d6c4df0912111c6fc583 Mon Sep 17 00:00:00 2001 From: Simon Cruanes Date: Tue, 18 Aug 2026 16:49:26 -0400 Subject: [PATCH 09/13] perf: improve allocation profile of `CCString.Split` internal state --- src/core/CCString.ml | 166 +++++++++++++++++++++---------------------- 1 file changed, 81 insertions(+), 85 deletions(-) diff --git a/src/core/CCString.ml b/src/core/CCString.ml index 90ad4a6c5..0ea12d0b4 100644 --- a/src/core/CCString.ml +++ b/src/core/CCString.ml @@ -346,67 +346,72 @@ module Split = struct let no_drop = { first = false; last = false } let default_drop = no_drop - type split_state = - | SplitStop - | SplitAt of { - search_start: int; - token_start: int; - } - - type split_next = - | SplitNone - | SplitSome of { - st: split_state; - off: int; - len: int; - } - - let rec _split ~by s state : split_next = - match state with - | SplitStop -> SplitNone - | SplitAt { search_start; token_start } -> - _split_search ~by s ~token_start ~search_start - - and _split_search ~by s ~token_start ~search_start : split_next = - let j = Find.find ~start:search_start ~pattern:by s in - if j < 0 then - SplitSome - { - st = SplitStop; - off = token_start; - len = String.length s - token_start; - } + type split_state = { + s: string; (** The string to split *) + by: [ `Direct ] Find.pattern; + mutable stopped: bool; + mutable is_first: bool; + mutable is_last: bool; + mutable search_start: int; + mutable token_start: int; + mutable off: int; (** Offset of the current result in [s] *) + mutable len: int; (** Length of the current result in [s] *) + } + + let _make_state ~by s : split_state = + { + s; + by = Find.compile by; + stopped = false; + is_first = true; + is_last = false; + search_start = 0; + token_start = 0; + off = 0; + len = 0; + } + + (* Advance [self] and return whether [off] and [len] contain a result. *) + let _split (self : split_state) : bool = + if self.stopped then + false else ( - let pat_len = Find.pattern_length by in - SplitSome - { - st = - SplitAt - { - token_start = j + pat_len; - (* empty pattern: must still move +1 *) - search_start = j + max 1 pat_len; - }; - off = token_start; - len = j - token_start; - } + let j = Find.find ~start:self.search_start ~pattern:self.by self.s in + if j < 0 then ( + self.stopped <- true; + self.is_last <- true; + self.off <- self.token_start; + self.len <- String.length self.s - self.token_start + ) else ( + let pat_len = Find.pattern_length self.by in + self.is_last <- false; + (* empty pattern: must still move +1 *) + self.search_start <- j + max 1 pat_len; + self.off <- self.token_start; + self.len <- j - self.token_start; + self.token_start <- j + pat_len + ); + true ) + let _should_drop ~(drop : drop_if_empty) (st : split_state) : bool = + let should_drop = + st.len = 0 && ((st.is_first && drop.first) || (st.is_last && drop.last)) + in + st.is_first <- false; + should_drop + let[@inline] _tuple3 x y z = x, y, z let _mkgen ~drop ~by s k : _ gen = - let state = ref (SplitAt { search_start = 0; token_start = 0 }) in - let by = Find.compile by in + let st = _make_state ~by s in let rec next () = - match _split ~by s !state with - | SplitNone -> None - | SplitSome { st = state'; off = 0; len = 0 } when drop.first -> - state := state'; + if not (_split st) then + None + else if _should_drop ~drop st then next () - | SplitSome { off = i; len = 0; _ } when drop.last && i = length s -> None - | SplitSome { st = state'; off = i; len } -> - state := state'; - Some (k s i len) + else + Some (k s st.off st.len) in next @@ -414,52 +419,43 @@ module Split = struct let gen_cpy ?(drop = default_drop) ~by s = _mkgen ~drop ~by s String.sub let _mklist ~drop ~by s k : _ list = - let by = Find.compile by in - let rec build acc state = - match _split ~by s state with - | SplitNone -> List.rev acc - | SplitSome { st = state'; off = 0; len = 0 } when drop.first -> - build acc state' - | SplitSome { off = i; len = 0; _ } when drop.last && i = length s -> + let st = _make_state ~by s in + let rec build acc = + if not (_split st) then List.rev acc - | SplitSome { st = state'; off = i; len } -> - build (k s i len :: acc) state' + else if _should_drop ~drop st then + build acc + else + build (k s st.off st.len :: acc) in - build [] (SplitAt { search_start = 0; token_start = 0 }) + build [] let list_ ?(drop = default_drop) ~by s = _mklist ~drop ~by s _tuple3 let list_cpy ?(drop = default_drop) ~by s = _mklist ~drop ~by s String.sub let _mkseq ~drop ~by s k : _ Seq.t = - let by = Find.compile by in - let rec make state () = - match _split ~by s state with - | SplitNone -> Seq.Nil - | SplitSome { st = state'; off = 0; len = 0 } when drop.first -> - make state' () - | SplitSome { off = i; len = 0; _ } when drop.last && i = length s -> + let rec make st = + if not (_split st) then Seq.Nil - | SplitSome { st = state'; off = i; len } -> - Seq.Cons (k s i len, make state') + else if _should_drop ~drop st then + make st + else + Seq.Cons (k s st.off st.len, delay st) + and delay st () = + (* need to copy [st] because it's mutable *) + let st = { st with stopped = st.stopped } in + make st in - make (SplitAt { search_start = 0; token_start = 0 }) + delay (_make_state ~by s) let seq ?(drop = default_drop) ~by s = _mkseq ~drop ~by s _tuple3 let seq_cpy ?(drop = default_drop) ~by s = _mkseq ~drop ~by s String.sub let _mk_iter ~drop ~by s f k = - let by = Find.compile by in - let rec aux state = - match _split ~by s state with - | SplitNone -> () - | SplitSome { st = state'; off = 0; len = 0 } when drop.first -> - aux state' - | SplitSome { off = i; len = 0; _ } when drop.last && i = length s -> () - | SplitSome { st = state'; off = i; len } -> - k (f s i len); - aux state' - in - aux (SplitAt { search_start = 0; token_start = 0 }) + let st = _make_state ~by s in + while _split st do + if not (_should_drop ~drop st) then k (f s st.off st.len) + done let iter ?(drop = default_drop) ~by s = _mk_iter ~drop ~by s _tuple3 let iter_cpy ?(drop = default_drop) ~by s = _mk_iter ~drop ~by s String.sub From d29cbe3f788a59c68b8e3908de37513c0b67ef38 Mon Sep 17 00:00:00 2001 From: Simon Cruanes Date: Tue, 18 Aug 2026 16:49:40 -0400 Subject: [PATCH 10/13] persistence test for splitting to Seq.t --- tests/core/t_string.ml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/core/t_string.ml b/tests/core/t_string.ml index c6fa26b37..6cfd99945 100644 --- a/tests/core/t_string.ml +++ b/tests/core/t_string.ml @@ -198,6 +198,18 @@ assert_next None (g ()); true ;; +t ~name:__LOC__ @@ fun () -> +let seq = Split.seq_cpy ~by:"" "ab" in +let head () = + match seq () with + | Seq.Nil -> None + | Seq.Cons (x, _) -> Some x +in +assert_equal ~printer:Q.Print.(option string) (Some "") (head ()); +assert_equal ~printer:Q.Print.(option string) (Some "") (head ()); +true +;; + t ~name:__LOC__ @@ fun () -> Split.left ~by:"" "ab" = Some ("", "ab");; t ~name:__LOC__ @@ fun () -> Split.right ~by:"" "ab" = Some ("ab", "");; t @@ fun () -> Split.left ~by:" " "ab cde f g " = Some ("ab", "cde f g ");; From 9dc551dd600f3db388bbc612ae957c780934e214 Mon Sep 17 00:00:00 2001 From: Simon Cruanes Date: Tue, 18 Aug 2026 16:54:47 -0400 Subject: [PATCH 11/13] benchmark CCString.Split --- benchs/run_benchs.ml | 87 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/benchs/run_benchs.ml b/benchs/run_benchs.ml index 50494701d..b8c0235bf 100644 --- a/benchs/run_benchs.ml +++ b/benchs/run_benchs.ml @@ -1636,6 +1636,91 @@ module Str = struct assert (CCList.equal CCInt.equal (mk_naive ()) (mk_current ())); B.throughputN 3 ~repeat [ "naive", mk_naive, (); "current", mk_current, () ] + module Previous_split = struct + type state = + | Stop + | At of { + search_start: int; + token_start: int; + } + + type next = + | None_ + | Some_ of { + state: state; + off: int; + len: int; + } + + let split ~pattern ~by_len s = function + | Stop -> None_ + | At { search_start; token_start } -> + let j = CCString.Find.find ~start:search_start ~pattern s in + if j < 0 then + Some_ + { + state = Stop; + off = token_start; + len = String.length s - token_start; + } + else + Some_ + { + state = At { search_start = j + by_len; token_start = j + by_len }; + off = token_start; + len = j - token_start; + } + + let iter ~by s yield = + let pattern = CCString.Find.compile by in + let by_len = String.length by in + let rec loop state = + match split ~pattern ~by_len s state with + | None_ -> () + | Some_ { state; off; len } -> + yield (s, off, len); + loop state + in + loop (At { search_start = 0; token_start = 0 }) + end + + let naive_split_iter ~by s yield = + let by_len = String.length by in + let rec loop token_start = + let j = find ~start:token_start ~sub:by s in + if j < 0 then + yield (s, token_start, String.length s - token_start) + else ( + yield (s, token_start, j - token_start); + loop (j + by_len) + ) + in + loop 0 + + let bench_split_iter n = + let sep = "abcdefg" in + let block = String.make 24 'x' ^ sep in + let haystack = CCString.repeat block (n / String.length block) in + let consume iter () = + let count = ref 0 and checksum = ref 0 in + iter (fun (_, off, len) -> + incr count; + checksum := !checksum * 65599 lxor off lxor len); + !count, !checksum + in + let mk_naive = consume (naive_split_iter ~by:sep haystack) + and mk_previous = consume (Previous_split.iter ~by:sep haystack) + and mk_current = consume (CCString.Split.iter ~by:sep haystack) in + let expected = mk_naive () in + assert (mk_previous () = expected); + assert (mk_current () = expected); + B.throughputN 3 ~repeat + [ + "naive", mk_naive, (); + "previous", mk_previous, (); + "current", mk_current, (); + ] + (* Inlined reference implementations to have a fixpoint to compare CCString with. @@ -1957,6 +2042,8 @@ module Str = struct [ 100_000; 500_000 ]; ]; ]; + "split_iter" + @>> app_ints bench_split_iter [ 100_000; 500_000; 1_000_000 ]; "rfind" @>>> [ "3" From b1b808dc3caf67b9f53c9fae44f89080bc2a1a9f Mon Sep 17 00:00:00 2001 From: Simon Cruanes Date: Wed, 19 Aug 2026 10:58:12 -0400 Subject: [PATCH 12/13] cleanups post review by @fardale --- benchs/run_benchs.ml | 4 ++-- src/core/CCString.ml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/benchs/run_benchs.ml b/benchs/run_benchs.ml index b8c0235bf..4110c411e 100644 --- a/benchs/run_benchs.ml +++ b/benchs/run_benchs.ml @@ -1808,8 +1808,8 @@ module Str = struct done; for k = 1 to len - 1 do let b = failure.(k) in - if b >= 0 && CCChar.equal pat.[k] pat.[b] then - failure.(k) <- failure.(b) + assert (b >= 0); + if CCChar.equal pat.[k] pat.[b] then failure.(k) <- failure.(b) done; { failure; pat } diff --git a/src/core/CCString.ml b/src/core/CCString.ml index 0ea12d0b4..fc42a79a8 100644 --- a/src/core/CCString.ml +++ b/src/core/CCString.ml @@ -104,8 +104,8 @@ module Find = struct Uses the direction-aware [get] so it works for both directions. *) for k = 1 to len - 1 do let b = failure.(k) in - if b >= 0 && CCChar.equal (get str k) (get str b) then - failure.(k) <- failure.(b) + assert (b >= 0); + if CCChar.equal (get str k) (get str b) then failure.(k) <- failure.(b) done; failure From 1f10d7cf6e5ad95163f2e5522d3b81420ac7fd94 Mon Sep 17 00:00:00 2001 From: Simon Cruanes Date: Thu, 20 Aug 2026 12:08:56 -0400 Subject: [PATCH 13/13] make sure we don't expose mutable state except in a couple specific places `gen` isn't thread safe here, but otherwise we make sure to use `compile` and not `compile_lazy` when intermediate state is reachable in the result (as opposed as used in a transient computation that yields a proper value as result). --- src/core/CCString.ml | 79 +++++++++++++++++++++++++------------------ src/core/CCString.mli | 15 +++++--- 2 files changed, 58 insertions(+), 36 deletions(-) diff --git a/src/core/CCString.ml b/src/core/CCString.ml index fc42a79a8..24b83c6b2 100644 --- a/src/core/CCString.ml +++ b/src/core/CCString.ml @@ -63,7 +63,8 @@ module Find = struct (* build the KMP failure table for [str], whose elements are read according to [dir] *) - let kmp_failure_ : type a. dir:a direction -> string -> int array = + let[@inline never] kmp_failure_ : type a. + dir:a direction -> string -> int array = fun ~dir str -> let len = length str in let get = get_ ~dir in @@ -109,7 +110,8 @@ module Find = struct done; failure - let kmp_compile_ : type a. dir:a direction -> string -> a kmp_pattern = + let[@inline] kmp_pattern_ : type a. dir:a direction -> string -> a kmp_pattern + = fun ~dir str -> { str; failure = [||]; dir } let[@inline] kmp_get_ (type a) (f : a kmp_pattern) : int array = @@ -117,8 +119,9 @@ module Find = struct f.failure <- kmp_failure_ ~dir:f.dir f.str; f.failure - let kmp_compile s = kmp_compile_ ~dir:Direct s - let kmp_rcompile s = kmp_compile_ ~dir:Reverse s + let[@inline] kmp_force_ s = ignore (kmp_get_ s : int array) + let[@inline] kmp_compile_ s = kmp_pattern_ ~dir:Direct s + let[@inline] kmp_rcompile_ s = kmp_pattern_ ~dir:Reverse s (* proper search function. [i] index in [s] @@ -206,17 +209,28 @@ module Find = struct | P_char _ -> 1 | P_KMP p -> kmp_pattern_length p - let compile sub : [ `Direct ] pattern = + let compile_ ~force sub : [ `Direct ] pattern = match length sub with | 0 -> P_empty | 1 -> P_char sub.[0] - | _ -> P_KMP (kmp_compile sub) + | _ -> + let p = kmp_compile_ sub in + if force then kmp_force_ p; + P_KMP p + + (** Careful to not use in multicore contexts, only local blocking uses *) + let compile_lazy sub = compile_ ~force:false sub + + let compile sub = compile_ ~force:true sub let rcompile sub : [ `Reverse ] pattern = match length sub with | 0 -> P_empty | 1 -> P_char sub.[0] - | _ -> P_KMP (kmp_rcompile sub) + | _ -> + let p = kmp_rcompile_ sub in + kmp_force_ p; + P_KMP p let find ?(start = 0) ~(pattern : [ `Direct ] pattern) s = match pattern with @@ -259,23 +273,22 @@ module Find = struct kmp_rfind ~pattern s start end -let find ?(start = 0) ~sub = - let pattern = Find.compile sub in - fun s -> Find.find ~start ~pattern s - -let find_all ?(start = 0) ~sub = - let pattern = Find.compile sub in - fun s -> - let i = ref start in - fun () -> - let res = Find.find ~start:!i ~pattern s in - if res = ~-1 then - None - else ( - i := res + 1; - (* possible overlap *) - Some res - ) +let find ?(start = 0) ~sub s = + let pattern = Find.compile_lazy sub in + Find.find ~start ~pattern s + +let find_all ?(start = 0) ~sub s = + let pattern = Find.compile_lazy sub in + let i = ref start in + fun () -> + let res = Find.find ~start:!i ~pattern s in + if res = ~-1 then + None + else ( + i := res + 1; + (* possible overlap *) + Some res + ) let find_all_l ?start ~sub s = let rec aux acc g = @@ -291,7 +304,7 @@ let rfind ~sub = let pattern = Find.rcompile sub in fun s -> Find.rfind ~pattern s -(* Replace substring [s.[pos] … s.[pos+len-1]] by [by] in [s] *) +(** Replace substring [s.[pos] … s.[pos+len-1]] by [by] in [s] *) let replace_at_ ~pos ~len ~by s = let b = Buffer.create (length s + length by - len) in Buffer.add_substring b s 0 pos; @@ -316,7 +329,7 @@ let replace ?(which = `All) ~sub ~by s = s | `All -> (* compile search pattern only once *) - let pattern = Find.compile sub in + let pattern = Find.compile_lazy sub in let b = Buffer.create (String.length s) in let start = ref 0 in @@ -358,10 +371,10 @@ module Split = struct mutable len: int; (** Length of the current result in [s] *) } - let _make_state ~by s : split_state = + let _make_state ~force ~by s : split_state = { s; - by = Find.compile by; + by = Find.compile_ ~force by; stopped = false; is_first = true; is_last = false; @@ -404,7 +417,7 @@ module Split = struct let[@inline] _tuple3 x y z = x, y, z let _mkgen ~drop ~by s k : _ gen = - let st = _make_state ~by s in + let st = _make_state ~force:false ~by s in let rec next () = if not (_split st) then None @@ -419,7 +432,7 @@ module Split = struct let gen_cpy ?(drop = default_drop) ~by s = _mkgen ~drop ~by s String.sub let _mklist ~drop ~by s k : _ list = - let st = _make_state ~by s in + let st = _make_state ~force:false ~by s in let rec build acc = if not (_split st) then List.rev acc @@ -446,13 +459,15 @@ module Split = struct let st = { st with stopped = st.stopped } in make st in - delay (_make_state ~by s) + + (* make sure to force the KMP pattern now, so the Seq is thread-safe *) + delay (_make_state ~force:true ~by s) let seq ?(drop = default_drop) ~by s = _mkseq ~drop ~by s _tuple3 let seq_cpy ?(drop = default_drop) ~by s = _mkseq ~drop ~by s String.sub let _mk_iter ~drop ~by s f k = - let st = _make_state ~by s in + let st = _make_state ~force:false ~by s in while _split st do if not (_should_drop ~drop st) then k (f s st.off st.len) done diff --git a/src/core/CCString.mli b/src/core/CCString.mli index 0118c3987..09f906bfe 100644 --- a/src/core/CCString.mli +++ b/src/core/CCString.mli @@ -109,6 +109,9 @@ val find : ?start:int -> sub:string -> string -> int val find_all : ?start:int -> sub:string -> string -> int gen (** [find_all ~start ~sub s] finds all occurrences of [sub] in [s], even overlapping instances and returns them in a generator [gen]. + + {b NOTE}: the generator is NOT multicore safe. Do not share between threads/domains. + @param start starting position in [s]. Returns all positions between start and the end of [s] if [sub=""] @since 0.17 *) @@ -403,7 +406,8 @@ module Split : sig val gen : ?drop:drop_if_empty -> by:string -> string -> (string * int * int) gen (** [gen ~drop ~by s] splits the given string [s] along the given separator [by]. - Returns a [gen] of slices. *) + Returns a [gen] of slices. + {b NOTE}: the generator isn't thread safe. *) val iter : ?drop:drop_if_empty -> by:string -> string -> (string * int * int) iter @@ -414,7 +418,8 @@ module Split : sig val seq : ?drop:drop_if_empty -> by:string -> string -> (string * int * int) Seq.t (** [seq ~drop ~by s] splits the given string [s] along the given separator [by]. - Returns a [Seq.t] of slices. + Returns a [Seq.t] of slices. The sequence is thread-safe and can be shared + between threads or domains. Renamed from [std_seq] since 3.0. @since 3.0 *) @@ -429,7 +434,8 @@ module Split : sig val gen_cpy : ?drop:drop_if_empty -> by:string -> string -> string gen (** [gen_cpy ~drop ~by s] splits the given string [s] along the given separator [by]. - Returns a [gen] of strings. *) + Returns a [gen] of strings. + {b NOTE}: the generator isn't thread safe. *) val iter_cpy : ?drop:drop_if_empty -> by:string -> string -> string iter (** [iter_cpy ~drop ~by s] splits the given string [s] along the given separator [by]. @@ -438,7 +444,8 @@ module Split : sig val seq_cpy : ?drop:drop_if_empty -> by:string -> string -> string Seq.t (** [seq_cpy ~drop ~by s] splits the given string [s] along the given separator [by]. - Returns a [Seq.t] of strings. + Returns a [Seq.t] of strings. The sequence is thread-safe and can be shared + between threads or domains. Renamed from [std_seq_cpy] since 3.0. @since 3.0 *)