This commit is contained in:
swrup 2025-11-11 02:07:51 +01:00
parent aa2ff7b2f0
commit 2f3113f55d
11742 changed files with 1223940 additions and 0 deletions

View file

@ -0,0 +1,21 @@
The MIT License
Copyright (c) 2016 Jane Street Group, LLC <opensource@janestreet.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View file

@ -0,0 +1,418 @@
module type Sexp = sig
type t =
| Atom of string
| List of t list
end
module type Monad = sig
type 'a t
val return : 'a -> 'a t
val bind : 'a t -> ('a -> 'b t) -> 'b t
end
module type S = sig
type sexp
val parse_string : string -> (sexp, int * string) result
val parse_string_many : string -> (sexp list, int * string) result
val input : in_channel -> (sexp, string) result
val input_opt : in_channel -> (sexp option, string) result
val input_many : in_channel -> (sexp list, string) result
val serialised_length : sexp -> int
val to_string : sexp -> string
val to_buffer : Buffer.t -> sexp -> unit
val to_channel : out_channel -> sexp -> unit
module Parser : sig
exception Parse_error of string
val premature_end_of_input : string
module Lexer : sig
type t
val create : unit -> t
type _ token =
| Await : [> `other ] token
| Lparen : [> `other ] token
| Rparen : [> `other ] token
| Atom : int -> [> `atom ] token
val feed : t -> char -> [ `other | `atom ] token
val feed_eoi : t -> unit
end
module Stack : sig
type t =
| Empty
| Open of t
| Sexp of sexp * t
val to_list : t -> sexp list
val open_paren : t -> t
val close_paren : t -> t
val add_atom : string -> t -> t
val add_token : [ `other ] Lexer.token -> t -> t
end
end
module type Input = sig
type t
module Monad : sig
type 'a t
val return : 'a -> 'a t
val bind : 'a t -> ('a -> 'b t) -> 'b t
end
val read_string : t -> int -> (string, string) result Monad.t
val read_char : t -> (char, string) result Monad.t
end
[@@deprecated "Use Parser module instead"]
[@@@warning "-3"]
module Make_parser (Input : Input) : sig
val parse : Input.t -> (sexp, string) result Input.Monad.t
val parse_many : Input.t -> (sexp list, string) result Input.Monad.t
end
[@@deprecated "Use Parser module instead"]
end
module Make (Sexp : Sexp) = struct
open Sexp
module Parser = struct
exception Parse_error of string
let parse_error msg = raise (Parse_error msg)
let parse_errorf f = Format.ksprintf parse_error f
let premature_end_of_input = "premature end of input"
module Lexer = struct
type state =
| Init
| Parsing_length
type t =
{ mutable state : state
; mutable n : int
}
let create () = { state = Init; n = 0 }
let int_of_digit c = Char.code c - Char.code '0'
type _ token =
| Await : [> `other ] token
| Lparen : [> `other ] token
| Rparen : [> `other ] token
| Atom : int -> [> `atom ] token
let feed t c =
match (t.state, c) with
| Init, '(' -> Lparen
| Init, ')' -> Rparen
| Init, '0' .. '9' ->
t.state <- Parsing_length;
t.n <- int_of_digit c;
Await
| Init, _ ->
parse_errorf "invalid character %C, expected '(', ')' or '0'..'9'" c
| Parsing_length, '0' .. '9' ->
let len = (t.n * 10) + int_of_digit c in
if len > Sys.max_string_length then
parse_error "atom too big to represent"
else (
t.n <- len;
Await)
| Parsing_length, ':' ->
t.state <- Init;
Atom t.n
| Parsing_length, _ ->
parse_errorf
"invalid character %C while parsing atom length, expected '0'..'9' \
or ':'"
c
let feed_eoi t =
match t.state with
| Init -> ()
| Parsing_length -> parse_error premature_end_of_input
end
module L = Lexer
module Stack = struct
type t =
| Empty
| Open of t
| Sexp of Sexp.t * t
let open_paren stack = Open stack
let close_paren =
let rec loop acc = function
| Empty ->
parse_error "right parenthesis without matching left parenthesis"
| Sexp (sexp, t) -> loop (sexp :: acc) t
| Open t -> Sexp (List acc, t)
in
fun t -> loop [] t
let to_list =
let rec loop acc = function
| Empty -> acc
| Sexp (sexp, t) -> loop (sexp :: acc) t
| Open _ -> parse_error premature_end_of_input
in
fun t -> loop [] t
let add_atom s stack = Sexp (Atom s, stack)
let add_token (x : [ `other ] Lexer.token) stack =
match x with
| L.Await -> stack
| L.Lparen -> open_paren stack
| L.Rparen -> close_paren stack
end
end
open Parser
let feed_eoi_single lexer stack =
match
Lexer.feed_eoi lexer;
Stack.to_list stack
with
| exception Parse_error msg -> Error msg
| [ x ] -> Ok x
| [] -> Error premature_end_of_input
| _ :: _ :: _ -> assert false
let feed_eoi_many lexer stack =
match
Lexer.feed_eoi lexer;
Stack.to_list stack
with
| exception Parse_error msg -> Error msg
| l -> Ok l
let one_token s pos len lexer stack k =
match Lexer.feed lexer (String.unsafe_get s pos) with
| exception Parse_error msg -> Error (pos, msg)
| L.Atom atom_len -> (
match String.sub s (pos + 1) atom_len with
| exception _ -> Error (len, premature_end_of_input)
| atom ->
let pos = pos + 1 + atom_len in
k s pos len lexer (Stack.add_atom atom stack))
| (L.Await | L.Lparen | L.Rparen) as x -> (
match Stack.add_token x stack with
| exception Parse_error msg -> Error (pos, msg)
| stack -> k s (pos + 1) len lexer stack)
[@@inline always]
let parse_string =
let rec loop s pos len lexer stack =
if pos = len then
match feed_eoi_single lexer stack with
| Error msg -> Error (pos, msg)
| Ok _ as ok -> ok
else one_token s pos len lexer stack cont
and cont s pos len lexer stack =
match stack with
| Stack.Sexp (sexp, Empty) ->
if pos = len then Ok sexp
else Error (pos, "data after canonical S-expression")
| stack -> loop s pos len lexer stack
in
fun s -> loop s 0 (String.length s) (Lexer.create ()) Empty
let parse_string_many =
let rec loop s pos len lexer stack =
if pos = len then
match feed_eoi_many lexer stack with
| Error msg -> Error (pos, msg)
| Ok _ as ok -> ok
else one_token s pos len lexer stack loop
in
fun s -> loop s 0 (String.length s) (Lexer.create ()) Empty
let one_token ic c lexer stack =
match Lexer.feed lexer c with
| L.Atom n -> (
match really_input_string ic n with
| exception End_of_file -> raise (Parse_error premature_end_of_input)
| s -> Stack.add_atom s stack)
| (L.Await | L.Lparen | L.Rparen) as x -> Stack.add_token x stack
let input_opt =
let rec loop ic lexer stack =
let c = input_char ic in
match one_token ic c lexer stack with
| Sexp (sexp, Empty) -> Ok (Some sexp)
| stack -> loop ic lexer stack
in
fun ic ->
let lexer = Lexer.create () in
match input_char ic with
| exception End_of_file -> Ok None
| c -> (
try
match Lexer.feed lexer c with
| L.Atom _ -> assert false
| (L.Await | L.Lparen | L.Rparen) as x ->
loop ic lexer (Stack.add_token x Empty)
with
| Parse_error msg -> Error msg
| End_of_file -> Error premature_end_of_input)
let input ic =
match input_opt ic with
| Ok None -> Error premature_end_of_input
| Ok (Some x) -> Ok x
| Error msg -> Error msg
let input_many =
let rec loop ic lexer stack =
match input_char ic with
| exception End_of_file ->
Lexer.feed_eoi lexer;
Ok (Stack.to_list stack)
| c -> loop ic lexer (one_token ic c lexer stack)
in
fun ic ->
try loop ic (Lexer.create ()) Empty with Parse_error msg -> Error msg
let serialised_length =
let rec loop acc t =
match t with
| Atom s ->
let len = String.length s in
let x = ref len in
let len_len = ref 1 in
while !x > 9 do
x := !x / 10;
incr len_len
done;
acc + !len_len + 1 + len
| List l -> 2 + List.fold_left loop acc l
in
fun t -> loop 0 t
let to_buffer buf sexp =
let rec loop = function
| Atom str ->
Buffer.add_string buf (string_of_int (String.length str));
Buffer.add_string buf ":";
Buffer.add_string buf str
| List e ->
Buffer.add_char buf '(';
List.iter loop e;
Buffer.add_char buf ')'
in
loop sexp
let to_string sexp =
let buf = Buffer.create (serialised_length sexp) in
to_buffer buf sexp;
Buffer.contents buf
let to_channel oc sexp =
let rec loop = function
| Atom str ->
output_string oc (string_of_int (String.length str));
output_char oc ':';
output_string oc str
| List l ->
output_char oc '(';
List.iter loop l;
output_char oc ')'
in
loop sexp
module type Input = sig
type t
module Monad : Monad
val read_string : t -> int -> (string, string) result Monad.t
val read_char : t -> (char, string) result Monad.t
end
module Make_parser (Input : Input) = struct
open Input.Monad
let ( >>= ) = bind
let ( >>=* ) m f =
m >>= function
| Error _ as err -> return err
| Ok x -> f x
let one_token input c lexer stack =
match Lexer.feed lexer c with
| exception Parse_error msg -> return (Error msg)
| L.Atom n ->
Input.read_string input n >>=* fun s ->
return (Ok (Stack.add_atom s stack))
| (L.Await | L.Lparen | L.Rparen) as x ->
return
(match Stack.add_token x stack with
| exception Parse_error msg -> Error msg
| stack -> Ok stack)
let parse =
let rec loop input lexer stack =
Input.read_char input >>= function
| Error _ -> return (feed_eoi_single lexer stack)
| Ok c -> (
one_token input c lexer stack >>=* function
| Sexp (sexp, Empty) -> return (Ok sexp)
| stack -> loop input lexer stack)
in
fun input -> loop input (Lexer.create ()) Empty
let parse_many =
let rec loop input lexer stack =
Input.read_char input >>= function
| Error _ -> return (feed_eoi_many lexer stack)
| Ok c ->
one_token input c lexer stack >>=* fun stack -> loop input lexer stack
in
fun input -> loop input (Lexer.create ()) Empty
end
end
module T = struct
type t =
| Atom of string
| List of t list
end
include T
include Make (T)

View file

@ -0,0 +1,378 @@
(** Canonical S-expressions *)
(** This module provides minimal support for reading and writing S-expressions
in canonical form.
https://en.wikipedia.org/wiki/Canonical_S-expressions
Note that because the canonical representation of S-expressions is so
simple, this module doesn't go out of his way to provide a fully generic
parser and printer and instead just provides a few simple functions. If you
are using fancy input sources, simply copy the parser and adapt it. The
format is so simple that it's pretty difficult to get it wrong by accident.
To avoid a dependency on a particular S-expression library, the only module
of this library is parameterised by the type of S-expressions.
{[
let rec print = function
| Atom str -> Printf.printf "%d:%s" (String.length s)
| List l -> List.iter print l
]} *)
module type Sexp = sig
type t =
| Atom of string
| List of t list
end
module type S = sig
(** {2 Parsing} *)
type sexp
(** [parse_string s] parses a single S-expression encoded in canonical form in
[s]. It is an error for [s] to contain a S-expression followed by more
data. In case of error, the offset of the error as well as an error
message is returned. *)
val parse_string : string -> (sexp, int * string) result
(** [parse_string s] parses a sequence of S-expressions encoded in canonical
form in [s] *)
val parse_string_many : string -> (sexp list, int * string) result
(** Read exactly one canonical S-expressions from the given channel. Note that
this function never raises [End_of_file]. Instead, it returns [Error]. *)
val input : in_channel -> (sexp, string) result
(** Same as [input] but returns [Ok None] if the end of file has already been
reached. If some more characters are available but the end of file is
reached before reading a complete S-expression, this function returns
[Error]. *)
val input_opt : in_channel -> (sexp option, string) result
(** Read many S-expressions until the end of input is reached. *)
val input_many : in_channel -> (sexp list, string) result
(** {2 Serialising} *)
(** The length of the serialised representation of a S-expression *)
val serialised_length : sexp -> int
(** [to_string sexp] converts S-expression [sexp] to a string in canonical
form. *)
val to_string : sexp -> string
(** [to_buffer buf sexp] outputs the S-expression [sexp] converted to its
canonical form to buffer [buf]. *)
val to_buffer : Buffer.t -> sexp -> unit
(** [output oc sexp] outputs the S-expression [sexp] converted to its
canonical form to channel [oc]. *)
val to_channel : out_channel -> sexp -> unit
(** {3 Low level parser}
For efficiently parsing from sources other than strings or input channel.
For instance in Lwt or Async programs. *)
module Parser : sig
(** The [Parser] module offers an API that is a balance between sharing the
common logic of parsing canonical S-expressions while allowing to write
parsers that are as efficient as possible, both in terms of speed and
allocations. A carefully written parser using this API will be:
- fast
- perform minimal allocations
- perform zero [caml_modify] (a slow function of the OCaml runtime that
is emitted when mutating a constructed value)
{2 Lexers}
To parse using this API, you must first create a lexer via
{!Lexer.create}. The lexer is responsible for scanning the input and
forming tokens. The user must feed characters read from the input one by
one to the lexer until it yields a token. For instance:
{[
# let lexer = Lexer.create ();;
val lexer : Lexer.t = <abstract>
# Lexer.feed lexer '(';;
- : [ `atom | `other ] Lexer.token = Lparen
# Lexer.feed lexer ')';;
- : [ `atom | `other ] Lexer.token = Rparen
]}
When the lexer doesn't have enough to return a token, it simply returns
the special token {!Lexer.Await}:
{[
# Lexer.feed lexer '1';;
- : [ `atom | `other ] Lexer.token = Await
]}
Note that since atoms of canonical S-expressions do not need quoting,
they are always represented as a contiguous sequence of characters that
don't need further processing. To achieve maximum efficiency, the lexer
only returns the length of the atom and it is the responsibility of the
caller to extract the atom from the input source:
{[
# Lexer.feed lexer '2';;
- : [ `atom | `other ] Lexer.token = Await
# Lexer.feed lexer ':';;
- : [ `atom | `other ] Lexer.token = Atom 2
]}
When getting [Atom n], the caller should then proceed to read the next
[n] characters of the input as a string. For instance, if the input is
an [in_channel] the caller should proceed with
[really_input_string ic n].
Finally, when the end of input is reached the user should call
{!Lexer.feed_eoi} to make sure the lexer is not awaiting more input. If
that is the case, {!Lexer.feed_eoi} will raise:
{[
# Lexer.feed lexer '1';;
- : [ `atom | `other ] Lexer.token = Await
# Lexer.feed_eoi lexer;;
Exception: Parse_error "premature end of input".
]}
{2 Parsing stacks}
The lexer doesn't keep track of the structure of the S-expressions. In
order to construct a whole structured S-expressions, the caller must
maintain a parsing stack via the {!Stack} module. A {!Stack.t} value
simply represent a parsed prefix in reverse order.
For instance, the prefix "1:x((1:y1:z)" will be represented as:
{[
Sexp (List [ Atom "y"; Atom "z" ], Open (Sexp (Atom "x", Empty)))
]}
The {!Stack} module offers various primitives to open or close
parentheses or insert an atom. And for convenience it provides a
function {!Stack.add_token} that takes the output of {!Lexer.feed}
directly:
{[
# Stack.add_token Rparen Empty;;
- : Stack.t = Open Empty
# Stack.add_token Lparen (Open Empty);;
- : Stack.t = Sexp (List [], Empty)
]}
Note that {!Stack.add_token} doesn't accept [Atom _]. This is enforced
at the type level by a GADT. The reason for this is that in order to
insert an atom, the user must have fetched the contents of the atom
themselves. In order to insert an atom into a stack, you can use the
function {!Stack.add_atom}:
{[
# Stack.add_atom "foo" (Open Empty);;
- : Stack.t = Sexp (Atom "foo", Open Empty)
]}
When parsing is finished, one may call the function {!Stack.to_list} in
order to extract all the toplevel S-expressions from the stack:
{[
# Stack.to_list (Sexp (Atom "x", Sexp (List [Atom "y"], Empty)));;
- : sexp list = [List [Atom "y"; Atom "x"]]
]}
If instead you want to stop parsing as soon a single full S-expression
has been discovered, you can match on the structure of the stack. If the
stack is of the form [Sexp (_, Empty)], then you know that exactly one
S-expression has been parsed and you can stop there.
{2 Parsing errors}
In order to reduce allocations to a minumim, parsing errors are reported
via the exception {!Parse_error}. It is the responsibility of the caller
to catch this exception and return it as an [Error _] value. Functions
that may raise [Parse_error] are documented as such.
When extracting an atom and the input doesn't have enough characters
left, the user may raise [Parse_error premature_end_of_input]. This will
produce an error message similar to what the various high-level
functions of this library produce.
{2 Building a parsing function}
Parsing functions should always follow the following pattern:
+ create a lexer and start with an empty parsing stack
+ iterate over the input, feeding the lexer characters one by one. When
the lexer returns [Atom n], fetch the next [n] characters from the
input to form an atom
+ update the stack via [Stack.add_atom] or [Stack.add_token]
+ if parsing the whole input, call [Lexer.feed_eoi] when the end of
input is reached, otherwise stop as soon as the stack is of the form
[Sexp (_, Empty)] -
For instance, to parse a string as a list of S-expressions:
{[
module Sexp = struct
type t =
| Atom of string
| List of t list
end
module Csexp = Csexp.Make (Sexp)
let extract_atom s pos len =
match String.sub s pos len with
| exception _ ->
(* Turn out-of-bounds errors into [Parse_error] *)
raise (Parse_error premature_end_of_input)
| s -> s
let parse_string =
let open Csexp.Parser in
let rec loop s pos len lexer stack =
if pos = len then (
Lexer.feed_eoi lexer;
Stack.to_list stack)
else
match Lexer.feed lexer (String.unsafe_get s pos) with
| Atom atom_len ->
let atom = extract_atom s (pos + 1) atom_len in
loop s (pos + 1 + atom) len lexer (Stack.add_atom atom stack)
| (Await | Lparen | Rparen) as x ->
loop s (pos + 1) len lexer (Stack.add_token x stack)
in
fun s ->
match loop s 0 (String.length s) (Lexer.create ()) Empty with
| v -> Ok v
| exception Parse_error msg -> Error msg
]} *)
exception Parse_error of string
(** Error message signaling the end of input was reached prematurely. You
can use this when extracting an atom from the input and the input
doesn't have enough characters. *)
val premature_end_of_input : string
module Lexer : sig
(** Lexical analyser *)
type t
val create : unit -> t
type _ token =
| Await : [> `other ] token
| Lparen : [> `other ] token
| Rparen : [> `other ] token
| Atom : int -> [> `atom ] token
(** Feed a character to the parser.
@raise Parse_error *)
val feed : t -> char -> [ `other | `atom ] token
(** Feed the end of input to the parser.
You should call this function when the end of input has been reached
in order to ensure that the lexer is not awaiting more input, which
would be an error.
@raise Parse_error if the lexer is awaiting more input *)
val feed_eoi : t -> unit
end
module Stack : sig
(** Parsing stack *)
type t =
| Empty
| Open of t
| Sexp of sexp * t
(** Extract the list of full S-expressions contained in a stack.
For instance:
{[
# to_list (Sexp (Atom "y", Sexp (Atom "x", Empty)));;
- : Stack.t list = [Atom "x"; Atom "y"]
]}
@raise Parse_error
if the stack contains open parentheses that has not been closed. *)
val to_list : t -> sexp list
(** Add a left parenthesis. *)
val open_paren : t -> t
(** Add a right parenthesis. Raise [Parse_error] if the stack contains no
opened parentheses.
For instance:
{[
# close_paren (Sexp (Atom "y", Sexp (Atom "x", Open Empty)));;
- : Stack.t = Sexp (List [Atom "x"; Atom "y"], Empty)
]}
@raise Parse_error if the stack contains no open open parenthesis. *)
val close_paren : t -> t
(** Insert an atom in the parsing stack:
{[
# add_atom "foo" Empty;;
- : Stack.t = Sexp (Atom "foo", Empty)
]} *)
val add_atom : string -> t -> t
(** Add a token as returned by the lexer.
@raise Parse_error *)
val add_token : [ `other ] Lexer.token -> t -> t
end
end
(** {3 Deprecated low-level parser} *)
(** The above are deprecated as the {!Input} signature does not allow to
distinguish between IO errors and end of input conditions. Additionally,
the use of monads tend to produce parsers that allocates a lot.
It is recommended to use the {!Parser} module instead. *)
module type Input = sig
type t
module Monad : sig
type 'a t
val return : 'a -> 'a t
val bind : 'a t -> ('a -> 'b t) -> 'b t
end
val read_string : t -> int -> (string, string) result Monad.t
val read_char : t -> (char, string) result Monad.t
end
[@@deprecated "Use Parser module instead"]
[@@@warning "-3"]
module Make_parser (Input : Input) : sig
val parse : Input.t -> (sexp, string) result Input.Monad.t
val parse_many : Input.t -> (sexp list, string) result Input.Monad.t
end
[@@deprecated "Use Parser module instead"]
end
module Make (Sexp : Sexp) : S with type sexp := Sexp.t
include Sexp
include S with type sexp := t

View file

@ -0,0 +1,5 @@
(library
(name csexp)
;; note that this library is deleted in release mode so that the one from opam
;; is used
(public_name stdune.csexp))