Commit 46b5387d authored by Robbert Krebbers's avatar Robbert Krebbers
Browse files

Support nat tokens in the tokenizer.

parent 33ca3791
coq-stdpp https://gitlab.mpi-sws.org/robbertkrebbers/coq-stdpp fd8a6717a2610c00dec920ee5f33b00b14ef1fbf coq-stdpp https://gitlab.mpi-sws.org/robbertkrebbers/coq-stdpp 6ffa20c8f23797f81f1bb55ab27432e897de71d5
...@@ -3,6 +3,7 @@ Set Default Proof Using "Type". ...@@ -3,6 +3,7 @@ Set Default Proof Using "Type".
Inductive token := Inductive token :=
| TName : string token | TName : string token
| TNat : nat token
| TAnom : token | TAnom : token
| TFrame : token | TFrame : token
| TBar : token | TBar : token
...@@ -26,42 +27,64 @@ Inductive token := ...@@ -26,42 +27,64 @@ Inductive token :=
| TMinus : token | TMinus : token
| TSep : token. | TSep : token.
Fixpoint cons_name (kn : string) (k : list token) : list token := Inductive state :=
match kn with "" => k | _ => TName (string_rev kn) :: k end. | SName : string state
Fixpoint tokenize_go (s : string) (k : list token) (kn : string) : list token := | SNat : nat state
| SNone : state.
Definition cons_state (kn : state) (k : list token) : list token :=
match kn with
| SNone => k
| SName s => TName (string_rev s) :: k
| SNat n => TNat n :: k
end.
Fixpoint tokenize_go (s : string) (k : list token) (kn : state) : list token :=
match s with match s with
| "" => reverse (cons_name kn k) | "" => reverse (cons_state kn k)
| String "?" s => tokenize_go s (TAnom :: cons_name kn k) "" | String "?" s => tokenize_go s (TAnom :: cons_state kn k) SNone
| String "$" s => tokenize_go s (TFrame :: cons_name kn k) "" | String "$" s => tokenize_go s (TFrame :: cons_state kn k) SNone
| String "[" s => tokenize_go s (TBracketL :: cons_name kn k) "" | String "[" s => tokenize_go s (TBracketL :: cons_state kn k) SNone
| String "]" s => tokenize_go s (TBracketR :: cons_name kn k) "" | String "]" s => tokenize_go s (TBracketR :: cons_state kn k) SNone
| String "|" s => tokenize_go s (TBar :: cons_name kn k) "" | String "|" s => tokenize_go s (TBar :: cons_state kn k) SNone
| String "(" s => tokenize_go s (TParenL :: cons_name kn k) "" | String "(" s => tokenize_go s (TParenL :: cons_state kn k) SNone
| String ")" s => tokenize_go s (TParenR :: cons_name kn k) "" | String ")" s => tokenize_go s (TParenR :: cons_state kn k) SNone
| String "&" s => tokenize_go s (TAmp :: cons_name kn k) "" | String "&" s => tokenize_go s (TAmp :: cons_state kn k) SNone
| String "{" s => tokenize_go s (TBraceL :: cons_name kn k) "" | String "{" s => tokenize_go s (TBraceL :: cons_state kn k) SNone
| String "}" s => tokenize_go s (TBraceR :: cons_name kn k) "" | String "}" s => tokenize_go s (TBraceR :: cons_state kn k) SNone
| String "%" s => tokenize_go s (TPure :: cons_name kn k) "" | String "%" s => tokenize_go s (TPure :: cons_state kn k) SNone
| String "#" s => tokenize_go s (TAlways :: cons_name kn k) "" | String "#" s => tokenize_go s (TAlways :: cons_state kn k) SNone
| String ">" s => tokenize_go s (TModal :: cons_name kn k) "" | String ">" s => tokenize_go s (TModal :: cons_state kn k) SNone
| String "!" (String "%" s) => tokenize_go s (TPureIntro :: cons_name kn k) "" | String "!" (String "%" s) => tokenize_go s (TPureIntro :: cons_state kn k) SNone
| String "!" (String "#" s) => tokenize_go s (TAlwaysIntro :: cons_name kn k) "" | String "!" (String "#" s) => tokenize_go s (TAlwaysIntro :: cons_state kn k) SNone
| String "!" (String ">" s) => tokenize_go s (TModalIntro :: cons_name kn k) "" | String "!" (String ">" s) => tokenize_go s (TModalIntro :: cons_state kn k) SNone
| String "/" (String "/" (String "=" s)) => | String "/" (String "/" (String "=" s)) =>
tokenize_go s (TSimpl :: TDone :: cons_name kn k) "" tokenize_go s (TSimpl :: TDone :: cons_state kn k) SNone
| String "/" (String "/" s) => tokenize_go s (TDone :: cons_name kn k) "" | String "/" (String "/" s) => tokenize_go s (TDone :: cons_state kn k) SNone
| String "/" (String "=" s) => tokenize_go s (TSimpl :: cons_name kn k) "" | String "/" (String "=" s) => tokenize_go s (TSimpl :: cons_state kn k) SNone
| String "*" (String "*" s) => tokenize_go s (TAll :: cons_name kn k) "" | String "*" (String "*" s) => tokenize_go s (TAll :: cons_state kn k) SNone
| String "*" s => tokenize_go s (TForall :: cons_name kn k) "" | String "*" s => tokenize_go s (TForall :: cons_state kn k) SNone
| String "-" s => tokenize_go s (TMinus :: cons_name kn k) "" | String "-" s => tokenize_go s (TMinus :: cons_state kn k) SNone
| String (Ascii.Ascii false true false false false true true true) (* unicode ∗ *) | String (Ascii.Ascii false true false false false true true true) (* unicode ∗ *)
(String (Ascii.Ascii false false false true false false false true) (String (Ascii.Ascii false false false true false false false true)
(String (Ascii.Ascii true true true false true false false true) s)) => (String (Ascii.Ascii true true true false true false false true) s)) =>
tokenize_go s (TSep :: cons_name kn k) "" tokenize_go s (TSep :: cons_state kn k) SNone
| String a s => | String a s =>
if is_space a then tokenize_go s (cons_name kn k) "" (* TODO: Complain about invalid characters, to future-proof this
else tokenize_go s k (String a kn) against making more characters special. *)
(* TODO: Complain about invalid characters, to future-proof this if is_space a then tokenize_go s (cons_state kn k) SNone else
against making more characters special. *) match kn with
| SNone =>
match is_nat a with
| Some n => tokenize_go s k (SNat n)
| None => tokenize_go s k (SName (String a ""))
end
| SName s' => tokenize_go s k (SName (String a s'))
| SNat n =>
match is_nat a with
| Some n' => tokenize_go s k (SNat (n' + 10 * n))
| None => tokenize_go s (TNat n :: k) (SName (String a ""))
end
end
end. end.
Definition tokenize (s : string) : list token := tokenize_go s [] "". Definition tokenize (s : string) : list token := tokenize_go s [] SNone.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment