coollang/lib/token.ml

(* 토큰 정의. 위치는 진단 품질이 헌법급이므로 모든 토큰이 들고 다닌다. *)

type pos = { line : int; col : int }

type kind =
  (* 리터럴과 이름 *)
  | Ident of string
  | Int of string
  | Str of string
  (* 키워드 *)
  | Kw_pub
  | Kw_fn
  | Kw_struct
  | Kw_enum
  | Kw_capability
  | Kw_const
  | Kw_import
  | Kw_as
  | Kw_reexport
  | Kw_let
  | Kw_mut
  | Kw_own
  | Kw_affine
  | Kw_copyable
  | Kw_effects
  | Kw_return
  | Kw_if
  | Kw_else
  | Kw_match
  | Kw_scope
  | Kw_true
  | Kw_false
  (* 구두점 *)
  | LParen
  | RParen
  | LBrace
  | RBrace
  | LBracket
  | RBracket
  | Comma
  | Colon
  | Dot
  | Arrow
  | FatArrow
  | Question
  | Underscore
  | Eq
  | EqEq
  | Bang
  | BangEq
  | Lt
  | Le
  | Gt
  | Ge
  | Plus
  | Minus
  | Star
  | Slash
  | Percent
  | AmpAmp
  | PipePipe
  | Pipe
  (* 렉서가 삽입하는 문 구분자 *)
  | Newline
  | Eof

type t = { kind : kind; pos : pos }

let keyword = function
  | "pub" -> Some Kw_pub
  | "fn" -> Some Kw_fn
  | "struct" -> Some Kw_struct
  | "enum" -> Some Kw_enum
  | "capability" -> Some Kw_capability
  | "const" -> Some Kw_const
  | "import" -> Some Kw_import
  | "as" -> Some Kw_as
  | "reexport" -> Some Kw_reexport
  | "let" -> Some Kw_let
  | "mut" -> Some Kw_mut
  | "own" -> Some Kw_own
  | "affine" -> Some Kw_affine
  | "copyable" -> Some Kw_copyable
  | "effects" -> Some Kw_effects
  | "return" -> Some Kw_return
  | "if" -> Some Kw_if
  | "else" -> Some Kw_else
  | "match" -> Some Kw_match
  | "scope" -> Some Kw_scope
  | "true" -> Some Kw_true
  | "false" -> Some Kw_false
  | _ -> None

let show_kind = function
  | Ident s -> Printf.sprintf "이름 %s" s
  | Int s -> Printf.sprintf "정수 %s" s
  | Str _ -> "문자열"
  | Kw_pub -> "pub"
  | Kw_fn -> "fn"
  | Kw_struct -> "struct"
  | Kw_enum -> "enum"
  | Kw_capability -> "capability"
  | Kw_const -> "const"
  | Kw_import -> "import"
  | Kw_as -> "as"
  | Kw_reexport -> "reexport"
  | Kw_let -> "let"
  | Kw_mut -> "mut"
  | Kw_own -> "own"
  | Kw_affine -> "affine"
  | Kw_copyable -> "copyable"
  | Kw_effects -> "effects"
  | Kw_return -> "return"
  | Kw_if -> "if"
  | Kw_else -> "else"
  | Kw_match -> "match"
  | Kw_scope -> "scope"
  | Kw_true -> "true"
  | Kw_false -> "false"
  | LParen -> "("
  | RParen -> ")"
  | LBrace -> "{"
  | RBrace -> "}"
  | LBracket -> "["
  | RBracket -> "]"
  | Comma -> ","
  | Colon -> ":"
  | Dot -> "."
  | Arrow -> "->"
  | FatArrow -> "=>"
  | Question -> "?"
  | Underscore -> "_"
  | Eq -> "="
  | EqEq -> "=="
  | Bang -> "!"
  | BangEq -> "!="
  | Lt -> "<"
  | Le -> "<="
  | Gt -> ">"
  | Ge -> ">="
  | Plus -> "+"
  | Minus -> "-"
  | Star -> "*"
  | Slash -> "/"
  | Percent -> "%"
  | AmpAmp -> "&&"
  | PipePipe -> "||"
  | Pipe -> "|"
  | Newline -> "줄바꿈"
  | Eof -> "파일 끝"

let show { kind; pos } =
  Printf.sprintf "%d:%d %s" pos.line pos.col (show_kind kind)

(* 모든 토큰 종류를 한 번씩 잇는 사슬. 아래 match가 전체 나열이므로 토큰을
   추가하면 컴파일러가 여기를 지적한다 — 목록이 조용히 낡지 않는다.
   문법 문서의 어휘 절이 이 나열에서 생성된다. *)
let next_kind = function
  | Ident _ -> Some (Int "")
  | Int _ -> Some (Str "")
  | Str _ -> Some Kw_pub
  | Kw_pub -> Some Kw_fn
  | Kw_fn -> Some Kw_struct
  | Kw_struct -> Some Kw_enum
  | Kw_enum -> Some Kw_capability
  | Kw_capability -> Some Kw_const
  | Kw_const -> Some Kw_import
  | Kw_import -> Some Kw_as
  | Kw_as -> Some Kw_reexport
  | Kw_reexport -> Some Kw_let
  | Kw_let -> Some Kw_mut
  | Kw_mut -> Some Kw_own
  | Kw_own -> Some Kw_affine
  | Kw_affine -> Some Kw_copyable
  | Kw_copyable -> Some Kw_effects
  | Kw_effects -> Some Kw_return
  | Kw_return -> Some Kw_if
  | Kw_if -> Some Kw_else
  | Kw_else -> Some Kw_match
  | Kw_match -> Some Kw_scope
  | Kw_scope -> Some Kw_true
  | Kw_true -> Some Kw_false
  | Kw_false -> Some LParen
  | LParen -> Some RParen
  | RParen -> Some LBrace
  | LBrace -> Some RBrace
  | RBrace -> Some LBracket
  | LBracket -> Some RBracket
  | RBracket -> Some Comma
  | Comma -> Some Colon
  | Colon -> Some Dot
  | Dot -> Some Arrow
  | Arrow -> Some FatArrow
  | FatArrow -> Some Question
  | Question -> Some Underscore
  | Underscore -> Some Eq
  | Eq -> Some EqEq
  | EqEq -> Some Bang
  | Bang -> Some BangEq
  | BangEq -> Some Lt
  | Lt -> Some Le
  | Le -> Some Gt
  | Gt -> Some Ge
  | Ge -> Some Plus
  | Plus -> Some Minus
  | Minus -> Some Star
  | Star -> Some Slash
  | Slash -> Some Percent
  | Percent -> Some AmpAmp
  | AmpAmp -> Some PipePipe
  | PipePipe -> Some Pipe
  | Pipe -> Some Newline
  | Newline -> Some Eof
  | Eof -> None

let all_kinds =
  let rec go k acc =
    match next_kind k with None -> List.rev (k :: acc) | Some n -> go n (k :: acc)
  in
  go (Ident "") []

(* 줄 끝에서 문 구분자를 삽입할지 결정한다 (grammar.ebnf 어휘 절).
   값으로 끝날 수 있는 토큰 뒤에서만 삽입하므로, 연산자나 여는 괄호로
   끝나는 줄은 다음 줄로 이어진다. *)
let can_end_statement = function
  | Ident _ | Int _ | Str _ | RParen | RBracket | RBrace | Question | Kw_return
  | Kw_true | Kw_false ->
      true
  | _ -> false