coolguy/doslang-mirror
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Ferro 파서를 Ferro 로, 그리고 그것이 드러낸 네 가지

Browse Source 
렉서 다음은 파서다. 노드는 한 배열에 살고 자식은 그 안의 인덱스다 -- 노드는
^Node 를 들 수 없고(여럿이며 한 번씩 소유하지 않는다) &Node 도 들 수 없다(R4).
인덱스는 둘 다 아니다. 소스도 필드가 아니라 매 단계에 같이 다닌다.

  unit demo / fn answer @2 / let n = (+ 1 (* 2 3)) / return n / balanced

전위 표기로 다시 찍는 것이 시험의 요점이다. 1 + 2 * 3 이 어떻게 묶였는지는
그렇게만 보인다.

쓰면서 나온 컴파일러 버그 넷:

1. 다른 유닛의 타입을 필드로 쓰면 그 필드 타입이 영영 UNKNOWN 이었다. 필드
   해석이 유닛마다 선언 직후에 돌아서, 아직 선언되지 않은 유닛의 타입을 찾다
   실패하고 그 답을 굳혔다. 이제 모든 유닛이 선언을 마친 뒤에 한 번 푼다.

2. 그리고 그 해석은 타입을 선언한 유닛에서 해야 한다. 필드 타입은 그 유닛의
   import 로 쓰였는데 아무 유닛에서나 풀고 있었다. 타입 계층에 enter/leave
   콜백을 두고 체커가 그 자리로 데려간다.

3. cycle_state 를 재귀 검사와 크기 계산이 같이 썼다. 첫 번째가 보는 중인 구조체
   가 두 번째에게는 다 끝난 것으로 보여서, 필드가 하나뿐인 것처럼 1 바이트로
   자리를 잡았다 -- Parser 가 그래서 자기 토큰을 밟았다. layout_state 로 나눴다.

4. 다른 유닛의 상수(ast.NONE)를 lowering 이 필드 접근으로 봤다. 체커가 이미
   링크 이름을 붙여두었으니 그것이 있으면 전역이다.

그리고 R1 을 실제로 지키게 했다: 소유자를 놓으면 그것이 가진 것도 놓는다.
전에는 자기 drop 이 있거나 자기가 owned 일 때만이어서, drop 을 가진 타입을
필드로 담은 구조체는 그것을 놓을 방법이 없었다(drop 은 손으로 못 부른다).
이제 release_at 이 drop 을 부르고 필드로 내려간다. 그 덕에 List/Arena/Map 의
drop 이 전부 필요 없어져서 지웠다 -- 버퍼가 owned 이니 R1 이 알아서 한다.

221/221, 29/29.
This commit is contained in:
coolguy
2026-08-17 12:49:53 +09:00
parent 8c4e80e246
commit e84f892147
15 changed files with 582 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fec/src/check.c
+35
View File
@@ -284,6 +284,9 @@ void enter_unit(FeCheck *c, unsigned index)
static int enter_decl_hook(void *owner, const char *unit);
static void leave_decl_hook(void *owner, int back);
void fe_check_init(FeCheck *c, FeBuild *build, FeDiags *diags,
unsigned pointer_bits, int no_checks)
{
@@ -301,6 +304,8 @@ void fe_check_init(FeCheck *c, FeBuild *build, FeDiags *diags,
c->types.unit_name = "unit";
c->types.instantiate = instantiate_type_node;
c->types.instantiate_owner = c;
c->types.enter_decl = enter_decl_hook;
c->types.leave_decl = leave_decl_hook;
c->instances = (FeInstance *)fe_arena_alloc(&c->arena,
(unsigned long)FE_GENERIC_INSTANCE_MAX * sizeof(FeInstance));
c->instance_count = 0;
@@ -354,6 +359,36 @@ FeType *unit_type(FeCheck *c, FeUnit *u, const char *name)
return 0;
}
/* A field type is written in the unit that declared the type, so it has to be
resolved with that unit's imports in scope -- not with whichever unit
happens to be current when the walk reaches it. Returns the index to go back
to, or -1 when there is nowhere to go. */
int enter_declaring_unit(FeCheck *c, const char *unit_name)
{
unsigned i;
unsigned here;
if (!unit_name || !c->build || !c->unit) return -1;
here = unit_index(c,c->unit);
for (i=0;i<c->build->count;++i)
if (strcmp(c->build->units[i].name,unit_name)==0) {
if (i==here) return -1;
enter_unit(c,i);
return (int)here;
}
return -1;
}
/* The type layer calls these; it knows nothing about units beyond a name. */
static int enter_decl_hook(void *owner, const char *unit)
{
return enter_declaring_unit((FeCheck *)owner, unit);
}
static void leave_decl_hook(void *owner, int back)
{
enter_unit((FeCheck *)owner, (unsigned)back);
}
/* The AST declaration of a type another unit declares, for its visibility and
for its methods. */
FeNode *unit_type_decl(FeCheck *c, FeUnit *u, const char *name)
fec/src/checkpri.h
+1
View File
@@ -104,6 +104,7 @@ FeUnit *binding_unit(FeCheckerState *s, FeNode *base);
int decl_is_public(const FeNode *decl);
FeSym *unit_member(FeCheck *c, FeUnit *u, const char *name);
FeType *unit_type(FeCheck *c, FeUnit *u, const char *name);
int enter_declaring_unit(FeCheck *c, const char *unit_name);
FeNode *unit_type_decl(FeCheck *c, FeUnit *u, const char *name);
FeType *node_type_in(FeCheck *c, const char *unit, FeNode *node);
FeNode *find_method(FeCheck *c, FeType *owner, const char *name);
fec/src/checkpro.c
+3 -1
View File
@@ -79,7 +79,6 @@ void declare_unit(FeCheck *c)
fe_type_declare_enum(&c->types,n);
for (n=c->ast->root ? c->ast->root->children : 0;n;n=n->next)
if (n->kind==FE_N_ERROR_DECL) fe_type_declare_error(&c->types,n);
check_type_cycles(c);
}
/* The unit's top-level names, in a scope of their own so that another unit
@@ -166,6 +165,9 @@ int fe_check_program(FeCheck *c)
s.fn_node=0;
fe_own_liveness_init(&s.liveness,&c->arena);
for (u=0;u<c->build->count;++u) { enter_unit(c,u); declare_unit(c); }
/* Only now: a field may name a type in a unit that had not declared it
yet, and resolving it early would freeze the wrong answer in place. */
for (u=0;u<c->build->count;++u) { enter_unit(c,u); check_type_cycles(c); }
fe_type_layout_all(&c->types);
for (u=0;u<c->build->count;++u) {
enter_unit(c,u);
fec/src/checkstm.c
+11 -4
View File
@@ -140,18 +140,25 @@ void check_type_cycle(FeCheck *c, FeType *t)
if (t->kind == FE_TYPE_ARRAY) {
check_type_cycle(c,t->elem);
} else if (t->kind == FE_TYPE_STRUCT) {
for (i=0;i<t->field_count;i++) {
int back=enter_declaring_unit(c,t->unit);
for (i=0;i<t->field_count;i++)
if (!t->fields[i].type && t->fields[i].ast_node)
t->fields[i].type=fe_type_from_ast(&c->types,t->fields[i].ast_node->a);
check_type_cycle(c,t->fields[i].type);
}
if (back>=0) enter_unit(c,(unsigned)back);
for (i=0;i<t->field_count;i++) check_type_cycle(c,t->fields[i].type);
} else if (t->kind == FE_TYPE_ENUM) {
int back=enter_declaring_unit(c,t->unit);
for (i=0;i<t->variant_count;i++) {
unsigned j;
for (j=0;j<t->variants[i].field_count;j++) {
for (j=0;j<t->variants[i].field_count;j++)
if (!t->variants[i].fields[j].type && t->variants[i].fields[j].ast_node)
t->variants[i].fields[j].type=fe_type_from_ast(&c->types,
t->variants[i].fields[j].ast_node->a);
}
if (back>=0) enter_unit(c,(unsigned)back);
for (i=0;i<t->variant_count;i++) {
unsigned j;
for (j=0;j<t->variants[i].field_count;j++) {
next=t->variants[i].fields[j].type;
check_type_cycle(c,next);
}
fec/src/lower.c
+10 -1
View File
@@ -113,9 +113,18 @@ unsigned as_address(Lower *L, Slot s, FeNode *n)
/* Does letting go of this type have to do something? */
int needs_release(const FeType *t)
{
unsigned i;
if (!t) return 0;
if (t->kind == FE_TYPE_OWNED) return 1;
return t->has_drop != 0;
if (t->has_drop) return 1;
/* SPEC 5 R1: letting go of an owner lets go of what it owns. A struct that
holds an owner has something to do even when it says nothing itself --
which is what lets one type hold another that has a `drop`, since
calling `drop` by hand is not allowed. */
if (t->kind == FE_TYPE_STRUCT)
for (i = 0; i < t->field_count; ++i)
if (needs_release(t->fields[i].type)) return 1;
return 0;
}
int lower_reserve(Lower *L, void **items, unsigned *capacity, unsigned needed,
fec/src/lowerexp.c
+38 -20
View File
@@ -171,6 +171,12 @@ Slot lower_expr_core(Lower *L, FeNode *n)
if (base && (base->kind == FE_TYPE_REF ||
base->kind == FE_TYPE_OWNED)) base = base->elem;
field = fe_type_field(base, n->b && n->b->text ? n->b->text : "");
/* `binding.name` is not a field of anything: it is a constant or a
global in another unit, and the checker already turned it into a
link name. */
if (!field && n->cname)
return slot_place(fe_ir_at_global(n->cname, 0), it,
ir_size(t));
if (!field) { fail(L, "an unresolved field", n); return slot_void(); }
b = lower_expr(L, n->a);
if (n->a->sem_type && (n->a->sem_type->kind == FE_TYPE_REF ||
@@ -305,6 +311,37 @@ const char *drop_name(Lower *L, const FeType *t)
return method->cname;
}
/* Let go of one value sitting at `at`. A type that says how to let go of
itself is asked first; then whatever it holds is let go of in turn, so a
struct that owns a struct that owns a buffer settles all three without
anyone writing a `drop` (SPEC 5 R1). */
void release_at(Lower *L, const FeType *t, FeIrPlace at)
{
unsigned args[1];
unsigned i;
if (!t) return;
if (t->has_drop) {
const char *how = drop_name(L, t);
args[0] = fe_ir_addr(L->m, L->b, at);
if (how) fe_ir_call(L->m, L->b, FE_IR_VOID, how, args, 1);
}
if (t->kind == FE_TYPE_OWNED) {
FeIrPlace p = at;
if (t->elem && t->elem->kind == FE_TYPE_SLICE)
p.offset += SLICE_PTR_OFFSET;
args[0] = fe_ir_load(L->m, L->b, FE_IR_PTR, p);
fe_ir_call(L->m, L->b, FE_IR_VOID, "fe_rt_free", args, 1);
return;
}
if (t->kind == FE_TYPE_STRUCT)
for (i = 0; i < t->field_count; ++i) {
FeIrPlace p = at;
if (!needs_release(t->fields[i].type)) continue;
p.offset += (long)t->fields[i].offset;
release_at(L, t->fields[i].type, p);
}
}
/* Settle what a scope owes, most recent first. A `return` in the middle of a
function still owes everything, so every exit path calls this. */
void run_deferred(Lower *L, unsigned from)
@@ -321,29 +358,10 @@ void run_deferred(Lower *L, unsigned from)
fe_ir_at_local(L->owed[i - 1].flag, 0));
FeIrBlock *doit = new_block(L);
FeIrBlock *skip = new_block(L);
unsigned args[1];
FeType *t = L->owed[i - 1].type;
fe_ir_br(L->b, live, doit->id, skip->id);
L->b = doit;
if (t && t->kind == FE_TYPE_OWNED && t->elem &&
t->elem->kind == FE_TYPE_SLICE) {
FeIrPlace at = fe_ir_at_local(L->owed[i - 1].local,
SLICE_PTR_OFFSET);
args[0] = fe_ir_load(L->m, L->b, FE_IR_PTR, at);
} else {
args[0] = fe_ir_load(L->m, L->b, FE_IR_PTR,
fe_ir_at_local(L->owed[i - 1].local, 0));
}
if (t && t->has_drop) {
/* A type that says how to let go of itself is asked to; the
name is the one its instance was given. */
const char *how = drop_name(L, t);
args[0] = fe_ir_addr(L->m, L->b,
fe_ir_at_local(L->owed[i - 1].local, 0));
if (how) fe_ir_call(L->m, L->b, FE_IR_VOID, how, args, 1);
} else {
fe_ir_call(L->m, L->b, FE_IR_VOID, "fe_rt_free", args, 1);
}
release_at(L, t, fe_ir_at_local(L->owed[i - 1].local, 0));
fe_ir_jmp(L->b, skip->id);
L->b = skip;
}
fec/src/lowerpri.h
+1
View File
@@ -145,6 +145,7 @@ Slot lower_call(Lower *L, FeNode *n);
Slot lower_expr(Lower *L, FeNode *n);
Slot lower_expr_core(Lower *L, FeNode *n);
const char *drop_name(Lower *L, const FeType *t);
void release_at(Lower *L, const FeType *t, FeIrPlace at);
void run_deferred(Lower *L, unsigned from);
Slot wrap_context(Lower *L, Slot v, FeNode *n);
unsigned wrapper_tag(Lower *L, Slot w, const FeType *t, FeNode *n);
fec/src/types.c
+52 -18
View File
@@ -47,6 +47,7 @@ static FeType *new_type(FeTypeCtx *ctx, const char *name, FeTypeKind kind)
t->next = ctx->types;
t->emit_state = 0;
t->cycle_state = 0;
t->layout_state = 0;
ctx->types = t;
return t;
}
@@ -61,6 +62,8 @@ void fe_types_init(FeTypeCtx *ctx, FeArena *arena, unsigned pointer_bits)
ctx->param_count = 0;
ctx->instantiate = 0;
ctx->instantiate_owner = 0;
ctx->enter_decl = 0;
ctx->leave_decl = 0;
}
/* Does this type answer to `name` for someone checking `unit`? A type with no
@@ -452,6 +455,26 @@ unsigned fe_type_align(const FeType *t)
return t && t->align ? t->align : 1U;
}
/* Resolve this type's fields where they were written. Without the callback
installed -- or for a type nobody declared -- everything stays where it is,
which is what the non-checking users of this layer want. */
static int enter_decl_unit(FeTypeCtx *ctx, const char *unit, const char **was)
{
*was = ctx->unit_name;
if (!ctx->enter_decl || !unit) return -1;
return ctx->enter_decl(ctx->instantiate_owner, unit);
}
/* Put back both halves: the unit the checker was in, and the name this layer
was interning under -- an instantiation moves the second without the
first, so restoring one is not restoring the other. */
static void leave_decl_unit(FeTypeCtx *ctx, int back, const char *was)
{
if (back >= 0 && ctx->leave_decl)
ctx->leave_decl(ctx->instantiate_owner, back);
ctx->unit_name = was;
}
static void layout_type(FeTypeCtx *ctx, FeType *t)
{
unsigned i;
@@ -460,14 +483,14 @@ static void layout_type(FeTypeCtx *ctx, FeType *t)
unsigned long max_size;
unsigned max_align;
if (!t || t->size) return;
if (t->cycle_state == 1) {
if (t->layout_state == 1) {
t->size = 1;
t->align = 1;
return;
}
t->cycle_state = 1;
t->layout_state = 1;
if (t->kind == FE_TYPE_VOID || t->kind == FE_TYPE_UNKNOWN ||
t->kind == FE_TYPE_ERROR) { t->size = 0; t->align = 1; t->cycle_state = 2; return; }
t->kind == FE_TYPE_ERROR) { t->size = 0; t->align = 1; t->layout_state = 2; return; }
if (t->kind == FE_TYPE_ERROR_UNION) {
if (t->error_value && t->error_value->kind != FE_TYPE_VOID) {
layout_type(ctx,t->error_value);
@@ -478,7 +501,7 @@ static void layout_type(FeTypeCtx *ctx, FeType *t)
t->size=2;
t->align=2U;
}
t->cycle_state = 2; return;
t->layout_state = 2; return;
}
if (t->kind == FE_TYPE_OPTIONAL) {
layout_type(ctx,t->elem);
@@ -490,44 +513,48 @@ static void layout_type(FeTypeCtx *ctx, FeType *t)
t->size=round_up(1UL,t->align)+fe_type_size(t->elem);
t->size=round_up(t->size,t->align);
}
t->cycle_state=2; return;
t->layout_state=2; return;
}
if (t->kind == FE_TYPE_BOOL || t->kind == FE_TYPE_CHAR) {
t->size = 1; t->align = 1; t->cycle_state = 2; return;
t->size = 1; t->align = 1; t->layout_state = 2; return;
}
if (t->kind == FE_TYPE_INT) {
t->size = (t->bits + 7U) / 8U;
t->align = t->size;
if (t->size > 4UL) t->size = 4UL;
t->cycle_state = 2; return;
t->layout_state = 2; return;
}
if (t->kind == FE_TYPE_REF || t->kind == FE_TYPE_RAW) {
t->size = FE_PTR_SIZE;
t->align = FE_PTR_ALIGN;
t->cycle_state = 2; return;
t->layout_state = 2; return;
}
if (t->kind == FE_TYPE_OWNED) {
t->size = t->elem && t->elem->kind==FE_TYPE_SLICE ?
2UL * FE_PTR_SIZE : FE_PTR_SIZE;
t->align = FE_PTR_ALIGN;
t->cycle_state = 2; return;
t->layout_state = 2; return;
}
if (t->kind == FE_TYPE_SLICE || t->kind == FE_TYPE_STR) {
t->size = 2UL * FE_PTR_SIZE;
t->align = FE_PTR_ALIGN;
t->cycle_state = 2; return;
t->layout_state = 2; return;
}
if (t->kind == FE_TYPE_ARRAY) {
layout_type(ctx, t->elem);
t->align = t->packed ? 1U : fe_type_align(t->elem);
t->size = t->length * fe_type_size(t->elem);
t->cycle_state = 2; return;
t->layout_state = 2; return;
}
if (t->kind == FE_TYPE_STRUCT) {
off = 0; max_align = 1;
for (i = 0; i < t->field_count; ++i) {
const char *was;
int back = enter_decl_unit(ctx, t->unit, &was);
for (i = 0; i < t->field_count; ++i)
if (!t->fields[i].type && t->fields[i].ast_node)
t->fields[i].type = fe_type_from_ast(ctx, t->fields[i].ast_node->a);
leave_decl_unit(ctx, back, was);
off = 0; max_align = 1;
for (i = 0; i < t->field_count; ++i) {
layout_type(ctx, t->fields[i].type);
align = t->packed ? 1U : fe_type_align(t->fields[i].type);
if (align > max_align) max_align = align;
@@ -537,18 +564,25 @@ static void layout_type(FeTypeCtx *ctx, FeType *t)
}
t->align = max_align;
t->size = round_up(off, max_align);
t->cycle_state = 2;
t->layout_state = 2;
return;
}
if (t->kind == FE_TYPE_ENUM) {
const char *was;
int back = enter_decl_unit(ctx, t->unit, &was);
for (i = 0; i < t->variant_count; ++i) {
unsigned j;
for (j = 0; j < t->variants[i].field_count; ++j)
if (!t->variants[i].fields[j].type && t->variants[i].fields[j].ast_node)
t->variants[i].fields[j].type = fe_type_from_ast(
ctx, t->variants[i].fields[j].ast_node->a);
}
leave_decl_unit(ctx, back, was);
max_size = 0; max_align = 1;
for (i = 0; i < t->variant_count; ++i) {
unsigned j;
off = 0;
for (j = 0; j < t->variants[i].field_count; ++j) {
if (!t->variants[i].fields[j].type && t->variants[i].fields[j].ast_node)
t->variants[i].fields[j].type = fe_type_from_ast(
ctx, t->variants[i].fields[j].ast_node->a);
layout_type(ctx, t->variants[i].fields[j].type);
if (fe_type_align(t->variants[i].fields[j].type) > max_align)
max_align = fe_type_align(t->variants[i].fields[j].type);
@@ -564,7 +598,7 @@ static void layout_type(FeTypeCtx *ctx, FeType *t)
off = round_up(t->bits / 8U, max_align);
t->size = round_up(off + max_size, max_align);
t->align = max_align;
t->cycle_state = 2;
t->layout_state = 2;
}
}
fec/src/types.h
+11
View File
@@ -98,6 +98,11 @@ struct FeType {
FeType *next;
int emit_state;
int cycle_state;
/* Separate from `cycle_state`: the checker's by-value recursion walk and
this layer's size computation run inside one another, and sharing one
marker made a struct in the middle of the first look complete to the
second -- one byte wide, with every field on top of the next. */
int layout_state;
};
typedef struct FeTypeCtx {
@@ -114,6 +119,12 @@ typedef struct FeTypeCtx {
so it installs this and the type layer calls back into it. */
FeType *(*instantiate)(void *owner, const FeNode *node);
void *instantiate_owner;
/* A field type is written in the unit that declared it, so resolving one
has to happen with that unit's imports in scope. The checker owns that
knowledge, so it installs this pair and the type layer calls back.
`enter` answers with what to hand `leave`, or -1 for "stayed put". */
int (*enter_decl)(void *owner, const char *unit);
void (*leave_decl)(void *owner, int back);
} FeTypeCtx;
void fe_types_init(FeTypeCtx *ctx, FeArena *arena, unsigned pointer_bits);
fec/std/list.fe
+4 -6
View File
@@ -1,8 +1,10 @@
unit std.list;
// A growable sequence. The buffer is owned, so a List owns its elements and
// releasing it releases them (SPEC 5 R1). Growth doubles, which keeps the
// total copying proportional to the number of pushes.
// releasing it releases them (SPEC 5 R1) -- which is also why there is no
// `drop` here: letting go of a List lets go of its buffer on its own. Growth
// doubles, which keeps the total copying proportional to the number of
// pushes.
pub struct List(T) {
items: ^[]mut T,
@@ -46,8 +48,4 @@ pub struct List(T) {
mem.destroy(old);
return;
}
pub fn drop(self: &mut Self) -> void {
mem.destroy(self.items);
}
}
fec/std/map.fe
+2 -6
View File
@@ -6,7 +6,8 @@ unit std.map;
// wrong shape for that. Keys are copied into one buffer the map owns and each
// slot records where in it the key sits -- the arena-and-handle shape R11 asks
// for, which also means letting go of the map is two frees and not one per
// entry.
// entry. Both buffers are owned, so R1 does the letting go and no `drop` is
// written here.
//
// Open addressing with linear probing. The table is a power of two so the
// index is a mask rather than a division, and it grows at three quarters full
@@ -155,11 +156,6 @@ pub struct Map(V) {
mem.destroy(old);
return;
}
pub fn drop(self: &mut Self) -> void {
mem.destroy(self.slots);
mem.destroy(self.bytes);
}
}
/// FNV-1a. Small, fast, and good enough for identifiers; nothing here has to
fec/std/mem.fe
+2 -2
View File
@@ -10,7 +10,8 @@ import std.sys;
/// SPEC R11 answers recursive and graph-shaped data with an arena that owns
/// the values and integer handles that reference them. This is that arena. A
/// handle is an offset, so it stays valid while the arena does, and comparing
/// two handles is comparing two numbers.
/// two handles is comparing two numbers. The block itself is owned, so nothing
/// here says how to let go of it -- R1 already does.
pub struct Arena {
bytes: ^[]mut u8,
used: usize,
@@ -56,5 +57,4 @@ pub struct Arena {
/// that is the trade an arena makes.
pub fn reset(self: &mut Self) -> void { self.used = 0; }
pub fn drop(self: &mut Self) -> void { mem.destroy(self.bytes); }
}
fec/tests/exec/lexer/ast.fe
+49
View File
@@ -0,0 +1,49 @@
unit ast;
// The tree the parser builds.
//
// Nodes live in one growing array and refer to each other by index, which is
// what SPEC R11 asks for: an owner that holds the values and handles that
// point at them. A node cannot hold a `^Node` for its children because a node
// has several and they are not each owned once; it cannot hold a `&Node`
// because R4 keeps borrows out of aggregate storage. An index is neither.
pub enum Shape {
Unit, // a: name
Fn, // a: name, b: first statement
Let, // a: name, b: value
Return, // a: value, or NONE
Binary, // a: left, b: right, from/len: the operator
Number,
Name,
Text,
Error,
}
/// No node. Zero is a real index, so the empty handle is the largest one.
pub const NONE: usize = 4294967295;
pub struct Node {
pub shape: Shape,
pub from: usize, // where in the source this came from
pub len: usize,
pub line: usize,
pub a: usize, // handles into the same tree
pub b: usize,
pub next: usize, // the following statement, when there is one
}
pub fn name_of(s: Shape) -> []u8 {
match s {
Unit => { return "unit"; }
Fn => { return "fn"; }
Let => { return "let"; }
Return => { return "return"; }
Binary => { return "binary"; }
Number => { return "number"; }
Name => { return "name"; }
Text => { return "text"; }
Error => { return "error"; }
}
return "?";
}
fec/tests/exec/lexer/parse.fe
+256
View File
@@ -0,0 +1,256 @@
unit parse;
import std.list;
import std.str;
import tok;
import scan;
import ast;
// Recursive descent over the lexer's tokens.
//
// The source is not a field: R4 keeps borrows out of aggregate storage, so
// `src` is passed to every step, the same way the lexer passes it. What the
// parser does own is the node array, and every parent points at its children
// by index into it.
//
// One token of lookahead lives in `cur`. That is all this grammar needs.
pub struct Parser {
nodes: list.List(ast.Node),
at: usize,
line: usize,
cur: tok.Token,
pub errors: usize,
pub fn on(src: []u8) -> !Self {
let nodes: list.List(ast.Node) =
try list.List(ast.Node).with_capacity(16);
var p: Self = Self{
nodes: nodes,
at: 0,
line: 1,
cur: tok.Token{ kind: tok.Kind.End, from: 0, len: 0, line: 1 },
errors: 0,
};
p.bump(src);
return p;
}
fn bump(self: &mut Self, src: []u8) -> void {
self.cur = scan.next(src, &mut self.at, &mut self.line);
return;
}
fn done(self: &Self) -> bool { return self.cur.kind == tok.Kind.End; }
/// Is the token in hand this exact spelling?
fn is(self: &Self, src: []u8, want: []u8) -> bool {
return str.eq(tok.text(src, self.cur), want);
}
/// Take the token in hand if it is this spelling; say whether it was.
fn eat(self: &mut Self, src: []u8, want: []u8) -> bool {
if not self.is(src, want) { return false; }
self.bump(src);
return true;
}
/// Demand this spelling. A miss is counted and the token stays put, so the
/// caller decides how to get back on its feet.
fn want(self: &mut Self, src: []u8, w: []u8) -> bool {
if self.eat(src, w) { return true; }
self.errors = self.errors + 1;
return false;
}
fn add(self: &mut Self, s: ast.Shape, t: tok.Token, a: usize,
b: usize) -> !usize {
let n: ast.Node = ast.Node{
shape: s, from: t.from, len: t.len, line: t.line,
a: a, b: b, next: ast.NONE,
};
let i: usize = self.nodes.count();
try self.nodes.push(n);
return i;
}
// -- reading the tree back ------------------------------------------
pub fn count(self: &Self) -> usize { return self.nodes.count(); }
pub fn node(self: &Self, i: usize) -> ast.Node {
return self.nodes.at(i);
}
// -- the grammar ----------------------------------------------------
//
// unit := "unit" NAME ";" item*
// item := "fn" NAME "(" ")" block
// block := "{" stmt* "}"
// stmt := "let" NAME "=" expr ";" | "return" expr? ";"
// expr := term (("+" | "-") term)*
// term := factor (("*" | "/") factor)*
// factor := NUMBER | NAME | TEXT | "(" expr ")"
pub fn unit_decl(self: &mut Self, src: []u8) -> !usize {
let ok: bool = self.want(src, "unit");
let name: tok.Token = self.cur;
if ok { self.bump(src); }
let semi: bool = self.want(src, ";");
let root: usize = try self.add(ast.Shape.Unit, name, ast.NONE,
ast.NONE);
var first: usize = ast.NONE;
var last: usize = ast.NONE;
while not self.done() {
let it: usize = try self.item(src);
if first == ast.NONE { first = it; }
else { self.link(last, it); }
last = it;
}
self.set_a(root, first);
return root;
}
/// Point one statement or item at the one after it.
fn link(self: &mut Self, from: usize, to: usize) -> void {
var n: ast.Node = self.nodes.at(from);
n.next = to;
self.nodes.set(from, n);
return;
}
fn set_a(self: &mut Self, at: usize, a: usize) -> void {
var n: ast.Node = self.nodes.at(at);
n.a = a;
self.nodes.set(at, n);
return;
}
fn item(self: &mut Self, src: []u8) -> !usize {
if not self.is(src, "fn") {
let bad: tok.Token = self.cur;
self.errors = self.errors + 1;
self.skip_stmt(src);
return try self.add(ast.Shape.Error, bad, ast.NONE, ast.NONE);
}
self.bump(src);
let name: tok.Token = self.cur;
self.bump(src);
let open: bool = self.want(src, "(");
let close: bool = self.want(src, ")");
let body: usize = try self.block(src);
return try self.add(ast.Shape.Fn, name, ast.NONE, body);
}
fn block(self: &mut Self, src: []u8) -> !usize {
let open: bool = self.want(src, "{");
var first: usize = ast.NONE;
var last: usize = ast.NONE;
while true {
if self.done() { break; }
if self.is(src, "}") { break; }
let s: usize = try self.stmt(src);
if first == ast.NONE { first = s; }
else { self.link(last, s); }
last = s;
}
let close: bool = self.want(src, "}");
return first;
}
fn stmt(self: &mut Self, src: []u8) -> !usize {
if self.is(src, "let") {
self.bump(src);
let name: tok.Token = self.cur;
self.bump(src);
let has_eq: bool = self.want(src, "=");
let value: usize = try self.expr(src);
let semi: bool = self.want(src, ";");
return try self.add(ast.Shape.Let, name, ast.NONE, value);
}
if self.is(src, "return") {
let head: tok.Token = self.cur;
self.bump(src);
if self.is(src, ";") {
self.bump(src);
return try self.add(ast.Shape.Return, head, ast.NONE,
ast.NONE);
}
let value: usize = try self.expr(src);
let semi: bool = self.want(src, ";");
return try self.add(ast.Shape.Return, head, value, ast.NONE);
}
let bad: tok.Token = self.cur;
self.errors = self.errors + 1;
self.skip_stmt(src);
return try self.add(ast.Shape.Error, bad, ast.NONE, ast.NONE);
}
/// Get back to a statement boundary after something unrecognised. Stopping
/// at `;` or `}` means one bad statement costs one diagnostic, not a run
/// of them.
fn skip_stmt(self: &mut Self, src: []u8) -> void {
while not self.done() {
if self.is(src, "}") { return; }
if self.is(src, ";") { self.bump(src); return; }
self.bump(src);
}
return;
}
fn expr(self: &mut Self, src: []u8) -> !usize {
var left: usize = try self.term(src);
while true {
if self.done() { break; }
let plus: bool = self.is(src, "+");
let minus: bool = self.is(src, "-");
if not plus and not minus { break; }
let op: tok.Token = self.cur;
self.bump(src);
let right: usize = try self.term(src);
left = try self.add(ast.Shape.Binary, op, left, right);
}
return left;
}
fn term(self: &mut Self, src: []u8) -> !usize {
var left: usize = try self.factor(src);
while true {
if self.done() { break; }
let star: bool = self.is(src, "*");
let slash: bool = self.is(src, "/");
if not star and not slash { break; }
let op: tok.Token = self.cur;
self.bump(src);
let right: usize = try self.factor(src);
left = try self.add(ast.Shape.Binary, op, left, right);
}
return left;
}
fn factor(self: &mut Self, src: []u8) -> !usize {
let t: tok.Token = self.cur;
if t.kind == tok.Kind.Number {
self.bump(src);
return try self.add(ast.Shape.Number, t, ast.NONE, ast.NONE);
}
if t.kind == tok.Kind.Name {
self.bump(src);
return try self.add(ast.Shape.Name, t, ast.NONE, ast.NONE);
}
if t.kind == tok.Kind.Text {
self.bump(src);
return try self.add(ast.Shape.Text, t, ast.NONE, ast.NONE);
}
if self.is(src, "(") {
self.bump(src);
let inner: usize = try self.expr(src);
let close: bool = self.want(src, ")");
return inner;
}
self.errors = self.errors + 1;
self.bump(src);
return try self.add(ast.Shape.Error, t, ast.NONE, ast.NONE);
}
}
fec/tests/exec/lexer/tree.fe
+107
View File
@@ -0,0 +1,107 @@
// EXIT:0
// OUTPUT:unit demo
// OUTPUT:fn answer @2
// OUTPUT: let n = (+ 1 (* 2 3))
// OUTPUT: return n
// OUTPUT:fn greet @3
// OUTPUT: let s = "hi"
// OUTPUT: return
// OUTPUT:fn muddle @4
// OUTPUT: let x = <error>
// OUTPUT:nodes 17 errors 2
// OUTPUT:balanced
unit tree;
import std.io;
import std.sys;
import tok;
import ast;
import parse;
// The Ferro parser, written in Ferro.
//
// The lexer next to this file showed that a token can say where it came from
// instead of holding the text. A tree is the same idea one level up: a node
// cannot hold `^Node` children -- it has several and owns none of them once --
// and R4 keeps `&Node` out of aggregate storage. So the parser owns one array
// of nodes and every child is an index into it.
//
// Printing the expressions back in prefix form is the point of the test: it is
// the only way to see that `1 + 2 * 3` bound the way the grammar says.
const SOURCE: str = "unit demo;\nfn answer() { let n = 1 + 2 * 3; return n; }\nfn greet() { let s = \"hi\"; return; }\nfn muddle() { let x = ; }\n";
fn show_expr(p: &parse.Parser, src: []u8, i: usize) -> void {
if i == ast.NONE { return; }
let n: ast.Node = p.node(i);
match n.shape {
Binary => {
@print("({} ", src[n.from..n.from + n.len]);
show_expr(p, src, n.a);
@print(" ");
show_expr(p, src, n.b);
@print(")");
}
Error => { @print("<error>"); }
_ => { @print("{}", src[n.from..n.from + n.len]); }
}
return;
}
fn show_stmt(p: &parse.Parser, src: []u8, i: usize) -> void {
let n: ast.Node = p.node(i);
match n.shape {
Let => {
@print(" let {} = ", src[n.from..n.from + n.len]);
show_expr(p, src, n.b);
@print("\n");
}
Return => {
if n.a == ast.NONE { @print(" return\n"); }
else {
@print(" return ");
show_expr(p, src, n.a);
@print("\n");
}
}
_ => { @print(" <error>\n"); }
}
return;
}
fn show_item(p: &parse.Parser, src: []u8, i: usize) -> void {
let n: ast.Node = p.node(i);
match n.shape {
Fn => {
@print("fn {} @{}\n", src[n.from..n.from + n.len], n.line);
var s: usize = n.b;
while s != ast.NONE {
show_stmt(p, src, s);
s = p.node(s).next;
}
}
_ => { @print(" <error>\n"); }
}
return;
}
fn run() -> !void {
var p: parse.Parser = try parse.Parser.on(SOURCE);
let root: usize = try p.unit_decl(SOURCE);
let head: ast.Node = p.node(root);
@print("unit {}\n", SOURCE[head.from..head.from + head.len]);
var it: usize = head.a;
while it != ast.NONE {
show_item(&p, SOURCE, it);
it = p.node(it).next;
}
@print("nodes {} errors {}\n", p.count(), p.errors);
return;
}
fn main() -> i32 {
run() catch |e| { @print("out of memory\n"); return 1; };
if sys.allocs() == sys.frees() { @print("balanced\n"); }
else { @print("leaked {}\n", sys.allocs() - sys.frees()); }
return 0;
}