From e84f89214762dd72a7dd14718465163657157696 Mon Sep 17 00:00:00 2001 From: Sebastian Jeong Date: Mon, 17 Aug 2026 12:49:53 +0900 Subject: [PATCH] =?UTF-8?q?Ferro=20=ED=8C=8C=EC=84=9C=EB=A5=BC=20Ferro=20?= =?UTF-8?q?=EB=A1=9C,=20=EA=B7=B8=EB=A6=AC=EA=B3=A0=20=EA=B7=B8=EA=B2=83?= =?UTF-8?q?=EC=9D=B4=20=EB=93=9C=EB=9F=AC=EB=82=B8=20=EB=84=A4=20=EA=B0=80?= =?UTF-8?q?=EC=A7=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 렉서 다음은 파서다. 노드는 한 배열에 살고 자식은 그 안의 인덱스다 -- 노드는 ^Node 를 들 수 없고(여럿이며 한 번씩 소유하지 않는다) &Node 도 들 수 없다(R4). 인덱스는 둘 다 아니다. 소스도 필드가 아니라 매 단계에 같이 다닌다. unit demo / fn answer @2 / let n = (+ 1 (* 2 3)) / return n / balanced 전위 표기로 다시 찍는 것이 시험의 요점이다. 1 + 2 * 3 이 어떻게 묶였는지는 그렇게만 보인다. 쓰면서 나온 컴파일러 버그 넷: 1. 다른 유닛의 타입을 필드로 쓰면 그 필드 타입이 영영 UNKNOWN 이었다. 필드 해석이 유닛마다 선언 직후에 돌아서, 아직 선언되지 않은 유닛의 타입을 찾다 실패하고 그 답을 굳혔다. 이제 모든 유닛이 선언을 마친 뒤에 한 번 푼다. 2. 그리고 그 해석은 타입을 선언한 유닛에서 해야 한다. 필드 타입은 그 유닛의 import 로 쓰였는데 아무 유닛에서나 풀고 있었다. 타입 계층에 enter/leave 콜백을 두고 체커가 그 자리로 데려간다. 3. cycle_state 를 재귀 검사와 크기 계산이 같이 썼다. 첫 번째가 보는 중인 구조체 가 두 번째에게는 다 끝난 것으로 보여서, 필드가 하나뿐인 것처럼 1 바이트로 자리를 잡았다 -- Parser 가 그래서 자기 토큰을 밟았다. layout_state 로 나눴다. 4. 다른 유닛의 상수(ast.NONE)를 lowering 이 필드 접근으로 봤다. 체커가 이미 링크 이름을 붙여두었으니 그것이 있으면 전역이다. 그리고 R1 을 실제로 지키게 했다: 소유자를 놓으면 그것이 가진 것도 놓는다. 전에는 자기 drop 이 있거나 자기가 owned 일 때만이어서, drop 을 가진 타입을 필드로 담은 구조체는 그것을 놓을 방법이 없었다(drop 은 손으로 못 부른다). 이제 release_at 이 drop 을 부르고 필드로 내려간다. 그 덕에 List/Arena/Map 의 drop 이 전부 필요 없어져서 지웠다 -- 버퍼가 owned 이니 R1 이 알아서 한다. 221/221, 29/29. --- fec/src/check.c | 35 +++++ fec/src/checkpri.h | 1 + fec/src/checkpro.c | 4 +- fec/src/checkstm.c | 15 +- fec/src/lower.c | 11 +- fec/src/lowerexp.c | 58 +++++--- fec/src/lowerpri.h | 1 + fec/src/types.c | 70 +++++++--- fec/src/types.h | 11 ++ fec/std/list.fe | 10 +- fec/std/map.fe | 8 +- fec/std/mem.fe | 4 +- fec/tests/exec/lexer/ast.fe | 49 +++++++ fec/tests/exec/lexer/parse.fe | 256 ++++++++++++++++++++++++++++++++++ fec/tests/exec/lexer/tree.fe | 107 ++++++++++++++ 15 files changed, 582 insertions(+), 58 deletions(-) create mode 100644 fec/tests/exec/lexer/ast.fe create mode 100644 fec/tests/exec/lexer/parse.fe create mode 100644 fec/tests/exec/lexer/tree.fe diff --git a/fec/src/check.c b/fec/src/check.c index ea1a250..57ed1cf 100644 --- a/fec/src/check.c +++ b/fec/src/check.c @@ -284,6 +284,9 @@ void enter_unit(FeCheck *c, unsigned index) +static int enter_decl_hook(void *owner, const char *unit); +static void leave_decl_hook(void *owner, int back); + void fe_check_init(FeCheck *c, FeBuild *build, FeDiags *diags, unsigned pointer_bits, int no_checks) { @@ -301,6 +304,8 @@ void fe_check_init(FeCheck *c, FeBuild *build, FeDiags *diags, c->types.unit_name = "unit"; c->types.instantiate = instantiate_type_node; c->types.instantiate_owner = c; + c->types.enter_decl = enter_decl_hook; + c->types.leave_decl = leave_decl_hook; c->instances = (FeInstance *)fe_arena_alloc(&c->arena, (unsigned long)FE_GENERIC_INSTANCE_MAX * sizeof(FeInstance)); c->instance_count = 0; @@ -354,6 +359,36 @@ FeType *unit_type(FeCheck *c, FeUnit *u, const char *name) return 0; } +/* A field type is written in the unit that declared the type, so it has to be + resolved with that unit's imports in scope -- not with whichever unit + happens to be current when the walk reaches it. Returns the index to go back + to, or -1 when there is nowhere to go. */ +int enter_declaring_unit(FeCheck *c, const char *unit_name) +{ + unsigned i; + unsigned here; + if (!unit_name || !c->build || !c->unit) return -1; + here = unit_index(c,c->unit); + for (i=0;ibuild->count;++i) + if (strcmp(c->build->units[i].name,unit_name)==0) { + if (i==here) return -1; + enter_unit(c,i); + return (int)here; + } + return -1; +} + +/* The type layer calls these; it knows nothing about units beyond a name. */ +static int enter_decl_hook(void *owner, const char *unit) +{ + return enter_declaring_unit((FeCheck *)owner, unit); +} + +static void leave_decl_hook(void *owner, int back) +{ + enter_unit((FeCheck *)owner, (unsigned)back); +} + /* The AST declaration of a type another unit declares, for its visibility and for its methods. */ FeNode *unit_type_decl(FeCheck *c, FeUnit *u, const char *name) diff --git a/fec/src/checkpri.h b/fec/src/checkpri.h index d3c04b7..fa75c95 100644 --- a/fec/src/checkpri.h +++ b/fec/src/checkpri.h @@ -104,6 +104,7 @@ FeUnit *binding_unit(FeCheckerState *s, FeNode *base); int decl_is_public(const FeNode *decl); FeSym *unit_member(FeCheck *c, FeUnit *u, const char *name); FeType *unit_type(FeCheck *c, FeUnit *u, const char *name); +int enter_declaring_unit(FeCheck *c, const char *unit_name); FeNode *unit_type_decl(FeCheck *c, FeUnit *u, const char *name); FeType *node_type_in(FeCheck *c, const char *unit, FeNode *node); FeNode *find_method(FeCheck *c, FeType *owner, const char *name); diff --git a/fec/src/checkpro.c b/fec/src/checkpro.c index 74acf81..0d67cd5 100644 --- a/fec/src/checkpro.c +++ b/fec/src/checkpro.c @@ -79,7 +79,6 @@ void declare_unit(FeCheck *c) fe_type_declare_enum(&c->types,n); for (n=c->ast->root ? c->ast->root->children : 0;n;n=n->next) if (n->kind==FE_N_ERROR_DECL) fe_type_declare_error(&c->types,n); - check_type_cycles(c); } /* The unit's top-level names, in a scope of their own so that another unit @@ -166,6 +165,9 @@ int fe_check_program(FeCheck *c) s.fn_node=0; fe_own_liveness_init(&s.liveness,&c->arena); for (u=0;ubuild->count;++u) { enter_unit(c,u); declare_unit(c); } + /* Only now: a field may name a type in a unit that had not declared it + yet, and resolving it early would freeze the wrong answer in place. */ + for (u=0;ubuild->count;++u) { enter_unit(c,u); check_type_cycles(c); } fe_type_layout_all(&c->types); for (u=0;ubuild->count;++u) { enter_unit(c,u); diff --git a/fec/src/checkstm.c b/fec/src/checkstm.c index 458bd1c..4d28c1e 100644 --- a/fec/src/checkstm.c +++ b/fec/src/checkstm.c @@ -140,18 +140,25 @@ void check_type_cycle(FeCheck *c, FeType *t) if (t->kind == FE_TYPE_ARRAY) { check_type_cycle(c,t->elem); } else if (t->kind == FE_TYPE_STRUCT) { - for (i=0;ifield_count;i++) { + int back=enter_declaring_unit(c,t->unit); + for (i=0;ifield_count;i++) if (!t->fields[i].type && t->fields[i].ast_node) t->fields[i].type=fe_type_from_ast(&c->types,t->fields[i].ast_node->a); - check_type_cycle(c,t->fields[i].type); - } + if (back>=0) enter_unit(c,(unsigned)back); + for (i=0;ifield_count;i++) check_type_cycle(c,t->fields[i].type); } else if (t->kind == FE_TYPE_ENUM) { + int back=enter_declaring_unit(c,t->unit); for (i=0;ivariant_count;i++) { unsigned j; - for (j=0;jvariants[i].field_count;j++) { + for (j=0;jvariants[i].field_count;j++) if (!t->variants[i].fields[j].type && t->variants[i].fields[j].ast_node) t->variants[i].fields[j].type=fe_type_from_ast(&c->types, t->variants[i].fields[j].ast_node->a); + } + if (back>=0) enter_unit(c,(unsigned)back); + for (i=0;ivariant_count;i++) { + unsigned j; + for (j=0;jvariants[i].field_count;j++) { next=t->variants[i].fields[j].type; check_type_cycle(c,next); } diff --git a/fec/src/lower.c b/fec/src/lower.c index 80d13f6..41ff46e 100644 --- a/fec/src/lower.c +++ b/fec/src/lower.c @@ -113,9 +113,18 @@ unsigned as_address(Lower *L, Slot s, FeNode *n) /* Does letting go of this type have to do something? */ int needs_release(const FeType *t) { + unsigned i; if (!t) return 0; if (t->kind == FE_TYPE_OWNED) return 1; - return t->has_drop != 0; + if (t->has_drop) return 1; + /* SPEC 5 R1: letting go of an owner lets go of what it owns. A struct that + holds an owner has something to do even when it says nothing itself -- + which is what lets one type hold another that has a `drop`, since + calling `drop` by hand is not allowed. */ + if (t->kind == FE_TYPE_STRUCT) + for (i = 0; i < t->field_count; ++i) + if (needs_release(t->fields[i].type)) return 1; + return 0; } int lower_reserve(Lower *L, void **items, unsigned *capacity, unsigned needed, diff --git a/fec/src/lowerexp.c b/fec/src/lowerexp.c index 869a36f..40de08c 100644 --- a/fec/src/lowerexp.c +++ b/fec/src/lowerexp.c @@ -171,6 +171,12 @@ Slot lower_expr_core(Lower *L, FeNode *n) if (base && (base->kind == FE_TYPE_REF || base->kind == FE_TYPE_OWNED)) base = base->elem; field = fe_type_field(base, n->b && n->b->text ? n->b->text : ""); + /* `binding.name` is not a field of anything: it is a constant or a + global in another unit, and the checker already turned it into a + link name. */ + if (!field && n->cname) + return slot_place(fe_ir_at_global(n->cname, 0), it, + ir_size(t)); if (!field) { fail(L, "an unresolved field", n); return slot_void(); } b = lower_expr(L, n->a); if (n->a->sem_type && (n->a->sem_type->kind == FE_TYPE_REF || @@ -305,6 +311,37 @@ const char *drop_name(Lower *L, const FeType *t) return method->cname; } +/* Let go of one value sitting at `at`. A type that says how to let go of + itself is asked first; then whatever it holds is let go of in turn, so a + struct that owns a struct that owns a buffer settles all three without + anyone writing a `drop` (SPEC 5 R1). */ +void release_at(Lower *L, const FeType *t, FeIrPlace at) +{ + unsigned args[1]; + unsigned i; + if (!t) return; + if (t->has_drop) { + const char *how = drop_name(L, t); + args[0] = fe_ir_addr(L->m, L->b, at); + if (how) fe_ir_call(L->m, L->b, FE_IR_VOID, how, args, 1); + } + if (t->kind == FE_TYPE_OWNED) { + FeIrPlace p = at; + if (t->elem && t->elem->kind == FE_TYPE_SLICE) + p.offset += SLICE_PTR_OFFSET; + args[0] = fe_ir_load(L->m, L->b, FE_IR_PTR, p); + fe_ir_call(L->m, L->b, FE_IR_VOID, "fe_rt_free", args, 1); + return; + } + if (t->kind == FE_TYPE_STRUCT) + for (i = 0; i < t->field_count; ++i) { + FeIrPlace p = at; + if (!needs_release(t->fields[i].type)) continue; + p.offset += (long)t->fields[i].offset; + release_at(L, t->fields[i].type, p); + } +} + /* Settle what a scope owes, most recent first. A `return` in the middle of a function still owes everything, so every exit path calls this. */ void run_deferred(Lower *L, unsigned from) @@ -321,29 +358,10 @@ void run_deferred(Lower *L, unsigned from) fe_ir_at_local(L->owed[i - 1].flag, 0)); FeIrBlock *doit = new_block(L); FeIrBlock *skip = new_block(L); - unsigned args[1]; FeType *t = L->owed[i - 1].type; fe_ir_br(L->b, live, doit->id, skip->id); L->b = doit; - if (t && t->kind == FE_TYPE_OWNED && t->elem && - t->elem->kind == FE_TYPE_SLICE) { - FeIrPlace at = fe_ir_at_local(L->owed[i - 1].local, - SLICE_PTR_OFFSET); - args[0] = fe_ir_load(L->m, L->b, FE_IR_PTR, at); - } else { - args[0] = fe_ir_load(L->m, L->b, FE_IR_PTR, - fe_ir_at_local(L->owed[i - 1].local, 0)); - } - if (t && t->has_drop) { - /* A type that says how to let go of itself is asked to; the - name is the one its instance was given. */ - const char *how = drop_name(L, t); - args[0] = fe_ir_addr(L->m, L->b, - fe_ir_at_local(L->owed[i - 1].local, 0)); - if (how) fe_ir_call(L->m, L->b, FE_IR_VOID, how, args, 1); - } else { - fe_ir_call(L->m, L->b, FE_IR_VOID, "fe_rt_free", args, 1); - } + release_at(L, t, fe_ir_at_local(L->owed[i - 1].local, 0)); fe_ir_jmp(L->b, skip->id); L->b = skip; } diff --git a/fec/src/lowerpri.h b/fec/src/lowerpri.h index 3c5522d..9ebd779 100644 --- a/fec/src/lowerpri.h +++ b/fec/src/lowerpri.h @@ -145,6 +145,7 @@ Slot lower_call(Lower *L, FeNode *n); Slot lower_expr(Lower *L, FeNode *n); Slot lower_expr_core(Lower *L, FeNode *n); const char *drop_name(Lower *L, const FeType *t); +void release_at(Lower *L, const FeType *t, FeIrPlace at); void run_deferred(Lower *L, unsigned from); Slot wrap_context(Lower *L, Slot v, FeNode *n); unsigned wrapper_tag(Lower *L, Slot w, const FeType *t, FeNode *n); diff --git a/fec/src/types.c b/fec/src/types.c index 1c1b707..b7b7cff 100644 --- a/fec/src/types.c +++ b/fec/src/types.c @@ -47,6 +47,7 @@ static FeType *new_type(FeTypeCtx *ctx, const char *name, FeTypeKind kind) t->next = ctx->types; t->emit_state = 0; t->cycle_state = 0; + t->layout_state = 0; ctx->types = t; return t; } @@ -61,6 +62,8 @@ void fe_types_init(FeTypeCtx *ctx, FeArena *arena, unsigned pointer_bits) ctx->param_count = 0; ctx->instantiate = 0; ctx->instantiate_owner = 0; + ctx->enter_decl = 0; + ctx->leave_decl = 0; } /* Does this type answer to `name` for someone checking `unit`? A type with no @@ -452,6 +455,26 @@ unsigned fe_type_align(const FeType *t) return t && t->align ? t->align : 1U; } +/* Resolve this type's fields where they were written. Without the callback + installed -- or for a type nobody declared -- everything stays where it is, + which is what the non-checking users of this layer want. */ +static int enter_decl_unit(FeTypeCtx *ctx, const char *unit, const char **was) +{ + *was = ctx->unit_name; + if (!ctx->enter_decl || !unit) return -1; + return ctx->enter_decl(ctx->instantiate_owner, unit); +} + +/* Put back both halves: the unit the checker was in, and the name this layer + was interning under -- an instantiation moves the second without the + first, so restoring one is not restoring the other. */ +static void leave_decl_unit(FeTypeCtx *ctx, int back, const char *was) +{ + if (back >= 0 && ctx->leave_decl) + ctx->leave_decl(ctx->instantiate_owner, back); + ctx->unit_name = was; +} + static void layout_type(FeTypeCtx *ctx, FeType *t) { unsigned i; @@ -460,14 +483,14 @@ static void layout_type(FeTypeCtx *ctx, FeType *t) unsigned long max_size; unsigned max_align; if (!t || t->size) return; - if (t->cycle_state == 1) { + if (t->layout_state == 1) { t->size = 1; t->align = 1; return; } - t->cycle_state = 1; + t->layout_state = 1; if (t->kind == FE_TYPE_VOID || t->kind == FE_TYPE_UNKNOWN || - t->kind == FE_TYPE_ERROR) { t->size = 0; t->align = 1; t->cycle_state = 2; return; } + t->kind == FE_TYPE_ERROR) { t->size = 0; t->align = 1; t->layout_state = 2; return; } if (t->kind == FE_TYPE_ERROR_UNION) { if (t->error_value && t->error_value->kind != FE_TYPE_VOID) { layout_type(ctx,t->error_value); @@ -478,7 +501,7 @@ static void layout_type(FeTypeCtx *ctx, FeType *t) t->size=2; t->align=2U; } - t->cycle_state = 2; return; + t->layout_state = 2; return; } if (t->kind == FE_TYPE_OPTIONAL) { layout_type(ctx,t->elem); @@ -490,44 +513,48 @@ static void layout_type(FeTypeCtx *ctx, FeType *t) t->size=round_up(1UL,t->align)+fe_type_size(t->elem); t->size=round_up(t->size,t->align); } - t->cycle_state=2; return; + t->layout_state=2; return; } if (t->kind == FE_TYPE_BOOL || t->kind == FE_TYPE_CHAR) { - t->size = 1; t->align = 1; t->cycle_state = 2; return; + t->size = 1; t->align = 1; t->layout_state = 2; return; } if (t->kind == FE_TYPE_INT) { t->size = (t->bits + 7U) / 8U; t->align = t->size; if (t->size > 4UL) t->size = 4UL; - t->cycle_state = 2; return; + t->layout_state = 2; return; } if (t->kind == FE_TYPE_REF || t->kind == FE_TYPE_RAW) { t->size = FE_PTR_SIZE; t->align = FE_PTR_ALIGN; - t->cycle_state = 2; return; + t->layout_state = 2; return; } if (t->kind == FE_TYPE_OWNED) { t->size = t->elem && t->elem->kind==FE_TYPE_SLICE ? 2UL * FE_PTR_SIZE : FE_PTR_SIZE; t->align = FE_PTR_ALIGN; - t->cycle_state = 2; return; + t->layout_state = 2; return; } if (t->kind == FE_TYPE_SLICE || t->kind == FE_TYPE_STR) { t->size = 2UL * FE_PTR_SIZE; t->align = FE_PTR_ALIGN; - t->cycle_state = 2; return; + t->layout_state = 2; return; } if (t->kind == FE_TYPE_ARRAY) { layout_type(ctx, t->elem); t->align = t->packed ? 1U : fe_type_align(t->elem); t->size = t->length * fe_type_size(t->elem); - t->cycle_state = 2; return; + t->layout_state = 2; return; } if (t->kind == FE_TYPE_STRUCT) { - off = 0; max_align = 1; - for (i = 0; i < t->field_count; ++i) { + const char *was; + int back = enter_decl_unit(ctx, t->unit, &was); + for (i = 0; i < t->field_count; ++i) if (!t->fields[i].type && t->fields[i].ast_node) t->fields[i].type = fe_type_from_ast(ctx, t->fields[i].ast_node->a); + leave_decl_unit(ctx, back, was); + off = 0; max_align = 1; + for (i = 0; i < t->field_count; ++i) { layout_type(ctx, t->fields[i].type); align = t->packed ? 1U : fe_type_align(t->fields[i].type); if (align > max_align) max_align = align; @@ -537,18 +564,25 @@ static void layout_type(FeTypeCtx *ctx, FeType *t) } t->align = max_align; t->size = round_up(off, max_align); - t->cycle_state = 2; + t->layout_state = 2; return; } if (t->kind == FE_TYPE_ENUM) { + const char *was; + int back = enter_decl_unit(ctx, t->unit, &was); + for (i = 0; i < t->variant_count; ++i) { + unsigned j; + for (j = 0; j < t->variants[i].field_count; ++j) + if (!t->variants[i].fields[j].type && t->variants[i].fields[j].ast_node) + t->variants[i].fields[j].type = fe_type_from_ast( + ctx, t->variants[i].fields[j].ast_node->a); + } + leave_decl_unit(ctx, back, was); max_size = 0; max_align = 1; for (i = 0; i < t->variant_count; ++i) { unsigned j; off = 0; for (j = 0; j < t->variants[i].field_count; ++j) { - if (!t->variants[i].fields[j].type && t->variants[i].fields[j].ast_node) - t->variants[i].fields[j].type = fe_type_from_ast( - ctx, t->variants[i].fields[j].ast_node->a); layout_type(ctx, t->variants[i].fields[j].type); if (fe_type_align(t->variants[i].fields[j].type) > max_align) max_align = fe_type_align(t->variants[i].fields[j].type); @@ -564,7 +598,7 @@ static void layout_type(FeTypeCtx *ctx, FeType *t) off = round_up(t->bits / 8U, max_align); t->size = round_up(off + max_size, max_align); t->align = max_align; - t->cycle_state = 2; + t->layout_state = 2; } } diff --git a/fec/src/types.h b/fec/src/types.h index 4109839..d8db1ea 100644 --- a/fec/src/types.h +++ b/fec/src/types.h @@ -98,6 +98,11 @@ struct FeType { FeType *next; int emit_state; int cycle_state; + /* Separate from `cycle_state`: the checker's by-value recursion walk and + this layer's size computation run inside one another, and sharing one + marker made a struct in the middle of the first look complete to the + second -- one byte wide, with every field on top of the next. */ + int layout_state; }; typedef struct FeTypeCtx { @@ -114,6 +119,12 @@ typedef struct FeTypeCtx { so it installs this and the type layer calls back into it. */ FeType *(*instantiate)(void *owner, const FeNode *node); void *instantiate_owner; + /* A field type is written in the unit that declared it, so resolving one + has to happen with that unit's imports in scope. The checker owns that + knowledge, so it installs this pair and the type layer calls back. + `enter` answers with what to hand `leave`, or -1 for "stayed put". */ + int (*enter_decl)(void *owner, const char *unit); + void (*leave_decl)(void *owner, int back); } FeTypeCtx; void fe_types_init(FeTypeCtx *ctx, FeArena *arena, unsigned pointer_bits); diff --git a/fec/std/list.fe b/fec/std/list.fe index 58fff67..cdf7c36 100644 --- a/fec/std/list.fe +++ b/fec/std/list.fe @@ -1,8 +1,10 @@ unit std.list; // A growable sequence. The buffer is owned, so a List owns its elements and -// releasing it releases them (SPEC 5 R1). Growth doubles, which keeps the -// total copying proportional to the number of pushes. +// releasing it releases them (SPEC 5 R1) -- which is also why there is no +// `drop` here: letting go of a List lets go of its buffer on its own. Growth +// doubles, which keeps the total copying proportional to the number of +// pushes. pub struct List(T) { items: ^[]mut T, @@ -46,8 +48,4 @@ pub struct List(T) { mem.destroy(old); return; } - - pub fn drop(self: &mut Self) -> void { - mem.destroy(self.items); - } } diff --git a/fec/std/map.fe b/fec/std/map.fe index 3427f19..3b50ec7 100644 --- a/fec/std/map.fe +++ b/fec/std/map.fe @@ -6,7 +6,8 @@ unit std.map; // wrong shape for that. Keys are copied into one buffer the map owns and each // slot records where in it the key sits -- the arena-and-handle shape R11 asks // for, which also means letting go of the map is two frees and not one per -// entry. +// entry. Both buffers are owned, so R1 does the letting go and no `drop` is +// written here. // // Open addressing with linear probing. The table is a power of two so the // index is a mask rather than a division, and it grows at three quarters full @@ -155,11 +156,6 @@ pub struct Map(V) { mem.destroy(old); return; } - - pub fn drop(self: &mut Self) -> void { - mem.destroy(self.slots); - mem.destroy(self.bytes); - } } /// FNV-1a. Small, fast, and good enough for identifiers; nothing here has to diff --git a/fec/std/mem.fe b/fec/std/mem.fe index f4524f7..831cccc 100644 --- a/fec/std/mem.fe +++ b/fec/std/mem.fe @@ -10,7 +10,8 @@ import std.sys; /// SPEC R11 answers recursive and graph-shaped data with an arena that owns /// the values and integer handles that reference them. This is that arena. A /// handle is an offset, so it stays valid while the arena does, and comparing -/// two handles is comparing two numbers. +/// two handles is comparing two numbers. The block itself is owned, so nothing +/// here says how to let go of it -- R1 already does. pub struct Arena { bytes: ^[]mut u8, used: usize, @@ -56,5 +57,4 @@ pub struct Arena { /// that is the trade an arena makes. pub fn reset(self: &mut Self) -> void { self.used = 0; } - pub fn drop(self: &mut Self) -> void { mem.destroy(self.bytes); } } diff --git a/fec/tests/exec/lexer/ast.fe b/fec/tests/exec/lexer/ast.fe new file mode 100644 index 0000000..9219e43 --- /dev/null +++ b/fec/tests/exec/lexer/ast.fe @@ -0,0 +1,49 @@ +unit ast; + +// The tree the parser builds. +// +// Nodes live in one growing array and refer to each other by index, which is +// what SPEC R11 asks for: an owner that holds the values and handles that +// point at them. A node cannot hold a `^Node` for its children because a node +// has several and they are not each owned once; it cannot hold a `&Node` +// because R4 keeps borrows out of aggregate storage. An index is neither. + +pub enum Shape { + Unit, // a: name + Fn, // a: name, b: first statement + Let, // a: name, b: value + Return, // a: value, or NONE + Binary, // a: left, b: right, from/len: the operator + Number, + Name, + Text, + Error, +} + +/// No node. Zero is a real index, so the empty handle is the largest one. +pub const NONE: usize = 4294967295; + +pub struct Node { + pub shape: Shape, + pub from: usize, // where in the source this came from + pub len: usize, + pub line: usize, + pub a: usize, // handles into the same tree + pub b: usize, + pub next: usize, // the following statement, when there is one +} + +pub fn name_of(s: Shape) -> []u8 { + match s { + Unit => { return "unit"; } + Fn => { return "fn"; } + Let => { return "let"; } + Return => { return "return"; } + Binary => { return "binary"; } + Number => { return "number"; } + Name => { return "name"; } + Text => { return "text"; } + Error => { return "error"; } + } + return "?"; +} diff --git a/fec/tests/exec/lexer/parse.fe b/fec/tests/exec/lexer/parse.fe new file mode 100644 index 0000000..ffd5d2b --- /dev/null +++ b/fec/tests/exec/lexer/parse.fe @@ -0,0 +1,256 @@ +unit parse; + +import std.list; +import std.str; +import tok; +import scan; +import ast; + +// Recursive descent over the lexer's tokens. +// +// The source is not a field: R4 keeps borrows out of aggregate storage, so +// `src` is passed to every step, the same way the lexer passes it. What the +// parser does own is the node array, and every parent points at its children +// by index into it. +// +// One token of lookahead lives in `cur`. That is all this grammar needs. + +pub struct Parser { + nodes: list.List(ast.Node), + at: usize, + line: usize, + cur: tok.Token, + pub errors: usize, + + pub fn on(src: []u8) -> !Self { + let nodes: list.List(ast.Node) = + try list.List(ast.Node).with_capacity(16); + var p: Self = Self{ + nodes: nodes, + at: 0, + line: 1, + cur: tok.Token{ kind: tok.Kind.End, from: 0, len: 0, line: 1 }, + errors: 0, + }; + p.bump(src); + return p; + } + + fn bump(self: &mut Self, src: []u8) -> void { + self.cur = scan.next(src, &mut self.at, &mut self.line); + return; + } + + fn done(self: &Self) -> bool { return self.cur.kind == tok.Kind.End; } + + /// Is the token in hand this exact spelling? + fn is(self: &Self, src: []u8, want: []u8) -> bool { + return str.eq(tok.text(src, self.cur), want); + } + + /// Take the token in hand if it is this spelling; say whether it was. + fn eat(self: &mut Self, src: []u8, want: []u8) -> bool { + if not self.is(src, want) { return false; } + self.bump(src); + return true; + } + + /// Demand this spelling. A miss is counted and the token stays put, so the + /// caller decides how to get back on its feet. + fn want(self: &mut Self, src: []u8, w: []u8) -> bool { + if self.eat(src, w) { return true; } + self.errors = self.errors + 1; + return false; + } + + fn add(self: &mut Self, s: ast.Shape, t: tok.Token, a: usize, + b: usize) -> !usize { + let n: ast.Node = ast.Node{ + shape: s, from: t.from, len: t.len, line: t.line, + a: a, b: b, next: ast.NONE, + }; + let i: usize = self.nodes.count(); + try self.nodes.push(n); + return i; + } + + // -- reading the tree back ------------------------------------------ + + pub fn count(self: &Self) -> usize { return self.nodes.count(); } + + pub fn node(self: &Self, i: usize) -> ast.Node { + return self.nodes.at(i); + } + + + // -- the grammar ---------------------------------------------------- + // + // unit := "unit" NAME ";" item* + // item := "fn" NAME "(" ")" block + // block := "{" stmt* "}" + // stmt := "let" NAME "=" expr ";" | "return" expr? ";" + // expr := term (("+" | "-") term)* + // term := factor (("*" | "/") factor)* + // factor := NUMBER | NAME | TEXT | "(" expr ")" + + pub fn unit_decl(self: &mut Self, src: []u8) -> !usize { + let ok: bool = self.want(src, "unit"); + let name: tok.Token = self.cur; + if ok { self.bump(src); } + let semi: bool = self.want(src, ";"); + let root: usize = try self.add(ast.Shape.Unit, name, ast.NONE, + ast.NONE); + var first: usize = ast.NONE; + var last: usize = ast.NONE; + while not self.done() { + let it: usize = try self.item(src); + if first == ast.NONE { first = it; } + else { self.link(last, it); } + last = it; + } + self.set_a(root, first); + return root; + } + + /// Point one statement or item at the one after it. + fn link(self: &mut Self, from: usize, to: usize) -> void { + var n: ast.Node = self.nodes.at(from); + n.next = to; + self.nodes.set(from, n); + return; + } + + fn set_a(self: &mut Self, at: usize, a: usize) -> void { + var n: ast.Node = self.nodes.at(at); + n.a = a; + self.nodes.set(at, n); + return; + } + + fn item(self: &mut Self, src: []u8) -> !usize { + if not self.is(src, "fn") { + let bad: tok.Token = self.cur; + self.errors = self.errors + 1; + self.skip_stmt(src); + return try self.add(ast.Shape.Error, bad, ast.NONE, ast.NONE); + } + self.bump(src); + let name: tok.Token = self.cur; + self.bump(src); + let open: bool = self.want(src, "("); + let close: bool = self.want(src, ")"); + let body: usize = try self.block(src); + return try self.add(ast.Shape.Fn, name, ast.NONE, body); + } + + fn block(self: &mut Self, src: []u8) -> !usize { + let open: bool = self.want(src, "{"); + var first: usize = ast.NONE; + var last: usize = ast.NONE; + while true { + if self.done() { break; } + if self.is(src, "}") { break; } + let s: usize = try self.stmt(src); + if first == ast.NONE { first = s; } + else { self.link(last, s); } + last = s; + } + let close: bool = self.want(src, "}"); + return first; + } + + fn stmt(self: &mut Self, src: []u8) -> !usize { + if self.is(src, "let") { + self.bump(src); + let name: tok.Token = self.cur; + self.bump(src); + let has_eq: bool = self.want(src, "="); + let value: usize = try self.expr(src); + let semi: bool = self.want(src, ";"); + return try self.add(ast.Shape.Let, name, ast.NONE, value); + } + if self.is(src, "return") { + let head: tok.Token = self.cur; + self.bump(src); + if self.is(src, ";") { + self.bump(src); + return try self.add(ast.Shape.Return, head, ast.NONE, + ast.NONE); + } + let value: usize = try self.expr(src); + let semi: bool = self.want(src, ";"); + return try self.add(ast.Shape.Return, head, value, ast.NONE); + } + let bad: tok.Token = self.cur; + self.errors = self.errors + 1; + self.skip_stmt(src); + return try self.add(ast.Shape.Error, bad, ast.NONE, ast.NONE); + } + + /// Get back to a statement boundary after something unrecognised. Stopping + /// at `;` or `}` means one bad statement costs one diagnostic, not a run + /// of them. + fn skip_stmt(self: &mut Self, src: []u8) -> void { + while not self.done() { + if self.is(src, "}") { return; } + if self.is(src, ";") { self.bump(src); return; } + self.bump(src); + } + return; + } + + fn expr(self: &mut Self, src: []u8) -> !usize { + var left: usize = try self.term(src); + while true { + if self.done() { break; } + let plus: bool = self.is(src, "+"); + let minus: bool = self.is(src, "-"); + if not plus and not minus { break; } + let op: tok.Token = self.cur; + self.bump(src); + let right: usize = try self.term(src); + left = try self.add(ast.Shape.Binary, op, left, right); + } + return left; + } + + fn term(self: &mut Self, src: []u8) -> !usize { + var left: usize = try self.factor(src); + while true { + if self.done() { break; } + let star: bool = self.is(src, "*"); + let slash: bool = self.is(src, "/"); + if not star and not slash { break; } + let op: tok.Token = self.cur; + self.bump(src); + let right: usize = try self.factor(src); + left = try self.add(ast.Shape.Binary, op, left, right); + } + return left; + } + + fn factor(self: &mut Self, src: []u8) -> !usize { + let t: tok.Token = self.cur; + if t.kind == tok.Kind.Number { + self.bump(src); + return try self.add(ast.Shape.Number, t, ast.NONE, ast.NONE); + } + if t.kind == tok.Kind.Name { + self.bump(src); + return try self.add(ast.Shape.Name, t, ast.NONE, ast.NONE); + } + if t.kind == tok.Kind.Text { + self.bump(src); + return try self.add(ast.Shape.Text, t, ast.NONE, ast.NONE); + } + if self.is(src, "(") { + self.bump(src); + let inner: usize = try self.expr(src); + let close: bool = self.want(src, ")"); + return inner; + } + self.errors = self.errors + 1; + self.bump(src); + return try self.add(ast.Shape.Error, t, ast.NONE, ast.NONE); + } +} diff --git a/fec/tests/exec/lexer/tree.fe b/fec/tests/exec/lexer/tree.fe new file mode 100644 index 0000000..4f7e2e6 --- /dev/null +++ b/fec/tests/exec/lexer/tree.fe @@ -0,0 +1,107 @@ +// EXIT:0 +// OUTPUT:unit demo +// OUTPUT:fn answer @2 +// OUTPUT: let n = (+ 1 (* 2 3)) +// OUTPUT: return n +// OUTPUT:fn greet @3 +// OUTPUT: let s = "hi" +// OUTPUT: return +// OUTPUT:fn muddle @4 +// OUTPUT: let x = +// OUTPUT:nodes 17 errors 2 +// OUTPUT:balanced +unit tree; + +import std.io; +import std.sys; +import tok; +import ast; +import parse; + +// The Ferro parser, written in Ferro. +// +// The lexer next to this file showed that a token can say where it came from +// instead of holding the text. A tree is the same idea one level up: a node +// cannot hold `^Node` children -- it has several and owns none of them once -- +// and R4 keeps `&Node` out of aggregate storage. So the parser owns one array +// of nodes and every child is an index into it. +// +// Printing the expressions back in prefix form is the point of the test: it is +// the only way to see that `1 + 2 * 3` bound the way the grammar says. + +const SOURCE: str = "unit demo;\nfn answer() { let n = 1 + 2 * 3; return n; }\nfn greet() { let s = \"hi\"; return; }\nfn muddle() { let x = ; }\n"; + +fn show_expr(p: &parse.Parser, src: []u8, i: usize) -> void { + if i == ast.NONE { return; } + let n: ast.Node = p.node(i); + match n.shape { + Binary => { + @print("({} ", src[n.from..n.from + n.len]); + show_expr(p, src, n.a); + @print(" "); + show_expr(p, src, n.b); + @print(")"); + } + Error => { @print(""); } + _ => { @print("{}", src[n.from..n.from + n.len]); } + } + return; +} + +fn show_stmt(p: &parse.Parser, src: []u8, i: usize) -> void { + let n: ast.Node = p.node(i); + match n.shape { + Let => { + @print(" let {} = ", src[n.from..n.from + n.len]); + show_expr(p, src, n.b); + @print("\n"); + } + Return => { + if n.a == ast.NONE { @print(" return\n"); } + else { + @print(" return "); + show_expr(p, src, n.a); + @print("\n"); + } + } + _ => { @print(" \n"); } + } + return; +} + +fn show_item(p: &parse.Parser, src: []u8, i: usize) -> void { + let n: ast.Node = p.node(i); + match n.shape { + Fn => { + @print("fn {} @{}\n", src[n.from..n.from + n.len], n.line); + var s: usize = n.b; + while s != ast.NONE { + show_stmt(p, src, s); + s = p.node(s).next; + } + } + _ => { @print(" \n"); } + } + return; +} + +fn run() -> !void { + var p: parse.Parser = try parse.Parser.on(SOURCE); + let root: usize = try p.unit_decl(SOURCE); + let head: ast.Node = p.node(root); + @print("unit {}\n", SOURCE[head.from..head.from + head.len]); + var it: usize = head.a; + while it != ast.NONE { + show_item(&p, SOURCE, it); + it = p.node(it).next; + } + @print("nodes {} errors {}\n", p.count(), p.errors); + return; +} + +fn main() -> i32 { + run() catch |e| { @print("out of memory\n"); return 1; }; + if sys.allocs() == sys.frees() { @print("balanced\n"); } + else { @print("leaked {}\n", sys.allocs() - sys.frees()); } + return 0; +}