Ferro 파서를 Ferro 로, 그리고 그것이 드러낸 네 가지

렉서 다음은 파서다. 노드는 한 배열에 살고 자식은 그 안의 인덱스다 -- 노드는
^Node 를 들 수 없고(여럿이며 한 번씩 소유하지 않는다) &Node 도 들 수 없다(R4).
인덱스는 둘 다 아니다. 소스도 필드가 아니라 매 단계에 같이 다닌다.

  unit demo / fn answer @2 / let n = (+ 1 (* 2 3)) / return n / balanced

전위 표기로 다시 찍는 것이 시험의 요점이다. 1 + 2 * 3 이 어떻게 묶였는지는
그렇게만 보인다.

쓰면서 나온 컴파일러 버그 넷:

1. 다른 유닛의 타입을 필드로 쓰면 그 필드 타입이 영영 UNKNOWN 이었다. 필드
   해석이 유닛마다 선언 직후에 돌아서, 아직 선언되지 않은 유닛의 타입을 찾다
   실패하고 그 답을 굳혔다. 이제 모든 유닛이 선언을 마친 뒤에 한 번 푼다.

2. 그리고 그 해석은 타입을 선언한 유닛에서 해야 한다. 필드 타입은 그 유닛의
   import 로 쓰였는데 아무 유닛에서나 풀고 있었다. 타입 계층에 enter/leave
   콜백을 두고 체커가 그 자리로 데려간다.

3. cycle_state 를 재귀 검사와 크기 계산이 같이 썼다. 첫 번째가 보는 중인 구조체
   가 두 번째에게는 다 끝난 것으로 보여서, 필드가 하나뿐인 것처럼 1 바이트로
   자리를 잡았다 -- Parser 가 그래서 자기 토큰을 밟았다. layout_state 로 나눴다.

4. 다른 유닛의 상수(ast.NONE)를 lowering 이 필드 접근으로 봤다. 체커가 이미
   링크 이름을 붙여두었으니 그것이 있으면 전역이다.

그리고 R1 을 실제로 지키게 했다: 소유자를 놓으면 그것이 가진 것도 놓는다.
전에는 자기 drop 이 있거나 자기가 owned 일 때만이어서, drop 을 가진 타입을
필드로 담은 구조체는 그것을 놓을 방법이 없었다(drop 은 손으로 못 부른다).
이제 release_at 이 drop 을 부르고 필드로 내려간다. 그 덕에 List/Arena/Map 의
drop 이 전부 필요 없어져서 지웠다 -- 버퍼가 owned 이니 R1 이 알아서 한다.

221/221, 29/29.

fec/src/check.c

@@ -284,6 +284,9 @@ void enter_unit(FeCheck *c, unsigned index)
 
 
 
+static int enter_decl_hook(void *owner, const char *unit);
+static void leave_decl_hook(void *owner, int back);
+
 void fe_check_init(FeCheck *c, FeBuild *build, FeDiags *diags,
                    unsigned pointer_bits, int no_checks)
 {
@@ -301,6 +304,8 @@ void fe_check_init(FeCheck *c, FeBuild *build, FeDiags *diags,
     c->types.unit_name = "unit";
     c->types.instantiate = instantiate_type_node;
     c->types.instantiate_owner = c;
+    c->types.enter_decl = enter_decl_hook;
+    c->types.leave_decl = leave_decl_hook;
     c->instances = (FeInstance *)fe_arena_alloc(&c->arena,
         (unsigned long)FE_GENERIC_INSTANCE_MAX * sizeof(FeInstance));
     c->instance_count = 0;
@@ -354,6 +359,36 @@ FeType *unit_type(FeCheck *c, FeUnit *u, const char *name)
     return 0;
 }
 
+/* A field type is written in the unit that declared the type, so it has to be
+   resolved with that unit's imports in scope -- not with whichever unit
+   happens to be current when the walk reaches it. Returns the index to go back
+   to, or -1 when there is nowhere to go. */
+int enter_declaring_unit(FeCheck *c, const char *unit_name)
+{
+    unsigned i;
+    unsigned here;
+    if (!unit_name || !c->build || !c->unit) return -1;
+    here = unit_index(c,c->unit);
+    for (i=0;i<c->build->count;++i)
+        if (strcmp(c->build->units[i].name,unit_name)==0) {
+            if (i==here) return -1;
+            enter_unit(c,i);
+            return (int)here;
+        }
+    return -1;
+}
+
+/* The type layer calls these; it knows nothing about units beyond a name. */
+static int enter_decl_hook(void *owner, const char *unit)
+{
+    return enter_declaring_unit((FeCheck *)owner, unit);
+}
+
+static void leave_decl_hook(void *owner, int back)
+{
+    enter_unit((FeCheck *)owner, (unsigned)back);
+}
+
 /* The AST declaration of a type another unit declares, for its visibility and
    for its methods. */
 FeNode *unit_type_decl(FeCheck *c, FeUnit *u, const char *name)

fec/src/checkpri.h

@@ -104,6 +104,7 @@ FeUnit *binding_unit(FeCheckerState *s, FeNode *base);
 int decl_is_public(const FeNode *decl);
 FeSym *unit_member(FeCheck *c, FeUnit *u, const char *name);
 FeType *unit_type(FeCheck *c, FeUnit *u, const char *name);
+int enter_declaring_unit(FeCheck *c, const char *unit_name);
 FeNode *unit_type_decl(FeCheck *c, FeUnit *u, const char *name);
 FeType *node_type_in(FeCheck *c, const char *unit, FeNode *node);
 FeNode *find_method(FeCheck *c, FeType *owner, const char *name);

fec/src/checkpro.c

@@ -79,7 +79,6 @@ void declare_unit(FeCheck *c)
             fe_type_declare_enum(&c->types,n);
     for (n=c->ast->root ? c->ast->root->children : 0;n;n=n->next)
         if (n->kind==FE_N_ERROR_DECL) fe_type_declare_error(&c->types,n);
-    check_type_cycles(c);
 }
 
 /* The unit's top-level names, in a scope of their own so that another unit
@@ -166,6 +165,9 @@ int fe_check_program(FeCheck *c)
     s.fn_node=0;
     fe_own_liveness_init(&s.liveness,&c->arena);
     for (u=0;u<c->build->count;++u) { enter_unit(c,u); declare_unit(c); }
+    /* Only now: a field may name a type in a unit that had not declared it
+       yet, and resolving it early would freeze the wrong answer in place. */
+    for (u=0;u<c->build->count;++u) { enter_unit(c,u); check_type_cycles(c); }
     fe_type_layout_all(&c->types);
     for (u=0;u<c->build->count;++u) {
         enter_unit(c,u);

fec/src/checkstm.c

@@ -140,18 +140,25 @@ void check_type_cycle(FeCheck *c, FeType *t)
     if (t->kind == FE_TYPE_ARRAY) {
         check_type_cycle(c,t->elem);
     } else if (t->kind == FE_TYPE_STRUCT) {
-        for (i=0;i<t->field_count;i++) {
+        int back=enter_declaring_unit(c,t->unit);
+        for (i=0;i<t->field_count;i++)
             if (!t->fields[i].type && t->fields[i].ast_node)
                 t->fields[i].type=fe_type_from_ast(&c->types,t->fields[i].ast_node->a);
-            check_type_cycle(c,t->fields[i].type);
+        if (back>=0) enter_unit(c,(unsigned)back);
-        }
+        for (i=0;i<t->field_count;i++) check_type_cycle(c,t->fields[i].type);
     } else if (t->kind == FE_TYPE_ENUM) {
+        int back=enter_declaring_unit(c,t->unit);
         for (i=0;i<t->variant_count;i++) {
             unsigned j;
-            for (j=0;j<t->variants[i].field_count;j++) {
+            for (j=0;j<t->variants[i].field_count;j++)
                 if (!t->variants[i].fields[j].type && t->variants[i].fields[j].ast_node)
                     t->variants[i].fields[j].type=fe_type_from_ast(&c->types,
                         t->variants[i].fields[j].ast_node->a);
+        }
+        if (back>=0) enter_unit(c,(unsigned)back);
+        for (i=0;i<t->variant_count;i++) {
+            unsigned j;
+            for (j=0;j<t->variants[i].field_count;j++) {
                 next=t->variants[i].fields[j].type;
                 check_type_cycle(c,next);
             }

fec/src/lower.c

@@ -113,9 +113,18 @@ unsigned as_address(Lower *L, Slot s, FeNode *n)
 /* Does letting go of this type have to do something? */
 int needs_release(const FeType *t)
 {
+    unsigned i;
     if (!t) return 0;
     if (t->kind == FE_TYPE_OWNED) return 1;
-    return t->has_drop != 0;
+    if (t->has_drop) return 1;
+    /* SPEC 5 R1: letting go of an owner lets go of what it owns. A struct that
+       holds an owner has something to do even when it says nothing itself --
+       which is what lets one type hold another that has a `drop`, since
+       calling `drop` by hand is not allowed. */
+    if (t->kind == FE_TYPE_STRUCT)
+        for (i = 0; i < t->field_count; ++i)
+            if (needs_release(t->fields[i].type)) return 1;
+    return 0;
 }
 
 int lower_reserve(Lower *L, void **items, unsigned *capacity, unsigned needed,

fec/src/lowerexp.c

@@ -171,6 +171,12 @@ Slot lower_expr_core(Lower *L, FeNode *n)
             if (base && (base->kind == FE_TYPE_REF ||
                          base->kind == FE_TYPE_OWNED)) base = base->elem;
             field = fe_type_field(base, n->b && n->b->text ? n->b->text : "");
+            /* `binding.name` is not a field of anything: it is a constant or a
+               global in another unit, and the checker already turned it into a
+               link name. */
+            if (!field && n->cname)
+                return slot_place(fe_ir_at_global(n->cname, 0), it,
+                                  ir_size(t));
             if (!field) { fail(L, "an unresolved field", n); return slot_void(); }
             b = lower_expr(L, n->a);
             if (n->a->sem_type && (n->a->sem_type->kind == FE_TYPE_REF ||
@@ -305,6 +311,37 @@ const char *drop_name(Lower *L, const FeType *t)
     return method->cname;
 }
 
+/* Let go of one value sitting at `at`. A type that says how to let go of
+   itself is asked first; then whatever it holds is let go of in turn, so a
+   struct that owns a struct that owns a buffer settles all three without
+   anyone writing a `drop` (SPEC 5 R1). */
+void release_at(Lower *L, const FeType *t, FeIrPlace at)
+{
+    unsigned args[1];
+    unsigned i;
+    if (!t) return;
+    if (t->has_drop) {
+        const char *how = drop_name(L, t);
+        args[0] = fe_ir_addr(L->m, L->b, at);
+        if (how) fe_ir_call(L->m, L->b, FE_IR_VOID, how, args, 1);
+    }
+    if (t->kind == FE_TYPE_OWNED) {
+        FeIrPlace p = at;
+        if (t->elem && t->elem->kind == FE_TYPE_SLICE)
+            p.offset += SLICE_PTR_OFFSET;
+        args[0] = fe_ir_load(L->m, L->b, FE_IR_PTR, p);
+        fe_ir_call(L->m, L->b, FE_IR_VOID, "fe_rt_free", args, 1);
+        return;
+    }
+    if (t->kind == FE_TYPE_STRUCT)
+        for (i = 0; i < t->field_count; ++i) {
+            FeIrPlace p = at;
+            if (!needs_release(t->fields[i].type)) continue;
+            p.offset += (long)t->fields[i].offset;
+            release_at(L, t->fields[i].type, p);
+        }
+}
+
 /* Settle what a scope owes, most recent first. A `return` in the middle of a
    function still owes everything, so every exit path calls this. */
 void run_deferred(Lower *L, unsigned from)
@@ -321,29 +358,10 @@ void run_deferred(Lower *L, unsigned from)
                                        fe_ir_at_local(L->owed[i - 1].flag, 0));
             FeIrBlock *doit = new_block(L);
             FeIrBlock *skip = new_block(L);
-            unsigned args[1];
             FeType *t = L->owed[i - 1].type;
             fe_ir_br(L->b, live, doit->id, skip->id);
             L->b = doit;
-            if (t && t->kind == FE_TYPE_OWNED && t->elem &&
+            release_at(L, t, fe_ir_at_local(L->owed[i - 1].local, 0));
-                t->elem->kind == FE_TYPE_SLICE) {
-                FeIrPlace at = fe_ir_at_local(L->owed[i - 1].local,
-                                              SLICE_PTR_OFFSET);
-                args[0] = fe_ir_load(L->m, L->b, FE_IR_PTR, at);
-            } else {
-                args[0] = fe_ir_load(L->m, L->b, FE_IR_PTR,
-                                     fe_ir_at_local(L->owed[i - 1].local, 0));
-            }
-            if (t && t->has_drop) {
-                /* A type that says how to let go of itself is asked to; the
-                   name is the one its instance was given. */
-                const char *how = drop_name(L, t);
-                args[0] = fe_ir_addr(L->m, L->b,
-                                     fe_ir_at_local(L->owed[i - 1].local, 0));
-                if (how) fe_ir_call(L->m, L->b, FE_IR_VOID, how, args, 1);
-            } else {
-                fe_ir_call(L->m, L->b, FE_IR_VOID, "fe_rt_free", args, 1);
-            }
             fe_ir_jmp(L->b, skip->id);
             L->b = skip;
         }

fec/src/lowerpri.h

@@ -145,6 +145,7 @@ Slot lower_call(Lower *L, FeNode *n);
 Slot lower_expr(Lower *L, FeNode *n);
 Slot lower_expr_core(Lower *L, FeNode *n);
 const char *drop_name(Lower *L, const FeType *t);
+void release_at(Lower *L, const FeType *t, FeIrPlace at);
 void run_deferred(Lower *L, unsigned from);
 Slot wrap_context(Lower *L, Slot v, FeNode *n);
 unsigned wrapper_tag(Lower *L, Slot w, const FeType *t, FeNode *n);

fec/src/types.c

@@ -47,6 +47,7 @@ static FeType *new_type(FeTypeCtx *ctx, const char *name, FeTypeKind kind)
     t->next = ctx->types;
     t->emit_state = 0;
     t->cycle_state = 0;
+    t->layout_state = 0;
     ctx->types = t;
     return t;
 }
@@ -61,6 +62,8 @@ void fe_types_init(FeTypeCtx *ctx, FeArena *arena, unsigned pointer_bits)
     ctx->param_count = 0;
     ctx->instantiate = 0;
     ctx->instantiate_owner = 0;
+    ctx->enter_decl = 0;
+    ctx->leave_decl = 0;
 }
 
 /* Does this type answer to `name` for someone checking `unit`? A type with no
@@ -452,6 +455,26 @@ unsigned fe_type_align(const FeType *t)
     return t && t->align ? t->align : 1U;
 }
 
+/* Resolve this type's fields where they were written. Without the callback
+   installed -- or for a type nobody declared -- everything stays where it is,
+   which is what the non-checking users of this layer want. */
+static int enter_decl_unit(FeTypeCtx *ctx, const char *unit, const char **was)
+{
+    *was = ctx->unit_name;
+    if (!ctx->enter_decl || !unit) return -1;
+    return ctx->enter_decl(ctx->instantiate_owner, unit);
+}
+
+/* Put back both halves: the unit the checker was in, and the name this layer
+   was interning under -- an instantiation moves the second without the
+   first, so restoring one is not restoring the other. */
+static void leave_decl_unit(FeTypeCtx *ctx, int back, const char *was)
+{
+    if (back >= 0 && ctx->leave_decl)
+        ctx->leave_decl(ctx->instantiate_owner, back);
+    ctx->unit_name = was;
+}
+
 static void layout_type(FeTypeCtx *ctx, FeType *t)
 {
     unsigned i;
@@ -460,14 +483,14 @@ static void layout_type(FeTypeCtx *ctx, FeType *t)
     unsigned long max_size;
     unsigned max_align;
     if (!t || t->size) return;
-    if (t->cycle_state == 1) {
+    if (t->layout_state == 1) {
         t->size = 1;
         t->align = 1;
         return;
     }
-    t->cycle_state = 1;
+    t->layout_state = 1;
     if (t->kind == FE_TYPE_VOID || t->kind == FE_TYPE_UNKNOWN ||
-        t->kind == FE_TYPE_ERROR) { t->size = 0; t->align = 1; t->cycle_state = 2; return; }
+        t->kind == FE_TYPE_ERROR) { t->size = 0; t->align = 1; t->layout_state = 2; return; }
     if (t->kind == FE_TYPE_ERROR_UNION) {
         if (t->error_value && t->error_value->kind != FE_TYPE_VOID) {
             layout_type(ctx,t->error_value);
@@ -478,7 +501,7 @@ static void layout_type(FeTypeCtx *ctx, FeType *t)
             t->size=2;
             t->align=2U;
         }
-        t->cycle_state = 2; return;
+        t->layout_state = 2; return;
     }
     if (t->kind == FE_TYPE_OPTIONAL) {
         layout_type(ctx,t->elem);
@@ -490,44 +513,48 @@ static void layout_type(FeTypeCtx *ctx, FeType *t)
             t->size=round_up(1UL,t->align)+fe_type_size(t->elem);
             t->size=round_up(t->size,t->align);
         }
-        t->cycle_state=2; return;
+        t->layout_state=2; return;
     }
     if (t->kind == FE_TYPE_BOOL || t->kind == FE_TYPE_CHAR) {
-        t->size = 1; t->align = 1; t->cycle_state = 2; return;
+        t->size = 1; t->align = 1; t->layout_state = 2; return;
     }
     if (t->kind == FE_TYPE_INT) {
         t->size = (t->bits + 7U) / 8U;
         t->align = t->size;
         if (t->size > 4UL) t->size = 4UL;
-        t->cycle_state = 2; return;
+        t->layout_state = 2; return;
     }
     if (t->kind == FE_TYPE_REF || t->kind == FE_TYPE_RAW) {
         t->size = FE_PTR_SIZE;
         t->align = FE_PTR_ALIGN;
-        t->cycle_state = 2; return;
+        t->layout_state = 2; return;
     }
     if (t->kind == FE_TYPE_OWNED) {
         t->size = t->elem && t->elem->kind==FE_TYPE_SLICE ?
             2UL * FE_PTR_SIZE : FE_PTR_SIZE;
         t->align = FE_PTR_ALIGN;
-        t->cycle_state = 2; return;
+        t->layout_state = 2; return;
     }
     if (t->kind == FE_TYPE_SLICE || t->kind == FE_TYPE_STR) {
         t->size = 2UL * FE_PTR_SIZE;
         t->align = FE_PTR_ALIGN;
-        t->cycle_state = 2; return;
+        t->layout_state = 2; return;
     }
     if (t->kind == FE_TYPE_ARRAY) {
         layout_type(ctx, t->elem);
         t->align = t->packed ? 1U : fe_type_align(t->elem);
         t->size = t->length * fe_type_size(t->elem);
-        t->cycle_state = 2; return;
+        t->layout_state = 2; return;
     }
     if (t->kind == FE_TYPE_STRUCT) {
-        off = 0; max_align = 1;
+        const char *was;
-        for (i = 0; i < t->field_count; ++i) {
+        int back = enter_decl_unit(ctx, t->unit, &was);
+        for (i = 0; i < t->field_count; ++i)
             if (!t->fields[i].type && t->fields[i].ast_node)
                 t->fields[i].type = fe_type_from_ast(ctx, t->fields[i].ast_node->a);
+        leave_decl_unit(ctx, back, was);
+        off = 0; max_align = 1;
+        for (i = 0; i < t->field_count; ++i) {
             layout_type(ctx, t->fields[i].type);
             align = t->packed ? 1U : fe_type_align(t->fields[i].type);
             if (align > max_align) max_align = align;
@@ -537,18 +564,25 @@ static void layout_type(FeTypeCtx *ctx, FeType *t)
         }
         t->align = max_align;
         t->size = round_up(off, max_align);
-        t->cycle_state = 2;
+        t->layout_state = 2;
         return;
     }
     if (t->kind == FE_TYPE_ENUM) {
+        const char *was;
+        int back = enter_decl_unit(ctx, t->unit, &was);
+        for (i = 0; i < t->variant_count; ++i) {
+            unsigned j;
+            for (j = 0; j < t->variants[i].field_count; ++j)
+                if (!t->variants[i].fields[j].type && t->variants[i].fields[j].ast_node)
+                    t->variants[i].fields[j].type = fe_type_from_ast(
+                        ctx, t->variants[i].fields[j].ast_node->a);
+        }
+        leave_decl_unit(ctx, back, was);
         max_size = 0; max_align = 1;
         for (i = 0; i < t->variant_count; ++i) {
             unsigned j;
             off = 0;
             for (j = 0; j < t->variants[i].field_count; ++j) {
-                if (!t->variants[i].fields[j].type && t->variants[i].fields[j].ast_node)
-                    t->variants[i].fields[j].type = fe_type_from_ast(
-                        ctx, t->variants[i].fields[j].ast_node->a);
                 layout_type(ctx, t->variants[i].fields[j].type);
                 if (fe_type_align(t->variants[i].fields[j].type) > max_align)
                     max_align = fe_type_align(t->variants[i].fields[j].type);
@@ -564,7 +598,7 @@ static void layout_type(FeTypeCtx *ctx, FeType *t)
         off = round_up(t->bits / 8U, max_align);
         t->size = round_up(off + max_size, max_align);
         t->align = max_align;
-        t->cycle_state = 2;
+        t->layout_state = 2;
     }
 }
 

fec/src/types.h

@@ -98,6 +98,11 @@ struct FeType {
     FeType *next;
     int emit_state;
     int cycle_state;
+    /* Separate from `cycle_state`: the checker's by-value recursion walk and
+       this layer's size computation run inside one another, and sharing one
+       marker made a struct in the middle of the first look complete to the
+       second -- one byte wide, with every field on top of the next. */
+    int layout_state;
 };
 
 typedef struct FeTypeCtx {
@@ -114,6 +119,12 @@ typedef struct FeTypeCtx {
        so it installs this and the type layer calls back into it. */
     FeType *(*instantiate)(void *owner, const FeNode *node);
     void *instantiate_owner;
+    /* A field type is written in the unit that declared it, so resolving one
+       has to happen with that unit's imports in scope. The checker owns that
+       knowledge, so it installs this pair and the type layer calls back.
+       `enter` answers with what to hand `leave`, or -1 for "stayed put". */
+    int (*enter_decl)(void *owner, const char *unit);
+    void (*leave_decl)(void *owner, int back);
 } FeTypeCtx;
 
 void fe_types_init(FeTypeCtx *ctx, FeArena *arena, unsigned pointer_bits);

fec/std/list.fe

@@ -1,8 +1,10 @@
 unit std.list;
 
 // A growable sequence. The buffer is owned, so a List owns its elements and
-// releasing it releases them (SPEC 5 R1). Growth doubles, which keeps the
+// releasing it releases them (SPEC 5 R1) -- which is also why there is no
-// total copying proportional to the number of pushes.
+// `drop` here: letting go of a List lets go of its buffer on its own. Growth
+// doubles, which keeps the total copying proportional to the number of
+// pushes.
 
 pub struct List(T) {
     items: ^[]mut T,
@@ -46,8 +48,4 @@ pub struct List(T) {
         mem.destroy(old);
         return;
     }
-
-    pub fn drop(self: &mut Self) -> void {
-        mem.destroy(self.items);
-    }
 }

fec/std/map.fe

@@ -6,7 +6,8 @@ unit std.map;
 // wrong shape for that. Keys are copied into one buffer the map owns and each
 // slot records where in it the key sits -- the arena-and-handle shape R11 asks
 // for, which also means letting go of the map is two frees and not one per
-// entry.
+// entry. Both buffers are owned, so R1 does the letting go and no `drop` is
+// written here.
 //
 // Open addressing with linear probing. The table is a power of two so the
 // index is a mask rather than a division, and it grows at three quarters full
@@ -155,11 +156,6 @@ pub struct Map(V) {
         mem.destroy(old);
         return;
     }
-
-    pub fn drop(self: &mut Self) -> void {
-        mem.destroy(self.slots);
-        mem.destroy(self.bytes);
-    }
 }
 
 /// FNV-1a. Small, fast, and good enough for identifiers; nothing here has to

fec/std/mem.fe

@@ -10,7 +10,8 @@ import std.sys;
 /// SPEC R11 answers recursive and graph-shaped data with an arena that owns
 /// the values and integer handles that reference them. This is that arena. A
 /// handle is an offset, so it stays valid while the arena does, and comparing
-/// two handles is comparing two numbers.
+/// two handles is comparing two numbers. The block itself is owned, so nothing
+/// here says how to let go of it -- R1 already does.
 pub struct Arena {
     bytes: ^[]mut u8,
     used: usize,
@@ -56,5 +57,4 @@ pub struct Arena {
     /// that is the trade an arena makes.
     pub fn reset(self: &mut Self) -> void { self.used = 0; }
 
-    pub fn drop(self: &mut Self) -> void { mem.destroy(self.bytes); }
 }

fec/tests/exec/lexer/ast.fe

@@ -0,0 +1,49 @@
+unit ast;
+
+// The tree the parser builds.
+//
+// Nodes live in one growing array and refer to each other by index, which is
+// what SPEC R11 asks for: an owner that holds the values and handles that
+// point at them. A node cannot hold a `^Node` for its children because a node
+// has several and they are not each owned once; it cannot hold a `&Node`
+// because R4 keeps borrows out of aggregate storage. An index is neither.
+
+pub enum Shape {
+    Unit,       // a: name
+    Fn,         // a: name, b: first statement
+    Let,        // a: name, b: value
+    Return,     // a: value, or NONE
+    Binary,     // a: left, b: right, from/len: the operator
+    Number,
+    Name,
+    Text,
+    Error,
+}
+
+/// No node. Zero is a real index, so the empty handle is the largest one.
+pub const NONE: usize = 4294967295;
+
+pub struct Node {
+    pub shape: Shape,
+    pub from: usize,        // where in the source this came from
+    pub len: usize,
+    pub line: usize,
+    pub a: usize,           // handles into the same tree
+    pub b: usize,
+    pub next: usize,        // the following statement, when there is one
+}
+
+pub fn name_of(s: Shape) -> []u8 {
+    match s {
+        Unit => { return "unit"; }
+        Fn => { return "fn"; }
+        Let => { return "let"; }
+        Return => { return "return"; }
+        Binary => { return "binary"; }
+        Number => { return "number"; }
+        Name => { return "name"; }
+        Text => { return "text"; }
+        Error => { return "error"; }
+    }
+    return "?";
+}

fec/tests/exec/lexer/parse.fe

@@ -0,0 +1,256 @@
+unit parse;
+
+import std.list;
+import std.str;
+import tok;
+import scan;
+import ast;
+
+// Recursive descent over the lexer's tokens.
+//
+// The source is not a field: R4 keeps borrows out of aggregate storage, so
+// `src` is passed to every step, the same way the lexer passes it. What the
+// parser does own is the node array, and every parent points at its children
+// by index into it.
+//
+// One token of lookahead lives in `cur`. That is all this grammar needs.
+
+pub struct Parser {
+    nodes: list.List(ast.Node),
+    at: usize,
+    line: usize,
+    cur: tok.Token,
+    pub errors: usize,
+
+    pub fn on(src: []u8) -> !Self {
+        let nodes: list.List(ast.Node) =
+            try list.List(ast.Node).with_capacity(16);
+        var p: Self = Self{
+            nodes: nodes,
+            at: 0,
+            line: 1,
+            cur: tok.Token{ kind: tok.Kind.End, from: 0, len: 0, line: 1 },
+            errors: 0,
+        };
+        p.bump(src);
+        return p;
+    }
+
+    fn bump(self: &mut Self, src: []u8) -> void {
+        self.cur = scan.next(src, &mut self.at, &mut self.line);
+        return;
+    }
+
+    fn done(self: &Self) -> bool { return self.cur.kind == tok.Kind.End; }
+
+    /// Is the token in hand this exact spelling?
+    fn is(self: &Self, src: []u8, want: []u8) -> bool {
+        return str.eq(tok.text(src, self.cur), want);
+    }
+
+    /// Take the token in hand if it is this spelling; say whether it was.
+    fn eat(self: &mut Self, src: []u8, want: []u8) -> bool {
+        if not self.is(src, want) { return false; }
+        self.bump(src);
+        return true;
+    }
+
+    /// Demand this spelling. A miss is counted and the token stays put, so the
+    /// caller decides how to get back on its feet.
+    fn want(self: &mut Self, src: []u8, w: []u8) -> bool {
+        if self.eat(src, w) { return true; }
+        self.errors = self.errors + 1;
+        return false;
+    }
+
+    fn add(self: &mut Self, s: ast.Shape, t: tok.Token, a: usize,
+           b: usize) -> !usize {
+        let n: ast.Node = ast.Node{
+            shape: s, from: t.from, len: t.len, line: t.line,
+            a: a, b: b, next: ast.NONE,
+        };
+        let i: usize = self.nodes.count();
+        try self.nodes.push(n);
+        return i;
+    }
+
+    // -- reading the tree back ------------------------------------------
+
+    pub fn count(self: &Self) -> usize { return self.nodes.count(); }
+
+    pub fn node(self: &Self, i: usize) -> ast.Node {
+        return self.nodes.at(i);
+    }
+
+
+    // -- the grammar ----------------------------------------------------
+    //
+    //   unit   := "unit" NAME ";" item*
+    //   item   := "fn" NAME "(" ")" block
+    //   block  := "{" stmt* "}"
+    //   stmt   := "let" NAME "=" expr ";" | "return" expr? ";"
+    //   expr   := term (("+" | "-") term)*
+    //   term   := factor (("*" | "/") factor)*
+    //   factor := NUMBER | NAME | TEXT | "(" expr ")"
+
+    pub fn unit_decl(self: &mut Self, src: []u8) -> !usize {
+        let ok: bool = self.want(src, "unit");
+        let name: tok.Token = self.cur;
+        if ok { self.bump(src); }
+        let semi: bool = self.want(src, ";");
+        let root: usize = try self.add(ast.Shape.Unit, name, ast.NONE,
+                                       ast.NONE);
+        var first: usize = ast.NONE;
+        var last: usize = ast.NONE;
+        while not self.done() {
+            let it: usize = try self.item(src);
+            if first == ast.NONE { first = it; }
+            else { self.link(last, it); }
+            last = it;
+        }
+        self.set_a(root, first);
+        return root;
+    }
+
+    /// Point one statement or item at the one after it.
+    fn link(self: &mut Self, from: usize, to: usize) -> void {
+        var n: ast.Node = self.nodes.at(from);
+        n.next = to;
+        self.nodes.set(from, n);
+        return;
+    }
+
+    fn set_a(self: &mut Self, at: usize, a: usize) -> void {
+        var n: ast.Node = self.nodes.at(at);
+        n.a = a;
+        self.nodes.set(at, n);
+        return;
+    }
+
+    fn item(self: &mut Self, src: []u8) -> !usize {
+        if not self.is(src, "fn") {
+            let bad: tok.Token = self.cur;
+            self.errors = self.errors + 1;
+            self.skip_stmt(src);
+            return try self.add(ast.Shape.Error, bad, ast.NONE, ast.NONE);
+        }
+        self.bump(src);
+        let name: tok.Token = self.cur;
+        self.bump(src);
+        let open: bool = self.want(src, "(");
+        let close: bool = self.want(src, ")");
+        let body: usize = try self.block(src);
+        return try self.add(ast.Shape.Fn, name, ast.NONE, body);
+    }
+
+    fn block(self: &mut Self, src: []u8) -> !usize {
+        let open: bool = self.want(src, "{");
+        var first: usize = ast.NONE;
+        var last: usize = ast.NONE;
+        while true {
+            if self.done() { break; }
+            if self.is(src, "}") { break; }
+            let s: usize = try self.stmt(src);
+            if first == ast.NONE { first = s; }
+            else { self.link(last, s); }
+            last = s;
+        }
+        let close: bool = self.want(src, "}");
+        return first;
+    }
+
+    fn stmt(self: &mut Self, src: []u8) -> !usize {
+        if self.is(src, "let") {
+            self.bump(src);
+            let name: tok.Token = self.cur;
+            self.bump(src);
+            let has_eq: bool = self.want(src, "=");
+            let value: usize = try self.expr(src);
+            let semi: bool = self.want(src, ";");
+            return try self.add(ast.Shape.Let, name, ast.NONE, value);
+        }
+        if self.is(src, "return") {
+            let head: tok.Token = self.cur;
+            self.bump(src);
+            if self.is(src, ";") {
+                self.bump(src);
+                return try self.add(ast.Shape.Return, head, ast.NONE,
+                                    ast.NONE);
+            }
+            let value: usize = try self.expr(src);
+            let semi: bool = self.want(src, ";");
+            return try self.add(ast.Shape.Return, head, value, ast.NONE);
+        }
+        let bad: tok.Token = self.cur;
+        self.errors = self.errors + 1;
+        self.skip_stmt(src);
+        return try self.add(ast.Shape.Error, bad, ast.NONE, ast.NONE);
+    }
+
+    /// Get back to a statement boundary after something unrecognised. Stopping
+    /// at `;` or `}` means one bad statement costs one diagnostic, not a run
+    /// of them.
+    fn skip_stmt(self: &mut Self, src: []u8) -> void {
+        while not self.done() {
+            if self.is(src, "}") { return; }
+            if self.is(src, ";") { self.bump(src); return; }
+            self.bump(src);
+        }
+        return;
+    }
+
+    fn expr(self: &mut Self, src: []u8) -> !usize {
+        var left: usize = try self.term(src);
+        while true {
+            if self.done() { break; }
+            let plus: bool = self.is(src, "+");
+            let minus: bool = self.is(src, "-");
+            if not plus and not minus { break; }
+            let op: tok.Token = self.cur;
+            self.bump(src);
+            let right: usize = try self.term(src);
+            left = try self.add(ast.Shape.Binary, op, left, right);
+        }
+        return left;
+    }
+
+    fn term(self: &mut Self, src: []u8) -> !usize {
+        var left: usize = try self.factor(src);
+        while true {
+            if self.done() { break; }
+            let star: bool = self.is(src, "*");
+            let slash: bool = self.is(src, "/");
+            if not star and not slash { break; }
+            let op: tok.Token = self.cur;
+            self.bump(src);
+            let right: usize = try self.factor(src);
+            left = try self.add(ast.Shape.Binary, op, left, right);
+        }
+        return left;
+    }
+
+    fn factor(self: &mut Self, src: []u8) -> !usize {
+        let t: tok.Token = self.cur;
+        if t.kind == tok.Kind.Number {
+            self.bump(src);
+            return try self.add(ast.Shape.Number, t, ast.NONE, ast.NONE);
+        }
+        if t.kind == tok.Kind.Name {
+            self.bump(src);
+            return try self.add(ast.Shape.Name, t, ast.NONE, ast.NONE);
+        }
+        if t.kind == tok.Kind.Text {
+            self.bump(src);
+            return try self.add(ast.Shape.Text, t, ast.NONE, ast.NONE);
+        }
+        if self.is(src, "(") {
+            self.bump(src);
+            let inner: usize = try self.expr(src);
+            let close: bool = self.want(src, ")");
+            return inner;
+        }
+        self.errors = self.errors + 1;
+        self.bump(src);
+        return try self.add(ast.Shape.Error, t, ast.NONE, ast.NONE);
+    }
+}

fec/tests/exec/lexer/tree.fe

@@ -0,0 +1,107 @@
+// EXIT:0
+// OUTPUT:unit demo
+// OUTPUT:fn answer @2
+// OUTPUT:  let n = (+ 1 (* 2 3))
+// OUTPUT:  return n
+// OUTPUT:fn greet @3
+// OUTPUT:  let s = "hi"
+// OUTPUT:  return
+// OUTPUT:fn muddle @4
+// OUTPUT:  let x = <error>
+// OUTPUT:nodes 17 errors 2
+// OUTPUT:balanced
+unit tree;
+
+import std.io;
+import std.sys;
+import tok;
+import ast;
+import parse;
+
+// The Ferro parser, written in Ferro.
+//
+// The lexer next to this file showed that a token can say where it came from
+// instead of holding the text. A tree is the same idea one level up: a node
+// cannot hold `^Node` children -- it has several and owns none of them once --
+// and R4 keeps `&Node` out of aggregate storage. So the parser owns one array
+// of nodes and every child is an index into it.
+//
+// Printing the expressions back in prefix form is the point of the test: it is
+// the only way to see that `1 + 2 * 3` bound the way the grammar says.
+
+const SOURCE: str = "unit demo;\nfn answer() { let n = 1 + 2 * 3; return n; }\nfn greet() { let s = \"hi\"; return; }\nfn muddle() { let x = ; }\n";
+
+fn show_expr(p: &parse.Parser, src: []u8, i: usize) -> void {
+    if i == ast.NONE { return; }
+    let n: ast.Node = p.node(i);
+    match n.shape {
+        Binary => {
+            @print("({} ", src[n.from..n.from + n.len]);
+            show_expr(p, src, n.a);
+            @print(" ");
+            show_expr(p, src, n.b);
+            @print(")");
+        }
+        Error => { @print("<error>"); }
+        _ => { @print("{}", src[n.from..n.from + n.len]); }
+    }
+    return;
+}
+
+fn show_stmt(p: &parse.Parser, src: []u8, i: usize) -> void {
+    let n: ast.Node = p.node(i);
+    match n.shape {
+        Let => {
+            @print("  let {} = ", src[n.from..n.from + n.len]);
+            show_expr(p, src, n.b);
+            @print("\n");
+        }
+        Return => {
+            if n.a == ast.NONE { @print("  return\n"); }
+            else {
+                @print("  return ");
+                show_expr(p, src, n.a);
+                @print("\n");
+            }
+        }
+        _ => { @print("  <error>\n"); }
+    }
+    return;
+}
+
+fn show_item(p: &parse.Parser, src: []u8, i: usize) -> void {
+    let n: ast.Node = p.node(i);
+    match n.shape {
+        Fn => {
+            @print("fn {} @{}\n", src[n.from..n.from + n.len], n.line);
+            var s: usize = n.b;
+            while s != ast.NONE {
+                show_stmt(p, src, s);
+                s = p.node(s).next;
+            }
+        }
+        _ => { @print("  <error>\n"); }
+    }
+    return;
+}
+
+fn run() -> !void {
+    var p: parse.Parser = try parse.Parser.on(SOURCE);
+    let root: usize = try p.unit_decl(SOURCE);
+    let head: ast.Node = p.node(root);
+    @print("unit {}\n", SOURCE[head.from..head.from + head.len]);
+    var it: usize = head.a;
+    while it != ast.NONE {
+        show_item(&p, SOURCE, it);
+        it = p.node(it).next;
+    }
+    @print("nodes {} errors {}\n", p.count(), p.errors);
+    return;
+}
+
+fn main() -> i32 {
+    run() catch |e| { @print("out of memory\n"); return 1; };
+    if sys.allocs() == sys.frees() { @print("balanced\n"); }
+    else { @print("leaked {}\n", sys.allocs() - sys.frees()); }
+    return 0;
+}