GOAL P3-4: StringInterner, 그리고 그것이 드러낸 배치 버그

이름 하나당 사본 하나와 그것을 대신하는 숫자. 컴파일러는 이름을 끊임없이
비교하고 사방에 저장하는데, StrId 둘을 비교하는 것은 정수 둘을 비교하는
것이고 하나를 저장하는 것은 4바이트에 소유권 없음이다.

str 을 꺼내는 API 를 일부러 두지 않았다. 텍스트를 빌리는 것은 interner 를
빌리는 것인데, interner 는 이름을 든 채로 계속 더 넣고 싶은 바로 그 물건이다
-- 파서는 식별자를 읽으면서 같은 숨에 다음 것을 등록한다. 텍스트를 열어서
하려던 일은 전부 여기 있다: eq, len_of, hash_of, find, copy_into.

그리고 이것이 제네릭 인스턴스를 필드로 담는 구조체를 통째로 깨뜨리던 버그를
드러냈다.

  Holder{ bytes: ^[]mut u8, used: usize, seen: map.Map(u32) }
  36 바이트여야 하는데 12 로 잡혔다.

Map(u32) 를 짓는 중에 그 안의 Slot(u32) 를 인스턴스화하면 거기서 배치 패스가
다시 돈다. 그때 Map(u32) 는 field_count 는 4 인데 필드 배열이 아직 아무것도
말하지 않는 상태라, 크기 0 으로 확정되고 굳었다. 이미 크기가 있는 타입은
아무도 다시 계산하지 않으니 Holder 는 그 0 을 읽었다.

셋을 고쳤다: 짓는 중인 인스턴스는 building 을 세워 배치를 거절하고, 멤버가
아직 자리를 못 잡은 집합 타입은 틀린 답으로 굳느니 물러나며, 배치 패스는
움직임이 없을 때까지 돈다.

245/245, 38/38.

fec/src/checkgen.c

@@ -210,6 +210,7 @@ FeType *build_struct_instance(FeCheck *c, FeUnit *home, FeNode *decl,
     t=fe_type_intern_unit(&c->types,home->name,key);
     if (!t || t->kind!=FE_TYPE_UNKNOWN) return t;
     t->kind=FE_TYPE_STRUCT;
+    t->building=1;
     t->packed=(decl->flags & FE_NODE_PACKED)!=0;
     t->decl_node=decl;
     t->bind_count=0;
@@ -246,6 +247,7 @@ FeType *build_struct_instance(FeCheck *c, FeUnit *home, FeNode *decl,
         pop_bindings(c,&save);
         c->types.unit_name=save_unit;
     }
+    t->building=0;
     fe_type_layout_all(&c->types);
     /* A type that says how to let go of itself needs that method to exist for
        every instance, whether or not anyone calls it by name: scope cleanup

fec/src/types.c

@@ -48,6 +48,7 @@ static FeType *new_type(FeTypeCtx *ctx, const char *name, FeTypeKind kind)
     t->emit_state = 0;
     t->cycle_state = 0;
     t->layout_state = 0;
+    t->building = 0;
     ctx->types = t;
     return t;
 }
@@ -475,6 +476,28 @@ static void leave_decl_unit(FeTypeCtx *ctx, int back, const char *was)
     ctx->unit_name = was;
 }
 
+/* Did every field end up with a size? A struct whose members are not settled
+   cannot be settled either -- and freezing it here is worse than leaving it,
+   because nothing recomputes a type that already has a size. */
+static int members_ready(const FeType *t)
+{
+    unsigned i;
+    unsigned j;
+    if (t->kind == FE_TYPE_STRUCT) {
+        for (i = 0; i < t->field_count; ++i) {
+            if (!t->fields[i].type) return 0;
+            if (t->fields[i].type->layout_state != 2) return 0;
+        }
+        return 1;
+    }
+    for (i = 0; i < t->variant_count; ++i)
+        for (j = 0; j < t->variants[i].field_count; ++j) {
+            if (!t->variants[i].fields[j].type) return 0;
+            if (t->variants[i].fields[j].type->layout_state != 2) return 0;
+        }
+    return 1;
+}
+
 static void layout_type(FeTypeCtx *ctx, FeType *t)
 {
     unsigned i;
@@ -483,6 +506,7 @@ static void layout_type(FeTypeCtx *ctx, FeType *t)
     unsigned long max_size;
     unsigned max_align;
     if (!t || t->size) return;
+    if (t->building) return;
     if (t->layout_state == 1) {
         t->size = 1;
         t->align = 1;
@@ -562,6 +586,7 @@ static void layout_type(FeTypeCtx *ctx, FeType *t)
             t->fields[i].offset = off;
             off += fe_type_size(t->fields[i].type);
         }
+        if (!members_ready(t)) { t->layout_state = 0; return; }
         t->align = max_align;
         t->size = round_up(off, max_align);
         t->layout_state = 2;
@@ -594,6 +619,7 @@ static void layout_type(FeTypeCtx *ctx, FeType *t)
             }
             if (off > max_size) max_size = off;
         }
+        if (!members_ready(t)) { t->layout_state = 0; return; }
         t->bits = t->variant_count > 256U ? 16U : 8U;
         off = round_up(t->bits / 8U, max_align);
         t->size = round_up(off + max_size, max_align);
@@ -605,7 +631,20 @@ static void layout_type(FeTypeCtx *ctx, FeType *t)
 void fe_type_layout_all(FeTypeCtx *ctx)
 {
     FeType *t;
-    for (t = ctx->types; t; t = t->next) layout_type(ctx, t);
+    int again = 1;
+    unsigned rounds = 0;
+    /* One pass settles a type only if everything under it is already settled,
+       so a type that had to wait is picked up by the next round. Sixteen is
+       far past any real nesting; it is here so a cycle cannot spin. */
+    while (again && rounds < 16U) {
+        again = 0;
+        for (t = ctx->types; t; t = t->next) {
+            if (t->size || t->layout_state == 2) continue;
+            layout_type(ctx, t);
+            if (t->layout_state == 2) again = 1;
+        }
+        ++rounds;
+    }
 }
 
 FeFieldType *fe_type_field(FeType *t, const char *name)

fec/src/types.h

@@ -103,6 +103,10 @@ struct FeType {
        marker made a struct in the middle of the first look complete to the
        second -- one byte wide, with every field on top of the next. */
     int layout_state;
+    /* Set while a generic instance is being filled in. Its field array
+       exists but says nothing yet, and a size taken from it would be
+       wrong and would then be frozen. */
+    int building;
 };
 
 typedef struct FeTypeCtx {

fec/std/intern.fe

@@ -0,0 +1,131 @@
+unit std.intern;
+
+import std.map;
+import std.mem;
+
+// One copy of every distinct name, and a number that stands for it.
+//
+// A compiler compares names constantly and stores them everywhere. Comparing
+// two `StrId` is comparing two integers; storing one costs four bytes and no
+// ownership. That is the whole point.
+//
+// There is deliberately no way to get a `str` back out. A borrow of the text
+// would be a borrow of the interner, and the interner is exactly the thing you
+// want to keep adding to while holding names -- the parser reads an identifier
+// and registers the next one in the same breath. Everything you would open the
+// text for is here instead: compare it, measure it, hash it, write it.
+
+/// A name, as a number. Copy, four bytes, and meaningless to any other
+/// interner -- which is fine, because a program has one.
+pub struct StrId {
+    pub raw: u32,
+
+    pub fn same(self: &Self, other: StrId) -> bool {
+        return self.raw == other.raw;
+    }
+}
+
+/// No name.
+pub const NONE: u32 = 4294967295;
+
+struct Entry {
+    at: usize,
+    len: usize,
+}
+
+pub struct Interner {
+    /// Every name end to end. Nothing is ever removed, so an offset stays
+    /// good for as long as the interner does.
+    bytes: ^[]mut u8,
+    used: usize,
+    names: ^[]mut Entry,
+    count: usize,
+    /// Text to id, so interning the same name twice gives the same number.
+    seen: map.Map(u32),
+
+    pub fn with_capacity(n: usize) -> !Self {
+        let text: ^[]mut u8 = try mem.alloc_slice(u8, n * 8);
+        let table: ^[]mut Entry = try mem.alloc_slice(Entry, n);
+        let index: map.Map(u32) = try map.Map(u32).with_capacity(n);
+        return Self{ bytes: text, used: 0, names: table, count: 0,
+                     seen: index };
+    }
+
+    pub fn count_of(self: &Self) -> usize { return self.count; }
+
+    /// The number for this name, making one if it is new.
+    pub fn intern(self: &mut Self, text: []u8) -> !StrId {
+        let found: u32 = self.seen.get(text, NONE);
+        if found != NONE { return StrId{ raw: found }; }
+        if self.count == self.names.^.n { return error.OutOfMemory; }
+        let at: usize = self.used;
+        if at + text.n > self.bytes.^.n { return error.OutOfMemory; }
+        var i: usize = 0;
+        while i < text.n {
+            self.bytes.^[at + i] = text[i];
+            i = i + 1;
+        }
+        let id: usize = self.count;
+        self.names.^[id].at = at;
+        self.names.^[id].len = text.n;
+        self.used = at + text.n;
+        self.count = id + 1;
+        try self.seen.put(text, id as u32);
+        return StrId{ raw: id as u32 };
+    }
+
+    /// Is this a name it has seen? `NONE` when not.
+    pub fn find(self: &Self, text: []u8) -> u32 {
+        return self.seen.get(text, NONE);
+    }
+
+    pub fn len_of(self: &Self, id: StrId) -> usize {
+        if (id.raw as usize) >= self.count { return 0; }
+        return self.names.^[id.raw as usize].len;
+    }
+
+    /// Does this id spell this text? The comparison every `if name == "fn"`
+    /// in a parser turns into.
+    pub fn eq(self: &Self, id: StrId, text: []u8) -> bool {
+        if (id.raw as usize) >= self.count { return false; }
+        let e: usize = id.raw as usize;
+        if self.names.^[e].len != text.n { return false; }
+        var i: usize = 0;
+        while i < text.n {
+            if self.bytes.^[self.names.^[e].at + i] != text[i] { return false; }
+            i = i + 1;
+        }
+        return true;
+    }
+
+    /// FNV-1a over the stored bytes, for anything that wants to bucket names
+    /// without opening them.
+    pub fn hash_of(self: &Self, id: StrId) -> u32 {
+        if (id.raw as usize) >= self.count { return 0; }
+        let e: usize = id.raw as usize;
+        var h: u32 = 2166136261;
+        var i: usize = 0;
+        while i < self.names.^[e].len {
+            h = h ^ (self.bytes.^[self.names.^[e].at + i] as u32);
+            h = h * 16777619;
+            i = i + 1;
+        }
+        return h;
+    }
+
+    /// Copy the name into `out` and say how many bytes it took. This is how a
+    /// name reaches a diagnostic without the interner being borrowed past the
+    /// statement.
+    pub fn copy_into(self: &Self, id: StrId, out: []mut u8) -> usize {
+        if (id.raw as usize) >= self.count { return 0; }
+        let e: usize = id.raw as usize;
+        var n: usize = self.names.^[e].len;
+        if n > out.n { n = out.n; }
+        var i: usize = 0;
+        while i < n {
+            out[i] = self.bytes.^[self.names.^[e].at + i];
+            i = i + 1;
+        }
+        return n;
+    }
+}

fec/tests/exec/interns.fe

@@ -0,0 +1,62 @@
+// EXIT:0
+// OUTPUT:ids 0 1 2 count 3
+// OUTPUT:again 0 same yes count 3
+// OUTPUT:eq yes no len 4
+// OUTPUT:find 1 missing yes
+// OUTPUT:hash steady yes apart yes
+// OUTPUT:copied unit 4
+// OUTPUT:balanced
+unit interns;
+
+import std.io;
+import std.sys;
+import std.intern;
+
+// One copy of every distinct name, and a number that stands for it. Comparing
+// two names is comparing two integers; storing one costs four bytes and no
+// ownership.
+//
+// There is deliberately no way to get a `str` back out: a borrow of the text
+// would be a borrow of the interner, and the interner is exactly what a parser
+// wants to keep adding to while it holds names.
+
+fn run() -> !void {
+    var t: intern.Interner = try intern.Interner.with_capacity(8);
+    let a: intern.StrId = try t.intern("unit");
+    let b: intern.StrId = try t.intern("fn");
+    let c: intern.StrId = try t.intern("struct");
+    @print("ids {} {} {} count {}\n", a.raw, b.raw, c.raw, t.count_of());
+
+    // The same name twice is the same number, and costs nothing new.
+    let again: intern.StrId = try t.intern("unit");
+    @print("again {} same {} count {}\n", again.raw, yesno(a.same(again)),
+           t.count_of());
+
+    @print("eq {} {} len {}\n", yesno(t.eq(a, "unit")), yesno(t.eq(a, "fn")),
+           t.len_of(a));
+
+    @print("find {} missing {}\n", t.find("fn"),
+           yesno(t.find("nope") == intern.NONE));
+
+    // A name hashes the same every time, and two names do not collide here.
+    @print("hash steady {} apart {}\n", yesno(t.hash_of(a) == t.hash_of(again)),
+           yesno(t.hash_of(a) != t.hash_of(b)));
+
+    // The only way to see the text: copy it somewhere you own.
+    var buf: [8]u8 = undefined;
+    let n: usize = t.copy_into(a, buf[..]);
+    @print("copied {} {}\n", buf[0..n], n);
+    return;
+}
+
+fn yesno(b: bool) -> []u8 {
+    if b { return "yes"; }
+    return "no";
+}
+
+fn main() -> i32 {
+    run() catch |e| { @print("failed\n"); return 1; };
+    if sys.allocs() == sys.frees() { @print("balanced\n"); }
+    else { @print("leaked\n"); }
+    return 0;
+}