From b7ce16f65e513b633ce5fbb7ec52c2ed86ec4ad4 Mon Sep 17 00:00:00 2001 From: Sebastian Jeong Date: Mon, 17 Aug 2026 16:27:47 +0900 Subject: [PATCH] =?UTF-8?q?GOAL=20P3-4:=20StringInterner,=20=EA=B7=B8?= =?UTF-8?q?=EB=A6=AC=EA=B3=A0=20=EA=B7=B8=EA=B2=83=EC=9D=B4=20=EB=93=9C?= =?UTF-8?q?=EB=9F=AC=EB=82=B8=20=EB=B0=B0=EC=B9=98=20=EB=B2=84=EA=B7=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 이름 하나당 사본 하나와 그것을 대신하는 숫자. 컴파일러는 이름을 끊임없이 비교하고 사방에 저장하는데, StrId 둘을 비교하는 것은 정수 둘을 비교하는 것이고 하나를 저장하는 것은 4바이트에 소유권 없음이다. str 을 꺼내는 API 를 일부러 두지 않았다. 텍스트를 빌리는 것은 interner 를 빌리는 것인데, interner 는 이름을 든 채로 계속 더 넣고 싶은 바로 그 물건이다 -- 파서는 식별자를 읽으면서 같은 숨에 다음 것을 등록한다. 텍스트를 열어서 하려던 일은 전부 여기 있다: eq, len_of, hash_of, find, copy_into. 그리고 이것이 제네릭 인스턴스를 필드로 담는 구조체를 통째로 깨뜨리던 버그를 드러냈다. Holder{ bytes: ^[]mut u8, used: usize, seen: map.Map(u32) } 36 바이트여야 하는데 12 로 잡혔다. Map(u32) 를 짓는 중에 그 안의 Slot(u32) 를 인스턴스화하면 거기서 배치 패스가 다시 돈다. 그때 Map(u32) 는 field_count 는 4 인데 필드 배열이 아직 아무것도 말하지 않는 상태라, 크기 0 으로 확정되고 굳었다. 이미 크기가 있는 타입은 아무도 다시 계산하지 않으니 Holder 는 그 0 을 읽었다. 셋을 고쳤다: 짓는 중인 인스턴스는 building 을 세워 배치를 거절하고, 멤버가 아직 자리를 못 잡은 집합 타입은 틀린 답으로 굳느니 물러나며, 배치 패스는 움직임이 없을 때까지 돈다. 245/245, 38/38. --- fec/src/checkgen.c | 2 + fec/src/types.c | 41 +++++++++++- fec/src/types.h | 4 ++ fec/std/intern.fe | 131 ++++++++++++++++++++++++++++++++++++++ fec/tests/exec/interns.fe | 62 ++++++++++++++++++ 5 files changed, 239 insertions(+), 1 deletion(-) create mode 100644 fec/std/intern.fe create mode 100644 fec/tests/exec/interns.fe diff --git a/fec/src/checkgen.c b/fec/src/checkgen.c index 619213f..e2432df 100644 --- a/fec/src/checkgen.c +++ b/fec/src/checkgen.c @@ -210,6 +210,7 @@ FeType *build_struct_instance(FeCheck *c, FeUnit *home, FeNode *decl, t=fe_type_intern_unit(&c->types,home->name,key); if (!t || t->kind!=FE_TYPE_UNKNOWN) return t; t->kind=FE_TYPE_STRUCT; + t->building=1; t->packed=(decl->flags & FE_NODE_PACKED)!=0; t->decl_node=decl; t->bind_count=0; @@ -246,6 +247,7 @@ FeType *build_struct_instance(FeCheck *c, FeUnit *home, FeNode *decl, pop_bindings(c,&save); c->types.unit_name=save_unit; } + t->building=0; fe_type_layout_all(&c->types); /* A type that says how to let go of itself needs that method to exist for every instance, whether or not anyone calls it by name: scope cleanup diff --git a/fec/src/types.c b/fec/src/types.c index b7b7cff..46bb024 100644 --- a/fec/src/types.c +++ b/fec/src/types.c @@ -48,6 +48,7 @@ static FeType *new_type(FeTypeCtx *ctx, const char *name, FeTypeKind kind) t->emit_state = 0; t->cycle_state = 0; t->layout_state = 0; + t->building = 0; ctx->types = t; return t; } @@ -475,6 +476,28 @@ static void leave_decl_unit(FeTypeCtx *ctx, int back, const char *was) ctx->unit_name = was; } +/* Did every field end up with a size? A struct whose members are not settled + cannot be settled either -- and freezing it here is worse than leaving it, + because nothing recomputes a type that already has a size. */ +static int members_ready(const FeType *t) +{ + unsigned i; + unsigned j; + if (t->kind == FE_TYPE_STRUCT) { + for (i = 0; i < t->field_count; ++i) { + if (!t->fields[i].type) return 0; + if (t->fields[i].type->layout_state != 2) return 0; + } + return 1; + } + for (i = 0; i < t->variant_count; ++i) + for (j = 0; j < t->variants[i].field_count; ++j) { + if (!t->variants[i].fields[j].type) return 0; + if (t->variants[i].fields[j].type->layout_state != 2) return 0; + } + return 1; +} + static void layout_type(FeTypeCtx *ctx, FeType *t) { unsigned i; @@ -483,6 +506,7 @@ static void layout_type(FeTypeCtx *ctx, FeType *t) unsigned long max_size; unsigned max_align; if (!t || t->size) return; + if (t->building) return; if (t->layout_state == 1) { t->size = 1; t->align = 1; @@ -562,6 +586,7 @@ static void layout_type(FeTypeCtx *ctx, FeType *t) t->fields[i].offset = off; off += fe_type_size(t->fields[i].type); } + if (!members_ready(t)) { t->layout_state = 0; return; } t->align = max_align; t->size = round_up(off, max_align); t->layout_state = 2; @@ -594,6 +619,7 @@ static void layout_type(FeTypeCtx *ctx, FeType *t) } if (off > max_size) max_size = off; } + if (!members_ready(t)) { t->layout_state = 0; return; } t->bits = t->variant_count > 256U ? 16U : 8U; off = round_up(t->bits / 8U, max_align); t->size = round_up(off + max_size, max_align); @@ -605,7 +631,20 @@ static void layout_type(FeTypeCtx *ctx, FeType *t) void fe_type_layout_all(FeTypeCtx *ctx) { FeType *t; - for (t = ctx->types; t; t = t->next) layout_type(ctx, t); + int again = 1; + unsigned rounds = 0; + /* One pass settles a type only if everything under it is already settled, + so a type that had to wait is picked up by the next round. Sixteen is + far past any real nesting; it is here so a cycle cannot spin. */ + while (again && rounds < 16U) { + again = 0; + for (t = ctx->types; t; t = t->next) { + if (t->size || t->layout_state == 2) continue; + layout_type(ctx, t); + if (t->layout_state == 2) again = 1; + } + ++rounds; + } } FeFieldType *fe_type_field(FeType *t, const char *name) diff --git a/fec/src/types.h b/fec/src/types.h index d8db1ea..c15d9a3 100644 --- a/fec/src/types.h +++ b/fec/src/types.h @@ -103,6 +103,10 @@ struct FeType { marker made a struct in the middle of the first look complete to the second -- one byte wide, with every field on top of the next. */ int layout_state; + /* Set while a generic instance is being filled in. Its field array + exists but says nothing yet, and a size taken from it would be + wrong and would then be frozen. */ + int building; }; typedef struct FeTypeCtx { diff --git a/fec/std/intern.fe b/fec/std/intern.fe new file mode 100644 index 0000000..03ddba8 --- /dev/null +++ b/fec/std/intern.fe @@ -0,0 +1,131 @@ +unit std.intern; + +import std.map; +import std.mem; + +// One copy of every distinct name, and a number that stands for it. +// +// A compiler compares names constantly and stores them everywhere. Comparing +// two `StrId` is comparing two integers; storing one costs four bytes and no +// ownership. That is the whole point. +// +// There is deliberately no way to get a `str` back out. A borrow of the text +// would be a borrow of the interner, and the interner is exactly the thing you +// want to keep adding to while holding names -- the parser reads an identifier +// and registers the next one in the same breath. Everything you would open the +// text for is here instead: compare it, measure it, hash it, write it. + +/// A name, as a number. Copy, four bytes, and meaningless to any other +/// interner -- which is fine, because a program has one. +pub struct StrId { + pub raw: u32, + + pub fn same(self: &Self, other: StrId) -> bool { + return self.raw == other.raw; + } +} + +/// No name. +pub const NONE: u32 = 4294967295; + +struct Entry { + at: usize, + len: usize, +} + +pub struct Interner { + /// Every name end to end. Nothing is ever removed, so an offset stays + /// good for as long as the interner does. + bytes: ^[]mut u8, + used: usize, + names: ^[]mut Entry, + count: usize, + /// Text to id, so interning the same name twice gives the same number. + seen: map.Map(u32), + + pub fn with_capacity(n: usize) -> !Self { + let text: ^[]mut u8 = try mem.alloc_slice(u8, n * 8); + let table: ^[]mut Entry = try mem.alloc_slice(Entry, n); + let index: map.Map(u32) = try map.Map(u32).with_capacity(n); + return Self{ bytes: text, used: 0, names: table, count: 0, + seen: index }; + } + + pub fn count_of(self: &Self) -> usize { return self.count; } + + /// The number for this name, making one if it is new. + pub fn intern(self: &mut Self, text: []u8) -> !StrId { + let found: u32 = self.seen.get(text, NONE); + if found != NONE { return StrId{ raw: found }; } + if self.count == self.names.^.n { return error.OutOfMemory; } + let at: usize = self.used; + if at + text.n > self.bytes.^.n { return error.OutOfMemory; } + var i: usize = 0; + while i < text.n { + self.bytes.^[at + i] = text[i]; + i = i + 1; + } + let id: usize = self.count; + self.names.^[id].at = at; + self.names.^[id].len = text.n; + self.used = at + text.n; + self.count = id + 1; + try self.seen.put(text, id as u32); + return StrId{ raw: id as u32 }; + } + + /// Is this a name it has seen? `NONE` when not. + pub fn find(self: &Self, text: []u8) -> u32 { + return self.seen.get(text, NONE); + } + + pub fn len_of(self: &Self, id: StrId) -> usize { + if (id.raw as usize) >= self.count { return 0; } + return self.names.^[id.raw as usize].len; + } + + /// Does this id spell this text? The comparison every `if name == "fn"` + /// in a parser turns into. + pub fn eq(self: &Self, id: StrId, text: []u8) -> bool { + if (id.raw as usize) >= self.count { return false; } + let e: usize = id.raw as usize; + if self.names.^[e].len != text.n { return false; } + var i: usize = 0; + while i < text.n { + if self.bytes.^[self.names.^[e].at + i] != text[i] { return false; } + i = i + 1; + } + return true; + } + + /// FNV-1a over the stored bytes, for anything that wants to bucket names + /// without opening them. + pub fn hash_of(self: &Self, id: StrId) -> u32 { + if (id.raw as usize) >= self.count { return 0; } + let e: usize = id.raw as usize; + var h: u32 = 2166136261; + var i: usize = 0; + while i < self.names.^[e].len { + h = h ^ (self.bytes.^[self.names.^[e].at + i] as u32); + h = h * 16777619; + i = i + 1; + } + return h; + } + + /// Copy the name into `out` and say how many bytes it took. This is how a + /// name reaches a diagnostic without the interner being borrowed past the + /// statement. + pub fn copy_into(self: &Self, id: StrId, out: []mut u8) -> usize { + if (id.raw as usize) >= self.count { return 0; } + let e: usize = id.raw as usize; + var n: usize = self.names.^[e].len; + if n > out.n { n = out.n; } + var i: usize = 0; + while i < n { + out[i] = self.bytes.^[self.names.^[e].at + i]; + i = i + 1; + } + return n; + } +} diff --git a/fec/tests/exec/interns.fe b/fec/tests/exec/interns.fe new file mode 100644 index 0000000..dfcbb62 --- /dev/null +++ b/fec/tests/exec/interns.fe @@ -0,0 +1,62 @@ +// EXIT:0 +// OUTPUT:ids 0 1 2 count 3 +// OUTPUT:again 0 same yes count 3 +// OUTPUT:eq yes no len 4 +// OUTPUT:find 1 missing yes +// OUTPUT:hash steady yes apart yes +// OUTPUT:copied unit 4 +// OUTPUT:balanced +unit interns; + +import std.io; +import std.sys; +import std.intern; + +// One copy of every distinct name, and a number that stands for it. Comparing +// two names is comparing two integers; storing one costs four bytes and no +// ownership. +// +// There is deliberately no way to get a `str` back out: a borrow of the text +// would be a borrow of the interner, and the interner is exactly what a parser +// wants to keep adding to while it holds names. + +fn run() -> !void { + var t: intern.Interner = try intern.Interner.with_capacity(8); + let a: intern.StrId = try t.intern("unit"); + let b: intern.StrId = try t.intern("fn"); + let c: intern.StrId = try t.intern("struct"); + @print("ids {} {} {} count {}\n", a.raw, b.raw, c.raw, t.count_of()); + + // The same name twice is the same number, and costs nothing new. + let again: intern.StrId = try t.intern("unit"); + @print("again {} same {} count {}\n", again.raw, yesno(a.same(again)), + t.count_of()); + + @print("eq {} {} len {}\n", yesno(t.eq(a, "unit")), yesno(t.eq(a, "fn")), + t.len_of(a)); + + @print("find {} missing {}\n", t.find("fn"), + yesno(t.find("nope") == intern.NONE)); + + // A name hashes the same every time, and two names do not collide here. + @print("hash steady {} apart {}\n", yesno(t.hash_of(a) == t.hash_of(again)), + yesno(t.hash_of(a) != t.hash_of(b))); + + // The only way to see the text: copy it somewhere you own. + var buf: [8]u8 = undefined; + let n: usize = t.copy_into(a, buf[..]); + @print("copied {} {}\n", buf[0..n], n); + return; +} + +fn yesno(b: bool) -> []u8 { + if b { return "yes"; } + return "no"; +} + +fn main() -> i32 { + run() catch |e| { @print("failed\n"); return 1; }; + if sys.allocs() == sys.frees() { @print("balanced\n"); } + else { @print("leaked\n"); } + return 0; +}