doslang-mirror/fec/tests/exec/lexer/scan.fe

unit scan;
import tok;

// A scanner over a byte slice. The position travels in a `&mut usize` beside
// the source rather than inside a struct with it, because a struct cannot hold
// a slice: a slice is a borrowed view and R4 keeps borrows out of aggregates.

fn is_space(c: u8) -> bool { return c == 32 or c == 9 or c == 13 or c == 10; }
fn is_digit(c: u8) -> bool { return c >= 48 and c <= 57; }

fn is_name_start(c: u8) -> bool {
    if c >= 97 and c <= 122 { return true; }
    if c >= 65 and c <= 90 { return true; }
    return c == 95;
}

fn is_name_part(c: u8) -> bool {
    return is_name_start(c) or is_digit(c);
}

const KEYWORDS: usize = 12;

fn is_keyword(word: []u8) -> bool {
    if same(word, "unit") { return true; }
    if same(word, "import") { return true; }
    if same(word, "pub") { return true; }
    if same(word, "fn") { return true; }
    if same(word, "struct") { return true; }
    if same(word, "enum") { return true; }
    if same(word, "let") { return true; }
    if same(word, "var") { return true; }
    if same(word, "if") { return true; }
    if same(word, "else") { return true; }
    if same(word, "while") { return true; }
    if same(word, "return") { return true; }
    return false;
}

fn same(a: []u8, b: []u8) -> bool {
    if a.n != b.n { return false; }
    var i: usize = 0;
    while i < a.n {
        if a[i] != b[i] { return false; }
        i = i + 1;
    }
    return true;
}

/// Step over anything that is not a token: spaces, newlines, and `//` to the
/// end of the line. `line` counts what was crossed so a token can say where it
/// came from.
fn skip_gaps(src: []u8, at: &mut usize, line: &mut usize) -> void {
    while at.^ < src.n {
        let c: u8 = src[at.^];
        if c == 10 { line.^ = line.^ + 1; at.^ = at.^ + 1; }
        else if is_space(c) { at.^ = at.^ + 1; }
        else if c == 47 and at.^ + 1 < src.n and src[at.^ + 1] == 47 {
            while at.^ < src.n {
                if src[at.^] == 10 { break; }
                at.^ = at.^ + 1;
            }
        }
        else { break; }
    }
}

pub fn next(src: []u8, at: &mut usize, line: &mut usize) -> tok.Token {
    skip_gaps(src, at, line);
    let start: usize = at.^;
    let where: usize = line.^;
    if start >= src.n {
        return tok.Token{ kind: tok.Kind.End, from: start, len: 0, line: where };
    }
    let c: u8 = src[start];
    if is_name_start(c) {
        while at.^ < src.n {
            if not is_name_part(src[at.^]) { break; }
            at.^ = at.^ + 1;
        }
        let word: []u8 = src[start..at.^];
        var kind: tok.Kind = tok.Kind.Name;
        if is_keyword(word) { kind = tok.Kind.Keyword; }
        return tok.Token{ kind: kind, from: start, len: at.^ - start,
                          line: where };
    }
    if is_digit(c) {
        while at.^ < src.n {
            if not is_digit(src[at.^]) { break; }
            at.^ = at.^ + 1;
        }
        return tok.Token{ kind: tok.Kind.Number, from: start,
                          len: at.^ - start, line: where };
    }
    if c == 34 {
        at.^ = at.^ + 1;
        while at.^ < src.n {
            if src[at.^] == 34 { break; }
            if src[at.^] == 92 and at.^ + 1 < src.n { at.^ = at.^ + 1; }
            at.^ = at.^ + 1;
        }
        if at.^ >= src.n {
            return tok.Token{ kind: tok.Kind.Bad, from: start,
                              len: at.^ - start, line: where };
        }
        at.^ = at.^ + 1;
        return tok.Token{ kind: tok.Kind.Text, from: start, len: at.^ - start,
                          line: where };
    }
    at.^ = at.^ + 1;
    // Two-byte punctuation the language actually uses.
    if at.^ < src.n {
        let d: u8 = src[at.^];
        if c == 45 and d == 62 { at.^ = at.^ + 1; }
        else if c == 61 and d == 61 { at.^ = at.^ + 1; }
        else if c == 33 and d == 61 { at.^ = at.^ + 1; }
        else if c == 60 and d == 61 { at.^ = at.^ + 1; }
        else if c == 62 and d == 61 { at.^ = at.^ + 1; }
        else if c == 46 and d == 46 { at.^ = at.^ + 1; }
    }
    return tok.Token{ kind: tok.Kind.Punct, from: start, len: at.^ - start,
                      line: where };
}