feat: support glue in parser

src/Ast.zig

@@ -9,8 +9,8 @@ const assert = std.debug.assert;
 
 filename: []const u8,
 source: []const u8,
-root: *Node,
 errors: []const Error,
+root: *Node,
 
 pub const Node = struct {
     tag: Tag,
@@ -48,9 +48,6 @@ pub const Node = struct {
         logical_lesser_expr,
         call_expr,
         choice_expr,
-        choice_start_expr,
-        choice_option_expr,
-        choice_inner_expr,
         divert_expr,
         selector_expr,
         assign_stmt,
@@ -93,6 +90,7 @@ pub const Node = struct {
 
     pub const Data = union {
         leaf: void,
+        // TODO: Add unary?
         bin: struct {
             lhs: ?*Node,
             rhs: ?*Node,
@@ -100,6 +98,11 @@ pub const Node = struct {
         list: struct {
             items: []*Node,
         },
+        content: struct {
+            items: []*Node,
+            leading_glue: bool,
+            trailing_glue: bool,
+        },
         choice_expr: struct {
             start_expr: ?*Node,
             option_expr: ?*Node,
@@ -154,6 +157,27 @@ pub const Node = struct {
         return node;
     }
 
+    pub fn createContent(
+        gpa: std.mem.Allocator,
+        tag: Tag,
+        span: Span,
+        options: struct {
+            items: []*Node,
+            leading_glue: bool,
+            trailing_glue: bool,
+        },
+    ) !*Node {
+        const node = try Node.create(gpa, tag, span);
+        node.data = .{
+            .content = .{
+                .items = options.items,
+                .leading_glue = options.leading_glue,
+                .trailing_glue = options.trailing_glue,
+            },
+        };
+        return node;
+    }
+
     pub fn createChoice(
         gpa: std.mem.Allocator,
         tag: Tag,

src/Ast/Render.zig

@@ -82,9 +82,6 @@ fn nodeTagToString(tag: Ast.Node.Tag) []const u8 {
         .selector_expr => "SelectorExpr",
         .call_expr => "CallExpr",
         .choice_expr => "ChoiceContentExpr",
-        .choice_start_expr => "ChoiceStartContentExpr",
-        .choice_option_expr => "ChoiceOptionContentExpr",
-        .choice_inner_expr => "ChoiceInnerContentExpr",
         .assign_stmt => "AssignStmt",
         .block_stmt => "BlockStmt",
         .content_stmt => "ContentStmt",
@@ -267,10 +264,11 @@ fn renderAstNode(
         try r.tty_config.setColor(writer, .reset);
     }
 
-    switch (node.tag) {
-        .file => {
     try r.writeType(writer, node);
     try writer.writeByte(' ');
+
+    switch (node.tag) {
+        .file => {
             try writer.writeByte('"');
             try writer.writeAll(r.tree.filename);
             try writer.writeByte('"');
@@ -285,8 +283,6 @@ fn renderAstNode(
         .multi_if_stmt,
         .switch_stmt,
         => {
-            try r.writeType(writer, node);
-            try writer.writeByte(' ');
             try r.writeLineSpan(writer, node);
         },
         .assign_stmt,
@@ -302,13 +298,8 @@ fn renderAstNode(
         .temp_decl,
         .var_decl,
         => {
-            try r.writeType(writer, node);
-            try writer.writeByte(' ');
             try r.writeLineColumnSpan(writer, node);
         },
-        .choice_start_expr,
-        .choice_option_expr,
-        .choice_inner_expr,
         .identifier,
         .number_literal,
         .parameter_decl,
@@ -316,15 +307,21 @@ fn renderAstNode(
         .string_literal,
         .string_expr,
         => {
-            try r.writeType(writer, node);
-            try writer.writeByte(' ');
             try r.writeLexeme(writer, node);
             try writer.writeByte(' ');
             try r.writeColumnSpan(writer, node);
         },
+        .content => {
+            const data = node.data.content;
+            if (data.leading_glue or data.trailing_glue) {
+                try writer.print("[leading_glue: {s}, trailing_glue: {s}'] ", .{
+                    if (data.leading_glue) "true" else "false",
+                    if (data.trailing_glue) "true" else "false",
+                });
+            }
+            try r.writeColumnSpan(writer, node);
+        },
         else => {
-            try r.writeType(writer, node);
-            try writer.writeByte(' ');
             try r.writeColumnSpan(writer, node);
         },
     }
@@ -353,20 +350,20 @@ fn renderAstWalk(
         .identifier,
         .parameter_decl,
         .ref_parameter_decl,
-        .choice_start_expr,
-        .choice_option_expr,
-        .choice_inner_expr,
         => {},
         .file,
         .argument_list,
         .parameter_list,
         .block_stmt,
         .choice_stmt,
-        .content,
         => {
             const data = node.data.list;
             for (data.items) |child_node| try children.append(gpa, child_node);
         },
+        .content => {
+            const data = node.data.content;
+            for (data.items) |child_node| try children.append(gpa, child_node);
+        },
         .choice_expr => {
             const data = node.data.choice_expr;
             if (data.start_expr) |lhs| try children.append(gpa, lhs);

src/AstGen.zig

@@ -734,9 +734,6 @@ fn expr(gi: *GenIr, scope: *Scope, optional_node: ?*const Ast.Node) InnerError!I
         .logical_lesser_or_equal_expr => return binaryOp(gi, scope, node, .cmp_lte),
         .call_expr => return callExpr(gi, scope, node, .call),
         .choice_expr => unreachable,
-        .choice_start_expr => unreachable,
-        .choice_option_expr => unreachable,
-        .choice_inner_expr => unreachable,
         .divert_expr => unreachable,
         .selector_expr => return fieldAccess(gi, scope, node),
         .assign_stmt => unreachable,
@@ -1055,7 +1052,7 @@ fn switchStmt(gi: *GenIr, scope: *Scope, node: *const Ast.Node) InnerError!Ir.In
 }
 
 fn contentExpr(block: *GenIr, scope: *Scope, node: *const Ast.Node) InnerError!Ir.Inst.Ref {
-    const data = node.data.list;
+    const data = node.data.content;
     for (data.items) |child_node| {
         switch (child_node.tag) {
             .string_literal => {

src/Parse.zig

@@ -245,34 +245,30 @@ fn popScratch(p: *Parse, context: *const StmtContext) *Ast.Node {
         @panic("BUG: Scratch buffer popped when empty!");
 }
 
-fn nodeListFromScratch(p: *Parse, start_offset: usize, end_offset: usize) Error![]*Ast.Node {
+fn makeNodeSliceFromScratch(p: *Parse, start: usize) Error![]*Ast.Node {
-    const span = end_offset - start_offset;
+    defer p.scratch.shrinkRetainingCapacity(start);
-    assert(span >= 0);
+    return p.arena.dupe(*Ast.Node, p.scratch.items[start..]);
-
-    const list = try p.arena.alloc(*Ast.Node, span);
-    defer p.scratch.shrinkRetainingCapacity(start_offset);
-
-    var li: usize = 0;
-    var i: usize = start_offset;
-    while (i < end_offset) : (i += 1) {
-        list[li] = p.scratch.items[i];
-        li += 1;
-    }
-    return list;
 }
 
-fn makeNodeSequence(
+fn makeNodeSliceFrom(
     p: *Parse,
     context: *const StmtContext,
     tag: Ast.Node.Tag,
     loc: Ast.Node.Span,
     scratch_offset: usize,
 ) Error!*Ast.Node {
-    if (!p.isScratchEmpty(context)) {
+    assert(scratch_offset >= context.scratch_top);
-        const list = try p.nodeListFromScratch(scratch_offset, p.scratch.items.len);
+    const list = try p.makeNodeSliceFromScratch(scratch_offset);
     return .createList(p.arena, tag, loc, list);
 }
-    return .createList(p.arena, tag, loc, &.{});
+
+fn makeNodeSlice(
+    p: *Parse,
+    context: *const StmtContext,
+    tag: Ast.Node.Tag,
+    loc: Ast.Node.Span,
+) Error!*Ast.Node {
+    return p.makeNodeSliceFrom(context, tag, loc, context.scratch_top);
 }
 
 fn isBlockStackEmpty(p: *Parse, context: *const StmtContext) bool {
@@ -342,7 +338,7 @@ fn collectBlock(p: *Parse, context: *StmtContext, level: usize) Error!?*Ast.Node
         span_end = span_start;
     }
 
-    var node = try p.makeNodeSequence(context, .block_stmt, .{
+    var node = try p.makeNodeSliceFrom(context, .block_stmt, .{
         .start = span_start,
         .end = span_end,
     }, block.scratch_offset);
@@ -399,7 +395,7 @@ fn collectContext(
             }
         }
 
-        const node = try p.makeNodeSequence(context, .choice_stmt, .{
+        const node = try p.makeNodeSliceFrom(context, .choice_stmt, .{
             .start = choice_state.source_offset,
             .end = p.token.loc.start,
         }, choice_state.scratch_offset);
@@ -436,7 +432,7 @@ fn collectKnot(p: *Parse, context: *StmtContext) Error!?*Ast.Node {
     const proto = p.scratch.items[p.knot_offset];
     if (proto.tag != .knot_prototype) return null;
 
-    const list = try p.nodeListFromScratch(p.knot_offset + 1, p.scratch.items.len);
+    const list = try p.makeNodeSliceFromScratch(p.knot_offset + 1);
     defer _ = p.popScratch(context);
 
     return .createKnot(p.arena, .knot_decl, .{
@@ -693,19 +689,6 @@ fn parseStringExpr(p: *Parse) Error!*Ast.Node {
     }, expr, null);
 }
 
-fn parseContentString(p: *Parse, token_set: []const Token.Tag) Error!?*Ast.Node {
-    const main_token = p.token;
-    while (!p.checkTokenInSet(token_set)) _ = p.nextToken();
-
-    return .createLeaf(p.arena, if (main_token.loc.start == p.token.loc.start)
-        .empty_string
-    else
-        .string_literal, .{
-        .start = main_token.loc.start,
-        .end = p.token.loc.start,
-    });
-}
-
 fn parseExprStmt(p: *Parse, lhs: ?*Ast.Node) Error!*Ast.Node {
     const main_token = p.token;
     const node = try parseInfixExpr(p, lhs, .none);
@@ -825,44 +808,18 @@ fn parseDivertStmt(p: *Parse) Error!*Ast.Node {
 }
 
 fn parseChoiceExpr(p: *Parse) Error!?*Ast.Node {
-    const token_set = [_]Token.Tag{
-        .left_brace,    .left_bracket, .right_brace,
-        .right_bracket, .right_arrow,  .newline,
-        .eof,
-    };
     const main_token = p.token;
     var lhs: ?*Ast.Node = null;
     var mhs: ?*Ast.Node = null;
     var rhs: ?*Ast.Node = null;
 
-    lhs = try parseContentString(p, &token_set);
+    lhs = try parseContent(p, .{ .ignore_brackets = false });
-    if (lhs) |n| {
-        if (n.tag != .empty_string) {
-            n.tag = .choice_start_expr;
-        }
-    }
     if (p.checkToken(.left_bracket)) {
         _ = p.nextToken();
         p.eatToken(.whitespace);
-
+        mhs = try parseContent(p, .{ .ignore_brackets = false });
-        if (!p.checkToken(.right_bracket)) {
-            mhs = try parseContentString(p, &token_set);
-            if (mhs) |n| {
-                if (n.tag != .empty_string) {
-                    n.tag = .choice_option_expr;
-                }
-            }
-        }
-
         _ = try p.expectToken(.right_bracket, false);
-        if (!p.checkTokenInSet(&token_set)) {
+        rhs = try parseContent(p, .{});
-            rhs = try parseContentString(p, &token_set);
-            if (rhs) |n| {
-                if (n.tag != .empty_string) {
-                    n.tag = .choice_inner_expr;
-                }
-            }
-        }
     }
     return .createChoice(p.arena, .choice_expr, .{
         .start = main_token.loc.start,
@@ -919,7 +876,7 @@ fn parseConditional(p: *Parse, main_token: Token, expr: ?*Ast.Node) Error!?*Ast.
     const node = try p.collectContext(&context, 0, false);
     if (node) |n| try p.scratch.append(p.gpa, n);
 
-    const list = try p.nodeListFromScratch(context.scratch_top, p.scratch.items.len);
+    const list = try p.makeNodeSliceFromScratch(context.scratch_top);
     return .createSwitch(p.arena, if (expr != null and !context.is_block_created)
         .switch_stmt
     else if (expr == null and !context.is_block_created)
@@ -932,18 +889,14 @@ fn parseConditional(p: *Parse, main_token: Token, expr: ?*Ast.Node) Error!?*Ast.
 }
 
 fn parseInlineIf(p: *Parse, main_token: Token, lhs: ?*Ast.Node) Error!?*Ast.Node {
-    const token_set = [_]Token.Tag{
-        .left_brace, .right_brace, .right_arrow,
-        .glue,       .newline,     .eof,
-    };
     p.eatToken(.whitespace);
 
-    const content_node = try parseContentExpr(p, &token_set);
+    const content = try parseContent(p, .{});
     const end_token = try p.expectToken(.right_brace, true);
     return .createBinary(p.arena, .inline_if_stmt, .{
         .start = main_token.loc.start,
         .end = end_token.loc.end,
-    }, lhs, content_node);
+    }, lhs, content);
 }
 
 fn parseLbraceExpr(p: *Parse) Error!?*Ast.Node {
@@ -981,38 +934,88 @@ fn parseLbraceExpr(p: *Parse) Error!?*Ast.Node {
     }
 }
 
-fn parseContentExpr(p: *Parse, token_set: []const Token.Tag) Error!?*Ast.Node {
+const ContentOptions = struct {
+    ignore_brackets: bool = true,
+    ignore_parens: bool = true,
+};
+
+fn parseContentString(p: *Parse, options: ContentOptions) Error!?*Ast.Node {
     const main_token = p.token;
-    const context = makeStmtContext(p, .block, null);
 
     while (true) {
-        var node: ?*Ast.Node = null;
+        switch (p.token.tag) {
-        if (!p.checkTokenInSet(token_set)) {
+            .eof,
-            node = try parseContentString(p, token_set);
+            .newline,
-        } else switch (p.token.tag) {
+            .left_arrow,
-            .eof, .newline, .right_brace => break,
+            .right_arrow,
-            .left_brace => node = try parseLbraceExpr(p),
+            .left_brace,
-            .right_arrow => node = try parseDivertStmt(p),
+            .right_brace,
-            //.INK_TT_GLUE => node = ink_parse_glue(p),
+            .glue,
-            else => {
+            => break,
-                return p.fail(.unexpected_token, p.token);
+            .left_bracket, .right_bracket => |tag| if (!options.ignore_brackets)
-            },
+                break
+            else
+                p.eatToken(tag),
+            else => |tag| p.eatToken(tag),
         }
-        if (node) |n| try p.scratch.append(p.gpa, n);
     }
-    return p.makeNodeSequence(&context, .content, .{
+    if (main_token.loc.start == p.token.loc.start) return null;
+
+    return .createLeaf(p.arena, .string_literal, .{
         .start = main_token.loc.start,
         .end = p.token.loc.start,
-    }, context.scratch_top);
+    });
+}
+
+fn parseContent(p: *Parse, options: ContentOptions) Error!?*Ast.Node {
+    const main_token = p.token;
+    const scratch_top = p.scratch.items.len;
+    var leading_glue = false;
+    var trailing_glue = false;
+
+    if (p.token.tag == .glue) {
+        leading_glue = true;
+        _ = p.nextToken();
+    }
+    while (true) {
+        const node: ?*Ast.Node = switch (p.token.tag) {
+            .eof, .newline, .left_arrow, .right_brace => break,
+            .left_brace => try parseLbraceExpr(p),
+            .right_arrow => try parseDivertStmt(p),
+            .glue => blk: {
+                const next_token = p.nextToken();
+                switch (next_token.tag) {
+                    .eof, .newline, .right_brace => {
+                        trailing_glue = true;
+                        break;
+                    },
+                    else => break :blk null,
+                }
+            },
+            else => |tag| blk: {
+                switch (tag) {
+                    .left_bracket, .right_bracket => if (!options.ignore_brackets) break,
+                    else => {},
+                }
+                break :blk try parseContentString(p, options);
+            },
+        };
+        if (node) |n| try p.scratch.append(p.gpa, n);
+    }
+    if (main_token.loc.start == p.token.loc.start) return null;
+    return .createContent(p.arena, .content, .{
+        .start = main_token.loc.start,
+        .end = p.token.loc.start,
+    }, .{
+        .items = try p.makeNodeSliceFromScratch(scratch_top),
+        .leading_glue = leading_glue,
+        .trailing_glue = trailing_glue,
+    });
 }
 
 fn parseContentStmt(p: *Parse) Error!*Ast.Node {
-    const token_set = [_]Token.Tag{
-        .left_brace, .right_brace, .right_arrow,
-        .glue,       .newline,     .eof,
-    };
     const main_token = p.token;
-    const node = try parseContentExpr(p, &token_set);
+    const node = try parseContent(p, .{});
     const end_token = try p.expectNewline();
     return .createBinary(p.arena, .content_stmt, .{
         .start = main_token.loc.start,
@@ -1052,10 +1055,10 @@ fn parseParameterList(p: *Parse) Error!?*Ast.Node {
     }
 
     _ = try p.expectToken(.right_paren, true);
-    return p.makeNodeSequence(&context, .parameter_list, .{
+    return p.makeNodeSlice(&context, .parameter_list, .{
         .start = main_token.loc.start,
         .end = p.token.loc.start,
-    }, context.scratch_top);
+    });
 }
 
 fn parseArgumentList(p: *Parse) Error!?*Ast.Node {
@@ -1078,10 +1081,10 @@ fn parseArgumentList(p: *Parse) Error!?*Ast.Node {
     }
 
     _ = try p.expectToken(.right_paren, false);
-    return p.makeNodeSequence(&context, .argument_list, .{
+    return p.makeNodeSlice(&context, .argument_list, .{
         .start = main_token.loc.start,
         .end = p.token.loc.start,
-    }, context.scratch_top);
+    });
 }
 
 fn parseConditionalBranch(p: *Parse, tag: Ast.Node.Tag) Error!?*Ast.Node {
@@ -1249,8 +1252,8 @@ pub fn parseFile(p: *Parse) Error!*Ast.Node {
     const node = try p.collectKnot(&context);
     if (node) |n| try p.scratch.append(p.gpa, n);
 
-    return p.makeNodeSequence(&context, .file, .{
+    return p.makeNodeSlice(&context, .file, .{
         .start = main_token.loc.start,
         .end = p.token.loc.end,
-    }, context.scratch_top);
+    });
 }

src/Story/testdata/I036/story.ink

@@ -6,4 +6,3 @@ A
 B
 === function string()
     ~ return "{3}"
-}

src/parser_tests.zig

@@ -116,13 +116,16 @@ test "parser: choice statements, simple" {
         \\   `--ChoiceStmt <line:1, line:3>
         \\      |--ChoiceStarStmt <line:1, col:1:4>
         \\      |  `--ChoiceContentExpr <col:3, col:4>
-        \\      |     `--ChoiceStartContentExpr `A` <col:3, col:4>
+        \\      |     `--Content <col:3, col:4>
+        \\      |        `--StringLiteral `A` <col:3, col:4>
         \\      |--ChoiceStarStmt <line:2, col:1:4>
         \\      |  `--ChoiceContentExpr <col:3, col:4>
-        \\      |     `--ChoiceStartContentExpr `B` <col:3, col:4>
+        \\      |     `--Content <col:3, col:4>
+        \\      |        `--StringLiteral `B` <col:3, col:4>
         \\      `--ChoiceStarStmt <line:3, col:1:4>
         \\         `--ChoiceContentExpr <col:3, col:4>
-        \\            `--ChoiceStartContentExpr `C` <col:3, col:4>
+        \\            `--Content <col:3, col:4>
+        \\               `--StringLiteral `C` <col:3, col:4>
         \\
         ,
     );
@@ -137,7 +140,6 @@ test "parser: choice statements, empty branch" {
         \\   `--ChoiceStmt <line:1, line:1>
         \\      `--ChoiceStarStmt <line:1, col:1:2>
         \\         `--ChoiceContentExpr <col:2, col:2>
-        \\            `--EmptyString <col:2, col:2>
         \\
         ,
     );
@@ -156,19 +158,24 @@ test "parser: choice statements, level normalization" {
         \\   `--ChoiceStmt <line:1, line:5>
         \\      |--ChoiceStarStmt <line:1, col:1:8>
         \\      |  `--ChoiceContentExpr <col:7, col:8>
-        \\      |     `--ChoiceStartContentExpr `A` <col:7, col:8>
+        \\      |     `--Content <col:7, col:8>
+        \\      |        `--StringLiteral `A` <col:7, col:8>
         \\      |--ChoiceStarStmt <line:2, col:1:7>
         \\      |  `--ChoiceContentExpr <col:6, col:7>
-        \\      |     `--ChoiceStartContentExpr `B` <col:6, col:7>
+        \\      |     `--Content <col:6, col:7>
+        \\      |        `--StringLiteral `B` <col:6, col:7>
         \\      |--ChoiceStarStmt <line:3, col:1:6>
         \\      |  `--ChoiceContentExpr <col:5, col:6>
-        \\      |     `--ChoiceStartContentExpr `C` <col:5, col:6>
+        \\      |     `--Content <col:5, col:6>
+        \\      |        `--StringLiteral `C` <col:5, col:6>
         \\      |--ChoiceStarStmt <line:4, col:1:5>
         \\      |  `--ChoiceContentExpr <col:4, col:5>
-        \\      |     `--ChoiceStartContentExpr `D` <col:4, col:5>
+        \\      |     `--Content <col:4, col:5>
+        \\      |        `--StringLiteral `D` <col:4, col:5>
         \\      `--ChoiceStarStmt <line:5, col:1:4>
         \\         `--ChoiceContentExpr <col:3, col:4>
-        \\            `--ChoiceStartContentExpr `E` <col:3, col:4>
+        \\            `--Content <col:3, col:4>
+        \\               `--StringLiteral `E` <col:3, col:4>
         \\
         ,
     );
@@ -183,8 +190,10 @@ test "parser: choice statements, mixed context" {
         \\   `--ChoiceStmt <line:1, line:1>
         \\      `--ChoiceStarStmt <line:1, col:1:7>
         \\         `--ChoiceContentExpr <col:3, col:7>
-        \\            |--ChoiceStartContentExpr `A` <col:3, col:4>
+        \\            |--Content <col:3, col:4>
-        \\            `--ChoiceInnerContentExpr `B` <col:6, col:7>
+        \\            |  `--StringLiteral `A` <col:3, col:4>
+        \\            `--Content <col:6, col:7>
+        \\               `--StringLiteral `B` <col:6, col:7>
         \\
         ,
     );