From d325cdf9653f5ac2a8fc395cc14882a3bf2344e9 Mon Sep 17 00:00:00 2001 From: Brett Broadhurst Date: Mon, 30 Mar 2026 08:38:30 -0600 Subject: [PATCH] feat: support glue in parser --- src/Ast.zig | 32 ++++- src/Ast/Render.zig | 37 +++--- src/AstGen.zig | 5 +- src/Parse.zig | 201 +++++++++++++++--------------- src/Story/testdata/I036/story.ink | 1 - src/parser_tests.zig | 31 +++-- 6 files changed, 168 insertions(+), 139 deletions(-) diff --git a/src/Ast.zig b/src/Ast.zig index 1d56585..686163b 100644 --- a/src/Ast.zig +++ b/src/Ast.zig @@ -9,8 +9,8 @@ const assert = std.debug.assert; filename: []const u8, source: []const u8, -root: *Node, errors: []const Error, +root: *Node, pub const Node = struct { tag: Tag, @@ -48,9 +48,6 @@ pub const Node = struct { logical_lesser_expr, call_expr, choice_expr, - choice_start_expr, - choice_option_expr, - choice_inner_expr, divert_expr, selector_expr, assign_stmt, @@ -93,6 +90,7 @@ pub const Node = struct { pub const Data = union { leaf: void, + // TODO: Add unary? bin: struct { lhs: ?*Node, rhs: ?*Node, @@ -100,6 +98,11 @@ pub const Node = struct { list: struct { items: []*Node, }, + content: struct { + items: []*Node, + leading_glue: bool, + trailing_glue: bool, + }, choice_expr: struct { start_expr: ?*Node, option_expr: ?*Node, @@ -154,6 +157,27 @@ pub const Node = struct { return node; } + pub fn createContent( + gpa: std.mem.Allocator, + tag: Tag, + span: Span, + options: struct { + items: []*Node, + leading_glue: bool, + trailing_glue: bool, + }, + ) !*Node { + const node = try Node.create(gpa, tag, span); + node.data = .{ + .content = .{ + .items = options.items, + .leading_glue = options.leading_glue, + .trailing_glue = options.trailing_glue, + }, + }; + return node; + } + pub fn createChoice( gpa: std.mem.Allocator, tag: Tag, diff --git a/src/Ast/Render.zig b/src/Ast/Render.zig index f2e1a3d..543845b 100644 --- a/src/Ast/Render.zig +++ b/src/Ast/Render.zig @@ -82,9 +82,6 @@ fn nodeTagToString(tag: Ast.Node.Tag) []const u8 { .selector_expr => "SelectorExpr", .call_expr => "CallExpr", .choice_expr => "ChoiceContentExpr", - .choice_start_expr => "ChoiceStartContentExpr", - .choice_option_expr => "ChoiceOptionContentExpr", - .choice_inner_expr => "ChoiceInnerContentExpr", .assign_stmt => "AssignStmt", .block_stmt => "BlockStmt", .content_stmt => "ContentStmt", @@ -267,10 +264,11 @@ fn renderAstNode( try r.tty_config.setColor(writer, .reset); } + try r.writeType(writer, node); + try writer.writeByte(' '); + switch (node.tag) { .file => { - try r.writeType(writer, node); - try writer.writeByte(' '); try writer.writeByte('"'); try writer.writeAll(r.tree.filename); try writer.writeByte('"'); @@ -285,8 +283,6 @@ fn renderAstNode( .multi_if_stmt, .switch_stmt, => { - try r.writeType(writer, node); - try writer.writeByte(' '); try r.writeLineSpan(writer, node); }, .assign_stmt, @@ -302,13 +298,8 @@ fn renderAstNode( .temp_decl, .var_decl, => { - try r.writeType(writer, node); - try writer.writeByte(' '); try r.writeLineColumnSpan(writer, node); }, - .choice_start_expr, - .choice_option_expr, - .choice_inner_expr, .identifier, .number_literal, .parameter_decl, @@ -316,15 +307,21 @@ fn renderAstNode( .string_literal, .string_expr, => { - try r.writeType(writer, node); - try writer.writeByte(' '); try r.writeLexeme(writer, node); try writer.writeByte(' '); try r.writeColumnSpan(writer, node); }, + .content => { + const data = node.data.content; + if (data.leading_glue or data.trailing_glue) { + try writer.print("[leading_glue: {s}, trailing_glue: {s}'] ", .{ + if (data.leading_glue) "true" else "false", + if (data.trailing_glue) "true" else "false", + }); + } + try r.writeColumnSpan(writer, node); + }, else => { - try r.writeType(writer, node); - try writer.writeByte(' '); try r.writeColumnSpan(writer, node); }, } @@ -353,20 +350,20 @@ fn renderAstWalk( .identifier, .parameter_decl, .ref_parameter_decl, - .choice_start_expr, - .choice_option_expr, - .choice_inner_expr, => {}, .file, .argument_list, .parameter_list, .block_stmt, .choice_stmt, - .content, => { const data = node.data.list; for (data.items) |child_node| try children.append(gpa, child_node); }, + .content => { + const data = node.data.content; + for (data.items) |child_node| try children.append(gpa, child_node); + }, .choice_expr => { const data = node.data.choice_expr; if (data.start_expr) |lhs| try children.append(gpa, lhs); diff --git a/src/AstGen.zig b/src/AstGen.zig index 19675f0..d709833 100644 --- a/src/AstGen.zig +++ b/src/AstGen.zig @@ -734,9 +734,6 @@ fn expr(gi: *GenIr, scope: *Scope, optional_node: ?*const Ast.Node) InnerError!I .logical_lesser_or_equal_expr => return binaryOp(gi, scope, node, .cmp_lte), .call_expr => return callExpr(gi, scope, node, .call), .choice_expr => unreachable, - .choice_start_expr => unreachable, - .choice_option_expr => unreachable, - .choice_inner_expr => unreachable, .divert_expr => unreachable, .selector_expr => return fieldAccess(gi, scope, node), .assign_stmt => unreachable, @@ -1055,7 +1052,7 @@ fn switchStmt(gi: *GenIr, scope: *Scope, node: *const Ast.Node) InnerError!Ir.In } fn contentExpr(block: *GenIr, scope: *Scope, node: *const Ast.Node) InnerError!Ir.Inst.Ref { - const data = node.data.list; + const data = node.data.content; for (data.items) |child_node| { switch (child_node.tag) { .string_literal => { diff --git a/src/Parse.zig b/src/Parse.zig index f0907a0..c555a87 100644 --- a/src/Parse.zig +++ b/src/Parse.zig @@ -245,34 +245,30 @@ fn popScratch(p: *Parse, context: *const StmtContext) *Ast.Node { @panic("BUG: Scratch buffer popped when empty!"); } -fn nodeListFromScratch(p: *Parse, start_offset: usize, end_offset: usize) Error![]*Ast.Node { - const span = end_offset - start_offset; - assert(span >= 0); - - const list = try p.arena.alloc(*Ast.Node, span); - defer p.scratch.shrinkRetainingCapacity(start_offset); - - var li: usize = 0; - var i: usize = start_offset; - while (i < end_offset) : (i += 1) { - list[li] = p.scratch.items[i]; - li += 1; - } - return list; +fn makeNodeSliceFromScratch(p: *Parse, start: usize) Error![]*Ast.Node { + defer p.scratch.shrinkRetainingCapacity(start); + return p.arena.dupe(*Ast.Node, p.scratch.items[start..]); } -fn makeNodeSequence( +fn makeNodeSliceFrom( p: *Parse, context: *const StmtContext, tag: Ast.Node.Tag, loc: Ast.Node.Span, scratch_offset: usize, ) Error!*Ast.Node { - if (!p.isScratchEmpty(context)) { - const list = try p.nodeListFromScratch(scratch_offset, p.scratch.items.len); - return .createList(p.arena, tag, loc, list); - } - return .createList(p.arena, tag, loc, &.{}); + assert(scratch_offset >= context.scratch_top); + const list = try p.makeNodeSliceFromScratch(scratch_offset); + return .createList(p.arena, tag, loc, list); +} + +fn makeNodeSlice( + p: *Parse, + context: *const StmtContext, + tag: Ast.Node.Tag, + loc: Ast.Node.Span, +) Error!*Ast.Node { + return p.makeNodeSliceFrom(context, tag, loc, context.scratch_top); } fn isBlockStackEmpty(p: *Parse, context: *const StmtContext) bool { @@ -342,7 +338,7 @@ fn collectBlock(p: *Parse, context: *StmtContext, level: usize) Error!?*Ast.Node span_end = span_start; } - var node = try p.makeNodeSequence(context, .block_stmt, .{ + var node = try p.makeNodeSliceFrom(context, .block_stmt, .{ .start = span_start, .end = span_end, }, block.scratch_offset); @@ -399,7 +395,7 @@ fn collectContext( } } - const node = try p.makeNodeSequence(context, .choice_stmt, .{ + const node = try p.makeNodeSliceFrom(context, .choice_stmt, .{ .start = choice_state.source_offset, .end = p.token.loc.start, }, choice_state.scratch_offset); @@ -436,7 +432,7 @@ fn collectKnot(p: *Parse, context: *StmtContext) Error!?*Ast.Node { const proto = p.scratch.items[p.knot_offset]; if (proto.tag != .knot_prototype) return null; - const list = try p.nodeListFromScratch(p.knot_offset + 1, p.scratch.items.len); + const list = try p.makeNodeSliceFromScratch(p.knot_offset + 1); defer _ = p.popScratch(context); return .createKnot(p.arena, .knot_decl, .{ @@ -693,19 +689,6 @@ fn parseStringExpr(p: *Parse) Error!*Ast.Node { }, expr, null); } -fn parseContentString(p: *Parse, token_set: []const Token.Tag) Error!?*Ast.Node { - const main_token = p.token; - while (!p.checkTokenInSet(token_set)) _ = p.nextToken(); - - return .createLeaf(p.arena, if (main_token.loc.start == p.token.loc.start) - .empty_string - else - .string_literal, .{ - .start = main_token.loc.start, - .end = p.token.loc.start, - }); -} - fn parseExprStmt(p: *Parse, lhs: ?*Ast.Node) Error!*Ast.Node { const main_token = p.token; const node = try parseInfixExpr(p, lhs, .none); @@ -825,44 +808,18 @@ fn parseDivertStmt(p: *Parse) Error!*Ast.Node { } fn parseChoiceExpr(p: *Parse) Error!?*Ast.Node { - const token_set = [_]Token.Tag{ - .left_brace, .left_bracket, .right_brace, - .right_bracket, .right_arrow, .newline, - .eof, - }; const main_token = p.token; var lhs: ?*Ast.Node = null; var mhs: ?*Ast.Node = null; var rhs: ?*Ast.Node = null; - lhs = try parseContentString(p, &token_set); - if (lhs) |n| { - if (n.tag != .empty_string) { - n.tag = .choice_start_expr; - } - } + lhs = try parseContent(p, .{ .ignore_brackets = false }); if (p.checkToken(.left_bracket)) { _ = p.nextToken(); p.eatToken(.whitespace); - - if (!p.checkToken(.right_bracket)) { - mhs = try parseContentString(p, &token_set); - if (mhs) |n| { - if (n.tag != .empty_string) { - n.tag = .choice_option_expr; - } - } - } - + mhs = try parseContent(p, .{ .ignore_brackets = false }); _ = try p.expectToken(.right_bracket, false); - if (!p.checkTokenInSet(&token_set)) { - rhs = try parseContentString(p, &token_set); - if (rhs) |n| { - if (n.tag != .empty_string) { - n.tag = .choice_inner_expr; - } - } - } + rhs = try parseContent(p, .{}); } return .createChoice(p.arena, .choice_expr, .{ .start = main_token.loc.start, @@ -919,7 +876,7 @@ fn parseConditional(p: *Parse, main_token: Token, expr: ?*Ast.Node) Error!?*Ast. const node = try p.collectContext(&context, 0, false); if (node) |n| try p.scratch.append(p.gpa, n); - const list = try p.nodeListFromScratch(context.scratch_top, p.scratch.items.len); + const list = try p.makeNodeSliceFromScratch(context.scratch_top); return .createSwitch(p.arena, if (expr != null and !context.is_block_created) .switch_stmt else if (expr == null and !context.is_block_created) @@ -932,18 +889,14 @@ fn parseConditional(p: *Parse, main_token: Token, expr: ?*Ast.Node) Error!?*Ast. } fn parseInlineIf(p: *Parse, main_token: Token, lhs: ?*Ast.Node) Error!?*Ast.Node { - const token_set = [_]Token.Tag{ - .left_brace, .right_brace, .right_arrow, - .glue, .newline, .eof, - }; p.eatToken(.whitespace); - const content_node = try parseContentExpr(p, &token_set); + const content = try parseContent(p, .{}); const end_token = try p.expectToken(.right_brace, true); return .createBinary(p.arena, .inline_if_stmt, .{ .start = main_token.loc.start, .end = end_token.loc.end, - }, lhs, content_node); + }, lhs, content); } fn parseLbraceExpr(p: *Parse) Error!?*Ast.Node { @@ -981,38 +934,88 @@ fn parseLbraceExpr(p: *Parse) Error!?*Ast.Node { } } -fn parseContentExpr(p: *Parse, token_set: []const Token.Tag) Error!?*Ast.Node { +const ContentOptions = struct { + ignore_brackets: bool = true, + ignore_parens: bool = true, +}; + +fn parseContentString(p: *Parse, options: ContentOptions) Error!?*Ast.Node { const main_token = p.token; - const context = makeStmtContext(p, .block, null); while (true) { - var node: ?*Ast.Node = null; - if (!p.checkTokenInSet(token_set)) { - node = try parseContentString(p, token_set); - } else switch (p.token.tag) { - .eof, .newline, .right_brace => break, - .left_brace => node = try parseLbraceExpr(p), - .right_arrow => node = try parseDivertStmt(p), - //.INK_TT_GLUE => node = ink_parse_glue(p), - else => { - return p.fail(.unexpected_token, p.token); - }, + switch (p.token.tag) { + .eof, + .newline, + .left_arrow, + .right_arrow, + .left_brace, + .right_brace, + .glue, + => break, + .left_bracket, .right_bracket => |tag| if (!options.ignore_brackets) + break + else + p.eatToken(tag), + else => |tag| p.eatToken(tag), } - if (node) |n| try p.scratch.append(p.gpa, n); } - return p.makeNodeSequence(&context, .content, .{ + if (main_token.loc.start == p.token.loc.start) return null; + + return .createLeaf(p.arena, .string_literal, .{ .start = main_token.loc.start, .end = p.token.loc.start, - }, context.scratch_top); + }); +} + +fn parseContent(p: *Parse, options: ContentOptions) Error!?*Ast.Node { + const main_token = p.token; + const scratch_top = p.scratch.items.len; + var leading_glue = false; + var trailing_glue = false; + + if (p.token.tag == .glue) { + leading_glue = true; + _ = p.nextToken(); + } + while (true) { + const node: ?*Ast.Node = switch (p.token.tag) { + .eof, .newline, .left_arrow, .right_brace => break, + .left_brace => try parseLbraceExpr(p), + .right_arrow => try parseDivertStmt(p), + .glue => blk: { + const next_token = p.nextToken(); + switch (next_token.tag) { + .eof, .newline, .right_brace => { + trailing_glue = true; + break; + }, + else => break :blk null, + } + }, + else => |tag| blk: { + switch (tag) { + .left_bracket, .right_bracket => if (!options.ignore_brackets) break, + else => {}, + } + break :blk try parseContentString(p, options); + }, + }; + if (node) |n| try p.scratch.append(p.gpa, n); + } + if (main_token.loc.start == p.token.loc.start) return null; + return .createContent(p.arena, .content, .{ + .start = main_token.loc.start, + .end = p.token.loc.start, + }, .{ + .items = try p.makeNodeSliceFromScratch(scratch_top), + .leading_glue = leading_glue, + .trailing_glue = trailing_glue, + }); } fn parseContentStmt(p: *Parse) Error!*Ast.Node { - const token_set = [_]Token.Tag{ - .left_brace, .right_brace, .right_arrow, - .glue, .newline, .eof, - }; const main_token = p.token; - const node = try parseContentExpr(p, &token_set); + const node = try parseContent(p, .{}); const end_token = try p.expectNewline(); return .createBinary(p.arena, .content_stmt, .{ .start = main_token.loc.start, @@ -1052,10 +1055,10 @@ fn parseParameterList(p: *Parse) Error!?*Ast.Node { } _ = try p.expectToken(.right_paren, true); - return p.makeNodeSequence(&context, .parameter_list, .{ + return p.makeNodeSlice(&context, .parameter_list, .{ .start = main_token.loc.start, .end = p.token.loc.start, - }, context.scratch_top); + }); } fn parseArgumentList(p: *Parse) Error!?*Ast.Node { @@ -1078,10 +1081,10 @@ fn parseArgumentList(p: *Parse) Error!?*Ast.Node { } _ = try p.expectToken(.right_paren, false); - return p.makeNodeSequence(&context, .argument_list, .{ + return p.makeNodeSlice(&context, .argument_list, .{ .start = main_token.loc.start, .end = p.token.loc.start, - }, context.scratch_top); + }); } fn parseConditionalBranch(p: *Parse, tag: Ast.Node.Tag) Error!?*Ast.Node { @@ -1249,8 +1252,8 @@ pub fn parseFile(p: *Parse) Error!*Ast.Node { const node = try p.collectKnot(&context); if (node) |n| try p.scratch.append(p.gpa, n); - return p.makeNodeSequence(&context, .file, .{ + return p.makeNodeSlice(&context, .file, .{ .start = main_token.loc.start, .end = p.token.loc.end, - }, context.scratch_top); + }); } diff --git a/src/Story/testdata/I036/story.ink b/src/Story/testdata/I036/story.ink index 43e5e04..8ee92d3 100644 --- a/src/Story/testdata/I036/story.ink +++ b/src/Story/testdata/I036/story.ink @@ -6,4 +6,3 @@ A B === function string() ~ return "{3}" -} diff --git a/src/parser_tests.zig b/src/parser_tests.zig index 40c209a..9dce462 100644 --- a/src/parser_tests.zig +++ b/src/parser_tests.zig @@ -116,13 +116,16 @@ test "parser: choice statements, simple" { \\ `--ChoiceStmt \\ |--ChoiceStarStmt \\ | `--ChoiceContentExpr - \\ | `--ChoiceStartContentExpr `A` + \\ | `--Content + \\ | `--StringLiteral `A` \\ |--ChoiceStarStmt \\ | `--ChoiceContentExpr - \\ | `--ChoiceStartContentExpr `B` + \\ | `--Content + \\ | `--StringLiteral `B` \\ `--ChoiceStarStmt \\ `--ChoiceContentExpr - \\ `--ChoiceStartContentExpr `C` + \\ `--Content + \\ `--StringLiteral `C` \\ , ); @@ -137,7 +140,6 @@ test "parser: choice statements, empty branch" { \\ `--ChoiceStmt \\ `--ChoiceStarStmt \\ `--ChoiceContentExpr - \\ `--EmptyString \\ , ); @@ -156,19 +158,24 @@ test "parser: choice statements, level normalization" { \\ `--ChoiceStmt \\ |--ChoiceStarStmt \\ | `--ChoiceContentExpr - \\ | `--ChoiceStartContentExpr `A` + \\ | `--Content + \\ | `--StringLiteral `A` \\ |--ChoiceStarStmt \\ | `--ChoiceContentExpr - \\ | `--ChoiceStartContentExpr `B` + \\ | `--Content + \\ | `--StringLiteral `B` \\ |--ChoiceStarStmt \\ | `--ChoiceContentExpr - \\ | `--ChoiceStartContentExpr `C` + \\ | `--Content + \\ | `--StringLiteral `C` \\ |--ChoiceStarStmt \\ | `--ChoiceContentExpr - \\ | `--ChoiceStartContentExpr `D` + \\ | `--Content + \\ | `--StringLiteral `D` \\ `--ChoiceStarStmt \\ `--ChoiceContentExpr - \\ `--ChoiceStartContentExpr `E` + \\ `--Content + \\ `--StringLiteral `E` \\ , ); @@ -183,8 +190,10 @@ test "parser: choice statements, mixed context" { \\ `--ChoiceStmt \\ `--ChoiceStarStmt \\ `--ChoiceContentExpr - \\ |--ChoiceStartContentExpr `A` - \\ `--ChoiceInnerContentExpr `B` + \\ |--Content + \\ | `--StringLiteral `A` + \\ `--Content + \\ `--StringLiteral `B` \\ , );