module: begin improved query implementation

Signed-off-by: Stephen Gutekanst <stephen@hexops.com>

src/module/Archetype.zig

@@ -292,7 +292,7 @@ pub fn Slicer(comptime modules: anytype) type {
                 @tagName(component_name),
             ).type;
             if (namespace_name == .entity and component_name == .id) {
-                const name_id = slicer.archetype.component_names.index("id").?;
+                const name_id = slicer.archetype.component_names.index("entity.id").?;
                 return slicer.archetype.getColumnValues(name_id, Type).?[0..slicer.archetype.len];
             }
             const name = @tagName(namespace_name) ++ "." ++ @tagName(component_name);

src/module/entities.zig

@@ -9,6 +9,11 @@ const StringTable = @import("StringTable.zig");
 const ComponentTypesByName = @import("module.zig").ComponentTypesByName;
 const merge = @import("main.zig").merge;
 const builtin_modules = @import("main.zig").builtin_modules;
+const EntityModule = @import("main.zig").EntityModule;
+const ModuleName = @import("module.zig").ModuleName;
+const ComponentNameM = @import("module.zig").ComponentNameM;
+const ComponentName = @import("module.zig").ComponentName;
+const ModsByName = @import("module.zig").ModsByName;
 
 /// An entity ID uniquely identifies an entity globally within an Entities set.
 pub const EntityID = u64;
@@ -92,6 +97,8 @@ pub fn Entities(comptime modules: anytype) type {
         component_names: *StringTable,
         id_name: StringTable.Index = 0,
 
+        active_queries: std.ArrayListUnmanaged(query2.State) = .{},
+
         const Self = @This();
 
         /// Points to where an entity is stored, specifically in which archetype table and in which row
@@ -124,7 +131,7 @@ pub fn Entities(comptime modules: anytype) type {
                 .component_names = component_names,
                 .buckets = buckets,
             };
-            entities.id_name = try entities.component_names.indexOrPut(allocator, "id");
+            entities.id_name = entities.componentName(EntityModule.name, .id);
 
             const columns = try allocator.alloc(Archetype.Column, 1);
             columns[0] = .{
@@ -153,6 +160,7 @@ pub fn Entities(comptime modules: anytype) type {
             entities.allocator.free(entities.buckets);
             for (entities.archetypes.items) |*archetype| archetype.deinit(entities.allocator);
             entities.archetypes.deinit(entities.allocator);
+            entities.active_queries.deinit(entities.allocator);
         }
 
         fn archetypeOrPut(
@@ -267,10 +275,18 @@ pub fn Entities(comptime modules: anytype) type {
         /// The set of components used is expected to be static for the lifetime of the Entities,
         /// and as such this function allocates component names but there is no way to release that
         /// memory until Entities.deinit() is called.
-        pub fn componentName(entities: *Self, name_str: []const u8) StringTable.Index {
+        pub fn componentNameString(entities: *Self, name_str: []const u8) StringTable.Index {
             return entities.component_names.indexOrPut(entities.allocator, name_str) catch @panic("TODO: implement stateful OOM");
         }
 
+        pub fn componentName(
+            entities: *Self,
+            comptime module_name: ModuleName(modules),
+            comptime component_name: ComponentName(modules),
+        ) StringTable.Index {
+            return entities.componentNameString(@tagName(module_name) ++ "." ++ @tagName(component_name));
+        }
+
         /// Returns the archetype storage for the given entity.
         pub inline fn archetypeByID(entities: *Self, entity: EntityID) *Archetype {
             const ptr = entities.entities.get(entity).?;
@@ -619,19 +635,81 @@ pub fn Entities(comptime modules: anytype) type {
             return ArchetypeIterator(modules).init(entities, q);
         }
 
-        // TODO: queryDynamic
+        pub const query2 = struct {
+            /// Represents a dynamic (runtime-generated, non type safe) query.
+            pub const Dynamic = union(enum) {
+                /// Logical AND operator for query expressions
+                op_and: []const @This(),
 
-        // TODO: iteration over all entities
+                /// Match a specific module component.
-        // TODO: iteration over all entities with components (U, V, ...)
+                // TODO: add component name type and consider replacing StringTable approach with global enum
-        // TODO: iteration over all entities with type T
+                component: StringTable.Index,
-        // TODO: iteration over all entities with type T and components (U, V, ...)
 
-        // TODO: "indexes" - a few ideas we could express:
+                pub fn match(q: @This(), archetype: *Archetype) bool {
-        //
+                    switch (q) {
-        // * Graph relations index: e.g. parent-child entity relations for a DOM / UI / scene graph.
+                        .op_and => |qs| {
-        // * Spatial index: "give me all entities within 5 units distance from (x, y, z)"
+                            for (qs) |and_q| if (!and_q.match(archetype)) return false;
-        // * Generic index: "give me all entities where arbitraryFunction(e) returns true"
+                            return true;
-        //
+                        },
+                        .component => |component_name| {
+                            for (archetype.columns) |column| if (column.name == component_name) return true;
+                            return false;
+                        },
+                    }
+                }
+            };
+
+            /// When a query is first performed, it becomes active and its iterator state is stored
+            /// and maintained. When all results in the iterator have been consumed, it is marked
+            /// as finished and later recycled.
+            pub const State = struct {
+                q: query2.Dynamic,
+                next_index: u31 = 0, // archetypes index
+                finished: bool = false,
+            };
+
+            /// Represents a dynamic (runtime-generated, non type safe) query result.
+            pub const DynamicResult = struct {
+                entities: *Self,
+                index: u32, // active_queries index
+
+                pub fn next(q: *DynamicResult) ?*Archetype {
+                    const state = &q.entities.active_queries.items[q.index];
+                    if (state.finished) @panic("query iterator already finished, invoking next() is illegal");
+
+                    while (state.next_index < q.entities.archetypes.items.len) {
+                        const archetype = &q.entities.archetypes.items[state.next_index];
+                        state.next_index += 1;
+                        if (state.q.match(archetype)) return archetype;
+                    }
+
+                    state.finished = true;
+                    q.entities.reuseInactiveQueries();
+                    return null;
+                }
+            };
+        };
+
+        /// Performs a dynamic (runtime-generated, non type safe) query.
+        pub fn queryDynamic(entities: *Self, q: query2.Dynamic) !query2.DynamicResult {
+            const new_query = query2.DynamicResult{
+                .entities = entities,
+                .index = @intCast(entities.active_queries.items.len),
+            };
+            const state = query2.State{ .q = q };
+            try entities.active_queries.append(entities.allocator, state);
+            return new_query;
+        }
+
+        /// Releases any inactive queries entities.active_queries state memory space at the end of
+        /// the list, enabling reuse of it.
+        fn reuseInactiveQueries(entities: *Self) void {
+            var new_len: usize = entities.active_queries.items.len;
+            while (new_len > 0 and entities.active_queries.items[new_len - 1].finished) {
+                new_len -= 1;
+            }
+            entities.active_queries.shrinkRetainingCapacity(new_len);
+        }
 
         // TODO: ability to remove archetype entirely, deleting all entities in it
         // TODO: ability to remove archetypes with no entities (garbage collection)
@@ -679,7 +757,7 @@ pub fn ArchetypeIterator(comptime modules: anytype) type {
                                     const name = switch (component) {
                                         inline else => |c| std.fmt.bufPrint(&buf, "{s}.{s}", .{ @tagName(namespace), @tagName(c) }) catch break,
                                     };
-                                    const name_id = iter.entities.componentName(name);
+                                    const name_id = iter.entities.componentNameString(name);
                                     var has_column = false;
                                     for (consideration.columns) |column| {
                                         if (column.name == name_id) {
@@ -723,20 +801,20 @@ test "dynamic" {
     const Rotation = struct { degrees: f32 };
 
     // Create a world.
-    var world = try Entities(.{}).init(allocator);
+    var world = try Entities(merge(.{builtin_modules})).init(allocator);
     defer world.deinit();
 
     // Create an entity and add dynamic components.
     const player1 = try world.new();
-    try world.setComponentDynamic(player1, world.componentName("game.name"), "jane", @alignOf([]const u8), 100);
+    try world.setComponentDynamic(player1, world.componentNameString("game.name"), "jane", @alignOf([]const u8), 100);
-    try world.setComponentDynamic(player1, world.componentName("game.name"), "joey", @alignOf([]const u8), 100);
+    try world.setComponentDynamic(player1, world.componentNameString("game.name"), "joey", @alignOf([]const u8), 100);
-    try world.setComponentDynamic(player1, world.componentName("game.location"), asBytes(&Location{ .x = 1, .y = 2, .z = 3 }), @alignOf(Location), 101);
+    try world.setComponentDynamic(player1, world.componentNameString("game.location"), asBytes(&Location{ .x = 1, .y = 2, .z = 3 }), @alignOf(Location), 101);
 
     // Get components
-    try testing.expect(world.getComponentDynamic(player1, world.componentName("game.rotation"), @sizeOf(Rotation), @alignOf(Rotation), 102) == null);
+    try testing.expect(world.getComponentDynamic(player1, world.componentNameString("game.rotation"), @sizeOf(Rotation), @alignOf(Rotation), 102) == null);
-    const loc = world.getComponentDynamic(player1, world.componentName("game.location"), @sizeOf(Location), @alignOf(Location), 101);
+    const loc = world.getComponentDynamic(player1, world.componentNameString("game.location"), @sizeOf(Location), @alignOf(Location), 101);
     try testing.expectEqual(Location{ .x = 1, .y = 2, .z = 3 }, std.mem.bytesToValue(Location, @as(*[12]u8, @ptrCast(loc.?.ptr))));
-    try testing.expectEqualStrings(world.getComponentDynamic(player1, world.componentName("game.name"), 4, @alignOf([]const u8), 100).?, "joey");
+    try testing.expectEqualStrings(world.getComponentDynamic(player1, world.componentNameString("game.name"), 4, @alignOf([]const u8), 100).?, "joey");
 }
 
 test "entity ID size" {
@@ -754,16 +832,18 @@ test "example" {
 
     const Rotation = struct { degrees: f32 };
 
+    const Game = struct {
+        pub const name = .game;
+        pub const components = .{
+            .name = .{ .type = []const u8 },
+            .location = .{ .type = Location },
+            .rotation = .{ .type = Rotation },
+        };
+    };
+
     const modules = merge(.{
         builtin_modules,
-        struct {
+        Game,
-            pub const name = .game;
-            pub const components = .{
-                .name = .{ .type = []const u8 },
-                .location = .{ .type = Location },
-                .rotation = .{ .type = Rotation },
-            };
-        },
     });
 
     //-------------------------------------------------------------------------
@@ -819,7 +899,7 @@ test "example" {
     // Resolve archetype by entity ID and print column names
     const columns = world.archetypeByID(player2).columns;
     try testing.expectEqual(@as(usize, 2), columns.len);
-    try testing.expectEqualStrings("id", world.component_names.string(columns[0].name));
+    try testing.expectEqualStrings("entity.id", world.component_names.string(columns[0].name));
     try testing.expectEqualStrings("game.rotation", world.component_names.string(columns[1].name));
 
     //-------------------------------------------------------------------------
@@ -833,6 +913,21 @@ test "example" {
         try testing.expectEqual(player2, ids[0]);
     }
 
+    // Dynamic queries (e.g. issued from another programming language without comptime)
+    var q = try world.queryDynamic(.{
+        .op_and = &.{
+            .{ .component = world.componentName(EntityModule.name, .id) },
+            .{ .component = world.componentName(Game.name, .rotation) },
+        },
+    });
+    while (q.next()) |archtype| {
+        try testing.expectEqual(@as(usize, 1), archtype.len);
+        try testing.expectEqual(@as(usize, 2), archtype.columns.len);
+
+        try testing.expectEqualStrings("entity.id", world.component_names.string(archtype.columns[0].name));
+        try testing.expectEqualStrings("game.rotation", world.component_names.string(archtype.columns[1].name));
+    }
+
     // TODO: iterating components an entity has not currently supported.
 
     //-------------------------------------------------------------------------
@@ -843,7 +938,7 @@ test "example" {
 test "empty_world" {
     const allocator = testing.allocator;
     //-------------------------------------------------------------------------
-    var world = try Entities(.{}).init(allocator);
+    var world = try Entities(merge(.{builtin_modules})).init(allocator);
     // Create a world.
     defer world.deinit();
 }
@@ -859,7 +954,8 @@ test "many entities" {
 
     const Rotation = struct { degrees: f32 };
 
-    const modules = .{
+    const modules = merge(.{
+        builtin_modules,
         struct {
             pub const name = .game;
             pub const components = .{
@@ -868,7 +964,7 @@ test "many entities" {
                 .rotation = .{ .type = Rotation },
             };
         },
-    };
+    });
 
     // Create many entities
     var world = try Entities(modules).init(allocator);
@@ -886,16 +982,16 @@ test "many entities" {
     // Confirm archetypes
     var columns = archetypes[0].columns;
     try testing.expectEqual(@as(usize, 1), columns.len);
-    try testing.expectEqualStrings("id", world.component_names.string(columns[0].name));
+    try testing.expectEqualStrings("entity.id", world.component_names.string(columns[0].name));
 
     columns = archetypes[1].columns;
     try testing.expectEqual(@as(usize, 2), columns.len);
-    try testing.expectEqualStrings("id", world.component_names.string(columns[0].name));
+    try testing.expectEqualStrings("entity.id", world.component_names.string(columns[0].name));
     try testing.expectEqualStrings("game.name", world.component_names.string(columns[1].name));
 
     columns = archetypes[2].columns;
     try testing.expectEqual(@as(usize, 3), columns.len);
-    try testing.expectEqualStrings("id", world.component_names.string(columns[0].name));
+    try testing.expectEqualStrings("entity.id", world.component_names.string(columns[0].name));
     try testing.expectEqualStrings("game.name", world.component_names.string(columns[1].name));
     try testing.expectEqualStrings("game.location", world.component_names.string(columns[2].name));
 }

src/module/module.zig

@@ -66,6 +66,7 @@ const testing = @import("../testing.zig");
 const Entities = @import("entities.zig").Entities;
 const EntityID = @import("entities.zig").EntityID;
 const is_debug = @import("Archetype.zig").is_debug;
+const builtin_modules = @import("main.zig").builtin_modules;
 
 const log = std.log.scoped(.mach);
 
@@ -534,7 +535,7 @@ pub fn Modules(comptime modules: anytype) type {
     };
 }
 
-fn ModsByName(comptime modules: anytype) type {
+pub fn ModsByName(comptime modules: anytype) type {
     var fields: []const std.builtin.Type.StructField = &[0]std.builtin.Type.StructField{};
     for (modules) |M| {
         const ModT = ModSet(modules).Mod(M);
@@ -627,8 +628,7 @@ pub fn ModSet(comptime modules: anytype) type {
                 pub inline fn set(
                     m: *@This(),
                     entity: EntityID,
-                    // TODO(important): cleanup comptime
+                    comptime component_name: ComponentNameM(M),
-                    comptime component_name: std.meta.FieldEnum(@TypeOf(components)),
                     component: @field(components, @tagName(component_name)).type,
                 ) !void {
                     try m.entities.setComponent(entity, module_tag, component_name, component);
@@ -639,8 +639,7 @@ pub fn ModSet(comptime modules: anytype) type {
                 pub inline fn get(
                     m: *@This(),
                     entity: EntityID,
-                    // TODO(important): cleanup comptime
+                    comptime component_name: ComponentNameM(M),
-                    comptime component_name: std.meta.FieldEnum(@TypeOf(components)),
                 ) ?@field(components, @tagName(component_name)).type {
                     return m.entities.getComponent(entity, module_tag, component_name);
                 }
@@ -649,8 +648,7 @@ pub fn ModSet(comptime modules: anytype) type {
                 pub inline fn remove(
                     m: *@This(),
                     entity: EntityID,
-                    // TODO(important): cleanup comptime
+                    comptime component_name: ComponentNameM(M),
-                    comptime component_name: std.meta.FieldEnum(@TypeOf(components)),
                 ) !void {
                     try m.entities.removeComponent(entity, module_tag, component_name);
                 }
@@ -908,8 +906,38 @@ fn GlobalEventEnumM(comptime M: anytype) type {
     });
 }
 
+/// enum describing component names for the given module only
+pub fn ComponentNameM(comptime M: type) type {
+    const components = ComponentTypesM(M){};
+    return std.meta.FieldEnum(@TypeOf(components));
+}
+
+/// enum describing component names for all of the modules
+pub fn ComponentName(comptime modules: anytype) type {
+    var enum_fields: []const std.builtin.Type.EnumField = &[0]std.builtin.Type.EnumField{};
+    var i: usize = 0;
+    inline for (modules) |M| {
+        search: for (@typeInfo(ComponentTypesM(M)).Struct.fields) |field| {
+            for (enum_fields) |existing| if (std.mem.eql(u8, existing.name, field.name)) continue :search;
+            enum_fields = enum_fields ++ [_]std.builtin.Type.EnumField{.{
+                .name = field.name,
+                .value = i,
+            }};
+            i += 1;
+        }
+    }
+    return @Type(.{
+        .Enum = .{
+            .tag_type = std.math.IntFittingRange(0, enum_fields.len - 1),
+            .fields = enum_fields,
+            .decls = &[_]std.builtin.Type.Declaration{},
+            .is_exhaustive = true,
+        },
+    });
+}
+
 /// enum describing every possible comptime-known module name
-fn ModuleName(comptime modules: anytype) type {
+pub fn ModuleName(comptime modules: anytype) type {
     var enum_fields: []const std.builtin.Type.EnumField = &[0]std.builtin.Type.EnumField{};
     for (modules, 0..) |M, i| {
         enum_fields = enum_fields ++ [_]std.builtin.Type.EnumField{.{ .name = @tagName(M.name), .value = i }};
@@ -1181,11 +1209,12 @@ test Modules {
         pub const name = .engine_sprite2d;
     });
 
-    var modules: Modules(.{
+    var modules: Modules(merge(.{
+        builtin_modules,
         Physics,
         Renderer,
         Sprite2D,
-    }) = undefined;
+    })) = undefined;
     try modules.init(testing.allocator);
     defer modules.deinit(testing.allocator);
     testing.refAllDeclsRecursive(Physics);
@@ -1237,11 +1266,12 @@ test "event name" {
         fn fooBar() void {}
     });
 
-    const Ms = Modules(.{
+    const Ms = Modules(merge(.{
+        builtin_modules,
         Physics,
         Renderer,
         Sprite2D,
-    });
+    }));
 
     const locals = @typeInfo(Ms.LocalEvent).Enum;
     try testing.expect(type, u1).eql(locals.tag_type);
@@ -1270,19 +1300,21 @@ test ModuleName {
     const Sprite2D = ModuleInterface(struct {
         pub const name = .engine_sprite2d;
     });
-    const modules = .{
+    const modules = merge(.{
+        builtin_modules,
         Physics,
         Renderer,
         Sprite2D,
-    };
+    });
     _ = Modules(modules);
     const info = @typeInfo(ModuleName(modules)).Enum;
 
     try testing.expect(type, u2).eql(info.tag_type);
-    try testing.expect(usize, 3).eql(info.fields.len);
+    try testing.expect(usize, 4).eql(info.fields.len);
-    try testing.expect([]const u8, "engine_physics").eql(info.fields[0].name);
+    try testing.expect([]const u8, "entity").eql(info.fields[0].name);
-    try testing.expect([]const u8, "engine_renderer").eql(info.fields[1].name);
+    try testing.expect([]const u8, "engine_physics").eql(info.fields[1].name);
-    try testing.expect([]const u8, "engine_sprite2d").eql(info.fields[2].name);
+    try testing.expect([]const u8, "engine_renderer").eql(info.fields[2].name);
+    try testing.expect([]const u8, "engine_sprite2d").eql(info.fields[3].name);
 }
 
 // TODO: remove this in favor of testing.expect
@@ -1444,10 +1476,11 @@ test "event name calling" {
         }
     });
 
-    const modules2 = .{
+    const modules2 = merge(.{
+        builtin_modules,
         Physics,
         Renderer,
-    };
+    });
     var modules: Modules(modules2) = undefined;
     try modules.init(testing.allocator);
     defer modules.deinit(testing.allocator);
@@ -1578,11 +1611,12 @@ test "dispatch" {
         }
     });
 
-    const modules2 = .{
+    const modules2 = merge(.{
+        builtin_modules,
         Minimal,
         Physics,
         Renderer,
-    };
+    });
     var modules: Modules(modules2) = undefined;
     try modules.init(testing.allocator);
     defer modules.deinit(testing.allocator);

src/module/query.zig

@@ -1,6 +1,8 @@
 const std = @import("std");
 const testing = std.testing;
 const ComponentTypesByName = @import("module.zig").ComponentTypesByName;
+const merge = @import("main.zig").merge;
+const builtin_modules = @import("main.zig").builtin_modules;
 
 pub const QueryTag = enum {
     any,
@@ -80,7 +82,8 @@ test "query" {
 
     const Rotation = struct { degrees: f32 };
 
-    const modules = .{
+    const modules = merge(.{
+        builtin_modules,
         struct {
             pub const name = .game;
             pub const components = .{
@@ -97,7 +100,7 @@ test "query" {
         struct {
             pub const name = .renderer;
         },
-    };
+    });
 
     const Q = Query(modules);