Build terms for cached queries

* Add support for components in both positions of pairs

* Export type function

* Rework query types

.github/workflows/styling.yaml

@@ -1,20 +0,0 @@
-name: Styling
-
-on: [push, pull_request, workflow_dispatch]
-
-jobs:
-    run:
-        name: Run Stylua
-        runs-on: ubuntu-latest
-
-        steps:
-            - name: Checkout Project
-              uses: actions/checkout@v4
-
-            - name: Run Stylua
-              uses: JohnnyMorganz/stylua-action@v4
-              with:
-                  token: ${{ secrets.GITHUB_TOKEN }}
-                  version: latest # NOTE: we recommend pinning to a specific version in case of formatting changes
-                  # CLI arguments
-                  args: --check jecs.luau

jecs.luau

@@ -78,31 +78,32 @@ type EntityIndex = {
 
 local HI_COMPONENT_ID = _G.__JECS_HI_COMPONENT_ID or 256
 -- stylua: ignore start
-local EcsOnAdd =          HI_COMPONENT_ID +  1
-local EcsOnRemove =       HI_COMPONENT_ID +  2
-local EcsOnSet =          HI_COMPONENT_ID +  3
-local EcsWildcard =       HI_COMPONENT_ID +  4
-local EcsChildOf =        HI_COMPONENT_ID +  5
-local EcsComponent =      HI_COMPONENT_ID +  6
-local EcsOnDelete =       HI_COMPONENT_ID +  7
-local EcsOnDeleteTarget = HI_COMPONENT_ID +  8
-local EcsDelete =         HI_COMPONENT_ID +  9
-local EcsRemove =         HI_COMPONENT_ID + 10
-local EcsName =           HI_COMPONENT_ID + 11
-local EcsTableCreate =    HI_COMPONENT_ID + 12
-local EcsRest =           HI_COMPONENT_ID + 13
+local EcsOnAdd =           HI_COMPONENT_ID +  1
+local EcsOnRemove =        HI_COMPONENT_ID +  2
+local EcsOnSet =           HI_COMPONENT_ID +  3
+local EcsWildcard =        HI_COMPONENT_ID +  4
+local EcsChildOf =         HI_COMPONENT_ID +  5
+local EcsComponent =       HI_COMPONENT_ID +  6
+local EcsOnDelete =        HI_COMPONENT_ID +  7
+local EcsOnDeleteTarget =  HI_COMPONENT_ID +  8
+local EcsDelete =          HI_COMPONENT_ID +  9
+local EcsRemove =          HI_COMPONENT_ID + 10
+local EcsName =            HI_COMPONENT_ID + 11
+local EcsArchetypeCreate = HI_COMPONENT_ID + 12
+local EcsArchetypeDelete = HI_COMPONENT_ID + 13
+local EcsRest =            HI_COMPONENT_ID + 14
 
 local ECS_PAIR_FLAG =                       0x8
 local ECS_ID_FLAGS_MASK =                  0x10
 local ECS_ENTITY_MASK =     bit32.lshift(1, 24)
 local ECS_GENERATION_MASK = bit32.lshift(1, 16)
 
-local ECS_ID_DELETE =              0b0000_0001
-local ECS_ID_IS_TAG =              0b0000_0010
-local ECS_ID_HAS_ON_ADD =          0b0000_0100
-local ECS_ID_HAS_ON_SET =          0b0000_1000
-local ECS_ID_HAS_ON_REMOVE =       0b0001_0000
-local ECS_ID_MASK =                0b0000_0000
+local ECS_ID_DELETE =               0b0000_0001
+local ECS_ID_IS_TAG =               0b0000_0010
+local ECS_ID_HAS_ON_ADD =           0b0000_0100
+local ECS_ID_HAS_ON_SET =           0b0000_1000
+local ECS_ID_HAS_ON_REMOVE =        0b0001_0000
+local ECS_ID_MASK =                 0b0000_0000
 -- stylua: ignore end
 local NULL_ARRAY = table.freeze({}) :: Column
 
@@ -251,55 +252,27 @@ local function ecs_pair_second(world, e)
 	return entity_index_get_alive(world.entity_index, ECS_ENTITY_T_LO(e))
 end
 
-local function query_match(query, archetype)
+local function query_match(terms: { {i53 }}, archetype: Archetype)
 	local records = archetype.records
-	for _, id in query.ids do
-		if not records[id] then
-			return false
-		end
-	end
 
-	local filters = query.filters
-	if filters then
-		local without = filters.without
-		if without then
-			for _, id in filters.without do
-				if records[id] then
-					return false
-				end
-			end
-		end
-		local with = filters.with
-		if with then
-			for _, id in filters.without do
-				if not records[id] then
-					return false
-				end
-			end
+	for _, term in terms do
+		local id = term[1]
+		local out = term[2]
+		local has = records[id] ~= nil
+		if has ~= not out then
+			return false
 		end
 	end
 
 	return true
 end
 
-local function observer_invoke(observer, event)
-	table.insert(observer.query.compatible_archetypes, event.archetype)
-end
-
-local function emit(world: World, event)
-	local map = world.observerable[event.id]
-	if not map then
-		return
-	end
-	local observer_list: {[string]: any} = map[event.component]
-	if not observer_list then
-		return
-	end
-	for _, observer in observer_list do
-		if query_match(observer.query, event.archetype) then
-			observer_invoke(observer, event)
-		end
+local function find_observers(world: World, event, component): { Observer }?
+	local cache = world.observerable[event]
+	if not cache then
+		return nil
 	end
+	return cache[component] :: any
 end
 
 local function archetype_move(entity_index: EntityIndex, to: Archetype, dst_row: i24, from: Archetype, src_row: i24)
@@ -309,7 +282,7 @@ local function archetype_move(entity_index: EntityIndex, to: Archetype, dst_row:
 	local src_entities = from.entities
 
 	local last = #src_entities
-	local types = from.types
+	local id_types = from.types
 	local records = to.records
 
 	for i, column in src_columns do
@@ -318,12 +291,13 @@ local function archetype_move(entity_index: EntityIndex, to: Archetype, dst_row:
 		end
 		-- Retrieves the new column index from the source archetype's record from each component
 		-- We have to do this because the columns are tightly packed and indexes may not correspond to each other.
-		local tr = records[types[i]]
+		local tr = records[id_types[i]]
 
 		-- Sometimes target column may not exist, e.g. when you remove a component.
 		if tr then
 			dst_columns[tr.column][dst_row] = column[src_row]
 		end
+
 		-- If the entity is the last row in the archetype then swapping it would be meaningless.
 		if src_row ~= last then
 			-- Swap rempves columns to ensure there are no holes in the archetype.
@@ -527,7 +501,8 @@ local function id_record_ensure(world: World, id: number): IdRecord
 	if not idr then
 		local flags = ECS_ID_MASK
 		local relation = id
-		if ECS_IS_PAIR(id) then
+		local is_pair = ECS_IS_PAIR(id)
+		if is_pair then
 			relation = ecs_pair_first(world, id)
 		end
 
@@ -544,6 +519,10 @@ local function id_record_ensure(world: World, id: number): IdRecord
 
 		local is_tag = not world_has_one_inline(world, relation, EcsComponent)
 
+		if is_tag and is_pair then
+			is_tag = not world_has_one_inline(world, ecs_pair_second(world, id), EcsComponent)
+		end
+
 		flags = bit32.bor(
 			flags,
 			if on_add then ECS_ID_HAS_ON_ADD else 0,
@@ -587,26 +566,6 @@ local function archetype_append_to_records(
 	end
 end
 
-local function create_observer_uni(world: World, component: number, event)
-	local map = world.observerable[event]
-	if not map then
-		map = {}
-		world.observerable[event] = map
-	end
-
-	local observer_list = map[component]
-	if not observer_list then
-		observer_list = {}
-		map[component] = observer_list
-	end
-
-	local observer = {}
-
-	table.insert(observer_list, observer)
-
-	return observer
-end
-
 local function archetype_create(world: World, id_types: { i24 }, ty, prev: i53?): Archetype
 	local archetype_id = (world.nextArchetypeId :: number) + 1
 	world.nextArchetypeId = archetype_id
@@ -650,11 +609,18 @@ local function archetype_create(world: World, id_types: { i24 }, ty, prev: i53?)
 		else
 			columns[i] = NULL_ARRAY
 		end
-
 	end
 
 	for _, id in id_types do
-		emit(world, { id = EcsTableCreate, component = id, archetype = archetype})
+		local observer_list = find_observers(world, EcsArchetypeCreate, id)
+		if not observer_list then
+			continue
+		end
+		for _, observer in observer_list do
+			if query_match(observer.terms, archetype) then
+				observer.callback(archetype)
+			end
+		end
 	end
 
 	world.archetypeIndex[ty] = archetype
@@ -671,22 +637,22 @@ local function world_parent(world: World, entity: i53)
 	return world_target(world, entity, EcsChildOf, 0)
 end
 
-local function archetype_ensure(world: World, types): Archetype
-	if #types < 1 then
+local function archetype_ensure(world: World, id_types): Archetype
+	if #id_types < 1 then
 		return world.ROOT_ARCHETYPE
 	end
 
-	local ty = hash(types)
+	local ty = hash(id_types)
 	local archetype = world.archetypeIndex[ty]
 	if archetype then
 		return archetype
 	end
 
-	return archetype_create(world, types, ty)
+	return archetype_create(world, id_types, ty)
 end
 
-local function find_insert(types: { i53 }, toAdd: i53): number
-	for i, id in types do
+local function find_insert(id_types: { i53 }, toAdd: i53): number
+	for i, id in id_types do
 		if id == toAdd then
 			return -1
 		end
@@ -694,17 +660,17 @@ local function find_insert(types: { i53 }, toAdd: i53): number
 			return i
 		end
 	end
-	return #types + 1
+	return #id_types + 1
 end
 
 local function find_archetype_with(world: World, node: Archetype, id: i53): Archetype
-	local types = node.types
+	local id_types = node.types
 	-- Component IDs are added incrementally, so inserting and sorting
 	-- them each time would be expensive. Instead this insertion sort can find the insertion
 	-- point in the types array.
 
 	local dst = table.clone(node.types) :: { i53 }
-	local at = find_insert(types, id)
+	local at = find_insert(id_types, id)
 	if at == -1 then
 		-- If it finds a duplicate, it just means it is the same archetype so it can return it
 		-- directly instead of needing to hash types for a lookup to the archetype.
@@ -716,13 +682,13 @@ local function find_archetype_with(world: World, node: Archetype, id: i53): Arch
 end
 
 local function find_archetype_without(world: World, node: Archetype, id: i53): Archetype
-	local types = node.types
-	local at = table.find(types, id)
+	local id_types = node.types
+	local at = table.find(id_types, id)
 	if at == nil then
 		return node
 	end
 
-	local dst = table.clone(types)
+	local dst = table.clone(id_types)
 	table.remove(dst, at)
 
 	return archetype_ensure(world, dst)
@@ -964,7 +930,7 @@ end
 local function archetype_delete(world: World, archetype: Archetype, row: number, destruct: boolean?)
 	local entityIndex = world.entity_index
 	local columns = archetype.columns
-	local types = archetype.types
+	local id_types = archetype.types
 	local entities = archetype.entities
 	local column_count = #entities
 	local last = #entities
@@ -983,7 +949,7 @@ local function archetype_delete(world: World, archetype: Archetype, row: number,
 
 	-- TODO: if last == 0 then deactivate table
 
-	for _, id in types do
+	for _, id in id_types do
 		local on_remove: (entity: i53) -> () = world_get_one_inline(world, id, EcsOnRemove)
 		if on_remove then
 			on_remove(delete)
@@ -991,9 +957,9 @@ local function archetype_delete(world: World, archetype: Archetype, row: number,
 	end
 
 	if row == last then
-		archetype_fast_delete_last(columns, column_count, types, delete)
+		archetype_fast_delete_last(columns, column_count, id_types, delete)
 	else
-		archetype_fast_delete(columns, column_count, row, types, delete)
+		archetype_fast_delete(columns, column_count, row, id_types, delete)
 	end
 end
 
@@ -1078,6 +1044,18 @@ local function archetype_destroy(world: World, archetype: Archetype)
 	world.archetypeIndex[archetype.type] = nil :: any
 	local records = archetype.records
 
+	for id in records do
+		local observer_list = find_observers(world, EcsArchetypeDelete, id)
+		if not observer_list then
+			continue
+		end
+		for _, observer in observer_list do
+			if query_match(observer.terms, archetype) then
+				observer.callback(archetype)
+			end
+		end
+	end
+
 	for id in records do
 		local idr = component_index[id]
 		idr.cache[archetype_id] = nil :: any
@@ -1136,23 +1114,30 @@ do
 		local idr = component_index[delete]
 
 		if idr then
-			local children = {}
-			for archetype_id in idr.cache do
-				local idr_archetype = archetypes[archetype_id]
-
-				for i, child in idr_archetype.entities do
-					table.insert(children, child)
-				end
-			end
 			local flags = idr.flags
 			if bit32.band(flags, ECS_ID_DELETE) ~= 0 then
-				for _, child in children do
-					-- Cascade deletion to children
-					world_delete(world, child)
+				for archetype_id in idr.cache do
+					local idr_archetype = archetypes[archetype_id]
+
+					local entities = idr_archetype.entities
+					local n = #entities
+					for i = n, 1, -1 do
+						world_delete(world, entities[i])
+					end
 				end
 			else
-				for _, child in children do
-					world_remove(world, child, delete)
+				for archetype_id in idr.cache do
+					local idr_archetype = archetypes[archetype_id]
+					local entities = idr_archetype.entities
+					local n = #entities
+					for i = n, 1, -1 do
+						world_remove(world, entities[i], delete)
+					end
+				end
+
+				for archetype_id in idr.cache do
+					local idr_archetype = archetypes[archetype_id]
+					archetype_destroy(world, idr_archetype)
 				end
 			end
 		end
@@ -1486,12 +1471,12 @@ local function query_iter(query): () -> (number, ...any)
 	return query_next
 end
 
-local function query_without(query: { compatible_archetypes: { Archetype } }, ...)
-	local filters = query.filters
+local function query_without(query: QueryInner, ...: i53)
+	local filters: { without: { i53 } } = query.filters :: any
 	local without = { ... }
 	if not filters then
 		filters = {}
-		query.filters = filters
+		query.filters = filters :: any
 	end
 	filters.without = without
 	local compatible_archetypes = query.compatible_archetypes
@@ -1521,13 +1506,13 @@ local function query_without(query: { compatible_archetypes: { Archetype } }, ..
 	return query :: any
 end
 
-local function query_with(query: { compatible_archetypes: { Archetype } }, ...)
+local function query_with(query: QueryInner, ...)
 	local compatible_archetypes = query.compatible_archetypes
-	local filters = query.filters
+	local filters: { with: { i53 } } = query.filters :: any
 	local with = { ... }
 	if not filters then
 		filters = {}
-		query.filters = filters
+		query.filters = filters :: any
 	end
 	filters.with = with
 	for i = #compatible_archetypes, 1, -1 do
@@ -1563,9 +1548,73 @@ local function query_archetypes(query)
 	return query.compatible_archetypes
 end
 
-local function query_cached(query)
-	local observer = create_observer_uni(query.world, query.ids[1], EcsTableCreate)
-	observer.query = query
+local function query_cached(query: QueryInner)
+	local archetypes = query.compatible_archetypes
+	local world = query.world :: World
+	-- Only need one observer for EcsArchetypeCreate and EcsArchetypeDelete respectively
+	-- because the event will be emitted for all components of that Archetype.
+	local first = query.ids[1]
+	local observerable = world.observerable
+	local on_create_action = observerable[EcsArchetypeCreate]
+	if not on_create_action then
+		on_create_action = {}
+		observerable[EcsArchetypeCreate] = on_create_action
+	end
+	local query_cache_on_create = on_create_action[first]
+	if not query_cache_on_create then
+		query_cache_on_create = {}
+		on_create_action[first] = query_cache_on_create
+	end
+
+	local on_delete_action = observerable[EcsArchetypeDelete]
+	if not on_delete_action then
+		on_delete_action = {}
+		observerable[EcsArchetypeDelete] = on_delete_action
+	end
+	local query_cache_on_delete = on_delete_action[first]
+	if not query_cache_on_delete then
+		query_cache_on_delete = {}
+		on_delete_action[first] = query_cache_on_delete
+	end
+
+	local function on_create_callback(archetype)
+		table.insert(archetypes, archetype)
+	end
+
+	local function on_delete_callback(archetype)
+		local i = table.find(archetypes, archetype)  :: number
+		local n = #archetypes
+		archetypes[i] = archetypes[n]
+		archetypes[n] = nil
+	end
+
+	local terms = {}
+
+	for _, id in query.ids do
+		table.insert(terms, { id })
+	end
+	local filters = query.filters
+	if filters then
+		local with = filters.with
+		if with then
+			for _, id in with do
+				table.insert(terms, { id })
+			end
+		end
+		local without = filters.without
+		if without then
+			for _, id in without do
+				table.insert(terms, { id, true })
+			end
+		end
+	end
+
+	local observer_for_create = { query = query, callback = on_create_callback, terms = terms }
+	local observer_for_delete = { query = query, callback = on_delete_callback, terms = terms }
+
+	table.insert(query_cache_on_create, observer_for_create)
+	table.insert(query_cache_on_delete, observer_for_delete)
+
 	return query
 end
 
@@ -1820,7 +1869,7 @@ function World.new()
 		nextComponentId = 0 :: number,
 		nextEntityId = 0 :: number,
 		ROOT_ARCHETYPE = (nil :: any) :: Archetype,
-		observerable = {}
+		observerable = {},
 	}, World) :: any
 
 	self.ROOT_ARCHETYPE = archetype_create(self, {}, "")
@@ -1860,23 +1909,21 @@ function World.new()
 	return self
 end
 
-export type Id<T = unknown> = Entity<T> | Pair<Entity<T>, Entity<unknown>>
+export type Id<T = unknown> = Entity<T>
 
-export type Pair<First, Second> = number & {
-	__relation: First,
-}
+type function ecs_entity_t(entity)
+	return entity:components()[2]:readproperty(types.singleton("__T"))
+end
 
--- type function _Pair(first, second)
---     local thing = first:components()[2]
+type function Pair(first, second)
+    local thing = first:components()[2]
 
--- 	if thing:readproperty(types.singleton("__T")):is("nil") then
--- 		return second
--- 	else
--- 		return first
--- 	end
--- end
-
--- type TestPair = _Pair<Entity<number>, Entity<Vector3>>
+	if thing:readproperty(types.singleton("__T")):is("nil") then
+		return second
+	else
+		return first
+	end
+end
 
 type Item<T...> = (self: Query<T...>) -> (Entity, T...)
 
@@ -1891,9 +1938,32 @@ export type Query<T...> = typeof(setmetatable({}, {
 	with: (self: Query<T...>, ...Id) -> Query<T...>,
 	without: (self: Query<T...>, ...Id) -> Query<T...>,
 	archetypes: (self: Query<T...>) -> { Archetype },
-	cached: (self: Query<T...>) -> Query<T...>
+	cached: (self: Query<T...>) -> Query<T...>,
 }
 
+type QueryInner = {
+	compatible_archetypes: { Archetype },
+	filters: {
+		without: { i53 }?,
+		with: { i53 }?,
+	}?,
+	ids: { i53 },
+	world: {} -- Downcasted to be serializable by the analyzer
+}
+
+type Observer = {
+ 	callback: (archetype: Archetype) -> (),
+   	query: QueryInner,
+}
+
+type function ecs_partial_t(ty)
+	local output = types.newtable()
+	for k, v in ty:properties() do
+		output:setproperty(k, types.unionof(v.write, types.singleton(nil)))
+	end
+	return output
+end
+
 export type World = {
 	archetypeIndex: { [string]: Archetype },
 	archetypes: Archetypes,
@@ -1905,7 +1975,7 @@ export type World = {
 	nextEntityId: number,
 	nextArchetypeId: number,
 
-	observerable: { [string]: { [Id]: { query: Query<i53> } } }
+	observerable: { [i53]: { [i53]: { { query: Query<i53> } } } },
 } & {
 	--- Creates a new entity
 	entity: (self: World) -> Entity,
@@ -1948,7 +2018,14 @@ export type World = {
 	children: (self: World, id: Id) -> () -> Entity,
 
 	--- Searches the world for entities that match a given query
-	query: (<A>(self: World, a: { __T: A }) -> Query<A>)
+	query: (<A>(World, A) -> Query<ecs_entity_t<A>>)
+        & (<A, B>(World, A, B) -> Query<ecs_entity_t<A>, ecs_entity_t<B>>)
+        & (<A, B, C>(World, A, B, C) -> Query<ecs_entity_t<A>, ecs_entity_t<B>, ecs_entity_t<C>>)
+        & (<A, B, C, D>(World, A, B, C, D) -> Query<ecs_entity_t<A>, ecs_entity_t<B>, ecs_entity_t<C>, ecs_entity_t<D>>)
+        & (<A, B, C, D, E>(World, A, B, C, D, E) -> Query<ecs_entity_t<A>, ecs_entity_t<B>, ecs_entity_t<C>, ecs_entity_t<D>, ecs_entity_t<E>>)
+        & (<A, B, C, D, E, F>(World, A, B, C, D, E, F) -> Query<ecs_entity_t<A>, ecs_entity_t<B>, ecs_entity_t<C>, ecs_entity_t<D>, ecs_entity_t<E>, ecs_entity_t<F>>)
+        & (<A, B, C, D, E, F, G>(World, A, B, C, D, E, F, G) -> Query<ecs_entity_t<A>, ecs_entity_t<B>, ecs_entity_t<C>, ecs_entity_t<D>, ecs_entity_t<E>, ecs_entity_t<F>, ecs_entity_t<G>>)
+        & (<A, B, C, D, E, F, G, H>(World, A, B, C, D, E, F, G, H) -> Query<ecs_entity_t<A>, ecs_entity_t<B>, ecs_entity_t<C>, ecs_entity_t<D>, ecs_entity_t<E>, ecs_entity_t<F>, ecs_entity_t<G>, ecs_entity_t<H>>)
 }
 
 return {

test/tests.luau

@@ -169,8 +169,7 @@ TEST("world:entity()", function()
 		CHECK(ECS_GENERATION(e) == 1) -- 1
 	end
 
-	do
-		CASE("pairs")
+	do CASE("pairs")
 		local world = jecs.World.new()
 		local _e = world:entity()
 		local e2 = world:entity()
@@ -179,11 +178,17 @@ TEST("world:entity()", function()
 		-- Incomplete pair, must have a bit flag that notes it is a pair
 		CHECK(IS_PAIR(world:entity()) == false)
 
-		local pair = pair(e2, e3)
-		CHECK(IS_PAIR(pair) == true)
+		local p = pair(e2, e3)
+		CHECK(IS_PAIR(p) == true)
 
-		CHECK(ecs_pair_first(world, pair) == e2)
-		CHECK(ecs_pair_second(world, pair) == e3)
+		CHECK(ecs_pair_first(world, p) == e2)
+		CHECK(ecs_pair_second(world, p) == e3)
+
+		world:delete(e2)
+		local e2v2 = world:entity()
+		CHECK(IS_PAIR(e2v2) == false)
+
+		CHECK(IS_PAIR(pair(e2v2, e3)) == true)
 	end
 
 	do CASE "Recycling"
@@ -285,7 +290,7 @@ TEST("world:set()", function()
 
 		CHECK(world:get(e, pair(C1, C2)))
 		CHECK(world:get(e, pair(C1, T1)))
-		CHECK(not world:get(e, pair(T1, C1)))
+		CHECK(world:get(e, pair(T1, C1)))
 		CHECK(not world:get(e, pair(T1, T2)))
 
 		local e2 = world:entity()
@@ -372,6 +377,8 @@ TEST("world:query()", function()
 		end
 		CHECK(#q:archetypes() == 1)
 		CHECK(not table.find(q:archetypes(), world.archetypes[table.concat({Foo, Bar, Baz}, "_")]))
+		world:delete(Foo)
+		CHECK(#q:archetypes() == 0)
 	end
 	do CASE("multiple iter")
 		local world = jecs.World.new()
@@ -419,7 +426,7 @@ TEST("world:query()", function()
 		for id, a, b, c, d in world:query(pair(C1, C2), pair(C1, T1), pair(T1, C1), pair(T1, T2)) do
 			CHECK(a == true)
 			CHECK(b == true)
-			CHECK(c == nil)
+			CHECK(c == true)
 			CHECK(d == nil)
 		end
 	end