Improve relationship performance

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## Unreleased
 
+### Added
+- Added signals that allow listening to relation part of pairs in signals.
+
+### Changed
+- `OnRemove` hooks so that they are allowed to move entity's archetype even during deletion.
+
+## 0.8.0
+
 ### Added
 - `jecs.Exclusive` trait for making exclusive relationships.
 

jecs.luau

@@ -54,8 +54,6 @@ export type Query<T...> = typeof(setmetatable(
 		archetypes: (self: Query<T...>) -> { Archetype },
 		cached: (self: Query<T...>) -> Query<T...>,
 		ids: { Id<any> },
-		patch: (self: Query<T...>, fn: (T...) -> (T...)) -> (),
-		view: (self: Query<T...>) -> View<T...>,
 		-- world: World
 	},
 	{} :: {
@@ -70,6 +68,14 @@ export type Observer = {
 	query: QueryInner,
 }
 
+type query = {
+	compatible_archetypes: { archetype },
+	ids: { i53 },
+	filter_with: { i53 },
+	filter_without: { i53 },
+	next: () -> (i53, ...any),
+	world: World,
+}
 
 export type observer = {
 	callback: (archetype: archetype) -> (),
@@ -86,7 +92,7 @@ type archetype = {
 }
 
 type componentrecord = {
-	records: { [i53]: number },
+	records: { [number]: number },
 	counts: { [i53]: number },
 	flags: number,
 	size: number,
@@ -94,6 +100,8 @@ type componentrecord = {
 	on_add: ((entity: i53, id: i53, value: any?) -> ())?,
 	on_change: ((entity: i53, id: i53, value: any) -> ())?,
 	on_remove: ((entity: i53, id: i53) -> ())?,
+
+	wildcard_pairs: { [number]: componentrecord },
 }
 type record = {
 	archetype: archetype,
@@ -889,17 +897,32 @@ local function archetype_create(world: world, id_types: { i53 }, ty, prev: i53?)
 		if ECS_IS_PAIR(component_id) then
 			local relation = ECS_PAIR_FIRST(component_id)
 			local object = ECS_PAIR_SECOND(component_id)
+
 			local r = ECS_PAIR(relation, EcsWildcard)
 			local idr_r = id_record_ensure(world, r)
-			idr_r.size += 1
 
+			idr_r.size += 1
 			archetype_append_to_records(idr_r, archetype_id, columns_map, r, i, column)
+			local idr_r_wc_pairs = idr_r.wildcard_pairs
+			if not idr_r_wc_pairs then
+				idr_r_wc_pairs = {} :: {[i53]: componentrecord }
+				idr_r.wildcard_pairs = idr_r_wc_pairs
+			end
+			idr_r_wc_pairs[component_id] = idr
 
 			local t = ECS_PAIR(EcsWildcard, object)
 			local idr_t = id_record_ensure(world, t)
-			idr_t.size += 1
 
+			idr_t.size += 1
 			archetype_append_to_records(idr_t, archetype_id, columns_map, t, i, column)
+
+			-- Hypothetically this should only capture leaf component records
+			local idr_t_wc_pairs = idr_t.wildcard_pairs
+			if not idr_t_wc_pairs then
+				idr_t_wc_pairs = {} :: {[i53]: componentrecord }
+				idr_t.wildcard_pairs = idr_t_wc_pairs
+			end
+			idr_t_wc_pairs[component_id] = idr
 		end
 	end
 
@@ -1086,15 +1109,12 @@ end
 
 local function archetype_delete(world: world, archetype: archetype, row: number)
 	local entity_index = world.entity_index
-	local component_index = world.component_index
 	local columns = archetype.columns
-	local id_types = archetype.types
 	local entities = archetype.entities
 	local column_count = #entities
 	local last = #entities
 	local move = entities[last]
 	-- We assume first that the entity is the last in the archetype
-	local delete = move
 
 	if row ~= last then
 		local record_to_move = entity_index_try_get_any(entity_index, move)
@@ -1102,18 +1122,9 @@ local function archetype_delete(world: world, archetype: archetype, row: number)
 			record_to_move.row = row
 		end
 
-		delete = entities[row]
 		entities[row] = move
 	end
 
-	for _, id in id_types do
-		local idr = component_index[id]
-		local on_remove = idr.on_remove
-		if on_remove then
-			on_remove(delete, id)
-		end
-	end
-
 	entities[last] = nil :: any
 
 	if row == last then
@@ -2365,6 +2376,22 @@ local function world_new()
 		return r
 	end
 
+	local function exclusive_traverse_add(
+		archetype: archetype,
+		cr: number,
+		id: i53
+	)
+		local edge = archetype_edges[archetype.id]
+		local to = edge[id]
+		if not to then
+			local dst = table.clone(archetype.types)
+			dst[cr] = id
+			to = archetype_ensure(world, dst)
+			edge[id] = to
+		end
+		return to
+	end
+
 	local function inner_world_set(world: world, entity: i53, id: i53, data): ()
 		local record = inner_entity_index_try_get_unsafe(entity)
 		if not record then
@@ -2395,22 +2422,19 @@ local function world_new()
 				local edge = archetype_edges[src.id]
 				to = edge[id]
 				if to == nil then
-					if idr and (bit32.btest(idr.flags) == true) then
+					if idr and (bit32.btest(idr.flags, ECS_ID_IS_EXCLUSIVE) == true) then
 						local cr = idr.records[src.id]
 						if cr then
 							local on_remove = idr.on_remove
 							local id_types = src.types
 							if on_remove then
 								on_remove(entity, id_types[cr])
-
 								src = record.archetype
 								id_types = src.types
 								cr = idr.records[src.id]
 							end
 
-							local dst = table.clone(id_types)
+							to = exclusive_traverse_add(src, cr, id)
-							dst[cr] = id
-							to = archetype_ensure(world, dst)
 						end
 					end
 
@@ -2425,30 +2449,27 @@ local function world_new()
 					archetype_edges[(to :: Archetype).id][id] = src
 				else
 					if bit32.btest(idr.flags, ECS_ID_IS_EXCLUSIVE) then
+						local on_remove = idr.on_remove
+						if on_remove then
 							local cr = idr.records[src.id]
 							if cr then
-							local on_remove = idr.on_remove
 								local id_types = src.types
-							if on_remove then
 								on_remove(entity, id_types[cr])
-								src = record.archetype
+								local arche = record.archetype
-								id_types = src.types
+								if src ~= arche then
-								cr = idr.records[src.id]
+									id_types = arche.types
-							end
+									cr = idr.records[arche.id]
-							local dst = table.clone(id_types)
+									to = exclusive_traverse_add(arche, cr, id)
-							dst[cr] = id
+								end
-							to = archetype_ensure(world, dst)
 							end
 						end
-					if not to then
-						to = find_archetype_with(world, id, src)
 					end
 				end
 			else
 				local edges = archetype_edges
 				local edge = edges[src.id]
 
-				to = edge[id] :: archetype
+				to = edge[id]
 				if not to then
 					to = find_archetype_with(world, id, src)
 					edge[id] = to
@@ -2501,7 +2522,7 @@ local function world_new()
 			local edge = archetype_edges[src.id]
 			to = edge[id]
 			if to == nil then
-				if idr and (bit32.btest(idr.flags) == true) then
+				if idr and (bit32.btest(idr.flags, ECS_ID_IS_EXCLUSIVE) == true) then
 					local cr = idr.records[src.id]
 					if cr then
 						local on_remove = idr.on_remove
@@ -2514,9 +2535,7 @@ local function world_new()
 							cr = idr.records[src.id]
 						end
 
-						local dst = table.clone(id_types)
+						to = exclusive_traverse_add(src, cr, id)
-						dst[cr] = id
-						to = archetype_ensure(world, dst)
 					end
 				end
 
@@ -2531,23 +2550,20 @@ local function world_new()
 				archetype_edges[(to :: Archetype).id][id] = src
 			else
 				if bit32.btest(idr.flags, ECS_ID_IS_EXCLUSIVE) then
+					local on_remove = idr.on_remove
+					if on_remove then
 						local cr = idr.records[src.id]
 						if cr then
-						local on_remove = idr.on_remove
 							local id_types = src.types
-						if on_remove then
 							on_remove(entity, id_types[cr])
-							src = record.archetype
+							local arche = record.archetype
-							id_types = src.types
+							if src ~= arche then
-							cr = idr.records[src.id]
+								id_types = arche.types
-						end
+								cr = idr.records[arche.id]
-						local dst = table.clone(id_types)
+								to = exclusive_traverse_add(arche, cr, id)
-						dst[cr] = id
+							end
-						to = archetype_ensure(world, dst)
 						end
 					end
-				if not to then
-					to = find_archetype_with(world, id, src)
 				end
 			end
 		else
@@ -2626,13 +2642,21 @@ local function world_new()
 				table.insert(listeners, existing_hook)
 			end
 
-			local idr = component_index[ECS_PAIR(component, EcsWildcard)] or component_index[component]
+			local idr_pair = component_index[ECS_PAIR(component, EcsWildcard)]
+
+			if idr_pair then
+				for id, cr in idr_pair.wildcard_pairs do
+					cr.on_add = on_add
+				end
+				idr_pair.on_add = on_add
+			else
+				local idr = component_index[component]
 				if idr then
 					idr.on_add = on_add
-			else
-				inner_world_set(world, component, EcsOnAdd, on_add)
 				end
 			end
+			inner_world_set(world, component, EcsOnAdd, on_add)
+		end
 		table.insert(listeners, fn)
 		return function()
 			local n = #listeners
@@ -2661,13 +2685,23 @@ local function world_new()
 				table.insert(listeners, existing_hook)
 			end
 
-			local idr = component_index[ECS_PAIR(component, EcsWildcard)] or component_index[component]
+			local idr_pair = component_index[ECS_PAIR(component, EcsWildcard)]
+
+			if idr_pair then
+				for _, cr in idr_pair.wildcard_pairs do
+					cr.on_change = on_change
+				end
+
+				idr_pair.on_change = on_change
+			else
+				local idr = component_index[component]
 				if idr then
 					idr.on_change = on_change
-			else
-				inner_world_set(world, component, EcsOnChange, on_change)
 				end
 			end
+
+			inner_world_set(world, component, EcsOnChange, on_change)
+		end
 		table.insert(listeners, fn)
 		return function()
 			local n = #listeners
@@ -2687,19 +2721,30 @@ local function world_new()
 					listener(entity, id)
 				end
 			end
+
 			local existing_hook = inner_world_get(world, component, EcsOnRemove) :: Listener<T>
 			if existing_hook then
 				table.insert(listeners, existing_hook)
 			end
 
-			local idr = component_index[ECS_PAIR(component, EcsWildcard)] or component_index[component]
+			local idr_pair = component_index[ECS_PAIR(component, EcsWildcard)]
+
+			if idr_pair then
+				for _, cr in idr_pair.wildcard_pairs do
+					cr.on_remove = on_remove
+				end
+
+				idr_pair.on_remove = on_remove
+			else
+				local idr = component_index[component]
 				if idr then
 					idr.on_remove = on_remove
-			else
-				inner_world_set(world, component, EcsOnRemove, on_remove)
 				end
 			end
 
+			inner_world_set(world, component, EcsOnRemove, on_remove)
+		end
+
 		table.insert(listeners, fn)
 
 		return function()
@@ -2860,6 +2905,7 @@ local function world_new()
 		if from.columns_map[id] then
 			local idr = world.component_index[id]
 			local on_remove = idr.on_remove
+
 			if on_remove then
 				on_remove(entity, id)
 			end
@@ -2964,12 +3010,16 @@ local function world_new()
 		end
 
 		local archetype = record.archetype
-		local row = record.row
 
 		if archetype then
-			-- In the future should have a destruct mode for
+			for _, id in archetype.types do
-			-- deleting archetypes themselves. Maybe requires recycling
+				local idr = component_index[id]
-			archetype_delete(world, archetype, row)
+				local on_remove = idr.on_remove
+				if on_remove then
+					on_remove(entity, id)
+				end
+			end
+			archetype_delete(world, record.archetype, record.row)
 		end
 
 		local component_index = world.component_index
@@ -3035,57 +3085,56 @@ local function world_new()
 			end
 		end
 		if idr_t then
-			local archetype_ids = idr_t.records
+			for id, cr in idr_t.wildcard_pairs do
-			for archetype_id in archetype_ids do
+				local flags = cr.flags
-				local idr_t_archetype = archetypes[archetype_id]
-				local node = idr_t_archetype
-				local idr_t_types = idr_t_archetype.types
-				local entities = idr_t_archetype.entities
-
-				local deleted = false
-				for _, id in idr_t_types do
-					if not ECS_IS_PAIR(id) then
-						continue
-					end
-					local object = entity_index_get_alive(
-					    entity_index, ECS_PAIR_SECOND(id))
-					if object ~= entity then
-						continue
-					end
-					local id_record = component_index[id]
-					local flags = id_record.flags
 				local flags_delete_mask = bit32.btest(flags, ECS_ID_DELETE)
+				local on_remove = cr.on_remove
 				if flags_delete_mask then
+					for archetype_id in cr.records do
+						local idr_t_archetype = archetypes[archetype_id]
+						local entities = idr_t_archetype.entities
 						for i = #entities, 1, -1 do
 							local child = entities[i]
 							inner_world_delete(world, child)
 						end
-						deleted = true
 						break
+					end
 				else
-						node = archetype_traverse_remove(world, id, node)
+					for archetype_id in cr.records do
-						local on_remove = component_index[id].on_remove
+						local idr_t_archetype = archetypes[archetype_id]
-						if on_remove then
+						local entities = idr_t_archetype.entities
-							for _, entity in entities do
+						-- archetype_traverse_remove is not idempotent meaning
-								on_remove(entity, id)
+						-- this access is actually unsafe because it can
-							end
+						-- incorrectly cache an edge despite a node of the
-						end
+						-- component id on the archetype does not exist. This
-					end
+						-- requires careful testing to ensure correct values are
-				end
+						-- being passed to the arguments.
+						local to = archetype_traverse_remove(world, id, idr_t_archetype)
 
-				if not deleted then
 						for i = #entities, 1, -1 do
 							local e = entities[i]
-						local r = inner_entity_index_try_get_unsafe(e) :: record
+							local r = eindex_sparse_array[ECS_ID(e :: number)]
-						inner_entity_move(entity_index, e, r, node)
+							if on_remove then
+								on_remove(e, id)
+
+								local from = r.archetype
+								if from ~= idr_t_archetype then
+									-- unfortunately the on_remove hook allows a window where `e` can have changed archetype
+									-- this is hypothetically not that expensive of an operation anyways
+									to = archetype_traverse_remove(world, id, from)
+								end
+							end
+
+							inner_entity_move(entity_index, e, r, to)
 						end
 					end
 				end
 
-			for archetype_id in archetype_ids do
+				for archetype_id in cr.records do
 					archetype_destroy(world, archetypes[archetype_id])
 				end
 			end
+		end
 
 		if idr_r then
 			local archetype_ids = idr_r.records
@@ -3286,6 +3335,9 @@ end
 
 return {
 	world = world_new :: () -> World,
+	World = {
+		new = world_new
+	},
 	component = (ECS_COMPONENT :: any) :: <T>() -> Entity<T>,
 	tag = (ECS_TAG :: any) :: <T>() -> Entity<T>,
 	meta = (ECS_META :: any) :: <T, a>(id: Entity<T>, id: Id<a>, value: a?) -> Entity<T>,

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@rbxts/jecs",
-	"version": "0.9.0-rc.8",
+	"version": "0.9.0-rc.9",
 	"description": "Stupidly fast Entity Component System",
 	"main": "jecs.luau",
 	"repository": {

test/tests.luau

@@ -333,6 +333,15 @@ TEST("world:add()", function()
 
 		CHECK(world:has(e, pair(A, B)) == false)
 		CHECK(world:has(e, pair(A, C)) == true)
+
+		-- We have to test the path that checks the uncached method
+		local e1 = world:entity()
+
+		world:add(e1, pair(A, B))
+		world:add(e1, pair(A, C))
+
+		CHECK(world:has(e1, pair(A, B)) == false)
+		CHECK(world:has(e1, pair(A, C)) == true)
 	end
 
 	do CASE "exclusive relations invoke hooks"
@@ -379,6 +388,44 @@ TEST("world:add()", function()
 		CHECK(world:has(e, pair(A, C)) == true)
 	end
 
+	do CASE "exclusive relations invoke on_remove hooks that should allow side effects"
+		local world = jecs.world()
+		local A = world:component()
+		local B = world:component()
+		local C = world:component()
+		local D = world:component()
+
+		world:add(A, jecs.Exclusive)
+		local call_count = 0
+		world:set(A, jecs.OnRemove, function(e, id)
+			call_count += 1
+			if call_count == 1 then
+				world:add(e, C)
+			else
+				world:add(e, D)
+			end
+		end)
+
+		local e = world:entity()
+		world:add(e, pair(A, B))
+		world:add(e, pair(A, C))
+
+		CHECK(world:has(e, pair(A, B)) == false)
+		CHECK(world:has(e, pair(A, C)) == true)
+		CHECK(world:has(e, C))
+
+
+		-- We have to ensure that it actually invokes hooks everytime it
+		-- traverses the archetype
+		e = world:entity()
+		world:add(e, pair(A, B))
+		world:add(e, pair(A, C))
+
+		CHECK(world:has(e, pair(A, B)) == false)
+		CHECK(world:has(e, pair(A, C)) == true)
+		CHECK(world:has(e, D))
+	end
+
 	do CASE "idempotent"
 		local world = jecs.world()
 		local d = dwi(world)
@@ -621,6 +668,57 @@ TEST("world:contains()", function()
 end)
 
 TEST("world:delete()", function()
+	do CASE "idr_t//delete_mask@3102..3108"
+		local world = jecs.world()
+		local A = world:component()
+		world:add(A, pair(jecs.OnDeleteTarget, jecs.Delete))
+		local B = world:component()
+		local B_OnAdd_called = false
+		local B_OnRemove_called = false
+		world:set(B, jecs.OnAdd, function()
+			B_OnAdd_called = true
+		end)
+		world:set(B, jecs.OnRemove, function()
+			B_OnRemove_called = true
+		end)
+
+		world:set(A, jecs.OnRemove, function(entity, id)
+			world:set(entity, B, true)
+		end)
+
+		local e1 = world:entity()
+		local e2 = world:entity()
+		world:set(e2, pair(A, e1), true)
+
+		world:delete(e1)
+
+		CHECK(not world:has(e2, pair(A, e1)))
+		CHECK(not world:has(e2, B))
+		CHECK(not world:contains(e1))
+		CHECK(not world:contains(e2))
+		CHECK(B_OnAdd_called)
+		-- CHECK(B_OnRemove_called)
+	end
+
+	do CASE "idr_t//remove//on_remove//changed_archetype@3123..3126"
+		local world = jecs.world()
+		local A = world:component()
+		local B = world:component()
+		local C = world:component()
+		world:set(A, jecs.OnRemove, function(entity, id)
+			world:set(entity, B, true)
+		end)
+
+		local e1 = world:entity()
+		local e2 = world:entity()
+		world:add(e2, pair(A, e2))
+		world:set(e2, pair(A, e1), true)
+
+		world:delete(e1)
+
+		CHECK(not world:has(e2, pair(A, e1)))
+	end
+
 	do CASE "pair(OnDelete, Delete)"
 	    local world = jecs.world()
 	    local ct = world:component()
@@ -1076,19 +1174,45 @@ TEST("world:added", function()
 		CHECK(ran)
 	end
 
+	do CASE ""
+		local world = jecs.world()
+		local IsNearby = world:component()
+		world:set(IsNearby, jecs.Name, "IsNearby")
+		local person1, person2 = world:entity(), world:entity()
+
+		world:add(person2, jecs.pair(IsNearby, person1))
+		local IsNearby_added_called = false
+		world:added(IsNearby, function(...) -- This prints fine
+			IsNearby_added_called = true
+		end)
+		local IsNearby_removed_called = false
+		world:removed(IsNearby, function(...)
+			IsNearby_removed_called = true
+		end)
+
+		world:remove(person2, pair(IsNearby, person1))
+		world:add(person2, pair(IsNearby, person1))
+		world:remove(person2, pair(IsNearby, person1))
+
+		CHECK(IsNearby_added_called)
+		CHECK(IsNearby_removed_called)
+	end
+
+
 
 	do CASE "Should work even if set after the pair has been used"
 		local A = world:component()
 		local B = world:component()
 
-		world:set(world:entity(), A, 2)
 		world:set(world:entity(), pair(A, B), 2)
 
+		local ran = false
 		world:added(A, function()
 			ran = true
 		end)
 
 		local entity = world:entity()
+		print(pair(A, B))
 		world:set(entity, pair(A, B), 3)
 		CHECK(ran)
 	end
@@ -1099,6 +1223,7 @@ TEST("world:added", function()
 
 		world:add(world:entity(), A)
 
+		local ran = false
 		world:added(A, function()
 			ran = true
 		end)
@@ -1113,6 +1238,7 @@ TEST("world:added", function()
 
 		world:add(world:entity(), pair(A, B))
 
+		local ran = false
 		world:added(A, function()
 			ran = true
 		end)
@@ -2199,7 +2325,6 @@ TEST("#repro", function()
 	local types1 = { pair(Attacks, e1), pair(Eats, e1) }
 	table.sort(types1)
 
-
 	CHECK(d.tbl(e1).type == "")
 	CHECK(d.tbl(e3).type == table.concat(types1, "_"))
 

wally.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ukendio/jecs"
-version = "0.9.0-rc.8"
+version = "0.9.0-rc.9"
 registry = "https://github.com/UpliftGames/wally-index"
 realm = "shared"
 license = "MIT"