Merge 96bed9bd7e into b26fc39fce

Prevent invalidation during removal of pairs with deleted targe

.gitignore

@@ -65,3 +65,7 @@ drafts/
 
 # Luau tools
 profile.*
+
+# Patch files
+
+*.patch

.luaurc

@@ -1,8 +1,9 @@
 {
     "aliases": {
         "jecs": "jecs",
-        "testkit": "test/testkit",
-        "mirror": "mirror"
+        "testkit": "tools/testkit",
+        "mirror": "mirror",
+        "tools": "tools",
     },
     "languageMode": "strict"
 }

benches/visual/despawn.bench.luau

@@ -5,41 +5,60 @@ local ReplicatedStorage = game:GetService("ReplicatedStorage")
 local Matter = require(ReplicatedStorage.DevPackages.Matter)
 local ecr = require(ReplicatedStorage.DevPackages.ecr)
 local jecs = require(ReplicatedStorage.Lib)
+local pair = jecs.pair
 local newWorld = Matter.World.new()
 local ecs = jecs.World.new()
+local mirror = require(ReplicatedStorage.mirror)
+local mcs = mirror.World.new()
 
-local A, B = Matter.component(), Matter.component()
-local C, D = ecs:component(), ecs:component()
+local C1 = ecs:component()
+local C2 = ecs:entity()
+ecs:add(C2, pair(jecs.OnDeleteTarget, jecs.Delete))
+local C3 = ecs:entity()
+ecs:add(C3, pair(jecs.OnDeleteTarget, jecs.Delete))
+local C4 = ecs:entity()
+ecs:add(C4, pair(jecs.OnDeleteTarget, jecs.Delete))
+local E1 = mcs:component()
+local E2 = mcs:entity()
+mcs:add(E2, pair(jecs.OnDeleteTarget, jecs.Delete))
+local E3 = mcs:entity()
+mcs:add(E3, pair(jecs.OnDeleteTarget, jecs.Delete))
+local E4 = mcs:entity()
+mcs:add(E4, pair(jecs.OnDeleteTarget, jecs.Delete))
+
+local registry2 = ecr.registry()
 
 return {
 	ParameterGenerator = function()
-		local matter_entities = {}
-		local jecs_entities = {}
-		local entities = {
-			matter = matter_entities,
-			jecs = jecs_entities,
-		}
+		local j = ecs:entity()
+		ecs:set(j, C1, true)
+		local m = mcs:entity()
+		mcs:set(m, E1, true)
 		for i = 1, 1000 do
-			table.insert(matter_entities, newWorld:spawn(A(), B()))
-			local e = ecs:entity()
-			ecs:set(e, C, {})
-			ecs:set(e, D, {})
-			table.insert(jecs_entities, e)
+			local friend1 = ecs:entity()
+			local friend2 = mcs:entity()
+
+			ecs:add(friend1, pair(C2, j))
+			ecs:add(friend1, pair(C3, j))
+			ecs:add(friend1, pair(C4, j))
+
+			mcs:add(friend2, pair(E2, m))
+			mcs:add(friend2, pair(E3, m))
+			mcs:add(friend2, pair(E4, m))
 		end
-		return entities
+		return {
+			m = m,
+			j = j,
+		}
 	end,
 
 	Functions = {
-		Matter = function(_, entities)
-			for _, entity in entities.matter do
-				newWorld:despawn(entity)
-			end
+		Mirror = function(_, a)
+			mcs:delete(a.m)
 		end,
 
-		Jecs = function(_, entities)
-			for _, entity in entities.jecs do
-				ecs:delete(entity)
-			end
+		Jecs = function(_, a)
+			ecs:delete(a.j)
 		end,
 	},
 }

demo/src/ReplicatedStorage/track.luau

@@ -0,0 +1,48 @@
+local events = {}
+
+local function trackers_invoke(event, component, entity, ...)
+	local trackers = events[event][component]
+	if not trackers then
+		return
+	end
+
+	for _, tracker in trackers do
+		tracker(entity, data)
+	end
+end
+
+local function trackers_init(event, component, fn)
+	local ob = events[event]
+
+	return {
+		connect = function(component, fn)
+			local trackers = ob[component]
+			if not trackers then
+				trackers = {}
+				ob[component] = trackers
+			end
+
+			table.insert(trackers, fn)
+		end,
+		invoke = function(component, ...)
+			trackers_invoke(event, component, ...)
+		end
+	}
+	return function(component, fn)
+		local trackers = ob[component]
+		if not trackers then
+			trackers = {}
+			ob[component] = trackers
+		end
+
+		table.insert(trackers, fn)
+	end
+end
+
+local trackers = {
+	emplace = trackers_init("emplace"),
+	add = trackers_init("added"),
+	remove = trackers_init("removed")
+}
+
+return trackers

jecs.luau

@@ -46,7 +46,7 @@ export type Record = {
 }
 
 type IdRecord = {
-	columns: { number },
+	cache: { number },
 	counts: { number },
 	flags: number,
 	size: number,
@@ -480,7 +480,7 @@ local function world_target(world: World, entity: i53, relation: i24, index: num
 		nth = nth + count + 1
 	end
 
-	local tr = idr.columns[archetype_id]
+	local tr = idr.cache[archetype_id]
 
 	nth = archetype.types[nth + tr]
 
@@ -537,7 +537,7 @@ local function id_record_ensure(world: World, id: number): IdRecord
 
 		idr = {
 			size = 0,
-			columns = {},
+			cache = {},
 			counts = {},
 			flags = flags,
 			hooks = {
@@ -562,7 +562,7 @@ local function archetype_append_to_records(
 	local archetype_id = archetype.id
 	local archetype_records = archetype.records
 	local archetype_counts = archetype.counts
-	local idr_columns = idr.columns
+	local idr_columns = idr.cache
 	local idr_counts = idr.counts
 	local tr = idr_columns[archetype_id]
 	if not tr then
@@ -1063,7 +1063,7 @@ local function archetype_destroy(world: World, archetype: Archetype)
 
 	for id in records do
 		local idr = component_index[id]
-		idr.columns[archetype_id] = nil :: any
+		idr.cache[archetype_id] = nil :: any
 		idr.counts[archetype_id] = nil
 		idr.size -= 1
 		records[id] = nil :: any
@@ -1122,7 +1122,7 @@ do
 		if idr then
 			local flags = idr.flags
 			if bit32.band(flags, ECS_ID_DELETE) ~= 0 then
-				for archetype_id in idr.columns do
+				for archetype_id in idr.cache do
 					local idr_archetype = archetypes[archetype_id]
 
 					local entities = idr_archetype.entities
@@ -1134,7 +1134,7 @@ do
 					archetype_destroy(world, idr_archetype)
 				end
 			else
-				for archetype_id in idr.columns do
+				for archetype_id in idr.cache do
 					local idr_archetype = archetypes[archetype_id]
 					local entities = idr_archetype.entities
 					local n = #entities
@@ -1147,55 +1147,66 @@ do
 			end
 		end
 
-		local sparse_array = entity_index.sparse_array
 		local dense_array = entity_index.dense_array
 
 		if idr_t then
-			for archetype_id in idr_t.columns do
-				local children = {}
+			local children
+			local ids
+			local count = 0
+			local archetype_ids = idr_t.cache
+			for archetype_id in archetype_ids do
 				local idr_t_archetype = archetypes[archetype_id]
-
 				local idr_t_types = idr_t_archetype.types
-
-				for _, child in idr_t_archetype.entities do
-					table.insert(children, child)
-				end
-
-				local n = #children
+				local entities = idr_t_archetype.entities
+				local removal_queued = false
 
 				for _, id in idr_t_types do
 					if not ECS_IS_PAIR(id) then
 						continue
 					end
 					local object = ecs_pair_second(world, id)
-					if object == delete then
-						local id_record = component_index[id]
-						local flags = id_record.flags
-						local flags_delete_mask: number = bit32.band(flags, ECS_ID_DELETE)
-						if flags_delete_mask ~= 0 then
-							for i = n, 1, -1 do
-								world_delete(world, children[i])
-							end
-							break
-						else
-							local on_remove = id_record.hooks.on_remove
-							local to = archetype_traverse_remove(world, id, idr_t_archetype)
-							local empty = #to.types == 0
-							for i = n, 1, -1 do
-								local child = children[i]
-								if on_remove then
-									on_remove(child)
-								end
-								local r = sparse_array[ECS_ENTITY_T_LO(child)]
-								if not empty then
-									entity_move(entity_index, child, r, to)
-								end
-							end
+					if object ~= delete then
+						continue
+					end
+					local id_record = component_index[id]
+					local flags = id_record.flags
+					local flags_delete_mask: number = bit32.band(flags, ECS_ID_DELETE)
+					if flags_delete_mask ~= 0 then
+						for i = #entities, 1, -1 do
+							local child = entities[i]
+							world_delete(world, child)
 						end
+						break
+					else
+					    if not ids then
+							ids = {}
+						end
+						ids[id] = true
+						removal_queued = true
 					end
 				end
 
-				archetype_destroy(world, idr_t_archetype)
+				if not removal_queued then
+					continue
+				end
+				if not children then
+					children = {}
+				end
+				local n = #entities
+				table.move(entities, 1, n, count + 1, children)
+				count += n
+			end
+
+			if ids then
+				for id in ids do
+					for _, child in children do
+						world_remove(world, child, id)
+					end
+				end
+			end
+
+			for archetype_id in archetype_ids do
+				archetype_destroy(world, archetypes[archetype_id])
 			end
 		end
 
@@ -2096,7 +2107,7 @@ local function world_query(world: World, ...)
 		return q
 	end
 
-	for archetype_id in idr.columns do
+	for archetype_id in idr.cache do
 		local compatibleArchetype = archetypes[archetype_id]
 		if #compatibleArchetype.entities == 0 then
 			continue
@@ -2130,9 +2141,9 @@ local function world_each(world: World, id): () -> ()
 		return NOOP
 	end
 
-	local idr_columns = idr.columns
+	local idr_cache = idr.cache
 	local archetypes = world.archetypes
-	local archetype_id = next(idr_columns, nil) :: number
+	local archetype_id = next(idr_cache, nil) :: number
 	local archetype = archetypes[archetype_id]
 	if not archetype then
 		return NOOP
@@ -2144,7 +2155,7 @@ local function world_each(world: World, id): () -> ()
 	return function(): any
 		local entity = entities[row]
 		while not entity do
-			archetype_id = next(idr_columns, archetype_id) :: number
+			archetype_id = next(idr_cache, archetype_id) :: number
 			if not archetype_id then
 				return
 			end
@@ -2477,6 +2488,7 @@ export type World = {
 
 return {
 	World = World :: { new: () -> World },
+	world = World.new :: () -> World,
 
 	OnAdd = EcsOnAdd :: Entity<(entity: Entity) -> ()>,
 	OnRemove = EcsOnRemove :: Entity<(entity: Entity) -> ()>,
@@ -2500,6 +2512,8 @@ return {
 	ECS_GENERATION = ECS_GENERATION,
 	ECS_ID_IS_WILDCARD = ECS_ID_IS_WILDCARD,
 
+	ECS_ID_DELETE = ECS_ID_DELETE,
+
 	IS_PAIR = ECS_IS_PAIR,
 	pair_first = ecs_pair_first,
 	pair_second = ecs_pair_second,

test/devtools_test.luau

@@ -0,0 +1,25 @@
+local jecs = require("@jecs")
+local pair = jecs.pair
+local ChildOf = jecs.ChildOf
+local lifetime_tracker_add = require("@tools/lifetime_tracker")
+local pe = require("@tools/entity_visualiser").prettify
+local world = lifetime_tracker_add(jecs.world(), {padding_enabled=false})
+local FriendsWith = world:component()
+local _1 = world:print_snapshot()
+local e1 = world:entity()
+local e2 = world:entity()
+world:delete(e2)
+
+local _2 = world:print_snapshot()
+local e3 = world:entity()
+world:add(e3, pair(ChildOf, e1))
+local e4 = world:entity()
+world:add(e4, pair(FriendsWith, e3))
+local _3 = world:print_snapshot()
+world:delete(e1)
+world:delete(e3)
+local _4 = world:print_snapshot()
+world:print_entity_index()
+world:entity()
+world:entity()
+local _5 = world:print_snapshot()

test/tests.luau

@@ -26,6 +26,50 @@ local N = 2 ^ 8
 type World = jecs.World
 type Entity<T=nil> = jecs.Entity<T>
 
+local c = {
+	white_underline = function(s: any)
+		return `\27[1;4m{s}\27[0m`
+	end,
+
+	white = function(s: any)
+		return `\27[37;1m{s}\27[0m`
+	end,
+
+	green = function(s: any)
+		return `\27[32;1m{s}\27[0m`
+	end,
+
+	red = function(s: any)
+		return `\27[31;1m{s}\27[0m`
+	end,
+
+	yellow = function(s: any)
+		return `\27[33;1m{s}\27[0m`
+	end,
+
+	red_highlight = function(s: any)
+		return `\27[41;1;30m{s}\27[0m`
+	end,
+
+	green_highlight = function(s: any)
+		return `\27[42;1;30m{s}\27[0m`
+	end,
+
+	gray = function(s: any)
+		return `\27[30;1m{s}\27[0m`
+	end,
+}
+
+local function pe(e)
+	local gen = ECS_GENERATION(e)
+	return c.green(`e{ECS_ID(e)}`)..c.yellow(`v{gen}`)
+end
+
+local function pp(e)
+	local gen = ECS_GENERATION(e)
+	return c.green(`e{ECS_ID(e)}`)..c.yellow(`v{jecs.ECS_ENTITY_T_HI(e)}`)
+end
+
 local function debug_world_inspect(world: World)
 	local function record(e): jecs.Record
 		return entity_index_try_get_any(world.entity_index, e) :: any
@@ -67,10 +111,61 @@ local function debug_world_inspect(world: World)
 	}
 end
 
+local dwi = debug_world_inspect
+
 local function name(world, e)
 	return world:get(e, jecs.Name)
 end
 
+TEST("#repro", function()
+	local world = world_new()
+
+	local function getTargets(relation)
+		local tgts = {}
+		local pairwildcard = pair(relation, jecs.Wildcard)
+		for _, archetype in world:query(pairwildcard):archetypes() do
+		local tr = archetype.records[pairwildcard]
+		local count = archetype.counts[pairwildcard]
+		local types = archetype.types
+		for _, entity in archetype.entities do
+		    for i = 0, count - 1 do
+		    local tgt = jecs.pair_second(world, types[i + tr])
+		    table.insert(tgts, tgt)
+		    end
+		end
+		end
+		return tgts
+	end
+
+	local Attacks = world:component()
+	local Eats = world:component()
+
+	local function setAttacksAndEats(entity1, entity2)
+		world:add(entity1, pair(Attacks, entity2))
+		world:add(entity1, pair(Eats, entity2))
+	end
+
+	local e1 = world:entity()
+	local e2 = world:entity()
+	local e3 = world:entity()
+	print(e1, e2, e3)
+	setAttacksAndEats(e3, e1)
+	setAttacksAndEats(e3, e2)
+	setAttacksAndEats(e1, e2)
+	print("---------------- delete e2 ---------------")
+	local d = dwi(world)
+	for archetype_id in world.component_index[pair(jecs.Wildcard, e2)].cache do
+		local archetype = world.archetypes[archetype_id].type
+		testkit.print(archetype)
+	end
+	world:delete(e2)
+	print("-----------------------------")
+	testkit.print(d.tbl(e1).types)
+	-- testkit.print(d.tbl(e3).types)
+	-- testkit.print(getTargets(Attacks))
+	-- testkit.print(getTargets(Eats))
+end)
+
 TEST("archetype", function()
 	local archetype_traverse_add = jecs.archetype_traverse_add
 	local archetype_traverse_remove = jecs.archetype_traverse_remove
@@ -1199,6 +1294,7 @@ TEST("world:delete", function()
 			world:delete(e)
 		end)
 
+		local d = debug_world_inspect(world)
 		for i, friend in friends do
 			CHECK(not world:has(friend, pair(FriendsWith, e)))
 			CHECK(world:has(friend, Health))

tools/ansi.luau

@@ -0,0 +1,33 @@
+return {
+	white_underline = function(s: any)
+		return `\27[1;4m{s}\27[0m`
+	end,
+
+	white = function(s: any)
+		return `\27[37;1m{s}\27[0m`
+	end,
+
+	green = function(s: any)
+		return `\27[32;1m{s}\27[0m`
+	end,
+
+	red = function(s: any)
+		return `\27[31;1m{s}\27[0m`
+	end,
+
+	yellow = function(s: any)
+		return `\27[33;1m{s}\27[0m`
+	end,
+
+	red_highlight = function(s: any)
+		return `\27[41;1;30m{s}\27[0m`
+	end,
+
+	green_highlight = function(s: any)
+		return `\27[42;1;30m{s}\27[0m`
+	end,
+
+	gray = function(s: any)
+		return `\27[30;1m{s}\27[0m`
+	end,
+}

tools/entity_visualiser.luau

@@ -0,0 +1,43 @@
+local jecs = require("@jecs")
+local ECS_GENERATION = jecs.ECS_GENERATION
+local ECS_ID = jecs.ECS_ID
+local ansi = require("@tools/ansi")
+
+local function pe(e: any)
+	local gen = ECS_GENERATION(e)
+	return ansi.green(`e{ECS_ID(e)}`)..ansi.yellow(`v{gen}`)
+end
+
+local function name(world: jecs.World, id: any)
+	return world:get(id, jecs.Name) or `${id}`
+end
+
+local function components(world: jecs.World, entity: any)
+	local r = jecs.entity_index_try_get(world.entity_index, entity)
+	if not r then
+		return false
+	end
+
+	local archetype = r.archetype
+	local row = r.row
+	print(`Entity {pe(entity)}`)
+	print("-----------------------------------------------------")
+	for i, column in archetype.columns do
+		local component = archetype.types[i]
+		local n
+		if jecs.IS_PAIR(component) then
+			n = `({name(world, jecs.pair_first(world, component))}, {name(world, jecs.pair_second(world, component))})`
+		else
+			n = name(world, component)
+		end
+		local data = column[row] or "TAG"
+		print(`| {n} | {data} |`)
+	end
+	print("-----------------------------------------------------")
+	return true
+end
+
+return {
+	components = components,
+	prettify = pe,
+}

tools/lifetime_tracker.luau

@@ -0,0 +1,212 @@
+local jecs = require("@jecs")
+local ECS_GENERATION = jecs.ECS_GENERATION
+local ECS_ID = jecs.ECS_ID
+local __ = jecs.Wildcard
+local pair = jecs.pair
+
+local prettify = require("@tools/entity_visualiser").prettify
+
+local pe = prettify
+local ansi = require("@tools/ansi")
+
+function print_centered_entity(entity, width: number)
+    local entity_str = tostring(entity)
+    local entity_length = #entity_str
+
+    local padding_total = width - 2 - entity_length
+
+    local padding_left = math.floor(padding_total / 2)
+    local padding_right = padding_total - padding_left
+
+    local centered_str = string.rep(" ", padding_left) .. entity_str .. string.rep(" ", padding_right)
+
+    print("|" .. centered_str .. "|")
+end
+
+local function name(world, e)
+	return world:get(world, e, jecs.Name) or pe(e)
+end
+local padding_enabled = false
+local function pad()
+	if padding_enabled then
+		print("")
+	end
+end
+
+local function lifetime_tracker_add(world: jecs.World, opt)
+	local entity_index = world.entity_index
+	local dense_array = entity_index.dense_array
+	local component_index = world.component_index
+
+	local ENTITY_RANGE = (jecs.Rest :: any) + 1
+
+	local w = setmetatable({}, { __index = world })
+
+	padding_enabled = opt.padding_enabled
+
+	local world_entity = world.entity
+	w.entity = function(self)
+		local will_recycle = entity_index.max_id ~= entity_index.alive_count
+		local e = world_entity(world)
+		if will_recycle then
+			print(`*recycled {pe(e)}`)
+		else
+			print(`*created {pe(e)}`)
+		end
+		pad()
+		return e
+	end
+	w.print_entity_index = function(self)
+		local max_id = entity_index.max_id
+		local alive_count = entity_index.alive_count
+		local alive = table.move(dense_array, 1+jecs.Rest::any, alive_count, 1, {})
+		local dead = table.move(dense_array, alive_count + 1, max_id, 1, {})
+
+		local sep = "|--------|"
+		if #alive > 0 then
+			print("|-alive--|")
+			for i = 1, #alive do
+				local e = pe(alive[i])
+				print_centered_entity(e, 32)
+				print(sep)
+			end
+			print("\n")
+		end
+
+		if #dead > 0 then
+			print("|--dead--|")
+			for i = 1, #dead do
+				print_centered_entity(pe(dead[i]), 32)
+				print(sep)
+			end
+		end
+		pad()
+	end
+	local timelines = {}
+	w.print_snapshot = function(self)
+		local timeline = #timelines + 1
+		local entity_column_width = 10
+		local status_column_width = 8
+
+		local header = string.format("| %-" .. entity_column_width .. "s |", "Entity")
+		for i = 1, timeline do
+			header = header .. string.format(" %-" .. status_column_width .. "s |", string.format("T%d", i))
+		end
+
+		local max_id = entity_index.max_id
+		local alive_count = entity_index.alive_count
+		local alive = table.move(dense_array, 1+jecs.Rest::any, alive_count, 1, {})
+		local dead = table.move(dense_array, alive_count + 1, max_id, 1, {})
+
+		local data = {}
+		print("-------------------------------------------------------------------")
+		print(header)
+
+		-- Store the snapshot data for this timeline
+		for i = ENTITY_RANGE, max_id do
+			if dense_array[i] then
+				local entity = dense_array[i]
+				local id = ECS_ID(entity)
+				local status = "alive"
+				if id > alive_count then
+					status = "dead"
+				end
+				data[id] = status
+			end
+		end
+
+		table.insert(timelines, data)
+
+		-- Create a table to hold entity data for sorting
+		local entities = {}
+		for i = ENTITY_RANGE, max_id do
+			if dense_array[i] then
+				local entity = dense_array[i]
+				local id = ECS_ID(entity)
+				-- Push entity and id into the new `entities` table
+				table.insert(entities, {entity = entity, id = id})
+			end
+		end
+
+		-- Sort the entities by ECS_ID
+		table.sort(entities, function(a, b)
+			return a.id < b.id
+		end)
+
+		-- Print the sorted rows
+		for _, entity_data in ipairs(entities) do
+			local entity = entity_data.entity
+			local id = entity_data.id
+			local status = "alive"
+			if id > alive_count then
+				status = "dead"
+			end
+			local row = string.format("| %-" .. entity_column_width .. "s     |", pe(entity))
+			for j = 1, timeline do
+				local timeline_data = timelines[j]
+				local entity_data = timeline_data[id]
+				if entity_data then
+					row = row .. string.format(" %-" .. status_column_width .. "s |", entity_data)
+				else
+					row = row .. string.format(" %-" .. status_column_width .. "s |", "-")
+				end
+			end
+			print(row)
+		end
+		print("-------------------------------------------------------------------")
+		pad()
+	end
+	local world_add = world.add
+	local relations = {}
+	w.add = function(self, entity: any, component: any)
+		world_add(world, entity, component)
+		if jecs.IS_PAIR(component) then
+			local relation = jecs.pair_first(world, component)
+			local target = jecs.pair_second(world, component)
+			print(`*added ({pe(relation)}, {pe(target)}) to {pe(entity)}`)
+			pad()
+		end
+
+	end
+
+	local world_delete = world.delete
+	w.delete = function(self, e)
+		world_delete(world, e)
+
+		local idr_t = component_index[pair(__, e)]
+		if idr_t then
+			for archetype_id in idr_t.cache do
+				local archetype = world.archetypes[archetype_id]
+				for _, id in archetype.types do
+					if not jecs.IS_PAIR(id) then
+						continue
+					end
+					local object = jecs.pair_second(world, id)
+					if object ~= e then
+						continue
+					end
+					local id_record = component_index[id]
+					local flags = id_record.flags
+					local flags_delete_mask: number = bit32.band(flags, jecs.ECS_ID_DELETE)
+					if flags_delete_mask ~= 0 then
+						for _, entity in archetype.entities do
+							print(`*deleted dependant {pe(entity)} of {pe(e)}`)
+							pad()
+						end
+						break
+					else
+						for _, entity in archetype.entities do
+							print(`*removed dependency ({pe(jecs.pair_first(world, id))}, {pe(object)}) from {pe(entity)}`)
+						end
+					end
+				end
+			end
+		end
+
+		print(`*deleted {pe(e)}`)
+		pad()
+	end
+	return w
+end
+
+return lifetime_tracker_add