rewrite(phase2): schema-defined buffer

sourcemap.json

@@ -1 +1 @@
-{"name":"Warp","className":"ModuleScript","filePaths":["src\\init.luau","default.project.json"],"children":[{"name":"Buffer","className":"ModuleScript","filePaths":["src\\Buffer.luau"]},{"name":"Client","className":"ModuleScript","filePaths":["src\\Client\\init.luau"]},{"name":"Server","className":"ModuleScript","filePaths":["src\\Server\\init.luau"]},{"name":"Thread","className":"ModuleScript","filePaths":["src\\Thread.luau"]}]}
+{"name":"Warp","className":"ModuleScript","filePaths":["src\\init.luau","default.project.json"],"children":[{"name":"Client","className":"ModuleScript","filePaths":["src\\Client\\init.luau"]},{"name":"Server","className":"ModuleScript","filePaths":["src\\Server\\init.luau"]},{"name":"Thread","className":"ModuleScript","filePaths":["src\\Thread.luau"]},{"name":"Buffer","className":"ModuleScript","filePaths":["src\\Buffer\\init.luau"]}]}

src/Buffer/init.luau

@@ -947,8 +947,315 @@ local function reset(w: Writer)
 	table.clear(w.refs)
 end
 
+local Schema = {}
+export type SchemaType = { type: string, [any]: any }
+
+Schema.u8 = { type = "u8" }
+Schema.i8 = { type = "i8" }
+Schema.u16 = { type = "u16" }
+Schema.i16 = { type = "i16" }
+Schema.u32 = { type = "u32" }
+Schema.i32 = { type = "i32" }
+Schema.f32 = { type = "f32" }
+Schema.f64 = { type = "f64" }
+Schema.f16 = { type = "f16" }
+Schema.boolean = { type = "boolean" }
+Schema.vector3 = { type = "vector3" }
+Schema.vector2 = { type = "vector2" }
+Schema.cframe = { type = "cframe" }
+Schema.color3 = { type = "color3" }
+Schema.instance = { type = "instance" }
+Schema.string = { type = "string" }
+
+function Schema.optional(item: SchemaType): SchemaType
+	return { type = "optional", item = item }
+end
+
+function Schema.array(item: SchemaType): SchemaType
+	return { type = "array", item = item }
+end
+
+function Schema.map(key: SchemaType, value: SchemaType): SchemaType
+	return { type = "map", key = key, value = value }
+end
+
+function Schema.struct(fields: {[string]: SchemaType}): SchemaType
+	local orderedFields = {}
+	for k, v in fields do
+		table.insert(orderedFields, { key = k, schema = v })
+	end
+	table.sort(orderedFields, function(a, b) return a.key < b.key end)
+	return { type = "struct", fields = orderedFields }
+end
+
+local function compilePacker(s: SchemaType): (Writer, any) -> ()
+	if s.type == "u8" then return wByte end
+	if s.type == "i8" then return function(w, v) ensureSpace(w, 1) buffer.writei8(w.buf, w.cursor, v) w.cursor += 1 end end
+	if s.type == "u16" then return wU16 end
+	if s.type == "i16" then return wI16 end
+	if s.type == "u32" then return wU32 end
+	if s.type == "i32" then return wI32 end
+	if s.type == "f32" then return wF32 end
+	if s.type == "f64" then return wF64 end
+	if s.type == "f16" then return wF16 end
+	if s.type == "boolean" then return function(w, v) wByte(w, v and 1 or 0) end end
+	if s.type == "string" then return function(w, v) local len = #v wVarUInt(w, len) wString(w, v) end end
+	
+	if s.type == "vector3" then return function(w, v) wF16(w, f32ToF16(v.X)) wF16(w, f32ToF16(v.Y)) wF16(w, f32ToF16(v.Z)) end end
+	if s.type == "vector2" then return function(w, v) wF16(w, f32ToF16(v.X)) wF16(w, f32ToF16(v.Y)) end end
+	
+	if s.type == "cframe" then 
+		return function(w, v) 
+			local pos = v.Position
+			local rx, ry, rz = v:ToOrientation()
+			wF16(w, f32ToF16(pos.X)) wF16(w, f32ToF16(pos.Y)) wF16(w, f32ToF16(pos.Z))
+			wF16(w, f32ToF16(rx)) wF16(w, f32ToF16(ry)) wF16(w, f32ToF16(rz))
+		end 
+	end
+
+	if s.type == "color3" then 
+		return function(w, v) 
+			wByte(w, math.clamp(math.round(v.R * 255), 0, 255))
+			wByte(w, math.clamp(math.round(v.G * 255), 0, 255))
+			wByte(w, math.clamp(math.round(v.B * 255), 0, 255))
+		end 
+	end
+
+	if s.type == "instance" then return function(w, v) table.insert(w.refs, v) wVarUInt(w, #w.refs) end end
+	
+	if s.type == "struct" then
+		local fields = {}
+		for _, field in s.fields do
+			table.insert(fields, { key = field.key, packer = compilePacker(field.schema) })
+		end
+		return function(w, v)
+			if type(v) ~= "table" then error(`Expected table for struct, got {typeof(v)}`) end
+			for _, f in fields do
+				local val = v[f.key]
+				if val == nil then error(`Schema Error: Missing required field '{f.key}'`) end
+				f.packer(w, val)
+			end
+		end
+	end
+
+	if s.type == "array" then
+		local itemPacker = compilePacker(s.item)
+		return function(w, v)
+			if type(v) ~= "table" then error(`Expected table for array, got {typeof(v)}`) end
+			local len = #v
+			wVarUInt(w, len)
+			for i = 1, len do
+				if v[i] == nil then error(`Schema Error: Array item at index {i} is nil`) end
+				itemPacker(w, v[i])
+			end
+		end
+	end
+
+	if s.type == "map" then
+		local keyPacker = compilePacker(s.key)
+		local valPacker = compilePacker(s.value)
+		return function(w, v)
+			local count = 0
+			for _ in v do count += 1 end
+			wVarUInt(w, count)
+			for k, val in v do
+				keyPacker(w, k)
+				valPacker(w, val)
+			end
+		end
+	end
+
+	if s.type == "optional" then
+		local itemPacker = compilePacker(s.item)
+		return function(w, v)
+			if v == nil then
+				wByte(w, 0)
+			else
+				wByte(w, 1)
+				itemPacker(w, v)
+			end
+		end
+	end
+
+	return function() end
+end
+
+local function compileReader(s: SchemaType): (buffer, number, {any}?) -> (any, number)
+	if s.type == "u8" then return function(b, c) return buffer.readu8(b, c), c + 1 end end
+	if s.type == "i8" then return function(b, c) return buffer.readi8(b, c), c + 1 end end
+	if s.type == "u16" then return function(b, c) return buffer.readu16(b, c), c + 2 end end
+	if s.type == "i16" then return function(b, c) return buffer.readi16(b, c), c + 2 end end
+	if s.type == "u32" then return function(b, c) return buffer.readu32(b, c), c + 4 end end
+	if s.type == "i32" then return function(b, c) return buffer.readi32(b, c), c + 4 end end
+	if s.type == "f32" then return function(b, c) return buffer.readf32(b, c), c + 4 end end
+	if s.type == "f64" then return function(b, c) return buffer.readf64(b, c), c + 8 end end
+	if s.type == "f16" then return function(b, c) return f16ToF32(buffer.readu16(b, c)), c + 2 end end
+	if s.type == "boolean" then return function(b, c) return buffer.readu8(b, c) ~= 0, c + 1 end end
+	if s.type == "string" then 
+		return function(b, c) 
+			local len; len, c = readVarUInt(b, c) 
+			return buffer.readstring(b, c, len), c + len 
+		end 
+	end
+	if s.type == "vector3" then
+		return function(b, c)
+			local x = f16ToF32(buffer.readu16(b, c))
+			local y = f16ToF32(buffer.readu16(b, c + 2))
+			local z = f16ToF32(buffer.readu16(b, c + 4))
+			return Vector3.new(x, y, z), c + 6
+		end
+	end
+	
+	if s.type == "vector2" then
+		return function(b, c)
+			local x = f16ToF32(buffer.readu16(b, c))
+			local y = f16ToF32(buffer.readu16(b, c + 2))
+			return Vector2.new(x, y), c + 4
+		end
+	end
+	
+	if s.type == "color3" then
+		return function(b, c)
+			local r = buffer.readu8(b, c)
+			local g = buffer.readu8(b, c + 1)
+			local bVal = buffer.readu8(b, c + 2)
+			return Color3.fromRGB(r, g, bVal), c + 3
+		end
+	end
+
+	if s.type == "cframe" then
+		return function(b, c)
+			local px = f16ToF32(buffer.readu16(b, c))
+			local py = f16ToF32(buffer.readu16(b, c + 2))
+			local pz = f16ToF32(buffer.readu16(b, c + 4))
+			local rx = f16ToF32(buffer.readu16(b, c + 6))
+			local ry = f16ToF32(buffer.readu16(b, c + 8))
+			local rz = f16ToF32(buffer.readu16(b, c + 10))
+			return CFrame.new(px, py, pz) * CFrame.fromOrientation(rx, ry, rz), c + 12
+		end
+	end
+	if s.type == "instance" then
+		return function(b, c, refs)
+			local idx; idx, c = readVarUInt(b, c)
+			return refs and refs[idx] or nil, c
+		end
+	end
+	
+	if s.type == "struct" then
+		local fields = {}
+		for _, field in s.fields do
+			table.insert(fields, { key = field.key, reader = compileReader(field.schema) })
+		end
+		return function(b, c, refs)
+			local obj = {}
+			for _, f in fields do
+				obj[f.key], c = f.reader(b, c, refs)
+			end
+			return obj, c
+		end
+	end
+
+	if s.type == "array" then
+		local itemReader = compileReader(s.item)
+		return function(b, c, refs)
+			local len; len, c = readVarUInt(b, c)
+			local arr = table.create(len)
+			for i = 1, len do
+				arr[i], c = itemReader(b, c, refs)
+			end
+			return arr, c
+		end
+	end
+
+	if s.type == "map" then
+		local keyReader = compileReader(s.key)
+		local valReader = compileReader(s.value)
+		return function(b, c, refs)
+			local count; count, c = readVarUInt(b, c)
+			local map = {}
+			for _ = 1, count do
+				local k, val
+				k, c = keyReader(b, c, refs)
+				val, c = valReader(b, c, refs)
+				map[k] = val
+			end
+			return map, c
+		end
+	end
+
+	if s.type == "optional" then
+		local itemReader = compileReader(s.item)
+		return function(b, c, refs)
+			local exists = buffer.readu8(b, c) ~= 0
+			c += 1
+			if exists then
+				return itemReader(b, c, refs)
+			else
+				return nil, c
+			end
+		end
+	end
+	
+	return function(_, c) return nil, c end
+end
+
+local function packStrict(w: Writer, s: SchemaType, v: any)
+	local packer = compilePacker(s)
+	packer(w, v)
+end
+
+local function readStrict(buf: buffer, cursor: number, s: SchemaType, refs: { any }?): (any, number)
+	local reader = compileReader(s)
+	return reader(buf, cursor, refs)
+end
+
+local function writeEvents(w: Writer, events: {{any}}, schemas: {[string]: SchemaType})
+	local count = #events
+	wVarUInt(w, count)
+	for _, event in events do
+		local remote = event[1]
+		local args = event[2]
+		packValue(w, remote)
+		local schema = schemas[remote]
+		if schema then
+			packStrict(w, schema, args[1])
+		else
+			packValue(w, args)
+		end
+	end
+end
+
+local function readEvents(buf: buffer, refs: {any}?, schemas: {[string]: SchemaType}): {{any}}
+	local pos, count = 0
+	count, pos = readVarUInt(buf, pos)
+	local events = table.create(count)
+	for i = 1, count do
+		local remote
+		remote, pos = unpackValue(buf, pos, refs)
+		local args
+		local schema = schemas[remote]
+		if schema then
+			local val
+			val, pos = readStrict(buf, pos, schema, refs)
+			args = {val}
+		else
+			args, pos = unpackValue(buf, pos, refs)
+		end
+		events[i] = {remote, args}
+	end
+	return events
+end
+
 local BufferSerdes = {}
 
+BufferSerdes.writeEvents = writeEvents
+BufferSerdes.readEvents = readEvents
+BufferSerdes.Schema = Schema
+BufferSerdes.compilePacker = compilePacker
+BufferSerdes.compileReader = compileReader
+BufferSerdes.packStrict = packStrict
+BufferSerdes.readStrict = readStrict
+
 function BufferSerdes.write(data: any): (buffer, {any}?)
 	local w = createWriter()
 	packValue(w, data)
@@ -983,4 +1290,8 @@ BufferSerdes.readVarUInt = readVarUInt
 BufferSerdes.f32ToF16 = f32ToF16
 BufferSerdes.f16ToF32 = f16ToF32
 
-return BufferSerdes :: typeof(BufferSerdes)
+BufferSerdes.readTagged = unpackValue
+BufferSerdes.packTagged = packValue
+BufferSerdes.unpack = unpackValue
+
+return BufferSerdes :: typeof(BufferSerdes)

src/Client/init.luau

@@ -22,11 +22,15 @@ type Event = {
 
 local queueEvent: { { any } } = {}
 local eventListeners: { Event } = {}
+local eventSchemas: { [string]: Buffer.SchemaType } = {}
 
 local pendingInvokes: { [string]: thread } = {}
-local invokeHandlers: { [string]: (...any?) -> ...any? } = {}
 local invokeId = 0
 
+Client.useSchema = function(remoteName: string, schema: Buffer.SchemaType)
+	eventSchemas[remoteName] = schema
+end
+
 Client.Connect = function(remoteName: string, fn: (Player, ...any?) -> (...any?)): Connection
 	local detail = {
 		remote = remoteName,
@@ -86,21 +90,21 @@ end
 
 Client.Invoke = function(remoteName: string, timeout: number?, ...: any?): ...any?
 	invokeId += 1
-	local invokeId, thread = `{invokeId}`, coroutine.running()
+	local id, thread = `{invokeId}`, coroutine.running()
 
-	pendingInvokes[invokeId] = thread
+	pendingInvokes[id] = thread
 	task.delay(timeout or 2, function()
-		local pending = pendingInvokes[invokeId]
+		local pending = pendingInvokes[id]
 		if not pending then
 			return
 		end
 		task.spawn(pending, nil)
-		pendingInvokes[invokeId] = nil
+		pendingInvokes[id] = nil
 	end)
 
 	table.insert(queueEvent, {
 		"\0",
-		{ remoteName, invokeId, { ... } :: any } :: any
+		{ remoteName, id, { ... } :: any } :: any
 	})
 	return coroutine.yield()
 end
@@ -113,19 +117,19 @@ if RunService:IsClient() then
 			local remote = content[1]
 			local content = content[2]
 			if remote == "\1" then
-				local invokeId = content[1]
+				local id = content[1]
 				local results = content[2]
-				local pending = pendingInvokes[invokeId]
+				local pending = pendingInvokes[id]
 				if pending then
 					task.spawn(pending :: any, table.unpack(results))
-					pendingInvokes[invokeId] = nil
+					pendingInvokes[id] = nil
 				end
 				continue
 			end
 			if #eventListeners == 0 then continue end
 			if remote == "\0" then
 				local remoteName = content[1]
-				local invokeId = content[2]
+				local id = content[2]
 				local args = content[3]
 				for _, connection in eventListeners do
 					if connection.remote == remoteName then
@@ -133,7 +137,7 @@ if RunService:IsClient() then
 							local results = { connection.fn(table.unpack(args)) }
 							table.insert(queueEvent, {
 								"\1",
-								{ invokeId, results } :: any
+								{ id, results } :: any
 							})
 						end)
 						break
@@ -152,7 +156,7 @@ if RunService:IsClient() then
 		if deltaT < cycle then return end
 		deltaT = 0
 		if #queueEvent == 0 then return end
-		Buffer.pack(writer, queueEvent)
+		Buffer.writeEvents(writer, queueEvent, eventSchemas)
 		do
 			local buf, ref = Buffer.buildWithRefs(writer)
 			Buffer.reset(writer)
@@ -166,4 +170,11 @@ if RunService:IsClient() then
 	end)
 end
 
+--[[
+	@class Client
+	@schema
+	define a schema for your data and use a strict packing
+]]
+Client.Schema = Buffer.Schema
+
 return Client :: typeof(Client)

src/Server/init.luau

@@ -25,12 +25,16 @@ local queueEvent: {
 	[Player]: { { any } },
 } = {}
 local eventListeners: { Event } = {}
+local eventSchemas: { [string]: Buffer.SchemaType } = {}
 local players_ready: { Player } = {}
 
 local pendingInvokes: { [string]: thread } = {}
-local invokeHandlers: { [string]: (...any?) -> ...any? } = {}
 local invokeId = 0
 
+Server.useSchema = function(remoteName: string, schema: Buffer.SchemaType)
+	eventSchemas[remoteName] = schema
+end
+
 Server.Connect = function(remoteName: string, fn: (Player, ...any?) -> (...any?)): Connection
 	local detail = {
 		remote = remoteName,
@@ -99,21 +103,21 @@ end
 
 Server.Invoke = function(remoteName: string, player: Player, timeout: number?, ...: any?): ...any?
 	invokeId += 1
-	local invokeId, thread = `{invokeId}`, coroutine.running()
+	local id, thread = `{invokeId}`, coroutine.running()
 
-	pendingInvokes[invokeId] = thread
+	pendingInvokes[id] = thread
 	task.delay(timeout or 2, function()
-		local pending = pendingInvokes[invokeId]
+		local pending = pendingInvokes[id]
 		if not pending then
 			return
 		end
 		task.spawn(pending, nil)
-		pendingInvokes[invokeId] = nil
+		pendingInvokes[id] = nil
 	end)
 
 	table.insert(queueEvent[player], {
 		"\0",
-		{ remoteName, invokeId, { ... } :: any } :: any
+		{ remoteName, id, { ... } :: any } :: any
 	})
 	return coroutine.yield()
 end
@@ -121,24 +125,24 @@ end
 if RunService:IsServer() then
 	Event.OnServerEvent:Connect(function(player: Player, b: buffer, ref: { Instance? })
 		if type(b) ~= "buffer" then return end
-		local contents = Buffer.read(b, ref)
+		local contents = Buffer.readEvents(b, ref, eventSchemas)
 		for _, content in contents do
 			local remote = content[1]
 			local content = content[2]
 			if remote == "\1" then
-				local invokeId = content[1]
+				local id = content[1]
 				local results = content[2]
-				local pending = pendingInvokes[invokeId]
+				local pending = pendingInvokes[id]
 				if pending then
 					task.spawn(pending :: any, table.unpack(results))
-					pendingInvokes[invokeId] = nil
+					pendingInvokes[id] = nil
 				end
 				continue
 			end
 			if #eventListeners == 0 then continue end
 			if remote == "\0" then
 				local remoteName = content[1]
-				local invokeId = content[2]
+				local id = content[2]
 				local args = content[3]
 				for _, connection in eventListeners do
 					if connection.remote == remoteName then
@@ -146,7 +150,7 @@ if RunService:IsServer() then
 							local results = { connection.fn(table.unpack(args)) }
 							table.insert(queueEvent[player], {
 								"\1",
-								{ invokeId, results } :: any
+								{ id, results } :: any
 							})
 						end)
 						break
@@ -166,7 +170,7 @@ if RunService:IsServer() then
 		deltaT = 0
 		for player: Player, content in queueEvent do
 			if #content == 0 or player.Parent ~= Players then continue end
-			Buffer.pack(writer, content)
+			Buffer.writeEvents(writer, content, eventSchemas)
 			do
 				local buf, ref = Buffer.buildWithRefs(writer)
 				Buffer.reset(writer)
@@ -200,4 +204,11 @@ if RunService:IsServer() then
 	end
 end
 
+--[[
+	@class Server
+	@schema
+	define a schema for your data and use a strict packing
+]]
+Server.Schema = Buffer.Schema
+
 return Server :: typeof(Server)

src/init.luau

@@ -13,13 +13,29 @@ end
 
 local Client = require("@self/Client")
 local Server = require("@self/Server")
+local Buffer = require("@self/Buffer")
 
+--[[
+	@class Remote
+	@client
+]]
 Remote.Client = function()
 	return Client
 end
 
+--[[
+	@class Remote
+	@server
+]]
 Remote.Server = function()
 	return Server
 end
 
+--[[
+	@class Remote
+	@schema
+	define a schema for your data and use a strict packing
+]]
+Remote.Buffer = Buffer
+
 return Remote :: typeof(Remote)