From bb0592f3ebaf9e951db2eded80ad9610f1c146c0 Mon Sep 17 00:00:00 2001 From: Ryan Ward Date: Sun, 30 Jul 2023 00:47:27 -0400 Subject: [PATCH] new pack/unpack for tables, current issue is things being turned into strings --- init.lua | 2 +- integration/loveManager/extensions.lua | 3 +- integration/loveManager/init.lua | 11 +- integration/loveManager/serpent.lua | 176 ------------------------- integration/loveManager/threads.lua | 42 +----- integration/loveManager/utils.lua | 55 ++++++++ integration/sharedExtensions/init.lua | 2 +- lovethreads/main.lua | 132 ++++++++++++------- tests/threadtests.lua | 9 +- 9 files changed, 162 insertions(+), 270 deletions(-) delete mode 100644 integration/loveManager/serpent.lua create mode 100644 integration/loveManager/utils.lua diff --git a/init.lua b/init.lua index 821e0f1..aa87d1f 100644 --- a/init.lua +++ b/init.lua @@ -1287,7 +1287,7 @@ function thread.hold(n, opt) elseif type(n) == "function" then return yield(CMD, t_hold, n, nil, interval) else - multi.error("Invalid argument passed to thread.hold(...)!") + multi.error("Invalid argument passed to thread.hold(...) ".. type(n) .. "!") end end diff --git a/integration/loveManager/extensions.lua b/integration/loveManager/extensions.lua index c30adc9..163ccf6 100644 --- a/integration/loveManager/extensions.lua +++ b/integration/loveManager/extensions.lua @@ -11,7 +11,7 @@ function multi:newSystemThreadedQueue(name) c.Name = name c.Type = multi.SQUEUE - c.chan = love.thread.getChannel(name) + c.chan = love.thread.newChannel() function c:push(dat) self.chan:push(THREAD.packValue(dat)) @@ -26,7 +26,6 @@ function multi:newSystemThreadedQueue(name) end function c:init() - self.chan = love.thread.getChannel(self.Name) return self end diff --git a/integration/loveManager/init.lua b/integration/loveManager/init.lua index 0bc9577..472125c 100644 --- a/integration/loveManager/init.lua +++ b/integration/loveManager/init.lua @@ -4,10 +4,11 @@ end local ThreadFileData = [[ ISTHREAD = true -__FUNC, THREAD_ID, THREAD_NAME, __PACK = ... +args = {...} +THREAD_ID = table.remove(args, 1) +THREAD_NAME = table.remove(args, 1) GLOBAL, THREAD = require("multi.integration.loveManager.threads"):init() -__FUNC = THREAD.unpackValue(__FUNC) -__PACK = THREAD.unpackValue(__PACK) +__FUNC = THREAD.unpackValue(table.remove(args, 1)) math.randomseed(THREAD_ID) math.random() math.random() @@ -22,7 +23,7 @@ end multi, thread = require("multi"):init() require("multi.integration.loveManager.extensions") require("multi.integration.sharedExtensions") -stab["returns"] = {__FUNC(multi.unpack(__PACK))} +stab["returns"] = {__FUNC(multi.unpack(args))} ]] _G.THREAD_NAME = "MAIN_THREAD" @@ -44,7 +45,7 @@ function multi:newSystemThread(name, func, ...) c.Name = name c.ID = tid c.thread = love.thread.newThread(ThreadFileData) - c.thread:start(THREAD.packValue(func), c.ID, c.Name, THREAD.packValue({...})) + c.thread:start(c.ID, c.Name, THREAD.packValue(func), ...) c.stab = THREAD.createTable(name .. c.ID) c.creationTime = os.clock() c.OnDeath = multi:newConnection() diff --git a/integration/loveManager/serpent.lua b/integration/loveManager/serpent.lua deleted file mode 100644 index 5063f3a..0000000 --- a/integration/loveManager/serpent.lua +++ /dev/null @@ -1,176 +0,0 @@ ---[[ -Serpent source is released under the MIT License - -Copyright (c) 2012-2018 Paul Kulchenko (paul@kulchenko.com) - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -]] - -local n, v = "serpent", "0.303" -- (C) 2012-18 Paul Kulchenko; MIT License -local c, d = "Paul Kulchenko", "Lua serializer and pretty printer" -local snum = {[tostring(1/0)]='1/0 --[[math.huge]]',[tostring(-1/0)]='-1/0 --[[-math.huge]]',[tostring(0/0)]='0/0'} -local badtype = {thread = true, userdata = true, cdata = true} -local getmetatable = debug and debug.getmetatable or getmetatable -local pairs = function(t) return next, t end -- avoid using __pairs in Lua 5.2+ -local keyword, globals, G = {}, {}, (_G or _ENV) -for _,k in ipairs({'and', 'break', 'do', 'else', 'elseif', 'end', 'false', - 'for', 'function', 'goto', 'if', 'in', 'local', 'nil', 'not', 'or', 'repeat', - 'return', 'then', 'true', 'until', 'while'}) do keyword[k] = true end -for k,v in pairs(G) do globals[v] = k end -- build func to name mapping -for _,g in ipairs({'coroutine', 'debug', 'io', 'math', 'string', 'table', 'os'}) do - for k,v in pairs(type(G[g]) == 'table' and G[g] or {}) do globals[v] = g..'.'..k end end - -local function s(t, opts) - local name, indent, fatal, maxnum = opts.name, opts.indent, opts.fatal, opts.maxnum - local sparse, custom, huge = opts.sparse, opts.custom, not opts.nohuge - local space, maxl = (opts.compact and '' or ' '), (opts.maxlevel or math.huge) - local maxlen, metatostring = tonumber(opts.maxlength), opts.metatostring - local iname, comm = '_'..(name or ''), opts.comment and (tonumber(opts.comment) or math.huge) - local numformat = opts.numformat or "%.17g" - local seen, sref, syms, symn = {}, {'local '..iname..'={}'}, {}, 0 - local function gensym(val) return '_'..(tostring(tostring(val)):gsub("[^%w]",""):gsub("(%d%w+)", - -- tostring(val) is needed because __tostring may return a non-string value - function(s) if not syms[s] then symn = symn+1; syms[s] = symn end return tostring(syms[s]) end)) end - local function safestr(s) return type(s) == "number" and (huge and snum[tostring(s)] or numformat:format(s)) - or type(s) ~= "string" and tostring(s) -- escape NEWLINE/010 and EOF/026 - or ("%q"):format(s):gsub("\010","n"):gsub("\026","\\026") end - -- handle radix changes in some locales - if opts.fixradix and (".1f"):format(1.2) ~= "1.2" then - local origsafestr = safestr - safestr = function(s) return type(s) == "number" - and (nohuge and snum[tostring(s)] or numformat:format(s):gsub(",",".")) or origsafestr(s) - end - end - local function comment(s,l) return comm and (l or 0) < comm and ' --[['..select(2, pcall(tostring, s))..']]' or '' end - local function globerr(s,l) return globals[s] and globals[s]..comment(s,l) or not fatal - and safestr(select(2, pcall(tostring, s))) or error("Can't serialize "..tostring(s)) end - local function safename(path, name) -- generates foo.bar, foo[3], or foo['b a r'] - local n = name == nil and '' or name - local plain = type(n) == "string" and n:match("^[%l%u_][%w_]*$") and not keyword[n] - local safe = plain and n or '['..safestr(n)..']' - return (path or '')..(plain and path and '.' or '')..safe, safe end - local alphanumsort = type(opts.sortkeys) == 'function' and opts.sortkeys or function(k, o, n) -- k=keys, o=originaltable, n=padding - local maxn, to = tonumber(n) or 12, {number = 'a', string = 'b'} - local function padnum(d) return ("%0"..tostring(maxn).."d"):format(tonumber(d)) end - table.sort(k, function(a,b) - -- sort numeric keys first: k[key] is not nil for numerical keys - return (k[a] ~= nil and 0 or to[type(a)] or 'z')..(tostring(a):gsub("%d+",padnum)) - < (k[b] ~= nil and 0 or to[type(b)] or 'z')..(tostring(b):gsub("%d+",padnum)) end) end - local function val2str(t, name, indent, insref, path, plainindex, level) - local ttype, level, mt = type(t), (level or 0), getmetatable(t) - local spath, sname = safename(path, name) - local tag = plainindex and - ((type(name) == "number") and '' or name..space..'='..space) or - (name ~= nil and sname..space..'='..space or '') - if seen[t] then -- already seen this element - sref[#sref+1] = spath..space..'='..space..seen[t] - return tag..'nil'..comment('ref', level) - end - -- protect from those cases where __tostring may fail - if type(mt) == 'table' and metatostring ~= false then - local to, tr = pcall(function() return mt.__tostring(t) end) - local so, sr = pcall(function() return mt.__serialize(t) end) - if (to or so) then -- knows how to serialize itself - seen[t] = insref or spath - t = so and sr or tr - ttype = type(t) - end -- new value falls through to be serialized - end - if ttype == "table" then - if level >= maxl then return tag..'{}'..comment('maxlvl', level) end - seen[t] = insref or spath - if next(t) == nil then return tag..'{}'..comment(t, level) end -- table empty - if maxlen and maxlen < 0 then return tag..'{}'..comment('maxlen', level) end - local maxn, o, out = math.min(#t, maxnum or #t), {}, {} - for key = 1, maxn do o[key] = key end - if not maxnum or #o < maxnum then - local n = #o -- n = n + 1; o[n] is much faster than o[#o+1] on large tables - for key in pairs(t) do - if o[key] ~= key then n = n + 1; o[n] = key end - end - end - if maxnum and #o > maxnum then o[maxnum+1] = nil end - if opts.sortkeys and #o > maxn then alphanumsort(o, t, opts.sortkeys) end - local sparse = sparse and #o > maxn -- disable sparsness if only numeric keys (shorter output) - for n, key in ipairs(o) do - local value, ktype, plainindex = t[key], type(key), n <= maxn and not sparse - if opts.valignore and opts.valignore[value] -- skip ignored values; do nothing - or opts.keyallow and not opts.keyallow[key] - or opts.keyignore and opts.keyignore[key] - or opts.valtypeignore and opts.valtypeignore[type(value)] -- skipping ignored value types - or sparse and value == nil then -- skipping nils; do nothing - elseif ktype == 'table' or ktype == 'function' or badtype[ktype] then - if not seen[key] and not globals[key] then - sref[#sref+1] = 'placeholder' - local sname = safename(iname, gensym(key)) -- iname is table for local variables - sref[#sref] = val2str(key,sname,indent,sname,iname,true) - end - sref[#sref+1] = 'placeholder' - local path = seen[t]..'['..tostring(seen[key] or globals[key] or gensym(key))..']' - sref[#sref] = path..space..'='..space..tostring(seen[value] or val2str(value,nil,indent,path)) - else - out[#out+1] = val2str(value,key,indent,nil,seen[t],plainindex,level+1) - if maxlen then - maxlen = maxlen - #out[#out] - if maxlen < 0 then break end - end - end - end - local prefix = string.rep(indent or '', level) - local head = indent and '{\n'..prefix..indent or '{' - local body = table.concat(out, ','..(indent and '\n'..prefix..indent or space)) - local tail = indent and "\n"..prefix..'}' or '}' - return (custom and custom(tag,head,body,tail,level) or tag..head..body..tail)..comment(t, level) - elseif badtype[ttype] then - seen[t] = insref or spath - return tag..globerr(t, level) - elseif ttype == 'function' then - seen[t] = insref or spath - if opts.nocode then return tag.."function() --[[..skipped..]] end"..comment(t, level) end - local ok, res = pcall(string.dump, t) - local func = ok and "((loadstring or load)("..safestr(res)..",'@serialized'))"..comment(t, level) - return tag..(func or globerr(t, level)) - else return tag..safestr(t) end -- handle all other types - end - local sepr = indent and "\n" or ";"..space - local body = val2str(t, name, indent) -- this call also populates sref - local tail = #sref>1 and table.concat(sref, sepr)..sepr or '' - local warn = opts.comment and #sref>1 and space.."--[[incomplete output with shared/self-references skipped]]" or '' - return not name and body..warn or "do local "..body..sepr..tail.."return "..name..sepr.."end" -end - -local function deserialize(data, opts) - local env = (opts and opts.safe == false) and G - or setmetatable({}, { - __index = function(t,k) return t end, - __call = function(t,...) error("cannot call functions") end - }) - local f, res = (loadstring or load)('return '..data, nil, nil, env) - if not f then f, res = (loadstring or load)(data, nil, nil, env) end - if not f then return f, res end - if setfenv then setfenv(f, env) end - return pcall(f) -end - -local function merge(a, b) if b then for k,v in pairs(b) do a[k] = v end end; return a; end -return { _NAME = n, _COPYRIGHT = c, _DESCRIPTION = d, _VERSION = v, serialize = s, - load = deserialize, - dump = function(a, opts) return s(a, merge({name = '_', compact = true, sparse = true}, opts)) end, - line = function(a, opts) return s(a, merge({sortkeys = true, comment = true}, opts)) end, - block = function(a, opts) return s(a, merge({indent = ' ', sortkeys = true, comment = true}, opts)) end } \ No newline at end of file diff --git a/integration/loveManager/threads.lua b/integration/loveManager/threads.lua index 6610690..a0c6dc1 100644 --- a/integration/loveManager/threads.lua +++ b/integration/loveManager/threads.lua @@ -25,53 +25,19 @@ require("love.timer") require("love.system") require("love.data") require("love.thread") -local serpent = require("multi.integration.loveManager.serpent") +local utils = require("multi.integration.loveManager.utils") local multi, thread = require("multi"):init() -local function ltype(data) - local tp = type(data) - if tp == "userdata" then - return data:type() - end - return tp -end - local NIL = love.data.newByteData("\3") -- If a non table/function is supplied we just return it local function packValue(t) - local tp = type(t) - if tp == "table" then - return love.data.newByteData("\1"..serpent.dump(t,{safe = true})) - elseif tp == "function" then - return love.data.newByteData("\2"..serpent.dump({t,true},{safe = true})) - else - return t - end + return utils.pack(t) end -- If a non table/function is supplied we just return it local function unpackValue(d) - if ltype(d) == "ByteData" then - local data = d:getString() - if data:sub(1, 1) == "\1" then - local status, data = serpent.load(data:sub(2,-1),{safe = false}) - if not status then - multi.error(data) - end - return data - elseif data:sub(1, 1) =="\2" then - local status, data = serpent.load(data:sub(2,-1),{safe = false}) - if not status then - multi.error(data) - end - return data[1] - else - return d - end - else - return d - end + return utils.unpack(d) end local function createTable(n) @@ -126,7 +92,7 @@ function INIT() end repeat wait() - until GLOBAL[name] + until GLOBAL[name] ~= nil if type(GLOBAL[name].init) == "function" then return GLOBAL[name]:init() else diff --git a/integration/loveManager/utils.lua b/integration/loveManager/utils.lua new file mode 100644 index 0000000..bdfec3d --- /dev/null +++ b/integration/loveManager/utils.lua @@ -0,0 +1,55 @@ +require("love.data") +local sutils = {} +local NIL = {Type="nil"} + +--love.data.newByteData("\2"..serpent.dump({t,true},{safe = true})) + +local ltype = function(v) return v:type() end +local t = function(value) + local v = type(value) + if v == "userdata" then + local status, return_or_err = pcall(ltype, value) + if status then return return_or_err else return "userdata" end + else return v end +end + +function sutils.pack(tbl, seen) + if type(tbl) == "function" then return {["__$FUNC$__"] = love.data.newByteData(string.dump(tbl))} end + if type(tbl) ~= "table" then return tbl end + local seen = seen or {} + local result = {} + for i,v in pairs(tbl) do + if seen[v] then + result[i] = v + elseif t(v) == "table" then + seen[v] = true + result[i] = sutils.pack(v, seen) + elseif t(v) == "function" then + result["$F"..i] = love.data.newByteData(string.dump(v)) + elseif t{v} == "userdata" then + result[i] = tostring(v) + else -- Handle what we need to and pass the rest along as a value + result[i] = v + end + end + return result +end + +function sutils.unpack(tbl) + if type(tbl) ~= "table" then return tbl end + if tbl["__$FUNC$__"] then return loadstring(tbl["__$FUNC$__"]:getString()) end + for i,v in pairs(tbl) do + if type(i) == "string" and i:sub(1,2) == "$F" then + local rawfunc = v:getString() + v:release() + tbl[i] = nil + tbl[i:sub(3,-1)] = loadstring(rawfunc) + end + if type(v) == "table" then + sutils.unpack(v) + end + end + return tbl +end + +return sutils \ No newline at end of file diff --git a/integration/sharedExtensions/init.lua b/integration/sharedExtensions/init.lua index 1f6f082..9b29a83 100644 --- a/integration/sharedExtensions/init.lua +++ b/integration/sharedExtensions/init.lua @@ -135,7 +135,6 @@ function multi:newProxy(list) self.recv = THREAD.waitFor(self.name.."_R"):init() self.Type = multi.PROXY for _,v in pairs(funcs) do - print(v,_) if type(v) == "table" then -- We have a connection v[2]:init(proc_name) @@ -190,6 +189,7 @@ function multi:newProxy(list) THREAD = multi.integration.THREAD end local proxy = THREAD.waitFor(self.proxy_link) + print("Got:",proxy) proxy.funcs = self.funcs return proxy:init() end diff --git a/lovethreads/main.lua b/lovethreads/main.lua index 01b16c4..3b0434f 100644 --- a/lovethreads/main.lua +++ b/lovethreads/main.lua @@ -1,50 +1,94 @@ package.path = "../?/init.lua;../?.lua;"..package.path local multi, thread = require("multi"):init{print=true, warning = true, error=true} +local flat = require("flatten") -GLOBAL, THREAD = require("multi.integration.loveManager"):init() +local people = { + { + name = "Fred", + address = "16 Long Street", + phone = "123456" + }, + { + name = "Wilma", + address = "16 Long Street", + phone = "123456", + func = function() + print("Hi") + end + }, + { + name = "Barney", + address = "17 Long Street", + phone = "123457", + important = love.data.newByteData("TEST") + } + } -local queue = multi:newSystemThreadedQueue("TestQueue") -local tab = multi:newSystemThreadedTable("TestTable") - -local test = multi:newSystemThread("Test",function() - local queue = THREAD.waitFor("TestQueue") - local tab = THREAD.waitFor("TestTable") - print("THREAD_ID:",THREAD_ID) - queue:push("Did it work?") - tab["Test"] = true - return 1,2,3 -end) - -multi:newThread("QueueTest", function() - print(thread.hold(queue)) - print(thread.hold(tab, {key="Test"})) - print("Done!") -end) - -local jq = multi:newSystemThreadedJobQueue(n) - -jq:registerFunction("test2",function() - print("This works!") -end) - -jq:registerFunction("test",function(a, b, c) - print(a, b+c) - test2() - return a+b+c -end) - -print("Job:",jq:pushJob("test",1,2,3)) -print("Job:",jq:pushJob("test",2,3,4)) -print("Job:",jq:pushJob("test",5,6,7)) - -jq.OnJobCompleted(function(...) - print("Job Completed!", ...) -end) - -function love.draw() - -- +function dump(o) + if type(o) == 'table' then + local s = '{ ' + for k,v in pairs(o) do + if type(k) ~= 'number' then k = '"'..k..'"' end + s = s .. '['..k..'] = ' .. dump(v) .. ',' + end + return s .. '} ' + else + return tostring(o) + end end -function love.update() - multi:uManager() -end \ No newline at end of file +local fpeople = flat.flatten(people) + +print("Flatten", dump(fpeople)) + +local people = flat.unflatten(fpeople) + +print("Unflatten", dump(people)) + +-- GLOBAL, THREAD = require("multi.integration.loveManager"):init() + +-- local queue = multi:newSystemThreadedQueue("TestQueue") +-- local tab = multi:newSystemThreadedTable("TestTable") + +-- local test = multi:newSystemThread("Test",function() +-- local queue = THREAD.waitFor("TestQueue") +-- local tab = THREAD.waitFor("TestTable") +-- print("THREAD_ID:",THREAD_ID) +-- queue:push("Did it work?") +-- tab["Test"] = true +-- return 1,2,3 +-- end) + +-- multi:newThread("QueueTest", function() +-- print(thread.hold(queue)) +-- print(thread.hold(tab, {key="Test"})) +-- print("Done!") +-- end) + +-- local jq = multi:newSystemThreadedJobQueue(n) + +-- jq:registerFunction("test2",function() +-- print("This works!") +-- end) + +-- jq:registerFunction("test",function(a, b, c) +-- print(a, b+c) +-- test2() +-- return a+b+c +-- end) + +-- print("Job:",jq:pushJob("test",1,2,3)) +-- print("Job:",jq:pushJob("test",2,3,4)) +-- print("Job:",jq:pushJob("test",5,6,7)) + +-- jq.OnJobCompleted(function(...) +-- print("Job Completed!", ...) +-- end) + +-- function love.draw() +-- -- +-- end + +-- function love.update() +-- multi:uManager() +-- end \ No newline at end of file diff --git a/tests/threadtests.lua b/tests/threadtests.lua index f3d8fed..f6acf67 100644 --- a/tests/threadtests.lua +++ b/tests/threadtests.lua @@ -71,13 +71,13 @@ multi:newThread("Scheduler Thread",function() func = THREAD:newFunction(function(a,b,c) assert(a == 3, "First argument expected '3' got '".. a .."'!") - assert(b == 2, "Second argument expected '2' got '".. a .."'!") - assert(c == 1, "Third argument expected '1' got '".. a .."'!") + assert(b == 2, "Second argument expected '2' got '".. b .."'!") + assert(c == 1, "Third argument expected '1' got '".. c .."'!") return 1, 2, 3, {"a table"} end, true) -- Hold this a, b, c, d = func(3,2,1) - + print(a, b, c, d) assert(a == 1, "First return was not '1'!") assert(b == 2, "Second return was not '2'!") assert(c == 3, "Third return was not '3'!") @@ -194,6 +194,9 @@ multi:newThread("Scheduler Thread",function() multi:newSystemThread("Testing proxy copy THREAD",function(tloop) local multi, thread = require("multi"):init() + for i,v in pairs(tloop.funcs) do + print(i,v) + end tloop = tloop:init() multi.print("tloop type:",tloop.Type) multi.print("Testing proxies on other threads")