new pack/unpack for tables, current issue is things being turned into strings

init.lua

@@ -1287,7 +1287,7 @@ function thread.hold(n, opt)
 	elseif type(n) == "function" then
 		return yield(CMD, t_hold, n, nil, interval)
 	else
-		multi.error("Invalid argument passed to thread.hold(...)!")
+		multi.error("Invalid argument passed to thread.hold(...) ".. type(n) .. "!")
 	end
 end
 

integration/loveManager/extensions.lua

@@ -11,7 +11,7 @@ function multi:newSystemThreadedQueue(name)
 
 	c.Name = name
 	c.Type = multi.SQUEUE
-    c.chan = love.thread.getChannel(name)
+    c.chan = love.thread.newChannel()
 
     function c:push(dat)
         self.chan:push(THREAD.packValue(dat))
@@ -26,7 +26,6 @@ function multi:newSystemThreadedQueue(name)
     end
 
     function c:init()
-        self.chan = love.thread.getChannel(self.Name)
         return self
     end
 

integration/loveManager/init.lua

@@ -4,10 +4,11 @@ end
 
 local ThreadFileData = [[
 ISTHREAD = true
-__FUNC, THREAD_ID, THREAD_NAME, __PACK = ...
+args = {...}
+THREAD_ID = table.remove(args, 1)
+THREAD_NAME = table.remove(args, 1)
 GLOBAL, THREAD = require("multi.integration.loveManager.threads"):init()
-__FUNC = THREAD.unpackValue(__FUNC)
+__FUNC = THREAD.unpackValue(table.remove(args, 1))
-__PACK = THREAD.unpackValue(__PACK)
 math.randomseed(THREAD_ID)
 math.random()
 math.random()
@@ -22,7 +23,7 @@ end
 multi, thread = require("multi"):init()
 require("multi.integration.loveManager.extensions")
 require("multi.integration.sharedExtensions")
-stab["returns"] = {__FUNC(multi.unpack(__PACK))}
+stab["returns"] = {__FUNC(multi.unpack(args))}
 ]]
 
 _G.THREAD_NAME = "MAIN_THREAD"
@@ -44,7 +45,7 @@ function multi:newSystemThread(name, func, ...)
     c.Name = name
     c.ID = tid
     c.thread = love.thread.newThread(ThreadFileData)
-    c.thread:start(THREAD.packValue(func), c.ID, c.Name, THREAD.packValue({...}))
+    c.thread:start(c.ID, c.Name, THREAD.packValue(func), ...)
     c.stab = THREAD.createTable(name .. c.ID)
     c.creationTime = os.clock()
     c.OnDeath = multi:newConnection()

integration/loveManager/serpent.lua

@@ -1,176 +0,0 @@
---[[
-Serpent source is released under the MIT License
-
-Copyright (c) 2012-2018 Paul Kulchenko (paul@kulchenko.com)
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-]]
-
-local n, v = "serpent", "0.303" -- (C) 2012-18 Paul Kulchenko; MIT License
-local c, d = "Paul Kulchenko", "Lua serializer and pretty printer"
-local snum = {[tostring(1/0)]='1/0 --[[math.huge]]',[tostring(-1/0)]='-1/0 --[[-math.huge]]',[tostring(0/0)]='0/0'}
-local badtype = {thread = true, userdata = true, cdata = true}
-local getmetatable = debug and debug.getmetatable or getmetatable
-local pairs = function(t) return next, t end -- avoid using __pairs in Lua 5.2+
-local keyword, globals, G = {}, {}, (_G or _ENV)
-for _,k in ipairs({'and', 'break', 'do', 'else', 'elseif', 'end', 'false',
-  'for', 'function', 'goto', 'if', 'in', 'local', 'nil', 'not', 'or', 'repeat',
-  'return', 'then', 'true', 'until', 'while'}) do keyword[k] = true end
-for k,v in pairs(G) do globals[v] = k end -- build func to name mapping
-for _,g in ipairs({'coroutine', 'debug', 'io', 'math', 'string', 'table', 'os'}) do
-  for k,v in pairs(type(G[g]) == 'table' and G[g] or {}) do globals[v] = g..'.'..k end end
-
-local function s(t, opts)
-  local name, indent, fatal, maxnum = opts.name, opts.indent, opts.fatal, opts.maxnum
-  local sparse, custom, huge = opts.sparse, opts.custom, not opts.nohuge
-  local space, maxl = (opts.compact and '' or ' '), (opts.maxlevel or math.huge)
-  local maxlen, metatostring = tonumber(opts.maxlength), opts.metatostring
-  local iname, comm = '_'..(name or ''), opts.comment and (tonumber(opts.comment) or math.huge)
-  local numformat = opts.numformat or "%.17g"
-  local seen, sref, syms, symn = {}, {'local '..iname..'={}'}, {}, 0
-  local function gensym(val) return '_'..(tostring(tostring(val)):gsub("[^%w]",""):gsub("(%d%w+)",
-    -- tostring(val) is needed because __tostring may return a non-string value
-    function(s) if not syms[s] then symn = symn+1; syms[s] = symn end return tostring(syms[s]) end)) end
-  local function safestr(s) return type(s) == "number" and (huge and snum[tostring(s)] or numformat:format(s))
-    or type(s) ~= "string" and tostring(s) -- escape NEWLINE/010 and EOF/026
-    or ("%q"):format(s):gsub("\010","n"):gsub("\026","\\026") end
-  -- handle radix changes in some locales
-  if opts.fixradix and (".1f"):format(1.2) ~= "1.2" then
-    local origsafestr = safestr
-    safestr = function(s) return type(s) == "number"
-      and (nohuge and snum[tostring(s)] or numformat:format(s):gsub(",",".")) or origsafestr(s)
-    end
-  end
-  local function comment(s,l) return comm and (l or 0) < comm and ' --[['..select(2, pcall(tostring, s))..']]' or '' end
-  local function globerr(s,l) return globals[s] and globals[s]..comment(s,l) or not fatal
-    and safestr(select(2, pcall(tostring, s))) or error("Can't serialize "..tostring(s)) end
-  local function safename(path, name) -- generates foo.bar, foo[3], or foo['b a r']
-    local n = name == nil and '' or name
-    local plain = type(n) == "string" and n:match("^[%l%u_][%w_]*$") and not keyword[n]
-    local safe = plain and n or '['..safestr(n)..']'
-    return (path or '')..(plain and path and '.' or '')..safe, safe end
-  local alphanumsort = type(opts.sortkeys) == 'function' and opts.sortkeys or function(k, o, n) -- k=keys, o=originaltable, n=padding
-    local maxn, to = tonumber(n) or 12, {number = 'a', string = 'b'}
-    local function padnum(d) return ("%0"..tostring(maxn).."d"):format(tonumber(d)) end
-    table.sort(k, function(a,b)
-      -- sort numeric keys first: k[key] is not nil for numerical keys
-      return (k[a] ~= nil and 0 or to[type(a)] or 'z')..(tostring(a):gsub("%d+",padnum))
-           < (k[b] ~= nil and 0 or to[type(b)] or 'z')..(tostring(b):gsub("%d+",padnum)) end) end
-  local function val2str(t, name, indent, insref, path, plainindex, level)
-    local ttype, level, mt = type(t), (level or 0), getmetatable(t)
-    local spath, sname = safename(path, name)
-    local tag = plainindex and
-      ((type(name) == "number") and '' or name..space..'='..space) or
-      (name ~= nil and sname..space..'='..space or '')
-    if seen[t] then -- already seen this element
-      sref[#sref+1] = spath..space..'='..space..seen[t]
-      return tag..'nil'..comment('ref', level)
-    end
-    -- protect from those cases where __tostring may fail
-    if type(mt) == 'table' and metatostring ~= false then
-      local to, tr = pcall(function() return mt.__tostring(t) end)
-      local so, sr = pcall(function() return mt.__serialize(t) end)
-      if (to or so) then -- knows how to serialize itself
-        seen[t] = insref or spath
-        t = so and sr or tr
-        ttype = type(t)
-      end -- new value falls through to be serialized
-    end
-    if ttype == "table" then
-      if level >= maxl then return tag..'{}'..comment('maxlvl', level) end
-      seen[t] = insref or spath
-      if next(t) == nil then return tag..'{}'..comment(t, level) end -- table empty
-      if maxlen and maxlen < 0 then return tag..'{}'..comment('maxlen', level) end
-      local maxn, o, out = math.min(#t, maxnum or #t), {}, {}
-      for key = 1, maxn do o[key] = key end
-      if not maxnum or #o < maxnum then
-        local n = #o -- n = n + 1; o[n] is much faster than o[#o+1] on large tables
-        for key in pairs(t) do
-          if o[key] ~= key then n = n + 1; o[n] = key end
-        end
-      end
-      if maxnum and #o > maxnum then o[maxnum+1] = nil end
-      if opts.sortkeys and #o > maxn then alphanumsort(o, t, opts.sortkeys) end
-      local sparse = sparse and #o > maxn -- disable sparsness if only numeric keys (shorter output)
-      for n, key in ipairs(o) do
-        local value, ktype, plainindex = t[key], type(key), n <= maxn and not sparse
-        if opts.valignore and opts.valignore[value] -- skip ignored values; do nothing
-        or opts.keyallow and not opts.keyallow[key]
-        or opts.keyignore and opts.keyignore[key]
-        or opts.valtypeignore and opts.valtypeignore[type(value)] -- skipping ignored value types
-        or sparse and value == nil then -- skipping nils; do nothing
-        elseif ktype == 'table' or ktype == 'function' or badtype[ktype] then
-          if not seen[key] and not globals[key] then
-            sref[#sref+1] = 'placeholder'
-            local sname = safename(iname, gensym(key)) -- iname is table for local variables
-            sref[#sref] = val2str(key,sname,indent,sname,iname,true)
-          end
-          sref[#sref+1] = 'placeholder'
-          local path = seen[t]..'['..tostring(seen[key] or globals[key] or gensym(key))..']'
-          sref[#sref] = path..space..'='..space..tostring(seen[value] or val2str(value,nil,indent,path))
-        else
-          out[#out+1] = val2str(value,key,indent,nil,seen[t],plainindex,level+1)
-          if maxlen then
-            maxlen = maxlen - #out[#out]
-            if maxlen < 0 then break end
-          end
-        end
-      end
-      local prefix = string.rep(indent or '', level)
-      local head = indent and '{\n'..prefix..indent or '{'
-      local body = table.concat(out, ','..(indent and '\n'..prefix..indent or space))
-      local tail = indent and "\n"..prefix..'}' or '}'
-      return (custom and custom(tag,head,body,tail,level) or tag..head..body..tail)..comment(t, level)
-    elseif badtype[ttype] then
-      seen[t] = insref or spath
-      return tag..globerr(t, level)
-    elseif ttype == 'function' then
-      seen[t] = insref or spath
-      if opts.nocode then return tag.."function() --[[..skipped..]] end"..comment(t, level) end
-      local ok, res = pcall(string.dump, t)
-      local func = ok and "((loadstring or load)("..safestr(res)..",'@serialized'))"..comment(t, level)
-      return tag..(func or globerr(t, level))
-    else return tag..safestr(t) end -- handle all other types
-  end
-  local sepr = indent and "\n" or ";"..space
-  local body = val2str(t, name, indent) -- this call also populates sref
-  local tail = #sref>1 and table.concat(sref, sepr)..sepr or ''
-  local warn = opts.comment and #sref>1 and space.."--[[incomplete output with shared/self-references skipped]]" or ''
-  return not name and body..warn or "do local "..body..sepr..tail.."return "..name..sepr.."end"
-end
-
-local function deserialize(data, opts)
-  local env = (opts and opts.safe == false) and G
-    or setmetatable({}, {
-        __index = function(t,k) return t end,
-        __call = function(t,...) error("cannot call functions") end
-      })
-  local f, res = (loadstring or load)('return '..data, nil, nil, env)
-  if not f then f, res = (loadstring or load)(data, nil, nil, env) end
-  if not f then return f, res end
-  if setfenv then setfenv(f, env) end
-  return pcall(f)
-end
-
-local function merge(a, b) if b then for k,v in pairs(b) do a[k] = v end end; return a; end
-return { _NAME = n, _COPYRIGHT = c, _DESCRIPTION = d, _VERSION = v, serialize = s,
-  load = deserialize,
-  dump = function(a, opts) return s(a, merge({name = '_', compact = true, sparse = true}, opts)) end,
-  line = function(a, opts) return s(a, merge({sortkeys = true, comment = true}, opts)) end,
-  block = function(a, opts) return s(a, merge({indent = '  ', sortkeys = true, comment = true}, opts)) end }

integration/loveManager/threads.lua

@@ -25,53 +25,19 @@ require("love.timer")
 require("love.system")
 require("love.data")
 require("love.thread")
-local serpent = require("multi.integration.loveManager.serpent")
+local utils = require("multi.integration.loveManager.utils")
 local multi, thread = require("multi"):init()
 
-local function ltype(data)
-    local tp = type(data)
-    if tp == "userdata" then
-        return data:type()
-    end
-    return tp
-end
-
 local NIL = love.data.newByteData("\3")
 
 -- If a non table/function is supplied we just return it
 local function packValue(t)
-    local tp = type(t)
+    return utils.pack(t)
-    if tp == "table" then
-        return love.data.newByteData("\1"..serpent.dump(t,{safe = true}))
-    elseif tp == "function" then
-        return love.data.newByteData("\2"..serpent.dump({t,true},{safe = true}))
-    else
-        return t
-    end
 end
 
 -- If a non table/function is supplied we just return it
 local function unpackValue(d)
-    if ltype(d) == "ByteData" then
+    return utils.unpack(d)
-        local data = d:getString()
-        if data:sub(1, 1) == "\1" then
-            local status, data = serpent.load(data:sub(2,-1),{safe = false})
-            if not status then
-                multi.error(data)
-            end
-            return data
-        elseif data:sub(1, 1) =="\2" then
-            local status, data = serpent.load(data:sub(2,-1),{safe = false})
-            if not status then
-                multi.error(data)
-            end
-            return data[1]
-        else
-            return d
-        end
-    else
-        return d
-    end
 end
 
 local function createTable(n)
@@ -126,7 +92,7 @@ function INIT()
         end
         repeat
             wait()
-        until GLOBAL[name]
+        until GLOBAL[name] ~= nil
         if type(GLOBAL[name].init) == "function" then
             return GLOBAL[name]:init()
         else

integration/loveManager/utils.lua

@@ -0,0 +1,55 @@
+require("love.data")
+local sutils = {}
+local NIL = {Type="nil"}
+
+--love.data.newByteData("\2"..serpent.dump({t,true},{safe = true}))
+
+local ltype = function(v) return v:type() end
+local t = function(value)
+    local v = type(value)
+    if v == "userdata" then
+        local status, return_or_err = pcall(ltype, value)
+        if status then return return_or_err else return "userdata" end
+    else return v end
+end
+
+function sutils.pack(tbl, seen)
+    if type(tbl) == "function" then return {["__$FUNC$__"] = love.data.newByteData(string.dump(tbl))} end
+    if type(tbl) ~= "table" then return tbl end
+    local seen = seen or {}
+    local result = {}
+    for i,v in pairs(tbl) do
+        if seen[v] then
+            result[i] = v
+        elseif t(v) == "table" then
+            seen[v] = true
+            result[i] = sutils.pack(v, seen)
+        elseif t(v) == "function" then
+            result["$F"..i] = love.data.newByteData(string.dump(v))
+        elseif t{v} == "userdata" then
+            result[i] = tostring(v)
+        else -- Handle what we need to and pass the rest along as a value
+            result[i] = v
+        end
+    end
+    return result
+end
+
+function sutils.unpack(tbl)
+    if type(tbl) ~= "table" then return tbl end
+    if tbl["__$FUNC$__"] then return loadstring(tbl["__$FUNC$__"]:getString()) end
+    for i,v in pairs(tbl) do
+        if type(i) == "string" and i:sub(1,2) == "$F" then
+            local rawfunc = v:getString()
+            v:release()
+            tbl[i] = nil
+            tbl[i:sub(3,-1)] = loadstring(rawfunc)
+        end
+        if type(v) == "table" then
+            sutils.unpack(v)
+        end
+    end
+    return tbl
+end
+
+return sutils

integration/sharedExtensions/init.lua

@@ -135,7 +135,6 @@ function multi:newProxy(list)
 			self.recv = THREAD.waitFor(self.name.."_R"):init()
 			self.Type = multi.PROXY
 			for _,v in pairs(funcs) do
-				print(v,_)
 				if type(v) == "table" then
 					-- We have a connection
 					v[2]:init(proc_name)
@@ -190,6 +189,7 @@ function multi:newProxy(list)
 				THREAD = multi.integration.THREAD
 			end
 			local proxy = THREAD.waitFor(self.proxy_link)
+			print("Got:",proxy)
 			proxy.funcs = self.funcs
 			return proxy:init()
 		end

lovethreads/main.lua

@@ -1,50 +1,94 @@
 package.path = "../?/init.lua;../?.lua;"..package.path
 local multi, thread = require("multi"):init{print=true, warning = true, error=true}
+local flat = require("flatten")
 
-GLOBAL, THREAD = require("multi.integration.loveManager"):init()
+local people = {
+    {
+       name = "Fred",
+       address = "16 Long Street",
+       phone = "123456"
+    },
+    {
+       name = "Wilma",
+       address = "16 Long Street",
+       phone = "123456",
+       func = function()
+           print("Hi")
+       end
+    },
+    {
+       name = "Barney",
+       address = "17 Long Street",
+       phone = "123457",
+       important = love.data.newByteData("TEST")
+    }
+ }
 
-local queue = multi:newSystemThreadedQueue("TestQueue")
+function dump(o)
-local tab = multi:newSystemThreadedTable("TestTable")
+    if type(o) == 'table' then
-
+        local s = '{ '
-local test = multi:newSystemThread("Test",function()
+        for k,v in pairs(o) do
-    local queue = THREAD.waitFor("TestQueue")
+            if type(k) ~= 'number' then k = '"'..k..'"' end
-    local tab = THREAD.waitFor("TestTable")
+            s = s .. '['..k..'] = ' .. dump(v) .. ','
-    print("THREAD_ID:",THREAD_ID)
+        end
-    queue:push("Did it work?")
+        return s .. '} '
-    tab["Test"] = true
+    else
-    return 1,2,3
+        return tostring(o)
-end)
+    end
-
-multi:newThread("QueueTest", function()
-    print(thread.hold(queue))
-    print(thread.hold(tab, {key="Test"}))
-    print("Done!")
-end)
-
-local jq = multi:newSystemThreadedJobQueue(n)
-
-jq:registerFunction("test2",function()
-    print("This works!")
-end)
-
-jq:registerFunction("test",function(a, b, c)
-    print(a, b+c)
-    test2()
-    return a+b+c
-end)
-
-print("Job:",jq:pushJob("test",1,2,3))
-print("Job:",jq:pushJob("test",2,3,4))
-print("Job:",jq:pushJob("test",5,6,7))
-
-jq.OnJobCompleted(function(...)
-    print("Job Completed!", ...)
-end)
-
-function love.draw()
-    --
 end
 
-function love.update()
+local fpeople = flat.flatten(people)
-    multi:uManager()
+
-end
+print("Flatten", dump(fpeople))
+
+local people = flat.unflatten(fpeople)
+
+print("Unflatten", dump(people))
+ 
+-- GLOBAL, THREAD = require("multi.integration.loveManager"):init()
+
+-- local queue = multi:newSystemThreadedQueue("TestQueue")
+-- local tab = multi:newSystemThreadedTable("TestTable")
+
+-- local test = multi:newSystemThread("Test",function()
+--     local queue = THREAD.waitFor("TestQueue")
+--     local tab = THREAD.waitFor("TestTable")
+--     print("THREAD_ID:",THREAD_ID)
+--     queue:push("Did it work?")
+--     tab["Test"] = true
+--     return 1,2,3
+-- end)
+
+-- multi:newThread("QueueTest", function()
+--     print(thread.hold(queue))
+--     print(thread.hold(tab, {key="Test"}))
+--     print("Done!")
+-- end)
+
+-- local jq = multi:newSystemThreadedJobQueue(n)
+
+-- jq:registerFunction("test2",function()
+--     print("This works!")
+-- end)
+
+-- jq:registerFunction("test",function(a, b, c)
+--     print(a, b+c)
+--     test2()
+--     return a+b+c
+-- end)
+
+-- print("Job:",jq:pushJob("test",1,2,3))
+-- print("Job:",jq:pushJob("test",2,3,4))
+-- print("Job:",jq:pushJob("test",5,6,7))
+
+-- jq.OnJobCompleted(function(...)
+--     print("Job Completed!", ...)
+-- end)
+
+-- function love.draw()
+--     --
+-- end
+
+-- function love.update()
+--     multi:uManager()
+-- end

tests/threadtests.lua

@@ -71,13 +71,13 @@ multi:newThread("Scheduler Thread",function()
 
     func = THREAD:newFunction(function(a,b,c)
         assert(a == 3, "First argument expected '3' got '".. a .."'!")
-        assert(b == 2, "Second argument expected '2' got '".. a .."'!")
+        assert(b == 2, "Second argument expected '2' got '".. b .."'!")
-        assert(c == 1, "Third argument expected '1' got '".. a .."'!")
+        assert(c == 1, "Third argument expected '1' got '".. c .."'!")
         return 1, 2, 3, {"a table"}
     end, true) -- Hold this
 
     a, b, c, d = func(3,2,1)
-
+    print(a, b, c, d)
     assert(a == 1, "First return was not '1'!")
     assert(b == 2, "Second return was not '2'!")
     assert(c == 3, "Third return was not '3'!")
@@ -194,6 +194,9 @@ multi:newThread("Scheduler Thread",function()
 
     multi:newSystemThread("Testing proxy copy THREAD",function(tloop)
         local multi, thread = require("multi"):init()
+        for i,v in pairs(tloop.funcs) do
+            print(i,v)
+        end
         tloop = tloop:init()
         multi.print("tloop type:",tloop.Type)
         multi.print("Testing proxies on other threads")