Working on 16.0.0 (#53)

* Fixed spelling, started ideaing for 16.0.0

* Updated files

* Updated readme

* Updated version

* Concat conns now properly transfer events

* Testing types

* Connections can be % with functions

* Updated connections

* Fixed issue with double thread activations (Looking for another solution)

* Working on issue with love threaded functions not waiting when in a thread

* Working on issue where threads created in threads don't work

* Fixed broken threads for love

* Fixed some issues with threads

* removed test

* Updated changes.md

* Plan on testing parity between the threading modules

* Writing tests for system threading

* Added test cases for threading, fixed issues. Todo test love2d

* Fixed love2d to succeed with tests

* All tests working

* Updated files for testing

* Modified tests to make it more seamless

* removed extra __cores in lanes/pseudo

* Working on new priority scheme

* Working on priority management

* Working on custom prioritySchemes

* Fixed issues with missing code

* Threaded processors

* THREAD.exposeENV(), thread:newProcessor()

* Typo in changes.md

* Fixed typo in pseudoManager

* fixing

* Trying to fix exposeENV with pseudoThreading

* Changes to threads

* updated changes.md

* Working on systemthreadedprocess, and experimental newProxy for threading

* newProxy and STP work

* newProxy implemented

* Proxies work with connections now :D

* Added tstep to STP, updated changes.md

* thread.hold(proxy.conn)

* Clean up connection events when holding, working on scheduling tasks/threads to system threaded processors

* Getting loads of processors implemented

* Finished getLoad(type)

* Fixed some bugs

* Added an easy way to share a table, found some limitations with lanes threading.

* THREAD_NAME set for main thread, connections break the rules for proxies

* Testing

* Really close to portable proxies, currently extreamly unstable!

* Debugging what is going on...

* Fixed critical issue with coroutine based threads

* Removed extra bloat, proxies are portable now!

* Started work on the debugManager

* Testing actions, fixing bugs with lanes

* Testing...

* fixing actions

* typo fixed

* Throw an error when things break

* fixing stuff

* Fixed issue with errors not going through

* Removed system threaded connections, soon to be replaced by proxies

* Testing love2d tests

* Test love2d

* Use later love-build

* Use ubuntu for build

* Fixed path

* Use appimage

* use sudo

* No window for love2d

* Fixed love2d tests

* Testing love2d

* Use workspace

* Moved other tests while testing

* actually pull the repo

* packagepath set

* Fixed pull

* Update multi

* Removed link

* Edited symlink

* Added timeout to build

* Rewriting loveManager, too much outdated code

* Still implementing new love2d threading code

* Rewriting love2d threading binding

* Working on adding a Hold method to all objects. Will document how they all work when done.

* jobqueues having isues with stp

* new pack/unpack for tables, current issue is things being turned into strings

* Fixed packing of values into threads, need to fix system proxies and system threaded processors

* testing...

* Not hard crashing when error is encountered

* Should now push non 0 exit codes

* Push error when an error happens

* Closer to getting things working...

* Working on new type system, planning out debugmanager

* Fixed error code issue

* Test for 5.1

* Planning out debugManager

* Some work on the debug manager, proxies working on lanes, todo get pseudo manager and love2d working

* Working on getting pseudoThreading tests to work

* Added function / connection

* Added boost method

* Document new features to conns, todo fix newTask

* Fixed newTask()

* Updated changes.md and fixed some bugs

* Added thread.defer(func)

* Fixed tests on lanes and pseudo threading, todo fix love2d threading

* Fixed paths

* Working on paths

* Testing paths

* Add test for rockspec

* Fixed issues

* Fixed typo

* Added test for defer

* Threading working in love2d

* Fixed, conf

* lanes uses a threaded function like waitfor function

* Cleaned up changes.md

* added priorityManager to rockspec

integration/lanesManager/extensions.lua

@@ -22,29 +22,57 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 ]]
 local multi, thread = require("multi"):init()
+
 if not (GLOBAL and THREAD) then
-	local GLOBAL, THREAD = multi.integration.GLOBAL,multi.integration.THREAD
+	GLOBAL, THREAD = multi.integration.GLOBAL, multi.integration.THREAD
 else
 	lanes = require("lanes")
 end
+
 function multi:newSystemThreadedQueue(name)
 	local name = name or multi.randomString(16)
 	local c = {}
 	c.Name = name
 	c.linda = lanes.linda()
+	c.Type = multi.registerType("s_queue")
+
 	function c:push(v)
 		self.linda:send("Q", v)
 	end
+
 	function c:pop()
 		return ({self.linda:receive(0, "Q")})[2]
 	end
+
 	function c:peek()
 		return self.linda:get("Q")
 	end
+
 	function c:init()
 		return self
 	end
-	GLOBAL[name or "_"] = c
+
+	if multi.isMainThread then
+		multi.integration.GLOBAL[name] = c
+	else
+		GLOBAL[name] = c
+	end
+
+	function c:Hold(opt)
+		local multi, thread = require("multi"):init()
+        if opt.peek then
+            return thread.hold(function()
+                return self:peek()
+            end)
+        else
+            return thread.hold(function()
+                return self:pop()
+            end)
+        end
+	end
+
+	self:create(c)
+
 	return c
 end
 
@@ -53,37 +81,60 @@ function multi:newSystemThreadedTable(name)
     local c = {}
     c.link = lanes.linda()
 	c.Name = name
-    setmetatable(c,{
+	c.Type = multi.registerType("s_table")
+
+    function c:init()
+        return self
+    end
+	
+	setmetatable(c,{
         __index = function(t,k)
             return c.link:get(k)
         end,
         __newindex = function(t,k,v)
-            c.link:set(k,v)
+            c.link:set(k, v)
         end
     })
-    function c:init()
-        return self
+
+    if multi.isMainThread then
+		multi.integration.GLOBAL[name] = c
+	else
+		GLOBAL[name] = c
+	end
+
+	function c:Hold(opt)
+		local multi, thread = require("multi"):init()
+        if opt.key then
+            return thread.hold(function()
+                return self.tab[opt.key]
+            end)
+        else
+            multi.error("Must provide a key to check opt.key = 'key'")
+        end
     end
-    GLOBAL[name or "_"] = c
+
+	self:create(c)
+
 	return c
 end
 
 function multi:newSystemThreadedJobQueue(n)
     local c = {}
     c.cores = n or THREAD.getCores()*2
+	c.Type = multi.registerType("s_jobqueue")
     c.OnJobCompleted = multi:newConnection()
-    local funcs = multi:newSystemThreadedTable():init()
-    local queueJob = multi:newSystemThreadedQueue():init()
-    local queueReturn = multi:newSystemThreadedQueue():init()
-    local doAll = multi:newSystemThreadedQueue():init()
+    local funcs = multi:newSystemThreadedTable()
+    local queueJob = multi:newSystemThreadedQueue()
+    local queueReturn = multi:newSystemThreadedQueue()
+    local doAll = multi:newSystemThreadedQueue()
     local ID=1
     local jid = 1
     function c:isEmpty()
         return queueJob:peek()==nil
     end
-    function c:doToAll(func)
+    function c:doToAll(func,...)
         for i=1,c.cores do
-            doAll:push{ID,func}
+            doAll:push{ID,func,...}
         end
         ID = ID + 1
         return self
@@ -93,12 +144,12 @@ function multi:newSystemThreadedJobQueue(n)
         return self
     end
     function c:pushJob(name,...)
-        queueJob:push{name,jid,{...}}
+        queueJob:push{name,jid,multi.pack(...)}
         jid = jid + 1
         return jid-1
     end
     local nFunc = 0
-    function c:newFunction(name,func,holup) -- This registers with the queue
+    function c:newFunction(name, func, holup) -- This registers with the queue
         if type(name)=="function" then
             holup = func
             func = name
@@ -112,32 +163,38 @@ function multi:newSystemThreadedJobQueue(n)
             local rets
             link = c.OnJobCompleted(function(jid,...)
                 if id==jid then
-                    rets = {...}
-                    link:Destroy()
+                    rets = multi.pack(...)
                 end
             end)
             return thread.hold(function()
                 if rets then
-                    return unpack(rets) or multi.NIL
+					if #rets == 0 then
+						return multi.NIL
+					else
+                    	return multi.unpack(rets)
+					end
                 end
             end)
-        end,holup),name
+        end, holup), name
     end
     thread:newThread("JobQueueManager",function()
         while true do
             local job = thread.hold(function()
                 return queueReturn:pop()
             end)
-            local id = table.remove(job,1)
-            c.OnJobCompleted:Fire(id,unpack(job))
+			if job then
+				local id = table.remove(job,1)
+				c.OnJobCompleted:Fire(id,multi.unpack(job))
+			end
         end
     end)
     for i=1,c.cores do
-        multi:newSystemThread("SystemThreadedJobQueue",function(queue)
-            local multi,thread = require("multi"):init()
+        multi:newSystemThread("STJQ_"..multi.randomString(8),function(queue)
+            local multi, thread = require("multi"):init()
             local idle = os.clock()
             local clock = os.clock
             local ref = 0
+			_G["__QR"] = queueReturn
             setmetatable(_G,{__index = funcs})
             thread:newThread("JobHandler",function()
                 while true do
@@ -145,10 +202,12 @@ function multi:newSystemThreadedJobQueue(n)
                         return queueJob:pop()
                     end)
                     idle = clock()
-                    local name = table.remove(dat,1)
-                    local jid = table.remove(dat,1)
-                    local args = table.remove(dat,1)
-                    queueReturn:push{jid, funcs[name](unpack(args)),queue}
+					thread:newThread("JobQueue-Spawn",function()
+						local name = table.remove(dat, 1)
+						local jid = table.remove(dat, 1)
+						local args = table.remove(dat, 1)
+						queueReturn:push{jid, funcs[name](args[1],args[2],args[3],args[4],args[5],args[6],args[7],args[8]), queue}
+					end)
                 end
             end)
             thread:newThread("DoAllHandler",function()
@@ -158,9 +217,10 @@ function multi:newSystemThreadedJobQueue(n)
                     end)
                     if dat then
                         if dat[1]>ref then
+							ref = table.remove(dat, 1)
+							func = table.remove(dat, 1)
                             idle = clock()
-                            ref = dat[1]
-                            dat[2]()
+                            func(unpack(dat))
                             doAll:pop()
                         end
                     end
@@ -175,14 +235,22 @@ function multi:newSystemThreadedJobQueue(n)
                 end
             end)
             multi:mainloop()
-        end,i).priority = thread.Priority_Core
+        end,i)
     end
+
+	function c:Hold(opt)
+        return thread.hold(self.OnJobCompleted)
+    end
+
+	self:create(c)
+
     return c
 end
 
 function multi:newSystemThreadedConnection(name)
 	local name = name or multi.randomString(16)
 	local c = {}
+	c.Type = multi.registerType("s_connection")
 	c.CONN = 0x00
 	c.TRIG = 0x01
 	c.PING = 0x02
@@ -196,7 +264,7 @@ function multi:newSystemThreadedConnection(name)
 		end
 		return r
 	end
-	c.CID = THREAD.getID()
+	c.CID = THREAD_ID
 	c.subscribe = multi:newSystemThreadedQueue("SUB_STC_"..self.Name):init()
 	c.Name = name
 	c.links = {} -- All triggers sent from main connection. When a connection is triggered on another thread, they speak to the main then send stuff out.
@@ -233,7 +301,7 @@ function multi:newSystemThreadedConnection(name)
 
 	local function fire(...)
 		for _, link in pairs(c.links) do
-			link:push {c.TRIG, {...}}
+			link:push {c.TRIG, multi.pack(...)}
 		end
 	end
 
@@ -253,16 +321,16 @@ function multi:newSystemThreadedConnection(name)
 				end)
 				c.links[#c.links+1] = item[2]
 			elseif item[1] == c.TRIG then
-				fire(unpack(item[2]))
-				c.proxy_conn:Fire(unpack(item[2]))
+				fire(multi.unpack(item[2]))
+				c.proxy_conn:Fire(multi.unpack(item[2]))
 			end
 		end
 	end)
 	--- ^^^ This will only exist in the init thread
 
 	function c:Fire(...)
-		local args = {...}
-		if self.CID == THREAD.getID() then -- Host Call
+		local args = multi.pack(...)
+		if self.CID == THREAD_ID then -- Host Call
 			for _, link in pairs(self.links) do
 				link:push {self.TRIG, args}
 			end
@@ -277,8 +345,14 @@ function multi:newSystemThreadedConnection(name)
 		self.links = {}
 		self.proxy_conn = multi:newConnection()
 		local mt = getmetatable(self.proxy_conn)
-		setmetatable(self, {__index = self.proxy_conn, __call = function(t,func) self.proxy_conn(func) end, __add = mt.__add})
-		if self.CID == THREAD.getID() then return self end
+		local tempMT = {}
+		for i,v in pairs(mt) do
+			tempMT[i] = v
+		end
+		tempMT.__index = self.proxy_conn
+		tempMT.__call = function(t,func) self.proxy_conn(func) end
+		setmetatable(self, tempMT)
+		if self.CID == THREAD_ID then return self end
 		thread:newThread("STC_CONN_MAN"..name,function()
 			local item
 			local link_self_ref = multi:newSystemThreadedQueue()
@@ -296,7 +370,7 @@ function multi:newSystemThreadedConnection(name)
 					end
 					link_self_ref:pop()
 				elseif item[1] == self.TRIG then
-					self.proxy_conn:Fire(unpack(item[2]))
+					self.proxy_conn:Fire(multi.unpack(item[2]))
 					link_self_ref:pop()
 				else
 					-- This shouldn't be the case
@@ -306,7 +380,14 @@ function multi:newSystemThreadedConnection(name)
 		return self
 	end
 
-	GLOBAL[name] = c
+	if multi.isMainThread then
+		multi.integration.GLOBAL[name] = c
+	else
+		GLOBAL[name] = c
+	end
+
+	self:create(c)
 
 	return c
-end
+end
+require("multi.integration.sharedExtensions")

integration/lanesManager/init.lua

@@ -36,6 +36,9 @@ lanes = require("lanes").configure()
 multi.SystemThreads = {}
 multi.isMainThread = true
 
+_G.THREAD_NAME = "MAIN_THREAD"
+_G.THREAD_ID = 0
+
 function multi:canSystemThread()
 	return true
 end
@@ -55,10 +58,10 @@ local count = 1
 local started = false
 local livingThreads = {}
 
-function THREAD:newFunction(func,holdme)
+function THREAD:newFunction(func, holdme)
 	return thread:newFunctionBase(function(...)
 		return multi:newSystemThread("TempSystemThread",func,...)
-	end,holdme)()
+	end, holdme, multi.registerType("s_function"))()
 end
 
 function multi:newSystemThread(name, func, ...)
@@ -67,35 +70,43 @@ function multi:newSystemThread(name, func, ...)
 	local rand = math.random(1, 10000000)
 	local return_linda = lanes.linda()
 	c = {}
-	c.name = name
 	c.Name = name
-	c.Id = count
+	c.ID = count
 	c.loadString = {"base","package","os","io","math","table","string","coroutine"}
 	livingThreads[count] = {true, name}
 	c.returns = return_linda
-	c.Type = "sthread"
+	c.Type = multi.registerType("s_thread")
 	c.creationTime = os.clock()
 	c.alive = true
 	c.priority = THREAD.Priority_Normal
 	local multi_settings = multi.defaultSettings
-	for i,v in pairs(multi_settings) do
-		print(i,v)
+	local globe = {
+		THREAD_NAME = name,
+		THREAD_ID = count,
+		THREAD = THREAD,
+		GLOBAL = GLOBAL,
+		_Console = __ConsoleLinda,
+		_DEFER = {}
+	}
+	if GLOBAL["__env"] then
+		for i,v in pairs(GLOBAL["__env"]) do
+			globe[i] = v
+		end
 	end
 	c.thread = lanes.gen("*",
 	{
-		globals={ -- Set up some globals
-			THREAD_NAME = name,
-			THREAD_ID = count,
-			THREAD = THREAD,
-			GLOBAL = GLOBAL,
-			_Console = __ConsoleLinda
-		},
-		priority=c.priority
+		globals = globe,
+		priority = c.priority
 	},function(...)
-		require("multi"):init(multi_settings)
+		multi, thread = require("multi"):init(multi_settings)
 		require("multi.integration.lanesManager.extensions")
+		require("multi.integration.sharedExtensions")
 		local has_error = true
-		return_linda:set("returns",{func(...)})
+		returns = {pcall(func, ...)}
+		return_linda:set("returns", returns)
+		for i,v in pairs(_DEFER) do
+			pcall(v)
+		end
 		has_error = false
 	end)(...)
 	count = count + 1
@@ -110,10 +121,20 @@ function multi:newSystemThread(name, func, ...)
 	c.OnDeath = multi:newConnection()
 	c.OnError = multi:newConnection()
 	GLOBAL["__THREADS__"] = livingThreads
+	c.OnError(multi.error)
+
+	if self.isActor then
+		self:create(c)
+	else
+		multi.create(multi, c)
+	end
+
 	return c
 end
 
-THREAD.newSystemThread = multi.newSystemThread
+THREAD.newSystemThread = function(...)
+    multi:newSystemThread(...)
+end
 
 function multi.InitSystemThreadErrorHandler()
 	if started == true then
@@ -126,18 +147,26 @@ function multi.InitSystemThreadErrorHandler()
 		while true do
 			thread.yield()
 			_,data = __ConsoleLinda:receive(0, "Q")
-			if data then print(unpack(data)) end
+			if data then
+				--print(data[1])
+			end
 			for i = #threads, 1, -1 do
 				temp = threads[i]
 				status = temp.thread.status
-				push = __StatusLinda:get(temp.Id)
+				push = __StatusLinda:get(temp.ID)
 				if push then
-					temp.statusconnector:Fire(unpack(({__StatusLinda:receive(nil, temp.Id)})[2]))
+					temp.statusconnector:Fire(multi.unpack(({__StatusLinda:receive(nil, temp.ID)})[2]))
 				end
 				if status == "done" or temp.returns:get("returns") then
-					livingThreads[temp.Id] = {false, temp.Name}
+					returns = ({temp.returns:receive(0, "returns")})[2]
+					livingThreads[temp.ID] = {false, temp.Name}
 					temp.alive = false
-					temp.OnDeath:Fire(unpack(({temp.returns:receive(0, "returns")})[2]))
+					if returns[1] == false then
+						temp.OnError:Fire(temp, returns[2])
+					else
+						table.remove(returns,1)
+						temp.OnDeath:Fire(multi.unpack(returns))
+					end
 					GLOBAL["__THREADS__"] = livingThreads
 					table.remove(threads, i)
 				elseif status == "running" then
@@ -145,19 +174,15 @@ function multi.InitSystemThreadErrorHandler()
 				elseif status == "waiting" then
 					--
 				elseif status == "error" then
-					livingThreads[temp.Id] = {false, temp.Name}
-					temp.alive = false
-					temp.OnError:Fire(temp,unpack(temp.returns:receive(0,"returns") or {"Thread Killed!"}))
-					GLOBAL["__THREADS__"] = livingThreads
-					table.remove(threads, i)
+					-- The thread never really errors, we handle this through our linda object
 				elseif status == "cancelled" then
-					livingThreads[temp.Id] = {false, temp.Name}
+					livingThreads[temp.ID] = {false, temp.Name}
 					temp.alive = false
 					temp.OnError:Fire(temp,"thread_cancelled")
 					GLOBAL["__THREADS__"] = livingThreads
 					table.remove(threads, i)
 				elseif status == "killed" then
-					livingThreads[temp.Id] = {false, temp.Name}
+					livingThreads[temp.ID] = {false, temp.Name}
 					temp.alive = false
 					temp.OnError:Fire(temp,"thread_killed")
 					GLOBAL["__THREADS__"] = livingThreads
@@ -165,10 +190,10 @@ function multi.InitSystemThreadErrorHandler()
 				end
 			end
 		end
-	end)
+	end).OnError(multi.error)
 end
 
-multi.print("Integrated Lanes!")
+multi.print("Integrated Lanes Threading!")
 multi.integration = {} -- for module creators
 multi.integration.GLOBAL = GLOBAL
 multi.integration.THREAD = THREAD

integration/lanesManager/threads.lua

@@ -48,20 +48,12 @@ local function INIT(__GlobalLinda, __SleepingLinda, __StatusLinda, __Console)
     end
 
     function THREAD.waitFor(name)
-        local function wait()
+        local multi, thread = require("multi"):init()
+        return multi.hold(function()
             math.randomseed(os.time())
             __SleepingLinda:receive(.001, "__non_existing_variable")
-        end
-        repeat
-            wait()
-        until __GlobalLinda:get(name)
-        return __GlobalLinda:get(name)
-    end
-
-    if getOS() == "windows" then
-        THREAD.__CORES = tonumber(os.getenv("NUMBER_OF_PROCESSORS"))
-    else
-        THREAD.__CORES = tonumber(io.popen("nproc --all"):read("*n"))
+            return __GlobalLinda:get(name)
+        end)
     end
 
     function THREAD.getCores()
@@ -72,11 +64,11 @@ local function INIT(__GlobalLinda, __SleepingLinda, __StatusLinda, __Console)
         local c = {}
         c.queue = __Console
         function c.print(...)
-            c.queue:send("Q", {...})
+            c.queue:push("Q", table.concat(multi.pack(...), "\t"))
         end
         function c.error(err)
-            c.queue:push("Q",{"ERROR in <"..__THREADNAME__..">: "..err,__THREADID__})
-            error(err)
+            c.queue:push("Q", "Error in <"..THREAD_NAME..":" .. THREAD_ID .. ">: ".. err)
+            multi.error(err)
         end
         return c
     end
@@ -95,16 +87,12 @@ local function INIT(__GlobalLinda, __SleepingLinda, __StatusLinda, __Console)
         error("Thread was killed!\1")
     end
 
-    function THREAD.getName()
-        return THREAD_NAME
-    end
-
-    function THREAD.getID()
-        return THREAD_ID
+    function THREAD.sync()
+        -- Maybe do something...
     end
 	
     function THREAD.pushStatus(...)
-        local args = {...}
+        local args = multi.pack(...)
         __StatusLinda:send(nil,THREAD_ID, args)
     end
 
@@ -134,9 +122,30 @@ local function INIT(__GlobalLinda, __SleepingLinda, __StatusLinda, __Console)
 			__GlobalLinda:set(k, v)
 		end
 	})
+
+    function THREAD.setENV(env, name)
+        GLOBAL[name or "__env"] = env
+    end
+
+    function THREAD.getENV(name)
+        return GLOBAL[name or "__env"]
+    end
+
+    function THREAD.exposeENV(name)
+        name = name or "__env"
+        local env = THREAD.getENV(name)
+        for i,v in pairs(env) do
+            _G[i] = v
+        end
+    end
+
+    function THREAD.defer(func)
+        table.insert(_DEFER, func)
+    end
+
     return GLOBAL, THREAD
 end
 
-return {init = function(g,s,st,c)
-    return INIT(g,s,st,c)
+return {init = function(g,s,st,c,onexit)
+    return INIT(g,s,st,c,onexit)
 end}