From 614e032aa5d2d7597c90d808cdd85603e334e351 Mon Sep 17 00:00:00 2001 From: Ryan Ward Date: Tue, 4 Jul 2023 13:05:07 -0400 Subject: [PATCH] Testing actions, fixing bugs with lanes --- .github/workflows/nix_ci.yml | 42 ++++++++++++++++++++++ .github/workflows/win_ci.yml | 48 +++++++++++++++++++++++++ docs/changes.md | 15 ++++++++ init.lua | 26 ++++++++++---- integration/effilManager/extensions.lua | 0 integration/effilManager/init.lua | 46 ++++++++++++++++++++++++ integration/effilManager/threads.lua | 0 integration/lanesManager/extensions.lua | 14 +++++++- integration/lanesManager/init.lua | 10 +++--- integration/loveManager/extensions.lua | 32 +++++++++++++++-- integration/loveManager/init.lua | 6 ++-- integration/sharedExtensions/init.lua | 1 + tests/multi | 1 + tests/runtests.lua | 10 +++++- tests/threadtests.lua | 16 +++++---- 15 files changed, 242 insertions(+), 25 deletions(-) create mode 100644 .github/workflows/nix_ci.yml create mode 100644 .github/workflows/win_ci.yml create mode 100644 integration/effilManager/extensions.lua create mode 100644 integration/effilManager/init.lua create mode 100644 integration/effilManager/threads.lua create mode 120000 tests/multi diff --git a/.github/workflows/nix_ci.yml b/.github/workflows/nix_ci.yml new file mode 100644 index 0000000..ce8c1de --- /dev/null +++ b/.github/workflows/nix_ci.yml @@ -0,0 +1,42 @@ +name: build & run tests (NIX) + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + strategy: + fail-fast: false + matrix: + build-type: [Release] # Debug + lua: ["lua 5.1", "lua 5.2", "lua 5.3", "luajit 2.1.0-beta3"] + os: ["macos-latest", "ubuntu-latest", "windows-2019"] + include: + - os: macos-latest + macos_build_target: 10.0 + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + - uses: actions/setup-python@v2 + with: + python-version: '3.8' + - name: Setup env + env: + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos_build_target }} + run: | + pip install hererocks + hererocks lua-pkg --${{ matrix.lua }} -rlatest + source ${{github.workspace}}/lua-pkg/bin/activate + + - name: Install lanes + run: | + luarocks insatll lanes + + - name: Run Tests + run: | + lua tests/runtests.lua diff --git a/.github/workflows/win_ci.yml b/.github/workflows/win_ci.yml new file mode 100644 index 0000000..800c9fb --- /dev/null +++ b/.github/workflows/win_ci.yml @@ -0,0 +1,48 @@ +name: build & run tests (Win) + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + strategy: + fail-fast: false + matrix: + build-type: [Release] # Debug + lua: ["lua 5.1", "lua 5.2", "lua 5.3", "luajit 2.0"] + os: ["windows-2019"] + platform: [ + {"forLua": "vs_32", "forCMake": "Win32"}, + {"forLua": "vs_64", "forCMake": "x64"}, + ] + exclude: + - lua: "luajit 2.0" + platform: {"forLua": "vs_32", "forCMake": "Win32"} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + - uses: actions/setup-python@v2 + with: + python-version: '3.8' + - name: Setup env + run: | + pip install hererocks + hererocks lua-pkg --${{ matrix.lua }} -rlatest --target ${{matrix.platform.forLua}} + cmd ${{github.workspace}}\lua-pkg\bin\activate + + - name: Configure CMake & build + run: | + cmake -A ${{matrix.platform.forCMake}} -B ${{github.workspace}}\build -DLUA_INCLUDE_DIR="${{github.workspace}}/lua-pkg/include" -DLUA_LIBRARY="${{github.workspace}}/lua-pkg/lib/lua*.lib" + cmake --build ${{github.workspace}}/build --config ${{matrix.build-type}} + + - name: Test + working-directory: ${{github.workspace}}/build + env: + STRESS: 1 + run: | + ${{github.workspace}}/lua-pkg/bin/lua ../tests/lua/run_tests diff --git a/docs/changes.md b/docs/changes.md index 9bb4824..e2c9f88 100644 --- a/docs/changes.md +++ b/docs/changes.md @@ -67,6 +67,21 @@ multi, thread = require("multi"):init{print=true} GLOBAL, THREAD = require("multi.integration.lanesManager"):init() ``` +## Added New Integration: **effilManager** + +Another option for multithreading support, works just like all the other threading integrations, but uses the internals of effil and it's unique features. +- Refer to this [doc](https://www.lua.org/wshop18/Kupriyanov.pdf) to read more about it. +- Project github [page](https://github.com/effil/effil/tree/master). + +```lua +package.path = "?/init.lua;?.lua;"..package.path + +local multi, thread = require("multi"):init({print=true, warn=true, error=true}) +local THREAD, GLOBAL = require("multi.integration.effilManager"):init() + +-- Code as you would +``` + ## Added New Integration: **priorityManager** Allows the user to have multi auto set priorities (Requires chronos). Also adds the functionality to create your own runners (multi:mainloop(), multi:umanager()) that you can set using the priority manager. Even if you do not have `chronos` installed all other features will still work! diff --git a/init.lua b/init.lua index e26d4f7..beb866d 100644 --- a/init.lua +++ b/init.lua @@ -60,6 +60,7 @@ function multi.setType(obj,t) }) end end + setmetatable(multi.DestroyedObj, { __index = function(t,k) return setmetatable({},{__index = uni,__newindex = uni,__call = uni,__metatable = multi.DestroyedObj,__tostring = function() return "destroyed" end,__unm = uni,__add = uni,__sub = uni,__mul = uni,__div = uni,__mod = uni,__pow = uni,__concat = uni}) @@ -68,7 +69,8 @@ setmetatable(multi.DestroyedObj, { multi.DESTROYED = multi.DestroyedObj multi.ROOTPROCESS = "rootprocess" -multi.CONNECTOR = "connector" +multi.CONNECTOR = "connector" -- To be deprecated +multi.CONNECTION = "connector" -- To be changed to connection and replace connector (v17.x,x) multi.TIMEMASTER = "timemaster" multi.PROCESS = "process" multi.TIMER = "timer" @@ -81,8 +83,18 @@ multi.STEP = "step" multi.TSTEP = "tstep" multi.THREAD = "thread" multi.SERVICE = "service" +multi.THREADEDFUNCTION = "threaded_function" -- To be deprecated +multi.FUNCTION = "threaded_function" -- To be changed to connection and replace connector (v17.x,x) + +-- Extensions multi.PROXY = "proxy" -multi.THREADEDFUNCTION = "threaded_function" +multi.STHREAD = "s_thread" +multi.SQUEUE = "s_queue" +multi.STABLE = "s_table" +multi.SJOBQUEUE = "s_jobqueue" +multi.SCONNECTION = "s_connection" +multi.SPROCESS = "s_process" +multi.SFUNCTION = "s_function" if not _G["$multi"] then _G["$multi"] = {multi = multi, thread = thread} @@ -305,7 +317,7 @@ function multi:newConnection(protect,func,kill) return cn end}) - c.Type=multi.CONNECTOR + c.Type=multi.CONNECTION c.func={} c.ID=0 local protect=protect or false @@ -1286,7 +1298,7 @@ function thread.hold(n, opt) if type(n) == "number" then thread.getRunningThread().lastSleep = clock() return yield(CMD, t_sleep, n or 0, nil, interval) - elseif type(n) == "table" and n.Type == multi.CONNECTOR then + elseif type(n) == "table" and n.Type == multi.CONNECTION then return yield(CMD, t_hold, conn_test(n), nil, interval) elseif type(n) == "table" and n.Hold ~= nil then return n:Hold(opt) @@ -1367,7 +1379,7 @@ function thread.pushStatus(...) t.statusconnector:Fire(...) end -function thread:newFunctionBase(generator, holdme) +function thread:newFunctionBase(generator, holdme, TYPE) return function() local tfunc = {} tfunc.Active = true @@ -1449,7 +1461,7 @@ function thread:newFunctionBase(generator, holdme) return temp end setmetatable(tfunc, tfunc) - tfunc.Type = multi.THREADEDFUNCTION + tfunc.Type = TYPE or multi.FUNCTION return tfunc end end @@ -2043,7 +2055,7 @@ local function doOpt() if type(n) == "number" then thread.getRunningThread().lastSleep = clock() return yield(CMD, t_sleep, n or 0, nil, interval) - elseif type(n) == "table" and n.Type == multi.CONNECTOR then + elseif type(n) == "table" and n.Type == multi.CONNECTION then local rdy = function() return false end diff --git a/integration/effilManager/extensions.lua b/integration/effilManager/extensions.lua new file mode 100644 index 0000000..e69de29 diff --git a/integration/effilManager/init.lua b/integration/effilManager/init.lua new file mode 100644 index 0000000..53818e6 --- /dev/null +++ b/integration/effilManager/init.lua @@ -0,0 +1,46 @@ +local multi, thread = require("multi"):init{error=true} +multi.error("Currntly not supported!") +os.exit() +local effil = require("effil") + +-- I like some of the things that this library offers. +-- Current limitations prevent me from being able to use effil, +-- but I might fork and work on it myself. + +-- Configs +effil.allow_table_upvalues(false) + +local GLOBAL,THREAD = require("multi.integration.effilManager.threads").init() +local count = 1 +local started = false +local livingThreads = {} + +function multi:newSystemThread(name, func, ...) + local name = name or multi.randomString(16) + local rand = math.random(1, 10000000) + c = {} + c.name = name + c.Name = name + c.Id = count +end + +function THREAD:newFunction(func, holdme) + return thread:newFunctionBase(function(...) + return multi:newSystemThread("TempSystemThread",func,...) + end, holdme, multi.SFUNCTION)() +end + +THREAD.newSystemThread = function(...) + multi:newSystemThread(...) +end + +multi.print("Integrated Effil Threading!") +multi.integration = {} -- for module creators +multi.integration.GLOBAL = GLOBAL +multi.integration.THREAD = THREAD +require("multi.integration.effilManager.extensions") +return { + init = function() + return GLOBAL, THREAD + end +} \ No newline at end of file diff --git a/integration/effilManager/threads.lua b/integration/effilManager/threads.lua new file mode 100644 index 0000000..e69de29 diff --git a/integration/lanesManager/extensions.lua b/integration/lanesManager/extensions.lua index 26cc459..343dd8b 100644 --- a/integration/lanesManager/extensions.lua +++ b/integration/lanesManager/extensions.lua @@ -34,6 +34,7 @@ function multi:newSystemThreadedQueue(name) local c = {} c.Name = name c.linda = lanes.linda() + c.Type = multi.SQUEUE function c:push(v) self.linda:send("Q", v) @@ -57,6 +58,8 @@ function multi:newSystemThreadedQueue(name) GLOBAL[name] = c end + self:create(c) + return c end @@ -65,6 +68,7 @@ function multi:newSystemThreadedTable(name) local c = {} c.link = lanes.linda() c.Name = name + c.Type = multi.STABLE function c:init() return self @@ -85,12 +89,15 @@ function multi:newSystemThreadedTable(name) GLOBAL[name] = c end + self:create(c) + return c end function multi:newSystemThreadedJobQueue(n) local c = {} c.cores = n or THREAD.getCores()*2 + c.Type = multi.SJOBQUEUE c.OnJobCompleted = multi:newConnection() local funcs = multi:newSystemThreadedTable():init() local queueJob = multi:newSystemThreadedQueue():init() @@ -133,7 +140,6 @@ function multi:newSystemThreadedJobQueue(n) link = c.OnJobCompleted(function(jid,...) if id==jid then rets = multi.pack(...) - c.OnJobCompleted:Unconnect(link) end end) return thread.hold(function() @@ -207,12 +213,16 @@ function multi:newSystemThreadedJobQueue(n) multi:mainloop() end,i).OnError(multi.error) end + + self:create(c) + return c end function multi:newSystemThreadedConnection(name) local name = name or multi.randomString(16) local c = {} + c.Type = multi.SCONNECTION c.CONN = 0x00 c.TRIG = 0x01 c.PING = 0x02 @@ -348,6 +358,8 @@ function multi:newSystemThreadedConnection(name) GLOBAL[name] = c end + self:create(c) + return c end require("multi.integration.sharedExtensions") \ No newline at end of file diff --git a/integration/lanesManager/init.lua b/integration/lanesManager/init.lua index 0892741..b33b7ba 100644 --- a/integration/lanesManager/init.lua +++ b/integration/lanesManager/init.lua @@ -32,9 +32,7 @@ if multi.integration then -- This allows us to call the lanes manager from suppo } end -- Step 1 get lanes -lanes = require("lanes").configure{ - nb_keepers = 4, -} +lanes = require("lanes").configure() multi.SystemThreads = {} multi.isMainThread = true @@ -63,7 +61,7 @@ local livingThreads = {} function THREAD:newFunction(func, holdme) return thread:newFunctionBase(function(...) return multi:newSystemThread("TempSystemThread",func,...) - end, holdme)() + end, holdme, multi.SFUNCTION)() end function multi:newSystemThread(name, func, ...) @@ -143,7 +141,9 @@ function multi:newSystemThread(name, func, ...) return c end -THREAD.newSystemThread = multi.newSystemThread +THREAD.newSystemThread = function(...) + multi:newSystemThread(...) +end function multi.InitSystemThreadErrorHandler() if started == true then diff --git a/integration/loveManager/extensions.lua b/integration/loveManager/extensions.lua index 738f18f..1dd7295 100644 --- a/integration/loveManager/extensions.lua +++ b/integration/loveManager/extensions.lua @@ -35,6 +35,7 @@ function multi:newSystemThreadedQueue(name) local name = name or multi.randomString(16) local c = {} c.Name = name + c.Type = multi.SQUEUE local fRef = {"func",nil} function c:init() local q = {} @@ -66,33 +67,49 @@ function multi:newSystemThreadedQueue(name) end return q end + THREAD.package(name,c) + + self:create(c) + return c end function multi:newSystemThreadedTable(name) local name = name or multi.randomString(16) - local c = {} + + local c = {} + c.Name = name + c.Type = multi.STABLE + function c:init() return THREAD.createTable(self.Name) end + THREAD.package(name,c) + + self:create(c) + return c end local jqc = 1 function multi:newSystemThreadedJobQueue(n) local c = {} + c.cores = n or THREAD.getCores() c.registerQueue = {} + c.Type = multi.SJOBQUEUE c.funcs = THREAD.createStaticTable("__JobQueue_"..jqc.."_table") c.queue = love.thread.getChannel("__JobQueue_"..jqc.."_queue") c.queueReturn = love.thread.getChannel("__JobQueue_"..jqc.."_queueReturn") c.queueAll = love.thread.getChannel("__JobQueue_"..jqc.."_queueAll") c.id = 0 c.OnJobCompleted = multi:newConnection() + local allfunc = 0 + function c:doToAll(func) local f = THREAD.dump(func) for i = 1, self.cores do @@ -211,13 +228,20 @@ function multi:newSystemThreadedJobQueue(n) multi:mainloop() end,jqc) end + jqc = jqc + 1 + + self:create(c) + return c end function multi:newSystemThreadedConnection(name) local name = name or multi.randomString(16) + local c = {} + + c.Type = multi.SCONNECTION c.CONN = 0x00 c.TRIG = 0x01 c.PING = 0x02 @@ -284,9 +308,11 @@ function multi:newSystemThreadedConnection(name) end return r end + c.CID = THREAD_ID c.Name = name c.links = {} -- All triggers sent from main connection. When a connection is triggered on another thread, they speak to the main then send stuff out. + -- Locals will only live in the thread that creates the original object local ping local pong = function(link, links) @@ -317,7 +343,7 @@ function multi:newSystemThreadedConnection(name) thread.sleep(3) ping:Resume() - end,false) + end, false) local function fire(...) for _, link in pairs(c.links) do @@ -351,5 +377,7 @@ function multi:newSystemThreadedConnection(name) THREAD.package(name,c) + self:create(c) + return c end \ No newline at end of file diff --git a/integration/loveManager/init.lua b/integration/loveManager/init.lua index 0c23b6b..a2a5d5f 100644 --- a/integration/loveManager/init.lua +++ b/integration/loveManager/init.lua @@ -116,10 +116,12 @@ end function THREAD:newFunction(func, holdme) return thread:newFunctionBase(function(...) return multi:newSystemThread("SystemThreaded Function Handler", func, ...) - end, holdme)() + end, holdme, multi.SFUNCTION)() end -THREAD.newSystemThread = multi.newSystemThread +THREAD.newSystemThread = function(...) + multi:newSystemThread(...) +end function love.threaderror(thread, errorstr) multi.print("Thread error!\n" .. errorstr) diff --git a/integration/sharedExtensions/init.lua b/integration/sharedExtensions/init.lua index 806ce69..c1bcfea 100644 --- a/integration/sharedExtensions/init.lua +++ b/integration/sharedExtensions/init.lua @@ -213,6 +213,7 @@ function multi:newSystemThreadedProcessor(cores) setmetatable(c,{__index = multi}) + c.Type = multi.SPROCESS c.threads = {} c.cores = cores or 8 c.Name = name diff --git a/tests/multi b/tests/multi new file mode 120000 index 0000000..5ddae4f --- /dev/null +++ b/tests/multi @@ -0,0 +1 @@ +D:/VSCWorkspace/discord-lua/multi \ No newline at end of file diff --git a/tests/runtests.lua b/tests/runtests.lua index 049c26b..1a89df6 100644 --- a/tests/runtests.lua +++ b/tests/runtests.lua @@ -182,11 +182,19 @@ runTest = thread:newFunction(function() end end) -runTest().OnError(function(...) +local handle = runTest() + +handle.OnError(function(...) multi.error("Something went wrong with the test!") print(...) end) if not love then multi:mainloop() +else + local hold = thread:newFunction(function() + thread.hold(handle.OnError + handle.OnReturn) + end, true) + hold() + multi.print("Starting Threading tests!") end \ No newline at end of file diff --git a/tests/threadtests.lua b/tests/threadtests.lua index b67a7e1..e859f22 100644 --- a/tests/threadtests.lua +++ b/tests/threadtests.lua @@ -109,12 +109,10 @@ multi:newThread("Scheduler Thread",function() local ready = false jq = multi:newSystemThreadedJobQueue(5) -- Job queue with 4 worker threads - func = jq:newFunction("test-thread",function(a,b) THREAD.sleep(.2) return a+b end) - local count = 0 for i = 1,10 do func(i, i*3).OnReturn(function(data) @@ -134,6 +132,7 @@ multi:newThread("Scheduler Thread",function() multi.success("SystemThreadedJobQueues: Ok") queue2 = multi:newSystemThreadedQueue("Test_Queue2"):init() + print(1) multi:newSystemThread("Test_Thread_2",function() queue2 = THREAD.waitFor("Test_Queue2"):init() connOut = THREAD.waitFor("ConnectionNAMEHERE"):init() @@ -142,7 +141,7 @@ multi:newThread("Scheduler Thread",function() end) multi:mainloop() end).OnError(multi.error) - + print(2) multi:newSystemThread("Test_Thread_3",function() queue2 = THREAD.waitFor("Test_Queue2"):init() connOut = THREAD.waitFor("ConnectionNAMEHERE"):init() @@ -151,32 +150,35 @@ multi:newThread("Scheduler Thread",function() end) multi:mainloop() end).OnError(multi.error) - + print(3) connOut = multi:newSystemThreadedConnection("ConnectionNAMEHERE"):init() a=0 connOut(function(arg) queue2:push("Main") end) + print(4) for i=1,3 do thread.sleep(.1) connOut:Fire("Test From Main Thread: "..i.."\n") end + print(5) thread.sleep(2) local count = 0 multi:newThread(function() while count < 9 do if queue2:pop() then count = count + 1 + print("Popped", count) end end end).OnError(multi.error) - + print(6) _, err = thread.hold(function() return count == 9 end,{sleep=.3}) - + print(7) if err == multi.TIMEOUT then multi.error("SystemThreadedConnections: Failed") end - + print(8) multi.success("SystemThreadedConnections: Ok") we_good = true