Fixes, move spinlock to cc, minor changes

CMakeLists.txt

@@ -15,7 +15,7 @@ option(TD_NO_WAITS "If enabled, worker threads do not wait/sleep when idle, mini
 # define library
 
 file(GLOB_RECURSE SOURCES "src/*.cc")
-file(GLOB_RECURSE HEADERS "src/*.hh")
+file(GLOB_RECURSE HEADERS "src/*.hh" "src/*.h")
 
 arcana_add_library(TD task-dispatcher SOURCES HEADERS)
 

licenses/moodycamel.txt

@@ -0,0 +1,62 @@
+This license file applies to everything in this repository except that which
+is explicitly annotated as being written by other authors, i.e. the Boost
+queue (included in the benchmarks for comparison), Intel's TBB library (ditto),
+dlib::pipe (ditto),
+the CDSChecker tool (used for verification), the Relacy model checker (ditto),
+and Jeff Preshing's semaphore implementation (used in the blocking queue) which
+has a zlib license (embedded in lightweightsempahore.h).
+
+---
+
+Simplified BSD License:
+
+Copyright (c) 2013-2016, Cameron Desrochers.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+- Redistributions of source code must retain the above copyright notice, this list of
+conditions and the following disclaimer.
+- Redistributions in binary form must reproduce the above copyright notice, this list of
+conditions and the following disclaimer in the documentation and/or other materials
+provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+---
+
+I have also chosen to dual-license under the Boost Software License as an alternative to
+the Simplified BSD license above:
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.

src/task-dispatcher/AutoCounter.cc

@@ -0,0 +1,34 @@
+#include "AutoCounter.hh"
+
+#include "Scheduler.hh"
+
+td::AutoCounter::operator td::CounterHandle() &
+{
+    if (!handle.isValid())
+    {
+        handle = td::acquireCounter();
+    }
+
+    return handle;
+}
+
+int32_t td::waitForCounter(AutoCounter& autoCounter, bool bPinned)
+{
+    if (!autoCounter.handle.isValid())
+    {
+        // return immediately for uninitialized syncs
+        return 0;
+    }
+
+    // perform real wait
+    int32_t const res = td::waitForCounter(autoCounter.handle, bPinned);
+
+    // this call must not be raced on
+    if (td::releaseCounterIfOnZero(autoCounter.handle))
+    {
+        // mark the sync as uninitialized
+        autoCounter.handle.invalidate();
+    }
+
+    return res;
+}

src/task-dispatcher/AutoCounter.hh

@@ -0,0 +1,35 @@
+#pragma once
+
+#include <clean-core/assert.hh>
+
+#include <task-dispatcher/CounterHandle.hh>
+#include <task-dispatcher/common/api.hh>
+
+namespace td
+{
+// An automatically managed CounterHandle
+// acquires a counter on first usage and releases it when waiting
+struct TD_API AutoCounter
+{
+    AutoCounter() = default;
+
+    AutoCounter(AutoCounter&& rhs) noexcept : handle(rhs.handle) { rhs.handle.invalidate(); }
+    AutoCounter& operator=(AutoCounter&& rhs) noexcept
+    {
+        CC_ASSERT(!handle.isValid() && "Must call td::waitForCounter() on AutoCounter before dropping it");
+        handle = rhs.handle;
+        rhs.handle.invalidate();
+        return *this;
+    }
+
+    ~AutoCounter() { CC_ASSERT(!handle.isValid() && "Must call td::waitForCounter() on AutoCounter before dropping it"); }
+
+    operator CounterHandle() &;
+
+    [[deprecated("For less restricted usage, use raw CounterHandle")]] AutoCounter(AutoCounter const&) = delete;
+    [[deprecated("For less restricted usage, use raw CounterHandle")]] AutoCounter& operator=(AutoCounter const&) = delete;
+    [[deprecated("Would immediately drop AutoCounter")]] operator CounterHandle() && = delete;
+
+    CounterHandle handle = {};
+};
+}

src/task-dispatcher/BatchedSubmission.hh

@@ -0,0 +1,96 @@
+#pragma once
+
+#include <type_traits>
+
+#include <clean-core/allocator.hh>
+#include <clean-core/assert.hh>
+#include <clean-core/enable_if.hh>
+#include <clean-core/forward.hh>
+#include <clean-core/utility.hh>
+
+#include <task-dispatcher/CounterHandle.hh>
+#include <task-dispatcher/Scheduler.hh>
+#include <task-dispatcher/container/Task.hh>
+
+//
+// Batched submission
+//
+// larger header than the raw scheduler
+
+namespace td
+{
+// submits multiple tasks calling a lambda "void f(uint32_t i)" from 0 to num - 1
+template <class F>
+void submitNumbered(CounterHandle counter, F&& func, uint32_t numElements, cc::allocator* scratch, ETaskPriority priority = ETaskPriority::Default)
+{
+    static_assert(std::is_invocable_v<F, unsigned>, "function must be invocable with index argument");
+    static_assert(std::is_same_v<std::invoke_result_t<F, uint32_t>, void>, "return must be void");
+
+    td::Task* tasks = scratch->new_array_sized<td::Task>(numElements);
+
+    for (uint32_t i = 0u; i < numElements; ++i)
+    {
+        tasks[i].initWithLambda([=] { func(i); });
+    }
+
+    td::submitTasks(counter, cc::span{tasks, numElements}, priority);
+    scratch->delete_array_sized(tasks, numElements);
+}
+
+// submits tasks calling a lambda "void f(uint start, uint end, uint batchIdx)" for evenly sized batches from 0 to num - 1
+// maxNumBatches: maximum amount of batches to partition the range into
+// returns amount of batches
+template <class F>
+uint32_t submitBatched(CounterHandle counter, F&& func, uint32_t numElements, uint32_t maxNumBatches, cc::allocator* scratch, ETaskPriority priority = ETaskPriority::Default)
+{
+    static_assert(std::is_invocable_v<F, uint32_t, uint32_t, uint32_t>, "function must be invocable with element start, end, and index argument");
+
+    if (numElements == 0 || maxNumBatches == 0)
+    {
+        return 0;
+    }
+
+    uint32_t const batchSize = cc::int_div_ceil(numElements, maxNumBatches);
+    uint32_t const numBatches = cc::int_div_ceil(numElements, batchSize);
+    CC_ASSERT(numBatches <= maxNumBatches && "programmer error");
+
+    td::Task* tasks = scratch->new_array_sized<td::Task>(numBatches);
+
+    for (uint32_t batch = 0u, start = 0u, end = cc::min(batchSize, numElements); //
+         batch < numBatches;                                                     //
+         ++batch, start = batch * batchSize, end = cc::min((batch + 1) * batchSize, numElements))
+    {
+        tasks[batch].initWithLambda([=] { func(start, end, batch); });
+    }
+
+    td::submitTasks(counter, cc::span{tasks, numBatches}, priority);
+    scratch->delete_array_sized(tasks, numBatches);
+    return numBatches;
+}
+
+// submits batched tasks calling a lambda "void f(T& value, uint32_t idx, uint32_t batchIdx)" for each element in the span
+// data in 'values' must stay alive until all tasks ran through
+// returns amount of batches
+template <class T, class F>
+uint32_t submitBatchedOnArray(CounterHandle counter, F&& func, cc::span<T> values, uint32_t maxNumBatches, cc::allocator* scratch, ETaskPriority priority = ETaskPriority::Default)
+{
+    static_assert(std::is_invocable_v<F, T&, uint32_t, uint32_t>, "function must be invocable with element reference");
+    static_assert(std::is_same_v<std::invoke_result_t<F, T&, uint32_t, uint32_t>, void>, "return must be void");
+
+    T* const pValues = values.data();
+    uint32_t const numValues = uint32_t(values.size());
+
+    // copy the lambda
+    return td::submitBatched(
+        counter,
+        [func, pValues](uint32_t start, uint32_t end, uint32_t batchIdx)
+        {
+            for (uint32_t i = start; i < end; ++i)
+            {
+                func(pValues[i], i, batchIdx);
+            }
+        },
+        numValues, maxNumBatches, scratch, priority);
+}
+
+}

src/task-dispatcher/CallableSubmission.hh

@@ -0,0 +1,75 @@
+#pragma once
+
+#include <type_traits> // for is_invocable, is_class, is_member_function_pointer_v
+#include <utility>     // for tuple_size
+
+#include <clean-core/apply.hh>
+#include <clean-core/enable_if.hh>
+#include <clean-core/forward.hh>
+#include <clean-core/move.hh>
+#include <clean-core/tuple.hh>
+
+#include <task-dispatcher/CounterHandle.hh>
+#include <task-dispatcher/Scheduler.hh>
+#include <task-dispatcher/container/Task.hh>
+
+//
+// Callable submission
+//
+// large header, only required for exotic callables
+// prefer LambdaSubmission.hh if possible
+
+namespace td
+{
+// submit a task based on any callable (lambda, funcptr, type with operator()), with arguments passed to it
+// arguments are moved into the task capture immediately
+template <class F, class... Args>
+void submitCallable(CounterHandle counter, F&& func, Args&&... args)
+{
+    static_assert(std::is_invocable_v<F, Args...>, "function must be invocable with the given args");
+    static_assert(std::is_same_v<std::invoke_result_t<F, Args...>, void>, "return must be void");
+
+    // A lambda calling func(args...), moving the arguments into the capture (instead of copying)
+    Task dispatch(
+        [func, tup = cc::tuple(cc::move(args)...)]
+        {
+            // "apply" the argument tuple to go back to a parameter pack
+            cc::apply(
+                [&func](auto&&... args)
+                {
+                    // actually call the callable with expanded arguments
+                    func(decltype(args)(args)...);
+                },
+                tup);
+        });
+
+    td::submitTasks(counter, cc::span{dispatch});
+}
+
+// submit a task based on a member function, with arguments passed to it
+// arguments are moved into the task capture immediately
+template <class F, class FObj, class... Args, cc::enable_if<std::is_member_function_pointer_v<F>> = true>
+void submitMethod(CounterHandle counter, FObj* pInst, F pMemberFunc, Args&&... args)
+{
+    // NOTE: the SFINAE and static asserts here wouldn't be necessary if we could just use void (FObj::*pMemberFunc)(Args...)
+    // as the type for the member function ptr but that creates ambiguity about 'Args...'
+    static_assert(std::is_invocable_v<F, FObj, Args...>, "function must be invocable with the given args");
+    static_assert(std::is_same_v<std::invoke_result_t<F, FObj, Args...>, void>, "return must be void");
+
+    // A lambda calling pInst->func(args...), moving the arguments into the capture (instead of copying)
+    Task dispatch(
+        [pMemberFunc, pInst, tup = cc::tuple(cc::move(args)...)]
+        {
+            // "apply" the argument tuple to go back to a parameter pack
+            cc::apply(
+                [&pMemberFunc, pInst](auto&&... args)
+                {
+                    // actually call the method with expanded arguments
+                    (pInst->*pMemberFunc)(decltype(args)(args)...);
+                },
+                tup);
+        });
+
+    td::submitTasks(counter, cc::span{dispatch});
+}
+}

src/task-dispatcher/CounterHandle.hh

@@ -0,0 +1,23 @@
+#pragma once
+
+#include <stdint.h>
+
+namespace td
+{
+// Handle to a counter, the core synchronization mechanism
+struct CounterHandle
+{
+    uint32_t _value = 0;
+
+    CounterHandle() = default;
+    explicit constexpr CounterHandle(uint32_t val) : _value(val) {}
+
+    void invalidate() & { _value = 0; }
+    bool isValid() const { return _value != 0; }
+
+    bool operator==(CounterHandle rhs) const { return _value == rhs._value; }
+    bool operator!=(CounterHandle rhs) const { return _value != rhs._value; }
+};
+
+inline constexpr CounterHandle NullCounterHandle = CounterHandle{};
+}

src/task-dispatcher/CriticalSection.cc

@@ -0,0 +1,55 @@
+#include "CriticalSection.hh"
+
+#include <clean-core/assert.hh>
+
+#include "Scheduler.hh"
+
+namespace
+{
+enum CritSecState : int32_t
+{
+    CritSecState_Unlocked = 0,
+    CritSecState_Locked
+};
+}
+
+td::CriticalSection::CriticalSection()
+{
+    CC_ASSERT(td::isInsideScheduler() && "td::CriticalSection must only be used from inside a scheduler");
+    mCounter = td::acquireCounter();
+}
+
+td::CriticalSection::~CriticalSection() { destroy(); }
+
+void td::CriticalSection::lock(bool pinned) noexcept
+{
+    while (true)
+    {
+        // try to acquire
+        int32_t prevState = td::compareAndSwapCounter(mCounter, CritSecState_Unlocked, CritSecState_Locked);
+
+        if (prevState == CritSecState_Unlocked)
+        {
+            // CAS successful
+            return;
+        }
+
+        // wait until this is unlocked, then retry CAS
+        td::waitForCounter(mCounter, pinned);
+    }
+}
+
+void td::CriticalSection::unlock() noexcept
+{
+    int32_t prevState = td::compareAndSwapCounter(mCounter, CritSecState_Locked, CritSecState_Unlocked);
+    CC_ASSERT(prevState == CritSecState_Locked && "unlocked a critical section that was already unlocked");
+}
+
+void td::CriticalSection::destroy()
+{
+    if (mCounter.isValid())
+    {
+        td::releaseCounter(mCounter);
+        mCounter = {};
+    }
+}