Merge pull request opencv#18673 from dmatveev:dm/upstream_desync

Author: alalek

modules/gapi/CMakeLists.txt

@@ -74,6 +74,7 @@ set(gapi_srcs
     src/api/kernels_imgproc.cpp
     src/api/kernels_video.cpp
     src/api/kernels_nnparsers.cpp
+    src/api/kernels_streaming.cpp
     src/api/render.cpp
     src/api/render_ocv.cpp
     src/api/ginfer.cpp
@@ -97,6 +98,7 @@ set(gapi_srcs
     src/compiler/passes/pattern_matching.cpp
     src/compiler/passes/perform_substitution.cpp
     src/compiler/passes/streaming.cpp
+    src/compiler/passes/intrin.cpp
 
     # Executor
     src/executor/gexecutor.cpp

modules/gapi/include/opencv2/gapi.hpp

@@ -33,4 +33,8 @@
 #include <opencv2/gapi/gkernel.hpp>
 #include <opencv2/gapi/operators.hpp>
 
+// Include this file here to avoid cyclic dependency between
+// Desync & GKernel & GComputation & GStreamingCompiled.
+#include <opencv2/gapi/streaming/desync.hpp>
+
 #endif // OPENCV_GAPI_HPP

modules/gapi/include/opencv2/gapi/garray.hpp

@@ -284,6 +284,14 @@ namespace detail
             return static_cast<VectorRefT<T>&>(*m_ref).rref();
         }
 
+        // Check if was created for/from std::vector<T>
+        template <typename T> bool holds() const
+        {
+            if (!m_ref) return false;
+            using U = typename std::decay<T>::type;
+            return dynamic_cast<VectorRefT<U>*>(m_ref.get()) != nullptr;
+        }
+
         void mov(VectorRef &v)
         {
             m_ref->mov(*v.m_ref);
@@ -341,15 +349,18 @@ template<typename T> class GArray
     explicit GArray(detail::GArrayU &&ref) // GArrayU-based constructor
         : m_ref(ref) { putDetails(); }     //   (used by GCall, not for users)
 
-    detail::GArrayU strip() const { return m_ref; }
-
-private:
-    static void VCTor(detail::VectorRef& vref) {
+    /// @private
+    detail::GArrayU strip() const {
+        return m_ref;
+    }
+    /// @private
+    static void VCtor(detail::VectorRef& vref) {
         vref.reset<HT>();
-        vref.storeKind<HT>();
     }
+
+private:
     void putDetails() {
-        m_ref.setConstructFcn(&VCTor);
+        m_ref.setConstructFcn(&VCtor);
         m_ref.specifyType<HT>();  // FIXME: to unify those 2 to avoid excessive dynamic_cast
         m_ref.storeKind<HT>();    //
     }

modules/gapi/include/opencv2/gapi/gkernel.hpp

@@ -28,6 +28,7 @@ namespace cv {
 
 using GShapes = std::vector<GShape>;
 using GKinds = std::vector<cv::detail::OpaqueKind>;
+using GCtors  = std::vector<detail::HostCtor>;
 
 // GKernel describes kernel API to the system
 // FIXME: add attributes of a kernel, (e.g. number and types
@@ -41,6 +42,7 @@ struct GAPI_EXPORTS GKernel
     M           outMeta;    // generic adaptor to API::outMeta(...)
     GShapes     outShapes;  // types (shapes) kernel's outputs
     GKinds      inKinds;    // kinds of kernel's inputs (fixme: below)
+    GCtors      outCtors;   // captured constructors for template output types
 };
 // TODO: It's questionable if inKinds should really be here. Instead,
 // this information could come from meta.
@@ -60,30 +62,27 @@ namespace detail
     // yield() is used in graph construction time as a generic method to obtain
     // lazy "return value" of G-API operations
     //
-    namespace
+    template<typename T> struct Yield;
+    template<> struct Yield<cv::GMat>
     {
-        template<typename T> struct Yield;
-        template<> struct Yield<cv::GMat>
-        {
-            static inline cv::GMat yield(cv::GCall &call, int i) { return call.yield(i); }
-        };
-        template<> struct Yield<cv::GMatP>
-        {
-            static inline cv::GMatP yield(cv::GCall &call, int i) { return call.yieldP(i); }
-        };
-        template<> struct Yield<cv::GScalar>
-        {
-            static inline cv::GScalar yield(cv::GCall &call, int i) { return call.yieldScalar(i); }
-        };
-        template<typename U> struct Yield<cv::GArray<U> >
-        {
-            static inline cv::GArray<U> yield(cv::GCall &call, int i) { return call.yieldArray<U>(i); }
-        };
-        template<typename U> struct Yield<cv::GOpaque<U> >
-        {
-            static inline cv::GOpaque<U> yield(cv::GCall &call, int i) { return call.yieldOpaque<U>(i); }
-        };
-    } // anonymous namespace
+        static inline cv::GMat yield(cv::GCall &call, int i) { return call.yield(i); }
+    };
+    template<> struct Yield<cv::GMatP>
+    {
+        static inline cv::GMatP yield(cv::GCall &call, int i) { return call.yieldP(i); }
+    };
+    template<> struct Yield<cv::GScalar>
+    {
+        static inline cv::GScalar yield(cv::GCall &call, int i) { return call.yieldScalar(i); }
+    };
+    template<typename U> struct Yield<cv::GArray<U> >
+    {
+        static inline cv::GArray<U> yield(cv::GCall &call, int i) { return call.yieldArray<U>(i); }
+    };
+    template<typename U> struct Yield<cv::GOpaque<U> >
+    {
+        static inline cv::GOpaque<U> yield(cv::GCall &call, int i) { return call.yieldOpaque<U>(i); }
+    };
 
     ////////////////////////////////////////////////////////////////////////////
     // Helper classes which brings outputMeta() marshalling to kernel
@@ -215,7 +214,8 @@ class GKernelTypeM<K, std::function<std::tuple<R...>(Args...)> >
                               , K::tag()
                               , &K::getOutMeta
                               , {detail::GTypeTraits<R>::shape...}
-                              , {detail::GTypeTraits<Args>::op_kind...}});
+                              , {detail::GTypeTraits<Args>::op_kind...}
+                              , {detail::GObtainCtor<R>::get()...}});
         call.pass(args...); // TODO: std::forward() here?
         return yield(call, typename detail::MkSeq<sizeof...(R)>::type());
     }
@@ -240,7 +240,8 @@ class GKernelType<K, std::function<R(Args...)> >
                               , K::tag()
                               , &K::getOutMeta
                               , {detail::GTypeTraits<R>::shape}
-                              , {detail::GTypeTraits<Args>::op_kind...}});
+                              , {detail::GTypeTraits<Args>::op_kind...}
+                              , {detail::GObtainCtor<R>::get()}});
         call.pass(args...);
         return detail::Yield<R>::yield(call, 0);
     }
@@ -459,11 +460,6 @@ namespace gapi {
         std::vector<GTransform> m_transformations;
 
     protected:
-        /// @private
-        // Check if package contains ANY implementation of a kernel API
-        // by API textual id.
-        bool includesAPI(const std::string &id) const;
-
         /// @private
         // Remove ALL implementations of the given API (identified by ID)
         void removeAPI(const std::string &id);
@@ -566,6 +562,9 @@ namespace gapi {
             return includesAPI(KAPI::id());
         }
 
+        /// @private
+        bool includesAPI(const std::string &id) const;
+
         // FIXME: The below comment is wrong, and who needs this function?
         /**
          * @brief Find a kernel (by its API)

modules/gapi/include/opencv2/gapi/gopaque.hpp

@@ -295,25 +295,27 @@ namespace detail
 template<typename T> class GOpaque
 {
 public:
-    GOpaque() { putDetails(); }              // Empty constructor
-    explicit GOpaque(detail::GOpaqueU &&ref) // GOpaqueU-based constructor
-        : m_ref(ref) { putDetails(); }       // (used by GCall, not for users)
-
-    detail::GOpaqueU strip() const { return m_ref; }
-
-private:
     // Host type (or Flat type) - the type this GOpaque is actually
     // specified to.
     using HT = typename detail::flatten_g<util::decay_t<T>>::type;
 
-    static void CTor(detail::OpaqueRef& ref) {
+    GOpaque() { putDetails(); }              // Empty constructor
+    explicit GOpaque(detail::GOpaqueU &&ref) // GOpaqueU-based constructor
+        : m_ref(ref) { putDetails(); }       // (used by GCall, not for users)
+
+    /// @private
+    detail::GOpaqueU strip() const {
+        return m_ref;
+    }
+    /// @private
+    static void Ctor(detail::OpaqueRef& ref) {
         ref.reset<HT>();
-        ref.storeKind<HT>();
     }
+private:
     void putDetails() {
-        m_ref.setConstructFcn(&CTor);
-        m_ref.specifyType<HT>(); // FIXME: to unify those 2 to avoid excessive dynamic_cast
-        m_ref.storeKind<HT>();   //
+        m_ref.setConstructFcn(&Ctor);
+        m_ref.specifyType<HT>();
+        m_ref.storeKind<HT>();
     }
 
     detail::GOpaqueU m_ref;

modules/gapi/include/opencv2/gapi/gstreaming.hpp

@@ -8,15 +8,99 @@
 #ifndef OPENCV_GAPI_GSTREAMING_COMPILED_HPP
 #define OPENCV_GAPI_GSTREAMING_COMPILED_HPP
 
+#include <memory>
 #include <vector>
 
 #include <opencv2/gapi/opencv_includes.hpp>
 #include <opencv2/gapi/own/assert.hpp>
+#include <opencv2/gapi/util/optional.hpp>
 #include <opencv2/gapi/garg.hpp>
 #include <opencv2/gapi/streaming/source.hpp>
 
 namespace cv {
 
+template<class T> using optional = cv::util::optional<T>;
+
+namespace detail {
+template<typename T> struct wref_spec {
+    using type = T;
+};
+template<typename T> struct wref_spec<std::vector<T> > {
+    using type = T;
+};
+
+template<typename RefHolder>
+struct OptRef {
+    struct OptHolder {
+        virtual void mov(RefHolder &h) = 0;
+        virtual void reset() = 0;
+        virtual ~OptHolder() = default;
+        using Ptr = std::shared_ptr<OptHolder>;
+    };
+    template<class T> struct Holder final: OptHolder {
+        std::reference_wrapper<cv::optional<T> > m_opt_ref;
+
+        explicit Holder(cv::optional<T>& opt) : m_opt_ref(std::ref(opt)) {
+        }
+        virtual void mov(RefHolder &h) override {
+            using U = typename wref_spec<T>::type;
+            m_opt_ref.get() = cv::util::make_optional(std::move(h.template wref<U>()));
+        }
+        virtual void reset() override {
+            m_opt_ref.get().reset();
+        }
+    };
+    template<class T>
+    explicit OptRef(cv::optional<T>& t) : m_opt{new Holder<T>(t)} {}
+    void mov(RefHolder &h) { m_opt->mov(h); }
+    void reset()           { m_opt->reset();}
+private:
+    typename OptHolder::Ptr m_opt;
+};
+using OptionalVectorRef = OptRef<cv::detail::VectorRef>;
+using OptionalOpaqueRef = OptRef<cv::detail::OpaqueRef>;
+} // namespace detail
+
+// TODO: Keep it in sync with GRunArgP (derive the type automatically?)
+using GOptRunArgP = util::variant<
+    optional<cv::Mat>*,
+    optional<cv::RMat>*,
+    optional<cv::Scalar>*,
+    cv::detail::OptionalVectorRef,
+    cv::detail::OptionalOpaqueRef
+>;
+using GOptRunArgsP = std::vector<GOptRunArgP>;
+
+namespace detail {
+
+template<typename T> inline GOptRunArgP wrap_opt_arg(optional<T>& arg) {
+    // By default, T goes to an OpaqueRef. All other types are specialized
+    return GOptRunArgP{OptionalOpaqueRef(arg)};
+}
+
+template<typename T> inline GOptRunArgP wrap_opt_arg(optional<std::vector<T> >& arg) {
+    return GOptRunArgP{OptionalVectorRef(arg)};
+}
+
+template<> inline GOptRunArgP wrap_opt_arg(optional<cv::Mat> &m) {
+    return GOptRunArgP{&m};
+}
+
+template<> inline GOptRunArgP wrap_opt_arg(optional<cv::Scalar> &s) {
+    return GOptRunArgP{&s};
+}
+
+} // namespace detail
+
+// Now cv::gout() may produce an empty vector (see "dynamic graphs"), so
+// there may be a conflict between these two. State here that Opt version
+// _must_ have at least one input for this overload
+template<typename T, typename... Ts>
+inline GOptRunArgsP gout(optional<T>&arg, optional<Ts>&... args)
+{
+    return GOptRunArgsP{ detail::wrap_opt_arg(arg), detail::wrap_opt_arg(args)... };
+}
+
 /**
  * \addtogroup gapi_main_classes
  * @{
@@ -169,6 +253,44 @@ class GAPI_EXPORTS_W_SIMPLE GStreamingCompiled
     // NB: Used from python
     GAPI_WRAP std::tuple<bool, cv::GRunArgs> pull();
 
+    /**
+     * @brief Get some next available data from the pipeline.
+     *
+     * This method takes a vector of cv::optional object. An object is
+     * assigned to some value if this value is available (ready) at
+     * the time of the call, and resets the object to empty() if it is
+     * not.
+     *
+     * This is a blocking method which guarantees that some data has
+     * been written to the output vector on return.
+     *
+     * Using this method only makes sense if the graph has
+     * desynchronized parts (see cv::gapi::desync). If there is no
+     * desynchronized parts in the graph, the behavior of this
+     * method is identical to the regular pull() (all data objects are
+     * produced synchronously in the output vector).
+     *
+     * Use gout() to create an output parameter vector.
+     *
+     * Output vectors must have the same number of elements as defined
+     * in the cv::GComputation protocol (at the moment of its
+     * construction). Shapes of elements also must conform to protocol
+     * (e.g. cv::optional<cv::Mat> needs to be passed where cv::GMat
+     * has been declared as output, and so on). Run-time exception is
+     * generated on type mismatch.
+     *
+     * This method writes new data into objects passed via output
+     * vector.  If there is no data ready yet, this method blocks. Use
+     * try_pull() if you need a non-blocking version.
+     *
+     * @param outs vector of output parameters to obtain.
+     * @return true if next result has been obtained,
+     *    false marks end of the stream.
+     *
+     * @sa cv::gapi::desync
+     */
+    bool pull(cv::GOptRunArgsP &&outs);
+
     /**
      * @brief Try to get the next processed frame from the pipeline.
      *

modules/gapi/include/opencv2/gapi/gtype_traits.hpp

@@ -191,6 +191,29 @@ namespace detail
 
     template<typename T> using wrap_gapi_helper = WrapValue<typename std::decay<T>::type>;
     template<typename T> using wrap_host_helper = WrapValue<typename std::decay<g_type_of_t<T> >::type>;
+
+// Union type for various user-defined type constructors (GArray<T>,
+// GOpaque<T>, etc)
+//
+// TODO: Replace construct-only API with a more generic one (probably
+//    with bits of introspection)
+//
+// Not required for non-user-defined types (GMat, GScalar, etc)
+using HostCtor = util::variant
+    < util::monostate
+    , detail::ConstructVec
+    , detail::ConstructOpaque
+    >;
+
+template<typename T> struct GObtainCtor {
+    static HostCtor get() { return HostCtor{}; }
+};
+template<typename T> struct GObtainCtor<GArray<T> > {
+    static HostCtor get() { return HostCtor{ConstructVec{&GArray<T>::VCtor}}; };
+};
+template<typename T> struct GObtainCtor<GOpaque<T> > {
+    static HostCtor get() { return HostCtor{ConstructOpaque{&GOpaque<T>::Ctor}}; };
+};
 } // namespace detail
 } // namespace cv