image annotation export ++

Author: philipp1234

CMakeLists.txt

@@ -93,6 +93,7 @@ add_library(${LIBRARY_NAME}
     src/core/event.cpp
     src/core/handle.cpp
     src/core/history.cpp
+    src/core/image.cpp
     src/core/interaction.cpp
     src/core/loader.cpp
     src/core/log.cpp

src/annotations/image.cpp

@@ -7,23 +7,39 @@
 
 #include <QPainterPath>
 #include <QTransform>
+#include <QPainter>
 
-struct PointImageAnnotation : ImageAnnotationBase {
-  virtual void render() override {
-    if (controlPoints().empty()) {
-      shape = nullptr;
-      complete = false;
-    } else {
-      shape = std::make_shared<QPainterPath>();
-      shape->addPolygon(QPolygonF(QRectF(-10, -2, 20, 4))
-                            .united(QPolygonF(QRectF(-2, -10, 4, 20))));
-      shape->translate(controlPoints().front().x(),
-                       controlPoints().front().y());
-      complete = true;
-    }
+void ImageAnnotationBase::heatmap(QPainter &painter, const QColor &color) {
+  render();
+  if (shape) {
+    painter.setPen(Qt::NoPen);
+    painter.setBrush(color);
+    painter.drawPath(*shape);
   }
-};
-DECLARE_TYPE(PointImageAnnotation, ImageAnnotationBase);
+}
+
+void PointImageAnnotation::render() {
+  if (controlPoints().empty()) {
+    shape = nullptr;
+    complete = false;
+  } else {
+    shape = std::make_shared<QPainterPath>();
+    shape->addPolygon(QPolygonF(QRectF(-10, -2, 20, 4))
+                          .united(QPolygonF(QRectF(-2, -10, 4, 20))));
+    shape->translate(controlPoints().front().x(), controlPoints().front().y());
+    complete = true;
+  }
+}
+
+void PointImageAnnotation::heatmap(QPainter &painter, const QColor &color) {
+  if (!controlPoints().empty()) {
+    auto p = controlPoints().at(0);
+    painter.setPen(Qt::NoPen);
+    painter.setBrush(color);
+    // painter.drawPoint(QPointF(p[0], p[1]));
+    painter.drawEllipse(QPointF(p[0], p[1]), 3.0, 3.0);
+  }
+}
 
 struct RectangleImageAnnotation : ImageAnnotationBase {
   virtual void render() override {

src/annotations/image.h

@@ -6,17 +6,26 @@
 #include "../core/document.h"
 
 class QPainterPath;
+class QPainter;
+class QColor;
 
 struct ImageAnnotationBase : AnnotationBase {
-protected:
+ protected:
   ImageAnnotationBase() {}
 
-public:
+ public:
   PROPERTY(std::vector<Eigen::Vector2d>, controlPoints);
   PROPERTY(std::string, topic);
   bool complete = false;
   std::shared_ptr<QPainterPath> shape;
   virtual void render() = 0;
+  virtual void heatmap(QPainter& painter, const QColor& color);
   virtual void constrain() {}
 };
 DECLARE_TYPE(ImageAnnotationBase, AnnotationBase);
+
+struct PointImageAnnotation : ImageAnnotationBase {
+  virtual void render() override;
+  virtual void heatmap(QPainter& painter, const QColor& color) override;
+};
+DECLARE_TYPE(PointImageAnnotation, ImageAnnotationBase);

src/core/image.cpp

@@ -0,0 +1,140 @@
+// TAMSVIZ
+// (c) 2020-2024 Philipp Ruppel
+
+#include "image.h"
+
+#include "log.h"
+
+#include <cv_bridge/cv_bridge.h>
+
+#include <sensor_msgs/CompressedImage.h>
+#include <sensor_msgs/Image.h>
+#include <sensor_msgs/image_encodings.h>
+
+#include <QtWidgets>
+
+bool isImageMessageTypeName(const std::string &message_type_name) {
+  return message_type_name == "sensor_msgs/Image" ||
+         message_type_name == "sensor_msgs/CompressedImage";
+}
+
+bool isImageMessage(const Message &image) {
+  return isImageMessageTypeName(image.type()->name());
+}
+
+bool tryConvertImageMessageToMat(const Message &image, cv::Mat &mat,
+                                 std::string &encoding,
+                                 std_msgs::Header &header) {
+  if (!isImageMessage(image)) {
+    LOG_WARN_THROTTLE(1, "not an image message " << image.type()->name());
+    return false;
+  }
+  if (auto img = image.instantiate<sensor_msgs::Image>()) {
+    header = img->header;
+    if (sensor_msgs::image_encodings::isBayer(img->encoding) ||
+        img->encoding == sensor_msgs::image_encodings::YUV422) {
+      try {
+        cv_bridge::CvImageConstPtr cv_ptr =
+            cv_bridge::toCvCopy(*img, sensor_msgs::image_encodings::RGB8);
+        mat = cv_ptr->image;
+        encoding = cv_ptr->encoding;
+        return true;
+      } catch (cv_bridge::Exception &e) {
+        LOG_ERROR("cv_bridge exception: " << e.what());
+      }
+    }
+    try {
+      cv_bridge::CvImageConstPtr cv_ptr = cv_bridge::toCvCopy(*img);
+      mat = cv_ptr->image;
+      encoding = cv_ptr->encoding;
+      return true;
+    } catch (cv_bridge::Exception &e) {
+      LOG_ERROR("cv_bridge exception: " << e.what());
+    }
+  } else if (auto img = image.instantiate<sensor_msgs::CompressedImage>()) {
+    header = img->header;
+    try {
+      cv_bridge::CvImageConstPtr cv_ptr =
+          cv_bridge::toCvCopy(*img, sensor_msgs::image_encodings::RGB8);
+      mat = cv_ptr->image;
+      encoding = cv_ptr->encoding;
+      return true;
+    } catch (cv_bridge::Exception &e) {
+      LOG_ERROR("cv_bridge exception: " << e.what());
+    }
+  } else {
+    LOG_WARN("unsupported image message type" << image.type()->name());
+  }
+  if (mat.empty()) {
+    LOG_WARN("failed to decode image");
+  }
+  header = std_msgs::Header();
+  return false;
+}
+
+bool tryConvertMatToU8(const cv::Mat &in, cv::Mat &out) {
+  switch (in.type()) {
+    case CV_16UC1:
+      in.convertTo(out, CV_8U, 1.0 / 256);
+      break;
+    case CV_32FC1:
+      in.convertTo(out, CV_8U, 255.0);
+      break;
+    case CV_64FC1:
+      in.convertTo(out, CV_8U, 255.0);
+      break;
+    case CV_16UC3:
+      in.convertTo(out, CV_8UC3, 1.0 / 256.0);
+      break;
+    case CV_32FC3:
+      in.convertTo(out, CV_8UC3, 255.0);
+      break;
+    case CV_64FC3:
+      in.convertTo(out, CV_8UC3, 255.0);
+      break;
+    case CV_16UC4:
+      in.convertTo(out, CV_8UC4, 1.0 / 256.0);
+      break;
+    case CV_32FC4:
+      in.convertTo(out, CV_8UC4, 255.0);
+      break;
+    case CV_64FC4:
+      in.convertTo(out, CV_8UC4, 255.0);
+      break;
+    default:
+      out = in;
+      break;
+  }
+  return CV_MAT_DEPTH(out.type()) == CV_8U;
+}
+
+bool tryConvertMatToImage(const cv::Mat &in_mat,
+                          const std::string &img_encoding, QImage &image) {
+  {
+    cv::Mat mat;
+    if (tryConvertMatToU8(in_mat, mat)) {
+      switch (mat.type()) {
+        case CV_8UC1:
+          image = QImage((uchar *)mat.data, mat.cols, mat.rows, mat.step,
+                         QImage::Format_Grayscale8);
+          return true;
+        case CV_8UC3:
+          if (img_encoding == "bgr8") {
+            cv::cvtColor(mat, mat, cv::COLOR_BGR2RGB);
+          }
+          image = QImage((uchar *)mat.data, mat.cols, mat.rows, mat.step,
+                         QImage::Format_RGB888);
+          return true;
+        case CV_8UC4:
+          if (img_encoding == "bgra8") {
+            cv::cvtColor(mat, mat, cv::COLOR_BGRA2RGBA);
+          }
+          image = QImage((uchar *)mat.data, mat.cols, mat.rows, mat.step,
+                         QImage::Format_RGBA8888);
+          return true;
+      }
+    }
+  }
+  LOG_WARN("cv to qt image format not yet supported " << in_mat.type());
+  return false;
+}

src/core/image.h

@@ -0,0 +1,23 @@
+// TAMSVIZ
+// (c) 2020-2024 Philipp Ruppel
+
+#pragma once
+
+#include "message.h"
+
+#include <opencv2/imgproc/imgproc.hpp>
+
+class QImage;
+
+bool isImageMessageTypeName(const std::string &message_type_name);
+
+bool isImageMessage(const Message &image);
+
+bool tryConvertImageMessageToMat(const Message &image, cv::Mat &mat,
+                                 std::string &encoding,
+                                 std_msgs::Header &header);
+
+bool tryConvertMatToU8(cv::Mat &in, cv::Mat &out);
+
+bool tryConvertMatToImage(const cv::Mat &mat, const std::string &img_encoding,
+                          QImage &image);