Gaussian Splat read, write and conversion to and from ply and splat files.

lumurillo · jccruzna · diegointel · web-flow · commit 8893079a3f47 · 2025-03-27T17:10:23.000-07:00
* Gaussian Splat read, write and conversion to and from ply and splat files. * IsGaussianSplat() function to check if a point cloud is a valid Gaussian Splat. * Unit tests for reading and writing ply and splat (read-write-read consistency). * Unit tests for converting ply to splat. * Jupyter notebook tutorial. References: - 3DGS files save to .ply format: https://github.com/graphdeco-inria/gaussian-splatting/blob/main/scene/gaussian_model.py - 3DGS files saved to .splat format: https://github.com/antimatter15/splat/blob/main/convert.py - Reference viewer: https://projects.markkellogg.org/threejs/demo_gaussian_splats_3d.php - 3DGS to point cloud reference: https://github.com/Lewis-Stuart-11/3DGS-to-PC/blob/main/gauss_to_pc.py 3DGS tensor point cloud representation: - pcd.point[‘positions’] (N,3) – (x,y,z) for each splat. - pcd.point[‘opacity’] – (N,) opacity - pcd.point[‘rot’] (N, 4) - quaternion rotation of Gaussian. - pcd.point[‘scale'] (N, 3) - x, y, z scales for Gaussian. - pcd.point[‘f_dc’] (N, 3) – DC components for RGB colors. - pcd.point[‘f_rest’] (N, Nc, 3) – SH coeffs for RGB colors. (Nc = 3, 8 or 15) --------- Co-authored-by: Juan Cruz <juan.cruz.naranjo@intel.com> Co-authored-by: Gomez Rodriguez, Diego <diego.gomez.rodriguez@intel.com> Co-authored-by: Jose Rojas Chaves <joserochh@gmail.com> Co-authored-by: ikun03 <ikunal03@gmail.com> Co-authored-by: Jose Rojas Chaves <jose.rojas.chaves@intel.com> Co-authored-by: Sameer Sheorey <sameer.sheorey@intel.com>
diff --git a/cpp/open3d/t/geometry/PointCloud.cpp b/cpp/open3d/t/geometry/PointCloud.cpp
@@ -1369,6 +1369,21 @@ core::Tensor PointCloud::ComputeMetrics(const PointCloud &pcd2,
                                 metrics, params);
 }
 
+bool PointCloud::IsGaussianSplat() const {
+    auto num_points = GetPointPositions().GetLength();
+    bool have_all_attrs = HasPointAttr("opacity") && HasPointAttr("rot") &&
+                          HasPointAttr("scale") && HasPointAttr("f_dc");
+    if (!have_all_attrs) {  // not 3DGS, no messages.
+        return false;
+    }
+    // Existing but invalid attributes cause errors.
+    core::AssertTensorShape(GetPointAttr("opacity"), {num_points, 1});
+    core::AssertTensorShape(GetPointAttr("rot"), {num_points, 4});
+    core::AssertTensorShape(GetPointAttr("scale"), {num_points, 3});
+    core::AssertTensorShape(GetPointAttr("f_dc"), {num_points, 3});
+    // GaussianSplatGetSHOrder(); // TODO: Tests f_rest shape is valid.
+    return true;
+}
 }  // namespace geometry
 }  // namespace t
 }  // namespace open3d
diff --git a/cpp/open3d/t/geometry/PointCloud.h b/cpp/open3d/t/geometry/PointCloud.h
@@ -738,6 +738,14 @@ class PointCloud : public Geometry, public DrawableGeometry {
             std::vector<Metric> metrics = {Metric::ChamferDistance},
             MetricParameters params = MetricParameters()) const;
 
+    /// Check if this point cloud has all the attributes required for a Gaussian
+    /// Splat. This checks for the presence of scale, rot, opacity and f_dc
+    /// attributes.
+    /// \returns True if a valid 3DGS point cloud, else False.
+    /// \throws If point cloud has 3DGS attributes, but they are invalid (wrong
+    /// shape).
+    bool IsGaussianSplat() const;
+
 protected:
     core::Device device_ = core::Device("CPU:0");
     TensorMap point_attr_;
diff --git a/cpp/open3d/t/io/CMakeLists.txt b/cpp/open3d/t/io/CMakeLists.txt
@@ -13,6 +13,7 @@ target_sources(tio PRIVATE
     file_format/FileJPG.cpp
     file_format/FilePCD.cpp
     file_format/FilePLY.cpp
+    file_format/FileSPLAT.cpp
     file_format/FilePNG.cpp
     file_format/FilePTS.cpp
     file_format/FileTXT.cpp
diff --git a/cpp/open3d/t/io/PointCloudIO.cpp b/cpp/open3d/t/io/PointCloudIO.cpp
@@ -35,6 +35,7 @@ static const std::unordered_map<
                 {"xyzrgb", ReadPointCloudFromTXT},
                 {"pcd", ReadPointCloudFromPCD},
                 {"ply", ReadPointCloudFromPLY},
+                {"splat", ReadPointCloudFromSPLAT},
                 {"pts", ReadPointCloudFromPTS},
         };
 
@@ -52,6 +53,7 @@ static const std::unordered_map<
                 {"xyzrgb", WritePointCloudToTXT},
                 {"pcd", WritePointCloudToPCD},
                 {"ply", WritePointCloudToPLY},
+                {"splat", WritePointCloudToSPLAT},
                 {"pts", WritePointCloudToPTS},
         };
 
diff --git a/cpp/open3d/t/io/PointCloudIO.h b/cpp/open3d/t/io/PointCloudIO.h
@@ -74,6 +74,14 @@ bool WritePointCloudToPLY(const std::string &filename,
                           const geometry::PointCloud &pointcloud,
                           const WritePointCloudOption &params);
 
+bool ReadPointCloudFromSPLAT(const std::string &filename,
+                             geometry::PointCloud &pointcloud,
+                             const ReadPointCloudOption &params);
+
+bool WritePointCloudToSPLAT(const std::string &filename,
+                            const geometry::PointCloud &pointcloud,
+                            const WritePointCloudOption &params);
+
 bool ReadPointCloudFromPTS(const std::string &filename,
                            geometry::PointCloud &pointcloud,
                            const ReadPointCloudOption &params);
diff --git a/cpp/open3d/t/io/file_format/FilePLY.cpp b/cpp/open3d/t/io/file_format/FilePLY.cpp
@@ -22,6 +22,12 @@ namespace open3d {
 namespace t {
 namespace io {
 
+namespace {
+constexpr auto ROTATION_SUFFIX_INDEX = std::char_traits<char>::length("rot_");
+constexpr auto SCALE_SUFFIX_INDEX = std::char_traits<char>::length("scale_");
+constexpr auto F_DC_SUFFIX_INDEX = std::char_traits<char>::length("f_dc_");
+constexpr auto F_REST_SUFFIX_INDEX = std::char_traits<char>::length("f_rest_");
+
 struct PLYReaderState {
     struct AttrState {
         std::string name_;
@@ -37,7 +43,7 @@ struct PLYReaderState {
 };
 
 template <typename T>
-static int ReadAttributeCallback(p_ply_argument argument) {
+int ReadAttributeCallback(p_ply_argument argument) {
     PLYReaderState *state_ptr;
     long id;
     ply_get_argument_user_data(argument, reinterpret_cast<void **>(&state_ptr),
@@ -63,7 +69,7 @@ static int ReadAttributeCallback(p_ply_argument argument) {
 
 // Some of these datatypes are supported by Tensor but are added here just
 // for completeness.
-static std::string GetDtypeString(e_ply_type type) {
+std::string GetDtypeString(e_ply_type type) {
     if (type == PLY_INT8) {
         return "int8";
     } else if (type == PLY_UINT8) {
@@ -103,7 +109,7 @@ static std::string GetDtypeString(e_ply_type type) {
     }
 }
 
-static core::Dtype GetDtype(e_ply_type type) {
+core::Dtype GetDtype(e_ply_type type) {
     // PLY_LIST attribute is not supported.
     // Currently, we are not doing datatype conversions, so some of the ply
     // datatypes are not included.
@@ -123,7 +129,8 @@ static core::Dtype GetDtype(e_ply_type type) {
     }
 }
 
-static std::tuple<std::string, int, int> GetNameStrideOffsetForAttribute(
+// Map ply attributes to Open3D point cloud attributes
+std::tuple<std::string, int, int> GetNameStrideOffsetForAttribute(
         const std::string &name) {
     // Positions attribute.
     if (name == "x") return std::make_tuple("positions", 3, 0);
@@ -140,10 +147,32 @@ static std::tuple<std::string, int, int> GetNameStrideOffsetForAttribute(
     if (name == "green") return std::make_tuple("colors", 3, 1);
     if (name == "blue") return std::make_tuple("colors", 3, 2);
 
+    // 3DGS SH DC component.
+    if (name.rfind(std::string("f_dc") + "_", 0) == 0) {
+        int offset = std::stoi(name.substr(F_DC_SUFFIX_INDEX));
+        return std::make_tuple("f_dc", 3, offset);
+    }
+    // 3DGS SH higher order components. stride = 0 => set later
+    if (name.rfind(std::string("f_rest") + "_", 0) == 0) {
+        int offset = std::stoi(name.substr(F_REST_SUFFIX_INDEX));
+        return std::make_tuple("f_rest", 0, offset);
+    }
+    // 3DGS Gaussian scale attribute.
+    if (name.rfind(std::string("scale") + "_", 0) == 0) {
+        int offset = std::stoi(name.substr(SCALE_SUFFIX_INDEX));
+        return std::make_tuple("scale", 3, offset);
+    }
+    // 3DGS Gaussian rotation as a quaternion.
+    if (name.rfind(std::string("rot") + "_", 0) == 0) {
+        int offset = std::stoi(name.substr(ROTATION_SUFFIX_INDEX));
+        return std::make_tuple("rot", 4, offset);
+    }
     // Other attribute.
     return std::make_tuple(name, 1, 0);
 }
 
+}  // namespace
+
 bool ReadPointCloudFromPLY(const std::string &filename,
                            geometry::PointCloud &pointcloud,
                            const open3d::io::ReadPointCloudOption &params) {
@@ -177,12 +206,17 @@ bool ReadPointCloudFromPLY(const std::string &filename,
     // No element with name "vertex".
     if (!element) {
         utility::LogWarning("Read PLY failed: no vertex attribute.");
+        ply_close(ply_file);
         return false;
     }
 
     std::unordered_map<std::string, bool> primary_attr_init = {
-            {"positions", false}, {"normals", false}, {"colors", false}};
+            {"positions", false}, {"normals", false}, {"colors", false},
+            {"opacity", false},   {"rot", false},     {"scale", false},
+            {"f_dc", false},      {"f_rest", false},
+    };
 
+    int f_rest_count = 0;
     p_ply_property attribute = ply_get_next_property(element, nullptr);
 
     while (attribute) {
@@ -210,39 +244,59 @@ bool ReadPointCloudFromPLY(const std::string &filename,
                         "size of {} ({}).",
                         name, size, element_name, element_size);
             }
-            const std::string attr_name = std::string(name);
 
-            std::tie(attr_state->name_, attr_state->stride_,
-                     attr_state->offset_) =
+            const std::string attr_name = std::string(name);
+            auto [name, stride, offset] =
                     GetNameStrideOffsetForAttribute(attr_name);
+            if (name == "f_rest") {
+                f_rest_count++;
+            }
+            attr_state->name_ = name;
+            attr_state->stride_ = stride;
+            attr_state->offset_ = offset;
 
-            if (primary_attr_init.count(attr_state->name_)) {
-                if (primary_attr_init.at(attr_state->name_) == false) {
+            if (primary_attr_init.count(name)) {
+                if (primary_attr_init.at(name) == false) {
                     pointcloud.SetPointAttr(
-                            attr_state->name_,
-                            core::Tensor::Empty(
-                                    {element_size, attr_state->stride_},
-                                    GetDtype(type)));
-                    primary_attr_init[attr_state->name_] = true;
+                            name, core::Tensor::Empty({element_size, stride},
+                                                      GetDtype(type)));
+                    primary_attr_init[name] = true;
                 }
             } else {
                 pointcloud.SetPointAttr(
-                        attr_state->name_,
-                        core::Tensor::Empty({element_size, attr_state->stride_},
-                                            GetDtype(type)));
+                        name, core::Tensor::Empty({element_size, stride},
+                                                  GetDtype(type)));
             }
-
-            attr_state->data_ptr_ =
-                    pointcloud.GetPointAttr(attr_state->name_).GetDataPtr();
-
+            attr_state->data_ptr_ = pointcloud.GetPointAttr(name).GetDataPtr();
             attr_state->size_ = element_size;
             attr_state->current_size_ = 0;
             state.id_to_attr_state_.push_back(attr_state);
         }
-
         attribute = ply_get_next_property(element, attribute);
     }
 
+    if (f_rest_count > 0) {
+        if (f_rest_count % 3 != 0) {
+            utility::LogWarning(
+                    "Read PLY failed: 3DGS f_rest attribute has {} elements "
+                    "per point, which is not divisible by 3.",
+                    f_rest_count);
+            ply_close(ply_file);
+            return false;
+        }
+        core::Tensor new_f_rest =
+                core::Tensor::Empty({element_size, f_rest_count / 3, 3},
+                                    core::Float32, pointcloud.GetDevice());
+        pointcloud.SetPointAttr("f_rest", new_f_rest);
+        for (auto &attr_state : state.id_to_attr_state_) {
+            if (attr_state->name_ == "f_rest") {
+                attr_state->data_ptr_ =
+                        pointcloud.GetPointAttr("f_rest").GetDataPtr();
+                attr_state->stride_ = f_rest_count;
+            }
+        }
+    }
+
     utility::CountingProgressReporter reporter(params.update_progress);
     reporter.SetTotal(element_size);
     state.progress_bar_ = &reporter;
@@ -257,10 +311,15 @@ bool ReadPointCloudFromPLY(const std::string &filename,
     ply_close(ply_file);
     reporter.Finish();
 
+    if (pointcloud.IsGaussianSplat()) {  // validates 3DGS, if present.
+        utility::LogDebug("PLY file contains a Gaussian Splat.");
+    }
+
     return true;
 }
 
-static e_ply_type GetPlyType(const core::Dtype &dtype) {
+namespace {
+e_ply_type GetPlyType(const core::Dtype &dtype) {
     if (dtype == core::UInt8) {
         return PLY_UCHAR;
     } else if (dtype == core::UInt16) {
@@ -290,6 +349,7 @@ struct AttributePtr {
     const void *data_ptr_;
     const int group_size_;
 };
+}  // namespace
 
 bool WritePointCloudToPLY(const std::string &filename,
                           const geometry::PointCloud &pointcloud,
@@ -299,12 +359,19 @@ bool WritePointCloudToPLY(const std::string &filename,
         return false;
     }
 
-    geometry::TensorMap t_map(pointcloud.GetPointAttr().Contiguous());
+    if (pointcloud.IsGaussianSplat()) {  // validates 3DGS, if present.
+        utility::LogDebug("Writing Gaussian Splat point cloud to PLY file.");
+    }
+
+    geometry::TensorMap t_map =
+            pointcloud.To(core::Device("CPU:0")).GetPointAttr().Contiguous();
 
     long num_points =
             static_cast<long>(pointcloud.GetPointPositions().GetLength());
 
-    // Make sure all the attributes have same size.
+    // Verify that standard attributes have length equal to num_points.
+    // Extra attributes must have at least 2 dimensions: (num_points, channels,
+    // ...).
     for (auto const &it : t_map) {
         if (it.first == "positions" || it.first == "normals" ||
             it.first == "colors") {
@@ -315,15 +382,20 @@ bool WritePointCloudToPLY(const std::string &filename,
                         num_points, it.first, it.second.GetLength());
                 return false;
             }
-        } else if (it.second.GetShape() != core::SizeVector({num_points, 1})) {
-            utility::LogWarning(
-                    "Write PLY failed. PointCloud contains {} attribute which "
-                    "is not supported by PLY IO. Only points, normals, colors "
-                    "and attributes with shape (num_points, 1) are supported. "
-                    "Expected shape: {} but got {}.",
-                    it.first, core::SizeVector({num_points, 1}).ToString(),
-                    it.second.GetShape().ToString());
-            return false;
+        } else {
+            auto shape = it.second.GetShape();
+            // Only tensors with shape (num_points, channels, ...) are
+            // supported.
+            if (shape.size() < 2 || shape[0] != num_points) {
+                utility::LogWarning(
+                        "Write PLY failed. PointCloud contains {} attribute "
+                        "which is not supported by PLY IO. Only points, "
+                        "normals, colors and attributes with shape "
+                        "(num_points, channels, ...) are supported. Expected "
+                        "shape: {{{}, ...}} but got {}.",
+                        it.first, num_points, it.second.GetShape().ToString());
+                return false;
+            }
         }
     }
 
@@ -375,16 +447,38 @@ bool WritePointCloudToPLY(const std::string &filename,
                          pointColorType);
     }
 
+    // Process extra attributes.
+    // Extra attributes are expected to be tensors with shape (num_points,
+    // channels) or (num_points, C, D). For multi-channel attributes, the
+    // channels are flattened, and each channel is written as a separate
+    // property (e.g., "f_rest_0", "f_rest_1", ...).
     e_ply_type attributeType;
     for (auto const &it : t_map) {
-        if (it.first != "positions" && it.first != "colors" &&
-            it.first != "normals") {
+        if (it.first == "positions" || it.first == "colors" ||
+            it.first == "normals")
+            continue;
+        auto shape = it.second.GetShape();
+        int group_size = 1;
+        if (shape.size() == 2) {
+            group_size = shape[1];
+        } else if (shape.size() >= 3) {
+            group_size = shape[1] * shape[2];
+        }
+        if (group_size == 1) {
             attribute_ptrs.emplace_back(it.second.GetDtype(),
                                         it.second.GetDataPtr(), 1);
-
             attributeType = GetPlyType(it.second.GetDtype());
             ply_add_property(ply_file, it.first.c_str(), attributeType,
                              attributeType, attributeType);
+        } else {
+            for (int ch = 0; ch < group_size; ch++) {
+                std::string prop_name = it.first + "_" + std::to_string(ch);
+                attributeType = GetPlyType(it.second.GetDtype());
+                ply_add_property(ply_file, prop_name.c_str(), attributeType,
+                                 attributeType, attributeType);
+            }
+            attribute_ptrs.emplace_back(it.second.GetDtype(),
+                                        it.second.GetDataPtr(), group_size);
         }
     }
 
diff --git a/cpp/open3d/t/io/file_format/FileSPLAT.cpp b/cpp/open3d/t/io/file_format/FileSPLAT.cpp
diff --git a/cpp/tests/t/io/PointCloudIO.cpp b/cpp/tests/t/io/PointCloudIO.cpp
