[Version] Bump version to v0.6.2 (#28)

* Refactor SMPL-X parameter refinement

* Add `ConverterBlender` and fix bugs for refine_smplx

* Refactor Blender script: Added support for refining SMPL-XL model and exporting vertices

* Fix coordinate system conversion of unreal

* Revert "Fix coordinate system conversion of unreal"

This reverts commit cf9edee.

* Fix converter for unreal

* Fix vector type check in ConverterMotion class

* add TODO

* add ppm for lineart

* Add WHAC: World-grounded Humans and Cameras project

* Add lineart material and audio test

* Update lineart material path in constants.py

* tmp update

* [Fix] bugs

* add todo

* replace assert error with warning in motion_frame

* doc

* [Fix] add curve_names in skeleton_name

* Bump version to 0.6.2 in XRFeitoriaBpy and XRFeitoriaUnreal

* Add MotionFrame definition for morph targets

* Refactor get_skeleton_names function in utils_actor.py

* update docs

* Save sequence asset when closing in XRFeitoriaUnreal/Content/Python/sequence.py

---------

Co-authored-by: esukastudio <[email protected]>

README.md

@@ -114,6 +114,7 @@ Please refer to [FAQ](https://xrfeitoria.readthedocs.io/en/latest/faq.html).
 | [MatrixCity: A Large-scale City Dataset for City-scale Neural Rendering and Beyond](https://city-super.github.io/matrixcity/) | <a href=https://city-super.github.io/matrixcity/><img src="https://city-super.github.io/matrixcity/img/teaser.jpg"></a> | Unreal Engine |
 | [HumanLiff: Layer-wise 3D Human Generation with Diffusion Model](https://skhu101.github.io/HumanLiff/) | <a href=https://skhu101.github.io/HumanLiff/><img src="https://skhu101.github.io/HumanLiff/HumanLiff%20-%20Project%20Page_files/SHERF_teaser.png"/></a> | Blender |
 | [PrimDiffusion: Volumetric Primitives Diffusion for 3D Human Generation](https://frozenburning.github.io/projects/primdiffusion/) | <a href=https://frozenburning.github.io/projects/primdiffusion/><img src="https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrfeitoria/pics/PrimDiffusion.png"></a> | Blender |
+| [WHAC: World-grounded Humans and Cameras](https://wqyin.github.io/projects/WHAC/) | <a href=https://wqyin.github.io/projects/WHAC/><img src="https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrfeitoria/pics/whac.jpg"></a> | Unreal Engine |
 
 ## License
 

docs/en/faq.rst

@@ -150,15 +150,3 @@ you can set the environment variable ``BLENDER_PORT`` or ``UNREAL_PORT`` to chan
         .. code-block:: powershell
 
             $env:BLENDER_PORT=50051; python xxx.py
-
------------
-
-Known Issues
--------------
-
-Inaccurate vertices in Unreal Engine
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-.. _XRFeitoriaUnreal: https://github.com/openxrlab/xrfeitoria/tree/main/src/XRFeitoriaUnreal
-
-The vertices exported from actors in Unreal Engine may be inaccurate. This is a issue in `XRFeitoriaUnreal`_ plugin.

docs/en/index.rst

@@ -50,6 +50,8 @@ Requirements
 - ``Python >= 3.8``
 - (optional) ``Unreal Engine >= 5.1``
     |check_| Windows
+
+    |check_| Linux
 - (optional) ``Blender >= 3.0``
     |check_| Windows
 

samples/blender/07_amass.py

@@ -16,8 +16,8 @@
 import xrfeitoria as xf
 from xrfeitoria.data_structure.models import RenderPass
 from xrfeitoria.rpc import remote_blender
-from xrfeitoria.utils import setup_logger
-from xrfeitoria.utils.anim import dump_humandata, load_amass_motion
+from xrfeitoria.utils import ConverterBlender, setup_logger
+from xrfeitoria.utils.anim import dump_humandata, load_amass_motion, refine_smpl_x
 
 # prepare the assets
 ####################
@@ -42,6 +42,8 @@
 saved_humandata_file = seq_dir / 'smplx' / 'output.npz'
 saved_blend_file = seq_dir / 'output.blend'
 
+n_frames = 10  # number of frames to render
+
 
 @remote_blender()
 def apply_scale(actor_name: str):
@@ -57,34 +59,47 @@ def main(background: bool = False):
 
     motion = load_amass_motion(amass_file)
     motion.convert_fps(30)  # convert the motion from 120fps (amass) to 30fps
-    motion.cut_motion(end_frame=10)  # cut the motion to 10 frames, for demonstration purpose
+    # motion.cut_transl(end_frame=n_frames)  # cut the motion to the length of n_frames
+    motion.sample_motion(n_frames)  # sample the motion to the length of n_frames
     motion_data = motion.get_motion_data()
 
     # modify this to your blender executable path
     xf_runner = xf.init_blender(
-        exec_path='C:/Program Files/Blender Foundation/Blender 3.6/blender.exe', background=background
+        exec_path='C:/Program Files/Blender Foundation/Blender 3.6/blender.exe',
+        background=background,
     )
 
-    with xf_runner.sequence(seq_name=seq_name, seq_length=motion.n_frames) as seq:
+    location = (0.2, 0.3, 0.5)
+    rotation = (20, 30, 40)
+
+    with xf_runner.sequence(seq_name=seq_name, seq_length=n_frames) as seq:
         # Import SMPL-XL model
-        actor = xf_runner.Actor.import_from_file(smpl_xl_file)
+        actor = xf_runner.Actor.import_from_file(smpl_xl_file, location=location, rotation=rotation)
         apply_scale(actor.name)  # SMPL-XL model is imported with scale, we need to apply scale to it
 
         # Apply motion data to the actor
         logger.info('Applying motion data')
         xf_runner.utils.apply_motion_data_to_actor(motion_data=motion_data, actor_name=actor.name)
         # Save the motion data as annotation in humandata format defined in https://github.com/open-mmlab/mmhuman3d/blob/main/docs/human_data.md
         dump_humandata(motion, save_filepath=saved_humandata_file, meta_filepath=smpl_xl_meta_file)
+        refine_smpl_x(
+            smpl_x_filepath=saved_humandata_file,
+            meta_filepath=smpl_xl_meta_file,
+            offset_location=ConverterBlender.location_from_blender(location),
+            offset_rotation=ConverterBlender.rotation_from_blender(rotation),
+            replace_smpl_x_file=True,
+        )
 
-        # Modify the frame range to the length of the motion
-        frame_start, frame_end = xf_runner.utils.get_keys_range()
-        xf_runner.utils.set_frame_range(frame_start, frame_end)
         # env
         xf_runner.utils.set_env_color(color=(1, 1, 1, 1))
 
         #
         camera = xf_runner.Camera.spawn(location=(0, -2.5, 0.6), rotation=(90, 0, 0))
 
+        #
+        logger.info('Exporting vertices')
+        xf_runner.utils.export_vertices(export_path=seq_dir / 'vertices')
+
         seq.add_to_renderer(
             output_path=output_path,
             resolution=(1920, 1080),

samples/unreal/07_amass.py

@@ -91,7 +91,8 @@ def main(background: bool = False):
 
     # refine smplx parameters
     refine_smpl_x_from_actor_info(
-        smpl_x_file=saved_humandata_file,
+        smpl_x_filepath=saved_humandata_file,
+        meta_filepath=smpl_xl_meta_file,
         actor_info_file=seq_dir / 'actor_infos' / f'{actor_name}.npz',
         replace_smpl_x_file=True,
     )

src/XRFeitoriaBpy/__init__.py

@@ -4,11 +4,11 @@
 bl_info = {
     'name': 'XRFeitoriaBpy',
     'author': 'OpenXRLab',
-    'version': (0, 6, 1),
+    'version': (0, 6, 2),
     'blender': (3, 3, 0),
     'category': 'Tools',
 }
-__version__ = version = '0.6.1'
+__version__ = version = '0.6.2'
 
 
 def register():

src/XRFeitoriaBpy/constants.py

@@ -8,6 +8,8 @@
 
 Tuple3 = Tuple[float, float, float]
 PathLike = Union[str, Path]
+# MotionFrame = {'bone_name': {'location': [x, y, z], 'rotation': [w, x, y, z]}}
+# MotionFrame = {'morph_target_name': {'curve': float}}
 MotionFrame = Dict[str, Dict[str, Union[float, List[float]]]]
 
 

src/XRFeitoriaUnreal/Content/Python/constants.py

@@ -30,6 +30,7 @@ def get_plugin_path() -> Tuple[Path, Path, Path]:
     'specular': f'/{PLUGIN_NAME}/Materials/PPM_specular',
     'tangent': f'/{PLUGIN_NAME}/Materials/PPM_tangent',
     'basecolor': f'/{PLUGIN_NAME}/Materials/PPM_basecolor',
+    'lineart': f'/{PLUGIN_NAME}/Materials/PPM_lineart_default',
 }
 SHAPE_PATHS = {
     'cube': '/Engine/BasicShapes/Cube',
@@ -122,6 +123,7 @@ class UnrealRenderLayerEnum(EnumBase):
     specular = 'specular'
     tangent = 'tangent'
     basecolor = 'basecolor'
+    lineart = 'lineart'
 
     vertices = 'vertices'
     skeleton = 'skeleton'
@@ -197,6 +199,8 @@ def __post_init__(self):
 
 
 TransformKeys = Union[List[SequenceTransformKey], SequenceTransformKey]
+# MotionFrame = {'bone_name': {'location': [x, y, z], 'rotation': [w, x, y, z]}}
+# MotionFrame = {'morph_target_name': {'curve': float}}
 MotionFrame = Dict[str, Dict[str, Union[float, List[float]]]]
 color_type = TypedDict(
     'color',

src/XRFeitoriaUnreal/Content/Python/sequence.py

@@ -459,21 +459,22 @@ def add_fk_motion_to_binding(binding: unreal.SequencerBindingProxy, motion_data:
         )
     )
     rig_section: unreal.MovieSceneControlRigParameterSection = rig_track.get_section_to_key()
-    param_names = list(rig_section.get_parameter_names())
-    for bone_name, bone_data in motion_data[0].items():
-        if 'curve' in bone_data.keys():
-            bone_name = f'{bone_name}_CURVE_CONTROL'
-        else:
-            bone_name = f'{bone_name}_CONTROL'
-        assert bone_name in param_names, RuntimeError(f'bone name: {bone_name} not in param names: {param_names}')
+    # Note: assert bone_name is now achieved in function `xf_runner.SequenceUnreal.spawn_actor_with_keys` and `xf_runner.SequenceUnreal.spawn_actor`
+    # param_names = list(rig_section.get_parameter_names())
+    # for bone_name, bone_data in motion_data[0].items():
+    #     if 'curve' in bone_data.keys():
+    #         bone_name = f'{bone_name}_CURVE_CONTROL'
+    #     else:
+    #         bone_name = f'{bone_name}_CONTROL'
+    #     assert bone_name in param_names, RuntimeError(f'bone name: {bone_name} not in param names: {param_names}')
 
     if ENGINE_MAJOR_VERSION == 5 and ENGINE_MINOR_VERSION < 2:
         msg = 'FKRigExecuteMode is not supported in < UE5.2, may cause unexpected result using FK motion.'
         unreal.log_warning(msg)
     else:
         rig_proxies = unreal.ControlRigSequencerLibrary.get_control_rigs(binding.sequence)
         for rig_proxy in rig_proxies:
-            ### TODO: judge if the track belongs to this actor
+            # TODO: judge if the track belongs to this actor
             unreal.ControlRigSequencerLibrary.set_control_rig_apply_mode(
                 rig_proxy.control_rig, unreal.ControlRigFKRigExecuteMode.ADDITIVE
             )
@@ -974,6 +975,7 @@ def save(cls) -> None:
     def close(cls) -> None:
         if cls.sequence is not None:
             unreal.LevelSequenceEditorBlueprintLibrary.close_level_sequence()
+            unreal.EditorAssetLibrary.save_asset(cls.sequence_path)  # XXX: save the sequence asset
             del cls.sequence
         cls.map_path = None
         cls.sequence_path = None

src/XRFeitoriaUnreal/Content/Python/utils_actor.py

@@ -50,6 +50,31 @@ def get_actor_mesh_component(
     return mesh_component
 
 
+def get_skeleton_names(actor_asset_path: str) -> List[str]:
+    """Retrieves the names of the bones in the skeleton of a SkeletalMeshActor (also can
+    be child class of it).
+
+    Args:
+        actor_asset_path (str): The asset path of the SkeletalMeshActor.
+
+    Returns:
+        List[str]: A set of bone names in the skeleton.
+    """
+    actor = spawn_actor_from_object(unreal.load_asset(actor_asset_path))
+    try:
+        assert isinstance(actor, unreal.SkeletalMeshActor), f'{actor.get_name()} is not a SkeletalMeshActor'
+        skeletal_mesh = actor.skeletal_mesh_component.skeletal_mesh
+        bone_names = [str(bone_name) for bone_name in skeletal_mesh.skeleton.get_reference_pose().get_bone_names()]
+        curve_names = [morph.get_name() for morph in skeletal_mesh.morph_targets]
+        skeleton_names = bone_names + curve_names
+    except Exception as e:
+        unreal.log_warning(f'Error: {e}')
+        skeleton_names = []
+    finally:
+        destroy_actor(actor)
+    return skeleton_names
+
+
 def get_z_by_raycast(x: float, y: float, debug: bool = False):
     z_infinity_positive = 100000
     z_infinity_negative = -100000