Added some Blender python scripts

.gitattributes

@@ -12,3 +12,4 @@ fonts export-ignore
 poses export-ignore
 views export-ignore
 images export-ignore
+blender export-ignore

README.md

@@ -1,5 +1,12 @@
 # Talking Head (3D)
 
+> [!NOTE]
+> [<img src="images/geminicompetition.jpg" width="300"/>](https://ai.google.dev/competition/projects/recycling-advisor-3d)
+>
+> A TalkingHead web app, **Recycling Advisor 3D**, is currently competing for the People's Choice Award in the Gemini API Developer Competition! - Watch the short demo video [here](https://ai.google.dev/competition/projects/recycling-advisor-3d) and give it a vote if you like it. 🙏
+
+---
+
 ### Demo Videos
 
 Video | Description
@@ -357,11 +364,26 @@ https://docs.readyplayer.me/ready-player-me/api-reference/avatars/morph-targets/
 
 The TalkingHead class supports both separated mesh and texture atlasing.
 
+Here are some Blender Python scripts that could be useful in converting
+custom models:
+
+Script | Description
+--- | ---
+[rename-mixamo-bones.py](https://github.com/met4citizen/TalkingHead/blob/main/blender/rename-mixamo-bones.py) | If your model doesn't have a compatible rig, you can auto-rig your model easily at [Mixamo](https://www.mixamo.com) and use this Blender script to rename the Mixamo bones.
+[rename-rocketbox-shapekeys.py](https://github.com/met4citizen/TalkingHead/blob/main/blender/rename-rocketbox-shapekeys.py) | Rename [Microsoft Rocketbox](https://github.com/microsoft/Microsoft-Rocketbox) model shape keys.
+[rename-avatarsdk-shapekeys.py](https://github.com/met4citizen/TalkingHead/blob/main/blender/rename-avatarsdk-shapekeys.py) | Rename [Avatar SDK MetaPerson](https://github.com/avatarsdk) model shape keys.
+[build-extras-from-arkit.py](https://github.com/met4citizen/TalkingHead/blob/main/blender/build-extras-from-arkit.py) | Build RPM extras (mouthOpen, mouthSmile, eyesClosed, eyesLookUp, eyesLookDown) from ARKit blendshapes.
+[build-visemes-from-arkit.py](https://github.com/met4citizen/TalkingHead/blob/main/blender/build-visemes-from-arkit.py) | Build Oculus visemes from ARKit blendshapes. As models are all different, you should fine-tune the script for best result. EXPERIMENTAL
+
 ---
 
 ### Appendix B: Create API Proxies with JSON Web Token (JWT) Single Sign-On (SSO)
 
-1. Make a CGI script that generates a new JSON Web Token with an expiration time (exp). See [jwt.io](https://jwt.io) for more information about JWT and libraries that best fit your needs and architecture.
+1. Make a CGI script that generates a new JSON Web Token with an expiration time (exp). See [jwt.io](https://jwt.io) for more information about JWT and libraries that best fit your needs and architecture. In my own test setup, I return the generated JWT as JSON.
+
+```json
+{ "jwt": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c" }
+```
 
 2. Protect your CGI script with some authentication scheme. Below is an example Apache 2.4 directory config that uses Basic authentication (remember to always use HTTPS/SSL!). Put your CGI script `get` in the `jwt` directory.
 

blender/build-extras-from-arkit.py

@@ -0,0 +1,70 @@
+import bpy
+
+# Override existing extras?
+override = True
+
+# Extra blend shapes
+shapekeys = [
+    { "name": "mouthOpen", "mix": [
+        { "name": "jawOpen", "value": 0.7 }
+    ]},
+    { "name": "mouthSmile", "mix": [
+        { "name": "mouthSmileLeft", "value": 1.0 },
+        { "name": "mouthSmileRight", "value": 1.0 }
+    ]},
+    { "name": "eyesClosed", "mix": [
+        { "name": "eyeBlinkLeft", "value": 1.0 },
+        { "name": "eyeBlinkRight", "value": 1.0 }
+    ]},
+    { "name": "eyesLookUp", "mix": [
+        { "name": "eyeLookUpLeft", "value": 1.0 },
+        { "name": "eyeLookUpRight", "value": 1.0 }
+    ]},
+    { "name": "eyesLookDown", "mix": [
+        { "name": "eyeLookDownLeft", "value": 1.0 },
+        { "name": "eyeLookDownRight", "value": 1.0 }
+    ]}
+]
+
+# Recursive traverse
+def traverse(x):
+    yield x
+    if hasattr(x, 'children'):
+        for c in x.children:
+            yield from traverse(c)
+
+# Has shape keys
+def hasShapekeys(x):
+    return hasattr(x, 'data') and hasattr(x.data, 'shape_keys') and hasattr(x.data.shape_keys, 'key_blocks')
+
+# Build missing blend shapes
+for r in bpy.context.scene.objects:
+    for o in traverse(r):
+        if hasShapekeys(o):
+            keys = o.data.shape_keys.key_blocks
+            for b in shapekeys:
+                name = b["name"]
+                mix = b["mix"]
+                # Override
+                if override:
+                    if not keys.get(name) is None:
+                        o.shape_key_remove(keys.get(name))
+                # Check/verify the extra doesn't already exist
+                if keys.get(name) is None:
+                    # Check if some component exists
+                    for m in mix:
+                        if not keys.get(m["name"]) is None:
+                            # Reset all shapes
+                            for k in keys:
+                                k.value = 0
+                            # Create a mixed shape
+                            for m in mix:
+                                if not keys.get(m["name"]) is None:
+                                    keys.get(m["name"]).value = m["value"]
+                            # Create a new shape key from the mix
+                            o.shape_key_add(name=name,from_mix=True)
+                            # Reset all shapes
+                            for k in keys:
+                                k.value = 0
+                            break
+

blender/build-visemes-from-arkit.py

@@ -0,0 +1,136 @@
+import bpy
+
+# Override existing visemes?
+override = False
+
+# Oculus viseme blendshapes
+shapekeys = [
+    { "name": "viseme_aa", "mix": [
+        { "name": "jawOpen", "value": 0.6 }
+    ]},
+    { "name": "viseme_E", "mix": [
+        { "name": "mouthPressLeft", "value": 0.8 },
+        { "name": "mouthPressRight", "value": 0.8 },
+        { "name": "mouthDimpleLeft", "value": 1.0 },
+        { "name": "mouthDimpleRight", "value": 1.0 },
+        { "name": "jawOpen", "value": 0.3 }
+    ]},
+    { "name": "viseme_I", "mix": [
+        { "name": "mouthPressLeft", "value": 0.6 },
+        { "name": "mouthPressRight", "value": 0.6 },
+        { "name": "mouthDimpleLeft", "value": 0.6 },
+        { "name": "mouthDimpleRight", "value": 0.6 },
+        { "name": "jawOpen", "value": 0.2 }
+    ]},
+    { "name": "viseme_O", "mix": [
+        { "name": "mouthPucker", "value": 1.0 },
+        { "name": "jawForward", "value": 0.6 },
+        { "name": "jawOpen", "value": 0.2 }
+    ]},
+    { "name": "viseme_U", "mix": [
+        { "name": "mouthFunnel", "value": 1.0 }
+    ]},
+    { "name": "viseme_PP", "mix": [
+        { "name": "mouthRollLower", "value": 0.8 },
+        { "name": "mouthRollUpper", "value": 0.8 },
+        { "name": "mouthUpperUpLeft", "value": 0.3 },
+        { "name": "mouthUpperUpRight", "value": 0.3 }
+    ]},
+    { "name": "viseme_FF", "mix": [
+        { "name": "mouthPucker", "value": 1.0 },
+        { "name": "mouthShrugUpper", "value": 1.0 },
+        { "name": "mouthLowerDownLeft", "value": 0.2 },
+        { "name": "mouthLowerDownRight", "value": 0.2 },
+        { "name": "mouthDimpleLeft", "value": 1.0 },
+        { "name": "mouthDimpleRight", "value": 1.0 },
+        { "name": "mouthRollLower", "value": 1.0 }
+    ]},
+    { "name": "viseme_DD", "mix": [
+        { "name": "mouthPressLeft", "value": 0.8 },
+        { "name": "mouthPressRight", "value": 0.8 },
+        { "name": "mouthFunnel", "value": 0.5 },
+        { "name": "jawOpen", "value": 0.2 }
+    ]},
+    { "name": "viseme_SS", "mix": [
+        { "name": "mouthPressLeft", "value": 0.8 },
+        { "name": "mouthPressRight", "value": 0.8 },
+        { "name": "mouthLowerDownLeft", "value": 0.5 },
+        { "name": "mouthLowerDownRight", "value": 0.5 },
+        { "name": "jawOpen", "value": 0.1 }
+    ]},
+    { "name": "viseme_TH", "mix": [
+        { "name": "mouthRollUpper", "value": 0.6 },
+        { "name": "jawOpen", "value": 0.2 },
+        { "name": "tongueOut", "value": 0.4 }
+    ]},
+    { "name": "viseme_CH", "mix": [
+        { "name": "mouthPucker", "value": 0.5 },
+        { "name": "jawOpen", "value": 0.2 }
+    ]},
+    { "name": "viseme_RR", "mix": [
+        { "name": "mouthPucker", "value": 0.5 },
+        { "name": "jawOpen", "value": 0.2 }
+    ]},
+    { "name": "viseme_kk", "mix": [
+        { "name": "mouthLowerDownLeft", "value": 0.4 },
+        { "name": "mouthLowerDownRight", "value": 0.4 },
+        { "name": "mouthDimpleLeft", "value": 0.3 },
+        { "name": "mouthDimpleRight", "value": 0.3 },
+        { "name": "mouthFunnel", "value": 0.3 },
+        { "name": "mouthPucker", "value": 0.3 },
+        { "name": "jawOpen", "value": 0.15 }
+    ]},
+    { "name": "viseme_nn", "mix": [
+        { "name": "mouthLowerDownLeft", "value": 0.4 },
+        { "name": "mouthLowerDownRight", "value": 0.4 },
+        { "name": "mouthDimpleLeft", "value": 0.3 },
+        { "name": "mouthDimpleRight", "value": 0.3 },
+        { "name": "mouthFunnel", "value": 0.3 },
+        { "name": "mouthPucker", "value": 0.3 },
+        { "name": "jawOpen", "value": 0.15 },
+        { "name": "tongueOut", "value": 0.2 }
+    ]},
+    { "name": "viseme_sil", "mix": [] }
+]
+
+# Recursive traverse
+def traverse(x):
+    yield x
+    if hasattr(x, 'children'):
+        for c in x.children:
+            yield from traverse(c)
+
+# Has shape keys
+def hasShapekeys(x):
+    return hasattr(x, 'data') and hasattr(x.data, 'shape_keys') and hasattr(x.data.shape_keys, 'key_blocks')
+
+# Build missing blend shapes
+for r in bpy.context.scene.objects:
+    for o in traverse(r):
+        if hasShapekeys(o):
+            keys = o.data.shape_keys.key_blocks
+            for b in shapekeys:
+                name = b["name"]
+                mix = b["mix"]
+                # Override
+                if override:
+                    if not keys.get(name) is None:
+                        o.shape_key_remove(keys.get(name))
+                # Check/verify the extra doesn't already exist
+                if keys.get(name) is None:
+                    # Check if some component exists
+                    for m in mix:
+                        if not keys.get(m["name"]) is None:
+                            # Reset all shapes
+                            for k in keys:
+                                k.value = 0
+                            # Create a mixed shape
+                            for m in mix:
+                                if not keys.get(m["name"]) is None:
+                                    keys.get(m["name"]).value = m["value"]
+                            # Create a new shape key from the mix
+                            o.shape_key_add(name=name,from_mix=True)
+                            # Reset all shapes
+                            for k in keys:
+                                k.value = 0
+                            break

blender/rename-avatarsdk-shapekeys.py

@@ -0,0 +1,36 @@
+import bpy
+
+# Shape key maps for Avatar SDK
+shapekeyMap = [
+    [ "sil", "viseme_sil" ], [ "PP", "viseme_PP" ], [ "FF", "viseme_FF" ],
+    [ "TH", "viseme_TH" ], [ "DD", "viseme_DD" ], [ "kk", "viseme_kk" ],
+    [ "CH", "viseme_CH" ], [ "SS", "viseme_SS" ], [ "nn", "viseme_nn" ],
+    [ "RR", "viseme_RR" ], [ "aa", "viseme_aa" ], [ "E", "viseme_E" ],
+    [ "ih", "viseme_I" ], [ "oh", "viseme_O" ], [ "ou", "viseme_U" ],
+]
+
+# Recursive traverse
+def traverse(x):
+    yield x
+    if hasattr(x, 'children'):
+        for c in x.children:
+            yield from traverse(c)
+
+# Has shape keys
+def hasShapekeys(x):
+    return hasattr(x, 'data') and hasattr(x.data, 'shape_keys') and hasattr(x.data.shape_keys, 'key_blocks')
+
+# Rename avatar root
+idx = bpy.context.scene.objects.find("AvatarRoot")
+if idx != -1:
+    bpy.context.scene.objects[idx].name = "Armature"
+
+# Rename shape keys
+for r in bpy.context.scene.objects:
+    for o in traverse(r):
+        if hasShapekeys(o):
+            keys = o.data.shape_keys.key_blocks
+            for m in shapekeyMap:
+                idx = keys.find(m[0])
+                if idx != -1:
+                    keys[idx].name = m[1]

blender/rename-cc4-shapekeys.py

@@ -0,0 +1,77 @@
+import bpy
+
+# Shape key map for Microsoft Rocketbox
+shapekeyMap = [
+[ "A01_Brow_Inner_Up", "browInnerUp" ],
+[ "A02_Brow_Down_Left", "browDownLeft" ],
+[ "A03_Brow_Down_Right", "browDownRight" ],
+[ "A04_Brow_Outer_Up_Left", "browOuterUpLeft" ],
+[ "A05_Brow_Outer_Up_Right", "browOuterUpRight" ],
+[ "A06_Eye_Look_Up_Left", "eyeLookUpLeft" ],
+[ "A07_Eye_Look_Up_Right", "eyeLookUpRight" ],
+[ "A08_Eye_Look_Down_Left", "eyeLookDownLeft" ],
+[ "A09_Eye_Look_Down_Right", "eyeLookDownRight" ],
+[ "A10_Eye_Look_Out_Left", "eyeLookOutLeft" ],
+[ "A11_Eye_Look_In_Left", "eyeLookInLeft" ],
+[ "A12_Eye_Look_In_Right", "eyeLookInRight" ],
+[ "A13_Eye_Look_Out_Right", "eyeLookOutRight" ],
+[ "A14_Eye_Blink_Left", "eyeBlinkLeft" ],
+[ "A15_Eye_Blink_Right", "eyeBlinkRight" ],
+[ "A16_Eye_Squint_Left", "eyeSquintLeft" ],
+[ "A17_Eye_Squint_Right", "eyeSquintRight" ],
+[ "A18_Eye_Wide_Left", "eyeWideLeft" ],
+[ "A19_Eye_Wide_Right", "eyeWideRight" ],
+[ "A20_Cheek_Puff", "cheekPuff" ],
+[ "A21_Cheek_Squint_Left", "cheekSquintLeft" ],
+[ "A22_Cheek_Squint_Right", "cheekSquintRight" ],
+[ "A23_Nose_Sneer_Left", "noseSneerLeft" ],
+[ "A24_Nose_Sneer_Right", "noseSneerRight" ],
+[ "A25_Jaw_Open", "jawOpen" ],
+[ "A26_Jaw_Forward", "jawForward" ],
+[ "A27_Jaw_Left", "jawLeft" ],
+[ "A28_Jaw_Right", "jawRight" ],
+[ "A29_Mouth_Funnel", "mouthFunnel" ],
+[ "A30_Mouth_Pucker", "mouthPucker" ],
+[ "A31_Mouth_Left", "mouthLeft" ],
+[ "A32_Mouth_Right", "mouthRight" ],
+[ "A33_Mouth_Roll_Upper", "mouthRollUpper" ],
+[ "A34_Mouth_Roll_Lower", "mouthRollLower" ],
+[ "A35_Mouth_Shrug_Upper", "mouthShrugUpper" ],
+[ "A36_Mouth_Shrug_Lower", "mouthShrugLower" ],
+[ "A37_Mouth_Close", "mouthClose" ],
+[ "A38_Mouth_Smile_Left", "mouthSmileLeft" ],
+[ "A39_Mouth_Smile_Right", "mouthSmileRight" ],
+[ "A40_Mouth_Frown_Left", "mouthFrownLeft" ],
+[ "A41_Mouth_Frown_Right", "mouthFrownRight" ],
+[ "A42_Mouth_Dimple_Left", "mouthDimpleLeft" ],
+[ "A43_Mouth_Dimple_Right", "mouthDimpleRight" ],
+[ "A44_Mouth_Upper_Up_Left", "mouthUpperUpLeft" ],
+[ "A45_Mouth_Upper_Up_Right", "mouthUpperUpRight" ],
+[ "A46_Mouth_Lower_Down_Left", "mouthLowerDownLeft" ],
+[ "A47_Mouth_Lower_Down_Right", "mouthLowerDownRight" ],
+[ "A48_Mouth_Press_Left", "mouthPressLeft" ],
+[ "A49_Mouth_Press_Right", "mouthPressRight" ],
+[ "A50_Mouth_Stretch_Left", "mouthStretchLeft" ],
+[ "A51_Mouth_Stretch_Right", "mouthStretchRight" ]
+]
+
+# Recursive traverse
+def traverse(x):
+    yield x
+    if hasattr(x, 'children'):
+        for c in x.children:
+            yield from traverse(c)
+
+# Has shape keys
+def hasShapekeys(x):
+    return hasattr(x, 'data') and hasattr(x.data, 'shape_keys') and hasattr(x.data.shape_keys, 'key_blocks')
+
+# Rename shape keys
+for r in bpy.context.scene.objects:
+    for o in traverse(r):
+        if hasShapekeys(o):
+            keys = o.data.shape_keys.key_blocks
+            for m in shapekeyMap:
+                idx = keys.find(m[0])
+                if idx != -1:
+                    keys[idx].name = m[1]

blender/rename-mixamo-bones.py

@@ -0,0 +1,19 @@
+import bpy
+
+# Recursive traverse
+def traverse(x):
+    yield x
+    if hasattr(x, 'children'):
+        for c in x.children:
+            yield from traverse(c)
+
+# Has bones
+def hasBones(x):
+    return hasattr(x, 'data') and hasattr(x.data, 'bones')
+
+# Remove mixamorig: prefix
+for r in bpy.context.scene.objects:
+    for o in traverse(r):
+        if hasBones(o):
+            for b in o.data.bones:
+                b.name = b.name.replace( 'mixamorig:', '')