Example_video-reconstruction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f5137f06-8336-4340-b37e-51eda2edd4c7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Working Directory:  /home/peizhi/Documents/flame-head-tracker\n",
      "Jupyter environment detected. Enabling Open3D WebVisualizer.\n",
      "[Open3D INFO] WebRTC GUI backend enabled.\n",
      "[Open3D INFO] WebRTCWindowSystem: HTTP handshake server disabled.\n"
     ]
    }
   ],
   "source": [
    "## Enviroment Setup\n",
    "import os, sys\n",
    "WORKING_DIR = '/home/peizhi/Documents/flame-head-tracker'\n",
    "os.chdir(WORKING_DIR) # change the working directory to the project's absolute path\n",
    "print(\"Current Working Directory: \", os.getcwd())\n",
    "\n",
    "## Computing Device\n",
    "device = 'cuda:0'\n",
    "import torch\n",
    "torch.cuda.set_device(device) # this will solve the problem that OpenGL not on the same device with torch tensors\n",
    "\n",
    "sys.path.append(WORKING_DIR)\n",
    "sys.path.append('./utils/flame_lib/')\n",
    "sys.path.append('./utils/flame_fitting/')\n",
    "sys.path.append('./utils/face_parsing/')\n",
    "sys.path.append('./utils/decalib/')\n",
    "sys.path.append('./utils/mesh_renderer')\n",
    "sys.path.append('./utils/scene')\n",
    "\n",
    "from time import time\n",
    "\n",
    "from tracker_video import track_video_legacy   # will fit and save one frame at a time\n",
    "from tracker_video import track_video          # faster than the legacy code, save at the end\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5a59dc29-3e26-4ea0-8841-e7d470313125",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Conversion completed. 862\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n",
      "I0000 00:00:1734934561.420280  505855 gl_context_egl.cc:85] Successfully initialized EGL. Major : 1 Minor: 5\n",
      "I0000 00:00:1734934561.463554  506624 gl_context.cc:357] GL version: 3.2 (OpenGL ES 3.2 NVIDIA 550.120), renderer: NVIDIA RTX A6000/PCIe/SSE2\n",
      "W0000 00:00:1734934561.464040  505855 face_landmarker_graph.cc:174] Sets FaceBlendshapesGraph acceleration to xnnpack by default.\n",
      "INFO: Created TensorFlow Lite XNNPACK delegate for CPU.\n",
      "W0000 00:00:1734934561.472968  506627 inference_feedback_manager.cc:114] Feedback manager requires a model with a single signature inference. Disabling support for feedback tensors.\n",
      "W0000 00:00:1734934561.487236  506645 inference_feedback_manager.cc:114] Feedback manager requires a model with a single signature inference. Disabling support for feedback tensors.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "creating the FLAME Decoder\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/peizhi/Documents/flame-head-tracker/utils/flame_lib/FLAME.py:81: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  self.register_buffer('dynamic_lmk_faces_idx', torch.tensor(lmk_embeddings['dynamic_lmk_faces_idx'], dtype=torch.long))\n",
      "/home/peizhi/Documents/flame-head-tracker/utils/flame_lib/FLAME.py:82: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  self.register_buffer('dynamic_lmk_bary_coords', torch.tensor(lmk_embeddings['dynamic_lmk_bary_coords'], dtype=self.dtype))\n",
      "/home/peizhi/miniconda3/envs/tracker/lib/python3.10/site-packages/pytorch3d/io/obj_io.py:550: UserWarning: Mtl file does not exist: ./models/template.mtl\n",
      "  warnings.warn(f\"Mtl file does not exist: {f}\")\n",
      "/home/peizhi/miniconda3/envs/tracker/lib/python3.10/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
      "  warnings.warn(\n",
      "/home/peizhi/miniconda3/envs/tracker/lib/python3.10/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=None`.\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "creating the FLAME Decoder\n",
      "trained model found. load models/deca_model.tar\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/peizhi/miniconda3/envs/tracker/lib/python3.10/site-packages/pytorch3d/io/obj_io.py:550: UserWarning: Mtl file does not exist: models/template.mtl\n",
      "  warnings.warn(f\"Mtl file does not exist: {f}\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Flame Tracker ready.\n",
      "Estimating global shape code\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:16<00:00,  6.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing video: ./assets/IMG_2647.MOV\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/862 [00:00<?, ?it/s]W0000 00:00:1734934581.167225  506637 landmark_projection_calculator.cc:186] Using NORM_RECT without IMAGE_DIMENSIONS is only supported for the square ROI. Provide IMAGE_DIMENSIONS or use PROJECTION_MATRIX.\n",
      "/home/peizhi/miniconda3/envs/tracker/lib/python3.10/site-packages/google/protobuf/symbol_database.py:55: UserWarning: SymbolDatabase.GetPrototype() is deprecated. Please use message_factory.GetMessageClass() instead. SymbolDatabase.GetPrototype() will be removed soon.\n",
      "  warnings.warn('SymbolDatabase.GetPrototype() is deprecated. Please '\n",
      "  0%|          | 1/862 [00:10<2:34:25, 10.76s/it]/home/peizhi/miniconda3/envs/tracker/lib/python3.10/site-packages/google/protobuf/symbol_database.py:55: UserWarning: SymbolDatabase.GetPrototype() is deprecated. Please use message_factory.GetMessageClass() instead. SymbolDatabase.GetPrototype() will be removed soon.\n",
      "  warnings.warn('SymbolDatabase.GetPrototype() is deprecated. Please '\n",
      "100%|██████████| 862/862 [47:48<00:00,  3.33s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time used: 48.16623602708181 minutes\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "\n",
    "tracker_cfg = {\n",
    "    'mediapipe_face_landmarker_v2_path': './models/face_landmarker_v2_with_blendshapes.task',\n",
    "    'flame_model_path': './models/FLAME2020/generic_model.pkl',\n",
    "    'flame_lmk_embedding_path': './models/landmark_embedding.npy',\n",
    "    'ear_landmarker_path': './models/ear_landmarker.pth', # this is optional, if you do not want to use ear landmarks during fitting, just remove this line\n",
    "    'tex_space_path': './models/FLAME_albedo_from_BFM.npz',\n",
    "    'face_parsing_model_path': './utils/face_parsing/79999_iter.pth',\n",
    "    'uv_coord_mapping_file_path': './models/uv2vert_256.npy',\n",
    "    'template_mesh_file_path': './models/head_template.obj',\n",
    "    'result_img_size': 512,\n",
    "    'device': device,\n",
    "    ## following are used for video tracking\n",
    "    'original_fps': 60,        # input video fps\n",
    "    'subsample_fps': 30,       # subsample fps\n",
    "    'video_path': './assets/IMG_2647.MOV',  # example video file path\n",
    "    'save_path': './output',   # tracking result save path\n",
    "    'use_kalman_filter': False, # whether to use Kalman filter\n",
    "    'kalman_filter_measurement_noise_factor': 1e-5, # measurement noise level in Kalman filter \n",
    "    'kalman_filter_process_noise_factor': 1e-5,     # process noise level in Kalman filter \n",
    "}\n",
    "\n",
    "## Note that, the first frame will take longer time to process\n",
    "dt = time()\n",
    "track_video_legacy(tracker_cfg)\n",
    "print(f\"time used: {(time() - dt) / 60} minutes\")\n",
    "\n",
    "\n",
    "## Note that, the first frame will take longer time to process\n",
    "# dt = time()\n",
    "# track_video(tracker_cfg) # track_video sometimes is not as stable as track_video_legacy, I need to fix it later\n",
    "# print(f\"time used: {(time() - dt) / 60} minutes\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d21771c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tracker",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}