-
Notifications
You must be signed in to change notification settings - Fork 332
[DOCS] LTX Video Generation Docs #3232
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
4aa4e2a
514d34b
5ed8c69
c9100dc
8823294
df77aa7
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| import CodeBlock from '@theme/CodeBlock'; | ||
|
|
||
| <CodeBlock language="cpp" showLineNumbers> | ||
| {`#include "openvino/genai/video_generation/text2video_pipeline.hpp" | ||
| #include "imwrite_video.hpp" | ||
|
|
||
| int main(int argc, char* argv[]) { | ||
| const std::string models_path = argv[1], prompt = argv[2]; | ||
|
|
||
| ov::genai::Text2VideoPipeline pipe(models_path, "${props.device || 'CPU'}"); | ||
| ov::Tensor video = pipe.generate(prompt).video; | ||
|
|
||
| imwrite_video("genai_video.avi", video); | ||
| } | ||
| `} | ||
| </CodeBlock> |
| Original file line number | Diff line number | Diff line change | ||||||
|---|---|---|---|---|---|---|---|---|
| @@ -0,0 +1,33 @@ | ||||||||
| import CodeBlock from '@theme/CodeBlock'; | ||||||||
|
|
||||||||
| <CodeBlock language="python" showLineNumbers> | ||||||||
| {`import openvino_genai as ov_genai | ||||||||
| import cv2 | ||||||||
|
|
||||||||
| def save_video(filename: str, video_tensor, fps: int = 25): | ||||||||
| batch_size, num_frames, height, width, _ = video_tensor.shape | ||||||||
| video_data = video_tensor.data | ||||||||
|
|
||||||||
| for b in range(batch_size): | ||||||||
| if batch_size == 1: | ||||||||
| output_path = filename | ||||||||
| else: | ||||||||
| base, ext = filename.rsplit(".", 1) if "." in filename else (filename, "avi") | ||||||||
| output_path = f"{base}_b{b}.{ext}" | ||||||||
|
|
||||||||
| fourcc = cv2.VideoWriter_fourcc(*"MJPG") | ||||||||
| writer = cv2.VideoWriter(output_path, fourcc, fps, (width, height)) | ||||||||
|
|
||||||||
| for f in range(num_frames): | ||||||||
| writer.write(video_data[b, f]) | ||||||||
|
|
||||||||
| writer.release() | ||||||||
| print(f"Wrote {output_path} ({num_frames} frames, {width}x{height} @ {fps} fps)") | ||||||||
|
|
||||||||
|
|
||||||||
|
||||||||
| model_path = "path/to/text2video/model" |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| import Text2VideoCPP from './_text2video_cpp.mdx'; | ||
| import Text2VideoPython from './_text2video_python.mdx'; | ||
|
|
||
| ## Run Model Using OpenVINO GenAI | ||
|
|
||
| OpenVINO GenAI supports the following video generation pipeline: | ||
| - [`Text2VideoPipeline`](https://docs.openvino.ai/2026/api/genai_api/_autosummary/openvino_genai.Text2VideoPipeline.html) for creating videos from text prompts. | ||
|
|
||
| <LanguageTabs> | ||
| <TabItemPython> | ||
| <Tabs groupId="device"> | ||
| <TabItem label="CPU" value="cpu"> | ||
| <Text2VideoPython device="CPU" /> | ||
| </TabItem> | ||
| <TabItem label="GPU" value="gpu"> | ||
| <Text2VideoPython device="GPU" /> | ||
| </TabItem> | ||
| </Tabs> | ||
| </TabItemPython> | ||
| <TabItemCpp> | ||
| <Tabs groupId="device"> | ||
| <TabItem label="CPU" value="cpu"> | ||
| <Text2VideoCPP device="CPU" /> | ||
| </TabItem> | ||
| <TabItem label="GPU" value="gpu"> | ||
| <Text2VideoCPP device="GPU" /> | ||
| </TabItem> | ||
| </Tabs> | ||
| </TabItemCpp> | ||
| </LanguageTabs> | ||
|
|
||
| :::tip | ||
|
|
||
| Use CPU or GPU as devices without any other code change. | ||
|
|
||
| ::: |
| Original file line number | Diff line number | Diff line change | ||||||||
|---|---|---|---|---|---|---|---|---|---|---|
| @@ -0,0 +1,83 @@ | ||||||||||
| import GenerationConfigurationWorkflow from '@site/docs/use-cases/_shared/_generation_configuration_workflow.mdx'; | ||||||||||
|
|
||||||||||
| ## Additional Usage Options | ||||||||||
|
|
||||||||||
| :::tip | ||||||||||
| Check out [Python](https://github.com/openvinotoolkit/openvino.genai/tree/master/samples/python/video_generation) and [C++](https://github.com/openvinotoolkit/openvino.genai/tree/master/samples/cpp/video_generation) video generation samples. | ||||||||||
| ::: | ||||||||||
|
|
||||||||||
| ### Use Different Generation Parameters | ||||||||||
|
|
||||||||||
| <GenerationConfigurationWorkflow /> | ||||||||||
|
|
||||||||||
| #### Video Generation Configuration | ||||||||||
|
|
||||||||||
| You can adjust several parameters to control the video generation process, including dimensions and the number of inference steps: | ||||||||||
|
|
||||||||||
| <LanguageTabs> | ||||||||||
| <TabItemPython> | ||||||||||
| ```python | ||||||||||
| import openvino_genai as ov_genai | ||||||||||
|
|
||||||||||
| pipe = ov_genai.Text2VideoPipeline(model_path, "CPU") | ||||||||||
| fps = 25 | ||||||||||
| video = pipe.generate( | ||||||||||
| prompt, | ||||||||||
| # highlight-start | ||||||||||
| width=512, | ||||||||||
| height=512, | ||||||||||
| num_videos_per_prompt=1, | ||||||||||
| num_inference_steps=30, | ||||||||||
| num_frames=161, | ||||||||||
| guidance_scale=7.5, | ||||||||||
| frame_rate=fps | ||||||||||
| # highlight-end | ||||||||||
| ).video | ||||||||||
|
|
||||||||||
| save_video("genai_video.avi", video, fps) | ||||||||||
|
||||||||||
| save_video("genai_video.avi", video, fps) | |
| save_video("genai_video.avi", video_tensor, fps) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Please address
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
updated tensor name to video:
| save_video("genai_video.avi", video, fps) | |
| video = pipe.generate(...) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| --- | ||
| sidebar_position: 3 | ||
| --- | ||
| import OptimumCLI from '@site/src/components/OptimumCLI'; | ||
| import ConvertModelSection from '../_shared/_convert_model.mdx'; | ||
| import RunModelSection from './_sections/_run_model/index.mdx'; | ||
| import UsageOptionsSection from './_sections/_usage_options/index.mdx'; | ||
|
|
||
| # Video Generation Using Diffusers | ||
|
|
||
| ## LTX-Video Pipeline for Video Generation | ||
|
|
||
| LTX-Video is the DiT-based video generation model capable of generating high-quality videos. | ||
| This video generation pipeline turns a text prompt into a sequence of video frames: | ||
| 1. **Prompt → tokens**: the prompt is tokenized (and padded to the model’s max length). | ||
| 2. **Tokens → text embeddings**: a text encoder produces embeddings used to condition generation. | ||
| 3. **Denoising loop**: a Transformer predicts and refines video latents over multiple steps, while the scheduler controls the noise level at each step. | ||
| 4. **Latents → frames**: the final latents are decoded into RGB video frames. | ||
|
|
||
|  | ||
| *Source: [arXiv:2501.00103](https://arxiv.org/abs/2501.00103)* | ||
|
|
||
| <ConvertModelSection> | ||
| Download and convert LTX model [Lightricks/LTX-Video](https://huggingface.co/Lightricks/LTX-Video) to OpenVINO format from Hugging Face: | ||
|
|
||
| <OptimumCLI model='Lightricks/LTX-Video' outputDir='LTX_Video_ov' weightFormat='int8' trustRemoteCode /> | ||
| </ConvertModelSection> | ||
|
|
||
| <RunModelSection /> | ||
|
|
||
| <UsageOptionsSection /> |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| import Button from '@site/src/components/Button'; | ||
| import { LanguageTabs, TabItemCpp, TabItemPython } from '@site/src/components/LanguageTabs'; | ||
|
|
||
| import UseCaseCard from './UseCaseCard'; | ||
|
|
||
| import CodeExampleCpp from '@site/docs/use-cases/video-generation/_sections/_run_model/_text2video_cpp.mdx'; | ||
| import CodeExamplePython from '@site/docs/use-cases/video-generation/_sections/_run_model/_text2video_python.mdx'; | ||
|
|
||
| export const VideoGeneration = () => ( | ||
| <UseCaseCard> | ||
| <UseCaseCard.Title>Video Generation Using Diffusers</UseCaseCard.Title> | ||
| <UseCaseCard.Description> | ||
| Create videos with LTX-Video model using a diffusion-based generation pipeline | ||
| to produce high-quality clips for creative storytelling, marketing content, product demos, and rapid visual prototyping. | ||
| </UseCaseCard.Description> | ||
| <UseCaseCard.Features> | ||
| <li>Support for text-to-video pipeline</li> | ||
| <li>Control video generation by adjusting parameters (dimensions, iterations, etc.)</li> | ||
| <li>Generate videos per one request</li> | ||
| </UseCaseCard.Features> | ||
| <UseCaseCard.Code> | ||
| <LanguageTabs> | ||
| <TabItemPython> | ||
| <CodeExamplePython /> | ||
| </TabItemPython> | ||
| <TabItemCpp> | ||
| <CodeExampleCpp /> | ||
| </TabItemCpp> | ||
| </LanguageTabs> | ||
| </UseCaseCard.Code> | ||
| <UseCaseCard.Actions> | ||
| <Button label="Explore Use Case" link="docs/use-cases/video-generation" variant="primary" /> | ||
| <Button label="View Code Samples" link="docs/samples" variant="primary" outline /> | ||
| </UseCaseCard.Actions> | ||
| </UseCaseCard> | ||
| ); |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Please also add Video Generation use case to the home page and reuse
_text2video_cpp.mdx&_text2video_python.mdxsnippets.Here is the
UseCasesSection- https://github.com/likholat/openvino.genai/blob/video_gen_docs/site/src/pages/_sections/UseCasesSection/index.tsx#L21 , just add new componentVideoGenerationsimilary toImageGeneration