Skip to content

Commit 3ca6562

Browse files
authored
add flex.2 tutorial (#2959)
CVS-167875
1 parent 51af719 commit 3ca6562

File tree

6 files changed

+1007
-1
lines changed

6 files changed

+1007
-1
lines changed

.ci/ignore_treon_docker.txt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,4 +88,5 @@ notebooks/olmocr-pdf-vlm/olmocr-pdf-vlm.ipynb
8888
notebooks/minicpm-o-omnimodal-chatbot/minicpm-o-omnimodal-chatbot.ipynb
8989
notebooks/kokoro/kokoro.ipynb
9090
notebooks/qwen2.5-omni-chatbot/qwen2.5-omni-chatbot.ipynb
91-
notebooks/intern-video2-classiciation/intern-video2-classification.ipynb
91+
notebooks/intern-video2-classiciation/intern-video2-classification.ipynb
92+
notebooks/flex.2-image-generation/flex.2-image-generation.ipynb

.ci/skipped_notebooks.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -542,6 +542,10 @@
542542
- macos-13
543543
- ubuntu-22.04
544544
- windows-2019
545+
- notebook: notebooks/flex.2-image-generation/flex.2-image-generation.ipynb
546+
skips:
547+
- python:
548+
- "3.9"
545549
- notebook: notebooks/openvoice2-and-melotts/openvoice2-and-melotts.ipynb
546550
skips:
547551
- os:
Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# Image generation with universal control using Flex.2 and OpenVINO
2+
3+
<div class="alert alert-block alert-danger"> <b>Important note:</b> This notebook requires python >= 3.11. Please make sure that your environment fulfill to this requirement before running it </div>
4+
5+
Flex.2 is flexible text-to-image diffusion model based on Flux model architecture with built in support inpainting and universal control - model accepts pose, line, and depth inputs.
6+
7+
<img src="https://github.com/user-attachments/assets/6a9ab66a-387a-4538-8625-2bb3a16072b5" width="1024">
8+
9+
More details about model can be found in [model card](https://huggingface.co/ostris/Flex.2-preview).
10+
11+
In this tutorial we consider how to convert and optimize Flex.2 model using OpenVINO.
12+
13+
>**Note**: Some demonstrated models can require at least 32GB RAM for conversion and running.
14+
15+
### Notebook Contents
16+
17+
In this demonstration, you will learn how to perform text-to-image generation using Flex.2 and OpenVINO.
18+
19+
Example of model work:
20+
21+
![](https://github.com/user-attachments/assets/140685b7-2c5d-4cef-86fb-33df0849ec1a)
22+
23+
The tutorial consists of the following steps:
24+
25+
- Install prerequisites
26+
- Collect Pytorch model pipeline
27+
- Convert model to OpenVINO intermediate representation (IR) format
28+
- Compress weights using NNCF
29+
- Prepare OpenVINO Inference pipeline
30+
- Run Image generation
31+
- Launch interactive demo
32+
33+
## Installation Instructions
34+
35+
This is a self-contained example that relies solely on its own code.</br>
36+
We recommend running the notebook in a virtual environment. You only need a Jupyter server to start.
37+
For further details, please refer to [Installation Guide](../../README.md).
38+
39+
<img referrerpolicy="no-referrer-when-downgrade" src="https://static.scarf.sh/a.png?x-pxid=5b5a4db0-7875-4bfb-bdbd-01698b5b1a77&file=notebooks/flex.2-image-generation/README.md" />

notebooks/flex.2-image-generation/flex.2-image-generation.ipynb

Lines changed: 702 additions & 0 deletions
Large diffs are not rendered by default.
Lines changed: 252 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,252 @@
1+
import gradio as gr
2+
import numpy as np
3+
import torch
4+
import random
5+
from PIL import Image, ImageFilter
6+
7+
MAX_SEED = np.iinfo(np.int32).max
8+
MAX_IMAGE_SIZE = 2048
9+
10+
11+
def make_demo(pipe):
12+
def infer(
13+
edit_images,
14+
prompt,
15+
seed=42,
16+
randomize_seed=False,
17+
width=1024,
18+
height=1024,
19+
guidance_scale=3.5,
20+
control_strength=0.5,
21+
control_stop=0.33,
22+
num_inference_steps=50,
23+
progress=gr.Progress(track_tqdm=True),
24+
):
25+
image = edit_images["background"].convert("RGB")
26+
mask = Image.fromarray(np.array(edit_images["layers"][-1])[:, :, -1])
27+
if randomize_seed:
28+
seed = random.randint(0, MAX_SEED)
29+
out_image = pipe(
30+
prompt=prompt,
31+
inpaint_image=image,
32+
inpaint_mask=mask,
33+
height=height,
34+
width=width,
35+
guidance_scale=guidance_scale,
36+
control_strength=control_strength,
37+
control_stop=control_stop,
38+
num_inference_steps=num_inference_steps,
39+
generator=torch.Generator("cpu").manual_seed(seed),
40+
).images[0]
41+
return (image, out_image), seed
42+
43+
css = """
44+
:root {
45+
--primary-color: #7E57C2;
46+
--secondary-color: #5E35B1;
47+
--accent-color: #B39DDB;
48+
--background-color: #F5F5F7;
49+
--card-background: #FFFFFF;
50+
--text-color: #333333;
51+
--shadow: 0 4px 12px rgba(0, 0, 0, 0.08);
52+
--radius: 12px;
53+
}
54+
body {
55+
font-family: 'Inter', system-ui, sans-serif;
56+
background-color: var(--background-color);
57+
}
58+
#col-container {
59+
margin: 0 auto;
60+
max-width: 1200px;
61+
padding: 0;
62+
}
63+
.container {
64+
background-color: var(--card-background);
65+
border-radius: var(--radius);
66+
box-shadow: var(--shadow);
67+
padding: 24px;
68+
margin-bottom: 24px;
69+
}
70+
.header-container {
71+
background: linear-gradient(135deg, var(--primary-color), var(--secondary-color));
72+
border-radius: var(--radius);
73+
padding: 32px;
74+
margin-bottom: 24px;
75+
color: white;
76+
text-align: center;
77+
box-shadow: var(--shadow);
78+
}
79+
.header-container h1 {
80+
font-weight: 700;
81+
font-size: 2.5rem;
82+
margin-bottom: 8px;
83+
background: linear-gradient(to right, #ffffff, #e0e0e0);
84+
-webkit-background-clip: text;
85+
background-clip: text;
86+
-webkit-text-fill-color: transparent;
87+
}
88+
.header-container p {
89+
font-size: 1.1rem;
90+
opacity: 0.92;
91+
margin-bottom: 16px;
92+
}
93+
.header-container a {
94+
color: var(--accent-color);
95+
text-decoration: underline;
96+
transition: opacity 0.2s;
97+
}
98+
.header-container a:hover {
99+
opacity: 0.8;
100+
}
101+
.btn-primary {
102+
background: linear-gradient(90deg, var(--primary-color), var(--secondary-color));
103+
border: none;
104+
border-radius: 8px;
105+
color: white;
106+
font-weight: 600;
107+
padding: 12px 24px;
108+
font-size: 16px;
109+
cursor: pointer;
110+
transition: all 0.3s ease;
111+
box-shadow: 0 4px 12px rgba(126, 87, 194, 0.3);
112+
}
113+
.btn-primary:hover {
114+
transform: translateY(-2px);
115+
box-shadow: 0 6px 16px rgba(126, 87, 194, 0.4);
116+
}
117+
.image-editor-container {
118+
border-radius: var(--radius);
119+
overflow: hidden;
120+
box-shadow: var(--shadow);
121+
}
122+
.prompt-container {
123+
background-color: var(--card-background);
124+
border-radius: var(--radius);
125+
padding: 16px;
126+
box-shadow: var(--shadow);
127+
margin-top: 16px;
128+
}
129+
.result-container {
130+
border-radius: var(--radius);
131+
overflow: hidden;
132+
box-shadow: var(--shadow);
133+
}
134+
.settings-container {
135+
background-color: var(--card-background);
136+
border-radius: var(--radius);
137+
padding: 20px;
138+
box-shadow: var(--shadow);
139+
margin-top: 16px;
140+
}
141+
.accordion-header {
142+
font-weight: 600;
143+
color: var(--primary-color);
144+
}
145+
/* Custom slider styling */
146+
input[type="range"] {
147+
height: 6px;
148+
border-radius: 3px;
149+
background: linear-gradient(90deg, var(--primary-color), var(--secondary-color));
150+
}
151+
input[type="range"]::-webkit-slider-thumb {
152+
background: var(--primary-color);
153+
border: 2px solid white;
154+
height: 18px;
155+
width: 18px;
156+
}
157+
.footer {
158+
text-align: center;
159+
padding: 24px;
160+
color: #777;
161+
font-size: 14px;
162+
}
163+
/* Animate the result transition */
164+
@keyframes fadeIn {
165+
from { opacity: 0; }
166+
to { opacity: 1; }
167+
}
168+
.result-animation {
169+
animation: fadeIn 0.5s ease-in-out;
170+
}
171+
"""
172+
173+
with gr.Blocks(css=css, theme=gr.themes.Monochrome()) as demo:
174+
with gr.Column(elem_id="col-container"):
175+
# Header with gradient
176+
with gr.Column(elem_classes=["header-container"]):
177+
gr.HTML(
178+
"""
179+
<h1>Flex.2 Preview Inpainting OpenVINO Demo</h1>
180+
"""
181+
)
182+
# Main interface container
183+
with gr.Column(elem_classes=["container"]):
184+
with gr.Row():
185+
# Left column: Input
186+
with gr.Column(scale=1):
187+
edit_image = gr.ImageEditor(
188+
label="Upload and draw mask for inpainting",
189+
type="pil",
190+
sources=["upload", "webcam"],
191+
image_mode="RGB",
192+
layers=False,
193+
brush=gr.Brush(colors=["#FFFFFF"], color_mode="fixed"),
194+
height=500,
195+
)
196+
197+
with gr.Column(elem_classes=["prompt-container"]):
198+
prompt = gr.Text(
199+
label="Your creative prompt",
200+
show_label=True,
201+
max_lines=1,
202+
placeholder="Describe what you want to generate...",
203+
container=True,
204+
)
205+
206+
run_button = gr.Button("✨ Generate", elem_classes=["btn-primary"])
207+
# Right column: Output
208+
with gr.Column(scale=1, elem_classes=["result-container"]):
209+
result = gr.ImageSlider(label="Before & After", type="pil", image_mode="RGB", elem_classes=["result-animation"])
210+
211+
# Advanced settings in a nice container
212+
with gr.Column(elem_classes=["settings-container"]):
213+
with gr.Accordion("Advanced Settings", open=False, elem_classes=["accordion-header"]):
214+
with gr.Column():
215+
with gr.Row():
216+
seed = gr.Slider(
217+
label="Seed",
218+
minimum=0,
219+
maximum=MAX_SEED,
220+
step=1,
221+
value=0,
222+
)
223+
randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
224+
225+
with gr.Row():
226+
height = gr.Slider(64, 2048, value=512, step=64, label="Height")
227+
width = gr.Slider(64, 2048, value=512, step=64, label="Width")
228+
229+
with gr.Row():
230+
guidance_scale = gr.Slider(0.0, 20.0, value=3.5, step=0.1, label="Guidance Scale")
231+
control_strength = gr.Slider(0.0, 1.0, value=0.5, step=0.05, label="Control Strength")
232+
233+
with gr.Row():
234+
control_stop = gr.Slider(0.0, 1.0, value=0.33, step=0.05, label="Control Stop")
235+
num_inference_steps = gr.Slider(1, 100, value=20, step=1, label="Inference Steps")
236+
237+
# Footer
238+
gr.HTML(
239+
"""
240+
<div class="footer">
241+
<p>Flex.2 Preview Inpainting OpenVINO Demo</p>
242+
</div>
243+
"""
244+
)
245+
246+
run_button.click(
247+
fn=infer,
248+
inputs=[edit_image, prompt, seed, randomize_seed, width, height, guidance_scale, control_strength, control_stop, num_inference_steps],
249+
outputs=[result, seed],
250+
)
251+
252+
return demo
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
from optimum.intel.openvino import OVDiffusionPipeline
2+
from pipeline import Flex2Pipeline
3+
4+
5+
class OVFlex2Pipeline(OVDiffusionPipeline, Flex2Pipeline):
6+
main_input_name = "prompt"
7+
export_feature = "text-to-image"
8+
auto_model_class = Flex2Pipeline

0 commit comments

Comments
 (0)