Skip to content

Commit f7fb0fe

Browse files
igolaizolaigolaizola
committed
Add runway extend feature
1 parent 34d3b80 commit f7fb0fe

File tree

5 files changed

+149
-48
lines changed

5 files changed

+149
-48
lines changed

‎README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ This is a CLI tool for [RunwayML Gen-2](https://runwayml.com/) that adds some ex
77
## 🚀 Features
88

99
- Generate videos directly from the command line using a text or image prompt.
10+
- Use RunwayML's extend feature to generate longer videos.
1011
- Create or extend videos longer than 4 seconds by reusing the last frame of the video as the input for the next generation.
1112
- Other handy tools to edit videos, like generating loops or resizing videos.
1213

@@ -42,6 +43,12 @@ Generate a video from a text prompt:
4243
vidai generate --token RUNWAYML_TOKEN --text "a car in the middle of the road" --output car.mp4
4344
```
4445

46+
Generate a video from a image prompt and extend it twice (using RunwayML's extend feature):
47+
48+
```bash
49+
vidai generate --token RUNWAYML_TOKEN --image car.jpg --output car.mp4 --extend 2
50+
```
51+
4552
Extend a video by reusing the last frame twice:
4653

4754
```bash

‎cmd/vidai/main.go‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -114,18 +114,12 @@ func newGenerateCommand() *ffcli.Command {
114114
return fmt.Errorf("image or text is required")
115115
}
116116
c := vidai.New(&cfg)
117-
urls, err := c.Generate(ctx, *image, *text, *output, *extend,
117+
u, err := c.Generate(ctx, *image, *text, *output, *extend,
118118
*interpolate, *upscale, *watermark)
119119
if err != nil {
120120
return err
121121
}
122-
if len(urls) == 1 {
123-
fmt.Printf("Video URL: %s\n", urls[0])
124-
} else {
125-
for i, u := range urls {
126-
fmt.Printf("Video URL %d: %s\n", i+1, u)
127-
}
128-
}
122+
fmt.Printf("Video URL: %s\n", u)
129123
return nil
130124
},
131125
}

‎pkg/runway/runway.go‎

Lines changed: 36 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -198,14 +198,18 @@ type createTaskRequest struct {
198198
}
199199

200200
type gen2Options struct {
201-
Interpolate bool `json:"interpolate"`
202-
Seed int `json:"seed"`
203-
Upscale bool `json:"upscale"`
204-
TextPrompt string `json:"text_prompt"`
205-
Watermark bool `json:"watermark"`
206-
ImagePrompt string `json:"image_prompt"`
207-
InitImage string `json:"init_image"`
208-
Mode string `json:"mode"`
201+
Interpolate bool `json:"interpolate"`
202+
Seed int `json:"seed"`
203+
Upscale bool `json:"upscale"`
204+
TextPrompt string `json:"text_prompt"`
205+
Watermark bool `json:"watermark"`
206+
ImagePrompt string `json:"image_prompt,omitempty"`
207+
InitImage string `json:"init_image,omitempty"`
208+
Mode string `json:"mode"`
209+
InitVideo string `json:"init_video,omitempty"`
210+
MotionScore int `json:"motion_score"`
211+
UseMotionScore bool `json:"use_motion_score"`
212+
UseMotionVectors bool `json:"use_motion_vectors"`
209213
}
210214

211215
type taskResponse struct {
@@ -243,21 +247,21 @@ type artifact struct {
243247
ParentAssetGroupId string `json:"parentAssetGroupId"`
244248
Filename string `json:"filename"`
245249
URL string `json:"url"`
246-
FileSize int `json:"fileSize"`
250+
FileSize string `json:"fileSize"`
247251
IsDirectory bool `json:"isDirectory"`
248252
PreviewURLs []string `json:"previewUrls"`
249253
Private bool `json:"private"`
250254
PrivateInTeam bool `json:"privateInTeam"`
251255
Deleted bool `json:"deleted"`
252256
Reported bool `json:"reported"`
253257
Metadata struct {
254-
FrameRate int `json:"frameRate"`
255-
Duration int `json:"duration"`
256-
Dimensions []int `json:"dimensions"`
258+
FrameRate int `json:"frameRate"`
259+
Duration float32 `json:"duration"`
260+
Dimensions []int `json:"dimensions"`
257261
} `json:"metadata"`
258262
}
259263

260-
func (c *Client) Generate(ctx context.Context, imageURL, textPrompt string, interpolate, upscale, watermark bool) (string, error) {
264+
func (c *Client) Generate(ctx context.Context, assetURL, textPrompt string, interpolate, upscale, watermark, extend bool) (string, error) {
261265
// Load team ID
262266
if err := c.loadTeamID(ctx); err != nil {
263267
return "", fmt.Errorf("runway: couldn't load team id: %w", err)
@@ -266,6 +270,14 @@ func (c *Client) Generate(ctx context.Context, imageURL, textPrompt string, inte
266270
// Generate seed
267271
seed := rand.Intn(1000000000)
268272

273+
var imageURL string
274+
var videoURL string
275+
if extend {
276+
videoURL = assetURL
277+
} else {
278+
imageURL = assetURL
279+
}
280+
269281
// Create task
270282
createReq := &createTaskRequest{
271283
TaskType: "gen2",
@@ -279,14 +291,17 @@ func (c *Client) Generate(ctx context.Context, imageURL, textPrompt string, inte
279291
}{
280292
Seconds: 4,
281293
Gen2Options: gen2Options{
282-
Interpolate: interpolate,
283-
Seed: seed,
284-
Upscale: upscale,
285-
TextPrompt: textPrompt,
286-
Watermark: watermark,
287-
ImagePrompt: imageURL,
288-
InitImage: imageURL,
289-
Mode: "gen2",
294+
Interpolate: interpolate,
295+
Seed: seed,
296+
Upscale: upscale,
297+
TextPrompt: textPrompt,
298+
Watermark: watermark,
299+
ImagePrompt: imageURL,
300+
InitImage: imageURL,
301+
InitVideo: videoURL,
302+
Mode: "gen2",
303+
UseMotionScore: true,
304+
MotionScore: 22,
290305
},
291306
Name: fmt.Sprintf("Gen-2, %d", seed),
292307
AssetGroupName: "Gen-2",

‎pkg/runway/runway_test.go‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
package runway
2+
3+
import (
4+
"encoding/json"
5+
"testing"
6+
)
7+
8+
func TestUnmarshal(t *testing.T) {
9+
js := `{
10+
"task": {
11+
"id": "00000000-0000-0000-0000-000000000000",
12+
"name": "Gen-2, 100000",
13+
"image": null,
14+
"createdAt": "2024-01-01T01:01:01.001Z",
15+
"updatedAt": "2024-01-01T01:01:01.001Z",
16+
"taskType": "gen2",
17+
"options": {
18+
"seconds": 4,
19+
"gen2Options": {
20+
"interpolate": true,
21+
"seed": 100000,
22+
"upscale": true,
23+
"text_prompt": "",
24+
"watermark": false,
25+
"image_prompt": "https://a.url.test",
26+
"init_image": "https://a.url.test",
27+
"mode": "gen2",
28+
"motion_score": 22,
29+
"use_motion_score": true,
30+
"use_motion_vectors": false
31+
},
32+
"name": "Gen-2, 100000",
33+
"assetGroupName": "Gen-2",
34+
"exploreMode": false,
35+
"recordingEnabled": true
36+
},
37+
"status": "SUCCEEDED",
38+
"error": null,
39+
"progressText": null,
40+
"progressRatio": "1",
41+
"placeInLine": null,
42+
"estimatedTimeToStartSeconds": null,
43+
"artifacts": [
44+
{
45+
"id": "00000000-0000-0000-0000-000000000000",
46+
"createdAt": "2024-01-01T01:01:01.001Z",
47+
"updatedAt": "2024-01-01T01:01:01.001Z",
48+
"userId": 100000,
49+
"createdBy": 100000,
50+
"taskId": "00000000-0000-0000-0000-000000000000",
51+
"parentAssetGroupId": "00000000-0000-0000-0000-000000000000",
52+
"filename": "Gen-2, 100000.mp4",
53+
"url": "https://a.url.test",
54+
"fileSize": "100000",
55+
"isDirectory": false,
56+
"previewUrls": [
57+
"https://a.url.test",
58+
"https://a.url.test",
59+
"https://a.url.test",
60+
"https://a.url.test"
61+
],
62+
"private": true,
63+
"privateInTeam": true,
64+
"deleted": false,
65+
"reported": false,
66+
"metadata": {
67+
"frameRate": 24,
68+
"duration": 8.1,
69+
"dimensions": [
70+
2816,
71+
1536
72+
],
73+
"size": {
74+
"width": 2816,
75+
"height": 1536
76+
}
77+
}
78+
}
79+
],
80+
"sharedAsset": null
81+
}
82+
}`
83+
var resp taskResponse
84+
if err := json.Unmarshal([]byte(js), &resp); err != nil {
85+
t.Fatal(err)
86+
}
87+
}

‎vidai.go‎

Lines changed: 17 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -48,23 +48,31 @@ func New(cfg *Config) *Client {
4848

4949
// Generate generates a video from an image and a text prompt.
5050
func (c *Client) Generate(ctx context.Context, image, text, output string,
51-
extend int, interpolate, upscale, watermark bool) ([]string, error) {
51+
extend int, interpolate, upscale, watermark bool) (string, error) {
5252
b, err := os.ReadFile(image)
5353
if err != nil {
54-
return nil, fmt.Errorf("vidai: couldn't read image: %w", err)
54+
return "", fmt.Errorf("vidai: couldn't read image: %w", err)
5555
}
5656
name := filepath.Base(image)
5757

5858
var imageURL string
5959
if image != "" {
6060
imageURL, err = c.client.Upload(ctx, name, b)
6161
if err != nil {
62-
return nil, fmt.Errorf("vidai: couldn't upload image: %w", err)
62+
return "", fmt.Errorf("vidai: couldn't upload image: %w", err)
6363
}
6464
}
65-
videoURL, err := c.client.Generate(ctx, imageURL, text, interpolate, upscale, watermark)
65+
videoURL, err := c.client.Generate(ctx, imageURL, text, interpolate, upscale, watermark, false)
6666
if err != nil {
67-
return nil, fmt.Errorf("vidai: couldn't generate video: %w", err)
67+
return "", fmt.Errorf("vidai: couldn't generate video: %w", err)
68+
}
69+
70+
// Extend video
71+
for i := 0; i < extend; i++ {
72+
videoURL, err = c.client.Generate(ctx, videoURL, "", interpolate, upscale, watermark, true)
73+
if err != nil {
74+
return "", fmt.Errorf("vidai: couldn't extend video: %w", err)
75+
}
6876
}
6977

7078
// Use temp file if no output is set and we need to extend the video
@@ -77,24 +85,14 @@ func (c *Client) Generate(ctx context.Context, image, text, output string,
7785
// Download video
7886
if videoPath != "" {
7987
if err := c.download(ctx, videoURL, videoPath); err != nil {
80-
return nil, fmt.Errorf("vidai: couldn't download video: %w", err)
81-
}
82-
}
83-
84-
// Extend video
85-
if extend > 0 {
86-
extendURLs, err := c.Extend(ctx, videoPath, output, extend,
87-
interpolate, upscale, watermark)
88-
if err != nil {
89-
return nil, fmt.Errorf("vidai: couldn't extend video: %w", err)
88+
return "", fmt.Errorf("vidai: couldn't download video: %w", err)
9089
}
91-
return append([]string{output}, extendURLs...), nil
9290
}
9391

94-
return []string{videoURL}, nil
92+
return videoURL, nil
9593
}
9694

97-
// Extend extends a video using the last frame of the previous video.
95+
// Extend extends a video using the previous video.
9896
func (c *Client) Extend(ctx context.Context, input, output string, n int,
9997
interpolate, upscale, watermark bool) ([]string, error) {
10098
base := strings.TrimSuffix(filepath.Base(input), filepath.Ext(input))
@@ -133,7 +131,7 @@ func (c *Client) Extend(ctx context.Context, input, output string, n int,
133131
if err != nil {
134132
return nil, fmt.Errorf("vidai: couldn't upload image: %w", err)
135133
}
136-
videoURL, err := c.client.Generate(ctx, imageURL, "", interpolate, upscale, watermark)
134+
videoURL, err := c.client.Generate(ctx, imageURL, "", interpolate, upscale, watermark, false)
137135
if err != nil {
138136
return nil, fmt.Errorf("vidai: couldn't generate video: %w", err)
139137
}

0 commit comments

Comments
 (0)