Skip to content

Commit 6b03cf7

Browse files
Oleksandr Shestopaloshe-gi
authored andcommitted
copy: Add text-based progress output for non-TTY environments
When copying images in non-TTY environments (CI/CD pipelines, redirected output, piped commands), the visual mpb progress bars are discarded, leaving users with no visibility into transfer progress. This makes it difficult to detect stalled transfers or monitor long-running copies. This change adds a nonTTYProgressWriter that consumes progress events from the existing Progress channel and prints periodic aggregate progress lines suitable for log output: Progress: 13.1 MiB / 52.3 MiB Progress: 26.2 MiB / 52.3 MiB Progress: 52.3 MiB / 52.3 MiB The feature is enabled when, output is not a TTY, we check if option.Progress is set, otherwise create a new buffered channel for progress events. Note: Unbuffered channels are replaced with buffered ones to prevent blocking during parallel blob downloads. Callers who need custom consumption should provide a properly buffered channel. Relates-to: containers/skopeo#658 Signed-off-by: Oleksandr Shestopal <ar.shestopal-oshegithub@gmail.com>
1 parent 026c353 commit 6b03cf7

File tree

3 files changed

+272
-1
lines changed

3 files changed

+272
-1
lines changed

image/copy/copy.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -257,10 +257,12 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef,
257257

258258
// If reportWriter is not a TTY (e.g., when piping to a file), do not
259259
// print the progress bars to avoid long and hard to parse output.
260-
// Instead use printCopyInfo() to print single line "Copying ..." messages.
260+
// Instead use text-based aggregate progress via nonTTYProgressWriter.
261261
progressOutput := reportWriter
262262
if !isTTY(reportWriter) {
263263
progressOutput = io.Discard
264+
265+
setupNonTTYProgressWriter(reportWriter, options)
264266
}
265267

266268
c := &copier{

image/copy/progress_nontty.go

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
package copy
2+
3+
import (
4+
"fmt"
5+
"io"
6+
"time"
7+
8+
"github.com/vbauerster/mpb/v8/decor"
9+
"go.podman.io/image/v5/types"
10+
)
11+
12+
const (
13+
// nonTTYProgressChannelSize is the buffer size for the progress channel
14+
// in non-TTY mode. Buffered to prevent blocking during parallel downloads.
15+
nonTTYProgressChannelSize = 10
16+
17+
// nonTTYProgressInterval is how often aggregate progress is printed
18+
// in non-TTY mode.
19+
nonTTYProgressInterval = 500 * time.Millisecond
20+
)
21+
22+
// nonTTYProgressWriter consumes ProgressProperties from a channel and writes
23+
// aggregate text-based progress output suitable for non-TTY environments.
24+
// No mutex needed - single goroutine processes events sequentially from channel.
25+
type nonTTYProgressWriter struct {
26+
output io.Writer
27+
28+
// Aggregate tracking (no per-blob state needed)
29+
totalSize int64 // Sum of all known blob sizes
30+
downloaded int64 // Total bytes downloaded (accumulated from OffsetUpdate)
31+
32+
// Output throttling
33+
lastOutput time.Time
34+
outputInterval time.Duration
35+
progressChannel <-chan types.ProgressProperties
36+
}
37+
38+
// newNonTTYProgressWriter creates a writer that outputs aggregate download
39+
// progress as simple text lines, suitable for non-TTY environments like
40+
// CI/CD pipelines or redirected output.
41+
func newNonTTYProgressWriter(output io.Writer, interval time.Duration, pch chan types.ProgressProperties) *nonTTYProgressWriter {
42+
return &nonTTYProgressWriter{
43+
output: output,
44+
outputInterval: interval,
45+
progressChannel: pch,
46+
}
47+
}
48+
49+
// setupNonTTYProgressWriter configures text-based progress output for non-TTY
50+
// environments unless the caller already provided a buffered Progress channel.
51+
// Returns a cleanup function that must be deferred by the caller.
52+
// It relies on the idea that options.Progress channel is only used once, to track progress with a progress bar
53+
// Otherwise we must do some sort of fan-out
54+
func setupNonTTYProgressWriter(reportWriter io.Writer, options *Options) {
55+
// Use user's interval if greater than our default, otherwise use default.
56+
// This allows users to slow down output while maintaining a sensible minimum.
57+
interval := max(options.ProgressInterval, nonTTYProgressInterval)
58+
if options.ProgressInterval <= 0 {
59+
options.ProgressInterval = nonTTYProgressInterval
60+
}
61+
62+
if options.Progress == nil || cap(options.Progress) == 0 {
63+
options.Progress = make(chan types.ProgressProperties, nonTTYProgressChannelSize)
64+
}
65+
66+
pw := newNonTTYProgressWriter(reportWriter, interval, options.Progress)
67+
go pw.Run()
68+
}
69+
70+
// Run consumes progress events from the channel and prints throttled
71+
// aggregate progress. Blocks until the channel is closed. Intended to
72+
// be called as a goroutine: go tw.Run(progressChan)
73+
func (w *nonTTYProgressWriter) Run() {
74+
for props := range w.progressChannel {
75+
switch props.Event {
76+
case types.ProgressEventNewArtifact:
77+
// New blob starting - add its size to total
78+
w.totalSize += props.Artifact.Size
79+
80+
case types.ProgressEventRead:
81+
// Bytes downloaded - accumulate and maybe print
82+
w.downloaded += int64(props.OffsetUpdate)
83+
if time.Since(w.lastOutput) > w.outputInterval {
84+
fmt.Fprintf(w.output, "Progress: %.1f / %.1f\n",
85+
decor.SizeB1024(w.downloaded), decor.SizeB1024(w.totalSize))
86+
w.lastOutput = time.Now()
87+
}
88+
}
89+
}
90+
}

image/copy/progress_nontty_test.go

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
package copy
2+
3+
import (
4+
"bytes"
5+
"strings"
6+
"testing"
7+
"time"
8+
9+
"github.com/stretchr/testify/assert"
10+
"go.podman.io/image/v5/types"
11+
)
12+
13+
func TestNonTTYProgressWriterRun(t *testing.T) {
14+
tests := []struct {
15+
name string
16+
interval time.Duration
17+
events []types.ProgressProperties
18+
wantTotalSize int64
19+
wantDownloaded int64
20+
wantLines int
21+
wantContains string
22+
}{
23+
{
24+
name: "new artifacts only",
25+
interval: 500 * time.Millisecond,
26+
events: []types.ProgressProperties{
27+
{Event: types.ProgressEventNewArtifact, Artifact: types.BlobInfo{Size: 1024}},
28+
{Event: types.ProgressEventNewArtifact, Artifact: types.BlobInfo{Size: 2048}},
29+
},
30+
wantTotalSize: 3072,
31+
wantDownloaded: 0,
32+
wantLines: 0,
33+
},
34+
{
35+
name: "read events produce output",
36+
interval: -1 * time.Millisecond,
37+
events: []types.ProgressProperties{
38+
{Event: types.ProgressEventNewArtifact, Artifact: types.BlobInfo{Size: 10240}},
39+
{Event: types.ProgressEventRead, OffsetUpdate: 5120},
40+
},
41+
wantTotalSize: 10240,
42+
wantDownloaded: 5120,
43+
wantLines: 1,
44+
wantContains: "Progress:",
45+
},
46+
{
47+
name: "throttling limits output",
48+
interval: time.Hour,
49+
events: []types.ProgressProperties{
50+
{Event: types.ProgressEventNewArtifact, Artifact: types.BlobInfo{Size: 10240}},
51+
{Event: types.ProgressEventRead, OffsetUpdate: 1024},
52+
{Event: types.ProgressEventRead, OffsetUpdate: 1024},
53+
{Event: types.ProgressEventRead, OffsetUpdate: 1024},
54+
},
55+
wantTotalSize: 10240,
56+
wantDownloaded: 3072,
57+
wantLines: 1,
58+
},
59+
{
60+
name: "unknown events ignored",
61+
interval: 500 * time.Millisecond,
62+
events: []types.ProgressProperties{
63+
{Event: types.ProgressEventDone},
64+
},
65+
wantTotalSize: 0,
66+
wantDownloaded: 0,
67+
wantLines: 0,
68+
},
69+
}
70+
71+
for _, tt := range tests {
72+
t.Run(tt.name, func(t *testing.T) {
73+
var buf bytes.Buffer
74+
75+
ch := make(chan types.ProgressProperties, len(tt.events))
76+
pw := newNonTTYProgressWriter(&buf, tt.interval, ch)
77+
for _, e := range tt.events {
78+
ch <- e
79+
}
80+
close(ch)
81+
82+
pw.Run()
83+
84+
assert.Equal(t, tt.wantTotalSize, pw.totalSize)
85+
assert.Equal(t, tt.wantDownloaded, pw.downloaded)
86+
87+
output := buf.String()
88+
if tt.wantLines == 0 {
89+
assert.Empty(t, output)
90+
} else {
91+
lines := strings.Split(strings.TrimSpace(output), "\n")
92+
assert.Equal(t, tt.wantLines, len(lines))
93+
}
94+
if tt.wantContains != "" {
95+
assert.Contains(t, output, tt.wantContains)
96+
}
97+
})
98+
}
99+
}
100+
101+
func TestSetupNonTTYProgressWriter(t *testing.T) {
102+
tests := []struct {
103+
name string
104+
progress chan types.ProgressProperties
105+
progressInterval time.Duration
106+
wantProgressSet bool
107+
wantIntervalSet bool
108+
wantMinInterval time.Duration
109+
}{
110+
{
111+
name: "nil channel gets default setup",
112+
progress: nil,
113+
progressInterval: 0,
114+
wantProgressSet: true,
115+
wantIntervalSet: true,
116+
wantMinInterval: nonTTYProgressInterval,
117+
},
118+
{
119+
name: "unbuffered channel gets replaced",
120+
progress: make(chan types.ProgressProperties),
121+
progressInterval: 0,
122+
wantProgressSet: true,
123+
wantIntervalSet: true,
124+
wantMinInterval: nonTTYProgressInterval,
125+
},
126+
{
127+
name: "buffered channel is kept",
128+
progress: make(chan types.ProgressProperties, 5),
129+
progressInterval: 0,
130+
wantProgressSet: false,
131+
wantIntervalSet: false,
132+
},
133+
{
134+
name: "caller interval larger than default is respected",
135+
progress: nil,
136+
progressInterval: 2 * time.Second,
137+
wantProgressSet: true,
138+
wantIntervalSet: false,
139+
wantMinInterval: 2 * time.Second,
140+
},
141+
{
142+
name: "caller interval smaller than default is kept",
143+
progress: nil,
144+
progressInterval: 100 * time.Millisecond,
145+
wantProgressSet: true,
146+
wantIntervalSet: false,
147+
},
148+
}
149+
150+
for _, tt := range tests {
151+
t.Run(tt.name, func(t *testing.T) {
152+
var buf bytes.Buffer
153+
opts := &Options{
154+
Progress: tt.progress,
155+
ProgressInterval: tt.progressInterval,
156+
}
157+
originalProgress := opts.Progress
158+
159+
setupNonTTYProgressWriter(&buf, opts)
160+
if tt.wantProgressSet {
161+
assert.NotNil(t, opts.Progress)
162+
assert.Greater(t, cap(opts.Progress), 0)
163+
if originalProgress != nil {
164+
assert.NotEqual(t, originalProgress, opts.Progress)
165+
}
166+
} else {
167+
assert.Equal(t, originalProgress, opts.Progress)
168+
}
169+
170+
if tt.wantIntervalSet {
171+
assert.Equal(t, nonTTYProgressInterval, opts.ProgressInterval)
172+
}
173+
174+
if tt.wantMinInterval > 0 {
175+
assert.GreaterOrEqual(t, opts.ProgressInterval, tt.wantMinInterval)
176+
}
177+
})
178+
}
179+
}

0 commit comments

Comments
 (0)