Skip to content

Commit f489ae9

Browse files
natario1natario1
authored
Fix encoder buffer leak (#132)
* Improve demo app * Fix encoder buffer leak
1 parent a283dca commit f489ae9

File tree

6 files changed

+105
-51
lines changed

6 files changed

+105
-51
lines changed

‎demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java

+18-22
Original file line numberDiff line numberDiff line change
@@ -35,12 +35,17 @@
3535

3636
import java.io.File;
3737
import java.io.IOException;
38+
import java.util.ArrayList;
39+
import java.util.List;
3840
import java.util.concurrent.Future;
41+
import java.util.function.Consumer;
3942

4043
import androidx.annotation.NonNull;
4144
import androidx.appcompat.app.AppCompatActivity;
4245
import androidx.core.content.FileProvider;
4346

47+
import kotlin.collections.ArraysKt;
48+
4449

4550
public class TranscoderActivity extends AppCompatActivity implements
4651
TranscoderListener {
@@ -70,9 +75,6 @@ public class TranscoderActivity extends AppCompatActivity implements
7075
private boolean mIsTranscoding;
7176
private boolean mIsAudioOnly;
7277
private Future<Void> mTranscodeFuture;
73-
private Uri mTranscodeInputUri1;
74-
private Uri mTranscodeInputUri2;
75-
private Uri mTranscodeInputUri3;
7678
private Uri mAudioReplacementUri;
7779
private File mTranscodeOutputFile;
7880
private long mTranscodeStartTime;
@@ -241,15 +243,13 @@ protected void onActivityResult(int requestCode, int resultCode, final Intent da
241243
&& data != null) {
242244
if (data.getClipData() != null) {
243245
ClipData clipData = data.getClipData();
244-
mTranscodeInputUri1 = clipData.getItemAt(0).getUri();
245-
mTranscodeInputUri2 = clipData.getItemCount() >= 2 ? clipData.getItemAt(1).getUri() : null;
246-
mTranscodeInputUri3 = clipData.getItemCount() >= 3 ? clipData.getItemAt(2).getUri() : null;
247-
transcode();
246+
List<Uri> uris = new ArrayList<>();
247+
for (int i = 0; i < clipData.getItemCount(); i++) {
248+
uris.add(clipData.getItemAt(i).getUri());
249+
}
250+
transcode(uris.toArray(new Uri[0]));
248251
} else if (data.getData() != null) {
249-
mTranscodeInputUri1 = data.getData();
250-
mTranscodeInputUri2 = null;
251-
mTranscodeInputUri3 = null;
252-
transcode();
252+
transcode(data.getData());
253253
}
254254
}
255255
if (requestCode == REQUEST_CODE_PICK_AUDIO
@@ -262,7 +262,7 @@ protected void onActivityResult(int requestCode, int resultCode, final Intent da
262262
}
263263
}
264264

265-
private void transcode() {
265+
private void transcode(@NonNull Uri... uris) {
266266
// Create a temporary file for output.
267267
try {
268268
File outputDir = new File(getExternalFilesDir(null), "outputs");
@@ -296,20 +296,16 @@ private void transcode() {
296296
setIsTranscoding(true);
297297
LOG.e("Building transcoding options...");
298298
TranscoderOptions.Builder builder = Transcoder.into(mTranscodeOutputFile.getAbsolutePath());
299+
List<DataSource> sources = ArraysKt.map(uris, uri -> new UriDataSource(this, uri));
300+
sources.set(0, new TrimDataSource(sources.get(0), mTrimStartUs, mTrimEndUs));
299301
if (mAudioReplacementUri == null) {
300-
if (mTranscodeInputUri1 != null) {
301-
DataSource source = new UriDataSource(this, mTranscodeInputUri1);
302-
builder.addDataSource(new TrimDataSource(source, mTrimStartUs, mTrimEndUs));
302+
for (DataSource source : sources) {
303+
builder.addDataSource(source);
303304
}
304-
if (mTranscodeInputUri2 != null) builder.addDataSource(this, mTranscodeInputUri2);
305-
if (mTranscodeInputUri3 != null) builder.addDataSource(this, mTranscodeInputUri3);
306305
} else {
307-
if (mTranscodeInputUri1 != null) {
308-
DataSource source = new UriDataSource(this, mTranscodeInputUri1);
309-
builder.addDataSource(TrackType.VIDEO, new TrimDataSource(source, mTrimStartUs, mTrimEndUs));
306+
for (DataSource source : sources) {
307+
builder.addDataSource(TrackType.VIDEO, source);
310308
}
311-
if (mTranscodeInputUri2 != null) builder.addDataSource(TrackType.VIDEO, this, mTranscodeInputUri2);
312-
if (mTranscodeInputUri3 != null) builder.addDataSource(TrackType.VIDEO, this, mTranscodeInputUri3);
313309
builder.addDataSource(TrackType.AUDIO, this, mAudioReplacementUri);
314310
}
315311
LOG.e("Starting transcoding!");

‎lib/src/main/java/com/otaliastudios/transcoder/internal/audio/AudioEngine.kt

+16-3
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,10 @@ import com.otaliastudios.transcoder.internal.audio.remix.AudioRemixer
77
import com.otaliastudios.transcoder.internal.codec.*
88
import com.otaliastudios.transcoder.internal.pipeline.*
99
import com.otaliastudios.transcoder.internal.utils.Logger
10+
import com.otaliastudios.transcoder.internal.utils.trackMapOf
1011
import com.otaliastudios.transcoder.resample.AudioResampler
1112
import com.otaliastudios.transcoder.stretch.AudioStretcher
13+
import java.util.concurrent.atomic.AtomicInteger
1214
import kotlin.math.ceil
1315
import kotlin.math.floor
1416

@@ -22,7 +24,10 @@ internal class AudioEngine(
2224
private val targetFormat: MediaFormat
2325
): QueuedStep<DecoderData, DecoderChannel, EncoderData, EncoderChannel>(), DecoderChannel {
2426

25-
private val log = Logger("AudioEngine")
27+
companion object {
28+
private val ID = AtomicInteger(0)
29+
}
30+
private val log = Logger("AudioEngine(${ID.getAndIncrement()})")
2631

2732
override val channel = this
2833
private val buffers = ShortBuffers()
@@ -37,12 +42,14 @@ internal class AudioEngine(
3742
override fun handleSourceFormat(sourceFormat: MediaFormat): Surface? = null
3843

3944
override fun handleRawFormat(rawFormat: MediaFormat) {
45+
log.i("handleRawFormat($rawFormat)")
4046
this.rawFormat = rawFormat
4147
remixer = AudioRemixer[rawFormat.channels, targetFormat.channels]
4248
chunks = ChunkQueue(rawFormat.sampleRate, rawFormat.channels)
4349
}
4450

4551
override fun enqueueEos(data: DecoderData) {
52+
log.i("enqueueEos()")
4653
data.release(false)
4754
chunks.enqueueEos()
4855
}
@@ -55,8 +62,14 @@ internal class AudioEngine(
5562
}
5663

5764
override fun drain(): State<EncoderData> {
58-
if (chunks.isEmpty()) return State.Wait
59-
val (outBytes, outId) = next.buffer() ?: return State.Wait
65+
if (chunks.isEmpty()) {
66+
log.i("drain(): no chunks, waiting...")
67+
return State.Wait
68+
}
69+
val (outBytes, outId) = next.buffer() ?: return run {
70+
log.i("drain(): no next buffer, waiting...")
71+
State.Wait
72+
}
6073
val outBuffer = outBytes.asShortBuffer()
6174
return chunks.drain(
6275
eos = State.Eos(EncoderData(outBytes, outId, 0))

‎lib/src/main/java/com/otaliastudios/transcoder/internal/codec/Decoder.kt

+39-7
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package com.otaliastudios.transcoder.internal.codec
33
import android.media.MediaCodec.*
44
import android.media.MediaFormat
55
import android.view.Surface
6+
import com.otaliastudios.transcoder.common.trackType
67
import com.otaliastudios.transcoder.internal.data.ReaderChannel
78
import com.otaliastudios.transcoder.internal.data.ReaderData
89
import com.otaliastudios.transcoder.internal.media.MediaCodecBuffers
@@ -11,7 +12,11 @@ import com.otaliastudios.transcoder.internal.pipeline.Channel
1112
import com.otaliastudios.transcoder.internal.pipeline.QueuedStep
1213
import com.otaliastudios.transcoder.internal.pipeline.State
1314
import com.otaliastudios.transcoder.internal.utils.Logger
15+
import com.otaliastudios.transcoder.internal.utils.trackMapOf
1416
import java.nio.ByteBuffer
17+
import java.util.concurrent.atomic.AtomicInteger
18+
import kotlin.properties.Delegates
19+
import kotlin.properties.Delegates.observable
1520

1621

1722
internal open class DecoderData(
@@ -30,32 +35,52 @@ internal class Decoder(
3035
continuous: Boolean, // relevant if the source sends no-render chunks. should we compensate or not?
3136
) : QueuedStep<ReaderData, ReaderChannel, DecoderData, DecoderChannel>(), ReaderChannel {
3237

33-
private val log = Logger("Decoder")
38+
companion object {
39+
private val ID = trackMapOf(AtomicInteger(0), AtomicInteger(0))
40+
}
41+
42+
private val log = Logger("Decoder(${format.trackType},${ID[format.trackType].getAndIncrement()})")
3443
override val channel = this
44+
3545
private val codec = createDecoderByType(format.getString(MediaFormat.KEY_MIME)!!)
3646
private val buffers by lazy { MediaCodecBuffers(codec) }
3747
private var info = BufferInfo()
3848
private val dropper = DecoderDropper(continuous)
3949

50+
private var dequeuedInputs by observable(0) { _, _, _ -> printDequeued() }
51+
private var dequeuedOutputs by observable(0) { _, _, _ -> printDequeued() }
52+
private fun printDequeued() {
53+
// log.v("dequeuedInputs=$dequeuedInputs dequeuedOutputs=$dequeuedOutputs")
54+
}
55+
4056
override fun initialize(next: DecoderChannel) {
4157
super.initialize(next)
58+
log.i("initialize()")
4259
val surface = next.handleSourceFormat(format)
4360
codec.configure(format, surface, null, 0)
4461
codec.start()
4562
}
4663

4764
override fun buffer(): Pair<ByteBuffer, Int>? {
4865
val id = codec.dequeueInputBuffer(0)
49-
log.v("buffer(): id=$id")
50-
return if (id >= 0) buffers.getInputBuffer(id) to id else null
66+
return if (id >= 0) {
67+
dequeuedInputs++
68+
buffers.getInputBuffer(id) to id
69+
} else {
70+
log.i("buffer() failed. dequeuedInputs=$dequeuedInputs dequeuedOutputs=$dequeuedOutputs")
71+
null
72+
}
5173
}
5274

5375
override fun enqueueEos(data: ReaderData) {
76+
log.i("enqueueEos()!")
77+
dequeuedInputs--
5478
val flag = BUFFER_FLAG_END_OF_STREAM
5579
codec.queueInputBuffer(data.id, 0, 0, 0, flag)
5680
}
5781

5882
override fun enqueue(data: ReaderData) {
83+
dequeuedInputs--
5984
val (chunk, id) = data
6085
val flag = if (chunk.keyframe) BUFFER_FLAG_SYNC_FRAME else 0
6186
codec.queueInputBuffer(id, chunk.buffer.position(), chunk.buffer.remaining(), chunk.timeUs, flag)
@@ -65,36 +90,43 @@ internal class Decoder(
6590
override fun drain(): State<DecoderData> {
6691
val result = codec.dequeueOutputBuffer(info, 0)
6792
return when (result) {
68-
INFO_TRY_AGAIN_LATER -> State.Wait
93+
INFO_TRY_AGAIN_LATER -> {
94+
log.i("drain(): got INFO_TRY_AGAIN_LATER, waiting.")
95+
State.Wait
96+
}
6997
INFO_OUTPUT_FORMAT_CHANGED -> {
98+
log.i("drain(): got INFO_OUTPUT_FORMAT_CHANGED, handling format and retrying. format=${codec.outputFormat}")
7099
next.handleRawFormat(codec.outputFormat)
71100
State.Retry
72101
}
73102
INFO_OUTPUT_BUFFERS_CHANGED -> {
103+
log.i("drain(): got INFO_OUTPUT_BUFFERS_CHANGED, retrying.")
74104
buffers.onOutputBuffersChanged()
75105
State.Retry
76106
}
77107
else -> {
78108
val isEos = info.flags and BUFFER_FLAG_END_OF_STREAM != 0
79109
val timeUs = if (isEos) 0 else dropper.output(info.presentationTimeUs)
80110
if (timeUs != null /* && (isEos || info.size > 0) */) {
111+
dequeuedOutputs++
81112
val buffer = buffers.getOutputBuffer(result)
82113
val data = DecoderData(buffer, timeUs) {
83114
codec.releaseOutputBuffer(result, it)
115+
dequeuedOutputs--
84116
}
85-
// log.w("TDBG isEos=$isEos timeUs=$timeUs")
86117
if (isEos) State.Eos(data) else State.Ok(data)
87118
} else {
88119
codec.releaseOutputBuffer(result, false)
89120
State.Wait
121+
}.also {
122+
log.v("drain(): returning $it")
90123
}
91124
}
92-
}.also {
93-
log.v("Returning $it")
94125
}
95126
}
96127

97128
override fun release() {
129+
log.i("release(): releasing codec. dequeuedInputs=$dequeuedInputs dequeuedOutputs=$dequeuedOutputs")
98130
codec.stop()
99131
codec.release()
100132
}

‎lib/src/main/java/com/otaliastudios/transcoder/internal/codec/Encoder.kt

+27-16
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import android.media.MediaCodec
44
import android.media.MediaCodec.*
55
import android.view.Surface
66
import com.otaliastudios.transcoder.common.TrackType
7+
import com.otaliastudios.transcoder.common.trackType
78
import com.otaliastudios.transcoder.internal.Codecs
89
import com.otaliastudios.transcoder.internal.data.WriterChannel
910
import com.otaliastudios.transcoder.internal.data.WriterData
@@ -12,7 +13,9 @@ import com.otaliastudios.transcoder.internal.pipeline.Channel
1213
import com.otaliastudios.transcoder.internal.pipeline.QueuedStep
1314
import com.otaliastudios.transcoder.internal.pipeline.State
1415
import com.otaliastudios.transcoder.internal.utils.Logger
16+
import com.otaliastudios.transcoder.internal.utils.trackMapOf
1517
import java.nio.ByteBuffer
18+
import java.util.concurrent.atomic.AtomicInteger
1619
import kotlin.properties.Delegates
1720
import kotlin.properties.Delegates.observable
1821

@@ -44,13 +47,15 @@ internal class Encoder(
4447
)
4548

4649
companion object {
47-
// Debugging
48-
private val log = Logger("Encoder")
49-
private var dequeuedInputs by observable(0) { _, _, _ -> printDequeued() }
50-
private var dequeuedOutputs by observable(0) { _, _, _ -> printDequeued() }
51-
private fun printDequeued() {
52-
log.v("dequeuedInputs=$dequeuedInputs dequeuedOutputs=$dequeuedOutputs")
53-
}
50+
private val ID = trackMapOf(AtomicInteger(0), AtomicInteger(0))
51+
}
52+
53+
private val type = if (surface != null) TrackType.VIDEO else TrackType.AUDIO
54+
private val log = Logger("Encoder(${type},${ID[type].getAndIncrement()})")
55+
private var dequeuedInputs by observable(0) { _, _, _ -> printDequeued() }
56+
private var dequeuedOutputs by observable(0) { _, _, _ -> printDequeued() }
57+
private fun printDequeued() {
58+
log.v("dequeuedInputs=$dequeuedInputs dequeuedOutputs=$dequeuedOutputs")
5459
}
5560

5661
override val channel = this
@@ -69,22 +74,26 @@ internal class Encoder(
6974

7075
override fun buffer(): Pair<ByteBuffer, Int>? {
7176
val id = codec.dequeueInputBuffer(0)
72-
log.v("buffer(): id=$id")
73-
if (id >= 0) dequeuedInputs++
74-
return if (id >= 0) buffers.getInputBuffer(id) to id else null
77+
return if (id >= 0) {
78+
dequeuedInputs++
79+
buffers.getInputBuffer(id) to id
80+
} else {
81+
log.i("buffer() failed. dequeuedInputs=$dequeuedInputs dequeuedOutputs=$dequeuedOutputs")
82+
null
83+
}
7584
}
7685

7786
private var eosReceivedButNotEnqueued = false
7887

7988
override fun enqueueEos(data: EncoderData) {
80-
if (!ownsCodecStop) {
81-
eosReceivedButNotEnqueued = true
82-
} else if (surface != null) {
83-
codec.signalEndOfInputStream()
84-
} else {
85-
val flag = BUFFER_FLAG_END_OF_STREAM
89+
if (surface == null) {
90+
if (!ownsCodecStop) eosReceivedButNotEnqueued = true
91+
val flag = if (!ownsCodecStop) 0 else BUFFER_FLAG_END_OF_STREAM
8692
codec.queueInputBuffer(data.id, 0, 0, 0, flag)
8793
dequeuedInputs--
94+
} else {
95+
if (!ownsCodecStop) eosReceivedButNotEnqueued = true
96+
else codec.signalEndOfInputStream()
8897
}
8998
}
9099

@@ -104,6 +113,7 @@ internal class Encoder(
104113
if (eosReceivedButNotEnqueued) {
105114
// Horrible hack. When we don't own the MediaCodec, we can't enqueue EOS so we
106115
// can't dequeue them. INFO_TRY_AGAIN_LATER is returned. We assume this means EOS.
116+
log.i("Sending fake Eos. dequeuedInputs=$dequeuedInputs dequeuedOutputs=$dequeuedOutputs")
107117
val buffer = ByteBuffer.allocateDirect(0)
108118
State.Eos(WriterData(buffer, 0L, 0) {})
109119
} else {
@@ -145,6 +155,7 @@ internal class Encoder(
145155
}
146156

147157
override fun release() {
158+
log.i("release(): ownsStop=$ownsCodecStop dequeuedInputs=${dequeuedInputs} dequeuedOutputs=$dequeuedOutputs")
148159
if (ownsCodecStop) {
149160
codec.stop()
150161
}

‎lib/src/main/java/com/otaliastudios/transcoder/internal/pipeline/Pipeline.kt

+1-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ internal class Pipeline private constructor(name: String, private val chain: Lis
2929
log.v("execute(): step ${step.name} (#$index/${chain.size}) is waiting. headState=$headState headIndex=$headIndex")
3030
return State.Wait
3131
}
32-
log.v("execute(): executed ${step.name} (#$index/${chain.size}). result=$state")
32+
// log.v("execute(): executed ${step.name} (#$index/${chain.size}). result=$state")
3333
if (state is State.Eos) {
3434
log.i("execute(): EOS from ${step.name} (#$index/${chain.size}).")
3535
headState = state

‎lib/src/main/java/com/otaliastudios/transcoder/source/DefaultDataSource.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
import java.io.IOException;
1717
import java.util.HashSet;
18+
import java.util.concurrent.atomic.AtomicInteger;
1819

1920
import static android.media.MediaMetadataRetriever.METADATA_KEY_DURATION;
2021
import static android.media.MediaMetadataRetriever.METADATA_KEY_LOCATION;
@@ -26,7 +27,8 @@
2627
*/
2728
public abstract class DefaultDataSource implements DataSource {
2829

29-
private final Logger LOG = new Logger("DefaultDataSource(" + this.hashCode() + ")");
30+
private final static AtomicInteger ID = new AtomicInteger(0);
31+
private final Logger LOG = new Logger("DefaultDataSource(" + ID.getAndIncrement() + ")");
3032

3133
private final MutableTrackMap<MediaFormat> mFormat = mutableTrackMapOf(null);
3234
private final MutableTrackMap<Integer> mIndex = mutableTrackMapOf(null);
@@ -269,7 +271,7 @@ public int getOrientation() {
269271

270272
@Override
271273
public long getDurationUs() {
272-
LOG.v("getDurationUs()");
274+
// LOG.v("getDurationUs()");
273275
try {
274276
return Long.parseLong(mMetadata.extractMetadata(METADATA_KEY_DURATION)) * 1000;
275277
} catch (NumberFormatException e) {

0 commit comments

Comments
 (0)