Skip to content

Commit 8c87e75

Browse files
committed
#757 Fix another potential leek of Hadoop FSInputStream in BufferedFsDataInputStream.
1 parent 1435418 commit 8c87e75

1 file changed

Lines changed: 23 additions & 12 deletions

File tree

spark-cobol/src/main/scala/za/co/absa/cobrix/spark/cobol/source/streaming/BufferedFSDataInputStream.scala

Lines changed: 23 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import za.co.absa.cobrix.spark.cobol.utils.{FileUtils, GpgUtils}
2222

2323
import java.io.InputStream
2424
import scala.util.Try
25+
import scala.util.control.NonFatal
2526

2627
class BufferedFSDataInputStream(filePath: Path,
2728
hadoopConfig: Configuration,
@@ -32,7 +33,7 @@ class BufferedFSDataInputStream(filePath: Path,
3233
gpgPassphrase: Option[String]) extends AutoCloseable {
3334
val bytesInMegabyte: Int = 1048576
3435
private var isCompressedStream = false
35-
private var rawStream: FSDataInputStream = _
36+
private var rawStream: FSDataInputStream = _ // This is the base stream for GPG-encrypted files. Used for closing only, never for actual read.
3637

3738
if (bufferSizeInMegabytes <=0 || bufferSizeInMegabytes > 1000) {
3839
throw new IllegalArgumentException(s"Invalid buffer size $bufferSizeInMegabytes MB.")
@@ -51,13 +52,15 @@ class BufferedFSDataInputStream(filePath: Path,
5152
override def close(): Unit = {
5253
if (!isStreamClosed) {
5354
Try {
54-
if (rawStream != null) {
55-
rawStream.close()
56-
}
5755
if (in != null) {
5856
in.close()
5957
}
6058
}
59+
Try {
60+
if (rawStream != null) {
61+
rawStream.close()
62+
}
63+
}
6164
in = null
6265
rawStream = null
6366
isStreamClosed = true
@@ -168,15 +171,23 @@ class BufferedFSDataInputStream(filePath: Path,
168171
}
169172

170173
if (startOffset > 0) {
171-
if (!isCompressedStream) {
172-
baseStream.asInstanceOf[FSDataInputStream].seek(startOffset)
173-
} else {
174-
var toSkip = startOffset
175-
while (toSkip > 0) {
176-
val skipped = baseStream.skip(toSkip)
177-
if (skipped <= 0) return baseStream
178-
toSkip -= skipped
174+
try {
175+
if (!isCompressedStream) {
176+
baseStream.asInstanceOf[FSDataInputStream].seek(startOffset)
177+
} else {
178+
var toSkip = startOffset
179+
while (toSkip > 0) {
180+
val skipped = baseStream.skip(toSkip)
181+
if (skipped <= 0) return baseStream
182+
toSkip -= skipped
183+
}
179184
}
185+
} catch {
186+
case NonFatal(ex) =>
187+
Try {
188+
baseStream.close()
189+
}
190+
throw ex
180191
}
181192
}
182193
baseStream

0 commit comments

Comments
 (0)