From 44df1c1f0d45bf1c53d435a64e28346ad8e56c5c Mon Sep 17 00:00:00 2001 From: songpan Date: Thu, 4 Apr 2019 11:02:02 -0700 Subject: [PATCH] Add profiler for MMAP. Startblock: PiperOrigin-RevId: 241962624 --- .../bytesource/ByteArrayByteSource.java | 1 - .../shared/bytesource/MmapByteSource.java | 146 ++++++++++++++++++ .../shared/bytesource/MmapByteSourceTest.java | 48 ++++++ 3 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 shared/src/main/java/com/google/archivepatcher/shared/bytesource/MmapByteSource.java create mode 100644 shared/src/test/java/com/google/archivepatcher/shared/bytesource/MmapByteSourceTest.java diff --git a/shared/src/main/java/com/google/archivepatcher/shared/bytesource/ByteArrayByteSource.java b/shared/src/main/java/com/google/archivepatcher/shared/bytesource/ByteArrayByteSource.java index d6f7353f..ec447236 100644 --- a/shared/src/main/java/com/google/archivepatcher/shared/bytesource/ByteArrayByteSource.java +++ b/shared/src/main/java/com/google/archivepatcher/shared/bytesource/ByteArrayByteSource.java @@ -42,5 +42,4 @@ protected InputStream openStream(long offset, long length) throws IOException { public void close() throws IOException { // Nothing needs to be done. } - } diff --git a/shared/src/main/java/com/google/archivepatcher/shared/bytesource/MmapByteSource.java b/shared/src/main/java/com/google/archivepatcher/shared/bytesource/MmapByteSource.java new file mode 100644 index 00000000..170c4e7d --- /dev/null +++ b/shared/src/main/java/com/google/archivepatcher/shared/bytesource/MmapByteSource.java @@ -0,0 +1,146 @@ +// Copyright 2016 Google LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.archivepatcher.shared.bytesource; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel.MapMode; + +/** A {@link ByteSource} backed by a memory mapped file. */ +public class MmapByteSource extends ByteSource { + private final RandomAccessFile raf; + private ByteBuffer byteBuffer; + + public MmapByteSource(File file) throws IOException { + this.raf = new RandomAccessFile(file, "r"); + long length = file.length(); + if (length > Integer.MAX_VALUE) { + throw new IllegalArgumentException( + "RandomAccessMmapObject only supports file sizes up to " + "Integer.MAX_VALUE."); + } + this.byteBuffer = raf.getChannel().map(MapMode.READ_ONLY, 0, (int) length); + } + + @Override + public long length() { + return byteBuffer.capacity(); + } + + /** + * Note that this method is not thread safe since all streams will be sharing the same ByteBuffer. + */ + @Override + protected InputStream openStream(long offset, long length) throws IOException { + if (offset + length > length()) { + throw new IllegalArgumentException( + "Specified offset and length would read out of the bounds of the mapped byte buffer."); + } + return new ByteBufferInputStream(byteBuffer, (int) offset, (int) length); + } + + @Override + public void close() throws IOException { + raf.close(); + + // There is a long-standing bug with memory mapped objects in Java that requires the JVM to + // finalize the MappedByteBuffer reference before the unmap operation is performed. This leaks + // file handles and fills the virtual address space. Worse, on some systems (Windows for one) + // the active mmap prevents the temp file from being deleted - even if File.deleteOnExit() is + // used. The only safe way to ensure that file handles and actual files are not leaked by this + // class is to force an explicit full gc after explicitly nulling the MappedByteBuffer + // reference. This has to be done before attempting file deletion. + // + // See https://github.com/andrewhayden/archive-patcher/issues/5 for more information. + // Also http://bugs.java.com/view_bug.do?bug_id=6417205. + + byteBuffer = null; + System.gc(); + System.runFinalization(); + } + + private static class ByteBufferInputStream extends InputStream { + private final ByteBuffer buffer; + private final int readLimit; + // Position of next read. We cannot rely on internal state of the byte buffer since it will be + // shared across multiple streams + private int nextReadPos; + + public ByteBufferInputStream(ByteBuffer buffer, int offset, int length) { + this.buffer = buffer; + this.nextReadPos = offset; + this.readLimit = offset + length; + } + + @Override + public int available() throws IOException { + return readLimit - nextReadPos; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (endOfStream()) { + return -1; + } + buffer.position(nextReadPos); + + // Default behaviour for ByteBuffer when not enough data can be read is to through an + // Exception. Expected behaviour of an InputStream is to read as much as possible. + int remaining = readLimit - nextReadPos; + if (len > remaining) { + len = remaining; + } + + buffer.get(b, off, len); + nextReadPos += len; + return len; + } + + @Override + public int read() throws IOException { + if (endOfStream()) { + return -1; + } + buffer.position(nextReadPos); + + // InputStream.read() expects an unsigned byte. ByteBuffer.get() returns a signed one. Hence + // we need to convert it to unsigned. + ++nextReadPos; + return buffer.get() & 0xff; + } + + @Override + public long skip(long n) throws IOException { + if (n <= 0) { + return 0; + } + + int remaining = readLimit - nextReadPos; + if (n > remaining) { + nextReadPos = readLimit; + return remaining; + } else { + nextReadPos += (int)n; + return n; + } + } + + private boolean endOfStream() { + return nextReadPos >= readLimit; + } + } +} diff --git a/shared/src/test/java/com/google/archivepatcher/shared/bytesource/MmapByteSourceTest.java b/shared/src/test/java/com/google/archivepatcher/shared/bytesource/MmapByteSourceTest.java new file mode 100644 index 00000000..36b75eb2 --- /dev/null +++ b/shared/src/test/java/com/google/archivepatcher/shared/bytesource/MmapByteSourceTest.java @@ -0,0 +1,48 @@ +// Copyright 2016 Google LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.archivepatcher.shared.bytesource; + +import static com.google.archivepatcher.shared.TestUtils.storeInTempFile; + +import java.io.ByteArrayInputStream; +import java.io.File; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; + +public class MmapByteSourceTest extends ByteSourceBaseTest { + + private static File tempFile = null; + private static byte[] testData; + + @BeforeClass + public static void staticSetUp() throws Exception { + testData = getSampleTestData(); + tempFile = storeInTempFile(new ByteArrayInputStream(testData)); + } + + @Before + public void setUp() throws Exception { + byteSource = new MmapByteSource(tempFile); + expectedData = testData; + } + + @AfterClass + public static void staticTearDown() throws Exception { + if (tempFile != null) { + tempFile.delete(); + } + } +}