BCF 2.2 writing WIP

Author: Anders Leung

.travis.yml

@@ -13,6 +13,7 @@ cache:
 env:
   global:
     - HTSJDK_SAMTOOLS_BIN=/usr/bin/samtools
+    - HTSJDK_BCFTOOLS_BIN=/usr/bin/bcftools
 jdk:
   - oraclejdk8
   - openjdk8
@@ -32,6 +33,7 @@ matrix:
 
 before_install:
   - scripts/install-samtools.sh
+  - scripts/install-bcftools.sh
   - scripts/htsget-scripts/start-htsget-test-server.sh
 
 script:

scripts/install-bcftools.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+set -ex
+wget https://github.com/samtools/bcftools/releases/download/1.13/bcftools-1.13.tar.bz2
+tar -xjvf bcftools-1.13.tar.bz2
+cd bcftools-1.13 && ./configure --prefix=/usr && make && sudo make install

src/main/java/htsjdk/samtools/util/FileExtensions.java

@@ -65,6 +65,9 @@ public final class FileExtensions {
     public static final String VCF = ".vcf";
     public static final String VCF_INDEX = TRIBBLE_INDEX;
     public static final String BCF = ".bcf";
+    // Note that .bcf on its own may be gzip compressed and usually is,
+    // but files with the extension .bcf.gz to seem to exist in the wild and should be supported
+    public static final String COMPRESSED_BCF = ".bcf.gz";
     public static final String COMPRESSED_VCF = ".vcf.gz";
     public static final String COMPRESSED_VCF_INDEX = ".tbi";
     public static final List<String> VCF_LIST = Collections.unmodifiableList(Arrays.asList(VCF, COMPRESSED_VCF, BCF));

src/main/java/htsjdk/samtools/util/ListByteBufferOutputStream.java

@@ -0,0 +1,138 @@
+package htsjdk.samtools.util;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+/**
+ * Growable byte buffer backed by a list of byte arrays, which can
+ * be used to buffer data without reallocating an underlying array.
+ * Once data is accumulated, it can either be retrieved by converting
+ * into a byte[] for interfaces that require a contiguous block of bytes,
+ * or written directly to an OutputStream to avoid array copies.
+ */
+public class ListByteBufferOutputStream extends OutputStream {
+
+    private final int blockSize;
+    private final ArrayList<byte[]> blocks;
+    private byte[] currentBlock;
+    private int nextBlockIndex;
+    private int nextBytePosition;
+    private int size;
+
+    public ListByteBufferOutputStream(final int blockSize) {
+        this.blockSize = blockSize;
+        blocks = new ArrayList<>();
+        nextBlockIndex = 0;
+        advanceBlock();
+        size = 0;
+    }
+
+    @Override
+    public void write(final int b) {
+        if (nextBytePosition == blockSize) {
+            advanceBlock();
+        }
+        currentBlock[nextBytePosition++] = (byte) b;
+        size++;
+    }
+
+    public void write(final byte b, final int nCopies) {
+        assert nCopies >= 0;
+
+        int bytesRemaining = nCopies;
+        while (bytesRemaining > 0) {
+            if (nextBytePosition == blockSize) {
+                advanceBlock();
+            }
+            final int toIndex = Math.min(nextBytePosition + bytesRemaining, blockSize);
+            Arrays.fill(currentBlock, nextBytePosition, toIndex, b);
+            bytesRemaining -= toIndex - nextBytePosition;
+            nextBytePosition = toIndex;
+        }
+        size += nCopies;
+    }
+
+    @Override
+    public void write(final byte[] b) {
+        write(b, 0, b.length);
+    }
+
+    @Override
+    public void write(final byte[] b, int off, final int len) {
+        assert b != null;
+        assert off >= 0;
+        assert len >= 0;
+        assert off + len <= b.length;
+
+        int bytesRemaining = len;
+        while (bytesRemaining > 0) {
+            if (nextBytePosition == blockSize) {
+                advanceBlock();
+            }
+            final int lengthToWrite = Math.min(bytesRemaining, blockSize - nextBytePosition);
+            System.arraycopy(b, off, currentBlock, nextBytePosition, lengthToWrite);
+            nextBytePosition += lengthToWrite;
+            off += lengthToWrite;
+            bytesRemaining -= lengthToWrite;
+        }
+        size += len;
+    }
+
+    public int size() {
+        return size;
+    }
+
+    public void writeTo(final OutputStream out) throws IOException {
+        for (final byte[] b : blocks) {
+            if (b == currentBlock) {
+                out.write(b, 0, nextBytePosition);
+                break;
+            } else {
+                out.write(b);
+            }
+        }
+    }
+
+    public byte[] toByteArray() {
+        final byte[] bytes = new byte[size];
+        final ByteBuffer buff = ByteBuffer.wrap(bytes);
+        for (final byte[] b : blocks) {
+            if (b == currentBlock) {
+                buff.put(b, 0, nextBytePosition);
+                break;
+            } else {
+                buff.put(b);
+            }
+        }
+        return bytes;
+    }
+
+    public void reset() {
+        currentBlock = blocks.get(0);
+        nextBytePosition = 0;
+        nextBlockIndex = 1;
+        size = 0;
+    }
+
+    public void clear() {
+        reset();
+        // blocks always has at least 1 element
+        blocks.subList(1, blocks.size()).clear();
+    }
+
+    private void advanceBlock() {
+        if (nextBlockIndex == blocks.size()) {
+            // Need to add a new block
+            currentBlock = new byte[blockSize];
+            blocks.add(currentBlock);
+        } else {
+            // Reuse old block
+            currentBlock = blocks.get(nextBlockIndex);
+        }
+        nextBytePosition = 0;
+        nextBlockIndex++;
+    }
+}

src/main/java/htsjdk/tribble/TribbleIndexedFeatureReader.java

@@ -33,6 +33,7 @@
 import htsjdk.tribble.index.IndexFactory;
 import htsjdk.tribble.readers.PositionalBufferedStream;
 import htsjdk.tribble.util.ParsingUtils;
+import htsjdk.variant.vcf.VCFFileReader;
 
 import java.io.BufferedInputStream;
 import java.io.IOException;
@@ -252,7 +253,11 @@ private void readHeader() throws IOException {
         PositionalBufferedStream pbs = null;
         try {
             is = ParsingUtils.openInputStream(path, wrapper);
-            if (IOUtil.hasBlockCompressedExtension(new URI(URLEncoder.encode(path, "UTF-8")))) {
+            // BCFs are usually gzipped but do not have the .gz extension,
+            // so we explicitly check for the presence of a gzip header
+            if (IOUtil.hasBlockCompressedExtension(new URI(URLEncoder.encode(path, "UTF-8")))
+                || (VCFFileReader.isBCF(path) && IOUtil.isGZIPInputStream(is))
+            ) {
                 // TODO: TEST/FIX THIS! https://github.com/samtools/htsjdk/issues/944
                 // TODO -- warning I don't think this can work, the buffered input stream screws up position
                 is = new GZIPInputStream(new BufferedInputStream(is));
@@ -326,7 +331,8 @@ public WFIterator() throws IOException {
             final InputStream inputStream = ParsingUtils.openInputStream(path, wrapper);
 
             final PositionalBufferedStream pbs;
-            if (IOUtil.hasBlockCompressedExtension(path)) {
+            // BCFs can be gzipped but usually do not have a compressed extension, so an extra check is needed
+            if (IOUtil.hasBlockCompressedExtension(path) || (VCFFileReader.isBCF(path) && IOUtil.isGZIPInputStream(inputStream))) {
                 // Gzipped -- we need to buffer the GZIPInputStream methods as this class makes read() calls,
                 // and seekableStream does not support single byte reads
                 final InputStream is = new GZIPInputStream(new BufferedInputStream(inputStream, 512000));

src/main/java/htsjdk/tribble/util/ParsingUtils.java

@@ -101,8 +101,8 @@ public static InputStream openInputStream(final String uri, final Function<Seeka
         if (URL_SCHEMES.stream().anyMatch(uri::startsWith)) {
             inputStream = getURLHelper(new URL(uri)).openInputStream();
         } else if (!IOUtil.hasScheme(uri)) {
-            File file = new File(uri);
-            inputStream = Files.newInputStream(file.toPath());
+            final File file = new File(uri);
+            inputStream = new SeekablePathStream(file.toPath(), wrapper);
         } else {
             inputStream = new SeekablePathStream(IOUtil.getPath(uri), wrapper);
         }