Merge pull request #1 from 10XGenomics/paul/chunked-records

Update rust_htslib to master, fix issue with bam::Record::clone

Author: sreenathkrishnan

.gitignore

@@ -1,3 +1,4 @@
 *~
 target
 Cargo.lock
+htslib/*

CHANGELOG.md

@@ -2,6 +2,18 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## [0.15.0] - 2017-12-05
+### Changed
+- HeaderView of bam and bcf can now be cloned.
+
+
+## [0.14.0] - 2017-12-03
+### Added
+- An efficient ringbuffer for accessing BCF regions
+- An efficient ringbuffer for accessing BAM regions
+### Changed
+- Improved mutability annotation for readers.
+
 ## [0.13.0] - 2017-09-22
 ### Added
 - Ability to clone bam records.
@@ -21,7 +33,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 - Support seek and tell to handle virtual offsets.
 ### Changed
 - Renamed previous seek method into fetch (in line with pysam).
-- Improved CIGAR API. 
+- Improved CIGAR API.
 - Updated dependencies.
 
 ## [0.11.0] - 2017-05-01

Cargo.toml

@@ -1,14 +1,14 @@
 [package]
 name = "rust-htslib"
-version = "0.13.0"
+version = "0.15.0"
 authors = ["Christopher Schröder <[email protected]>", "Johannes Köster <[email protected]>"]
 build = "build.rs"
 description = "This library provides HTSlib bindings and a high level Rust API for reading and writing BAM files."
 readme = "README.md"
 keywords = ["htslib", "bam", "bioinformatics", "pileup", "sequencing"]
 license = "MIT"
 repository = "https://github.com/rust-bio/rust-htslib.git"
-documentation = "http://rust-bio.github.io/rust-htslib"
+documentation = "https://docs.rs/rust-htslib"
 
 
 [dependencies]

src/bam/buffer.rs

@@ -0,0 +1,155 @@
+// Copyright 2017 Johannes Köster.
+// Licensed under the MIT license (http://opensource.org/licenses/MIT)
+// This file may not be copied, modified, or distributed
+// except according to those terms.
+
+
+use std::collections::{vec_deque, VecDeque};
+use std::error::Error;
+use std::str;
+
+use bam;
+use bam::Read;
+
+
+/// A buffer for BAM records. This allows access regions in a sorted BAM file while iterating
+/// over it in a single pass.
+/// The buffer is implemented as a ringbuffer, such that extension or movement to the right has
+/// linear complexity. The buffer makes use of indexed random access. Hence, when fetching a
+/// region at the very end of the BAM, everything before is omitted without cost.
+pub struct RecordBuffer {
+    reader: bam::IndexedReader,
+    inner: VecDeque<bam::Record>,
+    overflow: Option<bam::Record>
+}
+
+
+unsafe impl Sync for RecordBuffer {}
+unsafe impl Send for RecordBuffer {}
+
+
+impl RecordBuffer {
+    /// Create a new `RecordBuffer`.
+    pub fn new(bam: bam::IndexedReader) -> Self {
+        RecordBuffer {
+            reader: bam,
+            inner: VecDeque::new(),
+            overflow: None
+        }
+    }
+
+    /// Return end position of buffer.
+    fn end(&self) -> Option<u32> {
+        self.inner.back().map(|rec| rec.pos() as u32)
+    }
+
+    fn tid(&self) -> Option<i32> {
+        self.inner.back().map(|rec| rec.tid())
+    }
+
+    /// Fill buffer at the given interval. The start coordinate has to be left of
+    /// the start coordinate of any previous `fill` operation.
+    /// Coordinates are 0-based, and end is exclusive.
+    /// Returns tuple with numbers of added and deleted records since the previous fetch.
+    pub fn fetch(&mut self, chrom: &[u8], start: u32, end: u32) -> Result<(usize, usize), Box<Error>> {
+        let mut added = 0;
+        // move overflow from last fetch into ringbuffer
+        if self.overflow.is_some() {
+            added += 1;
+            self.inner.push_back(self.overflow.take().unwrap());
+        }
+
+        if let Some(tid) = self.reader.header.tid(chrom) {
+            let mut deleted = 0;
+            let window_start = start;
+            if self.inner.is_empty() || self.end().unwrap() < window_start || self.tid().unwrap() != tid as i32 {
+                let end = self.reader.header.target_len(tid).unwrap();
+                try!(self.reader.fetch(tid, window_start, end));
+                deleted = self.inner.len();
+                self.inner.clear();
+            } else {
+                // remove records too far left
+                let to_remove = self.inner.iter().take_while(|rec| rec.pos() < window_start as i32).count();
+                for _ in 0..to_remove {
+                    self.inner.pop_front();
+                }
+                deleted = to_remove;
+            }
+
+            // extend to the right
+            loop {
+                let mut record = bam::Record::new();
+                if let Err(e) = self.reader.read(&mut record) {
+                    if e.is_eof() {
+                        break;
+                    }
+                    return Err(Box::new(e));
+                }
+
+                if record.is_unmapped() {
+                    continue;
+                }
+
+                let pos = record.pos();
+                if pos >= end as i32 {
+                    self.overflow = Some(record);
+                    break;
+                } else {
+                    self.inner.push_back(record);
+                    added += 1;
+                }
+            }
+
+            Ok((added, deleted))
+        } else {
+            Err(Box::new(RecordBufferError::UnknownSequence(str::from_utf8(chrom).unwrap().to_owned())))
+        }
+    }
+
+    /// Iterate over records that have been fetched with `fetch`.
+    pub fn iter(&self) -> vec_deque::Iter<bam::Record> {
+        self.inner.iter()
+    }
+
+    pub fn len(&self) -> usize {
+        self.inner.len()
+    }
+}
+
+
+quick_error! {
+    #[derive(Debug)]
+    pub enum RecordBufferError {
+        UnknownSequence(chrom: String) {
+            description("unknown sequence")
+            display("sequence {} cannot be found in BAM", chrom)
+        }
+    }
+}
+
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use itertools::Itertools;
+    use bam;
+
+    #[test]
+    fn test_buffer() {
+        let reader = bam::IndexedReader::from_path(&"test/test.bam").unwrap();
+
+        let mut buffer = RecordBuffer::new(reader);
+
+        buffer.fetch(b"CHROMOSOME_I", 1, 5).unwrap();
+        {
+            let records = buffer.iter().collect_vec();
+            assert_eq!(records.len(), 6);
+            assert_eq!(records[0].pos(), 1);
+            assert_eq!(records[1].pos(), 1);
+            assert_eq!(records[2].pos(), 1);
+            assert_eq!(records[3].pos(), 1);
+            assert_eq!(records[4].pos(), 1);
+            assert_eq!(records[5].pos(), 1);
+        }
+    }
+}

src/bam/header.rs

@@ -27,7 +27,7 @@ impl Header {
         // header which samtools (<=1.6) will complain
         // about
         while let Some(&last_char) = record.last() {
-            if last_char==b'\n' {
+            if last_char == b'\n' {
                 record.pop();
             } else {
                 break;