-
Notifications
You must be signed in to change notification settings - Fork 20
FiniteWriter trait + many important fixes #11
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
4653eeb
336d8d4
a3ddd48
c7ad707
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -33,6 +33,7 @@ This is an original implementation. | |
*/ | ||
|
||
use std::{io, vec}; | ||
use shared::FiniteWriter; | ||
|
||
pub type Symbol = u8; | ||
static symbol_bits: uint = 8; | ||
|
@@ -184,16 +185,6 @@ impl<W: Writer> Encoder<W> { | |
self.stream.write(bytes) | ||
} | ||
|
||
/// Finish decoding by writing the code tail word | ||
pub fn finish(mut self) -> (W, io::IoResult<()>) { | ||
assert!(border_bits == 32); | ||
self.bytes_written += 4; | ||
let code = self.range.get_code_tail(); | ||
let result = self.stream.write_be_u32(code); | ||
let result = result.and(self.stream.flush()); | ||
(self.stream, result) | ||
} | ||
|
||
/// Flush the output stream | ||
pub fn flush(&mut self) -> io::IoResult<()> { | ||
self.stream.flush() | ||
|
@@ -205,6 +196,24 @@ impl<W: Writer> Encoder<W> { | |
} | ||
} | ||
|
||
impl<W: FiniteWriter> Encoder<W> { | ||
/// Finish decoding by writing the code tail word | ||
pub fn finish(mut self) -> (W, io::IoResult<()>) { | ||
let ret = self.write_terminator(); | ||
(self.stream, ret) | ||
} | ||
|
||
/// Write code tail bits | ||
pub fn write_terminator(&mut self) -> io::IoResult<()> { | ||
assert!(border_bits == 32); | ||
self.bytes_written += 4; | ||
let code = self.range.get_code_tail(); | ||
let result = self.stream.write_be_u32(code); | ||
result.and(self.stream.write_terminator()) | ||
} | ||
} | ||
|
||
|
||
/// An arithmetic decoder helper | ||
pub struct Decoder<R> { | ||
priv stream: R, | ||
|
@@ -435,6 +444,7 @@ impl Model for FrequencyTable { | |
|
||
|
||
/// A basic byte-encoding arithmetic | ||
/// uses a special terminator code to end the stream | ||
pub struct ByteEncoder<W> { | ||
/// A lower level encoder | ||
encoder: Encoder<W>, | ||
|
@@ -448,7 +458,7 @@ impl<W: Writer> ByteEncoder<W> { | |
let freq_max = range_default_threshold >> 2; | ||
ByteEncoder { | ||
encoder: Encoder::new(w), | ||
freq: FrequencyTable::new_flat(symbol_total, freq_max), | ||
freq: FrequencyTable::new_flat(symbol_total+1, freq_max), | ||
} | ||
} | ||
} | ||
|
@@ -468,13 +478,32 @@ impl<W: Writer> Writer for ByteEncoder<W> { | |
} | ||
} | ||
|
||
impl<W: FiniteWriter> FiniteWriter for ByteEncoder<W> { | ||
fn write_terminator(&mut self) -> io::IoResult<()> { | ||
self.encoder.encode(symbol_total, &self.freq). | ||
and(self.encoder.write_terminator()) | ||
} | ||
} | ||
|
||
impl<W: FiniteWriter> ByteEncoder<W> { | ||
/// Finish encoding and return the underlying stream | ||
pub fn finish(mut self) -> (W, io::IoResult<()>) { | ||
let ret = self.write_terminator(); | ||
let (w, ret_encoder) = self.encoder.finish(); | ||
(w, ret.and(ret_encoder)) | ||
} | ||
} | ||
|
||
|
||
/// A basic byte-decoding arithmetic | ||
/// expects a special terminator code for the end of the stream | ||
pub struct ByteDecoder<R> { | ||
/// A lower level decoder | ||
decoder: Decoder<R>, | ||
/// A basic frequency table | ||
freq: FrequencyTable, | ||
/// Remember if we found the terminator code | ||
priv is_eof: bool, | ||
} | ||
|
||
impl<R: Reader> ByteDecoder<R> { | ||
|
@@ -484,7 +513,8 @@ impl<R: Reader> ByteDecoder<R> { | |
let freq_max = range_default_threshold >> 2; | ||
ByteDecoder { | ||
decoder: Decoder::new(r), | ||
freq: FrequencyTable::new_flat(symbol_total, freq_max), | ||
freq: FrequencyTable::new_flat(symbol_total+1, freq_max), | ||
is_eof: false, | ||
} | ||
} | ||
} | ||
|
@@ -494,20 +524,21 @@ impl<R: Reader> Reader for ByteDecoder<R> { | |
if self.decoder.tell() == 0 { | ||
if_ok!(self.decoder.start()); | ||
} | ||
let mut ret = Ok(dst.len()); | ||
if self.is_eof { | ||
return Err(io::standard_error(io::EndOfFile)) | ||
} | ||
let mut amount = 0u; | ||
for out_byte in dst.mut_iter() { | ||
match self.decoder.decode(&self.freq) { | ||
Ok(value) => { | ||
self.freq.update(value, 10, 1); | ||
*out_byte = value as u8; | ||
}, | ||
Err(e) => { | ||
ret = Err(e); | ||
break | ||
} | ||
let value = if_ok!(self.decoder.decode(&self.freq)); | ||
if value == symbol_total { | ||
self.is_eof = true; | ||
break | ||
} | ||
self.freq.update(value, 10, 1); | ||
*out_byte = value as u8; | ||
amount += 1; | ||
} | ||
ret | ||
Ok(amount) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can you add tests for your changes in this module? |
||
} | ||
} | ||
|
||
|
@@ -523,12 +554,12 @@ mod test { | |
info!("Roundtrip Ari of size {}", bytes.len()); | ||
let mut e = ByteEncoder::new(MemWriter::new()); | ||
e.write(bytes).unwrap(); | ||
let (e, r) = e.encoder.finish(); | ||
let (e, r) = e.finish(); | ||
r.unwrap(); | ||
let encoded = e.unwrap(); | ||
debug!("Roundtrip input {:?} encoded {:?}", bytes, encoded); | ||
let mut d = ByteDecoder::new(BufReader::new(encoded)); | ||
let decoded = d.read_bytes(bytes.len()).unwrap(); | ||
let decoded = d.read_to_end().unwrap(); | ||
assert_eq!(bytes.as_slice(), decoded.as_slice()); | ||
} | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -26,6 +26,7 @@ can be found at https://github.com/bkaradzic/go-lz4. | |
use std::io; | ||
use std::num; | ||
use std::vec; | ||
use shared::FiniteWriter; | ||
|
||
static MAGIC: u32 = 0x184d2204; | ||
|
||
|
@@ -243,6 +244,7 @@ impl<R: Reader> Decoder<R> { | |
// raw block to read | ||
n if n & 0x80000000 != 0 => { | ||
let amt = (n & 0x7fffffff) as uint; | ||
debug!("decoding a raw block of size {}", amt) | ||
self.output.truncate(0); | ||
self.output.reserve(amt); | ||
if_ok!(self.r.push_bytes(&mut self.output, amt)); | ||
|
@@ -252,12 +254,14 @@ impl<R: Reader> Decoder<R> { | |
|
||
// actual block to decompress | ||
n => { | ||
debug!("decoding a compressed block of size {}", n); | ||
let n = n as uint; | ||
self.temp.truncate(0); | ||
self.temp.reserve(n); | ||
if_ok!(self.r.push_bytes(&mut self.temp, n)); | ||
|
||
let target = num::min(self.max_block_size, 4 * n / 3); | ||
debug!("target size: {}", target); | ||
self.output.truncate(0); | ||
self.output.reserve(target); | ||
let mut decoder = BlockDecoder { | ||
|
@@ -269,13 +273,16 @@ impl<R: Reader> Decoder<R> { | |
}; | ||
self.start = 0; | ||
self.end = decoder.decode(); | ||
debug!("end of block: {}", self.end); | ||
} | ||
} | ||
|
||
if self.blk_checksum { | ||
let cksum = if_ok!(self.r.read_le_u32()); | ||
debug!("ignoring block checksum {:?}", cksum); | ||
} | ||
|
||
debug!("block is done"); | ||
return Ok(true); | ||
} | ||
|
||
|
@@ -357,15 +364,13 @@ impl<W: Writer> Encoder<W> { | |
false | ||
} | ||
|
||
/// This function is used to flag that this session of compression is done | ||
/// with. The stream is finished up (final bytes are written), and then the | ||
/// wrapped writer is returned. | ||
pub fn finish(mut self) -> (W, io::IoResult<()>) { | ||
let result = self.flush(); | ||
let result = result.and(self.w.write_le_u32(0)); | ||
// XXX: this checksum is wrong | ||
let result = result.and(self.w.write_le_u32(0)); | ||
(self.w, result) | ||
/// End the current block | ||
fn finish_block(&mut self) -> io::IoResult<()> { | ||
if self.buf.len() > 0 { | ||
self.encode_block() | ||
} else { | ||
Ok(()) | ||
} | ||
} | ||
} | ||
|
||
|
@@ -396,13 +401,31 @@ impl<W: Writer> Writer for Encoder<W> { | |
} | ||
|
||
fn flush(&mut self) -> io::IoResult<()> { | ||
if self.buf.len() > 0 { | ||
if_ok!(self.encode_block()); | ||
} | ||
self.w.flush() | ||
self.finish_block().and(self.w.flush()) | ||
} | ||
} | ||
|
||
impl<W: FiniteWriter> FiniteWriter for Encoder<W> { | ||
fn write_terminator(&mut self) -> io::IoResult<()> { | ||
let result = self.finish_block(); | ||
let result = result.and(self.w.write_le_u32(0)); | ||
// XXX: this checksum is wrong | ||
let result = result.and(self.w.write_le_u32(0)); | ||
result.and(self.w.write_terminator()) | ||
} | ||
} | ||
|
||
impl<W: FiniteWriter> Encoder<W> { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This means that you can't unwrap any stream other than a There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I understand your concern, and open to ideas on how to make it better. If we implement it for just There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. What exactly is the bug that prevents you from calling this in There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. rust-lang/rust#4252 |
||
/// This function is used to flag that this session of compression is done | ||
/// with. The stream is finished up (final bytes are written), and then the | ||
/// wrapped writer is returned. | ||
pub fn finish(mut self) -> (W, io::IoResult<()>) { | ||
let result = (&mut self as &mut FiniteWriter).write_terminator(); | ||
(self.w, result) | ||
} | ||
} | ||
|
||
|
||
#[cfg(test)] | ||
mod test { | ||
use extra::test; | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
As with below, it would be nice to have tests for these changes.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Will do.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Actually, I don't think more tests needed for this one, because tests call
finish()
, and that calls the newwrite_terminator
anyway.