Skip to content

Commit f6a996e

Browse files
alvraalvra
committed
Allow controlling the parameters of the fast-forward optimization.
1 parent 4396e58 commit f6a996e

File tree

2 files changed

+91
-31
lines changed

2 files changed

+91
-31
lines changed

‎Cargo.toml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[package]
22
name = "reqwest-file"
3-
version = "0.1.0"
3+
version = "0.2.0"
44
edition = "2021"
55
license = "MIT OR Apache-2.0"
66
authors = ["Alexander van Ratingen"]
@@ -20,6 +20,7 @@ tokio-util = { version = "0.7.0", default-features = false, features = ["io"] }
2020
reqwest = { version = "0.11.9", default-features = false, features = ["stream"] }
2121

2222
[dev-dependencies]
23+
paste = "1.0.6"
2324
serde = { version = "1.0.136", default-features = false, features = ["derive"] }
2425
tokio = { version = "1.16.1", features = ["rt", "macros"] }
2526
tokio-test = "0.4.2"

‎src/lib.rs

Lines changed: 89 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
//!
1616
//! let client = reqwest::Client::new();
1717
//! let request = client.get("http://httpbin.org/base64/aGVsbG8gd29ybGQ=");
18-
//! let mut file = RequestFile::new(request);
18+
//! let mut file: RequestFile = RequestFile::new(request);
1919
//!
2020
//! let mut buffer = [0; 5];
2121
//! assert_eq!(file.read(&mut buffer).await.unwrap(), 5);
@@ -126,16 +126,16 @@ fn send_request(request: &RequestBuilder, offset: u64) -> RequestStreamFuture {
126126
///
127127
/// Return `Ok` if the fastforward is complete,
128128
/// or if EOF is reached (at the first empty read).
129-
fn fastforward<R: AsyncRead>(
129+
fn fastforward<R: AsyncRead, const BUFFER: usize>(
130130
mut reader: Pin<&mut R>,
131131
delta: u64,
132132
context: &mut Context<'_>,
133133
) -> (u64, u64, Poll<Result<(), IoError>>) {
134-
let mut array = [std::mem::MaybeUninit::uninit(); FF_BUFFER];
134+
let mut array = [std::mem::MaybeUninit::uninit(); BUFFER];
135135
let mut remaining = delta;
136136
let poll = loop {
137137
assert!(remaining > 0);
138-
let buffer_size = (remaining as usize).min(FF_BUFFER);
138+
let buffer_size = (remaining as usize).min(BUFFER);
139139
let mut buffer = ReadBuf::uninit(&mut array[0..buffer_size]);
140140
match reader.as_mut().poll_read(context, &mut buffer) {
141141
Poll::Ready(Ok(())) => {
@@ -160,10 +160,10 @@ fn fastforward<R: AsyncRead>(
160160
}
161161

162162
/// The maximum fast-forward read length.
163-
const FF_WINDOW: u64 = 128 * 1024;
163+
const DEFAULT_FF_WINDOW: u64 = 128 * 1024;
164164

165165
/// The size of the fast-forward read buffer.
166-
const FF_BUFFER: usize = 4096;
166+
const DEFAULT_FF_BUFFER: usize = 4096;
167167

168168
/// State of the request file.
169169
enum State<P, R> {
@@ -236,19 +236,35 @@ enum State<P, R> {
236236
/// ## Fast-Forward
237237
///
238238
/// As an optimization, seeking forward by a small amount
239-
/// (currently less than 128KiB) will not perform
239+
/// (by default up to 128KiB) will not perform
240240
/// a new request, but rather fast-forward through
241241
/// the response body.
242242
///
243243
/// This type of seek is always allowed,
244-
/// even if the webserver does not support `Range` requests
244+
/// even if the webserver does not support `Range` requests.
245+
///
246+
/// ###### Customization
247+
///
248+
/// The settings for fast-forwards can be changed through
249+
/// two constant parameters.
250+
///
251+
/// * `FF_WINDOW` limits how much can be fast-forwarded;
252+
/// only seeking up to this number of bytes forwards
253+
/// will read through the request (discarding the data)
254+
/// to avoid sending out a new request.
255+
///
256+
/// * `FF_BUFFER` defines the internal buffer size
257+
/// used to read into during a fast-foward.
245258
///
246259
/// [`NotSeekable`]: std::io::ErrorKind::NotSeekable
247260
/// [`Unsupported`]: std::io::ErrorKind::Unsupported
248261
/// [`InvalidInput`]: std::io::ErrorKind::InvalidInput
249262
/// [`Other`]: std::io::ErrorKind::Other
250263
#[pin_project(project = RequestFileProjection)]
251-
pub struct RequestFile {
264+
pub struct RequestFile<
265+
const FF_WINDOW: u64 = DEFAULT_FF_WINDOW,
266+
const FF_BUFFER: usize = DEFAULT_FF_BUFFER,
267+
> {
252268
/// The request template.
253269
request: RequestBuilder,
254270
/// The state of the HTTP request.
@@ -259,7 +275,11 @@ pub struct RequestFile {
259275
position: u64,
260276
}
261277

262-
impl RequestFile {
278+
impl<
279+
const FF_WINDOW: u64,
280+
const FF_BUFFER: usize,
281+
>
282+
RequestFile<FF_WINDOW, FF_BUFFER> {
263283
/// Create a new file-like object for a web resource.
264284
///
265285
/// # Panics
@@ -328,7 +348,11 @@ impl RequestFile {
328348
}
329349
}
330350

331-
impl RequestFileProjection<'_> {
351+
impl<
352+
const FF_WINDOW: u64,
353+
const FF_BUFFER: usize,
354+
>
355+
RequestFileProjection<'_, FF_WINDOW, FF_BUFFER> {
332356
/// Compute the absolute seek position.
333357
fn resolve_seek_position(
334358
&self,
@@ -396,7 +420,7 @@ impl RequestFileProjection<'_> {
396420
) -> Poll<Result<(), IoError>> {
397421
match std::mem::replace(self.state, State::Transient) {
398422
State::Seeking(mut reader, delta) => {
399-
let (read, remaining, poll) = fastforward(
423+
let (read, remaining, poll) = fastforward::<_, {FF_BUFFER}>(
400424
reader.as_mut(), delta, context);
401425
*self.position = self.position.saturating_add(read);
402426
match poll {
@@ -443,7 +467,8 @@ impl RequestFileProjection<'_> {
443467
}
444468
}
445469

446-
impl AsyncRead for RequestFile {
470+
impl<const FF_WINDOW: u64, const FF_BUFFER: usize> AsyncRead
471+
for RequestFile<FF_WINDOW, FF_BUFFER> {
447472
fn poll_read(
448473
self: Pin<&mut Self>,
449474
context: &mut Context<'_>,
@@ -467,7 +492,8 @@ impl AsyncRead for RequestFile {
467492
}
468493
}
469494

470-
impl AsyncSeek for RequestFile {
495+
impl<const FF_WINDOW: u64, const FF_BUFFER: usize> AsyncSeek
496+
for RequestFile<FF_WINDOW, FF_BUFFER> {
471497
fn start_seek(
472498
self: Pin<&mut Self>,
473499
position: SeekFrom,
@@ -590,13 +616,41 @@ mod tests {
590616
macro_rules! test {
591617
(
592618
$name:ident
593-
[ $( SeekFrom::$seek_from:ident($offset:literal) => ( $($tell:tt)* ) );* $(;)? ]
619+
$( $token:tt )*
620+
) => {
621+
// Make versions of the test for various const param combos.
622+
test!(
623+
@concrete
624+
$name
625+
$( $token )*
626+
);
627+
test!(
628+
@concrete
629+
@ff_window 2
630+
@ff_buffer 1
631+
$name
632+
$( $token )*
633+
);
634+
};
635+
(
636+
@concrete
637+
$( @ff_window $ff_window:literal )?
638+
$( @ff_buffer $ff_buffer:literal )?
639+
$name:ident
640+
[
641+
$( SeekFrom::$seek_from:ident($offset:literal)
642+
=> ( $($tell:tt)* ) );* $(;)?
643+
]
594644
$( Content-Length = $content_length:literal )?
595645
$( Content-Range = $content_range:literal )?
596646
$data:literal => $result:tt
597-
) => {
647+
) => { paste::paste! {
598648
#[tokio::test]
599-
async fn $name() {
649+
async fn [<
650+
$name
651+
$( _ff_window_ $ff_window )?
652+
$( _ff_buffer_ $ff_buffer )?
653+
>]() {
600654
let url = start_server();
601655
let data = $data;
602656
let client = reqwest::Client::new();
@@ -615,7 +669,12 @@ mod tests {
615669
)?
616670

617671
let request = client.get(format!("{url}/?data={data}{params}"));
618-
let mut file = RequestFile::new(request);
672+
const FF_WINDOW: u64 = super::DEFAULT_FF_WINDOW
673+
$( + $ff_window - super::DEFAULT_FF_WINDOW )?;
674+
const FF_BUFFER: usize = super::DEFAULT_FF_BUFFER
675+
$( + $ff_buffer - super::DEFAULT_FF_BUFFER )?;
676+
let mut file: RequestFile::<FF_WINDOW, FF_BUFFER>
677+
= RequestFile::new(request);
619678

620679
$(
621680
let seek_from = SeekFrom::$seek_from($offset);
@@ -627,7 +686,7 @@ mod tests {
627686
let read_result = file.read_to_string(&mut response_data).await;
628687
test!(@check_read read_result response_data $result);
629688
}
630-
};
689+
}};
631690
(
632691
@check_seek $seek:ident $result:literal
633692
) => {
@@ -829,7 +888,7 @@ mod tests {
829888
let client = reqwest::Client::new();
830889
let data = "abc";
831890
let request = client.get(format!("{url}/?data={data}"));
832-
let mut file = RequestFile::new(request);
891+
let mut file: RequestFile = RequestFile::new(request);
833892

834893
let mut response_data = String::new();
835894
file.read_to_string(&mut response_data).await.expect("read error");
@@ -846,7 +905,7 @@ mod tests {
846905
let client = reqwest::Client::new();
847906
let data = "abcd";
848907
let request = client.get(format!("{url}/?data={data}"));
849-
let mut file = RequestFile::new(request);
908+
let mut file: RequestFile = RequestFile::new(request);
850909

851910
assert_eq!(file.size(), None);
852911
file.prepare().await.unwrap();
@@ -859,7 +918,7 @@ mod tests {
859918
let client = reqwest::Client::new();
860919
let data = "abcd";
861920
let request = client.get(format!("{url}/?data={data}&content_length=true"));
862-
let mut file = RequestFile::new(request);
921+
let mut file: RequestFile = RequestFile::new(request);
863922

864923
assert_eq!(file.size(), None);
865924
file.seek(SeekFrom::Start(2)).await.unwrap();
@@ -872,7 +931,7 @@ mod tests {
872931
let client = reqwest::Client::new();
873932
let data = "abcd";
874933
let request = client.get(format!("{url}/?data={data}&content_range=true"));
875-
let mut file = RequestFile::new(request);
934+
let mut file: RequestFile = RequestFile::new(request);
876935

877936
assert_eq!(file.size(), None);
878937
file.seek(SeekFrom::Start(2)).await.unwrap();
@@ -885,7 +944,7 @@ mod tests {
885944
let url = start_server();
886945
let client = reqwest::Client::new();
887946
let request = client.get(format!("{url}/404"));
888-
let mut file = RequestFile::new(request);
947+
let mut file: RequestFile = RequestFile::new(request);
889948

890949
match file.prepare().await {
891950
Ok(()) => unreachable!(),
@@ -906,7 +965,7 @@ mod tests {
906965
let url = start_server();
907966
let client = reqwest::Client::new();
908967
let request = client.get(format!("{url}/?data=abc&status=204"));
909-
let mut file = RequestFile::new(request);
968+
let mut file: RequestFile = RequestFile::new(request);
910969

911970
match file.prepare().await {
912971
Ok(()) => unreachable!(),
@@ -926,7 +985,7 @@ mod tests {
926985
let url = start_server();
927986
let client = reqwest::Client::new();
928987
let request = client.get(format!("{url}/no-range-support"));
929-
let mut file = RequestFile::new(request);
988+
let mut file: RequestFile = RequestFile::new(request);
930989

931990
file.seek(SeekFrom::Start(0)).await.expect("seek error");
932991
let data = read(&mut file).await.expect("read error");
@@ -939,7 +998,7 @@ mod tests {
939998
let url = start_server();
940999
let client = reqwest::Client::new();
9411000
let request = client.get(format!("{url}/no-range-support"));
942-
let mut file = RequestFile::new(request);
1001+
let mut file: RequestFile = RequestFile::new(request);
9431002

9441003
// seek beyond the fastforward window
9451004
file.seek(SeekFrom::Start(1_000_000_000)).await.expect("seek error");
@@ -950,7 +1009,7 @@ mod tests {
9501009
let url = start_server();
9511010
let client = reqwest::Client::new();
9521011
let request = client.get(format!("{url}/?data=abc"));
953-
let mut file = RequestFile::new(request);
1012+
let mut file: RequestFile = RequestFile::new(request);
9541013

9551014
let seek_from = SeekFrom::Start(u64::MAX - 10);
9561015
let pos = file.seek(seek_from).await.expect("seek error");
@@ -965,7 +1024,7 @@ mod tests {
9651024
let url = start_server();
9661025
let client = reqwest::Client::new();
9671026
let request = client.get(format!("{url}/?data=abc"));
968-
let mut file = RequestFile::with_size(request, u64::MAX - 10);
1027+
let mut file: RequestFile = RequestFile::with_size(request, u64::MAX - 10);
9691028

9701029
let pos = file.seek(SeekFrom::End(20)).await.expect("seek error");
9711030
assert_eq!(pos, u64::MAX);
@@ -975,7 +1034,7 @@ mod tests {
9751034
async fn test_reset() {
9761035
let client = reqwest::Client::new();
9771036
let request = client.get("http://example.com/");
978-
let mut file = RequestFile::new(request);
1037+
let mut file: RequestFile = RequestFile::new(request);
9791038
file.reset();
9801039
}
9811040
}

0 commit comments

Comments
 (0)