From ef10b7d95b06f6e617c548e90b5ff60015f87f13 Mon Sep 17 00:00:00 2001 From: Chase Wilson Date: Sun, 3 May 2020 23:36:08 -0500 Subject: [PATCH 1/6] Added some fault-tolerance to the build queue --- src/docbuilder/queue.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/docbuilder/queue.rs b/src/docbuilder/queue.rs index f0f1df98f..265c6ff3a 100644 --- a/src/docbuilder/queue.rs +++ b/src/docbuilder/queue.rs @@ -13,10 +13,10 @@ impl DocBuilder { pub fn get_new_crates(&mut self) -> Result { let conn = connect_db()?; let index = Index::from_path_or_cloned(&self.options.crates_io_index_path)?; - let mut changes = index.fetch_changes()?; + let (mut changes, oid) = index.peek_changes()?; let mut add_count: usize = 0; - // I belive this will fix ordering of queue if we get more than one crate from changes + // I believe this will fix ordering of queue if we get more than one crate from changes changes.reverse(); for krate in changes.iter().filter(|k| k.kind != ChangeKind::Yanked) { @@ -27,6 +27,8 @@ impl DocBuilder { add_count += 1; } + index.set_last_seen_reference(oid)?; + Ok(add_count) } From 731e2a63ea29ff76851e698679354a9e235470dc Mon Sep 17 00:00:00 2001 From: Chase Wilson Date: Mon, 4 May 2020 09:49:57 -0500 Subject: [PATCH 2/6] Added error reporting for crates that fail to be added --- src/docbuilder/queue.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/docbuilder/queue.rs b/src/docbuilder/queue.rs index 265c6ff3a..40a877c59 100644 --- a/src/docbuilder/queue.rs +++ b/src/docbuilder/queue.rs @@ -21,9 +21,15 @@ impl DocBuilder { for krate in changes.iter().filter(|k| k.kind != ChangeKind::Yanked) { let priority = get_crate_priority(&conn, &krate.name)?; - add_crate_to_queue(&conn, &krate.name, &krate.version, priority).ok(); - debug!("{}-{} added into build queue", krate.name, krate.version); + match add_crate_to_queue(&conn, &krate.name, &krate.version, priority) { + Ok(()) => debug!("{}-{} added into build queue", krate.name, krate.version), + Err(err) => error!( + "failed adding {}-{} into build queue: {}", + krate.name, krate.version, err + ), + } + add_count += 1; } From 822026308ff8250fa3f20ee374fc2d839d9c56de Mon Sep 17 00:00:00 2001 From: Chase Wilson Date: Wed, 6 May 2020 10:26:37 -0500 Subject: [PATCH 3/6] Made get_new_crates return the actual queue size --- docker-entrypoint.sh | 0 src/docbuilder/queue.rs | 9 +++++---- 2 files changed, 5 insertions(+), 4 deletions(-) mode change 100755 => 100644 docker-entrypoint.sh diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh old mode 100755 new mode 100644 diff --git a/src/docbuilder/queue.rs b/src/docbuilder/queue.rs index 40a877c59..274a9caf1 100644 --- a/src/docbuilder/queue.rs +++ b/src/docbuilder/queue.rs @@ -14,7 +14,6 @@ impl DocBuilder { let conn = connect_db()?; let index = Index::from_path_or_cloned(&self.options.crates_io_index_path)?; let (mut changes, oid) = index.peek_changes()?; - let mut add_count: usize = 0; // I believe this will fix ordering of queue if we get more than one crate from changes changes.reverse(); @@ -29,13 +28,15 @@ impl DocBuilder { krate.name, krate.version, err ), } - - add_count += 1; } index.set_last_seen_reference(oid)?; - Ok(add_count) + // Get the size of the queue + let query = conn.query("SELECT COUNT(*) as size FROM queue", &[])?; + let queue_size = query.iter().next().unwrap().get::<_, i64>("size") as usize; + + Ok(queue_size) } pub fn get_queue_count(&self) -> Result { From 7455892be4a8ef59e5cd5235f2e4a7c96673a8fc Mon Sep 17 00:00:00 2001 From: Chase Wilson Date: Wed, 6 May 2020 11:19:38 -0500 Subject: [PATCH 4/6] Rustfmt --- src/docbuilder/queue.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/docbuilder/queue.rs b/src/docbuilder/queue.rs index c13bbbab1..791396580 100644 --- a/src/docbuilder/queue.rs +++ b/src/docbuilder/queue.rs @@ -43,7 +43,7 @@ impl DocBuilder { ChangeKind::Added => { let priority = get_crate_priority(&conn, &krate.name)?; - + match add_crate_to_queue(&conn, &krate.name, &krate.version, priority) { Ok(()) => debug!("{}-{} added into build queue", krate.name, krate.version), Err(err) => error!( From db76db2430a430c1fe4b70bcc3a159bd8d967129 Mon Sep 17 00:00:00 2001 From: Chase Wilson Date: Wed, 6 May 2020 11:32:09 -0500 Subject: [PATCH 5/6] Changed comment to reflect code & added back the number of crates added --- src/docbuilder/queue.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/docbuilder/queue.rs b/src/docbuilder/queue.rs index 791396580..ef5cd243a 100644 --- a/src/docbuilder/queue.rs +++ b/src/docbuilder/queue.rs @@ -9,11 +9,12 @@ use log::{debug, error}; impl DocBuilder { /// Updates crates.io-index repository and adds new crates into build queue. - /// Returns size of queue + /// Returns the number of crates added pub fn get_new_crates(&mut self) -> Result { let conn = connect_db()?; let index = Index::from_path_or_cloned(&self.options.crates_io_index_path)?; let (mut changes, oid) = index.peek_changes()?; + let mut crates_added = 0; // I believe this will fix ordering of queue if we get more than one crate from changes changes.reverse(); @@ -45,7 +46,10 @@ impl DocBuilder { let priority = get_crate_priority(&conn, &krate.name)?; match add_crate_to_queue(&conn, &krate.name, &krate.version, priority) { - Ok(()) => debug!("{}-{} added into build queue", krate.name, krate.version), + Ok(()) => { + debug!("{}-{} added into build queue", krate.name, krate.version); + crates_added += 1; + } Err(err) => error!( "failed adding {}-{} into build queue: {}", krate.name, krate.version, err @@ -57,11 +61,7 @@ impl DocBuilder { index.set_last_seen_reference(oid)?; - // Get the size of the queue - let query = conn.query("SELECT COUNT(*) as size FROM queue", &[])?; - let queue_size = query.iter().next().unwrap().get::<_, i64>("size") as usize; - - Ok(queue_size) + Ok(crates_added) } pub fn get_queue_count(&self) -> Result { From bb3a85dfcdcef8369dfda63ffbcfbf4e072af3f4 Mon Sep 17 00:00:00 2001 From: Chase Wilson Date: Wed, 6 May 2020 19:44:52 -0500 Subject: [PATCH 6/6] Reverted accidental file perm change --- docker-entrypoint.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 docker-entrypoint.sh diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh old mode 100644 new mode 100755