diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 7c78bc97a..65124ee96 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -239,7 +239,7 @@ pub fn main() { let mut count = 1; let mut total = 0; while count != 0 { - count = db::file::move_to_s3(&conn, 5_000).expect("Failed to upload batch to S3"); + count = db::move_to_s3(&conn, 5_000).expect("Failed to upload batch to S3"); total += count; eprintln!( "moved {} rows to s3 in this batch, total moved so far: {}", diff --git a/src/db/file.rs b/src/db/file.rs index ad3609e46..0528822c3 100644 --- a/src/db/file.rs +++ b/src/db/file.rs @@ -42,7 +42,7 @@ fn get_file_list_from_dir>(path: P, } -pub fn get_file_list>(path: P) -> Result> { +fn get_file_list>(path: P) -> Result> { let path = path.as_ref(); let mut files = Vec::new(); diff --git a/src/db/migrate.rs b/src/db/migrate.rs index c4182b0f6..59e72dbda 100644 --- a/src/db/migrate.rs +++ b/src/db/migrate.rs @@ -10,6 +10,7 @@ use std::borrow::Cow; #[derive(Copy, Clone)] enum ApplyMode { Permanent, + #[cfg(test)] Temporary, } @@ -22,6 +23,7 @@ impl MigrationContext { fn format_query<'a>(&self, query: &'a str) -> Cow<'a, str> { match self.apply_mode { ApplyMode::Permanent => Cow::Borrowed(query), + #[cfg(test)] ApplyMode::Temporary => { Cow::Owned(query.replace("CREATE TABLE", "CREATE TEMPORARY TABLE")) } @@ -73,6 +75,7 @@ pub fn migrate(version: Option, conn: &Connection) -> CratesfyiResult<( migrate_inner(version, conn, ApplyMode::Permanent) } +#[cfg(test)] pub fn migrate_temporary(version: Option, conn: &Connection) -> CratesfyiResult<()> { migrate_inner(version, conn, ApplyMode::Temporary) } diff --git a/src/db/mod.rs b/src/db/mod.rs index e486b0b15..7ca8f36e8 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -3,8 +3,10 @@ pub(crate) use self::add_package::add_package_into_database; pub(crate) use self::add_package::add_build_into_database; pub(crate) use self::add_package::CratesIoData; -pub use self::file::add_path_into_database; -pub use self::migrate::{migrate, migrate_temporary}; +pub use self::file::{add_path_into_database, move_to_s3}; +pub use self::migrate::migrate; +#[cfg(test)] +pub(crate) use self::migrate::migrate_temporary; pub use self::delete_crate::delete_crate; use postgres::{Connection, TlsMode}; @@ -14,7 +16,7 @@ use r2d2; use r2d2_postgres; mod add_package; -pub mod file; +pub(crate) mod file; mod migrate; mod delete_crate; pub mod blacklist; @@ -29,7 +31,7 @@ pub fn connect_db() -> Result { } -pub fn create_pool() -> r2d2::Pool { +pub(crate) fn create_pool() -> r2d2::Pool { let db_url = env::var("CRATESFYI_DATABASE_URL") .expect("CRATESFYI_DATABASE_URL environment variable is not exists"); let manager = r2d2_postgres::PostgresConnectionManager::new(&db_url[..], diff --git a/src/docbuilder/metadata.rs b/src/docbuilder/metadata.rs index 5fdcd41b9..97f0ea963 100644 --- a/src/docbuilder/metadata.rs +++ b/src/docbuilder/metadata.rs @@ -22,7 +22,6 @@ use failure::err_msg; /// default-target = "x86_64-unknown-linux-gnu" /// rustc-args = [ "--example-rustc-arg" ] /// rustdoc-args = [ "--example-rustdoc-arg" ] -/// dependencies = [ "example-system-dependency" ] /// ``` /// /// You can define one or more fields in your `Cargo.toml`. @@ -49,11 +48,6 @@ pub struct Metadata { /// List of command line arguments for `rustdoc`. pub rustdoc_args: Option>, - - /// System dependencies. - /// - /// Docs.rs is running on a Debian jessie. - pub dependencies: Option>, } @@ -69,7 +63,7 @@ impl Metadata { Err(err_msg("Manifest not found")) } - pub fn from_manifest>(path: P) -> Metadata { + fn from_manifest>(path: P) -> Metadata { use std::fs::File; use std::io::Read; let mut f = match File::open(path) { @@ -93,7 +87,6 @@ impl Metadata { default_target: None, rustc_args: None, rustdoc_args: None, - dependencies: None, } } @@ -122,8 +115,6 @@ impl Metadata { .and_then(|f| f.iter().map(|v| v.as_str().map(|v| v.to_owned())).collect()); metadata.rustdoc_args = table.get("rustdoc-args").and_then(|f| f.as_array()) .and_then(|f| f.iter().map(|v| v.as_str().map(|v| v.to_owned())).collect()); - metadata.dependencies = table.get("dependencies").and_then(|f| f.as_array()) - .and_then(|f| f.iter().map(|v| v.as_str().map(|v| v.to_owned())).collect()); } metadata @@ -151,7 +142,6 @@ mod test { default-target = "x86_64-unknown-linux-gnu" rustc-args = [ "--example-rustc-arg" ] rustdoc-args = [ "--example-rustdoc-arg" ] - dependencies = [ "example-system-dependency" ] "#; let metadata = Metadata::from_str(manifest); @@ -176,9 +166,5 @@ mod test { let rustdoc_args = metadata.rustdoc_args.unwrap(); assert_eq!(rustdoc_args.len(), 1); assert_eq!(rustdoc_args[0], "--example-rustdoc-arg".to_owned()); - - let dependencies = metadata.dependencies.unwrap(); - assert_eq!(dependencies.len(), 1); - assert_eq!(dependencies[0], "example-system-dependency".to_owned()); } } diff --git a/src/docbuilder/mod.rs b/src/docbuilder/mod.rs index c6e5e8b7c..21b03d0b1 100644 --- a/src/docbuilder/mod.rs +++ b/src/docbuilder/mod.rs @@ -1,6 +1,6 @@ -pub mod options; -pub mod metadata; +pub(crate) mod options; +mod metadata; mod limits; mod rustwide_builder; mod crates; @@ -9,6 +9,7 @@ mod queue; pub use self::rustwide_builder::RustwideBuilder; pub(crate) use self::rustwide_builder::BuildResult; pub(crate) use self::limits::Limits; +pub(self) use self::metadata::Metadata; use std::fs; diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index a22bb1a92..af43361a2 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -16,7 +16,7 @@ use std::borrow::Cow; use std::collections::HashSet; use std::path::Path; use utils::{copy_doc_dir, parse_rustc_version, CargoMetadata}; -use Metadata; +use super::Metadata; const USER_AGENT: &str = "docs.rs builder (https://github.com/rust-lang/docs.rs)"; const DEFAULT_RUSTWIDE_WORKSPACE: &str = ".rustwide"; @@ -509,7 +509,7 @@ impl RustwideBuilder { target: &str, is_default_target: bool, ) -> Result<()> { - let source = target_dir.join(target); + let source = target_dir.join(target).join("doc"); let mut dest = local_storage.to_path_buf(); // only add target name to destination directory when we are copying a non-default target. @@ -522,8 +522,7 @@ impl RustwideBuilder { } info!("{} {}", source.display(), dest.display()); - copy_doc_dir(source, dest, self.rustc_version.trim())?; - Ok(()) + copy_doc_dir(source, dest) } fn upload_docs( diff --git a/src/error.rs b/src/error.rs index 3f200eff0..075ae5ea1 100644 --- a/src/error.rs +++ b/src/error.rs @@ -2,6 +2,6 @@ use std::result::Result as StdResult; -pub use failure::{Error, ResultExt}; +pub(crate) use failure::Error; pub type Result = StdResult; diff --git a/src/lib.rs b/src/lib.rs index 6723147d2..438dad802 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -47,10 +47,9 @@ extern crate once_cell; pub use self::docbuilder::RustwideBuilder; pub use self::docbuilder::DocBuilder; pub use self::docbuilder::options::DocBuilderOptions; -pub use self::docbuilder::metadata::Metadata; pub use self::web::Server; -pub mod error; +mod error; pub mod db; pub mod utils; mod docbuilder; diff --git a/src/utils/copy.rs b/src/utils/copy.rs index bc47c78f7..8c51ebe34 100644 --- a/src/utils/copy.rs +++ b/src/utils/copy.rs @@ -1,53 +1,21 @@ -// FIXME: There is so many PathBuf's in this module -// Conver them to Path - -use std::path::{Path, PathBuf}; +use std::path::Path; use std::fs; use error::Result; use regex::Regex; - -/// Copies files from source directory to destination directory. -pub fn copy_dir>(source: P, destination: P) -> Result<()> { - copy_files_and_handle_html(source.as_ref().to_path_buf(), - destination.as_ref().to_path_buf(), - false, - "") -} - - /// Copies documentation from a crate's target directory to destination. /// /// Target directory must have doc directory. /// -/// This function is designed to avoid file duplications. It is using rustc version string -/// to rename common files (css files, jquery.js, playpen.js, main.js etc.) in a standard rustdoc. -pub fn copy_doc_dir>(target: P, - destination: P, - rustc_version: &str) - -> Result<()> { - let source = PathBuf::from(target.as_ref()).join("doc"); - copy_files_and_handle_html(source, - destination.as_ref().to_path_buf(), - true, - rustc_version) -} - - -fn copy_files_and_handle_html(source: PathBuf, - destination: PathBuf, - handle_html: bool, - rustc_version: &str) - -> Result<()> { +/// This function is designed to avoid file duplications. +pub fn copy_doc_dir, Q: AsRef>(source: P, destination: Q) -> Result<()> { + let destination = destination.as_ref(); - // FIXME: handle_html is useless since we started using --resource-suffix - // argument with rustdoc - - // Make sure destination directory is exists + // Make sure destination directory exists if !destination.exists() { - fs::create_dir_all(&destination)?; + fs::create_dir_all(destination)?; } // Avoid copying common files @@ -55,21 +23,17 @@ fn copy_files_and_handle_html(source: PathBuf, r"(\.lock|\.txt|\.woff|\.svg|\.css|main-.*\.css|main-.*\.js|normalize-.*\.js|rustdoc-.*\.css|storage-.*\.js|theme-.*\.js)$") .unwrap(); - for file in source.read_dir()? { + for file in source.as_ref().read_dir()? { let file = file?; - let mut destination_full_path = PathBuf::from(&destination); - destination_full_path.push(file.file_name()); + let destination_full_path = destination.join(file.file_name()); let metadata = file.metadata()?; if metadata.is_dir() { fs::create_dir_all(&destination_full_path)?; - copy_files_and_handle_html(file.path(), - destination_full_path, - handle_html, - &rustc_version)? - } else if handle_html && dup_regex.is_match(&file.file_name().into_string().unwrap()[..]) { + copy_doc_dir(file.path(), destination_full_path)? + } else if dup_regex.is_match(&file.file_name().into_string().unwrap()[..]) { continue; } else { fs::copy(&file.path(), &destination_full_path)?; @@ -85,19 +49,26 @@ fn copy_files_and_handle_html(source: PathBuf, mod test { extern crate env_logger; use std::fs; - use std::path::Path; use super::*; #[test] - #[ignore] - fn test_copy_dir() { - let destination = tempdir::TempDir::new("cratesfyi").unwrap(); + fn test_copy_doc_dir() { + let source = tempdir::TempDir::new("cratesfyi-src").unwrap(); + let destination = tempdir::TempDir::new("cratesfyi-dst").unwrap(); + let doc = source.path().join("doc"); + fs::create_dir(&doc).unwrap(); + fs::create_dir(doc.join("inner")).unwrap(); + + fs::write(doc.join("index.html"), "spooky").unwrap(); + fs::write(doc.join("index.txt"), "spooky").unwrap(); + fs::write(doc.join("inner").join("index.html"), "spooky").unwrap(); + fs::write(doc.join("inner").join("index.txt"), "spooky").unwrap(); // lets try to copy a src directory to tempdir - let res = copy_dir(Path::new("src"), destination.path()); - // remove temp dir - fs::remove_dir_all(destination.path()).unwrap(); - - assert!(res.is_ok()); + copy_doc_dir(source.path().join("doc"), destination.path()).unwrap(); + assert!(destination.path().join("index.html").exists()); + assert!(!destination.path().join("index.txt").exists()); + assert!(destination.path().join("inner").join("index.html").exists()); + assert!(!destination.path().join("inner").join("index.txt").exists()); } } diff --git a/src/utils/github_updater.rs b/src/utils/github_updater.rs index f7a23ef91..a57206414 100644 --- a/src/utils/github_updater.rs +++ b/src/utils/github_updater.rs @@ -9,11 +9,11 @@ use failure::err_msg; /// Fields we need use in cratesfyi #[derive(Debug)] struct GitHubFields { - pub description: String, - pub stars: i64, - pub forks: i64, - pub issues: i64, - pub last_commit: time::Timespec, + description: String, + stars: i64, + forks: i64, + issues: i64, + last_commit: time::Timespec, } diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 06864b9bc..8f54a7511 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,12 +1,12 @@ //! Various utilities for cratesfyi -pub use self::copy::{copy_dir, copy_doc_dir}; +pub(crate) use self::copy::copy_doc_dir; pub use self::github_updater::github_updater; pub use self::release_activity_updater::update_release_activity; pub use self::daemon::start_daemon; -pub use self::rustc_version::{parse_rustc_version, get_current_versions, command_result}; -pub use self::html::extract_head_and_body; +pub(crate) use self::rustc_version::parse_rustc_version; +pub(crate) use self::html::extract_head_and_body; pub use self::queue::add_crate_to_queue; pub(crate) use self::cargo_metadata::{CargoMetadata, Package as MetadataPackage}; diff --git a/src/utils/rustc_version.rs b/src/utils/rustc_version.rs index b7fea0a04..6aca1e8a6 100644 --- a/src/utils/rustc_version.rs +++ b/src/utils/rustc_version.rs @@ -1,5 +1,4 @@ -use std::process::{Command, Output}; use regex::Regex; use error::Result; use failure::err_msg; @@ -18,26 +17,6 @@ pub fn parse_rustc_version>(version: S) -> Result { captures.get(2).unwrap().as_str())) } - -/// Returns current version of rustc and cratesfyi -pub fn get_current_versions() -> Result<(String, String)> { - let rustc_version = command_result(Command::new("rustc").arg("--version").output()?)?; - let cratesfyi_version = command_result(Command::new("rustc").arg("--version").output()?)?; - - Ok((rustc_version, cratesfyi_version)) -} - - -pub fn command_result(output: Output) -> Result { - let mut command_out = String::from_utf8_lossy(&output.stdout).into_owned(); - command_out.push_str(&String::from_utf8_lossy(&output.stderr).into_owned()[..]); - match output.status.success() { - true => Ok(command_out), - false => Err(err_msg(command_out)), - } -} - - #[test] fn test_parse_rustc_version() { assert_eq!(parse_rustc_version("rustc 1.10.0-nightly (57ef01513 2016-05-23)").unwrap(), diff --git a/src/web/crate_details.rs b/src/web/crate_details.rs index 44c8333ce..b70a27b15 100644 --- a/src/web/crate_details.rs +++ b/src/web/crate_details.rs @@ -43,7 +43,7 @@ pub struct CrateDetails { github_stars: Option, github_forks: Option, github_issues: Option, - pub metadata: MetaData, + pub(crate) metadata: MetaData, is_library: bool, doc_targets: Option, license: Option, diff --git a/src/web/mod.rs b/src/web/mod.rs index 9334435e0..c6b73b4ad 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -1,7 +1,7 @@ //! Web interface of cratesfyi -pub mod page; +pub(crate) mod page; /// ctry! (cratesfyitry) is extremely similar to try! and itry! /// except it returns an error page response instead of plain Err. @@ -45,7 +45,7 @@ mod builds; mod error; mod sitemap; mod routes; -pub mod metrics; +pub(crate) mod metrics; use std::{env, fmt}; use std::error::Error; @@ -106,7 +106,7 @@ impl CratesfyiHandler { chain } - pub fn new(pool_factory: PoolFactory) -> CratesfyiHandler { + fn new(pool_factory: PoolFactory) -> CratesfyiHandler { let routes = routes::build_routes(); let blacklisted_prefixes = routes.page_prefixes(); @@ -202,7 +202,7 @@ enum MatchVersion { impl MatchVersion { /// Convert this `MatchVersion` into an `Option`, discarding whether the matched version came /// from an exact version number or a semver requirement. - pub fn into_option(self) -> Option { + fn into_option(self) -> Option { match self { MatchVersion::Exact(v) | MatchVersion::Semver(v) => Some(v), MatchVersion::None => None, @@ -397,7 +397,7 @@ fn redirect(url: Url) -> Response { resp } -pub fn redirect_base(req: &Request) -> String { +fn redirect_base(req: &Request) -> String { // Try to get the scheme from CloudFront first, and then from iron let scheme = req.headers .get_raw("cloudfront-forwarded-proto") @@ -458,18 +458,18 @@ fn ico_handler(req: &mut Request) -> IronResult { /// MetaData used in header #[derive(Debug)] -pub struct MetaData { - pub name: String, - pub version: String, - pub description: Option, - pub target_name: Option, - pub rustdoc_status: bool, +pub(crate) struct MetaData { + name: String, + version: String, + description: Option, + target_name: Option, + rustdoc_status: bool, pub default_target: String, } impl MetaData { - pub fn from_crate(conn: &Connection, name: &str, version: &str) -> Option { + fn from_crate(conn: &Connection, name: &str, version: &str) -> Option { for row in &conn.query("SELECT crates.name, releases.version, releases.description, diff --git a/src/web/releases.rs b/src/web/releases.rs index bc5c454ee..20e1cc55a 100644 --- a/src/web/releases.rs +++ b/src/web/releases.rs @@ -330,7 +330,7 @@ pub fn releases_feed_handler(req: &mut Request) -> IronResult { } -pub fn releases_handler(packages: Vec, +fn releases_handler(packages: Vec, page_number: i64, release_type: &str, tab: &str, diff --git a/src/web/rustdoc.rs b/src/web/rustdoc.rs index d00a36a9f..f20ca28c2 100644 --- a/src/web/rustdoc.rs +++ b/src/web/rustdoc.rs @@ -23,14 +23,14 @@ use utils; #[derive(Debug)] struct RustdocPage { - pub head: String, - pub body: String, - pub body_class: String, - pub name: String, - pub full: String, - pub version: String, - pub description: Option, - pub crate_details: Option, + head: String, + body: String, + body_class: String, + name: String, + full: String, + version: String, + description: Option, + crate_details: Option, } diff --git a/src/web/source.rs b/src/web/source.rs index 8fa3b0255..44eaf13d2 100644 --- a/src/web/source.rs +++ b/src/web/source.rs @@ -82,7 +82,7 @@ impl FileList { /// This function is only returning FileList for requested directory. If is empty, /// it will return list of files (and dirs) for root directory. req_path must be a /// directory or empty for root directory. - pub fn from_path(conn: &Connection, + fn from_path(conn: &Connection, name: &str, version: &str, req_path: &str)