Merge #1134

1134: Record readme filename; use to determine whether to render as Markdown or not r=carols10cents

Fixes #995. Depends on rust-lang/cargo#4633 to have the desired effect, but won't break in the absence of that PR; we'll just assume the readme is called `"README.md"` if the client doesn't report one.

Small CSS fix included to wrap extra long lines.

Comparison of render of https://crates.io/crates/relm's README:

| Before | After |
| :-: | :-: |
| ![screen shot 2017-10-17 at 5 51 50 pm](https://user-images.githubusercontent.com/1915/31650690-42018c7a-b364-11e7-95f8-7e1b8e8d56a0.png) | ![screen shot 2017-10-17 at 5 51 51 pm](https://user-images.githubusercontent.com/1915/31650698-44d16588-b364-11e7-92cf-1c3201c0e60e.png) |

Author: bors-voyager[bot]

Cargo.lock

@@ -42,6 +42,7 @@ oauth2 = "0.3"
 log = "0.3"
 env_logger = "0.4"
 hex = "0.2"
+htmlescape = "0.3.1"
 license-exprs = "^1.3"
 dotenv = "0.10.0"
 toml = "0.4"

Cargo.toml

@@ -305,6 +305,7 @@
     }
     .crate-readme {
         line-height: 1.5;
+        overflow-wrap: break-word;
 
         img {
             max-width: 100%;

app/styles/crate.scss

@@ -0,0 +1 @@
+ALTER TABLE crates DROP COLUMN readme_file;

migrations/2017-10-17-052814_add_readme_file_to_crates/down.sql

@@ -0,0 +1 @@
+ALTER TABLE crates ADD COLUMN readme_file VARCHAR;

migrations/2017-10-17-052814_add_readme_file_to_crates/up.sql

@@ -1,5 +1,5 @@
 // Iterates over every crate versions ever uploaded and (re-)renders their
-// readme using the Markdown renderer from the cargo_registry crate.
+// readme using the readme renderer from the cargo_registry crate.
 //
 // Warning: this can take a lot of time.
 
@@ -34,7 +34,7 @@ use url::Url;
 
 use cargo_registry::{Config, Version};
 use cargo_registry::schema::*;
-use cargo_registry::render::markdown_to_html;
+use cargo_registry::render::readme_to_html;
 
 const DEFAULT_PAGE_SIZE: usize = 25;
 const USAGE: &'static str = "
@@ -255,7 +255,7 @@ fn get_readme(config: &Config, version: &Version, krate_name: &str) -> Option<St
             manifest.package.readme.unwrap()
         );
         let contents = find_file_by_path(&mut entries, Path::new(&path), &version, &krate_name);
-        markdown_to_html(&contents).expect(&format!(
+        readme_to_html(&contents, &path).expect(&format!(
             "[{}-{}] Couldn't render README",
             krate_name,
             version.num

src/bin/render-readmes.rs

@@ -49,6 +49,7 @@ pub struct Crate {
     pub homepage: Option<String>,
     pub documentation: Option<String>,
     pub readme: Option<String>,
+    pub readme_file: Option<String>,
     pub license: Option<String>,
     pub repository: Option<String>,
     pub max_upload_size: Option<i32>,
@@ -66,6 +67,7 @@ type AllColumns = (
     crates::homepage,
     crates::documentation,
     crates::readme,
+    crates::readme_file,
     crates::license,
     crates::repository,
     crates::max_upload_size,
@@ -81,6 +83,7 @@ pub const ALL_COLUMNS: AllColumns = (
     crates::homepage,
     crates::documentation,
     crates::readme,
+    crates::readme_file,
     crates::license,
     crates::repository,
     crates::max_upload_size,
@@ -132,6 +135,7 @@ pub struct NewCrate<'a> {
     pub homepage: Option<&'a str>,
     pub documentation: Option<&'a str>,
     pub readme: Option<&'a str>,
+    pub readme_file: Option<&'a str>,
     pub repository: Option<&'a str>,
     pub max_upload_size: Option<i32>,
     pub license: Option<&'a str>,

src/krate/mod.rs

@@ -66,6 +66,7 @@ pub fn publish(req: &mut Request) -> CargoResult<Response> {
             homepage: new_crate.homepage.as_ref().map(|s| &**s),
             documentation: new_crate.documentation.as_ref().map(|s| &**s),
             readme: new_crate.readme.as_ref().map(|s| &**s),
+            readme_file: new_crate.readme_file.as_ref().map(|s| &**s),
             repository: new_crate.repository.as_ref().map(|s| &**s),
             license: new_crate.license.as_ref().map(|s| &**s),
             max_upload_size: None,
@@ -124,7 +125,10 @@ pub fn publish(req: &mut Request) -> CargoResult<Response> {
 
         // Render the README for this crate
         let readme = match new_crate.readme.as_ref() {
-            Some(readme) => Some(render::markdown_to_html(&**readme)?),
+            Some(readme) => Some(render::readme_to_html(
+                &**readme,
+                new_crate.readme_file.as_ref().map_or("README.md", |s| &**s),
+            )?),
             None => None,
         };
 

src/krate/publish.rs

@@ -22,6 +22,7 @@ extern crate dotenv;
 extern crate flate2;
 extern crate git2;
 extern crate hex;
+extern crate htmlescape;
 extern crate lettre;
 extern crate license_exprs;
 #[macro_use]

src/lib.rs

@@ -1,17 +1,18 @@
 use ammonia::Builder;
 use comrak;
+use htmlescape::encode_minimal;
 
 use util::CargoResult;
 
 /// Context for markdown to HTML rendering.
 #[allow(missing_debug_implementations)]
-pub struct MarkdownRenderer<'a> {
+struct MarkdownRenderer<'a> {
     html_sanitizer: Builder<'a>,
 }
 
 impl<'a> MarkdownRenderer<'a> {
     /// Creates a new renderer instance.
-    pub fn new() -> MarkdownRenderer<'a> {
+    fn new() -> MarkdownRenderer<'a> {
         let tags = [
             "a",
             "b",
@@ -106,7 +107,7 @@ impl<'a> MarkdownRenderer<'a> {
     }
 
     /// Renders the given markdown to HTML using the current settings.
-    pub fn to_html(&self, text: &str) -> CargoResult<String> {
+    fn to_html(&self, text: &str) -> CargoResult<String> {
         let options = comrak::ComrakOptions {
             ext_autolink: true,
             ext_strikethrough: true,
@@ -127,22 +128,46 @@ impl<'a> Default for MarkdownRenderer<'a> {
     }
 }
 
-/// Renders a markdown text to sanitized HTML.
+/// Renders Markdown text to sanitized HTML.
+fn markdown_to_html(text: &str) -> CargoResult<String> {
+    let renderer = MarkdownRenderer::new();
+    renderer.to_html(text)
+}
+
+/// Any readme with a filename ending in one of these extensions will be rendered as Markdown.
+/// Note we also render a readme as Markdown if _no_ extension is on the filename.
+static MARKDOWN_EXTENSIONS: [&'static str; 7] = [
+    ".md",
+    ".markdown",
+    ".mdown",
+    ".mdwn",
+    ".mkd",
+    ".mkdn",
+    ".mkdown",
+];
+
+/// Renders a readme to sanitized HTML.  An appropriate rendering method is chosen depending
+/// on the extension of the supplied filename.
 ///
 /// The returned text should not contain any harmful HTML tag or attribute (such as iframe,
 /// onclick, onmouseover, etc.).
 ///
 /// # Examples
 ///
 /// ```
-/// use render::markdown_to_html;
+/// use render::render_to_html;
 ///
 /// let text = "[Rust](https://rust-lang.org/) is an awesome *systems programming* language!";
-/// let rendered = markdown_to_html(text)?;
+/// let rendered = readme_to_html(text, "README.md")?;
 /// ```
-pub fn markdown_to_html(text: &str) -> CargoResult<String> {
-    let renderer = MarkdownRenderer::new();
-    renderer.to_html(text)
+pub fn readme_to_html(text: &str, filename: &str) -> CargoResult<String> {
+    let filename = filename.to_lowercase();
+
+    if !filename.contains('.') || MARKDOWN_EXTENSIONS.iter().any(|e| filename.ends_with(e)) {
+        return markdown_to_html(text);
+    }
+
+    Ok(encode_minimal(text).replace("\n", "<br>\n"))
 }
 
 #[cfg(test)]
@@ -218,6 +243,26 @@ mod tests {
         assert_eq!(result, "<p>Hello World!</p>\n");
     }
 
+    #[test]
+    fn readme_to_html_renders_markdown() {
+        for f in &["README", "readme.md", "README.MARKDOWN", "whatever.mkd"] {
+            assert_eq!(
+                readme_to_html("*lobster*", f).unwrap(),
+                "<p><em>lobster</em></p>\n"
+            );
+        }
+    }
+
+    #[test]
+    fn readme_to_html_renders_other_things() {
+        for f in &["readme.exe", "readem.org", "blah.adoc"] {
+            assert_eq!(
+                readme_to_html("<script>lobster</script>\n\nis my friend\n", f).unwrap(),
+                "&lt;script&gt;lobster&lt;/script&gt;<br>\n<br>\nis my friend<br>\n"
+            );
+        }
+    }
+
     #[test]
     fn header_has_tags() {
         let text = "# My crate\n\nHello, world!\n";

src/render.rs

@@ -282,6 +282,12 @@ table! {
         ///
         /// (Automatically generated by Diesel.)
         readme -> Nullable<Varchar>,
+        /// The `readme_file` column of the `crates` table.
+        ///
+        /// Its SQL type is `Nullable<Varchar>`.
+        ///
+        /// (Automatically generated by Diesel.)
+        readme_file -> Nullable<Varchar>,
         /// The `textsearchable_index_col` column of the `crates` table.
         ///
         /// Its SQL type is `Tsvector`.

src/schema.rs

@@ -480,6 +480,7 @@ fn krate(name: &str) -> Crate {
         homepage: None,
         description: None,
         readme: None,
+        readme_file: None,
         license: None,
         repository: None,
         max_upload_size: None,
@@ -647,6 +648,7 @@ fn new_req_body(
             homepage: krate.homepage,
             documentation: krate.documentation,
             readme: krate.readme,
+            readme_file: krate.readme_file,
             keywords: Some(u::KeywordList(kws)),
             categories: Some(u::CategoryList(cats)),
             license: Some("MIT".to_string()),

src/tests/all.rs

@@ -75,6 +75,7 @@ fn new_crate(name: &str) -> u::NewCrate {
         homepage: None,
         documentation: None,
         readme: None,
+        readme_file: None,
         keywords: None,
         categories: None,
         license: Some("MIT".to_string()),

src/tests/krate.rs

@@ -23,6 +23,7 @@ pub struct NewCrate {
     pub homepage: Option<String>,
     pub documentation: Option<String>,
     pub readme: Option<String>,
+    pub readme_file: Option<String>,
     pub keywords: Option<KeywordList>,
     pub categories: Option<CategoryList>,
     pub license: Option<String>,