|
| 1 | + |
| 2 | +use ::db::connect_db; |
| 3 | +use regex::Regex; |
| 4 | +use DocBuilderError; |
| 5 | +use time; |
| 6 | + |
| 7 | + |
| 8 | + |
| 9 | +/// Fields we need use in cratesfyi |
| 10 | +#[derive(Debug)] |
| 11 | +struct GitHubFields { |
| 12 | + pub description: String, |
| 13 | + pub stars: i64, |
| 14 | + pub forks: i64, |
| 15 | + pub issues: i64, |
| 16 | + pub last_commit: time::Timespec, |
| 17 | +} |
| 18 | + |
| 19 | + |
| 20 | +/// Updates github fields in crates table |
| 21 | +pub fn github_updater() -> Result<(), DocBuilderError> { |
| 22 | + let conn = try!(connect_db()); |
| 23 | + |
| 24 | + // TODO: This query assumes repository field in Cargo.toml is |
| 25 | + // always the same across all versions of a crate |
| 26 | + for row in &try!(conn.query("SELECT DISTINCT ON (crates.name) crate.name, crates.id, \ |
| 27 | + repository_url FROM crates, releases WHERE releases.crate_id \ |
| 28 | + = crates.id AND repository_url ~ '^https*://github.com' AND \ |
| 29 | + github_last_update < NOW() + INTERVAL '1 day'", |
| 30 | + &[])) { |
| 31 | + let crate_name: String = row.get(0); |
| 32 | + let crate_id: i32 = row.get(1); |
| 33 | + let repository_url: String = row.get(2); |
| 34 | + |
| 35 | + if let Err(err) = get_github_path(&repository_url[..]) |
| 36 | + .ok_or(DocBuilderError::GenericError("Failed to get github path" |
| 37 | + .to_string())) |
| 38 | + .and_then(|path| get_github_fields(&path[..])) |
| 39 | + .and_then(|fields| { |
| 40 | + conn.execute("UPDATE crates SET github_description = $1, \ |
| 41 | + github_stars = $2, github_forks = $3, \ |
| 42 | + github_issues = $4, github_last_commit = $5, \ |
| 43 | + github_last_update = NOW() WHERE id = $6", |
| 44 | + &[&fields.description, |
| 45 | + &(fields.stars as i32), |
| 46 | + &(fields.forks as i32), |
| 47 | + &(fields.issues as i32), |
| 48 | + &(fields.last_commit), |
| 49 | + &crate_id]) |
| 50 | + .map_err(DocBuilderError::DatabaseError) |
| 51 | + }) { |
| 52 | + error!("Failed to update github fields of: {} {}", crate_name, err); |
| 53 | + } |
| 54 | + } |
| 55 | + |
| 56 | + Ok(()) |
| 57 | +} |
| 58 | + |
| 59 | + |
| 60 | +fn get_github_fields(path: &str) -> Result<GitHubFields, DocBuilderError> { |
| 61 | + use rustc_serialize::json::Json; |
| 62 | + |
| 63 | + let body = { |
| 64 | + use std::io::Read; |
| 65 | + use hyper::client::Client; |
| 66 | + use hyper::header::{UserAgent, Authorization, Basic}; |
| 67 | + use hyper::status::StatusCode; |
| 68 | + use std::env; |
| 69 | + |
| 70 | + let client = Client::new(); |
| 71 | + let mut body = String::new(); |
| 72 | + |
| 73 | + let mut resp = try!(client.get(&format!("https://api.github.com/repos/{}", path)[..]) |
| 74 | + .header(UserAgent(format!("cratesfyi/{}", |
| 75 | + env!("CARGO_PKG_VERSION")))) |
| 76 | + .header(Authorization(Basic { |
| 77 | + username: env::var("CRATESFYI_GITHUB_USERNAME") |
| 78 | + .ok() |
| 79 | + .and_then(|u| Some(u.to_string())) |
| 80 | + .unwrap_or("".to_string()), |
| 81 | + password: env::var("CRATESFYI_GITHUB_ACCESSTOKEN").ok(), |
| 82 | + })) |
| 83 | + .send()); |
| 84 | + |
| 85 | + if resp.status != StatusCode::Ok { |
| 86 | + return Err(DocBuilderError::GenericError("Failed to get github data".to_string())); |
| 87 | + } |
| 88 | + |
| 89 | + try!(resp.read_to_string(&mut body)); |
| 90 | + body |
| 91 | + }; |
| 92 | + |
| 93 | + let json = try!(Json::from_str(&body[..])); |
| 94 | + let obj = json.as_object().unwrap(); |
| 95 | + |
| 96 | + Ok(GitHubFields { |
| 97 | + description: obj.get("description").and_then(|d| d.as_string()).unwrap_or("").to_string(), |
| 98 | + stars: obj.get("stargazers_count").and_then(|d| d.as_i64()).unwrap_or(0), |
| 99 | + forks: obj.get("forks_count").and_then(|d| d.as_i64()).unwrap_or(0), |
| 100 | + issues: obj.get("open_issues").and_then(|d| d.as_i64()).unwrap_or(0), |
| 101 | + last_commit: time::strptime(obj.get("pushed_at") |
| 102 | + .and_then(|d| d.as_string()) |
| 103 | + .unwrap_or(""), |
| 104 | + "%Y-%m-%dT%H:%M:%S") |
| 105 | + .unwrap_or(time::now()) |
| 106 | + .to_timespec(), |
| 107 | + }) |
| 108 | +} |
| 109 | + |
| 110 | + |
| 111 | + |
| 112 | +fn get_github_path(url: &str) -> Option<String> { |
| 113 | + let re = Regex::new(r"https*://github.com/([\w_-]+)/([\w_-]+)(.git)*").unwrap(); |
| 114 | + match re.captures(url) { |
| 115 | + Some(cap) => Some(format!("{}/{}", cap.at(1).unwrap(), cap.at(2).unwrap())), |
| 116 | + None => None, |
| 117 | + } |
| 118 | +} |
| 119 | + |
| 120 | + |
| 121 | + |
| 122 | +#[cfg(test)] |
| 123 | +mod test { |
| 124 | + extern crate env_logger; |
| 125 | + use super::{get_github_path, get_github_fields, github_updater}; |
| 126 | + |
| 127 | + #[test] |
| 128 | + fn test_get_github_path() { |
| 129 | + assert_eq!(get_github_path("https://github.com/onur/cratesfyi"), |
| 130 | + Some("onur/cratesfyi".to_string())); |
| 131 | + assert_eq!(get_github_path("http://github.com/onur/cratesfyi"), |
| 132 | + Some("onur/cratesfyi".to_string())); |
| 133 | + assert_eq!(get_github_path("https://github.com/onur/cratesfyi.git"), |
| 134 | + Some("onur/cratesfyi".to_string())); |
| 135 | + assert_eq!(get_github_path("https://github.com/onur23cmD_M_R_L_/crates_fy-i"), |
| 136 | + Some("onur23cmD_M_R_L_/crates_fy-i".to_string())); |
| 137 | + } |
| 138 | + |
| 139 | + |
| 140 | + #[test] |
| 141 | + fn test_get_github_fields() { |
| 142 | + let _ = env_logger::init(); |
| 143 | + let fields = get_github_fields("onur/cratesfyi"); |
| 144 | + assert!(fields.is_ok()); |
| 145 | + |
| 146 | + let fields = fields.unwrap(); |
| 147 | + assert!(fields.description != "".to_string()); |
| 148 | + assert!(fields.stars >= 0); |
| 149 | + assert!(fields.forks >= 0); |
| 150 | + assert!(fields.issues >= 0); |
| 151 | + |
| 152 | + use time; |
| 153 | + assert!(fields.last_commit <= time::now().to_timespec()); |
| 154 | + } |
| 155 | + |
| 156 | + |
| 157 | + #[test] |
| 158 | + #[ignore] |
| 159 | + fn test_github_updater() { |
| 160 | + let _ = env_logger::init(); |
| 161 | + assert!(github_updater().is_ok()); |
| 162 | + } |
| 163 | +} |
0 commit comments