use chrono::{DateTime, Duration, Local}; use failure::{format_err, Error, Fail}; use futures::future::{select_all, BoxFuture, FutureExt}; use lazy_static::lazy_static; use log::{debug, warn}; use pulldown_cmark::{Event, Parser, Tag}; use regex::Regex; use reqwest::redirect::Policy; use reqwest::Client; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet}; use std::env; use std::fs; use std::io::Write; use std::time; use std::u8; use tokio::sync::Semaphore; use tokio::sync::SemaphorePermit; #[derive(Debug, Fail, Serialize, Deserialize)] enum CheckerError { #[fail(display = "http error: {}", status)] HttpError { status: u16, location: Option<String>, }, } struct MaxHandles { remaining: Semaphore, } struct Handle<'a> { _permit: SemaphorePermit<'a>, } impl MaxHandles { fn new(max: usize) -> MaxHandles { MaxHandles { remaining: Semaphore::new(max), } } async fn get<'a>(&'a self) -> Handle<'a> { let permit = self.remaining.acquire().await.unwrap(); return Handle { _permit: permit }; } } impl<'a> Drop for Handle<'a> { fn drop(&mut self) { debug!("Dropping"); } } lazy_static! { static ref CLIENT: Client = Client::builder() .danger_accept_invalid_certs(true) .user_agent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:68.0) Gecko/20100101 Firefox/68.0") .redirect(Policy::none()) .pool_max_idle_per_host(0) .timeout(time::Duration::from_secs(20)) .build().unwrap(); static ref HANDLES: MaxHandles = MaxHandles::new(20); } lazy_static! { static ref GITHUB_REPO_REGEX: Regex = Regex::new(r"^https: static ref GITHUB_API_REGEX: Regex = Regex::new(r"https: } #[derive(Deserialize, Debug)] struct RepoInfo { full_name: String, description: Option<String>, topics: Vec<String>, } async fn get_hacktoberfest_core(github_url: String) -> Result<Info, CheckerError> { warn!("Downloading Hacktoberfest label for {}", github_url); let rewritten = GITHUB_REPO_REGEX .replace_all(&github_url, "https: .to_string(); let mut req = CLIENT.get(&rewritten); if let Ok(username) = env::var("GITHUB_USERNAME") { if let Ok(password) = env::var("GITHUB_TOKEN") { req = req.basic_auth(username, Some(password)); } } let resp = req.send().await; match resp { Err(err) => { warn!("Error while getting {}: {}", github_url, err); return Err(CheckerError::HttpError { status: err.status().unwrap().as_u16(), location: Some(github_url.to_string()), }); } Ok(ok) => { if !ok.status().is_success() { return Err(CheckerError::HttpError { status: ok.status().as_u16(), location: None, }); } let raw = ok.text().await.unwrap(); match serde_json::from_str::<RepoInfo>(&raw) { Ok(val) => Ok(Info { name: val.full_name, description: val.description.unwrap_or_default(), hacktoberfest: val.topics.iter().find(|t| *t == "hacktoberfest").is_some(), }), Err(_) => { panic!("{}", raw); } } } } } fn get_hacktoberfest(url: String) -> BoxFuture<'static, (String, Result<Info, CheckerError>)> { debug!("Need handle for {}", url); async move { let _handle = HANDLES.get().await; return (url.clone(), get_hacktoberfest_core(url).await); } .boxed() } #[derive(Debug, Serialize, Deserialize)] struct Info { hacktoberfest: bool, name: String, description: String, } #[derive(Debug, Serialize, Deserialize)] struct Link { updated_at: DateTime<Local>, info: Info, } type Results = BTreeMap<String, Link>; #[tokio::main] async fn main() -> Result<(), Error> { env_logger::init(); let markdown_input = fs::read_to_string("README.md").expect("Can't read README.md"); let parser = Parser::new(&markdown_input); let mut used: BTreeSet<String> = BTreeSet::new(); let mut results: Results = fs::read_to_string("results/hacktoberfest.yaml") .map_err(|e| format_err!("{}", e)) .and_then(|x| serde_yaml::from_str(&x).map_err(|e| format_err!("{}", e))) .unwrap_or(Results::new()); let mut url_checks = vec![]; let mut do_check = |url: String| { if !url.starts_with("http") { return; } if used.contains(&url) { return; } used.insert(url.clone()); if let Some(_) = results.get(&url) { return; } let check = get_hacktoberfest(url).boxed(); url_checks.push(check); }; let mut to_check: Vec<String> = vec![]; #[derive(Debug)] struct ListInfo { location: usize, data: Vec<String>, } for (event, _) in parser.into_offset_iter() { match event { Event::Start(tag) => match tag { Tag::Link(_link_type, url, _title) | Tag::Image(_link_type, url, _title) => { if GITHUB_REPO_REGEX.is_match(&url) { to_check.push(url.to_string()); } } _ => {} }, _ => {} } } for url in to_check { do_check(url) } let results_keys = results.keys().cloned().collect::<BTreeSet<String>>(); let old_links = results_keys.difference(&used); for link in old_links { results.remove(link).unwrap(); } fs::write("results/results.yaml", serde_yaml::to_string(&results)?)?; let mut not_written = 0; let mut last_written = Local::now(); let mut failed: u32 = 0; while url_checks.len() > 0 { debug!("Waiting for {}", url_checks.len()); let ((url, res), _index, remaining) = select_all(url_checks).await; url_checks = remaining; match res { Ok(info) => { print!("\u{2714} "); if let Some(link) = results.get_mut(&url) { link.updated_at = Local::now(); link.info = info } else { results.insert( url.clone(), Link { updated_at: Local::now(), info: info, }, ); } } Err(_) => { print!("\u{2718} "); println!("{}", url); failed += 1; } } std::io::stdout().flush().unwrap(); not_written += 1; let duration = Local::now() - last_written; if duration > Duration::seconds(5) || not_written > 20 { fs::write( "results/hacktoberfest.yaml", serde_yaml::to_string(&results)?, )?; not_written = 0; last_written = Local::now(); } } fs::write( "results/hacktoberfest.yaml", serde_yaml::to_string(&results)?, )?; println!(""); if failed == 0 { println!("All awesome-rust repos tagged with 'hacktoberfest'"); let mut sorted_repos = results .keys() .map(|s| s.to_string()) .collect::<Vec<String>>(); sorted_repos.sort_by_key(|a| a.to_lowercase()); for name in sorted_repos { let link = results.get(&name).unwrap(); if link.info.hacktoberfest { println!( "* [{}]({}) - {}", link.info.name, name, link.info.description ) } } Ok(()) } else { Err(format_err!("{} urls with errors", failed)) } }