use chrono::{DateTime, Duration, Local};
use failure::{format_err, Error, Fail};
use futures::future::{select_all, BoxFuture, FutureExt};
use lazy_static::lazy_static;
use log::{debug, warn};
use pulldown_cmark::{Event, Parser, Tag};
use regex::Regex;
use reqwest::redirect::Policy;
use reqwest::Client;
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, BTreeSet};
use std::env;
use std::fs;
use std::io::Write;
use std::time;
use std::u8;
use tokio::sync::Semaphore;
use tokio::sync::SemaphorePermit;
#[derive(Debug, Fail, Serialize, Deserialize)]
enum CheckerError {
#[fail(display = "http error: {}", status)]
HttpError {
status: u16,
location: Option<String>,
},
}
struct MaxHandles {
remaining: Semaphore,
}
struct Handle<'a> {
_permit: SemaphorePermit<'a>,
}
impl MaxHandles {
fn new(max: usize) -> MaxHandles {
MaxHandles {
remaining: Semaphore::new(max),
}
}
async fn get<'a>(&'a self) -> Handle<'a> {
let permit = self.remaining.acquire().await.unwrap();
return Handle { _permit: permit };
}
}
impl<'a> Drop for Handle<'a> {
fn drop(&mut self) {
debug!("Dropping");
}
}
lazy_static! {
static ref CLIENT: Client = Client::builder()
.danger_accept_invalid_certs(true)
.user_agent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:68.0) Gecko/20100101 Firefox/68.0")
.redirect(Policy::none())
.pool_max_idle_per_host(0)
.timeout(time::Duration::from_secs(20))
.build().unwrap();
static ref HANDLES: MaxHandles = MaxHandles::new(20);
}
lazy_static! {
static ref GITHUB_REPO_REGEX: Regex =
Regex::new(r"^https:
static ref GITHUB_API_REGEX: Regex = Regex::new(r"https:
}
#[derive(Deserialize, Debug)]
struct RepoInfo {
full_name: String,
description: Option<String>,
topics: Vec<String>,
}
async fn get_hacktoberfest_core(github_url: String) -> Result<Info, CheckerError> {
warn!("Downloading Hacktoberfest label for {}", github_url);
let rewritten = GITHUB_REPO_REGEX
.replace_all(&github_url, "https:
.to_string();
let mut req = CLIENT.get(&rewritten);
if let Ok(username) = env::var("GITHUB_USERNAME") {
if let Ok(password) = env::var("GITHUB_TOKEN") {
req = req.basic_auth(username, Some(password));
}
}
let resp = req.send().await;
match resp {
Err(err) => {
warn!("Error while getting {}: {}", github_url, err);
return Err(CheckerError::HttpError {
status: err.status().unwrap().as_u16(),
location: Some(github_url.to_string()),
});
}
Ok(ok) => {
if !ok.status().is_success() {
return Err(CheckerError::HttpError {
status: ok.status().as_u16(),
location: None,
});
}
let raw = ok.text().await.unwrap();
match serde_json::from_str::<RepoInfo>(&raw) {
Ok(val) => Ok(Info {
name: val.full_name,
description: val.description.unwrap_or_default(),
hacktoberfest: val.topics.iter().find(|t| *t == "hacktoberfest").is_some(),
}),
Err(_) => {
panic!("{}", raw);
}
}
}
}
}
fn get_hacktoberfest(url: String) -> BoxFuture<'static, (String, Result<Info, CheckerError>)> {
debug!("Need handle for {}", url);
async move {
let _handle = HANDLES.get().await;
return (url.clone(), get_hacktoberfest_core(url).await);
}
.boxed()
}
#[derive(Debug, Serialize, Deserialize)]
struct Info {
hacktoberfest: bool,
name: String,
description: String,
}
#[derive(Debug, Serialize, Deserialize)]
struct Link {
updated_at: DateTime<Local>,
info: Info,
}
type Results = BTreeMap<String, Link>;
#[tokio::main]
async fn main() -> Result<(), Error> {
env_logger::init();
let markdown_input = fs::read_to_string("README.md").expect("Can't read README.md");
let parser = Parser::new(&markdown_input);
let mut used: BTreeSet<String> = BTreeSet::new();
let mut results: Results = fs::read_to_string("results/hacktoberfest.yaml")
.map_err(|e| format_err!("{}", e))
.and_then(|x| serde_yaml::from_str(&x).map_err(|e| format_err!("{}", e)))
.unwrap_or(Results::new());
let mut url_checks = vec![];
let mut do_check = |url: String| {
if !url.starts_with("http") {
return;
}
if used.contains(&url) {
return;
}
used.insert(url.clone());
if let Some(_) = results.get(&url) {
return;
}
let check = get_hacktoberfest(url).boxed();
url_checks.push(check);
};
let mut to_check: Vec<String> = vec![];
#[derive(Debug)]
struct ListInfo {
location: usize,
data: Vec<String>,
}
for (event, _) in parser.into_offset_iter() {
match event {
Event::Start(tag) => match tag {
Tag::Link(_link_type, url, _title) | Tag::Image(_link_type, url, _title) => {
if GITHUB_REPO_REGEX.is_match(&url) {
to_check.push(url.to_string());
}
}
_ => {}
},
_ => {}
}
}
for url in to_check {
do_check(url)
}
let results_keys = results.keys().cloned().collect::<BTreeSet<String>>();
let old_links = results_keys.difference(&used);
for link in old_links {
results.remove(link).unwrap();
}
fs::write("results/results.yaml", serde_yaml::to_string(&results)?)?;
let mut not_written = 0;
let mut last_written = Local::now();
let mut failed: u32 = 0;
while url_checks.len() > 0 {
debug!("Waiting for {}", url_checks.len());
let ((url, res), _index, remaining) = select_all(url_checks).await;
url_checks = remaining;
match res {
Ok(info) => {
print!("\u{2714} ");
if let Some(link) = results.get_mut(&url) {
link.updated_at = Local::now();
link.info = info
} else {
results.insert(
url.clone(),
Link {
updated_at: Local::now(),
info: info,
},
);
}
}
Err(_) => {
print!("\u{2718} ");
println!("{}", url);
failed += 1;
}
}
std::io::stdout().flush().unwrap();
not_written += 1;
let duration = Local::now() - last_written;
if duration > Duration::seconds(5) || not_written > 20 {
fs::write(
"results/hacktoberfest.yaml",
serde_yaml::to_string(&results)?,
)?;
not_written = 0;
last_written = Local::now();
}
}
fs::write(
"results/hacktoberfest.yaml",
serde_yaml::to_string(&results)?,
)?;
println!("");
if failed == 0 {
println!("All awesome-rust repos tagged with 'hacktoberfest'");
let mut sorted_repos = results
.keys()
.map(|s| s.to_string())
.collect::<Vec<String>>();
sorted_repos.sort_by_key(|a| a.to_lowercase());
for name in sorted_repos {
let link = results.get(&name).unwrap();
if link.info.hacktoberfest {
println!(
"* [{}]({}) - {}",
link.info.name, name, link.info.description
)
}
}
Ok(())
} else {
Err(format_err!("{} urls with errors", failed))
}
}