diff --git a/README.md b/README.md new file mode 100644 index 0000000..15a203b --- /dev/null +++ b/README.md @@ -0,0 +1,10 @@ +usage: + +```console +cargo build --release +./target/release/lonefire https://www.pixiv.net/artworks/111503285 +``` + +or, install from AUR +[https://aur.archlinux.org/packages/lonefire](https://aur.archlinux.org/packages/lonefire) + diff --git a/src/downloader.rs b/src/downloader.rs index d1ae7b9..dd5a7e4 100644 --- a/src/downloader.rs +++ b/src/downloader.rs @@ -1,7 +1,9 @@ use anyhow::{Context, Result, anyhow}; -use config::PixivResponse; +use config::{MAX_CONCURRENT_DOWNLOADS, PixivResponse}; use futures::stream::{self, StreamExt}; -use helpers::{format_filename, generate_image_urls, normalize_image_url, set_headers}; +use helpers::{ + filter_valid_urls, format_filename, generate_image_urls, normalize_image_url, set_headers, +}; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use reqwest::{ Client, @@ -15,10 +17,7 @@ use std::{ }; use url::Url; -use crate::{ - config::{self, MAX_CONCURRENT_DOWNLOADS}, - helpers, -}; +use crate::{config, helpers}; pub async fn download_pixiv_artwork(artwork_url: &str) -> Result<()> { let client = Client::new(); @@ -59,11 +58,11 @@ pub async fn download_pixiv_artwork(artwork_url: &str) -> Result<()> { let page_count = illust_data.page_count as usize; let image_urls: Vec = match &illust_data.urls.original { - Some(original_url) if original_url.contains(&artwork_id) => { - generate_image_urls(original_url, page_count) - } + Some(original_url) if original_url.contains(&artwork_id) => (0..page_count) + .map(|seq| original_url.replace("p0", &format!("p{}", seq))) + .collect(), _ => { - illust_data + let urls = illust_data .user_illusts .as_ref() .map(|user_illusts| { @@ -75,7 +74,9 @@ pub async fn download_pixiv_artwork(artwork_url: &str) -> Result<()> { .flat_map(|normalized_url| generate_image_urls(&normalized_url, page_count)) .collect::>() }) - .unwrap_or_default() + .unwrap_or_default(); + + filter_valid_urls(urls).await } }; if image_urls.is_empty() { @@ -84,10 +85,10 @@ pub async fn download_pixiv_artwork(artwork_url: &str) -> Result<()> { let m = MultiProgress::new(); let sty = ProgressStyle::with_template( - "[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}", + "{spinner:.green}[{percent}%][{wide_bar:.cyan/blue}] {bytes}/{total_bytes} {msg}", ) .unwrap() - .progress_chars("##-"); + .progress_chars("#>-"); stream::iter(image_urls.into_iter().map(|url| { let client = client.clone(); @@ -113,7 +114,7 @@ async fn download_image(client: Client, url: &str, dir: &Path, pb: &ProgressBar) let filepath = dir.join(&filename); let temp_path = filepath.with_extension("part"); if filepath.exists() { - pb.finish_with_message("Already exists"); + pb.finish_with_message(format!("{filename} already exists")); return Ok(()); } let existing_size = temp_path.metadata().map(|s| s.len()).unwrap_or(0); @@ -156,7 +157,7 @@ async fn download_image(client: Client, url: &str, dir: &Path, pb: &ProgressBar) pb.inc(chunk.len() as u64); } - pb.finish_with_message("Download completed."); + pb.finish_with_message(format!("{filename} 🗹")); rename(temp_path, filepath)?; Ok(()) } diff --git a/src/helpers.rs b/src/helpers.rs index d36de70..fc5c6a2 100644 --- a/src/helpers.rs +++ b/src/helpers.rs @@ -1,8 +1,11 @@ +use futures::{StreamExt, stream}; use regex::Regex; use reqwest::header::{ ACCEPT, ACCEPT_LANGUAGE, CONNECTION, HeaderMap, HeaderValue, REFERER, USER_AGENT, }; +use crate::config::MAX_CONCURRENT_DOWNLOADS; + pub fn normalize_image_url(base_url: &str) -> String { let re_custom_thumb = Regex::new(r"/c/250x250_80_a2/custom-thumb").unwrap(); let re_img_master = Regex::new(r"/c/250x250_80_a2/img-master").unwrap(); @@ -29,6 +32,23 @@ pub fn generate_image_urls(base_url: &str, page_count: usize) -> Vec { .collect() } +async fn probe_image_url(url: String) -> Option { + let probe_client = reqwest::Client::new(); + let headers = set_headers(); + match probe_client.head(&url).headers(headers).send().await { + Ok(response) if response.status().is_success() => Some(url), + _ => None, + } +} + +pub async fn filter_valid_urls(urls: Vec) -> Vec { + stream::iter(urls.into_iter().map(probe_image_url)) + .buffer_unordered(MAX_CONCURRENT_DOWNLOADS) + .filter_map(|url| async { url }) + .collect::>() + .await +} + pub fn format_filename(url: &str) -> Option { let re = Regex::new(r"/(\d+)_p(\d+)(\.\w+)$").unwrap(); if let Some(caps) = re.captures(url) { diff --git a/src/main.rs b/src/main.rs index 3b3c902..385f245 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,7 +9,7 @@ mod helpers; async fn main() -> Result<()> { let args: Vec = std::env::args().collect(); if args.len() < 2 { - eprintln!("Usage: pixiv_download "); + eprintln!("Usage: lonefire \nBTW, lonefire can't download GIF image"); std::process::exit(1); } let artwork_url = &args[1];