From 41c4778195fe3c9c6eaa04693465f69e40761ca6 Mon Sep 17 00:00:00 2001 From: "Yuki.N" Date: Tue, 25 Mar 2025 19:00:10 +0800 Subject: [PATCH 01/10] README --- README.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..d12eaa3 --- /dev/null +++ b/README.md @@ -0,0 +1,5 @@ +use: + ```console + cargo build --release + ./target/release/lonefire https://www.pixiv.net/artworks/111503285 + ``` \ No newline at end of file From f14a6107b680e613bd80bafb2dc1a77b25f8161b Mon Sep 17 00:00:00 2001 From: "Yuki.N" Date: Tue, 25 Mar 2025 19:01:11 +0800 Subject: [PATCH 02/10] Fixed typo --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d12eaa3..e4787f0 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ use: - ```console - cargo build --release - ./target/release/lonefire https://www.pixiv.net/artworks/111503285 - ``` \ No newline at end of file + +```console +cargo build --release +./target/release/lonefire https://www.pixiv.net/artworks/111503285 +``` \ No newline at end of file From 5853827d10d2f5c79fe57f96f1ff42540f3b9b1c Mon Sep 17 00:00:00 2001 From: "Yuki.N" Date: Wed, 26 Mar 2025 17:08:24 +0800 Subject: [PATCH 03/10] keep pixiv original url if not age-restricted content --- src/downloader.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/downloader.rs b/src/downloader.rs index d1ae7b9..9e8e874 100644 --- a/src/downloader.rs +++ b/src/downloader.rs @@ -59,9 +59,9 @@ pub async fn download_pixiv_artwork(artwork_url: &str) -> Result<()> { let page_count = illust_data.page_count as usize; let image_urls: Vec = match &illust_data.urls.original { - Some(original_url) if original_url.contains(&artwork_id) => { - generate_image_urls(original_url, page_count) - } + Some(original_url) if original_url.contains(&artwork_id) => (0..page_count) + .map(|seq| original_url.replace("p0", &format!("p{}", seq))) + .collect(), _ => { illust_data .user_illusts From 878a7fee693f7d07f2fa6e233b8e46dee5c05c5c Mon Sep 17 00:00:00 2001 From: "Yuki.N" Date: Thu, 27 Mar 2025 08:42:17 +0800 Subject: [PATCH 04/10] feat: probe valid url before downloading age-restricted content --- src/downloader.rs | 8 +++++--- src/helpers.rs | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/downloader.rs b/src/downloader.rs index 9e8e874..15e621a 100644 --- a/src/downloader.rs +++ b/src/downloader.rs @@ -17,7 +17,7 @@ use url::Url; use crate::{ config::{self, MAX_CONCURRENT_DOWNLOADS}, - helpers, + helpers::{self, filter_valid_urls}, }; pub async fn download_pixiv_artwork(artwork_url: &str) -> Result<()> { @@ -63,7 +63,7 @@ pub async fn download_pixiv_artwork(artwork_url: &str) -> Result<()> { .map(|seq| original_url.replace("p0", &format!("p{}", seq))) .collect(), _ => { - illust_data + let urls = illust_data .user_illusts .as_ref() .map(|user_illusts| { @@ -75,7 +75,9 @@ pub async fn download_pixiv_artwork(artwork_url: &str) -> Result<()> { .flat_map(|normalized_url| generate_image_urls(&normalized_url, page_count)) .collect::>() }) - .unwrap_or_default() + .unwrap_or_default(); + + filter_valid_urls(urls).await } }; if image_urls.is_empty() { diff --git a/src/helpers.rs b/src/helpers.rs index d36de70..fc5c6a2 100644 --- a/src/helpers.rs +++ b/src/helpers.rs @@ -1,8 +1,11 @@ +use futures::{StreamExt, stream}; use regex::Regex; use reqwest::header::{ ACCEPT, ACCEPT_LANGUAGE, CONNECTION, HeaderMap, HeaderValue, REFERER, USER_AGENT, }; +use crate::config::MAX_CONCURRENT_DOWNLOADS; + pub fn normalize_image_url(base_url: &str) -> String { let re_custom_thumb = Regex::new(r"/c/250x250_80_a2/custom-thumb").unwrap(); let re_img_master = Regex::new(r"/c/250x250_80_a2/img-master").unwrap(); @@ -29,6 +32,23 @@ pub fn generate_image_urls(base_url: &str, page_count: usize) -> Vec { .collect() } +async fn probe_image_url(url: String) -> Option { + let probe_client = reqwest::Client::new(); + let headers = set_headers(); + match probe_client.head(&url).headers(headers).send().await { + Ok(response) if response.status().is_success() => Some(url), + _ => None, + } +} + +pub async fn filter_valid_urls(urls: Vec) -> Vec { + stream::iter(urls.into_iter().map(probe_image_url)) + .buffer_unordered(MAX_CONCURRENT_DOWNLOADS) + .filter_map(|url| async { url }) + .collect::>() + .await +} + pub fn format_filename(url: &str) -> Option { let re = Regex::new(r"/(\d+)_p(\d+)(\.\w+)$").unwrap(); if let Some(caps) = re.captures(url) { From 7da57eee1ba9c9a844fc51ed612c7db55c398280 Mon Sep 17 00:00:00 2001 From: "Yuki.N" Date: Thu, 27 Mar 2025 08:50:28 +0800 Subject: [PATCH 05/10] chore: organized dependencies --- src/downloader.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/downloader.rs b/src/downloader.rs index 15e621a..6cedc0a 100644 --- a/src/downloader.rs +++ b/src/downloader.rs @@ -1,7 +1,9 @@ use anyhow::{Context, Result, anyhow}; -use config::PixivResponse; +use config::{MAX_CONCURRENT_DOWNLOADS, PixivResponse}; use futures::stream::{self, StreamExt}; -use helpers::{format_filename, generate_image_urls, normalize_image_url, set_headers}; +use helpers::{ + filter_valid_urls, format_filename, generate_image_urls, normalize_image_url, set_headers, +}; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use reqwest::{ Client, @@ -15,10 +17,7 @@ use std::{ }; use url::Url; -use crate::{ - config::{self, MAX_CONCURRENT_DOWNLOADS}, - helpers::{self, filter_valid_urls}, -}; +use crate::{config, helpers}; pub async fn download_pixiv_artwork(artwork_url: &str) -> Result<()> { let client = Client::new(); From 6367cc7d61ba085fec4e771e0ecdaf9e2018f28d Mon Sep 17 00:00:00 2001 From: "Yuki.N" Date: Thu, 27 Mar 2025 09:14:57 +0800 Subject: [PATCH 06/10] feat: polish progress bar --- src/downloader.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/downloader.rs b/src/downloader.rs index 6cedc0a..ca30eb4 100644 --- a/src/downloader.rs +++ b/src/downloader.rs @@ -85,10 +85,10 @@ pub async fn download_pixiv_artwork(artwork_url: &str) -> Result<()> { let m = MultiProgress::new(); let sty = ProgressStyle::with_template( - "[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}", + "{spinner:.green}[{eta}][{bar:40.cyan/blue}] {bytes}/{total_bytes} {msg}", ) .unwrap() - .progress_chars("##-"); + .progress_chars("#>-"); stream::iter(image_urls.into_iter().map(|url| { let client = client.clone(); @@ -114,7 +114,7 @@ async fn download_image(client: Client, url: &str, dir: &Path, pb: &ProgressBar) let filepath = dir.join(&filename); let temp_path = filepath.with_extension("part"); if filepath.exists() { - pb.finish_with_message("Already exists"); + pb.finish_with_message(format!("{filename} already exists")); return Ok(()); } let existing_size = temp_path.metadata().map(|s| s.len()).unwrap_or(0); @@ -157,7 +157,7 @@ async fn download_image(client: Client, url: &str, dir: &Path, pb: &ProgressBar) pb.inc(chunk.len() as u64); } - pb.finish_with_message("Download completed."); + pb.finish_with_message(format!("{filename} 🗹")); rename(temp_path, filepath)?; Ok(()) } From 3a82332beaa3f9ef68f701d6912bbe51ec41cec5 Mon Sep 17 00:00:00 2001 From: "Yuki.N" Date: Fri, 28 Mar 2025 08:12:56 +0800 Subject: [PATCH 07/10] feat: chose percent and dropped eta on progress bar --- src/downloader.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/downloader.rs b/src/downloader.rs index ca30eb4..f40e25c 100644 --- a/src/downloader.rs +++ b/src/downloader.rs @@ -85,7 +85,7 @@ pub async fn download_pixiv_artwork(artwork_url: &str) -> Result<()> { let m = MultiProgress::new(); let sty = ProgressStyle::with_template( - "{spinner:.green}[{eta}][{bar:40.cyan/blue}] {bytes}/{total_bytes} {msg}", + "{spinner:.green}[{percent}%][{wide_bar:.cyan/blue}] {bytes}/{total_bytes} {msg}", ) .unwrap() .progress_chars("#>-"); @@ -150,10 +150,13 @@ async fn download_image(client: Client, url: &str, dir: &Path, pb: &ProgressBar) .create(true) .open(&temp_path)?; + let mut downloaded_size = 0; let mut stream = response.bytes_stream(); while let Some(chunk) = stream.next().await { let chunk = chunk?; file.write_all(&chunk)?; + downloaded_size += chunk.len() as u64; + pb.set_length(downloaded_size); pb.inc(chunk.len() as u64); } From ce2f638c32faf6ea2b3be69057464cec9fefd4ab Mon Sep 17 00:00:00 2001 From: "Yuki.N" Date: Fri, 28 Mar 2025 08:13:59 +0800 Subject: [PATCH 08/10] chore: remove wrong logic code --- src/downloader.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/downloader.rs b/src/downloader.rs index f40e25c..dd5a7e4 100644 --- a/src/downloader.rs +++ b/src/downloader.rs @@ -150,13 +150,10 @@ async fn download_image(client: Client, url: &str, dir: &Path, pb: &ProgressBar) .create(true) .open(&temp_path)?; - let mut downloaded_size = 0; let mut stream = response.bytes_stream(); while let Some(chunk) = stream.next().await { let chunk = chunk?; file.write_all(&chunk)?; - downloaded_size += chunk.len() as u64; - pb.set_length(downloaded_size); pb.inc(chunk.len() as u64); } From 14482d2e9b2ab15857334a4eccd85153ca259199 Mon Sep 17 00:00:00 2001 From: "Yuki.N" Date: Fri, 28 Mar 2025 08:17:28 +0800 Subject: [PATCH 09/10] chore: updated hint msg --- README.md | 2 +- src/main.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e4787f0..759267c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -use: +usage: ```console cargo build --release diff --git a/src/main.rs b/src/main.rs index 3b3c902..385f245 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,7 +9,7 @@ mod helpers; async fn main() -> Result<()> { let args: Vec = std::env::args().collect(); if args.len() < 2 { - eprintln!("Usage: pixiv_download "); + eprintln!("Usage: lonefire \nBTW, lonefire can't download GIF image"); std::process::exit(1); } let artwork_url = &args[1]; From f1bc906d26ce0a0e17f0fca591fb0e19f8ebfa51 Mon Sep 17 00:00:00 2001 From: "Yuki.N" Date: Sat, 29 Mar 2025 21:07:42 +0800 Subject: [PATCH 10/10] chore: added AUR URL in README --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 759267c..15a203b 100644 --- a/README.md +++ b/README.md @@ -3,4 +3,8 @@ usage: ```console cargo build --release ./target/release/lonefire https://www.pixiv.net/artworks/111503285 -``` \ No newline at end of file +``` + +or, install from AUR +[https://aur.archlinux.org/packages/lonefire](https://aur.archlinux.org/packages/lonefire) +