From 58a9e6b4a22998ee451dbb490a5bd83663be0efd Mon Sep 17 00:00:00 2001 From: Peter Cai Date: Sat, 16 Aug 2025 17:22:51 -0400 Subject: [PATCH 01/10] Move generation to the same function --- src/update.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/update.rs b/src/update.rs index 628eb2d..82a4889 100644 --- a/src/update.rs +++ b/src/update.rs @@ -79,12 +79,7 @@ async fn update_latest_build() -> eyre::Result<()> { } info!("Magisk zip downloaded"); - let metadata = parse_magisk_zip(target_path, latest_run.run_number).await?; - info!( - "Generated update manifest: {}", - serde_json::to_string_pretty(&metadata)? - ); - *LATEST_METADATA.write().unwrap() = Some(metadata); + parse_magisk_zip_and_update(target_path, latest_run.run_number).await?; Ok(()) } @@ -99,8 +94,15 @@ pub struct MagiskZipMetadata { changelog: String, } -async fn parse_magisk_zip(path: String, run_number: u64) -> eyre::Result { - Ok(tokio::task::spawn_blocking(move || do_parse_magisk_zip(path, run_number)).await??) +async fn parse_magisk_zip_and_update(path: String, run_number: u64) -> eyre::Result<()> { + info!("Parsing {path}"); + let res = tokio::task::spawn_blocking(move || do_parse_magisk_zip(path, run_number)).await??; + info!( + "Generated update manifest: {}", + serde_json::to_string_pretty(&res)? + ); + *LATEST_METADATA.write().unwrap() = Some(res); + Ok(()) } fn do_parse_magisk_zip(path: String, run_number: u64) -> eyre::Result { From fad40be482c87c13e4d844cc6f5f91e1dbfd6574 Mon Sep 17 00:00:00 2001 From: Peter Cai Date: Sat, 16 Aug 2025 17:45:56 -0400 Subject: [PATCH 02/10] Implement loading from existing cache --- src/main.rs | 10 ++++++---- src/update.rs | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/src/main.rs b/src/main.rs index 2ac5062..dd3bc4a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,9 +1,7 @@ use std::sync::LazyLock; use axum::{Router, routing::get}; -use tracing::info; - -use crate::update::update_latest_build_loop; +use tracing::{info, warn}; mod update; @@ -14,7 +12,11 @@ pub static ROOT_DOMAIN: LazyLock = LazyLock::new(|| std::env::var("ROOT_ #[tokio::main] async fn main() -> eyre::Result<()> { tracing_subscriber::fmt::init(); - tokio::spawn(update_latest_build_loop()); + tokio::fs::create_dir_all(&*CACHE_DIR).await?; + if let Err(e) = crate::update::maybe_init().await { + warn!("Error while trying to init from existing cache: {e:?}, skipping"); + } + tokio::spawn(crate::update::update_latest_build_loop()); let app = Router::new().route("/", get(root)); diff --git a/src/update.rs b/src/update.rs index 82a4889..17f8586 100644 --- a/src/update.rs +++ b/src/update.rs @@ -31,6 +31,20 @@ struct WorkflowTasksResponse { workflow_runs: Vec, } +pub async fn maybe_init() -> eyre::Result<()> { + let Some((latest_zip, latest_run)) = cleanup_and_get_latest_zip().await? else { + info!( + "Could not find existing, cached latest magisk zip; will initialize after the first update loop run" + ); + return Ok(()); + }; + + let path = format!("{}/{}", *CACHE_DIR, latest_zip); + + info!("Found existing Magisk zip {path}, initializing metadata with it"); + parse_magisk_zip_and_update(path, latest_run).await +} + pub async fn update_latest_build_loop() { let mut interval = tokio::time::interval(Duration::from_mins(15)); @@ -45,6 +59,8 @@ pub async fn update_latest_build_loop() { cur_attempts += 1; tokio::time::sleep(Duration::from_secs(60)).await; } + + cleanup_and_get_latest_zip().await.ok(); } } @@ -136,3 +152,36 @@ fn do_parse_magisk_zip(path: String, run_number: u64) -> eyre::Result eyre::Result> { + let mut dir = tokio::fs::read_dir(&*CACHE_DIR).await?; + let mut zips = Vec::new(); + + while let Some(entry) = dir.next_entry().await? { + if entry.file_type().await?.is_file() { + let name = entry.file_name().into_string().unwrap(); + if name.ends_with(".zip") && name.starts_with("magisk-") { + let run_number: u64 = name + .replacen("magisk-", "", 1) + .replacen(".zip", "", 1) + .parse()?; + zips.push((name, run_number)); + } + } + } + + if zips.is_empty() { + return Ok(None); + } + + zips.sort_by_key(|z| z.1); + zips.reverse(); + + for zip in zips.iter().skip(1) { + let path = format!("{}/{}", *CACHE_DIR, zip.0); + info!("Deleting {path}"); + tokio::fs::remove_file(&path).await.ok(); + } + + Ok(Some(zips[0].clone())) +} From 3f6368c9d4eb5b777613caa8132280162e637bbf Mon Sep 17 00:00:00 2001 From: Peter Cai Date: Sat, 16 Aug 2025 17:49:07 -0400 Subject: [PATCH 03/10] Skip useless updates --- src/update.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/update.rs b/src/update.rs index 17f8586..14de9d8 100644 --- a/src/update.rs +++ b/src/update.rs @@ -78,6 +78,16 @@ async fn update_latest_build() -> eyre::Result<()> { resp.workflow_runs.reverse(); let latest_run = resp.workflow_runs[0].clone(); + if let Some(metadata) = &*LATEST_METADATA.read().unwrap() + && metadata.run_number == latest_run.run_number + { + info!( + "Latest run has the same run number as existing {}, skipping", + latest_run.run_number + ); + return Ok(()); + } + let latest_module_url = format!( "https://gitea.angry.im/PeterCxy/OpenEUICC/actions/runs/{}/artifacts/magisk-debug", latest_run.run_number @@ -108,6 +118,8 @@ pub struct MagiskZipMetadata { #[serde(rename = "zipUrl")] zip_url: String, changelog: String, + #[serde(skip)] + run_number: u64, } async fn parse_magisk_zip_and_update(path: String, run_number: u64) -> eyre::Result<()> { @@ -134,6 +146,7 @@ fn do_parse_magisk_zip(path: String, run_number: u64) -> eyre::Result Date: Sat, 16 Aug 2025 17:50:28 -0400 Subject: [PATCH 04/10] Change file name --- src/update.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/update.rs b/src/update.rs index 14de9d8..7f0e9e8 100644 --- a/src/update.rs +++ b/src/update.rs @@ -97,7 +97,10 @@ async fn update_latest_build() -> eyre::Result<()> { let download_req = client.get(latest_module_url).build()?; let download_buf = client.execute(download_req).await?.bytes().await?; - let target_path = format!("{}/magisk-{}.zip", *CACHE_DIR, latest_run.run_number); + let target_path = format!( + "{}/magisk-openeuicc-debug-{}.zip", + *CACHE_DIR, latest_run.run_number + ); info!("Downloading Magisk zip to {target_path}"); { let mut target_file = tokio::fs::File::create(&target_path).await?; @@ -144,7 +147,10 @@ fn do_parse_magisk_zip(path: String, run_number: u64) -> eyre::Result eyre::Result> { while let Some(entry) = dir.next_entry().await? { if entry.file_type().await?.is_file() { let name = entry.file_name().into_string().unwrap(); - if name.ends_with(".zip") && name.starts_with("magisk-") { + if name.ends_with(".zip") && name.starts_with("magisk-openeuicc-debug-") { let run_number: u64 = name - .replacen("magisk-", "", 1) + .replacen("magisk-openeuicc-debug-", "", 1) .replacen(".zip", "", 1) .parse()?; zips.push((name, run_number)); From 8090b03f134ec69fb0886975356cda325f976965 Mon Sep 17 00:00:00 2001 From: Peter Cai Date: Sat, 16 Aug 2025 17:53:06 -0400 Subject: [PATCH 05/10] Handle workflow being empty --- src/update.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/update.rs b/src/update.rs index 7f0e9e8..b6d2f27 100644 --- a/src/update.rs +++ b/src/update.rs @@ -67,7 +67,7 @@ pub async fn update_latest_build_loop() { async fn update_latest_build() -> eyre::Result<()> { let client = reqwest::Client::new(); let req = client - .get("https://gitea.angry.im/api/v1/repos/PeterCxy/OpenEUICC/actions/tasks") + .get("https://gitea.angry.im/api/v1/repos/PeterCxy/OpenEUICC/actions/tasks?limit=200") .header("Accept-Encoding", "application/json") .build()?; let mut resp: WorkflowTasksResponse = client.execute(req).await?.json().await?; @@ -77,6 +77,11 @@ async fn update_latest_build() -> eyre::Result<()> { resp.workflow_runs.sort_by_key(|run| run.run_number); resp.workflow_runs.reverse(); + if resp.workflow_runs.is_empty() { + tracing::error!("Couldn't find the latest successful workflow, aborting"); + return Ok(()); + } + let latest_run = resp.workflow_runs[0].clone(); if let Some(metadata) = &*LATEST_METADATA.read().unwrap() && metadata.run_number == latest_run.run_number From f89b35615ab6ff38f1e5cfa276a5520195e56ce2 Mon Sep 17 00:00:00 2001 From: Peter Cai Date: Sat, 16 Aug 2025 18:07:04 -0400 Subject: [PATCH 06/10] Implement the manifest JSON API and a wait-for-init mechanism --- src/main.rs | 5 ++++- src/update.rs | 44 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index dd3bc4a..95077a1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,7 +18,10 @@ async fn main() -> eyre::Result<()> { } tokio::spawn(crate::update::update_latest_build_loop()); - let app = Router::new().route("/", get(root)); + let app = Router::new().route("/", get(root)).route( + "/magisk/magisk-debug.json", + get(crate::update::serve_latest_metadata), + ); let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await?; info!("Listening on port 3000"); diff --git a/src/update.rs b/src/update.rs index b6d2f27..7b9664a 100644 --- a/src/update.rs +++ b/src/update.rs @@ -1,5 +1,10 @@ -use std::{io::Read, sync::RwLock, time::Duration}; +use std::{ + io::Read, + sync::{LazyLock, RwLock}, + time::Duration, +}; +use axum::Json; use serde_derive::{Deserialize, Serialize}; use tokio::io::AsyncWriteExt; use tracing::info; @@ -7,6 +12,33 @@ use tracing::info; use crate::{CACHE_DIR, ROOT_DOMAIN}; pub static LATEST_METADATA: RwLock> = RwLock::new(None); +pub static INITIALIZE_CHANNEL: LazyLock< + RwLock< + Option<( + tokio::sync::broadcast::Sender<()>, + tokio::sync::broadcast::Receiver<()>, + )>, + >, +> = LazyLock::new(|| RwLock::new(Some(tokio::sync::broadcast::channel(1)))); + +pub async fn serve_latest_metadata() -> Json { + if let Some(metadata) = &*LATEST_METADATA.read().unwrap() { + return Json(metadata.clone()); + } + + // Try waiting for initialization + let rx = if let Some((tx, _)) = &*INITIALIZE_CHANNEL.read().unwrap() { + Some(tx.subscribe()) + } else { + None + }; + + if let Some(mut rx) = rx { + rx.recv().await.ok(); + } + + Json(LATEST_METADATA.read().unwrap().clone().unwrap()) +} #[allow(unused)] #[derive(Clone, Deserialize)] @@ -118,7 +150,7 @@ async fn update_latest_build() -> eyre::Result<()> { Ok(()) } -#[derive(Serialize)] +#[derive(Clone, Serialize)] pub struct MagiskZipMetadata { version: String, #[serde(rename = "versionCode")] @@ -138,6 +170,14 @@ async fn parse_magisk_zip_and_update(path: String, run_number: u64) -> eyre::Res serde_json::to_string_pretty(&res)? ); *LATEST_METADATA.write().unwrap() = Some(res); + + // If the initialize channel is still there, tell everyone we have finished initialization and LATEST_METADATA is no longer None + if let Some((tx, _)) = &*INITIALIZE_CHANNEL.read().unwrap() { + tx.send(()).ok(); + }; + // Drop the sender here; any subscriber that might have raced with us will receive a closed error + INITIALIZE_CHANNEL.write().unwrap().take(); + Ok(()) } From 1c37106b30549d1b32320d88446cc42081c72b86 Mon Sep 17 00:00:00 2001 From: Peter Cai Date: Sat, 16 Aug 2025 18:09:45 -0400 Subject: [PATCH 07/10] Serve cached static files --- Cargo.lock | 33 +++++++++++++++++++++++++++++++++ Cargo.toml | 1 + src/main.rs | 16 +++++++++++----- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a328e69..14a9e25 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -513,6 +513,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-range-header" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c" + [[package]] name = "httparse" version = "1.10.1" @@ -882,6 +888,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -961,6 +977,7 @@ dependencies = [ "serde_derive", "serde_json", "tokio", + "tower-http", "tracing", "tracing-subscriber", "zip", @@ -1603,14 +1620,24 @@ checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" dependencies = [ "bitflags", "bytes", + "futures-core", "futures-util", "http", "http-body", + "http-body-util", + "http-range-header", + "httpdate", "iri-string", + "mime", + "mime_guess", + "percent-encoding", "pin-project-lite", + "tokio", + "tokio-util", "tower", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -1695,6 +1722,12 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" +[[package]] +name = "unicase" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" + [[package]] name = "unicode-ident" version = "1.0.18" diff --git a/Cargo.toml b/Cargo.toml index 2076b3f..4408792 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ serde = "1.0.219" serde_derive = "1.0.219" serde_json = "1.0.142" tokio = { version = "1.47.1", features = ["full"] } +tower-http = { version = "0.6.6", features = ["fs"] } tracing = "0.1.41" tracing-subscriber = "0.3.19" zip = "4.3.0" diff --git a/src/main.rs b/src/main.rs index 95077a1..605ba35 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ use std::sync::LazyLock; use axum::{Router, routing::get}; +use tower_http::services::ServeDir; use tracing::{info, warn}; mod update; @@ -18,10 +19,15 @@ async fn main() -> eyre::Result<()> { } tokio::spawn(crate::update::update_latest_build_loop()); - let app = Router::new().route("/", get(root)).route( - "/magisk/magisk-debug.json", - get(crate::update::serve_latest_metadata), - ); + let static_cache_files = ServeDir::new(&*CACHE_DIR); + + let app = Router::new() + .route("/", get(root)) + .route( + "/magisk/magisk-debug.json", + get(crate::update::serve_latest_metadata), + ) + .nest_service("/magisk", static_cache_files); let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await?; info!("Listening on port 3000"); @@ -30,5 +36,5 @@ async fn main() -> eyre::Result<()> { } async fn root() -> &'static str { - "Hello, world" + "This is the OpenEUICC homepage." } From a527c01e8f62e06f9e2f5f42f9155f4f8eef5301 Mon Sep 17 00:00:00 2001 From: Peter Cai Date: Sat, 16 Aug 2025 18:18:18 -0400 Subject: [PATCH 08/10] Add Dockerfile --- .dockerignore | 1 + Dockerfile | 11 +++++++++++ src/update.rs | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..eb5a316 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +target diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..cc2a12e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM rust:latest AS builder + +COPY ./ /src + +RUN cd /src && cargo build --release + +FROM debian:trixie + +COPY --from=builder /src/target/release/openeuicc-site /openeuicc-site + +CMD [ "/openeuicc-site" ] diff --git a/src/update.rs b/src/update.rs index 7b9664a..432f017 100644 --- a/src/update.rs +++ b/src/update.rs @@ -78,7 +78,7 @@ pub async fn maybe_init() -> eyre::Result<()> { } pub async fn update_latest_build_loop() { - let mut interval = tokio::time::interval(Duration::from_mins(15)); + let mut interval = tokio::time::interval(Duration::from_secs(10 * 60)); loop { interval.tick().await; From 98e9969d6a57128b6b9bc5989c4a0a62a276f989 Mon Sep 17 00:00:00 2001 From: Peter Cai Date: Sat, 16 Aug 2025 18:36:28 -0400 Subject: [PATCH 09/10] Add fly.toml --- Dockerfile | 2 ++ fly.toml | 30 ++++++++++++++++++++++++++++++ src/update.rs | 8 ++++---- 3 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 fly.toml diff --git a/Dockerfile b/Dockerfile index cc2a12e..84d8cf9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,8 @@ RUN cd /src && cargo build --release FROM debian:trixie +RUN apt-get update && apt-get install -y ca-certificates + COPY --from=builder /src/target/release/openeuicc-site /openeuicc-site CMD [ "/openeuicc-site" ] diff --git a/fly.toml b/fly.toml new file mode 100644 index 0000000..b93a5dc --- /dev/null +++ b/fly.toml @@ -0,0 +1,30 @@ +app = 'openeuicc' +primary_region = 'yyz' + +[build] + +[mounts] + source = "openeuicc_site_cache" + destination = "/cache" + +[env] + CACHE_DIR = "/cache" + ROOT_DOMAIN = "openeuicc.com" + +[http_service] + internal_port = 3000 + force_https = true + auto_stop_machines = 'stop' + auto_start_machines = true + min_machines_running = 0 + processes = ['app'] + +[[http_service.checks]] + grace_period = "10s" + interval = "30s" + method = "GET" + timeout = "5s" + path = "/" + +[[vm]] + size = 'shared-cpu-1x' diff --git a/src/update.rs b/src/update.rs index 432f017..97b4575 100644 --- a/src/update.rs +++ b/src/update.rs @@ -78,18 +78,18 @@ pub async fn maybe_init() -> eyre::Result<()> { } pub async fn update_latest_build_loop() { - let mut interval = tokio::time::interval(Duration::from_secs(10 * 60)); + let mut interval = tokio::time::interval(Duration::from_secs(5 * 60)); loop { interval.tick().await; let mut cur_attempts = 0; while let Err(e) = update_latest_build().await - && cur_attempts < 5 + && cur_attempts < 20 { - tracing::error!("Failed to fetch latest build: {e:?}, retrying in 60 seconds"); + tracing::error!("Failed to fetch latest build: {e:?}, retrying in 30 seconds"); cur_attempts += 1; - tokio::time::sleep(Duration::from_secs(60)).await; + tokio::time::sleep(Duration::from_secs(30)).await; } cleanup_and_get_latest_zip().await.ok(); From c786084dc48114cd42592396be68ce2be5e31251 Mon Sep 17 00:00:00 2001 From: Peter Cai Date: Sat, 16 Aug 2025 18:40:00 -0400 Subject: [PATCH 10/10] Configure soft/hard limits on Fly --- fly.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fly.toml b/fly.toml index b93a5dc..c022ab5 100644 --- a/fly.toml +++ b/fly.toml @@ -19,6 +19,11 @@ primary_region = 'yyz' min_machines_running = 0 processes = ['app'] + [http_service.concurrency] + type = "requests" + soft_limit = 7000 + hard_limit = 8000 + [[http_service.checks]] grace_period = "10s" interval = "30s"