WIP: introduce locking to manage access & download of archive index, prune unused indexes

Cargo.toml

@@ -64,7 +64,7 @@ sysinfo = { version = "0.37.2", default-features = false, features = ["system"]
 derive_builder = "0.20.2"
 
 # Async
-tokio = { version = "1.0", features = ["rt-multi-thread", "signal", "macros"] }
+tokio = { version = "1.0", features = ["rt-multi-thread", "signal", "macros", "sync"] }
 tokio-util = { version = "0.7.15", default-features = false, features = ["io"] }
 futures-util = "0.3.5"
 async-stream = "0.3.5"

src/bin/cratesfyi.rs

@@ -5,8 +5,9 @@ use docs_rs::{
     db::{self, CrateId, Overrides, add_path_into_database},
     start_background_metrics_webserver, start_web_server,
     utils::{
-        ConfigName, get_config, get_crate_pattern_and_priority, list_crate_priorities,
-        queue_builder, remove_crate_priority, set_config, set_crate_priority,
+        ConfigName, daemon::start_background_cache_cleaner, get_config,
+        get_crate_pattern_and_priority, list_crate_priorities, queue_builder,
+        remove_crate_priority, set_config, set_crate_priority,
     },
 };
 use futures_util::StreamExt;
@@ -214,6 +215,8 @@ impl CommandLine {
                 queue_builder(&ctx, RustwideBuilder::init(&ctx)?)?;
             }
             Self::StartWebServer { socket_addr } => {
+                start_background_cache_cleaner(&ctx)?;
+
                 // Blocks indefinitely
                 start_web_server(Some(socket_addr), &ctx)?;
             }
@@ -568,11 +571,17 @@ enum DatabaseSubcommand {
         #[arg(long)]
         dry_run: bool,
     },
+
+    PruneArchiveIndexCache,
 }
 
 impl DatabaseSubcommand {
     fn handle_args(self, ctx: Context) -> Result<()> {
         match self {
+            Self::PruneArchiveIndexCache => ctx
+                .runtime
+                .block_on(async { ctx.async_storage.prune_archive_index_cache().await })
+                .context("Failed to prune archive index cache")?,
             Self::Migrate { version } => ctx
                 .runtime
                 .block_on(async {

src/config.rs

@@ -1,6 +1,6 @@
 use crate::{cdn::CdnKind, storage::StorageKind};
 use anyhow::{Context, Result, anyhow, bail};
-use std::{env::VarError, error::Error, path::PathBuf, str::FromStr, time::Duration};
+use std::{env::VarError, error::Error, io, path, path::PathBuf, str::FromStr, time::Duration};
 use tracing::trace;
 use url::Url;
 
@@ -85,6 +85,9 @@ pub struct Config {
     // for the remote archives?
     pub(crate) local_archive_cache_path: PathBuf,
 
+    // How long do we want to keep the locally cached index files?
+    pub(crate) local_archive_cache_ttl: Duration,
+
     // Where to collect metrics for the metrics initiative.
     // When empty, we won't collect metrics.
     pub(crate) compiler_metrics_collection_path: Option<PathBuf>,
@@ -209,10 +212,14 @@ impl Config {
             .cdn_max_queued_age(Duration::from_secs(env("DOCSRS_CDN_MAX_QUEUED_AGE", 3600)?))
             .cloudfront_distribution_id_web(maybe_env("CLOUDFRONT_DISTRIBUTION_ID_WEB")?)
             .cloudfront_distribution_id_static(maybe_env("CLOUDFRONT_DISTRIBUTION_ID_STATIC")?)
-            .local_archive_cache_path(env(
+            .local_archive_cache_path(ensure_absolute_path(env(
                 "DOCSRS_ARCHIVE_INDEX_CACHE_PATH",
                 prefix.join("archive_cache"),
-            )?)
+            )?)?)
+            .local_archive_cache_ttl(Duration::from_secs(env(
+                "DOCSRS_ARCHIVE_INDEX_CACHE_TTL",
+                48 * 3600, // 48 hours
+            )?))
             .compiler_metrics_collection_path(maybe_env("DOCSRS_COMPILER_METRICS_PATH")?)
             .temp_dir(temp_dir)
             .rustwide_workspace(env(
@@ -235,6 +242,14 @@ impl Config {
     }
 }
 
+fn ensure_absolute_path(path: PathBuf) -> io::Result<PathBuf> {
+    if path.is_absolute() {
+        Ok(path)
+    } else {
+        Ok(path::absolute(&path)?)
+    }
+}
+
 fn env<T>(var: &str, default: T) -> Result<T>
 where
     T: FromStr,