WIP: migrate to cargo_metadata crate & and its structs

Cargo.toml

@@ -76,6 +76,7 @@ aws-sdk-cloudfront = "1.3.0"
 aws-smithy-types-convert = { version = "0.60.0", features = ["convert-chrono"] }
 http = "1.0.0"
 uuid = { version = "1.1.2", features = ["v4"]}
+cargo_metadata = { version = "0.23.0", features = ["builder"] }
 
 # Data serialization and deserialization
 serde = { version = "1.0", features = ["derive"] }

src/db/add_package.rs

@@ -4,10 +4,11 @@ use crate::{
     error::Result,
     registry_api::{CrateData, CrateOwner, ReleaseData},
     storage::CompressionAlgorithm,
-    utils::{Dependency, MetadataPackage, rustc_version::parse_rustc_date},
+    utils::{cargo_metadata::PackageExt as _, rustc_version::parse_rustc_date},
     web::crate_details::{latest_release, releases_for_crate},
 };
 use anyhow::{Context, anyhow};
+use cargo_metadata::{Dependency, DependencyBuilder, DependencyKind};
 use derive_more::{Deref, Display};
 use futures_util::stream::TryStreamExt;
 use semver::VersionReq;
@@ -34,12 +35,11 @@ pub struct ReleaseId(pub i32);
 #[sqlx(transparent)]
 pub struct BuildId(pub i32);
 
-type DepOut = (String, String, String, bool);
-type DepIn = (String, VersionReq, Option<String>, Option<bool>);
+type Dep = (String, VersionReq, Option<DependencyKind>, Option<bool>);
 
 /// A crate dependency in our internal representation for releases.dependencies json.
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Deref)]
-#[serde(from = "DepIn", into = "DepOut")]
+#[serde(from = "Dep", into = "Dep")]
 pub(crate) struct ReleaseDependency(Dependency);
 
 impl ReleaseDependency {
@@ -48,27 +48,31 @@ impl ReleaseDependency {
     }
 }
 
-impl From<DepIn> for ReleaseDependency {
-    fn from((name, req, kind, optional): DepIn) -> Self {
-        ReleaseDependency(Dependency {
-            name,
-            req,
-            kind,
-            optional: optional.unwrap_or(false),
-            rename: None,
-        })
+impl From<Dep> for ReleaseDependency {
+    fn from((name, req, kind, optional): Dep) -> Self {
+        ReleaseDependency(
+            DependencyBuilder::default()
+                .name(name)
+                .source(None)
+                .req(req)
+                .kind(kind.unwrap_or_default())
+                .optional(optional.unwrap_or(false))
+                .uses_default_features(false)
+                .features(vec![])
+                .target(None)
+                .rename(None)
+                .registry(None)
+                .path(None)
+                .build()
+                .expect("we know the data is correct"),
+        )
     }
 }
 
-impl From<ReleaseDependency> for DepOut {
+impl From<ReleaseDependency> for Dep {
     fn from(rd: ReleaseDependency) -> Self {
         let d = rd.0;
-        (
-            d.name,
-            d.req.to_string(),
-            d.kind.unwrap_or_else(|| "normal".into()),
-            d.optional,
-        )
+        (d.name, d.req, Some(d.kind), Some(d.optional))
     }
 }
 
@@ -84,7 +88,7 @@ pub(crate) async fn finish_release(
     conn: &mut sqlx::PgConnection,
     crate_id: CrateId,
     release_id: ReleaseId,
-    metadata_pkg: &MetadataPackage,
+    metadata_pkg: &cargo_metadata::Package,
     source_dir: &Path,
     default_target: &str,
     source_files: Value,
@@ -98,7 +102,7 @@ pub(crate) async fn finish_release(
     source_size: u64,
 ) -> Result<()> {
     debug!("updating release data");
-    let dependencies = convert_dependencies(metadata_pkg)?;
+    let dependencies = convert_dependencies(metadata_pkg);
     let rustdoc = get_rustdoc(metadata_pkg, source_dir).unwrap_or(None);
     let readme = get_readme(metadata_pkg, source_dir).unwrap_or(None);
     let features = get_features(metadata_pkg);
@@ -133,7 +137,7 @@ pub(crate) async fn finish_release(
            WHERE id = $1"#,
         release_id.0,
         registry_data.release_time,
-        dependencies,
+        serde_json::to_value(dependencies)?,
         metadata_pkg.package_name(),
         registry_data.yanked,
         has_docs,
@@ -432,18 +436,24 @@ pub(crate) async fn initialize_build(
     Ok(build_id)
 }
 
-/// Convert dependencies into our own internal JSON representation
-fn convert_dependencies(pkg: &MetadataPackage) -> Result<serde_json::Value> {
-    let dependencies: Vec<_> = pkg
-        .dependencies
+fn convert_dependencies(
+    pkg: &cargo_metadata::Package,
+) -> Vec<(String, VersionReq, DependencyKind, bool)> {
+    pkg.dependencies
         .iter()
-        .map(|dependency| ReleaseDependency(dependency.clone()))
-        .collect::<Vec<_>>();
-    Ok(serde_json::to_value(dependencies)?)
+        .map(|dependency| {
+            (
+                dependency.name.clone(),
+                dependency.req.clone(),
+                dependency.kind,
+                dependency.optional,
+            )
+        })
+        .collect()
 }
 
 /// Reads features and converts them to Vec<Feature> with default being first
-fn get_features(pkg: &MetadataPackage) -> Vec<Feature> {
+fn get_features(pkg: &cargo_metadata::Package) -> Vec<Feature> {
     let mut features = Vec::with_capacity(pkg.features.len());
     if let Some(subfeatures) = pkg.features.get("default") {
         features.push(Feature::new("default".into(), subfeatures.clone()));
@@ -458,8 +468,8 @@ fn get_features(pkg: &MetadataPackage) -> Vec<Feature> {
 }
 
 /// Reads readme if there is any read defined in Cargo.toml of a Package
-fn get_readme(pkg: &MetadataPackage, source_dir: &Path) -> Result<Option<String>> {
-    let readme_path = source_dir.join(pkg.readme.as_deref().unwrap_or("README.md"));
+fn get_readme(pkg: &cargo_metadata::Package, source_dir: &Path) -> Result<Option<String>> {
+    let readme_path = source_dir.join(pkg.readme.as_deref().unwrap_or("README.md".into()));
 
     if !readme_path.exists() {
         return Ok(None);
@@ -479,8 +489,8 @@ fn get_readme(pkg: &MetadataPackage, source_dir: &Path) -> Result<Option<String>
     }
 }
 
-fn get_rustdoc(pkg: &MetadataPackage, source_dir: &Path) -> Result<Option<String>> {
-    if let Some(src_path) = &pkg.targets.first().and_then(|t| t.src_path.as_ref()) {
+fn get_rustdoc(pkg: &cargo_metadata::Package, source_dir: &Path) -> Result<Option<String>> {
+    if let Some(src_path) = &pkg.targets.first().map(|t| &t.src_path) {
         let src_path = Path::new(src_path);
         if src_path.is_absolute() {
             read_rust_doc(src_path)
@@ -528,7 +538,7 @@ fn read_rust_doc(file_path: &Path) -> Result<Option<String>> {
 /// Adds keywords into database
 async fn add_keywords_into_database(
     conn: &mut sqlx::PgConnection,
-    pkg: &MetadataPackage,
+    pkg: &cargo_metadata::Package,
     release_id: ReleaseId,
 ) -> Result<()> {
     let wanted_keywords: HashMap<String, String> = pkg
@@ -680,11 +690,16 @@ mod test {
     use super::*;
     use crate::registry_api::OwnerKind;
     use crate::test::*;
-    use crate::utils::CargoMetadata;
+    use crate::utils::cargo_metadata::{MetadataExt as _, load_cargo_metadata_from_host_path};
     use chrono::NaiveDate;
     use std::slice;
     use test_case::test_case;
 
+    #[test]
+    fn test_dependency_serialization_for_database() {
+        todo!();
+    }
+
     #[test]
     fn test_set_build_to_error() {
         async_wrapper(|env| async move {
@@ -1226,7 +1241,7 @@ mod test {
         "#;
 
         std::fs::write(dir.path().join("Cargo.toml"), [base, extra].concat())?;
-        let metadata = CargoMetadata::load_from_host_path(dir.path())?;
+        let metadata = load_cargo_metadata_from_host_path(dir.path())?;
         let features = super::get_features(metadata.root());
         assert_eq!(features, expected.as_ref());
 

src/docbuilder/rustwide_builder.rs

@@ -17,8 +17,9 @@ use crate::{
         rustdoc_archive_path, rustdoc_json_path, source_archive_path,
     },
     utils::{
-        CargoMetadata, ConfigName, MetadataPackage, copy_dir_all, get_config, parse_rustc_version,
-        report_error, set_config,
+        ConfigName,
+        cargo_metadata::{MetadataExt as _, PackageExt as _, load_cargo_metadata_from_rustwide},
+        copy_dir_all, get_config, parse_rustc_version, report_error, set_config,
     },
 };
 use anyhow::{Context as _, Error, anyhow, bail};
@@ -379,14 +380,14 @@ impl RustwideBuilder {
     }
 
     pub fn build_local_package(&mut self, path: &Path) -> Result<BuildPackageSummary> {
-        let metadata = CargoMetadata::load_from_rustwide(&self.workspace, &self.toolchain, path)
+        let metadata = load_cargo_metadata_from_rustwide(&self.workspace, &self.toolchain, path)
             .map_err(|err| {
                 err.context(format!("failed to load local package {}", path.display()))
             })?;
         let package = metadata.root();
         self.build_package(
             &package.name,
-            &package.version,
+            &package.version(),
             PackageKind::Local(path),
             false,
         )
@@ -1001,7 +1002,7 @@ impl RustwideBuilder {
         create_essential_files: bool,
         collect_metrics: bool,
     ) -> Result<FullBuildResult> {
-        let cargo_metadata = CargoMetadata::load_from_rustwide(
+        let cargo_metadata = load_cargo_metadata_from_rustwide(
             &self.workspace,
             &self.toolchain,
             &build.host_source_dir(),
@@ -1232,7 +1233,7 @@ impl RustwideBuilder {
         copy_dir_all(source, dest).map_err(Into::into)
     }
 
-    fn get_repo(&self, metadata: &MetadataPackage) -> Result<Option<i32>> {
+    fn get_repo(&self, metadata: &cargo_metadata::Package) -> Result<Option<i32>> {
         self.runtime
             .block_on(self.repository_stats_updater.load_repository(metadata))
     }
@@ -1241,7 +1242,7 @@ impl RustwideBuilder {
 struct FullBuildResult {
     result: BuildResult,
     target: String,
-    cargo_metadata: CargoMetadata,
+    cargo_metadata: cargo_metadata::Metadata,
     doc_coverage: Option<DocCoverage>,
     build_log: String,
 }
@@ -1286,10 +1287,12 @@ impl Default for BuildPackageSummary {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::db::types::Feature;
-    use crate::registry_api::ReleaseData;
-    use crate::storage::{CompressionAlgorithm, compression};
-    use crate::test::{AxumRouterTestExt, TestEnvironment};
+    use crate::{
+        db::types::Feature,
+        registry_api::ReleaseData,
+        storage::{CompressionAlgorithm, compression},
+        test::{AxumRouterTestExt, TestEnvironment, dummy_metadata_package},
+    };
     use pretty_assertions::assert_eq;
     use std::{io, iter};
     use test_case::test_case;
@@ -1658,21 +1661,7 @@ mod tests {
                 &mut conn,
                 crate_id,
                 release_id,
-                &MetadataPackage {
-                    name: crate_.into(),
-                    version: version.clone(),
-                    id: "".into(),
-                    license: None,
-                    repository: None,
-                    homepage: None,
-                    description: None,
-                    documentation: None,
-                    dependencies: vec![],
-                    targets: vec![],
-                    readme: None,
-                    keywords: vec![],
-                    features: HashMap::new(),
-                },
+                &dummy_metadata_package().build().unwrap(),
                 Path::new("/unknown/"),
                 "x86_64-unknown-linux-gnu",
                 serde_json::Value::Array(vec![]),

src/repositories/updater.rs

@@ -1,6 +1,5 @@
 use crate::error::Result;
 use crate::repositories::{GitHub, GitLab, RateLimitReached};
-use crate::utils::MetadataPackage;
 use crate::{Config, db::Pool};
 use async_trait::async_trait;
 use chrono::{DateTime, Utc};
@@ -81,7 +80,10 @@ impl RepositoryStatsUpdater {
         Self { updaters, pool }
     }
 
-    pub(crate) async fn load_repository(&self, metadata: &MetadataPackage) -> Result<Option<i32>> {
+    pub(crate) async fn load_repository(
+        &self,
+        metadata: &cargo_metadata::Package,
+    ) -> Result<Option<i32>> {
         let url = match &metadata.repository {
             Some(url) => url,
             None => {