Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Implement PEP735 support #2448

Merged
merged 3 commits into from
Nov 18, 2024
Merged

feat: Implement PEP735 support #2448

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3e1b596
Implement PEP735 support
olivier-lacroix Nov 10, 2024
1fb1aa2
Handle dependency groups in init
olivier-lacroix Nov 11, 2024
0e2646a
Remove --location from CLI
olivier-lacroix Nov 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions crates/pixi_manifest/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,15 @@ pub enum DependencyOverwriteBehavior {
Error,
}

pub enum PypiDependencyLocation {
// The [pypi-dependencies] or [tool.pixi.pypi-dependencies] table
Pixi,
// The [project.optional-dependencies] table in a 'pyproject.toml' manifest
OptionalDependencies,
// The [dependency-groups] table in a 'pyproject.toml' manifest
DependencyGroups,
}

/// Converts an array of Platforms to a non-empty Vec of Option<Platform>
fn to_options(platforms: &[Platform]) -> Vec<Option<Platform>> {
match platforms.is_empty() {
Expand Down
6 changes: 4 additions & 2 deletions crates/pixi_manifest/src/manifests/manifest.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ use crate::{
pypi::PyPiPackageName,
pyproject::PyProjectManifest,
to_options, DependencyOverwriteBehavior, Environment, EnvironmentName, Feature, FeatureName,
GetFeatureError, ParsedManifest, PrioritizedChannel, SpecType, Target, TargetSelector, Task,
TaskName,
GetFeatureError, ParsedManifest, PrioritizedChannel, PypiDependencyLocation, SpecType, Target,
TargetSelector, Task, TaskName,
};

#[derive(Debug, Clone)]
Expand Down Expand Up @@ -384,6 +384,7 @@ impl Manifest {
feature_name: &FeatureName,
editable: Option<bool>,
overwrite_behavior: DependencyOverwriteBehavior,
location: &Option<PypiDependencyLocation>,
) -> miette::Result<bool> {
let mut any_added = false;
for platform in crate::to_options(platforms) {
Expand All @@ -398,6 +399,7 @@ impl Manifest {
platform,
feature_name,
editable,
location,
)?;
any_added = true;
}
Expand Down
151 changes: 95 additions & 56 deletions crates/pixi_manifest/src/manifests/project.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ use rattler_conda_types::{PackageName, Platform};
use toml_edit::{value, Array, Item, Table, Value};

use super::TomlManifest;
use crate::PypiDependencyLocation;
use crate::{consts, error::TomlError, pypi::PyPiPackageName, PyPiRequirement};
use crate::{consts::PYPROJECT_PIXI_PREFIX, FeatureName, SpecType, Task};

Expand Down Expand Up @@ -188,30 +189,43 @@ impl ManifestSource {
) -> Result<(), TomlError> {
// For 'pyproject.toml' manifest, try and remove the dependency from native
// arrays
let array = match self {
let remove_requirement =
|source: &mut ManifestSource, table, array_name| -> Result<(), TomlError> {
let array = source.manifest().get_toml_array(table, array_name)?;
if let Some(array) = array {
array.retain(|x| {
let req: pep508_rs::Requirement = x
.as_str()
.unwrap_or("")
.parse()
.expect("should be a valid pep508 dependency");
let name = PyPiPackageName::from_normalized(req.name);
name != *dep
});
if array.is_empty() {
source
.manifest()
.get_or_insert_nested_table(table)?
.remove(array_name);
}
}
Ok(())
};

match self {
ManifestSource::PyProjectToml(_) if feature_name.is_default() => {
self.manifest().get_toml_array("project", "dependencies")?
remove_requirement(self, "project", "dependencies")?;
}
ManifestSource::PyProjectToml(_) => {
let name = feature_name.to_string();
remove_requirement(self, "project.optional-dependencies", &name)?;
remove_requirement(self, "dependency-groups", &name)?;
}
ManifestSource::PyProjectToml(_) => self
.manifest()
.get_toml_array("project.optional-dependencies", &feature_name.to_string())?,
_ => None,
_ => (),
};
if let Some(array) = array {
array.retain(|x| {
let req: pep508_rs::Requirement = x
.as_str()
.unwrap_or("")
.parse()
.expect("should be a valid pep508 dependency");
let name = PyPiPackageName::from_normalized(req.name);
name != *dep
});
}

// For both 'pyproject.toml' and 'pixi.toml' manifest,
// try and remove the dependency from pixi native tables

let table_name = TableName::new()
.with_prefix(self.table_prefix())
.with_feature_name(Some(feature_name))
Expand Down Expand Up @@ -285,47 +299,72 @@ impl ManifestSource {
platform: Option<Platform>,
feature_name: &FeatureName,
editable: Option<bool>,
location: &Option<PypiDependencyLocation>,
) -> Result<(), TomlError> {
match self {
ManifestSource::PyProjectToml(_) => {
// Pypi dependencies can be stored in different places
// so we remove any potential dependency of the same name before adding it back
self.remove_pypi_dependency(
&PyPiPackageName::from_normalized(requirement.name.clone()),
platform,
feature_name,
)?;
if let FeatureName::Named(name) = feature_name {
self.manifest()
.get_or_insert_toml_array("project.optional-dependencies", name)?
.push(requirement.to_string())
} else {
self.manifest()
.get_or_insert_toml_array("project", "dependencies")?
.push(requirement.to_string())
}
}
ManifestSource::PixiToml(_) => {
let mut pypi_requirement =
PyPiRequirement::try_from(requirement.clone()).map_err(Box::new)?;
if let Some(editable) = editable {
pypi_requirement.set_editable(editable);
}
// Pypi dependencies can be stored in different places in pyproject.toml manifests
// so we remove any potential dependency of the same name before adding it back
if matches!(self, ManifestSource::PyProjectToml(_)) {
self.remove_pypi_dependency(
&PyPiPackageName::from_normalized(requirement.name.clone()),
platform,
feature_name,
)?;
}

let dependency_table = TableName::new()
.with_prefix(self.table_prefix())
.with_platform(platform.as_ref())
.with_feature_name(Some(feature_name))
.with_table(Some(consts::PYPI_DEPENDENCIES));

self.manifest()
.get_or_insert_nested_table(dependency_table.to_string().as_str())?
.insert(
requirement.name.as_ref(),
Item::Value(pypi_requirement.into()),
);
// The '[pypi-dependencies]' or '[tool.pixi.pypi-dependencies]' table is selected
// - For 'pixi.toml' manifests where it is the only choice
// - When explicitly requested
// - When a specific platform is requested, as markers are not supported (https://github.com/prefix-dev/pixi/issues/2149)
// - When an editable install is requested
if matches!(self, ManifestSource::PixiToml(_))
|| matches!(location, Some(PypiDependencyLocation::Pixi))
|| platform.is_some()
|| editable.is_some_and(|e| e)
{
let mut pypi_requirement =
PyPiRequirement::try_from(requirement.clone()).map_err(Box::new)?;
if let Some(editable) = editable {
pypi_requirement.set_editable(editable);
}
};

let dependency_table = TableName::new()
.with_prefix(self.table_prefix())
.with_platform(platform.as_ref())
.with_feature_name(Some(feature_name))
.with_table(Some(consts::PYPI_DEPENDENCIES));

self.manifest()
.get_or_insert_nested_table(dependency_table.to_string().as_str())?
.insert(
requirement.name.as_ref(),
Item::Value(pypi_requirement.into()),
);
return Ok(());
}

// Otherwise:
// - the [project.dependencies] array is selected for the default feature
// - the [dependency-groups.feature_name] array is selected unless
// - optional-dependencies is explicitly requested as location
let add_requirement =
|source: &mut ManifestSource, table, array| -> Result<(), TomlError> {
source
.manifest()
.get_or_insert_toml_array(table, array)?
.push(requirement.to_string());
Ok(())
};
if feature_name.is_default() {
add_requirement(self, "project", "dependencies")?
} else if matches!(location, Some(PypiDependencyLocation::OptionalDependencies)) {
add_requirement(
self,
"project.optional-dependencies",
&feature_name.to_string(),
)?
} else {
add_requirement(self, "dependency-groups", &feature_name.to_string())?
}
Ok(())
}

Expand Down
105 changes: 64 additions & 41 deletions crates/pixi_manifest/src/pyproject.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use miette::{Diagnostic, IntoDiagnostic, Report, WrapErr};
use pep440_rs::VersionSpecifiers;
use pep508_rs::Requirement;
use pixi_spec::PixiSpec;
use pyproject_toml::{self, Project};
use pyproject_toml::{self, pep735_resolve::Pep735Error, Contact, Project};
use rattler_conda_types::{PackageName, ParseStrictness::Lenient, VersionSpec};
use serde::Deserialize;
use thiserror::Error;
Expand Down Expand Up @@ -165,11 +165,10 @@ impl PyProjectManifest {
return Some(
pyproject_authors
.iter()
.filter_map(|contact| match (contact.name(), contact.email()) {
(Some(name), Some(email)) => Some(format!("{} <{}>", name, email)),
(Some(name), None) => Some(name.to_string()),
(None, Some(email)) => Some(email.to_string()),
(None, None) => None,
.map(|contact| match contact {
Contact::NameEmail { name, email } => format!("{} <{}>", name, email),
Contact::Name { name } => name.clone(),
Contact::Email { email } => email.clone(),
})
.collect(),
);
Expand All @@ -192,15 +191,19 @@ impl PyProjectManifest {
self.project().and_then(|p| p.optional_dependencies.clone())
}

/// Builds a list of pixi environments from pyproject groups of extra
/// dependencies:
/// - one environment is created per group of extra, with the same name as
/// the group of extra
/// - each environment includes the feature of the same name as the group
/// of extra
/// Returns dependency groups from the `[dependency-groups]` table
fn dependency_groups(&self) -> Option<Result<IndexMap<String, Vec<Requirement>>, Pep735Error>> {
self.dependency_groups.as_ref().map(|dg| dg.resolve())
}

/// Builds a list of pixi environments from pyproject groups of optional
/// dependencies and/or dependency groups:
/// - one environment is created per group with the same name
/// - each environment includes the feature of the same name
/// - it will also include other features inferred from any self references
/// to other groups of extras
pub fn environments_from_extras(&self) -> HashMap<String, Vec<String>> {
/// to other groups of optional dependencies (but won't for dependency groups,
/// as recursion between groups is resolved upstream)
pub fn environments_from_extras(&self) -> Result<HashMap<String, Vec<String>>, Pep735Error> {
let mut environments = HashMap::new();
if let Some(extras) = self.optional_dependencies() {
let pname = self.package_name();
Expand All @@ -218,14 +221,27 @@ impl PyProjectManifest {
environments.insert(extra.replace('_', "-").clone(), features);
}
}
environments

if let Some(groups) = self.dependency_groups().transpose()? {
for group in groups.into_keys() {
let normalised = group.replace('_', "-");
// Nothing to do if a group of optional dependencies has the same name as the dependency group
if !environments.contains_key(&normalised) {
environments.insert(normalised.clone(), vec![normalised]);
}
}
}

Ok(environments)
}
}

#[derive(Debug, Error, Diagnostic)]
pub enum PyProjectToManifestError {
#[error("Unsupported pep508 requirement: '{0}'")]
DependencyError(Requirement, #[source] DependencyError),
#[error(transparent)]
DependencyGroupError(#[from] Pep735Error),
}

impl TryFrom<PyProjectManifest> for ParsedManifest {
Expand Down Expand Up @@ -289,32 +305,37 @@ impl TryFrom<PyProjectManifest> for ParsedManifest {
}
}

// For each extra group, create a feature of the same name if it does not exist,
// and add pypi dependencies from project.optional-dependencies,
// filtering out self-references
if let Some(extras) = item.optional_dependencies() {
let project_name = item.package_name();
for (extra, reqs) in extras {
let feature_name = FeatureName::Named(extra.to_string());
let target = manifest
.features
.entry(feature_name.clone())
.or_insert_with(move || Feature::new(feature_name))
.targets
.default_mut();
for requirement in reqs.iter() {
// filter out any self references in groups of extra dependencies
if project_name.as_ref() != Some(&requirement.name) {
target
.try_add_pep508_dependency(
requirement,
None,
DependencyOverwriteBehavior::Error,
)
.map_err(|err| {
PyProjectToManifestError::DependencyError(requirement.clone(), err)
})?;
}
// Define an iterator over both optional dependencies and dependency groups
let groups = item
.optional_dependencies()
.into_iter()
.chain(item.dependency_groups().transpose()?)
.flat_map(|map| map.into_iter());

// For each group of optional dependency or dependency group,
// create a feature of the same name if it does not exist,
// and add pypi dependencies, filtering out self-references in optional dependencies
let project_name = item.package_name();
for (group, reqs) in groups {
let feature_name = FeatureName::Named(group.to_string());
let target = manifest
.features
.entry(feature_name.clone())
.or_insert_with(move || Feature::new(feature_name))
.targets
.default_mut();
for requirement in reqs.iter() {
// filter out any self references in groups of extra dependencies
if project_name.as_ref() != Some(&requirement.name) {
target
.try_add_pep508_dependency(
requirement,
None,
DependencyOverwriteBehavior::Error,
)
.map_err(|err| {
PyProjectToManifestError::DependencyError(requirement.clone(), err)
})?;
}
}
}
Expand Down Expand Up @@ -534,6 +555,7 @@ mod tests {
&FeatureName::Default,
None,
DependencyOverwriteBehavior::Overwrite,
&None,
)
.unwrap();

Expand All @@ -557,6 +579,7 @@ mod tests {
&FeatureName::Named("test".to_string()),
None,
DependencyOverwriteBehavior::Overwrite,
&None,
)
.unwrap();
assert!(manifest
Expand Down
Loading
Loading