Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add "provides" package metadata. #194

Merged
merged 9 commits into from
Apr 29, 2016
18 changes: 12 additions & 6 deletions simplesat/constraints/package_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
'depends': ConstraintKinds.install_requires,
'install_requires': ConstraintKinds.install_requires,
'conflicts': ConstraintKinds.conflicts,
'provides': ConstraintKinds.provides,
}


Expand All @@ -32,7 +33,7 @@ def parse(self, pretty_string):

Pretty package strings are of the form::

numpy 1.8.1-1; install_requires (MKL == 10.3, nose ^= 1.3.4); conflicts (numeric) # noqa
numpy 1.8.1-1; install_requires (MKL == 10.3, nose ^= 1.3.4); conflicts (numeric); provides (numeric) # noqa
"""
pretty_string = pretty_string.strip()
pkg = {}
Expand Down Expand Up @@ -91,22 +92,21 @@ def parse_to_package(self, package_string):
return PackageMetadata(distribution, version, **pkg_dict)


def constraints_to_pretty_strings(install_requires):
def constraints_to_pretty_strings(constraint_tuples):
""" Convert a sequence of constraint tuples as used in PackageMetadata to a
list of pretty constraint strings.

Parameters
----------
install_requires : seq
Sequence of constraint tuples, e.g. ("MKL", (("< 11", ">= 10.1"),))
constraint_tuples : tuple of constraint
Sequence of constraint tuples, e.g. (("MKL", (("< 11", ">= 10.1"),)),)
"""
flat_strings = [
"{} {}".format(dist, constraint_string).strip()
for dist, disjunction in install_requires
for dist, disjunction in constraint_tuples
for conjunction in disjunction
for constraint_string in conjunction
]

return flat_strings


Expand All @@ -116,8 +116,14 @@ def package_to_pretty_string(package):
constraint_kinds = (
(ConstraintKinds.install_requires, package.install_requires),
(ConstraintKinds.conflicts, package.conflicts),
(ConstraintKinds.provides, package.provides),
)
for constraint_kind, constraints in constraint_kinds:
# FIXME: perhaps 'provides' just shouldn't include the package name
if constraint_kind == ConstraintKinds.provides:
constraints = tuple((dist, disjunction)
for dist, disjunction in constraints
if dist != package.name)
if len(constraints) > 0:
string = ', '.join(constraints_to_pretty_strings(constraints))
template += "; {} ({})".format(constraint_kind.value, string)
Expand Down
111 changes: 102 additions & 9 deletions simplesat/constraints/tests/test_package_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -226,30 +226,36 @@ def test_complicated(self):

# Given
package_string = '; '.join((
"bokeh 0.2.0-3",
"bokeh_git 0.2.0-3",
"install_requires (zope *, numpy ^= 1.8.0, requests >= 0.2)",
"conflicts (requests ^= 0.2.5, requests > 0.4, bokeh_git)",
"conflicts (requests ^= 0.2.5, requests > 0.4, bokeh)",
"provides (webplot ^= 0.1, bokeh)",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How is this different from "provides with version numbers"?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is only testing our ability to parse, which separates the clauses and builds constraint tuples, but doesn't validate. In general we don't validate things up front due to the overhead of parsing version objects and the sheer number of packages. Also, we will not be parsing pretty strings during a normal work flow. IThe program which is using simplesat as a library will construct package metadata directly (which is why we guard against it below this level).

))
r_install_requires = (
("numpy", (("^= 1.8.0",),)),
("requests", ((">= 0.2",),)),
("zope", (("*",),)))
r_conflicts = (
("bokeh_git", (('',),)),
("bokeh", (('',),)),
("requests", (("^= 0.2.5", "> 0.4"),)))
r_provides = (
("bokeh", (('',),)),
("webplot", (("^= 0.1",),)))

# When
package = parse(package_string)
name = package['distribution']
version = package['version']
install_requires = package['install_requires']
conflicts = package['conflicts']
provides = package['provides']

# Then
self.assertEqual(name, "bokeh")
self.assertEqual(name, "bokeh_git")
self.assertEqual(version, V("0.2.0-3"))
self.assertEqual(install_requires, r_install_requires)
self.assertEqual(conflicts, r_conflicts)
self.assertEqual(provides, r_provides)


class TestPackagePrettyString(unittest.TestCase):
Expand Down Expand Up @@ -315,24 +321,53 @@ def test_conflicts(self):
# Then
self.assertEqual(pretty_string, r_pretty_string)

def test_provides(self):
# Given
provides = ((u"dance", ((u">= 10.3-1",),)),)
package = PackageMetadata(u"zumba", V("1.8.1-1"), provides=provides)

r_pretty_string = u"zumba 1.8.1-1; provides (dance >= 10.3-1)"

# When
pretty_string = package_to_pretty_string(package)

# Then
self.assertEqual(pretty_string, r_pretty_string)

# Given
provides = (("cardio", (("*",),)),)
package = PackageMetadata(u"zumba", V("1.8.1-1"), provides=provides)

r_pretty_string = "zumba 1.8.1-1; provides (cardio *)"

# When
pretty_string = package_to_pretty_string(package)

# Then
self.assertEqual(pretty_string, r_pretty_string)

def test_complicated(self):
# Given
install_requires = (
("numpy", (("^= 1.8.0",),)),
("requests", ((">= 0.2",),)),
("zope", (("*",),)))
conflicts = (
("bokeh_git", (('',),)),
("bokeh", (('',),)),
("requests", ((">= 0.2.5", "< 0.4"),)))
provides = (
("bokeh", (('*',),)),)
package = PackageMetadata(
u"bokeh", V("0.2.0-3"),
u"bokeh_git", V("0.2.0-3"),
install_requires=install_requires,
conflicts=conflicts)
conflicts=conflicts,
provides=provides)

r_pretty_string = '; '.join((
"bokeh 0.2.0-3",
"bokeh_git 0.2.0-3",
"install_requires (numpy ^= 1.8.0, requests >= 0.2, zope *)",
"conflicts (bokeh_git, requests >= 0.2.5, requests < 0.4)",
"conflicts (bokeh, requests >= 0.2.5, requests < 0.4)",
"provides (bokeh *)",
))

# When
Expand Down Expand Up @@ -370,6 +405,64 @@ def test_with_depends(self):
self.assertEqual(package.version, V('1.8.1'))
self.assertEqual(package.install_requires, (("MKL", (("^= 10.3",),)),))

def test_with_conflicts(self):
# Given
s = u"numpy 1.8.1; conflicts (MKL <= 10.3)"
parser = PrettyPackageStringParser(V)

# When
package = parser.parse_to_package(s)

# Then
self.assertEqual(package.name, "numpy")
self.assertEqual(package.version, V('1.8.1'))
self.assertEqual(package.conflicts, (("MKL", (("<= 10.3",),)),))

def test_with_provides(self):
# Given
s = u"numpy 1.8.1-4; provides (numeric)"
parser = PrettyPackageStringParser(V)

# When
package = parser.parse_to_package(s)

# Then
self.assertEqual(package.name, "numpy")
self.assertEqual(package.version, V('1.8.1-4'))
self.assertEqual(package.provides,
(('numpy', (('*',),)),
("numeric", (("",),))))

def test_complicated(self):
# Given
install_requires = (
("numpy", (("^= 1.8.0",),)),
("requests", ((">= 0.2",),)),
("zope", (("*",),)))
conflicts = (
("bokeh", (('',),)),
("requests", ((">= 0.2.5", "< 0.4"),)))
provides = (
("bokeh", (('*',),)),)
expected = PackageMetadata(
u"bokeh_git", V("0.2.0-3"),
install_requires=install_requires,
conflicts=conflicts,
provides=provides)
parser = PrettyPackageStringParser(V)

# When
s = '; '.join((
"bokeh_git 0.2.0-3",
"install_requires (numpy ^= 1.8.0, requests >= 0.2, zope *)",
"conflicts (bokeh, requests >= 0.2.5, requests < 0.4)",
"provides (bokeh *)",
))
result = parser.parse_to_package(s)

# Then
self.assertEqual(result, expected)


class TestParseScaryPackages(unittest.TestCase):

Expand Down
11 changes: 7 additions & 4 deletions simplesat/dependency_solver.py
Original file line number Diff line number Diff line change
Expand Up @@ -296,13 +296,16 @@ def _connected_packages(solution, root_ids, pool):
# Our strategy is as follows:
# ... -> pkg.install_requires -> pkg names -> ids -> _id_to_package -> ...

def get_name(pkg_id):
return pool.id_to_package(abs(pkg_id)).name
def get_names(pkg_id):
provides = pool.id_to_package(abs(pkg_id)).provides
return tuple(name for name, _ in provides)

root_names = {get_name(pkg_id) for pkg_id in root_ids}
root_names = {name for pkg_id in root_ids for name in get_names(pkg_id)}

solution_name_to_id = {
get_name(pkg_id): pkg_id for pkg_id in solution
name: pkg_id
for pkg_id in solution
for name in get_names(pkg_id)
if pkg_id > 0
}

Expand Down
45 changes: 39 additions & 6 deletions simplesat/package.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
class ConstraintKinds(enum.Enum):
install_requires = 'install_requires'
conflicts = 'conflicts'
provides = 'provides'


class IRepositoryInfo(six.with_metaclass(abc.ABCMeta)):
Expand Down Expand Up @@ -62,7 +63,8 @@ def _from_pretty_string(cls, s):
parser = PrettyPackageStringParser(EnpkgVersion.from_string)
return parser.parse_to_package(s)

def __init__(self, name, version, install_requires=None, conflicts=None):
def __init__(self, name, version, install_requires=None, conflicts=None,
provides=None):
""" Return a new PackageMetdata object.

Parameters
Expand All @@ -85,15 +87,24 @@ def __init__(self, name, version, install_requires=None, conflicts=None):
(("MKL", ((">= 10.1", "< 11"),)),
("nose", (("*",),)),
("six", (("> 1.2", "<= 1.2.3"), (">= 1.2.5-2",)))

conflicts : tuple(tuple(str, tuple(tuple(str))))
A tuple of tuples mapping distribution names to disjunctions of
conjunctions of version constraints.

This works the same way as install_requires, but instead denotes
packages that must *not* be installed with this package.
provides : iterable of package names
The packages that are provided by this distribution. Useful when
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This makes it sounds like only the names of actual packages can be specified. From the examples, it looks like this could be an arbitrary label listed as a requirement. Is this purely a semantic difference?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Indeed a package can provide anything it wants. If the provided name does not correspond to a real package somewhere, we call it a "virtual package." This nomenclature is borrowed from debian. I'll add that to the docstring somehow.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In fact, this docstring is out of date, since the provides are also in the "constraint tuple" format.

this does not match the package name.

For example, a package ``foo`` is abandoned by its maintainer and a
fork ``bar`` is created to continue development. If ``bar`` is
intended to be a transparent replacement for ``foo``, then ``bar``
`provides` ``foo``.

"""
self._name = name
self._provides = tuple(provides or ())
self._version = version
self._install_requires = install_requires or ()
self._conflicts = conflicts or ()
Expand All @@ -104,6 +115,12 @@ def __init__(self, name, version, install_requires=None, conflicts=None):
def name(self):
return self._name

@property
def provides(self):
constraint_str = "*"
this_pkg = ((self._name, ((constraint_str,),)),)
return this_pkg + self._provides

@property
def version(self):
return self._version
Expand All @@ -124,10 +141,16 @@ def __hash__(self):
return self._hash

def __eq__(self, other):
return self._key == other._key
try:
return self._key == other._key
except AttributeError:
return NotImplemented

def __ne__(self, other):
return self._key != other._key
try:
return self._key != other._key
except AttributeError:
return NotImplemented


class RepositoryPackageMetadata(object):
Expand All @@ -147,6 +170,10 @@ def __init__(self, package, repository_info):
def name(self):
return self._package.name

@property
def provides(self):
return self._package.provides

@property
def version(self):
return self._package.version
Expand All @@ -173,7 +200,13 @@ def __hash__(self):
return self._hash

def __eq__(self, other):
return self._key == other._key
try:
return self._key == other._key
except AttributeError:
return NotImplemented

def __ne__(self, other):
return self._key != other._key
try:
return self._key != other._key
except AttributeError:
return NotImplemented
9 changes: 7 additions & 2 deletions simplesat/pool.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
from __future__ import absolute_import

from .utils import DefaultOrderedDict
from simplesat.constraints import modify_requirement
from simplesat.constraints import (
ConstraintModifiers, Requirement, modify_requirement
)


class Pool(object):
Expand Down Expand Up @@ -47,7 +49,10 @@ def add_repository(self, repository):
self._id += 1
self._id_to_package_[current_id] = package
self._package_to_id_[package] = current_id
self._packages_by_name_[package.name].append(package)
for constraints in package.provides:
req = Requirement.from_constraints(constraints)
assert not req.has_any_version_constraint
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you explain this assertion?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah yes, I forgot to write about it. Really we need to move the entire package metadata spec into the docs here.

The long and short of it is that right now we don't have a good way to handle provides with version numbers. We might someday, but there aren't any good use-cases that I know of. This assertion is there to make sure package metadata authors haven't done that. It should probably be ValueError or InvalidConstraint or something instead. What do you think?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This does sound like an InvalidConstraint. If this exception trickles up to users, we would want to make sure that the message was clear, too

self._packages_by_name_[req.name].append(package)

def what_provides(self, requirement, use_modifiers=True):
""" Computes the list of packages fulfilling the given
Expand Down
Loading