Mercurial > repos > bcclaywell > argo_navis

comparison venv/lib/python2.7/site-packages/pip/req/req_set.py @ 0:d67268158946 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit a3f181f5f126803c654b3a66dd4e83a48f7e203b
author bcclaywell
date Mon, 12 Oct 2015 17:43:33 -0400
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:d67268158946
1 from __future__ import absolute_import
2
3 from collections import defaultdict
4 import functools
5 import itertools
6 import logging
7 import os
8
9 from pip._vendor import pkg_resources
10 from pip._vendor import requests
11
12 from pip.download import (url_to_path, unpack_url)
13 from pip.exceptions import (InstallationError, BestVersionAlreadyInstalled,
14 DistributionNotFound, PreviousBuildDirError)
15 from pip.locations import (PIP_DELETE_MARKER_FILENAME, build_prefix)
16 from pip.req.req_install import InstallRequirement
17 from pip.utils import (display_path, rmtree, dist_in_usersite, normalize_path)
18 from pip.utils.logging import indent_log
19 from pip.vcs import vcs
20
21
22 logger = logging.getLogger(__name__)
23
24
25 class Requirements(object):
26
27 def __init__(self):
28 self._keys = []
29 self._dict = {}
30
31 def keys(self):
32 return self._keys
33
34 def values(self):
35 return [self._dict[key] for key in self._keys]
36
37 def __contains__(self, item):
38 return item in self._keys
39
40 def __setitem__(self, key, value):
41 if key not in self._keys:
42 self._keys.append(key)
43 self._dict[key] = value
44
45 def __getitem__(self, key):
46 return self._dict[key]
47
48 def __repr__(self):
49 values = ['%s: %s' % (repr(k), repr(self[k])) for k in self.keys()]
50 return 'Requirements({%s})' % ', '.join(values)
51
52
53 class DistAbstraction(object):
54 """Abstracts out the wheel vs non-wheel prepare_files logic.
55
56 The requirements for anything installable are as follows:
57 - we must be able to determine the requirement name
58 (or we can't correctly handle the non-upgrade case).
59 - we must be able to generate a list of run-time dependencies
60 without installing any additional packages (or we would
61 have to either burn time by doing temporary isolated installs
62 or alternatively violate pips 'don't start installing unless
63 all requirements are available' rule - neither of which are
64 desirable).
65 - for packages with setup requirements, we must also be able
66 to determine their requirements without installing additional
67 packages (for the same reason as run-time dependencies)
68 - we must be able to create a Distribution object exposing the
69 above metadata.
70 """
71
72 def __init__(self, req_to_install):
73 self.req_to_install = req_to_install
74
75 def dist(self, finder):
76 """Return a setuptools Dist object."""
77 raise NotImplementedError(self.dist)
78
79 def prep_for_dist(self):
80 """Ensure that we can get a Dist for this requirement."""
81 raise NotImplementedError(self.dist)
82
83
84 def make_abstract_dist(req_to_install):
85 """Factory to make an abstract dist object.
86
87 Preconditions: Either an editable req with a source_dir, or satisfied_by or
88 a wheel link, or a non-editable req with a source_dir.
89
90 :return: A concrete DistAbstraction.
91 """
92 if req_to_install.editable:
93 return IsSDist(req_to_install)
94 elif req_to_install.link and req_to_install.link.is_wheel:
95 return IsWheel(req_to_install)
96 else:
97 return IsSDist(req_to_install)
98
99
100 class IsWheel(DistAbstraction):
101
102 def dist(self, finder):
103 return list(pkg_resources.find_distributions(
104 self.req_to_install.source_dir))[0]
105
106 def prep_for_dist(self):
107 # FIXME:https://github.com/pypa/pip/issues/1112
108 pass
109
110
111 class IsSDist(DistAbstraction):
112
113 def dist(self, finder):
114 dist = self.req_to_install.get_dist()
115 # FIXME: shouldn't be globally added:
116 if dist.has_metadata('dependency_links.txt'):
117 finder.add_dependency_links(
118 dist.get_metadata_lines('dependency_links.txt')
119 )
120 return dist
121
122 def prep_for_dist(self):
123 self.req_to_install.run_egg_info()
124 self.req_to_install.assert_source_matches_version()
125
126
127 class Installed(DistAbstraction):
128
129 def dist(self, finder):
130 return self.req_to_install.satisfied_by
131
132 def prep_for_dist(self):
133 pass
134
135
136 class RequirementSet(object):
137
138 def __init__(self, build_dir, src_dir, download_dir, upgrade=False,
139 ignore_installed=False, as_egg=False, target_dir=None,
140 ignore_dependencies=False, force_reinstall=False,
141 use_user_site=False, session=None, pycompile=True,
142 isolated=False, wheel_download_dir=None):
143 if session is None:
144 raise TypeError(
145 "RequirementSet() missing 1 required keyword argument: "
146 "'session'"
147 )
148
149 self.build_dir = build_dir
150 self.src_dir = src_dir
151 # XXX: download_dir and wheel_download_dir overlap semantically and may
152 # be combinable.
153 self.download_dir = download_dir
154 self.upgrade = upgrade
155 self.ignore_installed = ignore_installed
156 self.force_reinstall = force_reinstall
157 self.requirements = Requirements()
158 # Mapping of alias: real_name
159 self.requirement_aliases = {}
160 self.unnamed_requirements = []
161 self.ignore_dependencies = ignore_dependencies
162 self.successfully_downloaded = []
163 self.successfully_installed = []
164 self.reqs_to_cleanup = []
165 self.as_egg = as_egg
166 self.use_user_site = use_user_site
167 self.target_dir = target_dir # set from --target option
168 self.session = session
169 self.pycompile = pycompile
170 self.isolated = isolated
171 if wheel_download_dir:
172 wheel_download_dir = normalize_path(wheel_download_dir)
173 self.wheel_download_dir = wheel_download_dir
174 # Maps from install_req -> dependencies_of_install_req
175 self._dependencies = defaultdict(list)
176
177 def __str__(self):
178 reqs = [req for req in self.requirements.values()
179 if not req.comes_from]
180 reqs.sort(key=lambda req: req.name.lower())
181 return ' '.join([str(req.req) for req in reqs])
182
183 def __repr__(self):
184 reqs = [req for req in self.requirements.values()]
185 reqs.sort(key=lambda req: req.name.lower())
186 reqs_str = ', '.join([str(req.req) for req in reqs])
187 return ('<%s object; %d requirement(s): %s>'
188 % (self.__class__.__name__, len(reqs), reqs_str))
189
190 def add_requirement(self, install_req, parent_req_name=None):
191 """Add install_req as a requirement to install.
192
193 :param parent_req_name: The name of the requirement that needed this
194 added. The name is used because when multiple unnamed requirements
195 resolve to the same name, we could otherwise end up with dependency
196 links that point outside the Requirements set. parent_req must
197 already be added. Note that None implies that this is a user
198 supplied requirement, vs an inferred one.
199 :return: Additional requirements to scan. That is either [] if
200 the requirement is not applicable, or [install_req] if the
201 requirement is applicable and has just been added.
202 """
203 name = install_req.name
204 if ((not name or not self.has_requirement(name)) and not
205 install_req.match_markers()):
206 # Only log if we haven't already got install_req from somewhere.
207 logger.debug("Ignore %s: markers %r don't match",
208 install_req.name, install_req.markers)
209 return []
210
211 install_req.as_egg = self.as_egg
212 install_req.use_user_site = self.use_user_site
213 install_req.target_dir = self.target_dir
214 install_req.pycompile = self.pycompile
215 if not name:
216 # url or path requirement w/o an egg fragment
217 self.unnamed_requirements.append(install_req)
218 return [install_req]
219 else:
220 if parent_req_name is None and self.has_requirement(name):
221 raise InstallationError(
222 'Double requirement given: %s (already in %s, name=%r)'
223 % (install_req, self.get_requirement(name), name))
224 if not self.has_requirement(name):
225 # Add requirement
226 self.requirements[name] = install_req
227 # FIXME: what about other normalizations? E.g., _ vs. -?
228 if name.lower() != name:
229 self.requirement_aliases[name.lower()] = name
230 result = [install_req]
231 else:
232 # Canonicalise to the already-added object
233 install_req = self.get_requirement(name)
234 # No need to scan, this is a duplicate requirement.
235 result = []
236 if parent_req_name:
237 parent_req = self.get_requirement(parent_req_name)
238 self._dependencies[parent_req].append(install_req)
239 return result
240
241 def has_requirement(self, project_name):
242 for name in project_name, project_name.lower():
243 if name in self.requirements or name in self.requirement_aliases:
244 return True
245 return False
246
247 @property
248 def has_requirements(self):
249 return list(self.requirements.values()) or self.unnamed_requirements
250
251 @property
252 def is_download(self):
253 if self.download_dir:
254 self.download_dir = os.path.expanduser(self.download_dir)
255 if os.path.exists(self.download_dir):
256 return True
257 else:
258 logger.critical('Could not find download directory')
259 raise InstallationError(
260 "Could not find or access download directory '%s'"
261 % display_path(self.download_dir))
262 return False
263
264 def get_requirement(self, project_name):
265 for name in project_name, project_name.lower():
266 if name in self.requirements:
267 return self.requirements[name]
268 if name in self.requirement_aliases:
269 return self.requirements[self.requirement_aliases[name]]
270 raise KeyError("No project with the name %r" % project_name)
271
272 def uninstall(self, auto_confirm=False):
273 for req in self.requirements.values():
274 req.uninstall(auto_confirm=auto_confirm)
275 req.commit_uninstall()
276
277 def _walk_req_to_install(self, handler):
278 """Call handler for all pending reqs.
279
280 :param handler: Handle a single requirement. Should take a requirement
281 to install. Can optionally return an iterable of additional
282 InstallRequirements to cover.
283 """
284 # The list() here is to avoid potential mutate-while-iterating bugs.
285 discovered_reqs = []
286 reqs = itertools.chain(
287 list(self.unnamed_requirements), list(self.requirements.values()),
288 discovered_reqs)
289 for req_to_install in reqs:
290 more_reqs = handler(req_to_install)
291 if more_reqs:
292 discovered_reqs.extend(more_reqs)
293
294 def locate_files(self):
295 """Remove in 7.0: used by --no-download"""
296 self._walk_req_to_install(self._locate_file)
297
298 def _locate_file(self, req_to_install):
299 install_needed = True
300 if not self.ignore_installed and not req_to_install.editable:
301 req_to_install.check_if_exists()
302 if req_to_install.satisfied_by:
303 if self.upgrade:
304 # don't uninstall conflict if user install and
305 # conflict is not user install
306 if not (self.use_user_site and
307 not dist_in_usersite(
308 req_to_install.satisfied_by
309 )):
310 req_to_install.conflicts_with = \
311 req_to_install.satisfied_by
312 req_to_install.satisfied_by = None
313 else:
314 install_needed = False
315 logger.info(
316 'Requirement already satisfied (use --upgrade to '
317 'upgrade): %s',
318 req_to_install,
319 )
320
321 if req_to_install.editable:
322 if req_to_install.source_dir is None:
323 req_to_install.source_dir = req_to_install.build_location(
324 self.src_dir
325 )
326 elif install_needed:
327 req_to_install.source_dir = req_to_install.build_location(
328 self.build_dir,
329 )
330
331 if (req_to_install.source_dir is not None and not
332 os.path.isdir(req_to_install.source_dir)):
333 raise InstallationError(
334 'Could not install requirement %s because source folder %s'
335 ' does not exist (perhaps --no-download was used without '
336 'first running an equivalent install with --no-install?)' %
337 (req_to_install, req_to_install.source_dir)
338 )
339
340 def prepare_files(self, finder):
341 """
342 Prepare process. Create temp directories, download and/or unpack files.
343 """
344 self._walk_req_to_install(
345 functools.partial(self._prepare_file, finder))
346
347 def _check_skip_installed(self, req_to_install, finder):
348 """Check if req_to_install should be skipped.
349
350 This will check if the req is installed, and whether we should upgrade
351 or reinstall it, taking into account all the relevant user options.
352
353 After calling this req_to_install will only have satisfied_by set to
354 None if the req_to_install is to be upgraded/reinstalled etc. Any
355 other value will be a dist recording the current thing installed that
356 satisfies the requirement.
357
358 Note that for vcs urls and the like we can't assess skipping in this
359 routine - we simply identify that we need to pull the thing down,
360 then later on it is pulled down and introspected to assess upgrade/
361 reinstalls etc.
362
363 :return: A text reason for why it was skipped, or None.
364 """
365 # Check whether to upgrade/reinstall this req or not.
366 req_to_install.check_if_exists()
367 if req_to_install.satisfied_by:
368 skip_reason = 'satisfied (use --upgrade to upgrade)'
369 if self.upgrade:
370 best_installed = False
371 # For link based requirements we have to pull the
372 # tree down and inspect to assess the version #, so
373 # its handled way down.
374 if not (self.force_reinstall or req_to_install.link):
375 try:
376 finder.find_requirement(req_to_install, self.upgrade)
377 except BestVersionAlreadyInstalled:
378 skip_reason = 'up-to-date'
379 best_installed = True
380 except DistributionNotFound:
381 # No distribution found, so we squash the
382 # error - it will be raised later when we
383 # re-try later to do the install.
384 # Why don't we just raise here?
385 pass
386
387 if not best_installed:
388 # don't uninstall conflict if user install and
389 # conflict is not user install
390 if not (self.use_user_site and not
391 dist_in_usersite(req_to_install.satisfied_by)):
392 req_to_install.conflicts_with = \
393 req_to_install.satisfied_by
394 req_to_install.satisfied_by = None
395 return skip_reason
396 else:
397 return None
398
399 def _prepare_file(self, finder, req_to_install):
400 """Prepare a single requirements files.
401
402 :return: A list of addition InstallRequirements to also install.
403 """
404 # Tell user what we are doing for this requirement:
405 # obtain (editable), skipping, processing (local url), collecting
406 # (remote url or package name)
407 if req_to_install.editable:
408 logger.info('Obtaining %s', req_to_install)
409 else:
410 # satisfied_by is only evaluated by calling _check_skip_installed,
411 # so it must be None here.
412 assert req_to_install.satisfied_by is None
413 if not self.ignore_installed:
414 skip_reason = self._check_skip_installed(
415 req_to_install, finder)
416
417 if req_to_install.satisfied_by:
418 assert skip_reason is not None, (
419 '_check_skip_installed returned None but '
420 'req_to_install.satisfied_by is set to %r'
421 % (req_to_install.satisfied_by,))
422 logger.info(
423 'Requirement already %s: %s', skip_reason,
424 req_to_install)
425 else:
426 if (req_to_install.link and
427 req_to_install.link.scheme == 'file'):
428 path = url_to_path(req_to_install.link.url)
429 logger.info('Processing %s', display_path(path))
430 else:
431 logger.info('Collecting %s', req_to_install)
432
433 with indent_log():
434 # ################################ #
435 # # vcs update or unpack archive # #
436 # ################################ #
437 if req_to_install.editable:
438 req_to_install.ensure_has_source_dir(self.src_dir)
439 req_to_install.update_editable(not self.is_download)
440 abstract_dist = make_abstract_dist(req_to_install)
441 abstract_dist.prep_for_dist()
442 if self.is_download:
443 req_to_install.archive(self.download_dir)
444 elif req_to_install.satisfied_by:
445 abstract_dist = Installed(req_to_install)
446 else:
447 # @@ if filesystem packages are not marked
448 # editable in a req, a non deterministic error
449 # occurs when the script attempts to unpack the
450 # build directory
451 req_to_install.ensure_has_source_dir(self.build_dir)
452 # If a checkout exists, it's unwise to keep going. version
453 # inconsistencies are logged later, but do not fail the
454 # installation.
455 # FIXME: this won't upgrade when there's an existing
456 # package unpacked in `req_to_install.source_dir`
457 if os.path.exists(
458 os.path.join(req_to_install.source_dir, 'setup.py')):
459 raise PreviousBuildDirError(
460 "pip can't proceed with requirements '%s' due to a"
461 " pre-existing build directory (%s). This is "
462 "likely due to a previous installation that failed"
463 ". pip is being responsible and not assuming it "
464 "can delete this. Please delete it and try again."
465 % (req_to_install, req_to_install.source_dir)
466 )
467 req_to_install.populate_link(finder, self.upgrade)
468 # We can't hit this spot and have populate_link return None.
469 # req_to_install.satisfied_by is None here (because we're
470 # guarded) and upgrade has no impact except when satisfied_by
471 # is not None.
472 # Then inside find_requirement existing_applicable -> False
473 # If no new versions are found, DistributionNotFound is raised,
474 # otherwise a result is guaranteed.
475 assert req_to_install.link
476 try:
477 if req_to_install.link.is_wheel and \
478 self.wheel_download_dir:
479 # when doing 'pip wheel`
480 download_dir = self.wheel_download_dir
481 do_download = True
482 else:
483 download_dir = self.download_dir
484 do_download = self.is_download
485 unpack_url(
486 req_to_install.link, req_to_install.source_dir,
487 download_dir, do_download, session=self.session,
488 )
489 except requests.HTTPError as exc:
490 logger.critical(
491 'Could not install requirement %s because '
492 'of error %s',
493 req_to_install,
494 exc,
495 )
496 raise InstallationError(
497 'Could not install requirement %s because '
498 'of HTTP error %s for URL %s' %
499 (req_to_install, exc, req_to_install.link)
500 )
501 abstract_dist = make_abstract_dist(req_to_install)
502 abstract_dist.prep_for_dist()
503 if self.is_download:
504 # Make a .zip of the source_dir we already created.
505 if req_to_install.link.scheme in vcs.all_schemes:
506 req_to_install.archive(self.download_dir)
507 # req_to_install.req is only avail after unpack for URL
508 # pkgs repeat check_if_exists to uninstall-on-upgrade
509 # (#14)
510 if not self.ignore_installed:
511 req_to_install.check_if_exists()
512 if req_to_install.satisfied_by:
513 if self.upgrade or self.ignore_installed:
514 # don't uninstall conflict if user install and
515 # conflict is not user install
516 if not (self.use_user_site and not
517 dist_in_usersite(
518 req_to_install.satisfied_by)):
519 req_to_install.conflicts_with = \
520 req_to_install.satisfied_by
521 req_to_install.satisfied_by = None
522 else:
523 logger.info(
524 'Requirement already satisfied (use '
525 '--upgrade to upgrade): %s',
526 req_to_install,
527 )
528
529 # ###################### #
530 # # parse dependencies # #
531 # ###################### #
532 dist = abstract_dist.dist(finder)
533 more_reqs = []
534
535 def add_req(subreq):
536 sub_install_req = InstallRequirement(
537 str(subreq),
538 req_to_install,
539 isolated=self.isolated,
540 )
541 more_reqs.extend(self.add_requirement(
542 sub_install_req, req_to_install.name))
543
544 # We add req_to_install before its dependencies, so that we
545 # can refer to it when adding dependencies.
546 if not self.has_requirement(req_to_install.name):
547 # 'unnamed' requirements will get added here
548 self.add_requirement(req_to_install, None)
549
550 if not self.ignore_dependencies:
551 if (req_to_install.extras):
552 logger.debug(
553 "Installing extra requirements: %r",
554 ','.join(req_to_install.extras),
555 )
556 missing_requested = sorted(
557 set(req_to_install.extras) - set(dist.extras)
558 )
559 for missing in missing_requested:
560 logger.warning(
561 '%s does not provide the extra \'%s\'',
562 dist, missing
563 )
564
565 available_requested = sorted(
566 set(dist.extras) & set(req_to_install.extras)
567 )
568 for subreq in dist.requires(available_requested):
569 add_req(subreq)
570
571 # cleanup tmp src
572 self.reqs_to_cleanup.append(req_to_install)
573
574 if not req_to_install.editable and not req_to_install.satisfied_by:
575 # XXX: --no-install leads this to report 'Successfully
576 # downloaded' for only non-editable reqs, even though we took
577 # action on them.
578 self.successfully_downloaded.append(req_to_install)
579
580 return more_reqs
581
582 def cleanup_files(self):
583 """Clean up files, remove builds."""
584 logger.debug('Cleaning up...')
585 with indent_log():
586 for req in self.reqs_to_cleanup:
587 req.remove_temporary_source()
588
589 if self._pip_has_created_build_dir():
590 logger.debug('Removing temporary dir %s...', self.build_dir)
591 rmtree(self.build_dir)
592
593 def _pip_has_created_build_dir(self):
594 return (
595 self.build_dir == build_prefix and
596 os.path.exists(
597 os.path.join(self.build_dir, PIP_DELETE_MARKER_FILENAME)
598 )
599 )
600
601 def _to_install(self):
602 """Create the installation order.
603
604 The installation order is topological - requirements are installed
605 before the requiring thing. We break cycles at an arbitrary point,
606 and make no other guarantees.
607 """
608 # The current implementation, which we may change at any point
609 # installs the user specified things in the order given, except when
610 # dependencies must come earlier to achieve topological order.
611 order = []
612 ordered_reqs = set()
613
614 def schedule(req):
615 if req.satisfied_by or req in ordered_reqs:
616 return
617 ordered_reqs.add(req)
618 for dep in self._dependencies[req]:
619 schedule(dep)
620 order.append(req)
621 for install_req in self.requirements.values():
622 schedule(install_req)
623 return order
624
625 def install(self, install_options, global_options=(), *args, **kwargs):
626 """
627 Install everything in this set (after having downloaded and unpacked
628 the packages)
629 """
630 to_install = self._to_install()
631
632 # DISTRIBUTE TO SETUPTOOLS UPGRADE HACK (1 of 3 parts)
633 # move the distribute-0.7.X wrapper to the end because it does not
634 # install a setuptools package. by moving it to the end, we ensure it's
635 # setuptools dependency is handled first, which will provide the
636 # setuptools package
637 # TODO: take this out later
638 distribute_req = pkg_resources.Requirement.parse("distribute>=0.7")
639 for req in to_install:
640 if (req.name == 'distribute' and
641 req.installed_version is not None and
642 req.installed_version in distribute_req):
643 to_install.remove(req)
644 to_install.append(req)
645
646 if to_install:
647 logger.info(
648 'Installing collected packages: %s',
649 ', '.join([req.name for req in to_install]),
650 )
651
652 with indent_log():
653 for requirement in to_install:
654
655 # DISTRIBUTE TO SETUPTOOLS UPGRADE HACK (1 of 3 parts)
656 # when upgrading from distribute-0.6.X to the new merged
657 # setuptools in py2, we need to force setuptools to uninstall
658 # distribute. In py3, which is always using distribute, this
659 # conversion is already happening in distribute's
660 # pkg_resources. It's ok *not* to check if setuptools>=0.7
661 # because if someone were actually trying to ugrade from
662 # distribute to setuptools 0.6.X, then all this could do is
663 # actually help, although that upgade path was certainly never
664 # "supported"
665 # TODO: remove this later
666 if requirement.name == 'setuptools':
667 try:
668 # only uninstall distribute<0.7. For >=0.7, setuptools
669 # will also be present, and that's what we need to
670 # uninstall
671 distribute_requirement = \
672 pkg_resources.Requirement.parse("distribute<0.7")
673 existing_distribute = \
674 pkg_resources.get_distribution("distribute")
675 if existing_distribute in distribute_requirement:
676 requirement.conflicts_with = existing_distribute
677 except pkg_resources.DistributionNotFound:
678 # distribute wasn't installed, so nothing to do
679 pass
680
681 if requirement.conflicts_with:
682 logger.info(
683 'Found existing installation: %s',
684 requirement.conflicts_with,
685 )
686 with indent_log():
687 requirement.uninstall(auto_confirm=True)
688 try:
689 requirement.install(
690 install_options,
691 global_options,
692 *args,
693 **kwargs
694 )
695 except:
696 # if install did not succeed, rollback previous uninstall
697 if (requirement.conflicts_with and not
698 requirement.install_succeeded):
699 requirement.rollback_uninstall()
700 raise
701 else:
702 if (requirement.conflicts_with and
703 requirement.install_succeeded):
704 requirement.commit_uninstall()
705 requirement.remove_temporary_source()
706
707 self.successfully_installed = to_install