| # -*- coding: utf-8 -*- |
| # |
| # Copyright (C) 2012-2015 Vinay Sajip. |
| # Licensed to the Python Software Foundation under a contributor agreement. |
| # See LICENSE.txt and CONTRIBUTORS.txt. |
| # |
| |
| import gzip |
| from io import BytesIO |
| import json |
| import logging |
| import os |
| import posixpath |
| import re |
| try: |
| import threading |
| except ImportError: # pragma: no cover |
| import dummy_threading as threading |
| import zlib |
| |
| from . import DistlibException |
| from .compat import (urljoin, urlparse, urlunparse, url2pathname, pathname2url, |
| queue, quote, unescape, build_opener, |
| HTTPRedirectHandler as BaseRedirectHandler, text_type, |
| Request, HTTPError, URLError) |
| from .database import Distribution, DistributionPath, make_dist |
| from .metadata import Metadata, MetadataInvalidError |
| from .util import (cached_property, ensure_slash, split_filename, get_project_data, |
| parse_requirement, parse_name_and_version, ServerProxy, |
| normalize_name) |
| from .version import get_scheme, UnsupportedVersionError |
| from .wheel import Wheel, is_compatible |
| |
| logger = logging.getLogger(__name__) |
| |
| HASHER_HASH = re.compile(r'^(\w+)=([a-f0-9]+)') |
| CHARSET = re.compile(r';\s*charset\s*=\s*(.*)\s*$', re.I) |
| HTML_CONTENT_TYPE = re.compile('text/html|application/x(ht)?ml') |
| DEFAULT_INDEX = 'https://pypi.org/pypi' |
| |
| def get_all_distribution_names(url=None): |
| """ |
| Return all distribution names known by an index. |
| :param url: The URL of the index. |
| :return: A list of all known distribution names. |
| """ |
| if url is None: |
| url = DEFAULT_INDEX |
| client = ServerProxy(url, timeout=3.0) |
| try: |
| return client.list_packages() |
| finally: |
| client('close')() |
| |
| class RedirectHandler(BaseRedirectHandler): |
| """ |
| A class to work around a bug in some Python 3.2.x releases. |
| """ |
| # There's a bug in the base version for some 3.2.x |
| # (e.g. 3.2.2 on Ubuntu Oneiric). If a Location header |
| # returns e.g. /abc, it bails because it says the scheme '' |
| # is bogus, when actually it should use the request's |
| # URL for the scheme. See Python issue #13696. |
| def http_error_302(self, req, fp, code, msg, headers): |
| # Some servers (incorrectly) return multiple Location headers |
| # (so probably same goes for URI). Use first header. |
| newurl = None |
| for key in ('location', 'uri'): |
| if key in headers: |
| newurl = headers[key] |
| break |
| if newurl is None: # pragma: no cover |
| return |
| urlparts = urlparse(newurl) |
| if urlparts.scheme == '': |
| newurl = urljoin(req.get_full_url(), newurl) |
| if hasattr(headers, 'replace_header'): |
| headers.replace_header(key, newurl) |
| else: |
| headers[key] = newurl |
| return BaseRedirectHandler.http_error_302(self, req, fp, code, msg, |
| headers) |
| |
| http_error_301 = http_error_303 = http_error_307 = http_error_302 |
| |
| class Locator(object): |
| """ |
| A base class for locators - things that locate distributions. |
| """ |
| source_extensions = ('.tar.gz', '.tar.bz2', '.tar', '.zip', '.tgz', '.tbz') |
| binary_extensions = ('.egg', '.exe', '.whl') |
| excluded_extensions = ('.pdf',) |
| |
| # A list of tags indicating which wheels you want to match. The default |
| # value of None matches against the tags compatible with the running |
| # Python. If you want to match other values, set wheel_tags on a locator |
| # instance to a list of tuples (pyver, abi, arch) which you want to match. |
| wheel_tags = None |
| |
| downloadable_extensions = source_extensions + ('.whl',) |
| |
| def __init__(self, scheme='default'): |
| """ |
| Initialise an instance. |
| :param scheme: Because locators look for most recent versions, they |
| need to know the version scheme to use. This specifies |
| the current PEP-recommended scheme - use ``'legacy'`` |
| if you need to support existing distributions on PyPI. |
| """ |
| self._cache = {} |
| self.scheme = scheme |
| # Because of bugs in some of the handlers on some of the platforms, |
| # we use our own opener rather than just using urlopen. |
| self.opener = build_opener(RedirectHandler()) |
| # If get_project() is called from locate(), the matcher instance |
| # is set from the requirement passed to locate(). See issue #18 for |
| # why this can be useful to know. |
| self.matcher = None |
| self.errors = queue.Queue() |
| |
| def get_errors(self): |
| """ |
| Return any errors which have occurred. |
| """ |
| result = [] |
| while not self.errors.empty(): # pragma: no cover |
| try: |
| e = self.errors.get(False) |
| result.append(e) |
| except self.errors.Empty: |
| continue |
| self.errors.task_done() |
| return result |
| |
| def clear_errors(self): |
| """ |
| Clear any errors which may have been logged. |
| """ |
| # Just get the errors and throw them away |
| self.get_errors() |
| |
| def clear_cache(self): |
| self._cache.clear() |
| |
| def _get_scheme(self): |
| return self._scheme |
| |
| def _set_scheme(self, value): |
| self._scheme = value |
| |
| scheme = property(_get_scheme, _set_scheme) |
| |
| def _get_project(self, name): |
| """ |
| For a given project, get a dictionary mapping available versions to Distribution |
| instances. |
| |
| This should be implemented in subclasses. |
| |
| If called from a locate() request, self.matcher will be set to a |
| matcher for the requirement to satisfy, otherwise it will be None. |
| """ |
| raise NotImplementedError('Please implement in the subclass') |
| |
| def get_distribution_names(self): |
| """ |
| Return all the distribution names known to this locator. |
| """ |
| raise NotImplementedError('Please implement in the subclass') |
| |
| def get_project(self, name): |
| """ |
| For a given project, get a dictionary mapping available versions to Distribution |
| instances. |
| |
| This calls _get_project to do all the work, and just implements a caching layer on top. |
| """ |
| if self._cache is None: # pragma: no cover |
| result = self._get_project(name) |
| elif name in self._cache: |
| result = self._cache[name] |
| else: |
| self.clear_errors() |
| result = self._get_project(name) |
| self._cache[name] = result |
| return result |
| |
| def score_url(self, url): |
| """ |
| Give an url a score which can be used to choose preferred URLs |
| for a given project release. |
| """ |
| t = urlparse(url) |
| basename = posixpath.basename(t.path) |
| compatible = True |
| is_wheel = basename.endswith('.whl') |
| is_downloadable = basename.endswith(self.downloadable_extensions) |
| if is_wheel: |
| compatible = is_compatible(Wheel(basename), self.wheel_tags) |
| return (t.scheme == 'https', 'pypi.org' in t.netloc, |
| is_downloadable, is_wheel, compatible, basename) |
| |
| def prefer_url(self, url1, url2): |
| """ |
| Choose one of two URLs where both are candidates for distribution |
| archives for the same version of a distribution (for example, |
| .tar.gz vs. zip). |
| |
| The current implementation favours https:// URLs over http://, archives |
| from PyPI over those from other locations, wheel compatibility (if a |
| wheel) and then the archive name. |
| """ |
| result = url2 |
| if url1: |
| s1 = self.score_url(url1) |
| s2 = self.score_url(url2) |
| if s1 > s2: |
| result = url1 |
| if result != url2: |
| logger.debug('Not replacing %r with %r', url1, url2) |
| else: |
| logger.debug('Replacing %r with %r', url1, url2) |
| return result |
| |
| def split_filename(self, filename, project_name): |
| """ |
| Attempt to split a filename in project name, version and Python version. |
| """ |
| return split_filename(filename, project_name) |
| |
| def convert_url_to_download_info(self, url, project_name): |
| """ |
| See if a URL is a candidate for a download URL for a project (the URL |
| has typically been scraped from an HTML page). |
| |
| If it is, a dictionary is returned with keys "name", "version", |
| "filename" and "url"; otherwise, None is returned. |
| """ |
| def same_project(name1, name2): |
| return normalize_name(name1) == normalize_name(name2) |
| |
| result = None |
| scheme, netloc, path, params, query, frag = urlparse(url) |
| if frag.lower().startswith('egg='): # pragma: no cover |
| logger.debug('%s: version hint in fragment: %r', |
| project_name, frag) |
| m = HASHER_HASH.match(frag) |
| if m: |
| algo, digest = m.groups() |
| else: |
| algo, digest = None, None |
| origpath = path |
| if path and path[-1] == '/': # pragma: no cover |
| path = path[:-1] |
| if path.endswith('.whl'): |
| try: |
| wheel = Wheel(path) |
| if not is_compatible(wheel, self.wheel_tags): |
| logger.debug('Wheel not compatible: %s', path) |
| else: |
| if project_name is None: |
| include = True |
| else: |
| include = same_project(wheel.name, project_name) |
| if include: |
| result = { |
| 'name': wheel.name, |
| 'version': wheel.version, |
| 'filename': wheel.filename, |
| 'url': urlunparse((scheme, netloc, origpath, |
| params, query, '')), |
| 'python-version': ', '.join( |
| ['.'.join(list(v[2:])) for v in wheel.pyver]), |
| } |
| except Exception as e: # pragma: no cover |
| logger.warning('invalid path for wheel: %s', path) |
| elif not path.endswith(self.downloadable_extensions): # pragma: no cover |
| logger.debug('Not downloadable: %s', path) |
| else: # downloadable extension |
| path = filename = posixpath.basename(path) |
| for ext in self.downloadable_extensions: |
| if path.endswith(ext): |
| path = path[:-len(ext)] |
| t = self.split_filename(path, project_name) |
| if not t: # pragma: no cover |
| logger.debug('No match for project/version: %s', path) |
| else: |
| name, version, pyver = t |
| if not project_name or same_project(project_name, name): |
| result = { |
| 'name': name, |
| 'version': version, |
| 'filename': filename, |
| 'url': urlunparse((scheme, netloc, origpath, |
| params, query, '')), |
| #'packagetype': 'sdist', |
| } |
| if pyver: # pragma: no cover |
| result['python-version'] = pyver |
| break |
| if result and algo: |
| result['%s_digest' % algo] = digest |
| return result |
| |
| def _get_digest(self, info): |
| """ |
| Get a digest from a dictionary by looking at a "digests" dictionary |
| or keys of the form 'algo_digest'. |
| |
| Returns a 2-tuple (algo, digest) if found, else None. Currently |
| looks only for SHA256, then MD5. |
| """ |
| result = None |
| if 'digests' in info: |
| digests = info['digests'] |
| for algo in ('sha256', 'md5'): |
| if algo in digests: |
| result = (algo, digests[algo]) |
| break |
| if not result: |
| for algo in ('sha256', 'md5'): |
| key = '%s_digest' % algo |
| if key in info: |
| result = (algo, info[key]) |
| break |
| return result |
| |
| def _update_version_data(self, result, info): |
| """ |
| Update a result dictionary (the final result from _get_project) with a |
| dictionary for a specific version, which typically holds information |
| gleaned from a filename or URL for an archive for the distribution. |
| """ |
| name = info.pop('name') |
| version = info.pop('version') |
| if version in result: |
| dist = result[version] |
| md = dist.metadata |
| else: |
| dist = make_dist(name, version, scheme=self.scheme) |
| md = dist.metadata |
| dist.digest = digest = self._get_digest(info) |
| url = info['url'] |
| result['digests'][url] = digest |
| if md.source_url != info['url']: |
| md.source_url = self.prefer_url(md.source_url, url) |
| result['urls'].setdefault(version, set()).add(url) |
| dist.locator = self |
| result[version] = dist |
| |
| def locate(self, requirement, prereleases=False): |
| """ |
| Find the most recent distribution which matches the given |
| requirement. |
| |
| :param requirement: A requirement of the form 'foo (1.0)' or perhaps |
| 'foo (>= 1.0, < 2.0, != 1.3)' |
| :param prereleases: If ``True``, allow pre-release versions |
| to be located. Otherwise, pre-release versions |
| are not returned. |
| :return: A :class:`Distribution` instance, or ``None`` if no such |
| distribution could be located. |
| """ |
| result = None |
| r = parse_requirement(requirement) |
| if r is None: # pragma: no cover |
| raise DistlibException('Not a valid requirement: %r' % requirement) |
| scheme = get_scheme(self.scheme) |
| self.matcher = matcher = scheme.matcher(r.requirement) |
| logger.debug('matcher: %s (%s)', matcher, type(matcher).__name__) |
| versions = self.get_project(r.name) |
| if len(versions) > 2: # urls and digests keys are present |
| # sometimes, versions are invalid |
| slist = [] |
| vcls = matcher.version_class |
| for k in versions: |
| if k in ('urls', 'digests'): |
| continue |
| try: |
| if not matcher.match(k): |
| pass # logger.debug('%s did not match %r', matcher, k) |
| else: |
| if prereleases or not vcls(k).is_prerelease: |
| slist.append(k) |
| # else: |
| # logger.debug('skipping pre-release ' |
| # 'version %s of %s', k, matcher.name) |
| except Exception: # pragma: no cover |
| logger.warning('error matching %s with %r', matcher, k) |
| pass # slist.append(k) |
| if len(slist) > 1: |
| slist = sorted(slist, key=scheme.key) |
| if slist: |
| logger.debug('sorted list: %s', slist) |
| version = slist[-1] |
| result = versions[version] |
| if result: |
| if r.extras: |
| result.extras = r.extras |
| result.download_urls = versions.get('urls', {}).get(version, set()) |
| d = {} |
| sd = versions.get('digests', {}) |
| for url in result.download_urls: |
| if url in sd: # pragma: no cover |
| d[url] = sd[url] |
| result.digests = d |
| self.matcher = None |
| return result |
| |
| |
| class PyPIRPCLocator(Locator): |
| """ |
| This locator uses XML-RPC to locate distributions. It therefore |
| cannot be used with simple mirrors (that only mirror file content). |
| """ |
| def __init__(self, url, **kwargs): |
| """ |
| Initialise an instance. |
| |
| :param url: The URL to use for XML-RPC. |
| :param kwargs: Passed to the superclass constructor. |
| """ |
| super(PyPIRPCLocator, self).__init__(**kwargs) |
| self.base_url = url |
| self.client = ServerProxy(url, timeout=3.0) |
| |
| def get_distribution_names(self): |
| """ |
| Return all the distribution names known to this locator. |
| """ |
| return set(self.client.list_packages()) |
| |
| def _get_project(self, name): |
| result = {'urls': {}, 'digests': {}} |
| versions = self.client.package_releases(name, True) |
| for v in versions: |
| urls = self.client.release_urls(name, v) |
| data = self.client.release_data(name, v) |
| metadata = Metadata(scheme=self.scheme) |
| metadata.name = data['name'] |
| metadata.version = data['version'] |
| metadata.license = data.get('license') |
| metadata.keywords = data.get('keywords', []) |
| metadata.summary = data.get('summary') |
| dist = Distribution(metadata) |
| if urls: |
| info = urls[0] |
| metadata.source_url = info['url'] |
| dist.digest = self._get_digest(info) |
| dist.locator = self |
| result[v] = dist |
| for info in urls: |
| url = info['url'] |
| digest = self._get_digest(info) |
| result['urls'].setdefault(v, set()).add(url) |
| result['digests'][url] = digest |
| return result |
| |
| class PyPIJSONLocator(Locator): |
| """ |
| This locator uses PyPI's JSON interface. It's very limited in functionality |
| and probably not worth using. |
| """ |
| def __init__(self, url, **kwargs): |
| super(PyPIJSONLocator, self).__init__(**kwargs) |
| self.base_url = ensure_slash(url) |
| |
| def get_distribution_names(self): |
| """ |
| Return all the distribution names known to this locator. |
| """ |
| raise NotImplementedError('Not available from this locator') |
| |
| def _get_project(self, name): |
| result = {'urls': {}, 'digests': {}} |
| url = urljoin(self.base_url, '%s/json' % quote(name)) |
| try: |
| resp = self.opener.open(url) |
| data = resp.read().decode() # for now |
| d = json.loads(data) |
| md = Metadata(scheme=self.scheme) |
| data = d['info'] |
| md.name = data['name'] |
| md.version = data['version'] |
| md.license = data.get('license') |
| md.keywords = data.get('keywords', []) |
| md.summary = data.get('summary') |
| dist = Distribution(md) |
| dist.locator = self |
| urls = d['urls'] |
| result[md.version] = dist |
| for info in d['urls']: |
| url = info['url'] |
| dist.download_urls.add(url) |
| dist.digests[url] = self._get_digest(info) |
| result['urls'].setdefault(md.version, set()).add(url) |
| result['digests'][url] = self._get_digest(info) |
| # Now get other releases |
| for version, infos in d['releases'].items(): |
| if version == md.version: |
| continue # already done |
| omd = Metadata(scheme=self.scheme) |
| omd.name = md.name |
| omd.version = version |
| odist = Distribution(omd) |
| odist.locator = self |
| result[version] = odist |
| for info in infos: |
| url = info['url'] |
| odist.download_urls.add(url) |
| odist.digests[url] = self._get_digest(info) |
| result['urls'].setdefault(version, set()).add(url) |
| result['digests'][url] = self._get_digest(info) |
| # for info in urls: |
| # md.source_url = info['url'] |
| # dist.digest = self._get_digest(info) |
| # dist.locator = self |
| # for info in urls: |
| # url = info['url'] |
| # result['urls'].setdefault(md.version, set()).add(url) |
| # result['digests'][url] = self._get_digest(info) |
| except Exception as e: |
| self.errors.put(text_type(e)) |
| logger.exception('JSON fetch failed: %s', e) |
| return result |
| |
| |
| class Page(object): |
| """ |
| This class represents a scraped HTML page. |
| """ |
| # The following slightly hairy-looking regex just looks for the contents of |
| # an anchor link, which has an attribute "href" either immediately preceded |
| # or immediately followed by a "rel" attribute. The attribute values can be |
| # declared with double quotes, single quotes or no quotes - which leads to |
| # the length of the expression. |
| _href = re.compile(""" |
| (rel\\s*=\\s*(?:"(?P<rel1>[^"]*)"|'(?P<rel2>[^']*)'|(?P<rel3>[^>\\s\n]*))\\s+)? |
| href\\s*=\\s*(?:"(?P<url1>[^"]*)"|'(?P<url2>[^']*)'|(?P<url3>[^>\\s\n]*)) |
| (\\s+rel\\s*=\\s*(?:"(?P<rel4>[^"]*)"|'(?P<rel5>[^']*)'|(?P<rel6>[^>\\s\n]*)))? |
| """, re.I | re.S | re.X) |
| _base = re.compile(r"""<base\s+href\s*=\s*['"]?([^'">]+)""", re.I | re.S) |
| |
| def __init__(self, data, url): |
| """ |
| Initialise an instance with the Unicode page contents and the URL they |
| came from. |
| """ |
| self.data = data |
| self.base_url = self.url = url |
| m = self._base.search(self.data) |
| if m: |
| self.base_url = m.group(1) |
| |
| _clean_re = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I) |
| |
| @cached_property |
| def links(self): |
| """ |
| Return the URLs of all the links on a page together with information |
| about their "rel" attribute, for determining which ones to treat as |
| downloads and which ones to queue for further scraping. |
| """ |
| def clean(url): |
| "Tidy up an URL." |
| scheme, netloc, path, params, query, frag = urlparse(url) |
| return urlunparse((scheme, netloc, quote(path), |
| params, query, frag)) |
| |
| result = set() |
| for match in self._href.finditer(self.data): |
| d = match.groupdict('') |
| rel = (d['rel1'] or d['rel2'] or d['rel3'] or |
| d['rel4'] or d['rel5'] or d['rel6']) |
| url = d['url1'] or d['url2'] or d['url3'] |
| url = urljoin(self.base_url, url) |
| url = unescape(url) |
| url = self._clean_re.sub(lambda m: '%%%2x' % ord(m.group(0)), url) |
| result.add((url, rel)) |
| # We sort the result, hoping to bring the most recent versions |
| # to the front |
| result = sorted(result, key=lambda t: t[0], reverse=True) |
| return result |
| |
| |
| class SimpleScrapingLocator(Locator): |
| """ |
| A locator which scrapes HTML pages to locate downloads for a distribution. |
| This runs multiple threads to do the I/O; performance is at least as good |
| as pip's PackageFinder, which works in an analogous fashion. |
| """ |
| |
| # These are used to deal with various Content-Encoding schemes. |
| decoders = { |
| 'deflate': zlib.decompress, |
| 'gzip': lambda b: gzip.GzipFile(fileobj=BytesIO(b)).read(), |
| 'none': lambda b: b, |
| } |
| |
| def __init__(self, url, timeout=None, num_workers=10, **kwargs): |
| """ |
| Initialise an instance. |
| :param url: The root URL to use for scraping. |
| :param timeout: The timeout, in seconds, to be applied to requests. |
| This defaults to ``None`` (no timeout specified). |
| :param num_workers: The number of worker threads you want to do I/O, |
| This defaults to 10. |
| :param kwargs: Passed to the superclass. |
| """ |
| super(SimpleScrapingLocator, self).__init__(**kwargs) |
| self.base_url = ensure_slash(url) |
| self.timeout = timeout |
| self._page_cache = {} |
| self._seen = set() |
| self._to_fetch = queue.Queue() |
| self._bad_hosts = set() |
| self.skip_externals = False |
| self.num_workers = num_workers |
| self._lock = threading.RLock() |
| # See issue #45: we need to be resilient when the locator is used |
| # in a thread, e.g. with concurrent.futures. We can't use self._lock |
| # as it is for coordinating our internal threads - the ones created |
| # in _prepare_threads. |
| self._gplock = threading.RLock() |
| self.platform_check = False # See issue #112 |
| |
| def _prepare_threads(self): |
| """ |
| Threads are created only when get_project is called, and terminate |
| before it returns. They are there primarily to parallelise I/O (i.e. |
| fetching web pages). |
| """ |
| self._threads = [] |
| for i in range(self.num_workers): |
| t = threading.Thread(target=self._fetch) |
| t.daemon = True |
| t.start() |
| self._threads.append(t) |
| |
| def _wait_threads(self): |
| """ |
| Tell all the threads to terminate (by sending a sentinel value) and |
| wait for them to do so. |
| """ |
| # Note that you need two loops, since you can't say which |
| # thread will get each sentinel |
| for t in self._threads: |
| self._to_fetch.put(None) # sentinel |
| for t in self._threads: |
| t.join() |
| self._threads = [] |
| |
| def _get_project(self, name): |
| result = {'urls': {}, 'digests': {}} |
| with self._gplock: |
| self.result = result |
| self.project_name = name |
| url = urljoin(self.base_url, '%s/' % quote(name)) |
| self._seen.clear() |
| self._page_cache.clear() |
| self._prepare_threads() |
| try: |
| logger.debug('Queueing %s', url) |
| self._to_fetch.put(url) |
| self._to_fetch.join() |
| finally: |
| self._wait_threads() |
| del self.result |
| return result |
| |
| platform_dependent = re.compile(r'\b(linux_(i\d86|x86_64|arm\w+)|' |
| r'win(32|_amd64)|macosx_?\d+)\b', re.I) |
| |
| def _is_platform_dependent(self, url): |
| """ |
| Does an URL refer to a platform-specific download? |
| """ |
| return self.platform_dependent.search(url) |
| |
| def _process_download(self, url): |
| """ |
| See if an URL is a suitable download for a project. |
| |
| If it is, register information in the result dictionary (for |
| _get_project) about the specific version it's for. |
| |
| Note that the return value isn't actually used other than as a boolean |
| value. |
| """ |
| if self.platform_check and self._is_platform_dependent(url): |
| info = None |
| else: |
| info = self.convert_url_to_download_info(url, self.project_name) |
| logger.debug('process_download: %s -> %s', url, info) |
| if info: |
| with self._lock: # needed because self.result is shared |
| self._update_version_data(self.result, info) |
| return info |
| |
| def _should_queue(self, link, referrer, rel): |
| """ |
| Determine whether a link URL from a referring page and with a |
| particular "rel" attribute should be queued for scraping. |
| """ |
| scheme, netloc, path, _, _, _ = urlparse(link) |
| if path.endswith(self.source_extensions + self.binary_extensions + |
| self.excluded_extensions): |
| result = False |
| elif self.skip_externals and not link.startswith(self.base_url): |
| result = False |
| elif not referrer.startswith(self.base_url): |
| result = False |
| elif rel not in ('homepage', 'download'): |
| result = False |
| elif scheme not in ('http', 'https', 'ftp'): |
| result = False |
| elif self._is_platform_dependent(link): |
| result = False |
| else: |
| host = netloc.split(':', 1)[0] |
| if host.lower() == 'localhost': |
| result = False |
| else: |
| result = True |
| logger.debug('should_queue: %s (%s) from %s -> %s', link, rel, |
| referrer, result) |
| return result |
| |
| def _fetch(self): |
| """ |
| Get a URL to fetch from the work queue, get the HTML page, examine its |
| links for download candidates and candidates for further scraping. |
| |
| This is a handy method to run in a thread. |
| """ |
| while True: |
| url = self._to_fetch.get() |
| try: |
| if url: |
| page = self.get_page(url) |
| if page is None: # e.g. after an error |
| continue |
| for link, rel in page.links: |
| if link not in self._seen: |
| try: |
| self._seen.add(link) |
| if (not self._process_download(link) and |
| self._should_queue(link, url, rel)): |
| logger.debug('Queueing %s from %s', link, url) |
| self._to_fetch.put(link) |
| except MetadataInvalidError: # e.g. invalid versions |
| pass |
| except Exception as e: # pragma: no cover |
| self.errors.put(text_type(e)) |
| finally: |
| # always do this, to avoid hangs :-) |
| self._to_fetch.task_done() |
| if not url: |
| #logger.debug('Sentinel seen, quitting.') |
| break |
| |
| def get_page(self, url): |
| """ |
| Get the HTML for an URL, possibly from an in-memory cache. |
| |
| XXX TODO Note: this cache is never actually cleared. It's assumed that |
| the data won't get stale over the lifetime of a locator instance (not |
| necessarily true for the default_locator). |
| """ |
| # http://peak.telecommunity.com/DevCenter/EasyInstall#package-index-api |
| scheme, netloc, path, _, _, _ = urlparse(url) |
| if scheme == 'file' and os.path.isdir(url2pathname(path)): |
| url = urljoin(ensure_slash(url), 'index.html') |
| |
| if url in self._page_cache: |
| result = self._page_cache[url] |
| logger.debug('Returning %s from cache: %s', url, result) |
| else: |
| host = netloc.split(':', 1)[0] |
| result = None |
| if host in self._bad_hosts: |
| logger.debug('Skipping %s due to bad host %s', url, host) |
| else: |
| req = Request(url, headers={'Accept-encoding': 'identity'}) |
| try: |
| logger.debug('Fetching %s', url) |
| resp = self.opener.open(req, timeout=self.timeout) |
| logger.debug('Fetched %s', url) |
| headers = resp.info() |
| content_type = headers.get('Content-Type', '') |
| if HTML_CONTENT_TYPE.match(content_type): |
| final_url = resp.geturl() |
| data = resp.read() |
| encoding = headers.get('Content-Encoding') |
| if encoding: |
| decoder = self.decoders[encoding] # fail if not found |
| data = decoder(data) |
| encoding = 'utf-8' |
| m = CHARSET.search(content_type) |
| if m: |
| encoding = m.group(1) |
| try: |
| data = data.decode(encoding) |
| except UnicodeError: # pragma: no cover |
| data = data.decode('latin-1') # fallback |
| result = Page(data, final_url) |
| self._page_cache[final_url] = result |
| except HTTPError as e: |
| if e.code != 404: |
| logger.exception('Fetch failed: %s: %s', url, e) |
| except URLError as e: # pragma: no cover |
| logger.exception('Fetch failed: %s: %s', url, e) |
| with self._lock: |
| self._bad_hosts.add(host) |
| except Exception as e: # pragma: no cover |
| logger.exception('Fetch failed: %s: %s', url, e) |
| finally: |
| self._page_cache[url] = result # even if None (failure) |
| return result |
| |
| _distname_re = re.compile('<a href=[^>]*>([^<]+)<') |
| |
| def get_distribution_names(self): |
| """ |
| Return all the distribution names known to this locator. |
| """ |
| result = set() |
| page = self.get_page(self.base_url) |
| if not page: |
| raise DistlibException('Unable to get %s' % self.base_url) |
| for match in self._distname_re.finditer(page.data): |
| result.add(match.group(1)) |
| return result |
| |
| class DirectoryLocator(Locator): |
| """ |
| This class locates distributions in a directory tree. |
| """ |
| |
| def __init__(self, path, **kwargs): |
| """ |
| Initialise an instance. |
| :param path: The root of the directory tree to search. |
| :param kwargs: Passed to the superclass constructor, |
| except for: |
| * recursive - if True (the default), subdirectories are |
| recursed into. If False, only the top-level directory |
| is searched, |
| """ |
| self.recursive = kwargs.pop('recursive', True) |
| super(DirectoryLocator, self).__init__(**kwargs) |
| path = os.path.abspath(path) |
| if not os.path.isdir(path): # pragma: no cover |
| raise DistlibException('Not a directory: %r' % path) |
| self.base_dir = path |
| |
| def should_include(self, filename, parent): |
| """ |
| Should a filename be considered as a candidate for a distribution |
| archive? As well as the filename, the directory which contains it |
| is provided, though not used by the current implementation. |
| """ |
| return filename.endswith(self.downloadable_extensions) |
| |
| def _get_project(self, name): |
| result = {'urls': {}, 'digests': {}} |
| for root, dirs, files in os.walk(self.base_dir): |
| for fn in files: |
| if self.should_include(fn, root): |
| fn = os.path.join(root, fn) |
| url = urlunparse(('file', '', |
| pathname2url(os.path.abspath(fn)), |
| '', '', '')) |
| info = self.convert_url_to_download_info(url, name) |
| if info: |
| self._update_version_data(result, info) |
| if not self.recursive: |
| break |
| return result |
| |
| def get_distribution_names(self): |
| """ |
| Return all the distribution names known to this locator. |
| """ |
| result = set() |
| for root, dirs, files in os.walk(self.base_dir): |
| for fn in files: |
| if self.should_include(fn, root): |
| fn = os.path.join(root, fn) |
| url = urlunparse(('file', '', |
| pathname2url(os.path.abspath(fn)), |
| '', '', '')) |
| info = self.convert_url_to_download_info(url, None) |
| if info: |
| result.add(info['name']) |
| if not self.recursive: |
| break |
| return result |
| |
| class JSONLocator(Locator): |
| """ |
| This locator uses special extended metadata (not available on PyPI) and is |
| the basis of performant dependency resolution in distlib. Other locators |
| require archive downloads before dependencies can be determined! As you |
| might imagine, that can be slow. |
| """ |
| def get_distribution_names(self): |
| """ |
| Return all the distribution names known to this locator. |
| """ |
| raise NotImplementedError('Not available from this locator') |
| |
| def _get_project(self, name): |
| result = {'urls': {}, 'digests': {}} |
| data = get_project_data(name) |
| if data: |
| for info in data.get('files', []): |
| if info['ptype'] != 'sdist' or info['pyversion'] != 'source': |
| continue |
| # We don't store summary in project metadata as it makes |
| # the data bigger for no benefit during dependency |
| # resolution |
| dist = make_dist(data['name'], info['version'], |
| summary=data.get('summary', |
| 'Placeholder for summary'), |
| scheme=self.scheme) |
| md = dist.metadata |
| md.source_url = info['url'] |
| # TODO SHA256 digest |
| if 'digest' in info and info['digest']: |
| dist.digest = ('md5', info['digest']) |
| md.dependencies = info.get('requirements', {}) |
| dist.exports = info.get('exports', {}) |
| result[dist.version] = dist |
| result['urls'].setdefault(dist.version, set()).add(info['url']) |
| return result |
| |
| class DistPathLocator(Locator): |
| """ |
| This locator finds installed distributions in a path. It can be useful for |
| adding to an :class:`AggregatingLocator`. |
| """ |
| def __init__(self, distpath, **kwargs): |
| """ |
| Initialise an instance. |
| |
| :param distpath: A :class:`DistributionPath` instance to search. |
| """ |
| super(DistPathLocator, self).__init__(**kwargs) |
| assert isinstance(distpath, DistributionPath) |
| self.distpath = distpath |
| |
| def _get_project(self, name): |
| dist = self.distpath.get_distribution(name) |
| if dist is None: |
| result = {'urls': {}, 'digests': {}} |
| else: |
| result = { |
| dist.version: dist, |
| 'urls': {dist.version: set([dist.source_url])}, |
| 'digests': {dist.version: set([None])} |
| } |
| return result |
| |
| |
| class AggregatingLocator(Locator): |
| """ |
| This class allows you to chain and/or merge a list of locators. |
| """ |
| def __init__(self, *locators, **kwargs): |
| """ |
| Initialise an instance. |
| |
| :param locators: The list of locators to search. |
| :param kwargs: Passed to the superclass constructor, |
| except for: |
| * merge - if False (the default), the first successful |
| search from any of the locators is returned. If True, |
| the results from all locators are merged (this can be |
| slow). |
| """ |
| self.merge = kwargs.pop('merge', False) |
| self.locators = locators |
| super(AggregatingLocator, self).__init__(**kwargs) |
| |
| def clear_cache(self): |
| super(AggregatingLocator, self).clear_cache() |
| for locator in self.locators: |
| locator.clear_cache() |
| |
| def _set_scheme(self, value): |
| self._scheme = value |
| for locator in self.locators: |
| locator.scheme = value |
| |
| scheme = property(Locator.scheme.fget, _set_scheme) |
| |
| def _get_project(self, name): |
| result = {} |
| for locator in self.locators: |
| d = locator.get_project(name) |
| if d: |
| if self.merge: |
| files = result.get('urls', {}) |
| digests = result.get('digests', {}) |
| # next line could overwrite result['urls'], result['digests'] |
| result.update(d) |
| df = result.get('urls') |
| if files and df: |
| for k, v in files.items(): |
| if k in df: |
| df[k] |= v |
| else: |
| df[k] = v |
| dd = result.get('digests') |
| if digests and dd: |
| dd.update(digests) |
| else: |
| # See issue #18. If any dists are found and we're looking |
| # for specific constraints, we only return something if |
| # a match is found. For example, if a DirectoryLocator |
| # returns just foo (1.0) while we're looking for |
| # foo (>= 2.0), we'll pretend there was nothing there so |
| # that subsequent locators can be queried. Otherwise we |
| # would just return foo (1.0) which would then lead to a |
| # failure to find foo (>= 2.0), because other locators |
| # weren't searched. Note that this only matters when |
| # merge=False. |
| if self.matcher is None: |
| found = True |
| else: |
| found = False |
| for k in d: |
| if self.matcher.match(k): |
| found = True |
| break |
| if found: |
| result = d |
| break |
| return result |
| |
| def get_distribution_names(self): |
| """ |
| Return all the distribution names known to this locator. |
| """ |
| result = set() |
| for locator in self.locators: |
| try: |
| result |= locator.get_distribution_names() |
| except NotImplementedError: |
| pass |
| return result |
| |
| |
| # We use a legacy scheme simply because most of the dists on PyPI use legacy |
| # versions which don't conform to PEP 440. |
| default_locator = AggregatingLocator( |
| # JSONLocator(), # don't use as PEP 426 is withdrawn |
| SimpleScrapingLocator('https://pypi.org/simple/', |
| timeout=3.0), |
| scheme='legacy') |
| |
| locate = default_locator.locate |
| |
| |
| class DependencyFinder(object): |
| """ |
| Locate dependencies for distributions. |
| """ |
| |
| def __init__(self, locator=None): |
| """ |
| Initialise an instance, using the specified locator |
| to locate distributions. |
| """ |
| self.locator = locator or default_locator |
| self.scheme = get_scheme(self.locator.scheme) |
| |
| def add_distribution(self, dist): |
| """ |
| Add a distribution to the finder. This will update internal information |
| about who provides what. |
| :param dist: The distribution to add. |
| """ |
| logger.debug('adding distribution %s', dist) |
| name = dist.key |
| self.dists_by_name[name] = dist |
| self.dists[(name, dist.version)] = dist |
| for p in dist.provides: |
| name, version = parse_name_and_version(p) |
| logger.debug('Add to provided: %s, %s, %s', name, version, dist) |
| self.provided.setdefault(name, set()).add((version, dist)) |
| |
| def remove_distribution(self, dist): |
| """ |
| Remove a distribution from the finder. This will update internal |
| information about who provides what. |
| :param dist: The distribution to remove. |
| """ |
| logger.debug('removing distribution %s', dist) |
| name = dist.key |
| del self.dists_by_name[name] |
| del self.dists[(name, dist.version)] |
| for p in dist.provides: |
| name, version = parse_name_and_version(p) |
| logger.debug('Remove from provided: %s, %s, %s', name, version, dist) |
| s = self.provided[name] |
| s.remove((version, dist)) |
| if not s: |
| del self.provided[name] |
| |
| def get_matcher(self, reqt): |
| """ |
| Get a version matcher for a requirement. |
| :param reqt: The requirement |
| :type reqt: str |
| :return: A version matcher (an instance of |
| :class:`distlib.version.Matcher`). |
| """ |
| try: |
| matcher = self.scheme.matcher(reqt) |
| except UnsupportedVersionError: # pragma: no cover |
| # XXX compat-mode if cannot read the version |
| name = reqt.split()[0] |
| matcher = self.scheme.matcher(name) |
| return matcher |
| |
| def find_providers(self, reqt): |
| """ |
| Find the distributions which can fulfill a requirement. |
| |
| :param reqt: The requirement. |
| :type reqt: str |
| :return: A set of distribution which can fulfill the requirement. |
| """ |
| matcher = self.get_matcher(reqt) |
| name = matcher.key # case-insensitive |
| result = set() |
| provided = self.provided |
| if name in provided: |
| for version, provider in provided[name]: |
| try: |
| match = matcher.match(version) |
| except UnsupportedVersionError: |
| match = False |
| |
| if match: |
| result.add(provider) |
| break |
| return result |
| |
| def try_to_replace(self, provider, other, problems): |
| """ |
| Attempt to replace one provider with another. This is typically used |
| when resolving dependencies from multiple sources, e.g. A requires |
| (B >= 1.0) while C requires (B >= 1.1). |
| |
| For successful replacement, ``provider`` must meet all the requirements |
| which ``other`` fulfills. |
| |
| :param provider: The provider we are trying to replace with. |
| :param other: The provider we're trying to replace. |
| :param problems: If False is returned, this will contain what |
| problems prevented replacement. This is currently |
| a tuple of the literal string 'cantreplace', |
| ``provider``, ``other`` and the set of requirements |
| that ``provider`` couldn't fulfill. |
| :return: True if we can replace ``other`` with ``provider``, else |
| False. |
| """ |
| rlist = self.reqts[other] |
| unmatched = set() |
| for s in rlist: |
| matcher = self.get_matcher(s) |
| if not matcher.match(provider.version): |
| unmatched.add(s) |
| if unmatched: |
| # can't replace other with provider |
| problems.add(('cantreplace', provider, other, |
| frozenset(unmatched))) |
| result = False |
| else: |
| # can replace other with provider |
| self.remove_distribution(other) |
| del self.reqts[other] |
| for s in rlist: |
| self.reqts.setdefault(provider, set()).add(s) |
| self.add_distribution(provider) |
| result = True |
| return result |
| |
| def find(self, requirement, meta_extras=None, prereleases=False): |
| """ |
| Find a distribution and all distributions it depends on. |
| |
| :param requirement: The requirement specifying the distribution to |
| find, or a Distribution instance. |
| :param meta_extras: A list of meta extras such as :test:, :build: and |
| so on. |
| :param prereleases: If ``True``, allow pre-release versions to be |
| returned - otherwise, don't return prereleases |
| unless they're all that's available. |
| |
| Return a set of :class:`Distribution` instances and a set of |
| problems. |
| |
| The distributions returned should be such that they have the |
| :attr:`required` attribute set to ``True`` if they were |
| from the ``requirement`` passed to ``find()``, and they have the |
| :attr:`build_time_dependency` attribute set to ``True`` unless they |
| are post-installation dependencies of the ``requirement``. |
| |
| The problems should be a tuple consisting of the string |
| ``'unsatisfied'`` and the requirement which couldn't be satisfied |
| by any distribution known to the locator. |
| """ |
| |
| self.provided = {} |
| self.dists = {} |
| self.dists_by_name = {} |
| self.reqts = {} |
| |
| meta_extras = set(meta_extras or []) |
| if ':*:' in meta_extras: |
| meta_extras.remove(':*:') |
| # :meta: and :run: are implicitly included |
| meta_extras |= set([':test:', ':build:', ':dev:']) |
| |
| if isinstance(requirement, Distribution): |
| dist = odist = requirement |
| logger.debug('passed %s as requirement', odist) |
| else: |
| dist = odist = self.locator.locate(requirement, |
| prereleases=prereleases) |
| if dist is None: |
| raise DistlibException('Unable to locate %r' % requirement) |
| logger.debug('located %s', odist) |
| dist.requested = True |
| problems = set() |
| todo = set([dist]) |
| install_dists = set([odist]) |
| while todo: |
| dist = todo.pop() |
| name = dist.key # case-insensitive |
| if name not in self.dists_by_name: |
| self.add_distribution(dist) |
| else: |
| #import pdb; pdb.set_trace() |
| other = self.dists_by_name[name] |
| if other != dist: |
| self.try_to_replace(dist, other, problems) |
| |
| ireqts = dist.run_requires | dist.meta_requires |
| sreqts = dist.build_requires |
| ereqts = set() |
| if meta_extras and dist in install_dists: |
| for key in ('test', 'build', 'dev'): |
| e = ':%s:' % key |
| if e in meta_extras: |
| ereqts |= getattr(dist, '%s_requires' % key) |
| all_reqts = ireqts | sreqts | ereqts |
| for r in all_reqts: |
| providers = self.find_providers(r) |
| if not providers: |
| logger.debug('No providers found for %r', r) |
| provider = self.locator.locate(r, prereleases=prereleases) |
| # If no provider is found and we didn't consider |
| # prereleases, consider them now. |
| if provider is None and not prereleases: |
| provider = self.locator.locate(r, prereleases=True) |
| if provider is None: |
| logger.debug('Cannot satisfy %r', r) |
| problems.add(('unsatisfied', r)) |
| else: |
| n, v = provider.key, provider.version |
| if (n, v) not in self.dists: |
| todo.add(provider) |
| providers.add(provider) |
| if r in ireqts and dist in install_dists: |
| install_dists.add(provider) |
| logger.debug('Adding %s to install_dists', |
| provider.name_and_version) |
| for p in providers: |
| name = p.key |
| if name not in self.dists_by_name: |
| self.reqts.setdefault(p, set()).add(r) |
| else: |
| other = self.dists_by_name[name] |
| if other != p: |
| # see if other can be replaced by p |
| self.try_to_replace(p, other, problems) |
| |
| dists = set(self.dists.values()) |
| for dist in dists: |
| dist.build_time_dependency = dist not in install_dists |
| if dist.build_time_dependency: |
| logger.debug('%s is a build-time dependency only.', |
| dist.name_and_version) |
| logger.debug('find done for %s', odist) |
| return dists, problems |
