356 lines
10 KiB
Python
356 lines
10 KiB
Python
|
# -*- coding: utf-8 -*-
|
||
|
|
#
|
||
|
|
# Copyright (C) 2013-2017 Vinay Sajip.
|
||
|
|
# Licensed to the Python Software Foundation under a contributor agreement.
|
||
|
|
# See LICENSE.txt and CONTRIBUTORS.txt.
|
||
|
|
#
|
||
|
|
from __future__ import unicode_literals
|
||
|
|
|
||
|
|
import bisect
|
||
|
|
import io
|
||
|
|
import logging
|
||
|
|
import os
|
||
|
|
import pkgutil
|
||
|
|
import shutil
|
||
|
|
import sys
|
||
|
|
import types
|
||
|
|
import zipimport
|
||
|
|
|
||
|
|
from . import DistlibException
|
||
|
|
from .util import cached_property, get_cache_base, path_to_cache_dir, Cache
|
||
|
|
|
||
|
|
logger = logging.getLogger(__name__)
|
||
|
|
|
||
|
|
|
||
|
|
cache = None # created when needed
|
||
|
|
|
||
|
|
|
||
|
|
class ResourceCache(Cache):
|
||
|
|
def __init__(self, base=None):
|
||
|
|
if base is None:
|
||
|
|
# Use native string to avoid issues on 2.x: see Python #20140.
|
||
|
|
base = os.path.join(get_cache_base(), str('resource-cache'))
|
||
|
|
super(ResourceCache, self).__init__(base)
|
||
|
|
|
||
|
|
def is_stale(self, resource, path):
|
||
|
|
"""
|
||
|
|
Is the cache stale for the given resource?
|
||
|
|
|
||
|
|
:param resource: The :class:`Resource` being cached.
|
||
|
|
:param path: The path of the resource in the cache.
|
||
|
|
:return: True if the cache is stale.
|
||
|
|
"""
|
||
|
|
# Cache invalidation is a hard problem :-)
|
||
|
|
return True
|
||
|
|
|
||
|
|
def get(self, resource):
|
||
|
|
"""
|
||
|
|
Get a resource into the cache,
|
||
|
|
|
||
|
|
:param resource: A :class:`Resource` instance.
|
||
|
|
:return: The pathname of the resource in the cache.
|
||
|
|
"""
|
||
|
|
prefix, path = resource.finder.get_cache_info(resource)
|
||
|
|
if prefix is None:
|
||
|
|
result = path
|
||
|
|
else:
|
||
|
|
result = os.path.join(self.base, self.prefix_to_dir(prefix), path)
|
||
|
|
dirname = os.path.dirname(result)
|
||
|
|
if not os.path.isdir(dirname):
|
||
|
|
os.makedirs(dirname)
|
||
|
|
if not os.path.exists(result):
|
||
|
|
stale = True
|
||
|
|
else:
|
||
|
|
stale = self.is_stale(resource, path)
|
||
|
|
if stale:
|
||
|
|
# write the bytes of the resource to the cache location
|
||
|
|
with open(result, 'wb') as f:
|
||
|
|
f.write(resource.bytes)
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
class ResourceBase(object):
|
||
|
|
def __init__(self, finder, name):
|
||
|
|
self.finder = finder
|
||
|
|
self.name = name
|
||
|
|
|
||
|
|
|
||
|
|
class Resource(ResourceBase):
|
||
|
|
"""
|
||
|
|
A class representing an in-package resource, such as a data file. This is
|
||
|
|
not normally instantiated by user code, but rather by a
|
||
|
|
:class:`ResourceFinder` which manages the resource.
|
||
|
|
"""
|
||
|
|
is_container = False # Backwards compatibility
|
||
|
|
|
||
|
|
def as_stream(self):
|
||
|
|
"""
|
||
|
|
Get the resource as a stream.
|
||
|
|
|
||
|
|
This is not a property to make it obvious that it returns a new stream
|
||
|
|
each time.
|
||
|
|
"""
|
||
|
|
return self.finder.get_stream(self)
|
||
|
|
|
||
|
|
@cached_property
|
||
|
|
def file_path(self):
|
||
|
|
global cache
|
||
|
|
if cache is None:
|
||
|
|
cache = ResourceCache()
|
||
|
|
return cache.get(self)
|
||
|
|
|
||
|
|
@cached_property
|
||
|
|
def bytes(self):
|
||
|
|
return self.finder.get_bytes(self)
|
||
|
|
|
||
|
|
@cached_property
|
||
|
|
def size(self):
|
||
|
|
return self.finder.get_size(self)
|
||
|
|
|
||
|
|
|
||
|
|
class ResourceContainer(ResourceBase):
|
||
|
|
is_container = True # Backwards compatibility
|
||
|
|
|
||
|
|
@cached_property
|
||
|
|
def resources(self):
|
||
|
|
return self.finder.get_resources(self)
|
||
|
|
|
||
|
|
|
||
|
|
class ResourceFinder(object):
|
||
|
|
"""
|
||
|
|
Resource finder for file system resources.
|
||
|
|
"""
|
||
|
|
|
||
|
|
if sys.platform.startswith('java'):
|
||
|
|
skipped_extensions = ('.pyc', '.pyo', '.class')
|
||
|
|
else:
|
||
|
|
skipped_extensions = ('.pyc', '.pyo')
|
||
|
|
|
||
|
|
def __init__(self, module):
|
||
|
|
self.module = module
|
||
|
|
self.loader = getattr(module, '__loader__', None)
|
||
|
|
self.base = os.path.dirname(getattr(module, '__file__', ''))
|
||
|
|
|
||
|
|
def _adjust_path(self, path):
|
||
|
|
return os.path.realpath(path)
|
||
|
|
|
||
|
|
def _make_path(self, resource_name):
|
||
|
|
# Issue #50: need to preserve type of path on Python 2.x
|
||
|
|
# like os.path._get_sep
|
||
|
|
if isinstance(resource_name, bytes): # should only happen on 2.x
|
||
|
|
sep = b'/'
|
||
|
|
else:
|
||
|
|
sep = '/'
|
||
|
|
parts = resource_name.split(sep)
|
||
|
|
parts.insert(0, self.base)
|
||
|
|
result = os.path.join(*parts)
|
||
|
|
return self._adjust_path(result)
|
||
|
|
|
||
|
|
def _find(self, path):
|
||
|
|
return os.path.exists(path)
|
||
|
|
|
||
|
|
def get_cache_info(self, resource):
|
||
|
|
return None, resource.path
|
||
|
|
|
||
|
|
def find(self, resource_name):
|
||
|
|
path = self._make_path(resource_name)
|
||
|
|
if not self._find(path):
|
||
|
|
result = None
|
||
|
|
else:
|
||
|
|
if self._is_directory(path):
|
||
|
|
result = ResourceContainer(self, resource_name)
|
||
|
|
else:
|
||
|
|
result = Resource(self, resource_name)
|
||
|
|
result.path = path
|
||
|
|
return result
|
||
|
|
|
||
|
|
def get_stream(self, resource):
|
||
|
|
return open(resource.path, 'rb')
|
||
|
|
|
||
|
|
def get_bytes(self, resource):
|
||
|
|
with open(resource.path, 'rb') as f:
|
||
|
|
return f.read()
|
||
|
|
|
||
|
|
def get_size(self, resource):
|
||
|
|
return os.path.getsize(resource.path)
|
||
|
|
|
||
|
|
def get_resources(self, resource):
|
||
|
|
def allowed(f):
|
||
|
|
return (f != '__pycache__' and not
|
||
|
|
f.endswith(self.skipped_extensions))
|
||
|
|
return set([f for f in os.listdir(resource.path) if allowed(f)])
|
||
|
|
|
||
|
|
def is_container(self, resource):
|
||
|
|
return self._is_directory(resource.path)
|
||
|
|
|
||
|
|
_is_directory = staticmethod(os.path.isdir)
|
||
|
|
|
||
|
|
def iterator(self, resource_name):
|
||
|
|
resource = self.find(resource_name)
|
||
|
|
if resource is not None:
|
||
|
|
todo = [resource]
|
||
|
|
while todo:
|
||
|
|
resource = todo.pop(0)
|
||
|
|
yield resource
|
||
|
|
if resource.is_container:
|
||
|
|
rname = resource.name
|
||
|
|
for name in resource.resources:
|
||
|
|
if not rname:
|
||
|
|
new_name = name
|
||
|
|
else:
|
||
|
|
new_name = '/'.join([rname, name])
|
||
|
|
child = self.find(new_name)
|
||
|
|
if child.is_container:
|
||
|
|
todo.append(child)
|
||
|
|
else:
|
||
|
|
yield child
|
||
|
|
|
||
|
|
|
||
|
|
class ZipResourceFinder(ResourceFinder):
|
||
|
|
"""
|
||
|
|
Resource finder for resources in .zip files.
|
||
|
|
"""
|
||
|
|
def __init__(self, module):
|
||
|
|
super(ZipResourceFinder, self).__init__(module)
|
||
|
|
archive = self.loader.archive
|
||
|
|
self.prefix_len = 1 + len(archive)
|
||
|
|
# PyPy doesn't have a _files attr on zipimporter, and you can't set one
|
||
|
|
if hasattr(self.loader, '_files'):
|
||
|
|
self._files = self.loader._files
|
||
|
|
else:
|
||
|
|
self._files = zipimport._zip_directory_cache[archive]
|
||
|
|
self.index = sorted(self._files)
|
||
|
|
|
||
|
|
def _adjust_path(self, path):
|
||
|
|
return path
|
||
|
|
|
||
|
|
def _find(self, path):
|
||
|
|
path = path[self.prefix_len:]
|
||
|
|
if path in self._files:
|
||
|
|
result = True
|
||
|
|
else:
|
||
|
|
if path and path[-1] != os.sep:
|
||
|
|
path = path + os.sep
|
||
|
|
i = bisect.bisect(self.index, path)
|
||
|
|
try:
|
||
|
|
result = self.index[i].startswith(path)
|
||
|
|
except IndexError:
|
||
|
|
result = False
|
||
|
|
if not result:
|
||
|
|
logger.debug('_find failed: %r %r', path, self.loader.prefix)
|
||
|
|
else:
|
||
|
|
logger.debug('_find worked: %r %r', path, self.loader.prefix)
|
||
|
|
return result
|
||
|
|
|
||
|
|
def get_cache_info(self, resource):
|
||
|
|
prefix = self.loader.archive
|
||
|
|
path = resource.path[1 + len(prefix):]
|
||
|
|
return prefix, path
|
||
|
|
|
||
|
|
def get_bytes(self, resource):
|
||
|
|
return self.loader.get_data(resource.path)
|
||
|
|
|
||
|
|
def get_stream(self, resource):
|
||
|
|
return io.BytesIO(self.get_bytes(resource))
|
||
|
|
|
||
|
|
def get_size(self, resource):
|
||
|
|
path = resource.path[self.prefix_len:]
|
||
|
|
return self._files[path][3]
|
||
|
|
|
||
|
|
def get_resources(self, resource):
|
||
|
|
path = resource.path[self.prefix_len:]
|
||
|
|
if path and path[-1] != os.sep:
|
||
|
|
path += os.sep
|
||
|
|
plen = len(path)
|
||
|
|
result = set()
|
||
|
|
i = bisect.bisect(self.index, path)
|
||
|
|
while i < len(self.index):
|
||
|
|
if not self.index[i].startswith(path):
|
||
|
|
break
|
||
|
|
s = self.index[i][plen:]
|
||
|
|
result.add(s.split(os.sep, 1)[0]) # only immediate children
|
||
|
|
i += 1
|
||
|
|
return result
|
||
|
|
|
||
|
|
def _is_directory(self, path):
|
||
|
|
path = path[self.prefix_len:]
|
||
|
|
if path and path[-1] != os.sep:
|
||
|
|
path += os.sep
|
||
|
|
i = bisect.bisect(self.index, path)
|
||
|
|
try:
|
||
|
|
result = self.index[i].startswith(path)
|
||
|
|
except IndexError:
|
||
|
|
result = False
|
||
|
|
return result
|
||
|
|
|
||
|
|
_finder_registry = {
|
||
|
|
type(None): ResourceFinder,
|
||
|
|
zipimport.zipimporter: ZipResourceFinder
|
||
|
|
}
|
||
|
|
|
||
|
|
try:
|
||
|
|
# In Python 3.6, _frozen_importlib -> _frozen_importlib_external
|
||
|
|
try:
|
||
|
|
import _frozen_importlib_external as _fi
|
||
|
|
except ImportError:
|
||
|
|
import _frozen_importlib as _fi
|
||
|
|
_finder_registry[_fi.SourceFileLoader] = ResourceFinder
|
||
|
|
_finder_registry[_fi.FileFinder] = ResourceFinder
|
||
|
|
del _fi
|
||
|
|
except (ImportError, AttributeError):
|
||
|
|
pass
|
||
|
|
|
||
|
|
|
||
|
|
def register_finder(loader, finder_maker):
|
||
|
|
_finder_registry[type(loader)] = finder_maker
|
||
|
|
|
||
|
|
_finder_cache = {}
|
||
|
|
|
||
|
|
|
||
|
|
def finder(package):
|
||
|
|
"""
|
||
|
|
Return a resource finder for a package.
|
||
|
|
:param package: The name of the package.
|
||
|
|
:return: A :class:`ResourceFinder` instance for the package.
|
||
|
|
"""
|
||
|
|
if package in _finder_cache:
|
||
|
|
result = _finder_cache[package]
|
||
|
|
else:
|
||
|
|
if package not in sys.modules:
|
||
|
|
__import__(package)
|
||
|
|
module = sys.modules[package]
|
||
|
|
path = getattr(module, '__path__', None)
|
||
|
|
if path is None:
|
||
|
|
raise DistlibException('You cannot get a finder for a module, '
|
||
|
|
'only for a package')
|
||
|
|
loader = getattr(module, '__loader__', None)
|
||
|
|
finder_maker = _finder_registry.get(type(loader))
|
||
|
|
if finder_maker is None:
|
||
|
|
raise DistlibException('Unable to locate finder for %r' % package)
|
||
|
|
result = finder_maker(module)
|
||
|
|
_finder_cache[package] = result
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
_dummy_module = types.ModuleType(str('__dummy__'))
|
||
|
|
|
||
|
|
|
||
|
|
def finder_for_path(path):
|
||
|
|
"""
|
||
|
|
Return a resource finder for a path, which should represent a container.
|
||
|
|
|
||
|
|
:param path: The path.
|
||
|
|
:return: A :class:`ResourceFinder` instance for the path.
|
||
|
|
"""
|
||
|
|
result = None
|
||
|
|
# calls any path hooks, gets importer into cache
|
||
|
|
pkgutil.get_importer(path)
|
||
|
|
loader = sys.path_importer_cache.get(path)
|
||
|
|
finder = _finder_registry.get(type(loader))
|
||
|
|
if finder:
|
||
|
|
module = _dummy_module
|
||
|
|
module.__file__ = os.path.join(path, '')
|
||
|
|
module.__loader__ = loader
|
||
|
|
result = finder(module)
|
||
|
|
return result
|