# # Cython Top Level # from __future__ import absolute_import, print_function import os import re import sys import io if sys.version_info[:2] < (2, 7) or (3, 0) <= sys.version_info[:2] < (3, 3): sys.stderr.write("Sorry, Cython requires Python 2.7 or 3.3+, found %d.%d\n" % tuple(sys.version_info[:2])) sys.exit(1) try: from __builtin__ import basestring except ImportError: basestring = str # Do not import Parsing here, import it when needed, because Parsing imports # Nodes, which globally needs debug command line options initialized to set a # conditional metaclass. These options are processed by CmdLine called from # main() in this file. # import Parsing from . import Errors from .StringEncoding import EncodedString from .Scanning import PyrexScanner, FileSourceDescriptor from .Errors import PyrexError, CompileError, error, warning from .Symtab import ModuleScope from .. import Utils from . import Options from .Options import CompilationOptions, default_options from .CmdLine import parse_command_line from .Lexicon import (unicode_start_ch_any, unicode_continuation_ch_any, unicode_start_ch_range, unicode_continuation_ch_range) def _make_range_re(chrs): out = [] for i in range(0, len(chrs), 2): out.append(u"{0}-{1}".format(chrs[i], chrs[i+1])) return u"".join(out) # py2 version looked like r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$" module_name_pattern = u"[{0}{1}][{0}{2}{1}{3}]*".format( unicode_start_ch_any, _make_range_re(unicode_start_ch_range), unicode_continuation_ch_any, _make_range_re(unicode_continuation_ch_range)) module_name_pattern = re.compile(u"{0}(\\.{0})*$".format(module_name_pattern)) standard_include_path = os.path.abspath( os.path.join(os.path.dirname(os.path.dirname(__file__)), 'Includes')) class Context(object): # This class encapsulates the context needed for compiling # one or more Cython implementation files along with their # associated and imported declaration files. It includes # the root of the module import namespace and the list # of directories to search for include files. # # modules {string : ModuleScope} # include_directories [string] # future_directives [object] # language_level int currently 2 or 3 for Python 2/3 cython_scope = None language_level = None # warn when not set but default to Py2 def __init__(self, include_directories, compiler_directives, cpp=False, language_level=None, options=None): # cython_scope is a hack, set to False by subclasses, in order to break # an infinite loop. # Better code organization would fix it. from . import Builtin, CythonScope self.modules = {"__builtin__" : Builtin.builtin_scope} self.cython_scope = CythonScope.create_cython_scope(self) self.modules["cython"] = self.cython_scope self.include_directories = include_directories self.future_directives = set() self.compiler_directives = compiler_directives self.cpp = cpp self.options = options self.pxds = {} # full name -> node tree self._interned = {} # (type(value), value, *key_args) -> interned_value if language_level is not None: self.set_language_level(language_level) self.legacy_implicit_noexcept = self.compiler_directives.get('legacy_implicit_noexcept', False) self.gdb_debug_outputwriter = None @classmethod def from_options(cls, options): return cls(options.include_path, options.compiler_directives, options.cplus, options.language_level, options=options) def set_language_level(self, level): from .Future import print_function, unicode_literals, absolute_import, division, generator_stop future_directives = set() if level == '3str': level = 3 else: level = int(level) if level >= 3: future_directives.add(unicode_literals) if level >= 3: future_directives.update([print_function, absolute_import, division, generator_stop]) self.language_level = level self.future_directives = future_directives if level >= 3: self.modules['builtins'] = self.modules['__builtin__'] def intern_ustring(self, value, encoding=None): key = (EncodedString, value, encoding) try: return self._interned[key] except KeyError: pass value = EncodedString(value) if encoding: value.encoding = encoding self._interned[key] = value return value # pipeline creation functions can now be found in Pipeline.py def process_pxd(self, source_desc, scope, module_name): from . import Pipeline if isinstance(source_desc, FileSourceDescriptor) and source_desc._file_type == 'pyx': source = CompilationSource(source_desc, module_name, os.getcwd()) result_sink = create_default_resultobj(source, self.options) pipeline = Pipeline.create_pyx_as_pxd_pipeline(self, result_sink) result = Pipeline.run_pipeline(pipeline, source) else: pipeline = Pipeline.create_pxd_pipeline(self, scope, module_name) result = Pipeline.run_pipeline(pipeline, source_desc) return result def nonfatal_error(self, exc): return Errors.report_error(exc) def _split_qualified_name(self, qualified_name, relative_import=False): # Splits qualified_name into parts in form of 2-tuples: (PART_NAME, IS_PACKAGE). qualified_name_parts = qualified_name.split('.') last_part = qualified_name_parts.pop() qualified_name_parts = [(p, True) for p in qualified_name_parts] if last_part != '__init__': # If Last part is __init__, then it is omitted. Otherwise, we need to check whether we can find # __init__.pyx/__init__.py file to determine if last part is package or not. is_package = False for suffix in ('.py', '.pyx'): path = self.search_include_directories( qualified_name, suffix=suffix, source_pos=None, source_file_path=None, sys_path=not relative_import) if path: is_package = self._is_init_file(path) break qualified_name_parts.append((last_part, is_package)) return qualified_name_parts @staticmethod def _is_init_file(path): return os.path.basename(path) in ('__init__.pyx', '__init__.py', '__init__.pxd') if path else False @staticmethod def _check_pxd_filename(pos, pxd_pathname, qualified_name): if not pxd_pathname: return pxd_filename = os.path.basename(pxd_pathname) if '.' in qualified_name and qualified_name == os.path.splitext(pxd_filename)[0]: warning(pos, "Dotted filenames ('%s') are deprecated." " Please use the normal Python package directory layout." % pxd_filename, level=1) def find_module(self, module_name, from_module=None, pos=None, need_pxd=1, absolute_fallback=True, relative_import=False): # Finds and returns the module scope corresponding to # the given relative or absolute module name. If this # is the first time the module has been requested, finds # the corresponding .pxd file and process it. # If from_module is not None, it must be a module scope, # and the module will first be searched for relative to # that module, provided its name is not a dotted name. debug_find_module = 0 if debug_find_module: print("Context.find_module: module_name = %s, from_module = %s, pos = %s, need_pxd = %s" % ( module_name, from_module, pos, need_pxd)) scope = None pxd_pathname = None if from_module: if module_name: # from .module import ... qualified_name = from_module.qualify_name(module_name) else: # from . import ... qualified_name = from_module.qualified_name scope = from_module from_module = None else: qualified_name = module_name if not module_name_pattern.match(qualified_name): raise CompileError(pos or (module_name, 0, 0), u"'%s' is not a valid module name" % module_name) if from_module: if debug_find_module: print("...trying relative import") scope = from_module.lookup_submodule(module_name) if not scope: pxd_pathname = self.find_pxd_file(qualified_name, pos, sys_path=not relative_import) self._check_pxd_filename(pos, pxd_pathname, qualified_name) if pxd_pathname: is_package = self._is_init_file(pxd_pathname) scope = from_module.find_submodule(module_name, as_package=is_package) if not scope: if debug_find_module: print("...trying absolute import") if absolute_fallback: qualified_name = module_name scope = self for name, is_package in self._split_qualified_name(qualified_name, relative_import=relative_import): scope = scope.find_submodule(name, as_package=is_package) if debug_find_module: print("...scope = %s" % scope) if not scope.pxd_file_loaded: if debug_find_module: print("...pxd not loaded") if not pxd_pathname: if debug_find_module: print("...looking for pxd file") # Only look in sys.path if we are explicitly looking # for a .pxd file. pxd_pathname = self.find_pxd_file(qualified_name, pos, sys_path=need_pxd and not relative_import) self._check_pxd_filename(pos, pxd_pathname, qualified_name) if debug_find_module: print("......found %s" % pxd_pathname) if not pxd_pathname and need_pxd: # Set pxd_file_loaded such that we don't need to # look for the non-existing pxd file next time. scope.pxd_file_loaded = True package_pathname = self.search_include_directories( qualified_name, suffix=".py", source_pos=pos, sys_path=not relative_import) if package_pathname and package_pathname.endswith(Utils.PACKAGE_FILES): pass else: error(pos, "'%s.pxd' not found" % qualified_name.replace('.', os.sep)) if pxd_pathname: scope.pxd_file_loaded = True try: if debug_find_module: print("Context.find_module: Parsing %s" % pxd_pathname) rel_path = module_name.replace('.', os.sep) + os.path.splitext(pxd_pathname)[1] if not pxd_pathname.endswith(rel_path): rel_path = pxd_pathname # safety measure to prevent printing incorrect paths source_desc = FileSourceDescriptor(pxd_pathname, rel_path) err, result = self.process_pxd(source_desc, scope, qualified_name) if err: raise err (pxd_codenodes, pxd_scope) = result self.pxds[module_name] = (pxd_codenodes, pxd_scope) except CompileError: pass return scope def find_pxd_file(self, qualified_name, pos=None, sys_path=True, source_file_path=None): # Search include path (and sys.path if sys_path is True) for # the .pxd file corresponding to the given fully-qualified # module name. # Will find either a dotted filename or a file in a # package directory. If a source file position is given, # the directory containing the source file is searched first # for a dotted filename, and its containing package root # directory is searched first for a non-dotted filename. pxd = self.search_include_directories( qualified_name, suffix=".pxd", source_pos=pos, sys_path=sys_path, source_file_path=source_file_path) if pxd is None and Options.cimport_from_pyx: return self.find_pyx_file(qualified_name, pos, sys_path=sys_path) return pxd def find_pyx_file(self, qualified_name, pos=None, sys_path=True, source_file_path=None): # Search include path for the .pyx file corresponding to the # given fully-qualified module name, as for find_pxd_file(). return self.search_include_directories( qualified_name, suffix=".pyx", source_pos=pos, sys_path=sys_path, source_file_path=source_file_path) def find_include_file(self, filename, pos=None, source_file_path=None): # Search list of include directories for filename. # Reports an error and returns None if not found. path = self.search_include_directories( filename, source_pos=pos, include=True, source_file_path=source_file_path) if not path: error(pos, "'%s' not found" % filename) return path def search_include_directories(self, qualified_name, suffix=None, source_pos=None, include=False, sys_path=False, source_file_path=None): include_dirs = self.include_directories if sys_path: include_dirs = include_dirs + sys.path # include_dirs must be hashable for caching in @cached_function include_dirs = tuple(include_dirs + [standard_include_path]) return search_include_directories( include_dirs, qualified_name, suffix or "", source_pos, include, source_file_path) def find_root_package_dir(self, file_path): return Utils.find_root_package_dir(file_path) def check_package_dir(self, dir, package_names): return Utils.check_package_dir(dir, tuple(package_names)) def c_file_out_of_date(self, source_path, output_path): if not os.path.exists(output_path): return 1 c_time = Utils.modification_time(output_path) if Utils.file_newer_than(source_path, c_time): return 1 pxd_path = Utils.replace_suffix(source_path, ".pxd") if os.path.exists(pxd_path) and Utils.file_newer_than(pxd_path, c_time): return 1 for kind, name in self.read_dependency_file(source_path): if kind == "cimport": dep_path = self.find_pxd_file(name, source_file_path=source_path) elif kind == "include": dep_path = self.search_include_directories(name, source_file_path=source_path) else: continue if dep_path and Utils.file_newer_than(dep_path, c_time): return 1 return 0 def find_cimported_module_names(self, source_path): return [ name for kind, name in self.read_dependency_file(source_path) if kind == "cimport" ] def is_package_dir(self, dir_path): return Utils.is_package_dir(dir_path) def read_dependency_file(self, source_path): dep_path = Utils.replace_suffix(source_path, ".dep") if os.path.exists(dep_path): with open(dep_path, "rU") as f: chunks = [ line.split(" ", 1) for line in (l.strip() for l in f) if " " in line ] return chunks else: return () def lookup_submodule(self, name): # Look up a top-level module. Returns None if not found. return self.modules.get(name, None) def find_submodule(self, name, as_package=False): # Find a top-level module, creating a new one if needed. scope = self.lookup_submodule(name) if not scope: scope = ModuleScope(name, parent_module = None, context = self, is_package=as_package) self.modules[name] = scope return scope def parse(self, source_desc, scope, pxd, full_module_name): if not isinstance(source_desc, FileSourceDescriptor): raise RuntimeError("Only file sources for code supported") source_filename = source_desc.filename scope.cpp = self.cpp # Parse the given source file and return a parse tree. num_errors = Errors.get_errors_count() try: with Utils.open_source_file(source_filename) as f: from . import Parsing s = PyrexScanner(f, source_desc, source_encoding = f.encoding, scope = scope, context = self) tree = Parsing.p_module(s, pxd, full_module_name) if self.options.formal_grammar: try: from ..Parser import ConcreteSyntaxTree except ImportError: raise RuntimeError( "Formal grammar can only be used with compiled Cython with an available pgen.") ConcreteSyntaxTree.p_module(source_filename) except UnicodeDecodeError as e: #import traceback #traceback.print_exc() raise self._report_decode_error(source_desc, e) if Errors.get_errors_count() > num_errors: raise CompileError() return tree def _report_decode_error(self, source_desc, exc): msg = exc.args[-1] position = exc.args[2] encoding = exc.args[0] line = 1 column = idx = 0 with io.open(source_desc.filename, "r", encoding='iso8859-1', newline='') as f: for line, data in enumerate(f, 1): idx += len(data) if idx >= position: column = position - (idx - len(data)) + 1 break return error((source_desc, line, column), "Decoding error, missing or incorrect coding= " "at top of source (cannot decode with encoding %r: %s)" % (encoding, msg)) def extract_module_name(self, path, options): # Find fully_qualified module name from the full pathname # of a source file. dir, filename = os.path.split(path) module_name, _ = os.path.splitext(filename) if "." in module_name: return module_name names = [module_name] while self.is_package_dir(dir): parent, package_name = os.path.split(dir) if parent == dir: break names.append(package_name) dir = parent names.reverse() return ".".join(names) def setup_errors(self, options, result): Errors.init_thread() if options.use_listing_file: path = result.listing_file = Utils.replace_suffix(result.main_source_file, ".lis") else: path = None Errors.open_listing_file(path=path, echo_to_stderr=options.errors_to_stderr) def teardown_errors(self, err, options, result): source_desc = result.compilation_source.source_desc if not isinstance(source_desc, FileSourceDescriptor): raise RuntimeError("Only file sources for code supported") Errors.close_listing_file() result.num_errors = Errors.get_errors_count() if result.num_errors > 0: err = True if err and result.c_file: try: Utils.castrate_file(result.c_file, os.stat(source_desc.filename)) except EnvironmentError: pass result.c_file = None def get_output_filename(source_filename, cwd, options): if options.cplus: c_suffix = ".cpp" else: c_suffix = ".c" suggested_file_name = Utils.replace_suffix(source_filename, c_suffix) if options.output_file: out_path = os.path.join(cwd, options.output_file) if os.path.isdir(out_path): return os.path.join(out_path, os.path.basename(suggested_file_name)) else: return out_path else: return suggested_file_name def create_default_resultobj(compilation_source, options): result = CompilationResult() result.main_source_file = compilation_source.source_desc.filename result.compilation_source = compilation_source source_desc = compilation_source.source_desc result.c_file = get_output_filename(source_desc.filename, compilation_source.cwd, options) result.embedded_metadata = options.embedded_metadata return result def run_pipeline(source, options, full_module_name=None, context=None): from . import Pipeline # ensure that the inputs are unicode (for Python 2) if sys.version_info[0] == 2: source = Utils.decode_filename(source) if full_module_name: full_module_name = Utils.decode_filename(full_module_name) source_ext = os.path.splitext(source)[1] options.configure_language_defaults(source_ext[1:]) # py/pyx if context is None: context = Context.from_options(options) # Set up source object cwd = os.getcwd() abs_path = os.path.abspath(source) full_module_name = full_module_name or context.extract_module_name(source, options) full_module_name = EncodedString(full_module_name) Utils.raise_error_if_module_name_forbidden(full_module_name) if options.relative_path_in_code_position_comments: rel_path = full_module_name.replace('.', os.sep) + source_ext if not abs_path.endswith(rel_path): rel_path = source # safety measure to prevent printing incorrect paths else: rel_path = abs_path source_desc = FileSourceDescriptor(abs_path, rel_path) source = CompilationSource(source_desc, full_module_name, cwd) # Set up result object result = create_default_resultobj(source, options) if options.annotate is None: # By default, decide based on whether an html file already exists. html_filename = os.path.splitext(result.c_file)[0] + ".html" if os.path.exists(html_filename): with io.open(html_filename, "r", encoding="UTF-8") as html_file: if u'