conda · msarahan · Dec 9, 2018 · Nov 30, 2018 · Dec 5, 2018 · Dec 5, 2018
diff --git a/conda/core/link.py b/conda/core/link.py
@@ -13,7 +13,7 @@
 import warnings
 
 from .package_cache_data import PackageCacheData
-from .path_actions import (CompilePycAction, CreateNonadminAction, CreatePrefixRecordAction,
+from .path_actions import (CompileMultiPycAction, CreateNonadminAction, CreatePrefixRecordAction,
                            CreatePythonEntryPointAction, LinkPathAction, MakeMenuAction,
                            RegisterEnvironmentLocationAction, RemoveLinkedPackageRecordAction,
                            RemoveMenuAction, UnlinkPathAction, UnregisterEnvironmentLocationAction,
@@ -386,28 +386,33 @@ def _verify_prefix_level(target_prefix, prefix_action_group):
         link_paths_dict = defaultdict(list)
         for axn in create_lpr_actions:
             for link_path_action in axn.all_link_path_actions:
-                path = link_path_action.target_short_path
-                path = lower_on_win(path)
-                link_paths_dict[path].append(axn)
-                if path not in unlink_paths and lexists(join(target_prefix, path)):
-                    # we have a collision; at least try to figure out where it came from
-                    colliding_prefix_rec = first(
-                        (prefix_rec for prefix_rec in PrefixData(target_prefix).iter_records()),
-                        key=lambda prefix_rec: path in prefix_rec.files
-                    )
-                    if colliding_prefix_rec:
-                        yield KnownPackageClobberError(
-                            path,
-                            axn.package_info.repodata_record.dist_str(),
-                            colliding_prefix_rec.dist_str(),
-                            context,
-                        )
-                    else:
-                        yield UnknownPackageClobberError(
-                            path,
-                            axn.package_info.repodata_record.dist_str(),
-                            context,
+                if isinstance(link_path_action, CompileMultiPycAction):
+                    target_short_paths = link_path_action.target_short_paths
+                else:
+                    target_short_paths = (link_path_action.target_short_path, )
+                for path in target_short_paths:
+                    path = lower_on_win(path)
+                    link_paths_dict[path].append(axn)
+                    if path not in unlink_paths and lexists(join(target_prefix, path)):
+                        # we have a collision; at least try to figure out where it came from
+                        colliding_prefix_rec = first(
+                            (prefix_rec for prefix_rec in
+                             PrefixData(target_prefix).iter_records()),
+                            key=lambda prefix_rec: path in prefix_rec.files
                         )
+                        if colliding_prefix_rec:
+                            yield KnownPackageClobberError(
+                                path,
+                                axn.package_info.repodata_record.dist_str(),
+                                colliding_prefix_rec.dist_str(),
+                                context,
+                            )
+                        else:
+                            yield UnknownPackageClobberError(
+                                path,
+                                axn.package_info.repodata_record.dist_str(),
+                                context,
+                            )
 
         # Verification 2. there's only a single instance of each path
         for path, axns in iteritems(link_paths_dict):
@@ -683,8 +688,8 @@ def _make_link_actions(transaction_context, package_info, target_prefix, request
         create_menu_actions = MakeMenuAction.create_actions(*required_quad)
 
         python_entry_point_actions = CreatePythonEntryPointAction.create_actions(*required_quad)
-        compile_pyc_actions = CompilePycAction.create_actions(*required_quad,
-                                                              file_link_actions=file_link_actions)
+        compile_pyc_actions = CompileMultiPycAction.create_actions(
+            *required_quad, file_link_actions=file_link_actions)
 
         # if requested_spec:
         #     application_entry_point_actions = CreateApplicationEntryPointAction.create_actions(

diff --git a/conda/core/path_actions.py b/conda/core/path_actions.py
@@ -16,6 +16,7 @@
 from .. import CondaError
 from .._vendor.auxlib.compat import with_metaclass
 from .._vendor.auxlib.ish import dals
+from .._vendor.toolz import concat
 from ..base.constants import CONDA_TARBALL_EXTENSION
 from ..base.context import context
 from ..common.compat import iteritems, on_win, text_type
@@ -26,8 +27,9 @@
 from ..common.url import has_platform, path_to_url, unquote
 from ..exceptions import CondaUpgradeError, CondaVerificationError, PaddingError, SafetyError
 from ..gateways.connection.download import download
-from ..gateways.disk.create import (compile_pyc, copy, create_hard_link_or_copy,
-                                    create_link, create_python_entry_point, extract_tarball,
+from ..gateways.disk.create import (compile_multiple_pyc, copy,
+                                    create_hard_link_or_copy, create_link,
+                                    create_python_entry_point, extract_tarball,
                                     make_menu, mkdir_p, write_as_json_to_file)
 from ..gateways.disk.delete import rm_rf, try_rmdir_all_empty
 from ..gateways.disk.permissions import make_writable
@@ -88,6 +90,44 @@ def __repr__(self):
         return "%s(%s)" % (self.__class__.__name__, ', '.join(args))
 
 
+@with_metaclass(ABCMeta)
+class MultiPathAction(object):
+
+    _verified = False
+
+    @abstractmethod
+    def verify(self):
+        # if verify fails, it should return an exception object rather than raise
+        #  at the end of a verification run, all errors will be raised as a CondaMultiError
+        # after successful verification, the verify method should set self._verified = True
+        raise NotImplementedError()
+
+    @abstractmethod
+    def execute(self):
+        raise NotImplementedError()
+
+    @abstractmethod
+    def reverse(self):
+        raise NotImplementedError()
+
+    @abstractmethod
+    def cleanup(self):
+        raise NotImplementedError()
+
+    @abstractproperty
+    def target_full_paths(self):
+        raise NotImplementedError()
+
+    @property
+    def verified(self):
+        return self._verified
+
+    def __repr__(self):
+        args = ('%s=%r' % (key, value) for key, value in iteritems(vars(self))
+                if key not in REPR_IGNORE_KWARGS)
+        return "%s(%s)" % (self.__class__.__name__, ', '.join(args))
+
+
 @with_metaclass(ABCMeta)
 class PrefixPathAction(PathAction):
 
@@ -444,7 +484,7 @@ def reverse(self):
             rm_rf(self.target_full_path)
 
 
-class CompilePycAction(CreateInPrefixPathAction):
+class CompileMultiPycAction(MultiPathAction):
 
     @classmethod
     def create_actions(cls, transaction_context, package_info, target_prefix, requested_link_type,
@@ -453,41 +493,69 @@ def create_actions(cls, transaction_context, package_info, target_prefix, reques
         if noarch is not None and noarch.type == NoarchType.python:
             noarch_py_file_re = re.compile(r'^site-packages[/\\][^\t\n\r\f\v]+\.py$')
             py_ver = transaction_context['target_python_version']
-            py_files = (axn.target_short_path for axn in file_link_actions
-                        if noarch_py_file_re.match(axn.source_short_path))
-            return tuple(cls(transaction_context, package_info, target_prefix,
-                             pf, pyc_path(pf, py_ver))
-                         for pf in py_files)
+            py_files = tuple((axn.target_short_path for axn in file_link_actions
+                              if noarch_py_file_re.match(axn.source_short_path)))
+            pyc_files = tuple((pyc_path(pf, py_ver) for pf in py_files))
+            return (cls(transaction_context, package_info, target_prefix, py_files, pyc_files), )
         else:
             return ()
 
     def __init__(self, transaction_context, package_info, target_prefix,
-                 source_short_path, target_short_path):
-        super(CompilePycAction, self).__init__(transaction_context, package_info,
-                                               target_prefix, source_short_path,
-                                               target_prefix, target_short_path)
-        self.prefix_path_data = PathDataV1(
-            _path=self.target_short_path,
-            path_type=PathType.pyc_file,
-        )
+                 source_short_paths, target_short_paths):
+        self.transaction_context = transaction_context
+        self.package_info = package_info
+        self.target_prefix = target_prefix
+        self.source_short_paths = source_short_paths
+        self.target_short_paths = target_short_paths
+        self.prefix_path_data = None
+        self.prefix_paths_data = [
+            PathDataV1(_path=p, path_type=PathType.pyc_file,) for p in self.target_short_paths]
         self._execute_successful = False
 
+    @property
+    def target_full_paths(self):
+        def join_or_none(prefix, short_path):
+            if prefix is None or short_path is None:
+                return None
+            else:
+                return join(prefix, win_path_ok(short_path))
+        return (join_or_none(self.target_prefix, p) for p in self.target_short_paths)
+
+    @property
+    def source_full_paths(self):
+        def join_or_none(prefix, short_path):
+            if prefix is None or short_path is None:
+                return None
+            else:
+                return join(prefix, win_path_ok(short_path))
+        return (join_or_none(self.target_prefix, p) for p in self.source_short_paths)
+
+    def verify(self):
+        self._verified = True
+
+    def cleanup(self):
+        # create actions typically won't need cleanup
+        pass
+
     def execute(self):
         # compile_pyc is sometimes expected to fail, for example a python 3.6 file
         #   installed into a python 2 environment, but no code paths actually importing it
         # technically then, this file should be removed from the manifest in conda-meta, but
         #   at the time of this writing that's not currently happening
-        log.trace("compiling %s", self.target_full_path)
+        log.trace("compiling %s", ' '.join(self.target_full_paths))
         target_python_version = self.transaction_context['target_python_version']
         python_short_path = get_python_short_path(target_python_version)
         python_full_path = join(self.target_prefix, win_path_ok(python_short_path))
-        compile_pyc(python_full_path, self.source_full_path, self.target_full_path)
+        compile_multiple_pyc(python_full_path, self.source_full_paths, self.target_full_paths,
+                             self.target_prefix, self.transaction_context['target_python_version'])
         self._execute_successful = True
 
     def reverse(self):
+        # this removes all pyc files even if they were not created
         if self._execute_successful:
-            log.trace("reversing pyc creation %s", self.target_full_path)
-            rm_rf(self.target_full_path)
+            log.trace("reversing pyc creation %s", ' '.join(self.target_full_paths))
+            for target_full_path in self.target_full_paths:
+                rm_rf(target_full_path)
 
 
 class CreatePythonEntryPointAction(CreateInPrefixPathAction):
@@ -754,12 +822,25 @@ def execute(self):
         package_tarball_full_path = extracted_package_dir + CONDA_TARBALL_EXTENSION
         # TODO: don't make above assumption; put package_tarball_full_path in package_info
 
-        files = (x.target_short_path for x in self.all_link_path_actions if x)
+        def files_from_action(link_path_action):
+            if isinstance(link_path_action, CompileMultiPycAction):
+                return link_path_action.target_short_paths
+            else:
+                return (link_path_action.target_short_path, )
+
+        def paths_from_action(link_path_action):
+            if isinstance(link_path_action, CompileMultiPycAction):
+                return link_path_action.prefix_paths_data
+            else:
+                if link_path_action.prefix_path_data is None:
+                    return ()
+                else:
+                    return (link_path_action.prefix_path_data, )
 
+        files = concat((files_from_action(x) for x in self.all_link_path_actions if x))
         paths_data = PathsData(
             paths_version=1,
-            paths=(x.prefix_path_data for x in self.all_link_path_actions
-                   if x and x.prefix_path_data),
+            paths=concat((paths_from_action(x) for x in self.all_link_path_actions if x)),
         )
 
         self.prefix_record = PrefixRecord.from_objects(

diff --git a/conda/gateways/disk/create.py b/conda/gateways/disk/create.py
@@ -11,6 +11,7 @@
 from shutil import copyfileobj, copystat
 import sys
 import tarfile
+import tempfile
 
 from . import mkdir_p
 from .delete import rm_rf
@@ -336,29 +337,53 @@ def create_link(src, dst, link_type=LinkType.hardlink, force=False):
         raise CondaError("Did not expect linktype=%r" % link_type)
 
 
-def compile_pyc(python_exe_full_path, py_full_path, pyc_full_path):
-    if lexists(pyc_full_path):
-        maybe_raise(BasicClobberError(None, pyc_full_path, context), context)
+def compile_multiple_pyc(python_exe_full_path, py_full_paths, pyc_full_paths, prefix, py_ver):
+    py_full_paths = tuple(py_full_paths)
+    pyc_full_paths = tuple(pyc_full_paths)
+    if len(py_full_paths) == 0:
+        return []
 
-    command = '"%s" -Wi -m py_compile "%s"' % (python_exe_full_path, py_full_path)
-    log.trace(command)
-    result = subprocess_call(command, raise_on_error=False)
+    for pyc_full_path in pyc_full_paths:
+        if lexists(pyc_full_path):
+            maybe_raise(BasicClobberError(None, pyc_full_path, context), context)
 
-    if not isfile(pyc_full_path):
-        message = dals("""
-        pyc file failed to compile successfully
-          python_exe_full_path: %s
-          py_full_path: %s
-          pyc_full_path: %s
-          compile rc: %s
-          compile stdout: %s
-          compile stderr: %s
-        """)
-        log.info(message, python_exe_full_path, py_full_path, pyc_full_path,
-                 result.rc, result.stdout, result.stderr)
-        return None
-
-    return pyc_full_path
+    fd, filename = tempfile.mkstemp()
+    try:
+        for f in py_full_paths:
+            f = os.path.relpath(f, prefix)
+            if hasattr(f, 'encode'):
+                f = f.encode(sys.getfilesystemencoding())
+            os.write(fd, f + b"\n")
+        os.close(fd)
+        command = ["-Wi", "-m", "compileall", "-q", "-l", "-i", filename]
+        # if the python version in the prefix is 3.5+, we have some extra args.
+        #    -j 0 will do the compilation in parallel, with os.cpu_count() cores
+        if int(py_ver[0]) >= 3 and int(py_ver.split('.')[1]) > 5:
+            command.extend(["-j", "0"])
+        command = '"%s" ' % python_exe_full_path + " ".join(command)
+        log.trace(command)
+        result = subprocess_call(command, raise_on_error=False, path=prefix)
+    finally:
+        os.remove(filename)
+
+    created_pyc_paths = []
+    for py_full_path, pyc_full_path in zip(py_full_paths, pyc_full_paths):
+        if not isfile(pyc_full_path):
+            message = dals("""
+            pyc file failed to compile successfully
+            python_exe_full_path: %s
+            py_full_path: %s
+            pyc_full_path: %s
+            compile rc: %s
+            compile stdout: %s
+            compile stderr: %s
+            """)
+            log.info(message, python_exe_full_path, py_full_path, pyc_full_path,
+                     result.rc, result.stdout, result.stderr)
+        else:
+            created_pyc_paths.append(pyc_full_path)
+
+    return created_pyc_paths
 
 
 def create_package_cache_directory(pkgs_dir):

diff --git a/conda/gateways/subprocess.py b/conda/gateways/subprocess.py
@@ -48,7 +48,7 @@ def subprocess_call(command, env=None, path=None, stdin=None, raise_on_error=Tru
     log.debug("executing>> %s", command_str)
     p = Popen(command_arg, cwd=cwd, stdin=PIPE, stdout=PIPE, stderr=PIPE, env=env)
     ACTIVE_SUBPROCESSES.add(p)
-    stdin = ensure_binary(stdin) if isinstance(stdin, string_types) else None
+    stdin = ensure_binary(stdin) if isinstance(stdin, string_types) else stdin
     stdout, stderr = p.communicate(input=stdin)
     rc = p.returncode
     ACTIVE_SUBPROCESSES.remove(p)