Merge pull request #478 from mrclary/ppm-syspath

PR: Add feature to prepend/append PYTHONPATH to sys.path

spyder_kernels/console/kernel.py

@@ -93,6 +93,11 @@ def __init__(self, *args, **kwargs):
         # To save the python env info
         self.pythonenv_info: PythonEnvInfo = {}
 
+        # Store original sys.path. Kernels are started with PYTHONPATH
+        # removed from environment variables, so this will never have
+        # user paths and should be clean.
+        self._sys_path = sys.path.copy()
+
     @property
     def kernel_info(self):
         # Used for checking correct version by spyder
@@ -770,27 +775,42 @@ def set_special_kernel(self, special):
         raise NotImplementedError(f"{special}")
 
     @comm_handler
-    def update_syspath(self, path_dict, new_path_dict):
+    def update_syspath(self, new_path, prioritize):
         """
         Update the PYTHONPATH of the kernel.
 
-        `path_dict` and `new_path_dict` have the paths as keys and the state
-        as values. The state is `True` for active and `False` for inactive.
-
-        `path_dict` corresponds to the previous state of the PYTHONPATH.
-        `new_path_dict` corresponds to the new state of the PYTHONPATH.
+        Parameters
+        ----------
+        new_path: list of str
+            List of PYTHONPATH paths.
+        prioritize: bool
+            Whether to place PYTHONPATH paths at the front (True) or
+            back (False) of sys.path.
+
+        Notes
+        -----
+        A copy of sys.path is made at instantiation, which should be clean,
+        so we can just prepend/append to the copy without having to explicitly
+        remove old user paths. PYTHONPATH can just be overwritten.
         """
-        # Remove old paths
-        for path in path_dict:
-            while path in sys.path:
-                sys.path.remove(path)
-
-        # Add new paths
-        pypath = [path for path, active in new_path_dict.items() if active]
-        if pypath:
-            sys.path.extend(pypath)
-            os.environ.update({'PYTHONPATH': os.pathsep.join(pypath)})
+        if new_path is not None:
+            # Overwrite PYTHONPATH
+            os.environ.update({'PYTHONPATH': os.pathsep.join(new_path)})
+
+            # Add new paths to original sys.path
+            if prioritize:
+                sys.path[:] = new_path + self._sys_path
+
+                # Ensure current directory is always first to imitate Python
+                # standard behavior
+                if '' in sys.path:
+                    sys.path.remove('')
+                    sys.path.insert(0, '')
+            else:
+                sys.path[:] = self._sys_path + new_path
         else:
+            # Restore original sys.path and remove PYTHONPATH
+            sys.path[:] = self._sys_path
             os.environ.pop('PYTHONPATH', None)
 
     @comm_handler

spyder_kernels/console/start.py

@@ -16,6 +16,14 @@
 import sys
 import site
 
+# Remove current directory from sys.path to prevent kernel crashes when people
+# name Python files or modules with the same name as standard library modules.
+# See spyder-ide/spyder#8007
+# Inject it back into sys.path after all imports in this module but
+# before the kernel is initialized
+while '' in sys.path:
+    sys.path.remove('')
+
 # Third-party imports
 from traitlets import DottedObjectName
 
@@ -29,13 +37,6 @@ def import_spydercustomize():
     parent = osp.dirname(here)
     customize_dir = osp.join(parent, 'customize')
 
-    # Remove current directory from sys.path to prevent kernel
-    # crashes when people name Python files or modules with
-    # the same name as standard library modules.
-    # See spyder-ide/spyder#8007
-    while '' in sys.path:
-        sys.path.remove('')
-
     # Import our customizations
     site.addsitedir(customize_dir)
     import spydercustomize  # noqa
@@ -46,6 +47,7 @@ def import_spydercustomize():
     except ValueError:
         pass
 
+
 def kernel_config():
     """Create a config object with IPython kernel options."""
     from IPython.core.application import get_ipython_dir
@@ -150,13 +152,6 @@ def main():
     # Import our customizations into the kernel
     import_spydercustomize()
 
-    # Remove current directory from sys.path to prevent kernel
-    # crashes when people name Python files or modules with
-    # the same name as standard library modules.
-    # See spyder-ide/spyder#8007
-    while '' in sys.path:
-        sys.path.remove('')
-
     # Main imports
     from ipykernel.kernelapp import IPKernelApp
     from spyder_kernels.console.kernel import SpyderKernel
@@ -189,6 +184,12 @@ def close(self):
         kernel.config = kernel_config()
     except:
         pass
+
+    # Re-add current working directory path into sys.path after all of the
+    # import statements, but before initializing the kernel.
+    if '' not in sys.path:
+        sys.path.insert(0, '')
+
     kernel.initialize()
 
     # Set our own magics

spyder_kernels/console/tests/test_console_kernel.py

@@ -77,13 +77,15 @@ def setup_kernel(cmd):
         )
         # wait for connection file to exist, timeout after 5s
         tic = time.time()
-        while not os.path.exists(connection_file) \
-            and kernel.poll() is None \
-            and time.time() < tic + SETUP_TIMEOUT:
+        while (
+            not os.path.exists(connection_file)
+            and kernel.poll() is None
+            and time.time() < tic + SETUP_TIMEOUT
+        ):
             time.sleep(0.1)
 
         if kernel.poll() is not None:
-            o,e = kernel.communicate()
+            o, e = kernel.communicate()
             raise IOError("Kernel failed to start:\n%s" % e)
 
         if not os.path.exists(connection_file):
@@ -229,7 +231,7 @@ def kernel(request):
             'True_'
         ],
         'minmax': False,
-        'filter_on':True
+        'filter_on': True
     }
 
     # Teardown
@@ -468,8 +470,11 @@ def test_is_defined(kernel):
 def test_get_doc(kernel):
     """Test to get object documentation dictionary."""
     objtxt = 'help'
-    assert ("Define the builtin 'help'" in kernel.get_doc(objtxt)['docstring'] or
-            "Define the built-in 'help'" in kernel.get_doc(objtxt)['docstring'])
+    assert (
+        "Define the builtin 'help'" in kernel.get_doc(objtxt)['docstring']
+        or "Define the built-in 'help'" in kernel.get_doc(objtxt)['docstring']
+    )
+
 
 def test_get_source(kernel):
     """Test to get object source."""
@@ -507,7 +512,7 @@ def test_cwd_in_sys_path():
     with setup_kernel(cmd) as client:
         reply = client.execute_interactive(
             "import sys; sys_path = sys.path",
-            user_expressions={'output':'sys_path'}, timeout=TIMEOUT)
+            user_expressions={'output': 'sys_path'}, timeout=TIMEOUT)
 
         # Transform value obtained through user_expressions
         user_expressions = reply['content']['user_expressions']
@@ -518,6 +523,21 @@ def test_cwd_in_sys_path():
         assert '' in value
 
 
+def test_prioritize(kernel):
+    """Test that user path priority is honored in sys.path."""
+    syspath = kernel.get_syspath()
+    append_path = ['/test/append/path']
+    prepend_path = ['/test/prepend/path']
+
+    kernel.update_syspath(append_path, prioritize=False)
+    new_syspath = kernel.get_syspath()
+    assert new_syspath == syspath + append_path
+
+    kernel.update_syspath(prepend_path, prioritize=True)
+    new_syspath = kernel.get_syspath()
+    assert new_syspath == prepend_path + syspath
+
+
 @flaky(max_runs=3)
 def test_multiprocessing(tmpdir):
     """
@@ -701,8 +721,10 @@ def test_runfile(tmpdir):
         assert content['found']
 
         # Run code file `u` with current namespace
-        msg = client.execute_interactive("%runfile {} --current-namespace"
-                                        .format(repr(str(u))), timeout=TIMEOUT)
+        msg = client.execute_interactive(
+            "%runfile {} --current-namespace".format(repr(str(u))),
+            timeout=TIMEOUT
+        )
         content = msg['content']
 
         # Verify that the variable `result3` is defined
@@ -727,7 +749,9 @@ def test_runfile(tmpdir):
     sys.platform == 'darwin' and sys.version_info[:2] == (3, 8),
     reason="Fails on Mac with Python 3.8")
 def test_np_threshold(kernel):
-    """Test that setting Numpy threshold doesn't make the Variable Explorer slow."""
+    """
+    Test that setting Numpy threshold doesn't make the Variable Explorer slow.
+    """
 
     cmd = "from spyder_kernels.console import start; start.main()"
 
@@ -786,7 +810,9 @@ def test_np_threshold(kernel):
         while "data" not in msg['content']:
             msg = client.get_shell_msg(timeout=TIMEOUT)
         content = msg['content']['data']['text/plain']
-        assert "{'float_kind': <built-in method format of str object" in content
+        assert (
+            "{'float_kind': <built-in method format of str object" in content
+        )
 
 
 @flaky(max_runs=3)
@@ -952,11 +978,12 @@ def test_comprehensions_with_locals_in_pdb(kernel):
 
     # Check that the variable is not reported as being part of globals.
     kernel.shell.pdb_session.default("in_globals = 'zz' in globals()")
-    assert kernel.get_value('in_globals') == False
+    assert kernel.get_value('in_globals') is False
 
     pdb_obj.curframe = None
     pdb_obj.curframe_locals = None
 
+
 def test_comprehensions_with_locals_in_pdb_2(kernel):
     """
     Test that evaluating comprehensions with locals works in Pdb.
@@ -1001,6 +1028,7 @@ def test_namespaces_in_pdb(kernel):
     # Create wrapper to check for errors
     old_error = pdb_obj.error
     pdb_obj._error_occured = False
+
     def error_wrapper(*args, **kwargs):
         print(args, kwargs)
         pdb_obj._error_occured = True
@@ -1052,7 +1080,6 @@ def test_functions_with_locals_in_pdb(kernel):
         'zz = fun_a()')
     assert kernel.get_value('zz') == 1
 
-
     pdb_obj.curframe = None
     pdb_obj.curframe_locals = None
 
@@ -1110,11 +1137,11 @@ def test_locals_globals_in_pdb(kernel):
 
     kernel.shell.pdb_session.default(
         'test = "a" in globals()')
-    assert kernel.get_value('test') == False
+    assert kernel.get_value('test') is False
 
     kernel.shell.pdb_session.default(
         'test = "a" in locals()')
-    assert kernel.get_value('test') == True
+    assert kernel.get_value('test') is True
 
     kernel.shell.pdb_session.default(
         'def f(): return a')
@@ -1128,11 +1155,11 @@ def test_locals_globals_in_pdb(kernel):
 
     kernel.shell.pdb_session.default(
         'test = "a" in globals()')
-    assert kernel.get_value('test') == False
+    assert kernel.get_value('test') is False
 
     kernel.shell.pdb_session.default(
         'test = "a" in locals()')
-    assert kernel.get_value('test') == True
+    assert kernel.get_value('test') is True
 
     pdb_obj.curframe = None
     pdb_obj.curframe_locals = None
@@ -1210,7 +1237,7 @@ def test_global_message(tmpdir):
 
         def check_found(msg):
             if "text" in msg["content"]:
-                if ("WARNING: This file contains a global statement"  in
+                if ("WARNING: This file contains a global statement" in
                         msg["content"]["text"]):
                     global found
                     found = True
@@ -1256,7 +1283,7 @@ def test_debug_namespace(tmpdir):
                 if 'hello' in msg["content"].get("text"):
                     break
 
-         # make sure that get_value('bb') returns 'hello'
+        # make sure that get_value('bb') returns 'hello'
         client.get_stdin_msg(timeout=TIMEOUT)
         client.input("get_ipython().kernel.get_value('bb')")
 

spyder_kernels/customize/spydercustomize.py

@@ -275,17 +275,3 @@ def restore_tmpdir():
                 pass
 
 restore_tmpdir()
-
-# =============================================================================
-# PYTHONPATH and sys.path Adjustments
-# =============================================================================
-# PYTHONPATH is not passed to kernel directly, see spyder-ide/spyder#13519
-# This allows the kernel to start without crashing if modules in PYTHONPATH
-# shadow standard library modules.
-def set_spyder_pythonpath():
-    pypath = os.environ.get('SPY_PYTHONPATH')
-    if pypath:
-        sys.path.extend(pypath.split(os.pathsep))
-        os.environ.update({'PYTHONPATH': pypath})
-
-set_spyder_pythonpath()