Automatically patch imported functions

- allows imports like 'from os import stat' to be patched

Author: mrbean-bremen

CHANGES.md

@@ -14,15 +14,19 @@ The release versions are PyPi releases.
     ```
 
 #### New Features
-  * a module imported as another name (`import os as _os`) is now correctly 
-    patched without the need of additional parameters
+  * improved automatic patching:
+    * automatically patch methods of a patched file system module imported like 
+      `from os.path import exists`
+    * a module imported as another name (`import os as _os`) is now correctly 
+      patched without the need of additional parameters 
+      ([#434](../../pull/434))
+    * automatically patch `Path` if imported like `from pathlib import Path` 
+      ([#440](../../issues/440))
+    * parameter `patch_path` has been removed from `UnitTest` and `Patcher`, 
+      the correct patching of `path` imports is now done automatically
+    * `UnitTest` /`Patcher` arguments can now also be set in `setUpPyfakefs()`
   * added possibility to set root user ([#431](../../issues/431))
-  * automatically patch `Path` if imported like `from pathlib import Path` 
-    ([#440](../../issues/440))
   * added side_effect option to fake files ([#433](../../pull/433))
-  * parameter `patch_path` has been removed from `UnitTest` and `Patcher`, 
-    the correct patching of `path` imports is now done automatically
-  * `UnitTest` /`Patcher` arguments can now also be set in `setUpPyfakefs()`
   * added pathlib2 support ([#408](../../issues/408)) ([#422](../../issues/422))
   * added some support for extended filesystem attributes under Linux 
   ([#423](../../issues/423)) 

docs/usage.rst

@@ -111,6 +111,14 @@ Pyfakefs automatically patches file system related modules that are:
   from os import path
   from pathlib import Path
 
+Additionally, functions from file system related modules are patched
+automatically if imported like:
+
+.. code:: python
+
+  from os.path import exists
+  from os import stat
+
 There are other cases where automatic patching does not work.
 Both ``fake_filesystem_unittest.Patcher`` and ``fake_filesystem_unittest.TestCase``
 provide a few additional arguments to handle such cases.
@@ -132,8 +140,15 @@ modules_to_reload
 This allows to pass a list of modules that shall be reloaded, thus allowing
 to patch modules not patched automatically.
 
+Here is a simple example for a default argument that is not patched
+automatically:
+
+.. code:: python
+
+  def check_if_exists(filepath, file_exists=os.path.exists):
+      return file_exists(filepath)
 
-If adding the module containing these imports to ``modules_to_reload``, they
+If adding the module containing this code to ``modules_to_reload``, it
 will be correctly patched.
 
 modules_to_patch

pyfakefs/fake_filesystem.py

@@ -3026,6 +3026,23 @@ class FakePathModule(object):
     """
     _OS_PATH_COPY = _copy_module(os.path)
 
+    @staticmethod
+    def dir():
+        """Return the list of patched function names. Used for patching
+        functions imported from the module.
+        """
+        dir = [
+            'abspath', 'dirname', 'exists', 'expanduser', 'getatime',
+            'getctime', 'getmtime', 'getsize', 'isabs', 'isdir', 'isfile',
+            'islink', 'ismount', 'join', 'lexists', 'normcase', 'normpath',
+            'realpath', 'relpath', 'split', 'splitdrive'
+        ]
+        if IS_PY2:
+            dir.append('walk')
+        if sys.platform != 'win32' or not IS_PY2:
+            dir.append('samefile')
+        return dir
+
     def __init__(self, filesystem, os_module=None):
         """Init.
 
@@ -3422,6 +3439,31 @@ class FakeOsModule(object):
 
     devnull = None
 
+    @staticmethod
+    def dir():
+        """Return the list of patched function names. Used for patching
+        functions imported from the module.
+        """
+        dir = [
+            'access', 'chdir', 'chmod', 'chown', 'close', 'fstat', 'fsync',
+            'getcwd', 'lchmod', 'link', 'listdir', 'lstat', 'makedirs',
+            'mkdir', 'mknod', 'open', 'read', 'readlink', 'remove',
+            'removedirs', 'rename', 'rmdir', 'stat', 'symlink', 'umask',
+            'unlink', 'utime', 'walk', 'write'
+        ]
+        if IS_PY2:
+            dir += ['getcwdu']
+        else:
+            dir += ['getcwdb', 'replace']
+            if sys.platform.startswith('linux'):
+                dir += [
+                    'fdatasync','getxattr', 'listxattr',
+                       'removexattr', 'setxattr'
+                ]
+        if use_scandir:
+            dir += ['scandir']
+        return dir
+
     def __init__(self, filesystem, os_path_module=None):
         """Also exposes self.path (to fake os.path).
 
@@ -3843,7 +3885,6 @@ def setxattr(self, path, attribute, value,
                 self.filesystem.raise_os_error(errno.EEXIST, file_obj.path)
             file_obj.xattr[attribute] = value
 
-
     if use_scandir:
         def scandir(self, path=''):
             """Return an iterator of DirEntry objects corresponding to the
@@ -4420,6 +4461,15 @@ class FakeIoModule(object):
     my_io_module = fake_filesystem.FakeIoModule(filesystem)
     """
 
+    @staticmethod
+    def dir():
+        """Return the list of patched function names. Used for patching
+        functions imported from the module.
+        """
+        # `open` would clash with build-in `open`, so don't patch it
+        # if imported like `from io import open`
+        return ()
+
     def __init__(self, filesystem):
         """
         Args:

pyfakefs/fake_filesystem_shutil.py

@@ -30,11 +30,22 @@
 import shutil
 import sys
 
+from pyfakefs.helpers import IS_PY2
+
 
 class FakeShutilModule(object):
     """Uses a FakeFilesystem to provide a fake replacement for shutil module.
     """
 
+    @staticmethod
+    def dir():
+        """Return the list of patched function names. Used for patching
+        functions imported from the module.
+        """
+        if not IS_PY2:
+            return 'disk_usage',
+        return ()
+
     def __init__(self, filesystem):
         """Construct fake shutil module using the fake filesystem.
 

pyfakefs/fake_filesystem_unittest.py

@@ -77,6 +77,7 @@
 else:
     import builtins
 
+OS_MODULE = 'nt' if sys.platform == 'win32' else 'posix'
 PATH_MODULE = 'ntpath' if sys.platform == 'win32' else 'posixpath'
 
 def load_doctests(loader, tests, ignore, module,
@@ -284,7 +285,7 @@ class Patcher(object):
 
     IS_WINDOWS = sys.platform in ('win32', 'cygwin')
 
-    SKIPNAMES = {'os', 'path', 'io', 'genericpath'}
+    SKIPNAMES = {'os', 'path', 'io', 'genericpath', OS_MODULE, PATH_MODULE}
     if pathlib:
         SKIPNAMES.add('pathlib')
 
@@ -338,8 +339,32 @@ def __init__(self, additional_skip_names=None,
                         module_name)
                 self._fake_module_classes[name] = fake_module
 
+        # handle patching function imported separately like
+        # `from os import stat`
+        # each patched function name has to be looked up separately
+        self._fake_module_functions = {}
+        for mod_name, fake_module in self._fake_module_classes.items():
+            modnames = (
+                (mod_name, OS_MODULE) if mod_name == 'os' else (mod_name,)
+            )
+            for fct_name in fake_module.dir():
+                self._fake_module_functions[fct_name] = (
+                    modnames,
+                    getattr(fake_module, fct_name),
+                    mod_name
+                )
+        # special handling for functions in os.path
+        fake_module = fake_filesystem.FakePathModule
+        for fct_name in fake_module.dir():
+            self._fake_module_functions[fct_name] = (
+                ('genericpath', PATH_MODULE),
+                getattr(fake_module, fct_name),
+                PATH_MODULE
+            )
+
         # Attributes set by _refresh()
         self._modules = {}
+        self._fct_modules = {}
         self._stubs = None
         self.fs = None
         self.fake_open = None
@@ -366,6 +391,12 @@ def _find_modules(self):
         Later, `setUp()` will stub these with the fake file system
         modules.
         """
+        def is_fct(module, name):
+            fct = module.__dict__.get(name)
+            return (fct is not None and
+                    (inspect.isfunction(fct) or inspect.isbuiltin(fct)) and
+                    fct.__module__ in self._fake_module_functions[name][0])
+
         module_names = list(self._fake_module_classes.keys()) + [PATH_MODULE]
         for name, module in set(sys.modules.items()):
             try:
@@ -378,6 +409,7 @@ def _find_modules(self):
                 # where py.error has no __name__ attribute
                 # see https://github.com/pytest-dev/py/issues/73
                 continue
+
             modules = {name: mod for name, mod in module.__dict__.items()
                        if inspect.ismodule(mod) and
                        mod.__name__ in module_names
@@ -387,6 +419,11 @@ def _find_modules(self):
                 self._modules.setdefault(name, set()).add((module,
                                                            mod.__name__))
 
+            functions = [name for name in self._fake_module_functions
+                         if is_fct(module, name)]
+            for name in functions:
+                self._fct_modules.setdefault(name, set()).add(module)
+
     def _refresh(self):
         """Renew the fake file system and set the _isStale flag to `False`."""
         if self._stubs is not None:
@@ -416,10 +453,17 @@ def setUp(self, doctester=None):
             # file() was eliminated in Python3
             self._stubs.smart_set(builtins, 'file', self.fake_open)
         self._stubs.smart_set(builtins, 'open', self.fake_open)
-        for name in self._modules:
-            for module, attr in self._modules[name]:
+        for name, modules in self._modules.items():
+            for module, attr in modules:
                 self._stubs.smart_set(module, name, self.fake_modules[attr])
 
+        for name, modules in self._fct_modules.items():
+            _, method, mod_name = self._fake_module_functions[name]
+            fake_module = self.fake_modules[mod_name]
+            attr = method.__get__(fake_module, fake_module.__class__)
+            for module in modules:
+                self._stubs.smart_set(module, name, attr)
+
         self._dyn_patcher = DynamicPatcher(self)
         sys.meta_path.insert(0, self._dyn_patcher)
 

pyfakefs/fake_pathlib.py

@@ -633,6 +633,13 @@ class FakePathlibModule(object):
     `fake_pathlib_module = fake_filesystem.FakePathlibModule(filesystem)`
     """
 
+    @staticmethod
+    def dir():
+        """Return an empty list as `Path` methods will always be called
+        on the instances.
+        """
+        return ()
+
     def __init__(self, filesystem):
         """
         Initializes the module with the given filesystem.
@@ -676,6 +683,10 @@ class FakePathlibPathModule(object):
     """Patches `pathlib.Path` by passing all calls to FakePathlibModule."""
     fake_pathlib = None
 
+    @staticmethod
+    def dir():
+        return ()
+
     def __init__(self, filesystem):
         if self.fake_pathlib is None:
             self.__class__.fake_pathlib = FakePathlibModule(filesystem)

pyfakefs/fake_scandir.py

@@ -258,6 +258,13 @@ class FakeScanDirModule(object):
     `fake_scandir_module = fake_filesystem.FakeScanDirModule(filesystem)`
     """
 
+    @staticmethod
+    def dir():
+        """Return the list of patched function names. Used for patching
+        functions imported from the module.
+        """
+        return 'scandir', 'walk'
+
     def __init__(self, filesystem):
         self.filesystem = filesystem
 

pyfakefs/tests/fake_filesystem_unittest_test.py

@@ -160,6 +160,18 @@ def test_import_path_from_pathlib(self):
             self.assertTrue(
                 pyfakefs.tests.import_as_example.check_if_exists3(file_path))
 
+    def test_import_function_from_os_path(self):
+        file_path = '/foo/bar'
+        self.fs.create_dir(file_path)
+        self.assertTrue(
+            pyfakefs.tests.import_as_example.check_if_exists5(file_path))
+
+    def test_import_function_from_os(self):
+        file_path = '/foo/bar'
+        self.fs.create_file(file_path, contents=b'abc')
+        stat_result = pyfakefs.tests.import_as_example.file_stat(file_path)
+        self.assertEqual(3, stat_result.st_size)
+
 
 class TestAttributesWithFakeModuleNames(TestPyfakefsUnittestBase):
     """Test that module attributes with names like `path` or `io` are not

pyfakefs/tests/import_as_example.py

@@ -15,7 +15,9 @@
 to be patched under another name.
 """
 from os import path
+from os.path import exists
 import os as my_os
+from os import stat
 
 try:
     from pathlib import Path
@@ -45,3 +47,13 @@ def check_if_exists3(filepath):
 def check_if_exists4(filepath, exists=my_os.path.exists):
     # this is a similar case as in the tempfile implementation under Posix
     return exists(filepath)
+
+
+def check_if_exists5(filepath):
+    # tests patching `exists` imported from os.path
+    return exists(filepath)
+
+
+def file_stat(filepath):
+    # tests patching `stat` imported from os
+    return stat(filepath)