pythongh-121735: Fix module-adjacent references in zip files (python#123037)

* pythongh-116608: Apply style and compatibility changes from importlib_metadata.

* pythongh-121735: Ensure module-adjacent resources are loadable from a zipfile.

* pythongh-121735: Allow all modules to be processed by the ZipReader.

* Add blurb

* Remove update-zips script, unneeded.

* Remove unnecessary references to removed static fixtures.

* Remove zipdata fixtures, unused.

Author: jaraco

.gitattributes

@@ -27,8 +27,6 @@ Lib/test/cjkencodings/*                    noeol
 Lib/test/tokenizedata/coding20731.py       noeol
 Lib/test/decimaltestdata/*.decTest         noeol
 Lib/test/test_email/data/*.txt             noeol
-Lib/test/test_importlib/resources/data01/*           noeol
-Lib/test/test_importlib/resources/namespacedata01/*  noeol
 Lib/test/xmltestdata/*                     noeol
 
 # Shell scripts should have LF even on Windows because of Cygwin

Lib/importlib/resources/readers.py

@@ -34,8 +34,10 @@ def files(self):
 
 class ZipReader(abc.TraversableResources):
     def __init__(self, loader, module):
-        _, _, name = module.rpartition('.')
-        self.prefix = loader.prefix.replace('\\', '/') + name + '/'
+        self.prefix = loader.prefix.replace('\\', '/')
+        if loader.is_package(module):
+            _, _, name = module.rpartition('.')
+            self.prefix += name + '/'
         self.archive = loader.archive
 
     def open_resource(self, resource):

Lib/test/test_importlib/resources/data01/__init__.py

@@ -1,7 +1,6 @@
 import unittest
 from importlib import resources
 
-from . import data01
 from . import util
 
 
@@ -19,25 +18,21 @@ def test_contents(self):
         assert self.expected <= contents
 
 
-class ContentsDiskTests(ContentsTests, unittest.TestCase):
-    def setUp(self):
-        self.data = data01
+class ContentsDiskTests(ContentsTests, util.DiskSetup, unittest.TestCase):
+    pass
 
 
 class ContentsZipTests(ContentsTests, util.ZipSetup, unittest.TestCase):
     pass
 
 
-class ContentsNamespaceTests(ContentsTests, unittest.TestCase):
+class ContentsNamespaceTests(ContentsTests, util.DiskSetup, unittest.TestCase):
+    MODULE = 'namespacedata01'
+
     expected = {
         # no __init__ because of namespace design
         'binary.file',
         'subdirectory',
         'utf-16.file',
         'utf-8.file',
     }
-
-    def setUp(self):
-        from . import namespacedata01
-
-        self.data = namespacedata01

Lib/test/test_importlib/resources/data01/binary.file

@@ -6,11 +6,7 @@
 
 from importlib import resources
 from importlib.resources.abc import Traversable
-from . import data01
 from . import util
-from . import _path
-from test.support import os_helper
-from test.support import import_helper
 
 
 @contextlib.contextmanager
@@ -48,70 +44,96 @@ def test_old_parameter(self):
             resources.files(package=self.data)
 
 
-class OpenDiskTests(FilesTests, unittest.TestCase):
-    def setUp(self):
-        self.data = data01
+class OpenDiskTests(FilesTests, util.DiskSetup, unittest.TestCase):
+    pass
 
 
 class OpenZipTests(FilesTests, util.ZipSetup, unittest.TestCase):
     pass
 
 
-class OpenNamespaceTests(FilesTests, unittest.TestCase):
-    def setUp(self):
-        from . import namespacedata01
-
-        self.data = namespacedata01
+class OpenNamespaceTests(FilesTests, util.DiskSetup, unittest.TestCase):
+    MODULE = 'namespacedata01'
 
 
 class OpenNamespaceZipTests(FilesTests, util.ZipSetup, unittest.TestCase):
     ZIP_MODULE = 'namespacedata01'
 
 
-class SiteDir:
-    def setUp(self):
-        self.fixtures = contextlib.ExitStack()
-        self.addCleanup(self.fixtures.close)
-        self.site_dir = self.fixtures.enter_context(os_helper.temp_dir())
-        self.fixtures.enter_context(import_helper.DirsOnSysPath(self.site_dir))
-        self.fixtures.enter_context(import_helper.isolated_modules())
+class DirectSpec:
+    """
+    Override behavior of ModuleSetup to write a full spec directly.
+    """
+
+    MODULE = 'unused'
+
+    def load_fixture(self, name):
+        self.tree_on_path(self.spec)
 
 
-class ModulesFilesTests(SiteDir, unittest.TestCase):
+class ModulesFiles:
+    spec = {
+        'mod.py': '',
+        'res.txt': 'resources are the best',
+    }
+
     def test_module_resources(self):
         """
         A module can have resources found adjacent to the module.
         """
-        spec = {
-            'mod.py': '',
-            'res.txt': 'resources are the best',
-        }
-        _path.build(spec, self.site_dir)
         import mod
 
         actual = resources.files(mod).joinpath('res.txt').read_text(encoding='utf-8')
-        assert actual == spec['res.txt']
+        assert actual == self.spec['res.txt']
+
+
+class ModuleFilesDiskTests(DirectSpec, util.DiskSetup, ModulesFiles, unittest.TestCase):
+    pass
+
+
+class ModuleFilesZipTests(DirectSpec, util.ZipSetup, ModulesFiles, unittest.TestCase):
+    pass
+
 
+class ImplicitContextFiles:
+    set_val = textwrap.dedent(
+        """
+        import importlib.resources as res
+        val = res.files().joinpath('res.txt').read_text(encoding='utf-8')
+        """
+    )
+    spec = {
+        'somepkg': {
+            '__init__.py': set_val,
+            'submod.py': set_val,
+            'res.txt': 'resources are the best',
+        },
+    }
 
-class ImplicitContextFilesTests(SiteDir, unittest.TestCase):
-    def test_implicit_files(self):
+    def test_implicit_files_package(self):
         """
         Without any parameter, files() will infer the location as the caller.
         """
-        spec = {
-            'somepkg': {
-                '__init__.py': textwrap.dedent(
-                    """
-                    import importlib.resources as res
-                    val = res.files().joinpath('res.txt').read_text(encoding='utf-8')
-                    """
-                ),
-                'res.txt': 'resources are the best',
-            },
-        }
-        _path.build(spec, self.site_dir)
         assert importlib.import_module('somepkg').val == 'resources are the best'
 
+    def test_implicit_files_submodule(self):
+        """
+        Without any parameter, files() will infer the location as the caller.
+        """
+        assert importlib.import_module('somepkg.submod').val == 'resources are the best'
+
+
+class ImplicitContextFilesDiskTests(
+    DirectSpec, util.DiskSetup, ImplicitContextFiles, unittest.TestCase
+):
+    pass
+
+
+class ImplicitContextFilesZipTests(
+    DirectSpec, util.ZipSetup, ImplicitContextFiles, unittest.TestCase
+):
+    pass
+
 
 if __name__ == '__main__':
     unittest.main()

Lib/test/test_importlib/resources/data01/subdirectory/__init__.py

@@ -1,26 +1,38 @@
 import unittest
 import os
+import importlib
 
 from test.support import warnings_helper
 
 from importlib import resources
 
+from . import util
+
 # Since the functional API forwards to Traversable, we only test
 # filesystem resources here -- not zip files, namespace packages etc.
 # We do test for two kinds of Anchor, though.
 
 
 class StringAnchorMixin:
-    anchor01 = 'test.test_importlib.resources.data01'
-    anchor02 = 'test.test_importlib.resources.data02'
+    anchor01 = 'data01'
+    anchor02 = 'data02'
 
 
 class ModuleAnchorMixin:
-    from . import data01 as anchor01
-    from . import data02 as anchor02
+    @property
+    def anchor01(self):
+        return importlib.import_module('data01')
+
+    @property
+    def anchor02(self):
+        return importlib.import_module('data02')
+
 
+class FunctionalAPIBase(util.DiskSetup):
+    def setUp(self):
+        super().setUp()
+        self.load_fixture('data02')
 
-class FunctionalAPIBase:
     def _gen_resourcetxt_path_parts(self):
         """Yield various names of a text file in anchor02, each in a subTest"""
         for path_parts in (
@@ -228,16 +240,16 @@ def test_text_errors(self):
 
 
 class FunctionalAPITest_StringAnchor(
-    unittest.TestCase,
-    FunctionalAPIBase,
     StringAnchorMixin,
+    FunctionalAPIBase,
+    unittest.TestCase,
 ):
     pass
 
 
 class FunctionalAPITest_ModuleAnchor(
-    unittest.TestCase,
-    FunctionalAPIBase,
     ModuleAnchorMixin,
+    FunctionalAPIBase,
+    unittest.TestCase,
 ):
     pass

Lib/test/test_importlib/resources/data01/subdirectory/binary.file

@@ -1,7 +1,6 @@
 import unittest
 
 from importlib import resources
-from . import data01
 from . import util
 
 
@@ -65,24 +64,20 @@ def test_open_text_FileNotFoundError(self):
             target.open(encoding='utf-8')
 
 
-class OpenDiskTests(OpenTests, unittest.TestCase):
-    def setUp(self):
-        self.data = data01
-
+class OpenDiskTests(OpenTests, util.DiskSetup, unittest.TestCase):
+    pass
 
-class OpenDiskNamespaceTests(OpenTests, unittest.TestCase):
-    def setUp(self):
-        from . import namespacedata01
 
-        self.data = namespacedata01
+class OpenDiskNamespaceTests(OpenTests, util.DiskSetup, unittest.TestCase):
+    MODULE = 'namespacedata01'
 
 
 class OpenZipTests(OpenTests, util.ZipSetup, unittest.TestCase):
     pass
 
 
 class OpenNamespaceZipTests(OpenTests, util.ZipSetup, unittest.TestCase):
-    ZIP_MODULE = 'namespacedata01'
+    MODULE = 'namespacedata01'
 
 
 if __name__ == '__main__':

Lib/test/test_importlib/resources/data01/utf-16.file

@@ -3,7 +3,6 @@
 import unittest
 
 from importlib import resources
-from . import data01
 from . import util
 
 
@@ -25,9 +24,7 @@ def test_reading(self):
             self.assertEqual('Hello, UTF-8 world!\n', path.read_text(encoding='utf-8'))
 
 
-class PathDiskTests(PathTests, unittest.TestCase):
-    data = data01
-
+class PathDiskTests(PathTests, util.DiskSetup, unittest.TestCase):
     def test_natural_path(self):
         # Guarantee the internal implementation detail that
         # file-system-backed resources do not get the tempdir

Lib/test/test_importlib/resources/data01/utf-8.file

@@ -1,7 +1,7 @@
 import unittest
 
 from importlib import import_module, resources
-from . import data01
+
 from . import util
 
 
@@ -51,8 +51,8 @@ def test_read_text_with_errors(self):
         )
 
 
-class ReadDiskTests(ReadTests, unittest.TestCase):
-    data = data01
+class ReadDiskTests(ReadTests, util.DiskSetup, unittest.TestCase):
+    pass
 
 
 class ReadZipTests(ReadTests, util.ZipSetup, unittest.TestCase):
@@ -68,15 +68,12 @@ def test_read_submodule_resource_by_name(self):
         self.assertEqual(result, bytes(range(4, 8)))
 
 
-class ReadNamespaceTests(ReadTests, unittest.TestCase):
-    def setUp(self):
-        from . import namespacedata01
-
-        self.data = namespacedata01
+class ReadNamespaceTests(ReadTests, util.DiskSetup, unittest.TestCase):
+    MODULE = 'namespacedata01'
 
 
 class ReadNamespaceZipTests(ReadTests, util.ZipSetup, unittest.TestCase):
-    ZIP_MODULE = 'namespacedata01'
+    MODULE = 'namespacedata01'
 
     def test_read_submodule_resource(self):
         submodule = import_module('namespacedata01.subdirectory')