Issue OpenCyphal#99 (modified) proposal

pydsdl/__init__.py

@@ -26,7 +26,6 @@
 
 # Never import anything that is not available here - API stability guarantees are only provided for the exposed items.
 from ._dsdl import PrintOutputHandler as PrintOutputHandler
-from ._dsdl import DsdlFile as DsdlFile
 from ._namespace import read_namespace as read_namespace
 from ._namespace import read_files as read_files
 

pydsdl/_dsdl.py

@@ -15,7 +15,9 @@
 class DsdlFile(ABC):
     """
     Interface for DSDL files. This interface is used by the parser to abstract DSDL type details inferred from the
-    filesystem.
+    filesystem. Where properties are duplicated between the composite type and this file the composite type is to be
+    considered canonical. The properties directly on this class are inferred from the dsdl file path before the
+    composite type has been parsed.
     """
 
     @property

pydsdl/_namespace.py

@@ -93,7 +93,7 @@ def read_namespace(
     allow_root_namespace_name_collision: bool = True,
 ) -> List[_serializable.CompositeType]:
     """
-    This function is the main entry point of the library.
+    This function is a main entry point for the library.
     It reads all DSDL definitions from the specified root namespace directory and produces the annotated AST.
 
     :param root_namespace_directory: The path of the root namespace directory that will be read.
@@ -120,10 +120,10 @@ def read_namespace(
              the same root namespace name multiple times in the lookup dirs. This will enable defining a namespace
              partially and let other entities define new messages or new sub-namespaces in the same root namespace.
 
-    :return: A list of :class:`pydsdl.CompositeType` sorted lexicographically by full data type name,
-             then by major version (newest version first), then by minor version (newest version first).
-             The ordering guarantee allows the caller to always find the newest version simply by picking
-             the first matching occurrence.
+    :return: A list of :class:`pydsdl.CompositeType` found under the `root_namespace_directory` and sorted
+             lexicographically by full data type name, then by major version (newest version first), then by minor
+             version (newest version first). The ordering guarantee allows the caller to always find the newest version
+             simply by picking the first matching occurrence.
 
     :raises: :class:`pydsdl.FrontendError`, :class:`MemoryError`, :class:`SystemError`,
         :class:`OSError` if directories do not exist or inaccessible,
@@ -163,17 +163,37 @@ def read_files(
     lookup_directories: Union[None, Path, str, Iterable[Union[Path, str]]] = None,
     print_output_handler: Optional[PrintOutputHandler] = None,
     allow_unregulated_fixed_port_id: bool = False,
-    allow_root_namespace_name_collision: bool = True,
-) -> Tuple[List[DsdlFile], List[DsdlFile]]:
+) -> Tuple[List[_serializable.CompositeType], List[_serializable.CompositeType]]:
     """
-    This function is the main entry point of the library.
-    It reads all DSDL definitions from the specified root namespace directory and produces the annotated AST.
+    This function is a main entry point for the library.
+    It reads all DSDL definitions from the specified `dsdl_files` and produces the annotated AST for these types and
+    the transitive closure of the types they depend on.
 
-    :param root_namespace_directory: The path of the root namespace directory that will be read.
-        For example, ``dsdl/uavcan`` to read the ``uavcan`` namespace.
+    :param dsdl_files: A list of paths to dsdl files to parse.
+
+    :param root_namespace_directories_or_names: This can be a set of names of root namespaces or relative paths to
+        root namespaces. All `dsdl_files` provided must be under one of these roots. For example, given:
+
+        ```
+        dsdl_files = [
+                        Path("workspace/project/types/animals/felines/Tabby.1.0"),
+                        Path("workspace/project/types/animals/canines/Boxer.1.0")
+                        Path("workspace/project/types/plants/trees/DouglasFir.1.0")
+                    ]
+        ```
+
+        then this argument must be one of:
+
+        ```
+        root_namespace_directories_or_names = ["animals", "plants"]
+        root_namespace_directories_or_names = [
+                                                    Path("workspace/project/types/animals"),
+                                                    Path("workspace/project/types/plants")
+                                            ]
+    ```
 
     :param lookup_directories: List of other namespace directories containing data type definitions that are
-        referred to from the target root namespace. For example, if you are reading a vendor-specific namespace,
+        referred to from the target dsdl files. For example, if you are reading vendor-specific types,
         the list of lookup directories should always include a path to the standard root namespace ``uavcan``,
         otherwise the types defined in the vendor-specific namespace won't be able to use data types from the
         standard namespace.
@@ -189,14 +209,11 @@ def read_files(
         This is a dangerous feature that must not be used unless you understand the risks.
         Please read https://opencyphal.org/guide.
 
-    :param allow_root_namespace_name_collision: Allow using the source root namespace name in the look up dirs or
-             the same root namespace name multiple times in the lookup dirs. This will enable defining a namespace
-             partially and let other entities define new messages or new sub-namespaces in the same root namespace.
-
-    :return: A list of :class:`pydsdl.CompositeType` sorted lexicographically by full data type name,
-             then by major version (newest version first), then by minor version (newest version first).
-             The ordering guarantee allows the caller to always find the newest version simply by picking
-             the first matching occurrence.
+    :return: A Tuple of lists of :class:`pydsdl.CompositeType`. The first index in the Tuple are the types parsed from
+        the `dsdl_files` argument. The second index are types that the target `dsdl_files` utilizes.
+        A note for using these values to describe build dependencies: each :class:`pydsdl.CompositeType` has two
+        fields that provide links back to the filesystem where the dsdl files read when parsing the type were found;
+        `source_file_path` and `source_file_path_to_root`.
 
     :raises: :class:`pydsdl.FrontendError`, :class:`MemoryError`, :class:`SystemError`,
         :class:`OSError` if directories do not exist or inaccessible,
@@ -219,11 +236,11 @@ def read_files(
     for x in target_dsdl_definitions:
         _logger.debug(_LOG_LIST_ITEM_PREFIX + str(x.file_path))
 
-    root_namespaces = dsdl_file_sort({f.root_namespace.resolve() for f in target_dsdl_definitions})
+    root_namespaces = {f.root_namespace_path.resolve() for f in target_dsdl_definitions}
     lookup_directories_path_list = _construct_lookup_directories_path_list(
         root_namespaces,
         dsdl_normalize_paths_argument(lookup_directories, cast(Callable[[Iterable], List[Path]], list)),
-        allow_root_namespace_name_collision,
+        True,
     )
 
     reader = _complete_read_function(
@@ -232,7 +249,7 @@ def read_files(
         NamespaceClosureReader(allow_unregulated_fixed_port_id, print_output_handler),
     )
 
-    return (reader.direct.files, reader.transitive.files)
+    return (reader.direct.types, reader.transitive.types)
 
 
 # +--[INTERNAL API::PUBLIC API HELPERS]-------------------------------------------------------------------------------+
@@ -284,7 +301,7 @@ def _complete_read_function(
 
 
 def _construct_lookup_directories_path_list(
-    root_namespace_directories: List[Path],
+    root_namespace_directories: Iterable[Path],
     lookup_directories_path_list: List[Path],
     allow_root_namespace_name_collision: bool,
 ) -> List[Path]:
@@ -515,7 +532,7 @@ def _ensure_minor_version_compatibility_pairwise(
 
 
 def _ensure_no_common_usage_errors(
-    root_namespace_directories: List[Path], lookup_directories: Iterable[Path], reporter: Callable[[str], None]
+    root_namespace_directories: Iterable[Path], lookup_directories: Iterable[Path], reporter: Callable[[str], None]
 ) -> None:
     suspicious_base_names = [
         "public_regulated_data_types",

pydsdl/_serializable/_composite.py

@@ -97,9 +97,29 @@ def __init__(  # pylint: disable=too-many-arguments
                 "Name is too long: %r is longer than %d characters" % (self._name, self.MAX_NAME_LENGTH)
             )
 
-        for component in self._name.split(self.NAME_COMPONENT_SEPARATOR):
+        self._name_components = self._name.split(self.NAME_COMPONENT_SEPARATOR)
+        for component in self._name_components:
             check_name(component)
 
+        def search_up_for_root(path: Path, namespace_components: typing.List[str]) -> Path:
+            if len(namespace_components) == 0:
+                raise InvalidNameError( "Path to file without a namepace. All dsdl files must be contained within "
+                                        f"folders corresponding to their namespaces ({self._source_file_path})"
+                )
+            if namespace_components[-1] != path.stem:
+                raise InvalidNameError(f"{path.stem} != {namespace_components[-1]}. Source file directory structure "
+                                       f"is not consistent with the type's namespace ({self._name_components}, "
+                                       f"{self._source_file_path})"
+                )
+            if len(namespace_components) == 1:
+                return path
+            return search_up_for_root(path.parent, namespace_components[:-1])
+
+        self._path_to_root_namespace = search_up_for_root(
+            self._source_file_path.parent,
+            (self.namespace_components if not self._has_parent_service else self.namespace_components[:-1])
+        )
+
         # Version check
         version_valid = (
             (0 <= self._version.major <= self.MAX_VERSION_NUMBER)
@@ -148,7 +168,12 @@ def full_name(self) -> str:
     @property
     def name_components(self) -> typing.List[str]:
         """Components of the full name as a list, e.g., ``['uavcan', 'node', 'Heartbeat']``."""
-        return self._name.split(CompositeType.NAME_COMPONENT_SEPARATOR)
+        return self._name_components
+
+    @property
+    def namespace_components(self) -> typing.List[str]:
+        """Components of the namspace as a list, e.g., ``['uavcan', 'node']``."""
+        return self._name_components[:-1]
 
     @property
     def short_name(self) -> str:
@@ -163,7 +188,7 @@ def doc(self) -> str:
     @property
     def full_namespace(self) -> str:
         """The full name without the short name, e.g., ``uavcan.node`` for ``uavcan.node.Heartbeat``."""
-        return str(CompositeType.NAME_COMPONENT_SEPARATOR.join(self.name_components[:-1]))
+        return str(CompositeType.NAME_COMPONENT_SEPARATOR.join(self.namespace_components))
 
     @property
     def root_namespace(self) -> str:
@@ -239,10 +264,30 @@ def has_fixed_port_id(self) -> bool:
     @property
     def source_file_path(self) -> Path:
         """
-        For synthesized types such as service request/response sections, this property is defined as an empty string.
+        The path to the dsdl file from which this type was read.
+        For synthesized types such as service request/response sections, this property is the path to the service type
+        since request and response types are defined within the service type's dsdl file.
         """
         return self._source_file_path
 
+    @property
+    def source_file_path_to_root(self) -> Path:
+        """
+        The path to the folder that is the root namespace folder for the `source_file_path` this type was read from.
+        The `source_file_path` will always be relative to the `source_file_path_to_root` but not all types that share
+        the same `root_namespace` will have the same path to their root folder since types may be contributed to a
+        root namespace from several different file trees. For example:
+
+        ```
+        path0 = "workspace_0/project_a/types/animal/feline/Tabby.1.0.dsdl"
+        path1 = "workspace_1/project_b/types/animal/canine/Boxer.1.0.dsdl"
+        ```
+
+        In these examples path0 and path1 will produce composite types with `animal` as the root namespace but both
+        with have different `source_file_path_to_root` paths.
+        """
+        return self._path_to_root_namespace
+
     @property
     def alignment_requirement(self) -> int:
         # This is more general than required by the Specification, but it is done this way in case if we decided

test/test_public_types.py

@@ -28,7 +28,7 @@ def dsdl_printer(dsdl_file: Path, line: int, message: str) -> None:
     logging.info(f"{dsdl_file}:{line}: {message}")
 
 
-def _unittest_public_types(public_types: Path) -> None:
+def _unittest_public_types_namespaces(public_types: Path) -> None:
     """
     Sanity check to ensure that the public types can be read. This also allows us to debug
     against a real dataset.
@@ -42,3 +42,21 @@ def _unittest_public_types(public_types: Path) -> None:
     ps = pstats.Stats(pr, stream=s).sort_stats(sortby)
     ps.print_stats()
     print(s.getvalue())
+
+
+def _unittest_public_types_files(public_types: Path) -> None:
+    """
+    Sanity check to ensure that the public types can be read. This also allows us to debug
+    against a real dataset.
+    """
+    pr = cProfile.Profile()
+    pr.enable()
+    node_types = list(public_types.glob("node/**/*.dsdl"))
+    assert(len(node_types) > 0)
+    _ = pydsdl.read_files(node_types, {"uavcan"})
+    pr.disable()
+    s = io.StringIO()
+    sortby = SortKey.TIME
+    ps = pstats.Stats(pr, stream=s).sort_stats(sortby)
+    ps.print_stats()
+    print(s.getvalue())