feat: generate showcase using docker image

.github/scripts/hermetic_library_generation.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -e
+set -ex
 # This script should be run at the root of the repository.
 # This script is used to, when a pull request changes the generation
 # configuration (generation_config.yaml by default) or Dockerfile:
@@ -95,6 +95,14 @@ changed_libraries=$(python hermetic_build/common/cli/get_changed_libraries.py cr
   --current-generation-config-path="${generation_config}")
 echo "Changed libraries are: ${changed_libraries:-"No changed library"}."
 
+# do not generate showcase automatically
+changed_libraries=$(echo "${changed_libraries}" | sed 's/showcase,//' | sed 's/,showcase//' | sed 's/showcase//')
+echo "${changed_libraries}"
+if [[ -z "${changed_libraries}" ]]; then
+  echo 'No libraries to generate. Aborting generation'
+  exit 0
+fi
+
 # run hermetic code generation docker image.
 docker run \
   --rm \

generation_config.yaml

@@ -1,8 +1,18 @@
 googleapis_commitish: 3cef095370b3176f67542e527ffde81e019df091
-# the libraries are ordered with respect to library name, which is
-# java-{library.library_name} or java-{library.api-shortname} when
-# library.library_name is not defined.
+# the libraries are ordered with respect to library name, which defaults to
+# {library.folder_name} or java-{library.library_name} or
+# java-{library.api-shortname} in sequence (i.e. if first is None, then try second and so on).
 libraries:
+- api_shortname: showcase
+  folder_name: showcase # prevents java-showcase
+  excluded_poms: gapic-showcase-bom
+  name_pretty: Showcase
+  api_description: Showcase module
+  product_documentation: https://cloud.google.com/dummy
+  distribution_name: com.google.cloud:gapic-showcase
+  library_type: OTHER
+  GAPICs:
+  - proto_path: schema/google/showcase/v1beta1
 - api_shortname: common-protos
   name_pretty: Common Protos
   product_documentation: https://github.com/googleapis/api-common-protos
@@ -14,18 +24,18 @@ libraries:
   excluded_poms: "proto-google-common-protos-bom,proto-google-common-protos"
   library_type: OTHER
   GAPICs:
-  - proto_path: google/api
-  - proto_path: google/apps/card/v1
-  - proto_path: google/cloud
-  - proto_path: google/cloud/audit
-  - proto_path: google/cloud/location
-  - proto_path: google/geo/type
-  - proto_path: google/logging/type
-  - proto_path: google/longrunning
-  - proto_path: google/rpc
-  - proto_path: google/rpc/context
-  - proto_path: google/shopping/type
-  - proto_path: google/type
+    - proto_path: google/api
+    - proto_path: google/apps/card/v1
+    - proto_path: google/cloud
+    - proto_path: google/cloud/audit
+    - proto_path: google/cloud/location
+    - proto_path: google/geo/type
+    - proto_path: google/logging/type
+    - proto_path: google/longrunning
+    - proto_path: google/rpc
+    - proto_path: google/rpc/context
+    - proto_path: google/shopping/type
+    - proto_path: google/type
 - api_shortname: iam
   name_pretty: IAM
   product_documentation: https://cloud.google.com/iam

hermetic_build/README.md

@@ -156,6 +156,7 @@ They are shared by all GAPICs of a library.
 | cloud_api             |    No    | `true` if not specified                                                                                                                   |
 | requires-billing      |    No    | `true` if not specified                                                                                                                   |
 | transport             |    No    | must be one of `grpc`, `rest` or `both`. This value would only be used for generating .repo-metadata.json and relevant sections in README |
+| folder_name           |    No    | Only used in monorepos. This allows to override the default folder name where the library is created (e.g. monorepo/my-folder)            |
 
 
 Note that `cloud_prefix` is `cloud-` if `cloud_api` is `true`; empty otherwise.

hermetic_build/common/model/gapic_inputs.py

@@ -19,24 +19,32 @@
 proto_library_pattern = r"""
 proto_library_with_info\(
 (.*?)
-\)
+\\?[\nn]\)
 """
 gapic_pattern = r"""
 java_gapic_library\(
 (.*?)
-\)
+\\?[\nn]\)
 """
 assembly_pattern = r"""
 java_gapic_assembly_gradle_pkg\(
 (.*?)
-\)
+\\?[\nn]\)
 """
 # match a line which the first character is "#".
 comment_pattern = r"^\s*\#+"
-pattern_to_proto = {
-    r"//google/cloud:common_resources_proto": "google/cloud/common_resources.proto",
-    r"//google/cloud/location:location_proto": "google/cloud/location/locations.proto",
-    r"//google/iam/v1:iam_policy_proto": "google/iam/v1/iam_policy.proto",
+# This pattern ignores api definitions that externally reference other repos.
+# For example: @googleapis_repo//google/cloud/...
+pattern_proto_library_to_additional_protos = {
+    r".*//google/cloud:common_resources_proto": "google/cloud/common_resources.proto",
+    r".*//google/cloud/location:location_proto": "google/cloud/location/locations.proto",
+    r".*//google/iam/v1:iam_policy_proto": "google/iam/v1/iam_policy.proto",
+}
+# This is a special case for showcase, where the additional protos
+# are not declared in the gapic library instead
+pattern_gapic_library_to_additional_protos = {
+    r".*//google/cloud/location:location_java_proto": "google/cloud/location/locations.proto",
+    r".*//google/iam/v1:iam_java_proto": "google/iam/v1/iam_policy.proto",
 }
 transport_pattern = r"transport = \"(.*?)\""
 rest_pattern = r"rest_numeric_enums = True"
@@ -108,9 +116,8 @@ def parse_build_str(build_str: str, versioned_path: str) -> GapicInputs:
         proto_library_pattern, re.DOTALL | re.VERBOSE
     ).findall(build_str)
     additional_protos = ""
-    if len(proto_library_target) > 0:
-        additional_protos = __parse_additional_protos(proto_library_target[0])
     gapic_target = re.compile(gapic_pattern, re.DOTALL | re.VERBOSE).findall(build_str)
+    additional_protos = __parse_additional_protos(proto_library_target, gapic_target)
     assembly_target = re.compile(assembly_pattern, re.DOTALL | re.VERBOSE).findall(
         build_str
     )
@@ -140,19 +147,42 @@ def parse_build_str(build_str: str, versioned_path: str) -> GapicInputs:
     )
 
 
-def __parse_additional_protos(proto_library_target: str) -> str:
-    res = [" "]
-    lines = proto_library_target.split("\n")
-    for line in lines:
+def __get_lines_without_comments(target: list[str]):
+    """
+    skips comment lines
+    """
+    if len(target) == 0:
+        return []
+    result = []
+    for line in target[0].split("\n"):
         if len(re.findall(comment_pattern, line)) != 0:
-            # skip a line which the first charactor is "#" since it's
+            # skip lines whose first character is "#" since it's
             # a comment.
             continue
-        for pattern in pattern_to_proto:
+        result.append(line)
+    return result
+
+
+def __parse_additional_protos(
+    proto_library_target: list[str], gapic_library_target: list[str]
+) -> str:
+    res = [" "]
+    lines = __get_lines_without_comments(proto_library_target)
+    # first, parse the proto library definition
+    for line in lines:
+        for pattern in pattern_proto_library_to_additional_protos:
+            if len(re.findall(pattern, line)) == 0:
+                continue
+            res.append(pattern_proto_library_to_additional_protos[pattern])
+    # then, parse the gapic library definition
+    lines = __get_lines_without_comments(gapic_library_target)
+    for line in lines:
+        for pattern in pattern_gapic_library_to_additional_protos:
             if len(re.findall(pattern, line)) == 0:
                 continue
-            res.append(pattern_to_proto[pattern])
-    return " ".join(res)
+            res.append(pattern_gapic_library_to_additional_protos[pattern])
+    # finally, return the additional protos without duplicates
+    return " ".join(list(dict.fromkeys(res)))
 
 
 def __parse_transport(gapic_target: str) -> str:

hermetic_build/common/model/generation_config.py

@@ -160,6 +160,7 @@ def from_yaml(path_to_yaml: str) -> GenerationConfig:
             recommended_package=__optional(library, "recommended_package", None),
             min_java_version=__optional(library, "min_java_version", None),
             transport=__optional(library, "transport", None),
+            folder_name=__optional(library, "folder_name", None),
         )
         parsed_libraries.append(new_library)
 

hermetic_build/common/model/library_config.py

@@ -54,6 +54,7 @@ def __init__(
         recommended_package: Optional[str] = None,
         min_java_version: Optional[int] = None,
         transport: Optional[str] = None,
+        folder_name: Optional[str] = None,
     ):
         self.api_shortname = api_shortname
         self.api_description = api_description
@@ -81,6 +82,18 @@ def __init__(
         self.min_java_version = min_java_version
         self.distribution_name = self.__get_distribution_name(distribution_name)
         self.transport = self.__validate_transport(transport)
+        self.folder_name = folder_name
+
+    def get_folder_name(self, language: str = "java") -> str:
+        """
+        This property is meant to be used for generating monorepos.
+        It will not have any effect when generating a non-monorepo library.
+        It allows specifying the final library's name inside the repo.
+        For example, sdk-platform-java/showcase
+        """
+        if self.folder_name is not None:
+            return self.folder_name
+        return f"{language}-{self.get_library_name()}"
 
     def get_library_name(self) -> str:
         """

hermetic_build/common/tests/model/gapic_inputs_unit_tests.py

@@ -24,6 +24,14 @@ class UtilitiesTest(unittest.TestCase):
                 "BUILD_iam_locations.bazel",
                 "  google/cloud/location/locations.proto google/iam/v1/iam_policy.proto",
             ),
+            (
+                "BUILD_additional_protos_in_gapic.bazel",
+                "  google/cloud/location/locations.proto google/iam/v1/iam_policy.proto",
+            ),
+            (
+                "BUILD_target_with_nested_parenthesis.bazel",
+                "  google/cloud/location/locations.proto google/iam/v1/iam_policy.proto",
+            ),
         ]
     )
     def test_gapic_inputs_parse_additional_protos(self, build_name, expected):

hermetic_build/common/tests/resources/misc/BUILD_additional_protos_in_gapic.bazel

@@ -0,0 +1,6 @@
+java_gapic_library(
+    deps = [
+        "//google/cloud/location:location_java_proto",
+        "//google/iam/v1:iam_java_proto"
+   ],
+)

hermetic_build/common/tests/resources/misc/BUILD_target_with_nested_parenthesis.bazel

@@ -0,0 +1,13 @@
+proto_library_with_info(
+  # this comment should not indicate an end of a pattern match )
+  deps = [
+    "//google/cloud/location:location_proto",
+  ]
+)
+
+java_gapic_library(
+  # this comment should not indicate an end of a pattern match )
+  deps = [
+    "//google/iam/v1:iam_java_proto"
+  ],
+)

hermetic_build/library_generation/cli/entry_point.py

@@ -133,12 +133,16 @@ def __generate_repo_impl(
     )
 
 
-def _needs_full_repo_generation(generation_config: GenerationConfig) -> bool:
+def _needs_full_repo_generation(
+    generation_config: GenerationConfig, includes: Optional[str]
+) -> bool:
     """
     Whether you should need a full repo generation, i.e., generate all
     libraries in the generation configuration.
+    If only showcase is specified, then it will return to false despite
+    other conditions (allows overriding contains_common_protos())
     """
-    return (
+    return (includes is None or includes.strip() != "showcase") and (
         not generation_config.is_monorepo()
         or generation_config.contains_common_protos()
     )
@@ -147,7 +151,7 @@ def _needs_full_repo_generation(generation_config: GenerationConfig) -> bool:
 def _parse_library_name_from(
     includes: Optional[str], generation_config: GenerationConfig
 ) -> Optional[list[str]]:
-    if includes is None or _needs_full_repo_generation(generation_config):
+    if includes is None or _needs_full_repo_generation(generation_config, includes):
         return None
     return [library_name.strip() for library_name in includes.split(",")]
 

hermetic_build/library_generation/utils/utilities.py

@@ -145,9 +145,9 @@ def prepare_repo(
     os.makedirs(output_folder, exist_ok=True)
     libraries = {}
     for library in library_config:
-        library_name = f"{language}-{library.get_library_name()}"
+        library_folder_name = library.get_folder_name(language)
         library_path = (
-            f"{repo_path}/{library_name}"
+            f"{repo_path}/{library_folder_name}"
             if gen_config.is_monorepo()
             else f"{repo_path}"
         )

showcase/.OwlBot-hermetic.yaml

@@ -0,0 +1,35 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+deep-remove-regex:
+- "/showcase/grpc-gapic-showcase-v1beta1/src"
+- "/showcase/proto-gapic-showcase-v1beta1/src"
+- "/showcase/gapic-showcase/src"
+- "/showcase/samples/snippets/generated"
+
+deep-preserve-regex:
+- "/showcase/gapic-showcase/src/test"
+
+deep-copy-regex:
+- source: "/schema/google/showcase/(v.*)/.*-java/proto-.*/src"
+  dest: "/owl-bot-staging/showcase/$1/proto-gapic-showcase-$1/src"
+- source: "/schema/google/showcase/(v.*)/.*-java/grpc-.*/src"
+  dest: "/owl-bot-staging/showcase/$1/grpc-gapic-showcase-$1/src"
+- source: "/schema/google/showcase/(v.*)/.*-java/gapic-.*/src"
+  dest: "/owl-bot-staging/showcase/$1/gapic-showcase/src"
+- source: "/schema/google/showcase/(v.*)/.*-java/samples/snippets/generated"
+  dest: "/owl-bot-staging/showcase/$1/samples/snippets/generated"
+
+api-name: showcase

showcase/.repo-metadata.json

@@ -0,0 +1,16 @@
+{
+  "api_shortname": "showcase",
+  "name_pretty": "Showcase",
+  "product_documentation": "https://cloud.google.com/dummy",
+  "api_description": "Showcase module",
+  "client_documentation": "https://cloud.google.com/java/docs/reference/gapic-showcase/latest/overview",
+  "release_level": "preview",
+  "transport": "both",
+  "language": "java",
+  "repo": "googleapis/sdk-platform-java",
+  "repo_short": "java-showcase",
+  "distribution_name": "com.google.cloud:gapic-showcase",
+  "library_type": "OTHER",
+  "requires_billing": true,
+  "excluded_poms": "gapic-showcase-bom"
+}

showcase/README.md

@@ -77,11 +77,6 @@ Note:
 ## Update the Golden Showcase Files
 
 Open a new terminal window in the root project directory.
-Note that the underlying scripts will modify your `$HOME` folder by creating a
-`.library_generation` folder.
-This is the well-known location of the artifacts the generation scripts use.
-For more details see the
-[development guide](https://github.com/googleapis/sdk-platform-java/blob/main/library_generation/DEVELOPMENT.md#the-hermetic-builds-well-known-folder)
 
 ```shell
 # In repository's root directory

showcase/gapic-showcase/pom.xml

@@ -1,7 +1,5 @@
 <?xml version='1.0' encoding='UTF-8'?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <groupId>com.google.cloud</groupId>
   <artifactId>gapic-showcase</artifactId>
@@ -104,6 +102,7 @@
     <dependency>
       <groupId>com.google.api.grpc</groupId>
       <artifactId>grpc-google-common-protos</artifactId>
+      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>com.google.api.grpc</groupId>