Add geneate proto workflow

tools/src/generate_proto_py/Pipfile

@@ -0,0 +1,13 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+gitpython = "*"
+pyyaml = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.9"

tools/src/generate_proto_py/generate_proto_wf.py

@@ -0,0 +1,149 @@
+import os
+import subprocess
+import tempfile
+from preprocessing import preprocess_openapi
+from git import Repo
+import argparse
+import yaml
+import re
+
+REPO_URL = "https://github.com/lucy66hw/openapi-generator.git"
+
+def clone_build(repo_url, branch_name="master"):
+    """
+    Clones the repository, builds it with maven, and runs the generated JAR.
+
+    Args:
+        repo_url (str): URL of the Git repository to clone.
+        branch_name (str): Branch to checkout.
+    """
+    clone_dir = tempfile.mkdtemp(dir=temp_dir)
+
+    try:
+        print(f"Cloning repository into: {clone_dir}")
+        repo = Repo.clone_from(repo_url, clone_dir)
+        print(f"Checking out branch: {branch_name}")
+        repo.git.checkout(branch_name)
+
+        print("Building the project with Maven...")
+        subprocess.run(["./mvnw", "clean", "install"], cwd=clone_dir, check=True)
+
+        jar_path = os.path.join(clone_dir, "modules", "openapi-generator-cli", "target", "openapi-generator-cli.jar")
+        if not os.path.exists(jar_path):
+            raise FileNotFoundError(f"JAR file not found: {jar_path}")
+
+        return jar_path
+
+    except FileNotFoundError as fnf_error:
+        print(f"File error: {fnf_error}")
+        raise
+    except subprocess.CalledProcessError as proc_error:
+        print(f"Process error during Maven or JAR execution: {proc_error}")
+        raise
+    except Exception as e:
+        print(f"Unexpected error: {e}")
+        raise
+
+def convert_proto(jar_path, jar_args=None):
+    jar_args = jar_args or []
+    print(f"Running the JAR: {jar_path}")
+    subprocess.run(["java", "-jar", jar_path] + jar_args, check=True)
+
+def run_merger_script(ts_file, source, output):
+    try:
+        result = subprocess.run(
+            ["ts-node", ts_file, "--source", source, "--output", output],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            check=True
+        )
+        print("Output from TypeScript script:")
+        print(result.stdout)
+    except subprocess.CalledProcessError as e:
+        print("Error while running TypeScript script:")
+        print(e.stderr)
+
+def get_git_root(current_path="."):
+    try:
+        repo = Repo(current_path, search_parent_directories=True)
+        return repo.git.rev_parse("--show-toplevel")
+    except Exception as e:
+        print(f"Error determining Git root: {e}")
+        raise
+
+def get_openapi_paths(openapi_file):
+    try:
+        with open(openapi_file, 'r') as file:
+            spec = yaml.safe_load(file)
+
+        paths = spec.get("paths", {})
+
+        return list(paths.keys())
+    except Exception as e:
+        print(f"Error reading OpenAPI file {openapi_file}: {e}")
+        return []
+
+def sanitize_path_name(path):
+    parts = path.split("/")
+
+    sanitized_parts = [re.sub(r'\W+', '', part).strip("_") for part in parts]
+
+    snake_case = "_".join(filter(None, sanitized_parts))
+    if snake_case == "":
+        return "root"
+    return snake_case.lower().strip("_")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Process OpenSearch API specification.")
+    parser.add_argument(
+        "-s", "--selected-paths",
+        nargs="+",
+        default=None,
+        help="API paths to convert proto."
+    )
+
+    parser.add_argument(
+        "-b", "--branch-name",
+        type=str,
+        default="master",
+        help="The branch name to checkout."
+    )
+
+    args = parser.parse_args()
+
+    current_path = os.getcwd()
+    git_root = get_git_root(current_path)
+
+    merger_file = os.path.join(git_root, "tools", "src", "merger", "merge.ts")
+    spec_source = os.path.join(git_root, "spec")
+    proto_output_dir = os.path.join(git_root, "proto")
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        # 1, merger
+        merger_output = os.path.join(temp_dir, "build", "opensearch-openapi.yaml")
+        run_merger_script(merger_file, spec_source, merger_output)
+
+
+        # 2, build tools
+        jar_path = clone_build(REPO_URL, args.branch_name)
+
+
+        selected_path = args.selected_paths or get_openapi_paths(merger_output)
+
+        for sp in selected_path:
+            # 3. preprocess openapi yaml file
+            path_name = sanitize_path_name(sp)
+            selected_output = os.path.join(temp_dir, "build", path_name, "filtered-opensearch-openapi.yaml")
+            preprocess_openapi(merger_output, selected_output, sp)
+            proto_out = os.path.join(proto_output_dir, path_name)
+
+            # 4. run tooling command
+            JAR_ARGS = [
+                "generate",
+                "-i", selected_output,
+                "-g", "protobuf-schema",
+                "-o",  proto_out,
+                "--additional-properties", "numberedFieldNumberList=true,startEnumsWithUnknown=true"
+            ]
+
+            convert_proto(jar_path, JAR_ARGS)

tools/src/generate_proto_py/preprocessing.py

@@ -0,0 +1,92 @@
+import yaml
+import os
+
+
+def collect_refs(item, used_refs, visited_refs, spec):
+    if isinstance(item, dict):
+        for key, value in item.items():
+            if key == '$ref':
+                used_refs.add(value)
+                resolve_nested_ref(value, used_refs, visited_refs, spec)
+            else:
+                collect_refs(value, used_refs, visited_refs, spec)
+    elif isinstance(item, list):
+        for i in item:
+            collect_refs(i, used_refs, visited_refs, spec)
+
+
+def resolve_nested_ref(ref, used_refs, visited_refs, spec):
+    if ref in visited_refs:
+        return
+
+    visited_refs.add(ref)
+
+    if ref.startswith("#/components/"):
+        ref_parts = ref.split('/')
+        schema = ref_parts[2]
+        component_name = ref_parts[3]
+
+        if schema in spec['components'] and component_name in spec['components'][schema]:
+            component = spec['components'][schema][component_name]
+            collect_refs(component, used_refs, visited_refs, spec)
+
+def rename_component(name):
+    if '___' in name:
+        return name.split('___')[-1]
+    return name
+
+
+def update_refs(item, rename_component):
+    if isinstance(item, dict):
+        for key, value in item.items():
+            if key == '$ref' and value.startswith("#/components/"):
+                ref_parts = value.split('/')
+                schema = ref_parts[2]
+                component_name = ref_parts[3]
+                new_name = rename_component(component_name)
+                item[key] = f"#/components/{schema}/{new_name}"
+            else:
+                update_refs(value, rename_component)
+    elif isinstance(item, list):
+        for i in item:
+            update_refs(i, rename_component)
+
+
+def preprocess_openapi(input_file, output_file, selected_path):
+    with open(input_file, 'r') as f:
+        spec = yaml.safe_load(f)
+
+    if selected_path in spec['paths']:
+        filtered_paths = {selected_path: spec['paths'][selected_path]}
+    else:
+        raise KeyError(f"The selected path '{selected_path}' is not found in the OpenAPI spec.")
+    spec['paths'] = filtered_paths
+
+    used_refs = set()
+    visited_refs = set()
+
+    for path_item in filtered_paths.values():
+        for method, operation in path_item.items():
+            if isinstance(operation, dict):
+                collect_refs(operation, used_refs, visited_refs, spec)
+
+    components = spec.get('components', {})
+    filtered_components = {}
+
+    for component_type, component_items in components.items():
+        renamed_items = {}
+        for name, details in component_items.items():
+            if f"#/components/{component_type}/{name}" in used_refs:
+                new_name = rename_component(name)
+                renamed_items[new_name] = details
+        filtered_components[component_type] = renamed_items
+
+    spec['components'] = filtered_components
+
+    update_refs(spec, rename_component)
+
+    os.makedirs(os.path.dirname(output_file), exist_ok=True)
+    with open(output_file, 'w') as f:
+        yaml.dump(spec, f, default_flow_style=False)
+
+    print(f"Filtered spec saved to {output_file}")

tools/tests/generate_proto_py/fixtures/animal.yaml

@@ -0,0 +1,52 @@
+openapi: 3.1.0
+info:
+  title: OpenSearch API
+  description: OpenSearch API
+  version: 1.0.0
+paths:
+  /pets:
+    get:
+      parameters:
+        - $ref: '#/components/parameters/parameter___Cat'
+  /animals:
+    get:
+      parameters:
+        - $ref: '#/components/parameters/parameter___Dog'
+      responses:
+        '200':
+          description: A list of animals
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/animals___Animal"
+components:
+  parameters:
+    parameter___Cat:
+      name: a
+      schema:
+        type: string
+      in: query
+    parameter___Dog:
+      name: b
+      schema:
+        type: string
+      in: query
+  schemas:
+    actions___Bark:
+      type: string
+    actions___Meow:
+      type: string
+    animals___Animal:
+      oneOf:
+        - $ref: '#/components/schemas/animals___Dog'
+        - $ref: '#/components/schemas/animals___Cat'
+    animals___Cat:
+      type: object
+      properties:
+        meow:
+          $ref: '#/components/schemas/actions___Meow'
+    animals___Dog:
+      type: object
+      properties:
+        bark:
+          $ref: '#/components/schemas/actions___Bark'

tools/tests/generate_proto_py/fixtures/expected.yaml

@@ -0,0 +1,43 @@
+openapi: 3.1.0
+info:
+  title: OpenSearch API
+  description: OpenSearch API
+  version: 1.0.0
+paths:
+  /animals:
+    get:
+      parameters:
+        - $ref: '#/components/parameters/Dog'
+      responses:
+        '200':
+          description: A list of animals
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Animal"
+components:
+  parameters:
+    Dog:
+      name: b
+      schema:
+        type: string
+      in: query
+  schemas:
+    Bark:
+      type: string
+    Meow:
+      type: string
+    Animal:
+      oneOf:
+        - $ref: '#/components/schemas/Dog'
+        - $ref: '#/components/schemas/Cat'
+    Cat:
+      type: object
+      properties:
+        meow:
+          $ref: '#/components/schemas/Meow'
+    Dog:
+      type: object
+      properties:
+        bark:
+          $ref: '#/components/schemas/Bark'