From 31a47b86ac4359f71f0f937c51287f9df130d031 Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Fri, 3 Jun 2022 13:03:01 -0700 Subject: [PATCH 1/2] add tab for config json schema --- metadata-ingestion/scripts/docgen.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/metadata-ingestion/scripts/docgen.py b/metadata-ingestion/scripts/docgen.py index 54cfbc7f9d4864..03434d291d3024 100644 --- a/metadata-ingestion/scripts/docgen.py +++ b/metadata-ingestion/scripts/docgen.py @@ -605,6 +605,11 @@ def generate( os.makedirs(config_dir, exist_ok=True) with open(f"{config_dir}/{plugin_name}_config.json", "w") as f: f.write(source_config_class.schema_json(indent=2)) + + create_or_update(source_documentation, + [platform_id, "plugins", plugin_name, "config_schema"], + source_config_class.schema_json(indent=2) or "", + ) table_md = gen_md_table_from_struct(source_config_class.schema()) create_or_update( @@ -644,6 +649,7 @@ def generate( with open(platform_doc_file, "w") as f: if "name" in platform_docs: + f.write(f"import Tabs from '@theme/Tabs';\nimport TabItem from '@theme/TabItem';\n\n") f.write(f"# {platform_docs['name']}\n") if len(platform_docs["plugins"].keys()) > 1: # More than one plugin used to provide integration with this platform @@ -716,6 +722,8 @@ def generate( f.write("\n```\n") if "config" in plugin_docs: f.write("\n### Config Details\n") + f.write(""" + \n\n""") f.write( "Note that a `.` is used to denote nested fields in the YAML recipe.\n\n" ) @@ -725,6 +733,15 @@ def generate( for doc in plugin_docs["config"]: f.write(doc) f.write("\n\n\n") + f.write(f""" + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below.\n\n +```javascript +{plugin_docs['config_schema']} +```\n\n + +\n\n""") # insert custom plugin docs after config details f.write(plugin_docs.get("custom_docs", "")) if "classname" in plugin_docs: From 1993f005fc7916ce5f1e02bbaf348239abfbe9cc Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Fri, 3 Jun 2022 14:07:05 -0700 Subject: [PATCH 2/2] upgrade dependency --- metadata-ingestion-modules/airflow-plugin/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-ingestion-modules/airflow-plugin/setup.py b/metadata-ingestion-modules/airflow-plugin/setup.py index ef6e49ab292aad..8ca4cad470f67e 100644 --- a/metadata-ingestion-modules/airflow-plugin/setup.py +++ b/metadata-ingestion-modules/airflow-plugin/setup.py @@ -29,7 +29,7 @@ def get_long_description(): "typing-inspect", "pydantic>=1.5.1", "apache-airflow >= 1.10.2", - "acryl-datahub[airflow]", + "acryl-datahub[airflow] >= 0.8.36", }