From 563d373386d74eebd2a69eb374e94cc6e5b1548f Mon Sep 17 00:00:00 2001 From: Vladimir Date: Thu, 28 Jan 2021 21:51:24 +0200 Subject: [PATCH] feat: add dynamic update the lists of related and references (#273) --- Dockerfile | 7 +- README.md | 2 +- bin/generate_related_references.py | 113 +++++++++++++++++++++++++++++ docs/targets.md | 1 + modules/readme/Makefile | 3 + 5 files changed, 124 insertions(+), 2 deletions(-) create mode 100755 bin/generate_related_references.py diff --git a/Dockerfile b/Dockerfile index e63898f3..9fceaa85 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,12 @@ RUN apk --update --no-cache add \ jq \ libc6-compat \ make \ - py-pip && \ + python3-dev \ + py-pip \ + py3-ruamel.yaml && \ + pip3 install --no-cache-dir \ + iteration-utilities==0.11.0 \ + PyGithub==1.54.1 && \ git config --global advice.detachedHead false SHELL ["/bin/bash", "-o", "pipefail", "-c"] diff --git a/README.md b/README.md index 1b40e0dd..7d8cc3a4 100644 --- a/README.md +++ b/README.md @@ -358,7 +358,7 @@ In general, PRs are welcome. We follow the typical "fork-and-pull" Git workflow. ## Copyrights -Copyright © 2016-2020 [Cloud Posse, LLC](https://cloudposse.com) +Copyright © 2016-2021 [Cloud Posse, LLC](https://cloudposse.com) diff --git a/bin/generate_related_references.py b/bin/generate_related_references.py new file mode 100755 index 00000000..dbc4895b --- /dev/null +++ b/bin/generate_related_references.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 + +import json +import os +import subprocess + +import requests +from github import Github +from iteration_utilities import unique_everseen +from ruamel.yaml import YAML + +GH_TOKEN = os.environ["GH_TOKEN"] +GH_ORG_NAME = os.getenv("GH_ORG_NAME", "cloudposse") +GH_SEARCH_PATTERN = os.getenv("GH_SEARCH_PATTERN", "terraform-") +TF_MODULE_PATH = os.getenv("TF_MODULE_PATH", ".") +TF_CONFIG_INSPECT_BINARY_PATH = os.getenv( + "TF_CONFIG_INSPECT_BINARY_PATH", "terraform-config-inspect" +) +TF_REGISTRY_URL = "https://registry.terraform.io" + +gh = Github(GH_TOKEN) +yaml = YAML(typ="rt") +yaml.default_flow_style = False +yaml.preserve_quotes = False + + +def parse_gh(): + gh_repos = [] + for repo in gh.get_organization(GH_ORG_NAME).get_repos(): + if GH_SEARCH_PATTERN in repo.name: + repo_object = {} + repo_object["name"] = repo.name + repo_object["description"] = repo.description + repo_object["url"] = repo.html_url + gh_repos.append(repo_object) + return gh_repos + + +def tf_config_inspect(): + output = json.loads( + subprocess.check_output( + [TF_CONFIG_INSPECT_BINARY_PATH, TF_MODULE_PATH, "--json"], + stderr=subprocess.STDOUT, + ) + ) + return output + + +def parse_tf_registry(src_data, src_type): + items = [] + src_item = "module_calls" + if src_type == "providers": + src_item = "required_providers" + + for k, v in src_data[src_item].items(): + item_object = {} + url = TF_REGISTRY_URL + "/v1/" + src_type + "/" + v["source"] + r = requests.get(url=url).json() + + if src_type == "providers": + name_pattern = "terraform-provider-{}".format(r["name"]) + else: + name_pattern = "terraform-{}-{}".format(r["provider"], r["name"]) + item_object["name"] = name_pattern + + if src_type == "providers": + # description on GitHub looks better than on terraform-registry + gh_repo_info = gh.get_repo("{}/{}".format(r["namespace"], name_pattern)) + item_object["description"] = gh_repo_info.description + item_object["url"] = TF_REGISTRY_URL + "/providers/{}/{}/latest".format( + r["namespace"], r["name"] + ) + else: + item_object["description"] = r["description"] + item_object["url"] = r["source"] + + items.append(item_object) + return items + + +if __name__ == "__main__": + related_list = [] + reference_list = [] + + inspected_data = tf_config_inspect() + modules_list = parse_tf_registry(inspected_data, "modules") + providers_list = parse_tf_registry(inspected_data, "providers") + gh_repos_list = parse_gh() + + # this can be done in one line but it requires itertools + # and additional step to remove empty dicts + for m in unique_everseen(modules_list): + related_list.append(m) + for g in unique_everseen(gh_repos_list): + related_list.append(g) + for p in unique_everseen(providers_list): + reference_list.append(p) + + with open("{}/README.yaml".format(TF_MODULE_PATH)) as f: + readme = yaml.load(f) + + readme["related"] = related_list + + # ensure that "references" key is present and then insert data + if readme.get("references"): + readme["references"] = reference_list + else: + # create key "references" after the "related" + readme.insert(list(readme.keys()).index("related") + 1, "references", []) + readme["references"] = reference_list + + with open("{}/README.yaml".format(TF_MODULE_PATH), "w") as f: + yaml.dump(readme, f) diff --git a/docs/targets.md b/docs/targets.md index ffa0b810..3c7eacce 100644 --- a/docs/targets.md +++ b/docs/targets.md @@ -108,6 +108,7 @@ Available targets: readme/build Create README.md by building it from README.yaml readme/init Create basic minimalistic .README.md template file readme/lint Verify the `README.md` is up to date + readme/generate-related-references Generate related references block (e.i. `related`) in the README.yaml semver/export Export semver vars slack/notify Send webhook notification to slack slack/notify/build Send notification to slack using "build" template diff --git a/modules/readme/Makefile b/modules/readme/Makefile index f8e77988..4155d134 100644 --- a/modules/readme/Makefile +++ b/modules/readme/Makefile @@ -32,3 +32,6 @@ readme/build: readme/deps $(README_DEPS) @gomplate --file $(README_TEMPLATE_FILE) \ --out $(README_FILE) @echo "Generated $(README_FILE) from $(README_TEMPLATE_FILE) using data from $(README_TEMPLATE_YAML)" + +readme/generate-related-references: + @$(BUILD_HARNESS_PATH)/bin/generate_related_references.py