From f1ee65b9f82b6ec3749e31186cd74ee6f0b7614f Mon Sep 17 00:00:00 2001 From: SweetOps Date: Tue, 19 Jan 2021 22:32:13 +0200 Subject: [PATCH 1/7] feat: add dynamic update the list of related references --- modules/readme/update_related_references.py | 88 +++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 modules/readme/update_related_references.py diff --git a/modules/readme/update_related_references.py b/modules/readme/update_related_references.py new file mode 100644 index 00000000..f465b520 --- /dev/null +++ b/modules/readme/update_related_references.py @@ -0,0 +1,88 @@ +from github import Github +import os +import json +import requests +import subprocess +from iteration_utilities import unique_everseen +import itertools +from ruamel.yaml import YAML + +GH_TOKEN = os.environ["GH_TOKEN"] +GH_ORG_NAME = os.getenv("GH_ORG_NAME", "cloudposse") +GH_SEARCH_PATTERN = os.getenv("GH_SEARCH_PATTERN", "terraform-aws-eks") +TF_MODULE_PATH = os.getenv("TF_MODULE_PATH", ".") +TF_CONFIG_INSPECT_BINARY_PATH = os.getenv( + "TF_CONFIG_INSPECT_BINARY_PATH", "/usr/local/bin/terraform-config-inspect" +) +TF_REGISTRY_URL = "https://registry.terraform.io/v1" + +gh = Github(GH_TOKEN) +yaml = YAML(typ="safe", pure=True) +yaml.default_flow_style = False +yaml.preserve_quotes = True + + +def get_list_of_repos(g): + repos = [] + for repo in gh.get_organization(GH_ORG_NAME).get_repos(): + name = repo.name + if GH_SEARCH_PATTERN in name: + repos.append(name) + return repos + + +def tf_config_inspect(): + list_of_modules = [] + list_of_providers = [] + list_of_related = [] + + output = json.loads( + subprocess.check_output( + [TF_CONFIG_INSPECT_BINARY_PATH, TF_MODULE_PATH, "--json"], + stderr=subprocess.STDOUT, + ) + ) + + for k, v in output["module_calls"].items(): + module_object = {} + url = TF_REGISTRY_URL + "/modules/" + v["source"] + module_info = requests.get(url=url).json() + + module_object["name"] = "terraform-{}-{}".format( + module_info["provider"], module_info["name"] + ) + module_object["description"] = module_info["description"] + module_object["url"] = module_info["source"] + list_of_modules.append(module_object) + + for k, v in output["required_providers"].items(): + provider_object = {} + url = TF_REGISTRY_URL + "/providers/" + v["source"] + provider_info = requests.get(url=url).json() + + provider_object["name"] = "terraform-provider-{}".format( + provider_info["namespace"] + ) + provider_object["description"] = provider_info["description"] + provider_object["url"] = provider_info["source"] + list_of_providers.append(provider_object) + + for m, p in itertools.zip_longest( + unique_everseen(list_of_modules), list_of_providers + ): + list_of_related.append(m) + list_of_related.append(p) + + return list_of_related + + +if __name__ == "__main__": + + with open("{}/README.yaml".format(TF_MODULE_PATH)) as f: + readme = yaml.load(f) + + new_related_list = yaml.load(json.dumps(tf_config_inspect())) + readme["related"] = new_related_list + + with open("{}/README.yaml".format(TF_MODULE_PATH), "w") as f: + yaml.dump(readme, f) From 06dd4f766f077aa55062fe5cc911c8e710994b04 Mon Sep 17 00:00:00 2001 From: SweetOps Date: Wed, 20 Jan 2021 18:23:10 +0200 Subject: [PATCH 2/7] upd --- modules/readme/update_related_references.py | 96 +++++++++++---------- 1 file changed, 50 insertions(+), 46 deletions(-) diff --git a/modules/readme/update_related_references.py b/modules/readme/update_related_references.py index f465b520..edf6bf27 100644 --- a/modules/readme/update_related_references.py +++ b/modules/readme/update_related_references.py @@ -1,10 +1,9 @@ -from github import Github import os import json -import requests import subprocess +from github import Github +import requests from iteration_utilities import unique_everseen -import itertools from ruamel.yaml import YAML GH_TOKEN = os.environ["GH_TOKEN"] @@ -12,77 +11,82 @@ GH_SEARCH_PATTERN = os.getenv("GH_SEARCH_PATTERN", "terraform-aws-eks") TF_MODULE_PATH = os.getenv("TF_MODULE_PATH", ".") TF_CONFIG_INSPECT_BINARY_PATH = os.getenv( - "TF_CONFIG_INSPECT_BINARY_PATH", "/usr/local/bin/terraform-config-inspect" + "TF_CONFIG_INSPECT_BINARY_PATH", + "terraform-config-inspect" ) TF_REGISTRY_URL = "https://registry.terraform.io/v1" gh = Github(GH_TOKEN) -yaml = YAML(typ="safe", pure=True) +yaml = YAML(typ="rt") yaml.default_flow_style = False -yaml.preserve_quotes = True +yaml.preserve_quotes = False -def get_list_of_repos(g): - repos = [] +def parse_gh(): + gh_repos = [] for repo in gh.get_organization(GH_ORG_NAME).get_repos(): - name = repo.name - if GH_SEARCH_PATTERN in name: - repos.append(name) - return repos + if GH_SEARCH_PATTERN in repo.name: + repo_object = {} + repo_object["name"] = repo.name + repo_object["description"] = repo.description + repo_object["url"] = repo.html_url + gh_repos.append(repo_object) + return gh_repos def tf_config_inspect(): - list_of_modules = [] - list_of_providers = [] - list_of_related = [] - output = json.loads( subprocess.check_output( [TF_CONFIG_INSPECT_BINARY_PATH, TF_MODULE_PATH, "--json"], stderr=subprocess.STDOUT, ) ) + return output - for k, v in output["module_calls"].items(): - module_object = {} - url = TF_REGISTRY_URL + "/modules/" + v["source"] - module_info = requests.get(url=url).json() - module_object["name"] = "terraform-{}-{}".format( - module_info["provider"], module_info["name"] - ) - module_object["description"] = module_info["description"] - module_object["url"] = module_info["source"] - list_of_modules.append(module_object) - - for k, v in output["required_providers"].items(): - provider_object = {} - url = TF_REGISTRY_URL + "/providers/" + v["source"] - provider_info = requests.get(url=url).json() - - provider_object["name"] = "terraform-provider-{}".format( - provider_info["namespace"] - ) - provider_object["description"] = provider_info["description"] - provider_object["url"] = provider_info["source"] - list_of_providers.append(provider_object) +def parse_tf_registry(src_data, src_type): + items = [] + src_item = "module_calls" + if src_type == "providers": + src_item = "required_providers" - for m, p in itertools.zip_longest( - unique_everseen(list_of_modules), list_of_providers - ): - list_of_related.append(m) - list_of_related.append(p) + for k, v in src_data[src_item].items(): + item_object = {} + url = TF_REGISTRY_URL + "/" + src_type + "/" + v["source"] + r = requests.get(url=url).json() - return list_of_related + if src_type == "providers": + name_pattern = "terraform-provider-{}".format(r["name"]) + else: + name_pattern = "terraform-{}-{}".format(r["provider"], r["name"]) + item_object["name"] = name_pattern + item_object["description"] = r["description"] + item_object["url"] = r["source"] + items.append(item_object) + return items if __name__ == "__main__": + related_reference_list = [] + + inspected_data = tf_config_inspect() + modules_list = parse_tf_registry(inspected_data, "modules") + providers_list = parse_tf_registry(inspected_data, "providers") + gh_repos_list = parse_gh() + + # this can be done in one line but it requires itertools + # and additional step to remove empty dicts + for m in unique_everseen(modules_list): + related_reference_list.append(m) + for g in unique_everseen(gh_repos_list): + related_reference_list.append(g) + for p in unique_everseen(providers_list): + related_reference_list.append(p) with open("{}/README.yaml".format(TF_MODULE_PATH)) as f: readme = yaml.load(f) - new_related_list = yaml.load(json.dumps(tf_config_inspect())) - readme["related"] = new_related_list + readme["related"] = related_reference_list with open("{}/README.yaml".format(TF_MODULE_PATH), "w") as f: yaml.dump(readme, f) From ff9cef3d5dd04666472f22520e6350ce27233553 Mon Sep 17 00:00:00 2001 From: SweetOps Date: Sun, 24 Jan 2021 20:54:18 +0200 Subject: [PATCH 3/7] move to bin --- .../generate_related_references.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename modules/readme/update_related_references.py => bin/generate_related_references.py (97%) mode change 100644 => 100755 diff --git a/modules/readme/update_related_references.py b/bin/generate_related_references.py old mode 100644 new mode 100755 similarity index 97% rename from modules/readme/update_related_references.py rename to bin/generate_related_references.py index edf6bf27..7d8883ee --- a/modules/readme/update_related_references.py +++ b/bin/generate_related_references.py @@ -8,7 +8,7 @@ GH_TOKEN = os.environ["GH_TOKEN"] GH_ORG_NAME = os.getenv("GH_ORG_NAME", "cloudposse") -GH_SEARCH_PATTERN = os.getenv("GH_SEARCH_PATTERN", "terraform-aws-eks") +GH_SEARCH_PATTERN = os.getenv("GH_SEARCH_PATTERN", "terraform-") TF_MODULE_PATH = os.getenv("TF_MODULE_PATH", ".") TF_CONFIG_INSPECT_BINARY_PATH = os.getenv( "TF_CONFIG_INSPECT_BINARY_PATH", From 5ff8429b54fb442b8b632007fb08961b82c00562 Mon Sep 17 00:00:00 2001 From: SweetOps Date: Mon, 25 Jan 2021 18:47:20 +0200 Subject: [PATCH 4/7] upd --- Dockerfile | 7 ++++++- bin/generate_related_references.py | 10 ++++++---- modules/readme/Makefile | 3 +++ 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index e63898f3..9fceaa85 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,12 @@ RUN apk --update --no-cache add \ jq \ libc6-compat \ make \ - py-pip && \ + python3-dev \ + py-pip \ + py3-ruamel.yaml && \ + pip3 install --no-cache-dir \ + iteration-utilities==0.11.0 \ + PyGithub==1.54.1 && \ git config --global advice.detachedHead false SHELL ["/bin/bash", "-o", "pipefail", "-c"] diff --git a/bin/generate_related_references.py b/bin/generate_related_references.py index 7d8883ee..27155cb9 100755 --- a/bin/generate_related_references.py +++ b/bin/generate_related_references.py @@ -1,8 +1,11 @@ -import os +#!/usr/bin/env python3 + import json +import os import subprocess -from github import Github + import requests +from github import Github from iteration_utilities import unique_everseen from ruamel.yaml import YAML @@ -11,8 +14,7 @@ GH_SEARCH_PATTERN = os.getenv("GH_SEARCH_PATTERN", "terraform-") TF_MODULE_PATH = os.getenv("TF_MODULE_PATH", ".") TF_CONFIG_INSPECT_BINARY_PATH = os.getenv( - "TF_CONFIG_INSPECT_BINARY_PATH", - "terraform-config-inspect" + "TF_CONFIG_INSPECT_BINARY_PATH", "terraform-config-inspect" ) TF_REGISTRY_URL = "https://registry.terraform.io/v1" diff --git a/modules/readme/Makefile b/modules/readme/Makefile index f8e77988..01504692 100644 --- a/modules/readme/Makefile +++ b/modules/readme/Makefile @@ -32,3 +32,6 @@ readme/build: readme/deps $(README_DEPS) @gomplate --file $(README_TEMPLATE_FILE) \ --out $(README_FILE) @echo "Generated $(README_FILE) from $(README_TEMPLATE_FILE) using data from $(README_TEMPLATE_YAML)" + +readme/generate-related-references: + @$(BUILD_HARNESS_PATH)/bin/generate_related_references.py \ No newline at end of file From 307b2953c157e5ad497ebb5b843eb937215dc444 Mon Sep 17 00:00:00 2001 From: SweetOps Date: Mon, 25 Jan 2021 21:44:32 +0200 Subject: [PATCH 5/7] upd targets.md --- docs/targets.md | 1 + modules/readme/Makefile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/targets.md b/docs/targets.md index ffa0b810..3c7eacce 100644 --- a/docs/targets.md +++ b/docs/targets.md @@ -108,6 +108,7 @@ Available targets: readme/build Create README.md by building it from README.yaml readme/init Create basic minimalistic .README.md template file readme/lint Verify the `README.md` is up to date + readme/generate-related-references Generate related references block (e.i. `related`) in the README.yaml semver/export Export semver vars slack/notify Send webhook notification to slack slack/notify/build Send notification to slack using "build" template diff --git a/modules/readme/Makefile b/modules/readme/Makefile index 01504692..4155d134 100644 --- a/modules/readme/Makefile +++ b/modules/readme/Makefile @@ -34,4 +34,4 @@ readme/build: readme/deps $(README_DEPS) @echo "Generated $(README_FILE) from $(README_TEMPLATE_FILE) using data from $(README_TEMPLATE_YAML)" readme/generate-related-references: - @$(BUILD_HARNESS_PATH)/bin/generate_related_references.py \ No newline at end of file + @$(BUILD_HARNESS_PATH)/bin/generate_related_references.py From 1cda3977432f1634770daf42bf44c95356f4ae0c Mon Sep 17 00:00:00 2001 From: SweetOps Date: Mon, 25 Jan 2021 22:01:44 +0200 Subject: [PATCH 6/7] upd README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1b40e0dd..7d8cc3a4 100644 --- a/README.md +++ b/README.md @@ -358,7 +358,7 @@ In general, PRs are welcome. We follow the typical "fork-and-pull" Git workflow. ## Copyrights -Copyright © 2016-2020 [Cloud Posse, LLC](https://cloudposse.com) +Copyright © 2016-2021 [Cloud Posse, LLC](https://cloudposse.com) From d884ff7dbf0014bb600bd1b272af9840bb9df990 Mon Sep 17 00:00:00 2001 From: SweetOps Date: Wed, 27 Jan 2021 16:30:24 +0200 Subject: [PATCH 7/7] upd --- bin/generate_related_references.py | 37 ++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/bin/generate_related_references.py b/bin/generate_related_references.py index 27155cb9..dbc4895b 100755 --- a/bin/generate_related_references.py +++ b/bin/generate_related_references.py @@ -16,7 +16,7 @@ TF_CONFIG_INSPECT_BINARY_PATH = os.getenv( "TF_CONFIG_INSPECT_BINARY_PATH", "terraform-config-inspect" ) -TF_REGISTRY_URL = "https://registry.terraform.io/v1" +TF_REGISTRY_URL = "https://registry.terraform.io" gh = Github(GH_TOKEN) yaml = YAML(typ="rt") @@ -54,7 +54,7 @@ def parse_tf_registry(src_data, src_type): for k, v in src_data[src_item].items(): item_object = {} - url = TF_REGISTRY_URL + "/" + src_type + "/" + v["source"] + url = TF_REGISTRY_URL + "/v1/" + src_type + "/" + v["source"] r = requests.get(url=url).json() if src_type == "providers": @@ -62,14 +62,25 @@ def parse_tf_registry(src_data, src_type): else: name_pattern = "terraform-{}-{}".format(r["provider"], r["name"]) item_object["name"] = name_pattern - item_object["description"] = r["description"] - item_object["url"] = r["source"] + + if src_type == "providers": + # description on GitHub looks better than on terraform-registry + gh_repo_info = gh.get_repo("{}/{}".format(r["namespace"], name_pattern)) + item_object["description"] = gh_repo_info.description + item_object["url"] = TF_REGISTRY_URL + "/providers/{}/{}/latest".format( + r["namespace"], r["name"] + ) + else: + item_object["description"] = r["description"] + item_object["url"] = r["source"] + items.append(item_object) return items if __name__ == "__main__": - related_reference_list = [] + related_list = [] + reference_list = [] inspected_data = tf_config_inspect() modules_list = parse_tf_registry(inspected_data, "modules") @@ -79,16 +90,24 @@ def parse_tf_registry(src_data, src_type): # this can be done in one line but it requires itertools # and additional step to remove empty dicts for m in unique_everseen(modules_list): - related_reference_list.append(m) + related_list.append(m) for g in unique_everseen(gh_repos_list): - related_reference_list.append(g) + related_list.append(g) for p in unique_everseen(providers_list): - related_reference_list.append(p) + reference_list.append(p) with open("{}/README.yaml".format(TF_MODULE_PATH)) as f: readme = yaml.load(f) - readme["related"] = related_reference_list + readme["related"] = related_list + + # ensure that "references" key is present and then insert data + if readme.get("references"): + readme["references"] = reference_list + else: + # create key "references" after the "related" + readme.insert(list(readme.keys()).index("related") + 1, "references", []) + readme["references"] = reference_list with open("{}/README.yaml".format(TF_MODULE_PATH), "w") as f: yaml.dump(readme, f)