From 6f1023a21d74c172576c495953429d4c34ba7af4 Mon Sep 17 00:00:00 2001 From: flosch62 Date: Fri, 17 Jan 2025 09:06:25 +0100 Subject: [PATCH 1/6] urllib3 --- src/eda.py | 71 +++++++++++++++++++++++++++++--------------------- src/helpers.py | 40 ++++++++++++++++++---------- 2 files changed, 67 insertions(+), 44 deletions(-) diff --git a/src/eda.py b/src/eda.py index ead5489..538fbf5 100644 --- a/src/eda.py +++ b/src/eda.py @@ -1,7 +1,7 @@ import json import logging -import requests +import urllib3 import yaml # configure logging @@ -34,21 +34,28 @@ def __init__(self, hostname, username, password, verify): self.version = None self.transactions = [] + # Create urllib3 connection pool + self.http = urllib3.PoolManager( + cert_reqs='CERT_REQUIRED' if verify else 'CERT_NONE', + retries=urllib3.Retry(3) + ) + def login(self): """ Retrieves an access_token and refresh_token from the EDA API """ payload = {"username": self.username, "password": self.password} - response = self.post("auth/login", payload, False).json() + response = self.post("auth/login", payload, False) + response_data = json.loads(response.data.decode('utf-8')) - if "code" in response and response["code"] != 200: + if "code" in response_data and response_data["code"] != 200: raise Exception( - f"Could not authenticate with EDA, error message: '{response['message']} {response['details']}'" + f"Could not authenticate with EDA, error message: '{response_data['message']} {response_data['details']}'" ) - self.access_token = response["access_token"] - self.refresh_token = response["refresh_token"] + self.access_token = response_data["access_token"] + self.refresh_token = response_data["refresh_token"] def get_headers(self, requires_auth): """ @@ -88,10 +95,10 @@ def get(self, api_path, requires_auth=True): url = f"{self.url}/{api_path}" logger.info(f"Performing GET request to '{url}'") - return requests.get( + return self.http.request( + 'GET', url, - verify=self.verify, - headers=self.get_headers(requires_auth), + headers=self.get_headers(requires_auth) ) def post(self, api_path, payload, requires_auth=True): @@ -110,11 +117,11 @@ def post(self, api_path, payload, requires_auth=True): """ url = f"{self.url}/{api_path}" logger.info(f"Performing POST request to '{url}'") - return requests.post( + return self.http.request( + 'POST', url, - verify=self.verify, - json=payload, headers=self.get_headers(requires_auth), + body=json.dumps(payload).encode('utf-8') ) def is_up(self): @@ -127,8 +134,9 @@ def is_up(self): """ logger.info("Checking whether EDA is up") health = self.get("core/about/health", requires_auth=False) - logger.debug(health.json()) - return health.json()["status"] == "UP" + health_data = json.loads(health.data.decode('utf-8')) + logger.debug(health_data) + return health_data["status"] == "UP" def get_version(self): """ @@ -140,7 +148,9 @@ def get_version(self): return self.version logger.info("Getting EDA version") - version = self.get("core/about/version").json()["eda"]["version"].split("-")[0] + version_response = self.get("core/about/version") + version_data = json.loads(version_response.data.decode('utf-8')) + version = version_data["eda"]["version"].split("-")[0] logger.info(f"EDA version is {version}") # storing this to make the tool backwards compatible @@ -254,18 +264,18 @@ def is_transaction_item_valid(self, item): logger.info("Validating transaction item") response = self.post("core/transaction/v1/validate", item) - if response.status_code == 204: + if response.status == 204: logger.info("Validation successful") return True - response = response.json() + response_data = json.loads(response.data.decode('utf-8')) # Need to decode response data - if "code" in response: - message = f"{response['message']}" - if "details" in response: - message = f"{message} - {response['details']}" + if "code" in response_data: + message = f"{response_data['message']}" + if "details" in response_data: + message = f"{message} - {response_data['details']}" logger.warning( - f"While validating a transaction item, the following validation error was returned (code {response['code']}): '{message}'" + f"While validating a transaction item, the following validation error was returned (code {response_data['code']}): '{message}'" ) return False @@ -295,16 +305,17 @@ def commit_transaction( logger.info(f"Committing transaction with {len(self.transactions)} item(s)") logger.debug(json.dumps(payload, indent=4)) - response = self.post("core/transaction/v1", payload).json() - if "id" not in response: - raise Exception(f"Could not find transaction ID in response {response}") + response = self.post("core/transaction/v1", payload) + response_data = json.loads(response.data.decode('utf-8')) + if "id" not in response_data: + raise Exception(f"Could not find transaction ID in response {response_data}") - transactionId = response["id"] + transactionId = response_data["id"] logger.info(f"Waiting for transaction with ID {transactionId} to complete") - result = self.get( + result = json.loads(self.get( f"core/transaction/v1/details/{transactionId}?waitForComplete=true&failOnErrors=true" - ).json() + ).data.decode('utf-8')) if "code" in result: message = f"{result['message']}" @@ -348,7 +359,7 @@ def revert_transaction(self, transactionId): ).json() response = self.post(f"core/transaction/v1/revert/{transactionId}", {}) - result = response.json() + result = json.loads(response.data.decode('utf-8')) if "code" in result and result["code"] != 0: message = f"{result['message']}" @@ -392,7 +403,7 @@ def restore_transaction(self, transactionId): ).json() response = self.post(f"core/transaction/v1/restore/{restore_point}", {}) - result = response.json() + result = json.loads(response.data.decode('utf-8')) if "code" in result and result["code"] != 0: message = f"{result['message']}" diff --git a/src/helpers.py b/src/helpers.py index 28400cb..6110e9c 100644 --- a/src/helpers.py +++ b/src/helpers.py @@ -5,7 +5,7 @@ import sys import tempfile -import requests +import urllib3 from jinja2 import Environment, FileSystemLoader import src.topology as topology @@ -17,6 +17,10 @@ # set up logging logger = logging.getLogger(__name__) +# Create a global urllib3 pool manager +http = urllib3.PoolManager( + retries=urllib3.Retry(3) +) def parse_topology(topology_file) -> topology.Topology: """ @@ -101,7 +105,7 @@ def apply_manifest_via_kubectl(yaml_str: str, namespace: str = "eda-system"): def get_artifact_from_github(owner: str, repo: str, version: str, asset_filter=None): """ - Queries GitHub for a specific release artifact. + Queries GitHub for a specific release artifact using urllib3. Parameters ---------- @@ -118,19 +122,27 @@ def get_artifact_from_github(owner: str, repo: str, version: str, asset_filter=N url = f"https://api.github.com/repos/{owner}/{repo}/releases/tags/{tag}" logger.info(f"Querying GitHub release {tag} from {owner}/{repo}") - resp = requests.get(url) - if resp.status_code != 200: - logger.warning(f"Failed to fetch release for {tag}, status={resp.status_code}") - return None, None - - data = resp.json() - assets = data.get("assets", []) - - for asset in assets: - name = asset.get("name", "") - if asset_filter is None or asset_filter(name): - return name, asset.get("browser_download_url") + try: + response = http.request('GET', url) + if response.status != 200: + logger.warning(f"Failed to fetch release for {tag}, status={response.status}") + return None, None + + data = json.loads(response.data.decode('utf-8')) + assets = data.get("assets", []) + + for asset in assets: + name = asset.get("name", "") + if asset_filter is None or asset_filter(name): + return name, asset.get("browser_download_url") + + except urllib3.exceptions.HTTPError as e: + logger.error(f"HTTP error occurred: {e}") + except json.JSONDecodeError as e: + logger.error(f"JSON decode error: {e}") + except Exception as e: + logger.error(f"Unexpected error: {e}") # No matching asset found return None, None From 7c674640dda5e3514dbd275edc68678aab06636b Mon Sep 17 00:00:00 2001 From: flosch62 Date: Fri, 17 Jan 2025 09:14:14 +0100 Subject: [PATCH 2/6] temp debug --- src/helpers.py | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/src/helpers.py b/src/helpers.py index 6110e9c..43d74d4 100644 --- a/src/helpers.py +++ b/src/helpers.py @@ -121,30 +121,52 @@ def get_artifact_from_github(owner: str, repo: str, version: str, asset_filter=N tag = f"v{version}" # Assume GitHub tags are prefixed with 'v' url = f"https://api.github.com/repos/{owner}/{repo}/releases/tags/{tag}" - logger.info(f"Querying GitHub release {tag} from {owner}/{repo}") - + # Log proxy environment + logger.info(f"HTTP_PROXY: {os.environ.get('HTTP_PROXY', 'not set')}") + logger.info(f"HTTPS_PROXY: {os.environ.get('HTTPS_PROXY', 'not set')}") + logger.info(f"NO_PROXY: {os.environ.get('NO_PROXY', 'not set')}") + + # Log request details + logger.info(f"Making request to: {url}") + logger.info(f"Using pool manager type: {type(http).__name__}") + try: response = http.request('GET', url) + logger.info(f"Response status: {response.status}") + logger.info(f"Response headers: {response.headers}") + if response.status != 200: - logger.warning(f"Failed to fetch release for {tag}, status={response.status}") + logger.info(f"Failed to fetch release for {tag}, status={response.status}") + logger.info(f"Response data: {response.data.decode('utf-8')}") return None, None data = json.loads(response.data.decode('utf-8')) assets = data.get("assets", []) + logger.info(f"Found {len(assets)} assets in release") for asset in assets: name = asset.get("name", "") + logger.info(f"Checking asset: {name}") if asset_filter is None or asset_filter(name): - return name, asset.get("browser_download_url") + download_url = asset.get("browser_download_url") + logger.info(f"Found matching asset: {name} with URL: {download_url}") + return name, download_url + else: + logger.info(f"Asset {name} did not match filter") except urllib3.exceptions.HTTPError as e: - logger.error(f"HTTP error occurred: {e}") + logger.info(f"HTTP error occurred: {e}") + logger.info(f"Error details: {str(e)}") except json.JSONDecodeError as e: - logger.error(f"JSON decode error: {e}") + logger.info(f"JSON decode error: {e}") + logger.info(f"Raw response: {response.data.decode('utf-8')}") except Exception as e: - logger.error(f"Unexpected error: {e}") + logger.info(f"Unexpected error: {e}") + logger.info(f"Error type: {type(e).__name__}") + logger.info(f"Error details: {str(e)}") # No matching asset found + logger.info("No matching asset found") return None, None From 1e84ccb7f4d8b4fae182937c5ae1344eb5957af9 Mon Sep 17 00:00:00 2001 From: flosch62 Date: Fri, 17 Jan 2025 10:11:39 +0100 Subject: [PATCH 3/6] better proxy handling --- src/eda.py | 9 +-- src/helpers.py | 75 +++++++++++++----------- src/http_client.py | 139 +++++++++++++++++++++++++++++++++++++++++++++ src/node_srl.py | 39 +++++++++---- 4 files changed, 213 insertions(+), 49 deletions(-) create mode 100644 src/http_client.py diff --git a/src/eda.py b/src/eda.py index 538fbf5..3d90c2b 100644 --- a/src/eda.py +++ b/src/eda.py @@ -1,9 +1,10 @@ import json import logging -import urllib3 import yaml +from src.http_client import create_pool_manager + # configure logging logger = logging.getLogger(__name__) @@ -34,11 +35,7 @@ def __init__(self, hostname, username, password, verify): self.version = None self.transactions = [] - # Create urllib3 connection pool - self.http = urllib3.PoolManager( - cert_reqs='CERT_REQUIRED' if verify else 'CERT_NONE', - retries=urllib3.Retry(3) - ) + self.http = create_pool_manager(url=self.url, verify=self.verify) def login(self): """ diff --git a/src/helpers.py b/src/helpers.py index 43d74d4..3f2f753 100644 --- a/src/helpers.py +++ b/src/helpers.py @@ -6,6 +6,7 @@ import tempfile import urllib3 + from jinja2 import Environment, FileSystemLoader import src.topology as topology @@ -17,10 +18,6 @@ # set up logging logger = logging.getLogger(__name__) -# Create a global urllib3 pool manager -http = urllib3.PoolManager( - retries=urllib3.Retry(3) -) def parse_topology(topology_file) -> topology.Topology: """ @@ -118,55 +115,67 @@ def get_artifact_from_github(owner: str, repo: str, version: str, asset_filter=N ------- Tuple of (filename, download_url) or (None, None) if not found """ + from src.http_client import create_pool_manager + tag = f"v{version}" # Assume GitHub tags are prefixed with 'v' url = f"https://api.github.com/repos/{owner}/{repo}/releases/tags/{tag}" - # Log proxy environment - logger.info(f"HTTP_PROXY: {os.environ.get('HTTP_PROXY', 'not set')}") - logger.info(f"HTTPS_PROXY: {os.environ.get('HTTPS_PROXY', 'not set')}") - logger.info(f"NO_PROXY: {os.environ.get('NO_PROXY', 'not set')}") - - # Log request details - logger.info(f"Making request to: {url}") - logger.info(f"Using pool manager type: {type(http).__name__}") - + http = create_pool_manager(url=url, verify=True) + + # Log proxy environment at debug level + logger.debug(f"HTTP_PROXY: {os.environ.get('HTTP_PROXY', 'not set')}") + logger.debug(f"HTTPS_PROXY: {os.environ.get('HTTPS_PROXY', 'not set')}") + logger.debug(f"NO_PROXY: {os.environ.get('NO_PROXY', 'not set')}") + logger.debug(f"Using pool manager type: {type(http).__name__}") + try: - response = http.request('GET', url) - logger.info(f"Response status: {response.status}") - logger.info(f"Response headers: {response.headers}") - + response = http.request("GET", url) + logger.debug(f"Response status: {response.status}") + logger.debug(f"Response headers: {response.headers}") + + if response.status == 403: + response_data = json.loads(response.data.decode('utf-8')) + if "rate limit exceeded" in response_data.get("message", "").lower(): + logger.warning(f"GitHub API rate limit exceeded. {response_data.get('message')}") + if response_data.get("documentation_url"): + logger.warning(f"See: {response_data['documentation_url']}") + return None, None + else: + logger.error(f"Access forbidden: {response_data.get('message', 'No message provided')}") + return None, None + if response.status != 200: - logger.info(f"Failed to fetch release for {tag}, status={response.status}") - logger.info(f"Response data: {response.data.decode('utf-8')}") + logger.error(f"Failed to fetch release {tag} (status={response.status})") + logger.debug(f"Response data: {response.data.decode('utf-8')}") return None, None - data = json.loads(response.data.decode('utf-8')) + data = json.loads(response.data.decode("utf-8")) assets = data.get("assets", []) - logger.info(f"Found {len(assets)} assets in release") + logger.debug(f"Found {len(assets)} assets in release") for asset in assets: name = asset.get("name", "") - logger.info(f"Checking asset: {name}") + logger.debug(f"Checking asset: {name}") if asset_filter is None or asset_filter(name): download_url = asset.get("browser_download_url") - logger.info(f"Found matching asset: {name} with URL: {download_url}") + logger.info(f"Found matching asset: {name}") + logger.debug(f"Download URL: {download_url}") return name, download_url else: - logger.info(f"Asset {name} did not match filter") + logger.debug(f"Asset {name} did not match filter") except urllib3.exceptions.HTTPError as e: - logger.info(f"HTTP error occurred: {e}") - logger.info(f"Error details: {str(e)}") + logger.error(f"HTTP error occurred while fetching {tag}: {e}") + logger.debug(f"Error details: {str(e)}") except json.JSONDecodeError as e: - logger.info(f"JSON decode error: {e}") - logger.info(f"Raw response: {response.data.decode('utf-8')}") + logger.error(f"Invalid JSON response for {tag}: {e}") + logger.debug(f"Raw response: {response.data.decode('utf-8')}") except Exception as e: - logger.info(f"Unexpected error: {e}") - logger.info(f"Error type: {type(e).__name__}") - logger.info(f"Error details: {str(e)}") + logger.error(f"Unexpected error while fetching {tag}: {type(e).__name__}") + logger.debug(f"Error details: {str(e)}") # No matching asset found - logger.info("No matching asset found") + logger.warning("No matching asset found") return None, None @@ -193,4 +202,4 @@ def normalize_name(name: str) -> str: if not safe_name[-1].isalnum(): safe_name = safe_name + "0" - return safe_name + return safe_name \ No newline at end of file diff --git a/src/http_client.py b/src/http_client.py new file mode 100644 index 0000000..9907471 --- /dev/null +++ b/src/http_client.py @@ -0,0 +1,139 @@ +import logging +import os +import re +import urllib3 +from urllib.parse import urlparse + +logger = logging.getLogger(__name__) + + +def get_proxy_settings(): + """ + Get proxy settings from environment variables. + Handles both upper and lowercase variants. + + Returns + ------- + tuple: (http_proxy, https_proxy, no_proxy) + """ + # Check both variants + http_upper = os.environ.get("HTTP_PROXY") + http_lower = os.environ.get("http_proxy") + https_upper = os.environ.get("HTTPS_PROXY") + https_lower = os.environ.get("https_proxy") + no_upper = os.environ.get("NO_PROXY") + no_lower = os.environ.get("no_proxy") + + # Log if both variants are set + if http_upper and http_lower and http_upper != http_lower: + logger.warning( + f"Both HTTP_PROXY ({http_upper}) and http_proxy ({http_lower}) are set with different values. Using HTTP_PROXY." + ) + + if https_upper and https_lower and https_upper != https_lower: + logger.warning( + f"Both HTTPS_PROXY ({https_upper}) and https_proxy ({https_lower}) are set with different values. Using HTTPS_PROXY." + ) + + if no_upper and no_lower and no_upper != no_lower: + logger.warning( + f"Both NO_PROXY ({no_upper}) and no_proxy ({no_lower}) are set with different values. Using NO_PROXY." + ) + + # Use uppercase variants if set, otherwise lowercase + http_proxy = http_upper if http_upper is not None else http_lower + https_proxy = https_upper if https_upper is not None else https_lower + no_proxy = no_upper if no_upper is not None else no_lower or "" + + return http_proxy, https_proxy, no_proxy + + +def should_bypass_proxy(url, no_proxy=None): + """ + Check if the given URL should bypass proxy based on NO_PROXY settings. + + Parameters + ---------- + url : str + The URL to check + no_proxy : str, optional + The NO_PROXY string to use. If None, gets from environment. + + Returns + ------- + bool + True if proxy should be bypassed, False otherwise + """ + if no_proxy is None: + _, _, no_proxy = get_proxy_settings() + + if not no_proxy: + return False + + parsed_url = urlparse(url if "//" in url else f"http://{url}") + hostname = parsed_url.hostname + + if not hostname: + return False + + # Split NO_PROXY into parts and clean them + no_proxy_parts = [p.strip() for p in no_proxy.split(",") if p.strip()] + + for no_proxy_value in no_proxy_parts: + # Convert .foo.com to foo.com + if no_proxy_value.startswith("."): + no_proxy_value = no_proxy_value[1:] + + # Handle IP addresses and CIDR notation + if re.match(r"^(?:\d{1,3}\.){3}\d{1,3}(?:/\d{1,2})?$", no_proxy_value): + # TODO: Implement CIDR matching if needed + if hostname == no_proxy_value: + return True + # Handle domain names with wildcards + else: + pattern = re.escape(no_proxy_value).replace(r"\*", ".*") + if re.match(f"^{pattern}$", hostname, re.IGNORECASE): + return True + + return False + + +def create_pool_manager(url=None, verify=True): + """ + Create a PoolManager or ProxyManager based on environment settings and URL + + Parameters + ---------- + url : str, optional + The URL that will be accessed with this pool manager + If provided, NO_PROXY rules will be checked + verify : bool + Whether to verify SSL certificates + + Returns + ------- + urllib3.PoolManager or urllib3.ProxyManager + """ + http_proxy, https_proxy, no_proxy = get_proxy_settings() + + # Check if this URL should bypass proxy + if url and should_bypass_proxy(url, no_proxy): + logger.debug(f"URL {url} matches NO_PROXY rules, creating direct PoolManager") + return urllib3.PoolManager( + cert_reqs="CERT_REQUIRED" if verify else "CERT_NONE", + retries=urllib3.Retry(3), + ) + + proxy_url = https_proxy or http_proxy + if proxy_url: + logger.debug(f"Creating ProxyManager with proxy URL: {proxy_url}") + return urllib3.ProxyManager( + proxy_url, + cert_reqs="CERT_REQUIRED" if verify else "CERT_NONE", + retries=urllib3.Retry(3), + ) + + logger.debug("Creating PoolManager without proxy") + return urllib3.PoolManager( + cert_reqs="CERT_REQUIRED" if verify else "CERT_NONE", retries=urllib3.Retry(3) + ) diff --git a/src/node_srl.py b/src/node_srl.py index f4c0d96..740dda1 100644 --- a/src/node_srl.py +++ b/src/node_srl.py @@ -18,7 +18,6 @@ class SRLNode(Node): - # this can be made part of the command line arguments, but this is not done (yet) SRL_USERNAME = "admin" SRL_PASSWORD = "NokiaSrl1!" NODE_TYPE = "srlinux" @@ -30,6 +29,8 @@ class SRLNode(Node): def __init__(self, name, kind, node_type, version, mgmt_ipv4): super().__init__(name, kind, node_type, version, mgmt_ipv4) + # Add cache for artifact info + self._artifact_info = None def test_ssh(self): """ @@ -116,9 +117,7 @@ def get_node_profile(self, topology): """ logger.info(f"Rendering node profile for {self}") - # Get artifact info first to construct the YANG path - artifact_name = self.get_artifact_name() - _, filename, _ = self.get_artifact_info() + artifact_name, filename = self.get_artifact_metadata() data = { "namespace": f"clab-{topology.name}", @@ -130,8 +129,7 @@ def get_node_profile(self, topology): # below evaluates to something like v24\.7\.1.* "version_match": "v{}.*".format(self.version.replace(".", "\.")), "yang_path": self.YANG_PATH.format( - artifact_name=artifact_name, - filename=filename + artifact_name=artifact_name, filename=filename ), "node_user": "admin", "onboarding_password": self.SRL_PASSWORD, @@ -231,14 +229,18 @@ def get_artifact_name(self): def get_artifact_info(self): """ - Gets SR Linux YANG models artifact information from GitHub + Gets SR Linux YANG models artifact information from GitHub. """ + # Return cached info if available + if self._artifact_info is not None: + return self._artifact_info + def srlinux_filter(name): return ( name.endswith(".zip") and name.startswith("srlinux-") and "Source code" not in name - ) + ) artifact_name = self.get_artifact_name() filename, download_url = helpers.get_artifact_from_github( @@ -248,7 +250,24 @@ def srlinux_filter(name): asset_filter=srlinux_filter, ) - return artifact_name, filename, download_url + # Cache the result + self._artifact_info = (artifact_name, filename, download_url) + return self._artifact_info + + def get_artifact_metadata(self): + """ + Returns just the artifact name and filename without making API calls. + Used when we don't need the download URL. + """ + if self._artifact_info is not None: + # Return cached info if available + artifact_name, filename, _ = self._artifact_info + return artifact_name, filename + + # If not cached, return basic info without API call + artifact_name = self.get_artifact_name() + filename = f"srlinux-{self.version}.zip" # Assume standard naming + return artifact_name, filename def get_artifact_yaml(self, artifact_name, filename, download_url): """ @@ -258,6 +277,6 @@ def get_artifact_yaml(self, artifact_name, filename, download_url): "artifact_name": artifact_name, "namespace": "eda-system", "artifact_filename": filename, - "artifact_url": download_url + "artifact_url": download_url, } return helpers.render_template("artifact.j2", data) From e34d2d11cbd9fb443ab4dbaf0d1c6c6749c7b30a Mon Sep 17 00:00:00 2001 From: flosch62 Date: Fri, 17 Jan 2025 10:33:31 +0100 Subject: [PATCH 4/6] formating --- src/eda.py | 38 ++++++++++++++++++++------------------ src/helpers.py | 14 +++++++++----- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/eda.py b/src/eda.py index 3d90c2b..241b5f5 100644 --- a/src/eda.py +++ b/src/eda.py @@ -44,7 +44,7 @@ def login(self): payload = {"username": self.username, "password": self.password} response = self.post("auth/login", payload, False) - response_data = json.loads(response.data.decode('utf-8')) + response_data = json.loads(response.data.decode("utf-8")) if "code" in response_data and response_data["code"] != 200: raise Exception( @@ -92,11 +92,7 @@ def get(self, api_path, requires_auth=True): url = f"{self.url}/{api_path}" logger.info(f"Performing GET request to '{url}'") - return self.http.request( - 'GET', - url, - headers=self.get_headers(requires_auth) - ) + return self.http.request("GET", url, headers=self.get_headers(requires_auth)) def post(self, api_path, payload, requires_auth=True): """ @@ -115,10 +111,10 @@ def post(self, api_path, payload, requires_auth=True): url = f"{self.url}/{api_path}" logger.info(f"Performing POST request to '{url}'") return self.http.request( - 'POST', + "POST", url, headers=self.get_headers(requires_auth), - body=json.dumps(payload).encode('utf-8') + body=json.dumps(payload).encode("utf-8"), ) def is_up(self): @@ -131,7 +127,7 @@ def is_up(self): """ logger.info("Checking whether EDA is up") health = self.get("core/about/health", requires_auth=False) - health_data = json.loads(health.data.decode('utf-8')) + health_data = json.loads(health.data.decode("utf-8")) logger.debug(health_data) return health_data["status"] == "UP" @@ -146,7 +142,7 @@ def get_version(self): logger.info("Getting EDA version") version_response = self.get("core/about/version") - version_data = json.loads(version_response.data.decode('utf-8')) + version_data = json.loads(version_response.data.decode("utf-8")) version = version_data["eda"]["version"].split("-")[0] logger.info(f"EDA version is {version}") @@ -265,7 +261,9 @@ def is_transaction_item_valid(self, item): logger.info("Validation successful") return True - response_data = json.loads(response.data.decode('utf-8')) # Need to decode response data + response_data = json.loads( + response.data.decode("utf-8") + ) # Need to decode response data if "code" in response_data: message = f"{response_data['message']}" @@ -303,16 +301,20 @@ def commit_transaction( logger.debug(json.dumps(payload, indent=4)) response = self.post("core/transaction/v1", payload) - response_data = json.loads(response.data.decode('utf-8')) + response_data = json.loads(response.data.decode("utf-8")) if "id" not in response_data: - raise Exception(f"Could not find transaction ID in response {response_data}") + raise Exception( + f"Could not find transaction ID in response {response_data}" + ) transactionId = response_data["id"] logger.info(f"Waiting for transaction with ID {transactionId} to complete") - result = json.loads(self.get( - f"core/transaction/v1/details/{transactionId}?waitForComplete=true&failOnErrors=true" - ).data.decode('utf-8')) + result = json.loads( + self.get( + f"core/transaction/v1/details/{transactionId}?waitForComplete=true&failOnErrors=true" + ).data.decode("utf-8") + ) if "code" in result: message = f"{result['message']}" @@ -356,7 +358,7 @@ def revert_transaction(self, transactionId): ).json() response = self.post(f"core/transaction/v1/revert/{transactionId}", {}) - result = json.loads(response.data.decode('utf-8')) + result = json.loads(response.data.decode("utf-8")) if "code" in result and result["code"] != 0: message = f"{result['message']}" @@ -400,7 +402,7 @@ def restore_transaction(self, transactionId): ).json() response = self.post(f"core/transaction/v1/restore/{restore_point}", {}) - result = json.loads(response.data.decode('utf-8')) + result = json.loads(response.data.decode("utf-8")) if "code" in result and result["code"] != 0: message = f"{result['message']}" diff --git a/src/helpers.py b/src/helpers.py index 3f2f753..2406aee 100644 --- a/src/helpers.py +++ b/src/helpers.py @@ -134,16 +134,20 @@ def get_artifact_from_github(owner: str, repo: str, version: str, asset_filter=N logger.debug(f"Response headers: {response.headers}") if response.status == 403: - response_data = json.loads(response.data.decode('utf-8')) + response_data = json.loads(response.data.decode("utf-8")) if "rate limit exceeded" in response_data.get("message", "").lower(): - logger.warning(f"GitHub API rate limit exceeded. {response_data.get('message')}") + logger.warning( + f"GitHub API rate limit exceeded. {response_data.get('message')}" + ) if response_data.get("documentation_url"): logger.warning(f"See: {response_data['documentation_url']}") return None, None else: - logger.error(f"Access forbidden: {response_data.get('message', 'No message provided')}") + logger.error( + f"Access forbidden: {response_data.get('message', 'No message provided')}" + ) return None, None - + if response.status != 200: logger.error(f"Failed to fetch release {tag} (status={response.status})") logger.debug(f"Response data: {response.data.decode('utf-8')}") @@ -202,4 +206,4 @@ def normalize_name(name: str) -> str: if not safe_name[-1].isalnum(): safe_name = safe_name + "0" - return safe_name \ No newline at end of file + return safe_name From 226128d3d43e53078333bd5861d35a6b984c4455 Mon Sep 17 00:00:00 2001 From: flosch62 Date: Fri, 17 Jan 2025 11:20:24 +0100 Subject: [PATCH 5/6] support gh token --- src/helpers.py | 60 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/src/helpers.py b/src/helpers.py index 2406aee..907718c 100644 --- a/src/helpers.py +++ b/src/helpers.py @@ -100,6 +100,46 @@ def apply_manifest_via_kubectl(yaml_str: str, namespace: str = "eda-system"): os.remove(tmp_path) +def get_github_token(): + """ + Get GitHub token from environment or GitHub CLI in priority order: + 1. GITHUB_TOKEN environment variable + 2. GH_TOKEN environment variable (GitHub CLI default) + 3. GitHub CLI authentication + + Returns + ------- + str or None + GitHub authentication token if found, None otherwise + """ + + # Check environment variables + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + if token: + logger.debug("Found GitHub token in environment variables") + return token + + # Try to get token from GitHub CLI + try: + # Check if gh is installed + result = subprocess.run(["gh", "--version"], capture_output=True, text=True) + if result.returncode == 0: + # Get auth token from gh + result = subprocess.run( + ["gh", "auth", "token"], capture_output=True, text=True + ) + if result.returncode == 0 and result.stdout.strip(): + logger.debug("Found GitHub token from GitHub CLI") + return result.stdout.strip() + except FileNotFoundError: + logger.debug("GitHub CLI (gh) not found") + except Exception as e: + logger.debug(f"Error getting GitHub CLI token: {e}") + + logger.debug("No GitHub token found") + return None + + def get_artifact_from_github(owner: str, repo: str, version: str, asset_filter=None): """ Queries GitHub for a specific release artifact using urllib3. @@ -110,7 +150,8 @@ def get_artifact_from_github(owner: str, repo: str, version: str, asset_filter=N repo: GitHub repository name version: Version tag to search for (without 'v' prefix) asset_filter: Optional function(asset_name) -> bool to filter assets - + token: Optional GitHub token. If None, will attempt to get from + environment or GitHub CLI Returns ------- Tuple of (filename, download_url) or (None, None) if not found @@ -120,6 +161,21 @@ def get_artifact_from_github(owner: str, repo: str, version: str, asset_filter=N tag = f"v{version}" # Assume GitHub tags are prefixed with 'v' url = f"https://api.github.com/repos/{owner}/{repo}/releases/tags/{tag}" + token = get_github_token() + + # Set up headers with authentication if token is available + headers = { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "container-lab/node-download" # Replace with your tool name + } + + if token: + headers["Authorization"] = f"Bearer {token}" + logger.debug("Using authenticated GitHub API request") + else: + logger.warning("No GitHub token found - using unauthenticated request (rate limit: 60 requests/hour)") + http = create_pool_manager(url=url, verify=True) # Log proxy environment at debug level @@ -129,7 +185,7 @@ def get_artifact_from_github(owner: str, repo: str, version: str, asset_filter=N logger.debug(f"Using pool manager type: {type(http).__name__}") try: - response = http.request("GET", url) + response = http.request("GET", url, headers=headers) logger.debug(f"Response status: {response.status}") logger.debug(f"Response headers: {response.headers}") From 5fa0fbbe67cf6db9ff43779aabcabcf9539da952 Mon Sep 17 00:00:00 2001 From: flosch62 Date: Fri, 17 Jan 2025 11:56:30 +0100 Subject: [PATCH 6/6] not using github api --- src/helpers.py | 142 ----------------------------------------------- src/integrate.py | 44 ++++++++++----- src/node_srl.py | 51 +++++------------ 3 files changed, 44 insertions(+), 193 deletions(-) diff --git a/src/helpers.py b/src/helpers.py index 907718c..a46d01c 100644 --- a/src/helpers.py +++ b/src/helpers.py @@ -5,8 +5,6 @@ import sys import tempfile -import urllib3 - from jinja2 import Environment, FileSystemLoader import src.topology as topology @@ -99,146 +97,6 @@ def apply_manifest_via_kubectl(yaml_str: str, namespace: str = "eda-system"): finally: os.remove(tmp_path) - -def get_github_token(): - """ - Get GitHub token from environment or GitHub CLI in priority order: - 1. GITHUB_TOKEN environment variable - 2. GH_TOKEN environment variable (GitHub CLI default) - 3. GitHub CLI authentication - - Returns - ------- - str or None - GitHub authentication token if found, None otherwise - """ - - # Check environment variables - token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") - if token: - logger.debug("Found GitHub token in environment variables") - return token - - # Try to get token from GitHub CLI - try: - # Check if gh is installed - result = subprocess.run(["gh", "--version"], capture_output=True, text=True) - if result.returncode == 0: - # Get auth token from gh - result = subprocess.run( - ["gh", "auth", "token"], capture_output=True, text=True - ) - if result.returncode == 0 and result.stdout.strip(): - logger.debug("Found GitHub token from GitHub CLI") - return result.stdout.strip() - except FileNotFoundError: - logger.debug("GitHub CLI (gh) not found") - except Exception as e: - logger.debug(f"Error getting GitHub CLI token: {e}") - - logger.debug("No GitHub token found") - return None - - -def get_artifact_from_github(owner: str, repo: str, version: str, asset_filter=None): - """ - Queries GitHub for a specific release artifact using urllib3. - - Parameters - ---------- - owner: GitHub repository owner - repo: GitHub repository name - version: Version tag to search for (without 'v' prefix) - asset_filter: Optional function(asset_name) -> bool to filter assets - token: Optional GitHub token. If None, will attempt to get from - environment or GitHub CLI - Returns - ------- - Tuple of (filename, download_url) or (None, None) if not found - """ - from src.http_client import create_pool_manager - - tag = f"v{version}" # Assume GitHub tags are prefixed with 'v' - url = f"https://api.github.com/repos/{owner}/{repo}/releases/tags/{tag}" - - token = get_github_token() - - # Set up headers with authentication if token is available - headers = { - "Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28", - "User-Agent": "container-lab/node-download" # Replace with your tool name - } - - if token: - headers["Authorization"] = f"Bearer {token}" - logger.debug("Using authenticated GitHub API request") - else: - logger.warning("No GitHub token found - using unauthenticated request (rate limit: 60 requests/hour)") - - http = create_pool_manager(url=url, verify=True) - - # Log proxy environment at debug level - logger.debug(f"HTTP_PROXY: {os.environ.get('HTTP_PROXY', 'not set')}") - logger.debug(f"HTTPS_PROXY: {os.environ.get('HTTPS_PROXY', 'not set')}") - logger.debug(f"NO_PROXY: {os.environ.get('NO_PROXY', 'not set')}") - logger.debug(f"Using pool manager type: {type(http).__name__}") - - try: - response = http.request("GET", url, headers=headers) - logger.debug(f"Response status: {response.status}") - logger.debug(f"Response headers: {response.headers}") - - if response.status == 403: - response_data = json.loads(response.data.decode("utf-8")) - if "rate limit exceeded" in response_data.get("message", "").lower(): - logger.warning( - f"GitHub API rate limit exceeded. {response_data.get('message')}" - ) - if response_data.get("documentation_url"): - logger.warning(f"See: {response_data['documentation_url']}") - return None, None - else: - logger.error( - f"Access forbidden: {response_data.get('message', 'No message provided')}" - ) - return None, None - - if response.status != 200: - logger.error(f"Failed to fetch release {tag} (status={response.status})") - logger.debug(f"Response data: {response.data.decode('utf-8')}") - return None, None - - data = json.loads(response.data.decode("utf-8")) - assets = data.get("assets", []) - logger.debug(f"Found {len(assets)} assets in release") - - for asset in assets: - name = asset.get("name", "") - logger.debug(f"Checking asset: {name}") - if asset_filter is None or asset_filter(name): - download_url = asset.get("browser_download_url") - logger.info(f"Found matching asset: {name}") - logger.debug(f"Download URL: {download_url}") - return name, download_url - else: - logger.debug(f"Asset {name} did not match filter") - - except urllib3.exceptions.HTTPError as e: - logger.error(f"HTTP error occurred while fetching {tag}: {e}") - logger.debug(f"Error details: {str(e)}") - except json.JSONDecodeError as e: - logger.error(f"Invalid JSON response for {tag}: {e}") - logger.debug(f"Raw response: {response.data.decode('utf-8')}") - except Exception as e: - logger.error(f"Unexpected error while fetching {tag}: {type(e).__name__}") - logger.debug(f"Error details: {str(e)}") - - # No matching asset found - logger.warning("No matching asset found") - return None, None - - def normalize_name(name: str) -> str: """ Returns a Kubernetes-compliant name by: diff --git a/src/integrate.py b/src/integrate.py index 7ebf2db..467a557 100644 --- a/src/integrate.py +++ b/src/integrate.py @@ -107,13 +107,12 @@ def prechecks(self): ) def create_artifacts(self): - """ - Creates artifacts needed by nodes in the topology - """ + """Creates artifacts needed by nodes that need them""" logger.info("Creating artifacts for nodes that need them") - processed = set() # Track which artifacts we've already created + nodes_by_artifact = {} # Track which nodes use which artifacts + # First pass: collect all nodes and their artifacts for node in self.topology.nodes: if not node.needs_artifact(): continue @@ -125,30 +124,49 @@ def create_artifacts(self): logger.warning(f"Could not get artifact details for {node}. Skipping.") continue - # Skip if we already processed this artifact - if artifact_name in processed: - continue - processed.add(artifact_name) + if artifact_name not in nodes_by_artifact: + nodes_by_artifact[artifact_name] = { + "nodes": [], + "filename": filename, + "download_url": download_url, + "version": node.version, + } + nodes_by_artifact[artifact_name]["nodes"].append(node.name) + + # Second pass: create artifacts + for artifact_name, info in nodes_by_artifact.items(): + first_node = info["nodes"][0] + logger.info( + f"Creating YANG artifact for node: {first_node} (version {info['version']})" + ) # Get the YAML and create the artifact - artifact_yaml = node.get_artifact_yaml( - artifact_name, filename, download_url + artifact_yaml = self.topology.nodes[0].get_artifact_yaml( + artifact_name, info["filename"], info["download_url"] ) + if not artifact_yaml: logger.warning( - f"Could not generate artifact YAML for {node}. Skipping." + f"Could not generate artifact YAML for {first_node}. Skipping." ) continue - logger.debug(f"Artifact yaml: {artifact_yaml}.") try: helpers.apply_manifest_via_kubectl( artifact_yaml, namespace="eda-system" ) logger.info(f"Artifact '{artifact_name}' has been created.") + # Log about other nodes using this artifact + other_nodes = info["nodes"][1:] + if other_nodes: + logger.info( + f"Using same artifact for nodes: {', '.join(other_nodes)}" + ) except RuntimeError as ex: if "AlreadyExists" in str(ex): - logger.info(f"Artifact '{artifact_name}' already exists, skipping.") + logger.info( + f"Artifact '{artifact_name}' already exists for nodes: {', '.join(info['nodes'])}" + ) else: logger.error(f"Error creating artifact '{artifact_name}': {ex}") diff --git a/src/node_srl.py b/src/node_srl.py index 740dda1..0d24ea3 100644 --- a/src/node_srl.py +++ b/src/node_srl.py @@ -27,6 +27,10 @@ class SRLNode(Node): SRL_IMAGE = "eda-system/srlimages/srlinux-{version}-bin/srlinux.bin" SRL_IMAGE_MD5 = "eda-system/srlimages/srlinux-{version}-bin/srlinux.bin.md5" + SUPPORTED_SCHEMA_PROFILES = { + "24.10.1": "https://github.com/nokia/srlinux-yang-models/releases/download/v24.10.1/srlinux-24.10.1-492.zip" + } + def __init__(self, name, kind, node_type, version, mgmt_ipv4): super().__init__(name, kind, node_type, version, mgmt_ipv4) # Add cache for artifact info @@ -117,7 +121,8 @@ def get_node_profile(self, topology): """ logger.info(f"Rendering node profile for {self}") - artifact_name, filename = self.get_artifact_metadata() + artifact_name = self.get_artifact_name() + filename = f"srlinux-{self.version}.zip" data = { "namespace": f"clab-{topology.name}", @@ -228,46 +233,16 @@ def get_artifact_name(self): return f"clab-srlinux-{self.version}" def get_artifact_info(self): - """ - Gets SR Linux YANG models artifact information from GitHub. - """ - # Return cached info if available - if self._artifact_info is not None: - return self._artifact_info - - def srlinux_filter(name): - return ( - name.endswith(".zip") - and name.startswith("srlinux-") - and "Source code" not in name - ) + """Gets artifact information for this SR Linux version""" + if self.version not in self.SUPPORTED_SCHEMA_PROFILES: + logger.warning(f"No schema profile URL defined for version {self.version}") + return None, None, None artifact_name = self.get_artifact_name() - filename, download_url = helpers.get_artifact_from_github( - owner="nokia", - repo="srlinux-yang-models", - version=self.version, - asset_filter=srlinux_filter, - ) + filename = f"srlinux-{self.version}.zip" + download_url = self.SUPPORTED_SCHEMA_PROFILES[self.version] - # Cache the result - self._artifact_info = (artifact_name, filename, download_url) - return self._artifact_info - - def get_artifact_metadata(self): - """ - Returns just the artifact name and filename without making API calls. - Used when we don't need the download URL. - """ - if self._artifact_info is not None: - # Return cached info if available - artifact_name, filename, _ = self._artifact_info - return artifact_name, filename - - # If not cached, return basic info without API call - artifact_name = self.get_artifact_name() - filename = f"srlinux-{self.version}.zip" # Assume standard naming - return artifact_name, filename + return artifact_name, filename, download_url def get_artifact_yaml(self, artifact_name, filename, download_url): """