Improvements to mindependency dev_requirement conflict resolution

eng/tox/install_depend_packages.py

@@ -5,21 +5,21 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
 
-
 import argparse
 import os
 import sys
 import logging
 import re
+
 from subprocess import check_call
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Callable, Optional
 from pkg_resources import parse_version, Requirement
 from pypi_tools.pypi import PyPIClient
 from packaging.specifiers import SpecifierSet
-from packaging.version import Version, parse
+from packaging.version import Version
 
 from ci_tools.parsing import ParsedSetup, parse_require
-from ci_tools.functions import compare_python_version
+from ci_tools.functions import compare_python_version, handle_incompatible_minimum_dev_reqs
 
 from typing import List
 
@@ -101,8 +101,15 @@ def install_dependent_packages(setup_py_file_path, dependency_type, temp_dir):
         override_added_packages.extend(check_pkg_against_overrides(pkg_spec))
 
     logging.info("%s released packages: %s", dependency_type, released_packages)
-    # filter released packages from dev_requirements and create a new file "new_dev_requirements.txt"
-    dev_req_file_path = filter_dev_requirements(setup_py_file_path, released_packages, temp_dir, dependency_type)
+
+    additionalFilterFn = None
+    if dependency_type == "Minimum":
+        additionalFilterFn = handle_incompatible_minimum_dev_reqs
+
+    # before september 2024, filter_dev_requirements only would remove any packages present in released_packages from the dev_requirements,
+    # then create a new file "new_dev_requirements.txt" without the problematic packages.
+    # after september 2024, filter_dev_requirements will also check for **compatibility** with the packages being installed when filtering the dev_requirements.
+    dev_req_file_path = filter_dev_requirements(setup_py_file_path, released_packages, temp_dir, additionalFilterFn)
 
     if override_added_packages:
         logging.info(f"Expanding the requirement set by the packages {override_added_packages}.")
@@ -249,7 +256,6 @@ def process_requirement(req, dependency_type, orig_pkg_name):
 
     # think of the various versions that come back from pypi as the top of a funnel
     # We apply generic overrides -> platform specific overrides -> package specific overrides
-
     versions = process_bounded_versions(orig_pkg_name, pkg_name, versions)
 
     # Search from lowest to latest in case of finding minimum dependency
@@ -301,17 +307,15 @@ def check_req_against_exclusion(req, req_to_exclude):
 
     return req_id == req_to_exclude
 
-# todo: remove when merging #37450
-def replace_identity(dev_requirement_line) -> str:
-    regex = r"azure[-_]identity"
-
-    if re.search(regex, dev_requirement_line):
-        return "azure-identity==1.17.0\n"
-    else:
-        return dev_requirement_line
 
+def filter_dev_requirements(setup_py_path, released_packages, temp_dir, additionalFilterFn: Optional[Callable[[str, List[str],List[Requirement]], List[str]]] = None):
+    """
+    This function takes an existing package path, a list of specific package specifiers that we have resolved, a temporary directory to write
+    the modified dev_requirements to, and an optional additionalFilterFn that can be used to further filter the dev_requirements file if necessary.
 
-def filter_dev_requirements(setup_py_path, released_packages, temp_dir, dependency_type):
+    The function will filter out any requirements present in the dev_requirements file that are present in the released_packages list (aka are required
+    by the package).
+    """
     # This method returns list of requirements from dev_requirements by filtering out packages in given list
     dev_req_path = os.path.join(os.path.dirname(setup_py_path), DEV_REQ_FILE)
     requirements = []
@@ -320,12 +324,13 @@ def filter_dev_requirements(setup_py_path, released_packages, temp_dir, dependen
 
     # filter out any package available on PyPI (released_packages)
     # include packages without relative reference and packages not available on PyPI
-    released_packages = [p.split("==")[0] for p in released_packages]
+    released_packages = [parse_require(p) for p in released_packages]
+    released_package_names = [p.key for p in released_packages]
     # find prebuilt whl paths in dev requiremente
     prebuilt_dev_reqs = [os.path.basename(req.replace("\n", "")) for req in requirements if os.path.sep in req]
     # filter any req if wheel is for a released package
-    req_to_exclude = [req for req in prebuilt_dev_reqs if req.split("-")[0].replace("_", "-") in released_packages]
-    req_to_exclude.extend(released_packages)
+    req_to_exclude = [req for req in prebuilt_dev_reqs if req.split("-")[0].replace("_", "-") in released_package_names]
+    req_to_exclude.extend(released_package_names)
 
     filtered_req = [
         req
@@ -334,9 +339,9 @@ def filter_dev_requirements(setup_py_path, released_packages, temp_dir, dependen
         and not any([check_req_against_exclusion(req, i) for i in req_to_exclude])
     ]
 
-    if dependency_type == "Minimum":
-        # replace identity with the minimum version of the package
-        filtered_req = [replace_identity(req) for req in filtered_req]
+    if additionalFilterFn:
+        # this filter function handles the case where a dev requirement is incompatible with the current set of targeted packages
+        filtered_req = additionalFilterFn(setup_py_path, filtered_req, released_packages)
 
     logging.info("Filtered dev requirements: %s", filtered_req)
 
@@ -345,7 +350,7 @@ def filter_dev_requirements(setup_py_path, released_packages, temp_dir, dependen
         # create new dev requirements file with different name for filtered requirements
         new_dev_req_path = os.path.join(temp_dir, NEW_DEV_REQ_FILE)
         with open(new_dev_req_path, "w") as dev_req_file:
-            dev_req_file.writelines(filtered_req)
+            dev_req_file.writelines(line if line.endswith("\n") else line + "\n" for line in filtered_req)
 
     return new_dev_req_path
 

scripts/devops_tasks/common_tasks.py

@@ -205,7 +205,7 @@ def is_required_version_on_pypi(package_name: str, spec: str) -> bool:
             versions = [str(v) for v in versions if v in specifier]
     except:
         logging.error("Package {} is not found on PyPI".format(package_name))
-    return versions
+    return bool(versions)
 
 
 def find_packages_missing_on_pypi(path: str) -> Iterable[str]:

sdk/identity/azure-identity-broker/pyproject.toml

@@ -1,4 +1,3 @@
 [tool.azure-sdk-build]
 type_check_samples = false
-pyright = false
-mindependency = false
+pyright = false