perf: added pinned python dependency count check (#532)

.github/workflows/ci.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest]
+        os: [ubuntu-20.04]
         python-version: ['3.12']
         toxenv: [python, quality]
 

pylintrc

@@ -64,7 +64,7 @@
 # SERIOUSLY.
 #
 # ------------------------------
-# Generated by edx-lint version: 5.3.6
+# Generated by edx-lint version: 5.4.0
 # ------------------------------
 [MASTER]
 ignore = ,.git,.tox
@@ -289,7 +289,9 @@ disable =
 	logging-fstring-interpolation,
 	django-not-available,
 	c-extension-no-member,
-	missing-timeout
+	missing-timeout,
+	possibly-used-before-assignment,
+	broad-exception-caught
 
 [REPORTS]
 output-format = text
@@ -386,4 +388,4 @@ int-import-graph =
 [EXCEPTIONS]
 overgeneral-exceptions = builtins.Exception
 
-# 6ba39b4accf3da730ae894307bd8cd4a4e122ab3
+# 0a144c96a050dac3a5cbf676c19bf5e8fa63eac2

pylintrc_tweaks

@@ -3,8 +3,10 @@
 ignore+ = ,.git,.tox
 
 [MESSAGES CONTROL]
-disable+ = 
+disable+ =
 	django-not-available,
 	c-extension-no-member,
-	missing-timeout
+	missing-timeout,
+	possibly-used-before-assignment,
+	broad-exception-caught
 

repo_health/check_pinned_python_dependencies.py

@@ -0,0 +1,42 @@
+"""
+Counts the python dependencies which are pinned
+"""
+import os
+
+import pytest
+
+from repo_health import get_file_content
+
+module_dict_key = "pinned_python_dependencies"
+
+
+def get_dependencies_count(repo_path, file_name):
+    """
+   entry point to read requirements from constraints and common-constraints
+   @param repo_path:
+   @param file_name:
+   @return: number.
+   """
+    full_path = os.path.join(repo_path, "requirements/{0}".format(file_name))
+    content = get_file_content(full_path)
+    lines = content.split('\n')
+    dependency_count = 0
+    pinned_dependencies_marker = ['==', '>', '<']
+    for line in lines:
+        if line.startswith('#'):
+            continue
+        if any(marker in line for marker in pinned_dependencies_marker):
+            dependency_count += 1
+        else:
+            continue
+    return dependency_count
+
+
+@pytest.mark.edx_health
+def check_pinned_python_dependencies(repo_path, all_results):
+    """
+    We shall read constraints file
+    """
+    constraints_count = get_dependencies_count(repo_path, 'common_constraints.txt')
+    common_constraints_count = get_dependencies_count(repo_path, 'constraints.txt')
+    all_results[module_dict_key] = constraints_count + common_constraints_count

repo_health/check_ubuntufiles.py

@@ -232,7 +232,7 @@ def get_playbook_data(self, playbook_path):
             packages = self._prepare_data(list(packages))
             return packages
 
-        except Exception as exc:  # pylint: disable=broad-exception-caught
+        except Exception as exc:
             logger.exception("Following error occurred while parsing yml playbook (%s) in configuration repo: %s",
                              playbook_path, exc)
             return []

scripts/repo-health-script.sh

@@ -7,7 +7,7 @@ export LANG=C.UTF-8
 
 WORKSPACE=$PWD
 # If the REPORT_DATE variable is set and not an empty string parse the date to standardize it.
-if [[ -n $REPORT_DATE ]]; then 
+if [[ -n $REPORT_DATE ]]; then
     REPORT_DATE=$(date '+%Y-%m-%d' -d "$REPORT_DATE")
 fi
 
@@ -104,7 +104,7 @@ while IFS= read -r line; do
             --output-path "${ORG_DATA_DIR}/${OUTPUT_FILE_NAME}" \
             -o log_cli=true --exitfirst --noconftest -v -c /dev/null
     }
-    
+
     if REPO_HEALTH_COMMAND; then
         true
     elif REPO_HEALTH_COMMAND; then

tests/fake_repos/python_pinned_dependencies/requirements/common_constraints.txt

@@ -0,0 +1,24 @@
+
+# A central location for most common version constraints
+# (across edx repos) for pip-installation.
+#
+# Similar to other constraint files this file doesn't install any packages.
+# It specifies version constraints that will be applied if a package is needed.
+# When pinning something here, please provide an explanation of why it is a good
+# idea to pin this package across all edx repos, Ideally, link to other information
+# that will help people in the future to remove the pin when possible.
+# Writing an issue against the offending project and linking to it here is good.
+#
+# Note: Changes to this file will automatically be used by other repos, referencing
+#  this file from Github directly. It does not require packaging in edx-lint.
+
+
+# using LTS django version
+Django<5.0
+
+# elasticsearch>=7.14.0 includes breaking changes in it which caused issues in discovery upgrade process.
+# elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html
+elasticsearch<7.14.0
+
+# django-simple-history>3.0.0 adds indexing and causes a lot of migrations to be affected
+django-simple-history==3.0.0

tests/fake_repos/python_pinned_dependencies/requirements/constraints.txt

@@ -0,0 +1,25 @@
+
+# A central location for most common version constraints
+# (across edx repos) for pip-installation.
+#
+# Similar to other constraint files this file doesn't install any packages.
+# It specifies version constraints that will be applied if a package is needed.
+# When pinning something here, please provide an explanation of why it is a good
+# idea to pin this package across all edx repos, Ideally, link to other information
+# that will help people in the future to remove the pin when possible.
+# Writing an issue against the offending project and linking to it here is good.
+#
+# Note: Changes to this file will automatically be used by other repos, referencing
+#  this file from Github directly. It does not require packaging in edx-lint.
+
+
+# using LTS django version
+Django<5.0
+
+# elasticsearch>=7.14.0 includes breaking changes in it which caused issues in discovery upgrade process.
+# elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html
+elasticsearch<7.14.0
+
+# django-simple-history>3.0.0 adds indexing and causes a lot of migrations to be affected
+django-simple-history==3.0.0
+