From 4b67615a59a14997da624133ba767ab524b2ac6d Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Tue, 10 Oct 2023 10:23:03 +0200 Subject: [PATCH] Add Median and 90th percentile to issue stats. Currently we are only looking at averages. This adds median and 90th percentile to issues stats. Those are much less sensitive to outliers. (Think issue cleanup sessions where very old issues get closed out but then dominate the issue stats at the end of the month). --- labels.py | 34 +++++++++------ markdown_writer.py | 37 ++++++++++------ test_labels.py | 7 +-- test_markdown_writer.py | 78 ++++++++++++++++++++++------------ test_time_to_answer.py | 2 +- test_time_to_close.py | 2 +- test_time_to_first_response.py | 2 +- time_to_answer.py | 9 +++- time_to_close.py | 11 ++++- time_to_first_response.py | 9 +++- 10 files changed, 129 insertions(+), 62 deletions(-) diff --git a/labels.py b/labels.py index 6af1d5b..38a90e9 100644 --- a/labels.py +++ b/labels.py @@ -3,6 +3,7 @@ from typing import List import github3 +import numpy import pytz from classes import IssueWithMetrics @@ -93,27 +94,34 @@ def get_average_time_in_labels( labels: List[str], ) -> dict[str, timedelta]: """Calculate the average time spent in each label.""" - average_time_in_labels = {} - number_of_issues_in_labels = {} + time_in_labels = {} for issue in issues_with_metrics: if issue.label_metrics: for label in issue.label_metrics: if issue.label_metrics[label] is None: continue - if label not in average_time_in_labels: - average_time_in_labels[label] = issue.label_metrics[label] - number_of_issues_in_labels[label] = 1 + if label not in time_in_labels: + time_in_labels[label] = [issue.label_metrics[label]] else: - average_time_in_labels[label] += issue.label_metrics[label] - number_of_issues_in_labels[label] += 1 + time_in_labels[label].append(issue.label_metrics[label]) - for label in average_time_in_labels: - average_time_in_labels[label] = ( - average_time_in_labels[label] / number_of_issues_in_labels[label] - ) + average_time_in_labels = {} + med_time_in_labels = {} + ninety_percentile_in_labels = {} + for label in time_in_labels: + average_time_in_labels[label] = numpy.average(time_in_labels[label]) + med_time_in_labels[label] = numpy.median(time_in_labels[label]) + ninety_percentile_in_labels[label] = numpy.percentile(time_in_labels[label], 90, axis=0) for label in labels: if label not in average_time_in_labels: average_time_in_labels[label] = None - - return average_time_in_labels + med_time_in_labels[label] = None + ninety_percentile_in_labels[label] = None + + stats = { + 'avg': average_time_in_labels, + 'med': med_time_in_labels, + '90p': ninety_percentile_in_labels + } + return stats diff --git a/markdown_writer.py b/markdown_writer.py index 82ed856..8f05bc2 100644 --- a/markdown_writer.py +++ b/markdown_writer.py @@ -170,10 +170,10 @@ def write_to_markdown( def write_overall_metrics_table( issues_with_metrics, - average_time_to_first_response, - average_time_to_close, - average_time_to_answer, - average_time_in_labels, + stats_time_to_first_response, + stats_time_to_close, + stats_time_to_answer, + stats_time_in_labels, num_issues_opened, num_issues_closed, labels, @@ -181,21 +181,34 @@ def write_overall_metrics_table( file, ): """Write the overall metrics table to the markdown file.""" - file.write("| Metric | Value |\n") - file.write("| --- | ---: |\n") + file.write("| Metric | Average | Median | 90th percentile |\n") + file.write("| --- | --- | --- | ---: |\n") if "Time to first response" in columns: + if (stats_time_to_first_response != None): file.write( - f"| Average time to first response | {average_time_to_first_response} |\n" + f"| Time to first response | {stats_time_to_first_response['avg']} | {stats_time_to_first_response['med']} | {stats_time_to_first_response['90p']} |\n" ) + else: + file.write(f"| Time to first response | None | None | None |\n") if "Time to close" in columns: - file.write(f"| Average time to close | {average_time_to_close} |\n") + if (stats_time_to_close != None): + file.write( + f"| Time to close | {stats_time_to_close['avg']} | {stats_time_to_close['med']} | {stats_time_to_close['90p']} |\n" + ) + else: + file.write(f"| Time to close | None | None | None |\n") if "Time to answer" in columns: - file.write(f"| Average time to answer | {average_time_to_answer} |\n") - if labels and average_time_in_labels: + if (stats_time_to_answer != None): + file.write( + f"| Time to answer | {stats_time_to_answer['avg']} | {stats_time_to_answer['med']} | {stats_time_to_answer['90p']} |\n" + ) + else: + file.write(f"| Time to answer | None | None | None |\n") + if labels and stats_time_in_labels: for label in labels: - if f"Time spent in {label}" in columns and label in average_time_in_labels: + if f"Time spent in {label}" in columns and label in stats_time_in_labels['avg']: file.write( - f"| Average time spent in {label} | {average_time_in_labels[label]} |\n" + f"| Time spent in {label} | {stats_time_in_labels['avg'][label]} | {stats_time_in_labels['med'][label]} | {stats_time_in_labels['90p'][label]} |\n" ) file.write(f"| Number of items that remain open | {num_issues_opened} |\n") file.write(f"| Number of items closed | {num_issues_closed} |\n") diff --git a/test_labels.py b/test_labels.py index a266647..a7b0d81 100644 --- a/test_labels.py +++ b/test_labels.py @@ -83,9 +83,10 @@ def test_get_average_time_in_labels(self): """Test get_average_time_in_labels""" labels = ["bug", "feature"] metrics = get_average_time_in_labels(self.issues_with_metrics, labels) - self.assertEqual(len(metrics), 2) - self.assertEqual(metrics["bug"], timedelta(days=2)) - self.assertIsNone(metrics.get("feature")) + print(metrics) + self.assertEqual(len(metrics['avg']), 2) + self.assertEqual(metrics['avg']["bug"], timedelta(days=2)) + self.assertIsNone(metrics['avg'].get("feature")) if __name__ == "__main__": diff --git a/test_markdown_writer.py b/test_markdown_writer.py index 11efeef..b1d0b43 100644 --- a/test_markdown_writer.py +++ b/test_markdown_writer.py @@ -17,7 +17,7 @@ class TestWriteToMarkdown(unittest.TestCase): """Test the write_to_markdown function.""" - + maxDiff = None def test_write_to_markdown(self): """Test that write_to_markdown writes the correct markdown file. @@ -48,20 +48,32 @@ def test_write_to_markdown(self): {"bug": timedelta(days=2)}, ), ] - average_time_to_first_response = timedelta(days=2) - average_time_to_close = timedelta(days=3) - average_time_to_answer = timedelta(days=4) - average_time_in_labels = {"bug": "1 day, 12:00:00"} + time_to_first_response = { + 'avg': timedelta(days=2), + 'med': timedelta(days=2), + '90p': timedelta(days=2)} + time_to_close = { + 'avg': timedelta(days=3), + 'med': timedelta(days=3), + '90p': timedelta(days=3)} + time_to_answer = { + 'avg': timedelta(days=4), + 'med': timedelta(days=4), + '90p': timedelta(days=4)} + time_in_labels = { + 'avg': {"bug": "1 day, 12:00:00"}, + 'med': {"bug": "1 day, 12:00:00"}, + '90p': {"bug": "1 day, 12:00:00"}} num_issues_opened = 2 num_issues_closed = 1 # Call the function write_to_markdown( issues_with_metrics=issues_with_metrics, - average_time_to_first_response=average_time_to_first_response, - average_time_to_close=average_time_to_close, - average_time_to_answer=average_time_to_answer, - average_time_in_labels=average_time_in_labels, + average_time_to_first_response=time_to_first_response, + average_time_to_close=time_to_close, + average_time_to_answer=time_to_answer, + average_time_in_labels=time_in_labels, num_issues_opened=num_issues_opened, num_issues_closed=num_issues_closed, labels=["bug"], @@ -73,12 +85,12 @@ def test_write_to_markdown(self): content = file.read() expected_content = ( "# Issue Metrics\n\n" - "| Metric | Value |\n" - "| --- | ---: |\n" - "| Average time to first response | 2 days, 0:00:00 |\n" - "| Average time to close | 3 days, 0:00:00 |\n" - "| Average time to answer | 4 days, 0:00:00 |\n" - "| Average time spent in bug | 1 day, 12:00:00 |\n" + "| Metric | Average | Median | 90th percentile |\n" + "| --- | --- | --- | ---: |\n" + "| Time to first response | 2 days, 0:00:00 | 2 days, 0:00:00 | 2 days, 0:00:00 |\n" + "| Time to close | 3 days, 0:00:00 | 3 days, 0:00:00 | 3 days, 0:00:00 |\n" + "| Time to answer | 4 days, 0:00:00 | 4 days, 0:00:00 | 4 days, 0:00:00 |\n" + "| Time spent in bug | 1 day, 12:00:00 | 1 day, 12:00:00 | 1 day, 12:00:00 |\n" "| Number of items that remain open | 2 |\n" "| Number of items closed | 1 |\n" "| Total number of items created | 2 |\n\n" @@ -125,10 +137,22 @@ def test_write_to_markdown_with_vertical_bar_in_title(self): {"bug": timedelta(days=2)}, ), ] - average_time_to_first_response = timedelta(days=2) - average_time_to_close = timedelta(days=3) - average_time_to_answer = timedelta(days=4) - average_time_in_labels = {"bug": "1 day, 12:00:00"} + average_time_to_first_response = { + 'avg' : timedelta(days=2), + 'med' : timedelta(days=2), + '90p' : timedelta(days=2)} + average_time_to_close = { + 'avg' : timedelta(days=3), + 'med' : timedelta(days=3), + '90p' : timedelta(days=3)} + average_time_to_answer = { + 'avg' : timedelta(days=4), + 'med' : timedelta(days=4), + '90p' : timedelta(days=4)} + average_time_in_labels = { + 'avg' : {"bug": "1 day, 12:00:00"}, + 'med' : {"bug": "1 day, 12:00:00"}, + '90p' : {"bug": "1 day, 12:00:00"}} num_issues_opened = 2 num_issues_closed = 1 @@ -149,12 +173,12 @@ def test_write_to_markdown_with_vertical_bar_in_title(self): content = file.read() expected_content = ( "# Issue Metrics\n\n" - "| Metric | Value |\n" - "| --- | ---: |\n" - "| Average time to first response | 2 days, 0:00:00 |\n" - "| Average time to close | 3 days, 0:00:00 |\n" - "| Average time to answer | 4 days, 0:00:00 |\n" - "| Average time spent in bug | 1 day, 12:00:00 |\n" + "| Metric | Average | Median | 90th percentile |\n" + "| --- | --- | --- | ---: |\n" + "| Time to first response | 2 days, 0:00:00 | 2 days, 0:00:00 | 2 days, 0:00:00 |\n" + "| Time to close | 3 days, 0:00:00 | 3 days, 0:00:00 | 3 days, 0:00:00 |\n" + "| Time to answer | 4 days, 0:00:00 | 4 days, 0:00:00 | 4 days, 0:00:00 |\n" + "| Time spent in bug | 1 day, 12:00:00 | 1 day, 12:00:00 | 1 day, 12:00:00 |\n" "| Number of items that remain open | 2 |\n" "| Number of items closed | 1 |\n" "| Total number of items created | 2 |\n\n" @@ -259,8 +283,8 @@ def test_writes_markdown_file_with_non_hidden_columns_only(self): content = file.read() expected_content = ( "# Issue Metrics\n\n" - "| Metric | Value |\n" - "| --- | ---: |\n" + "| Metric | Average | Median | 90th percentile |\n" + "| --- | --- | --- | ---: |\n" "| Number of items that remain open | 2 |\n" "| Number of items closed | 1 |\n" "| Total number of items created | 2 |\n\n" diff --git a/test_time_to_answer.py b/test_time_to_answer.py index 7097f95..aab97db 100644 --- a/test_time_to_answer.py +++ b/test_time_to_answer.py @@ -59,7 +59,7 @@ def test_returns_average_time_to_answer(self): ] # Act - result = get_average_time_to_answer(issues_with_metrics) + result = get_average_time_to_answer(issues_with_metrics)['avg'] # Assert self.assertEqual(result, timedelta(seconds=20)) diff --git a/test_time_to_close.py b/test_time_to_close.py index d4f1bea..723461d 100644 --- a/test_time_to_close.py +++ b/test_time_to_close.py @@ -44,7 +44,7 @@ def test_get_average_time_to_close(self): ] # Call the function and check the result - result = get_average_time_to_close(issues_with_metrics) + result = get_average_time_to_close(issues_with_metrics)['avg'] expected_result = timedelta(days=3) self.assertEqual(result, expected_result) diff --git a/test_time_to_first_response.py b/test_time_to_first_response.py index a5132bb..a54b296 100644 --- a/test_time_to_first_response.py +++ b/test_time_to_first_response.py @@ -334,7 +334,7 @@ def test_get_average_time_to_first_response(self): ] # Call the function and check the result - result = get_average_time_to_first_response(issues_with_metrics) + result = get_average_time_to_first_response(issues_with_metrics)['avg'] expected_result = timedelta(days=1.5) self.assertEqual(result, expected_result) diff --git a/time_to_answer.py b/time_to_answer.py index 27cf1d6..5b153d6 100644 --- a/time_to_answer.py +++ b/time_to_answer.py @@ -44,12 +44,19 @@ def get_average_time_to_answer( num_issues_with_time_to_answer = len(issues_with_time_to_answer) if num_issues_with_time_to_answer > 0: average_time_to_answer = numpy.average(answer_times) + med_time_to_answer = numpy.median(answer_times) + ninety_percentile_time_to_answer = numpy.percentile(answer_times, 90, axis=0) else: return None + stats = { + 'avg': timedelta(seconds=average_time_to_answer), + 'med': timedelta(seconds=med_time_to_answer), + '90p': timedelta(seconds=ninety_percentile_time_to_answer)} + # Print the average time to answer converting seconds to a readable time format print(f"Average time to answer: {timedelta(seconds=average_time_to_answer)}") - return timedelta(seconds=average_time_to_answer) + return stats def measure_time_to_answer(discussion: dict) -> Union[timedelta, None]: diff --git a/time_to_close.py b/time_to_close.py index 0a1e312..fe2ba1b 100644 --- a/time_to_close.py +++ b/time_to_close.py @@ -85,9 +85,16 @@ def get_average_time_to_close( num_issues_with_time_to_close = len(issues_with_time_to_close) if num_issues_with_time_to_close > 0 and total_time_to_close is not None: average_time_to_close = numpy.average(close_times) + med_time_to_close = numpy.median(close_times) + ninety_percentile_time_to_close = numpy.percentile(close_times, 90, axis=0) else: return None + stats = { + 'avg': timedelta(seconds=average_time_to_close), + 'med': timedelta(seconds=med_time_to_close), + '90p': timedelta(seconds=ninety_percentile_time_to_close)} + # Print the average time to close converting seconds to a readable time format - print(f"Average time to close: {timedelta(seconds=average_time_to_close)}") - return timedelta(seconds=average_time_to_close) + print(f"Time to close: {timedelta(seconds=average_time_to_close)}") + return stats diff --git a/time_to_first_response.py b/time_to_first_response.py index ae9d481..63b571a 100644 --- a/time_to_first_response.py +++ b/time_to_first_response.py @@ -146,10 +146,17 @@ def get_average_time_to_first_response( return None average_seconds_to_first_response = numpy.average(response_times) + med_seconds_to_first_response = numpy.median(response_times) + ninety_percentile_seconds_to_first_response = numpy.percentile(response_times, 90, axis=0) + + stats = { + 'avg': timedelta(seconds=average_seconds_to_first_response), + 'med': timedelta(seconds=med_seconds_to_first_response), + '90p': timedelta(seconds=ninety_percentile_seconds_to_first_response)} # Print the average time to first response converting seconds to a readable time format print( f"Average time to first response: {timedelta(seconds=average_seconds_to_first_response)}" ) - return timedelta(seconds=average_seconds_to_first_response) + return stats