feat: validate annual limit (#2002)

* feat: validate annual limit

* fix: modify wrapper to work with older redis keys

* test: add tests!

* chore: `utcnow` is deprecated, use `now` instead

* feat(annual limits): show error if user is over annual limits

* fix: move daily limit message into the appropriate template

* feat: add a client to get daily/yearly sent stats using caching where possible

* add translations

* chore: formatting

* fix send to work without FF too

* feat: fix sending; write tests; 🙏

* fix: only check for `send_exceeds_annual_limit` when FF is on

* fix(tests): only run the new tests with the FF on (they cant pass with it off!)

* chore: remove unused imports

* fix: move secondary message outside banner

* fix: undo ruff change made by accident

---------

Co-authored-by: William B <[email protected]>

app/__init__.py

@@ -226,7 +226,7 @@ def get_locale():
     application.jinja_env.globals["show_tou_prompt"] = show_tou_prompt
     application.jinja_env.globals["parse_ua"] = parse
     application.jinja_env.globals["events_key"] = EVENTS_KEY
-    application.jinja_env.globals["now"] = datetime.utcnow
+    application.jinja_env.globals["now"] = datetime.now
 
     # Initialize Salesforce Account list
     if application.config["FF_SALESFORCE_CONTACT"]:

app/main/views/send.py

@@ -1,5 +1,6 @@
 import itertools
 import json
+from datetime import datetime, timezone
 from string import ascii_uppercase
 from zipfile import BadZipFile
 
@@ -53,6 +54,7 @@
 )
 from app.main.views.dashboard import aggregate_notifications_stats
 from app.models.user import Users
+from app.notify_client.notification_counts_client import notification_counts_client
 from app.s3_client.s3_csv_client import (
     copy_bulk_send_file_to_uploads,
     list_bulk_send_uploads,
@@ -649,8 +651,8 @@ def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_
 
     sms_fragments_sent_today = daily_sms_fragment_count(service_id)
     emails_sent_today = daily_email_count(service_id)
-    remaining_sms_message_fragments = current_service.sms_daily_limit - sms_fragments_sent_today
-    remaining_email_messages = current_service.message_limit - emails_sent_today
+    remaining_sms_message_fragments_today = current_service.sms_daily_limit - sms_fragments_sent_today
+    remaining_email_messages_today = current_service.message_limit - emails_sent_today
 
     contents = s3download(service_id, upload_id)
 
@@ -659,7 +661,7 @@ def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_
     email_reply_to = None
     sms_sender = None
     recipients_remaining_messages = (
-        remaining_email_messages if db_template["template_type"] == "email" else remaining_sms_message_fragments
+        remaining_email_messages_today if db_template["template_type"] == "email" else remaining_sms_message_fragments_today
     )
 
     if db_template["template_type"] == "email":
@@ -743,8 +745,8 @@ def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_
         original_file_name=request.args.get("original_file_name", ""),
         upload_id=upload_id,
         form=CsvUploadForm(),
-        remaining_messages=remaining_email_messages,
-        remaining_sms_message_fragments=remaining_sms_message_fragments,
+        remaining_messages=remaining_email_messages_today,
+        remaining_sms_message_fragments=remaining_sms_message_fragments_today,
         sms_parts_to_send=sms_parts_to_send,
         is_sms_parts_estimated=is_sms_parts_estimated,
         choose_time_form=choose_time_form,
@@ -783,7 +785,29 @@ def check_messages(service_id, template_id, upload_id, row_index=2):
     data["original_file_name"] = SanitiseASCII.encode(data.get("original_file_name", ""))
     data["sms_parts_requested"] = data["stats_daily"]["sms"]["requested"]
     data["sms_parts_remaining"] = current_service.sms_daily_limit - daily_sms_fragment_count(service_id)
-    data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]
+
+    if current_app.config["FF_ANNUAL_LIMIT"]:
+        # Override the remaining messages counts with the remaining annual counts, if the latter are lower
+        stats_ytd = notification_counts_client.get_all_notification_counts_for_year(service_id, datetime.now(timezone.utc).year)
+        remaining_sms_this_year = current_service.sms_annual_limit - stats_ytd["sms"]
+        remaining_email_this_year = current_service.email_annual_limit - stats_ytd["email"]
+
+        # Show annual limit validation over the daily one (even if both are true)
+        data["send_exceeds_annual_limit"] = False
+        data["send_exceeds_daily_limit"] = False
+        if data["template"].template_type == "email":
+            if remaining_email_this_year < data["count_of_recipients"]:
+                data["recipients_remaining_messages"] = remaining_email_this_year
+                data["send_exceeds_annual_limit"] = True
+        else:
+            if remaining_sms_this_year < data["count_of_recipients"]:
+                data["recipients_remaining_messages"] = remaining_sms_this_year
+                data["send_exceeds_annual_limit"] = True
+            else:
+                data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]
+
+    else:
+        data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]
 
     if (
         data["recipients"].too_many_rows
@@ -804,6 +828,10 @@ def check_messages(service_id, template_id, upload_id, row_index=2):
     if data["send_exceeds_daily_limit"]:
         return render_template("views/check/column-errors.html", **data)
 
+    if current_app.config["FF_ANNUAL_LIMIT"]:
+        if data["send_exceeds_annual_limit"]:
+            return render_template("views/check/column-errors.html", **data)
+
     metadata_kwargs = {
         "notification_count": data["count_of_recipients"],
         "template_id": str(template_id),

app/notify_client/notification_counts_client.py

@@ -0,0 +1,75 @@
+from notifications_utils.clients.redis import (
+    email_daily_count_cache_key,
+    sms_daily_count_cache_key,
+)
+
+from app import redis_client, service_api_client, template_statistics_client
+
+
+class NotificationCounts:
+    def get_all_notification_counts_for_today(self, service_id):
+        # try to get today's stats from redis
+        todays_sms = redis_client.get(sms_daily_count_cache_key(service_id))
+        todays_email = redis_client.get(email_daily_count_cache_key(service_id))
+
+        if todays_sms is not None and todays_email is not None:
+            return {"sms": todays_sms, "email": todays_email}
+        # fallback to the API if the stats are not in redis
+        else:
+            stats = template_statistics_client.get_template_statistics_for_service(service_id, limit_days=1)
+            transformed_stats = _aggregate_notifications_stats(stats)
+
+            return transformed_stats
+
+    def get_all_notification_counts_for_year(self, service_id, year):
+        """
+        Get total number of notifications by type for the current service for the current year
+
+        Return value:
+        {
+            'sms': int,
+            'email': int
+        }
+
+        """
+        stats_today = self.get_all_notification_counts_for_today(service_id)
+        stats_this_year = service_api_client.get_monthly_notification_stats(service_id, year)["data"]
+        stats_this_year = _aggregate_stats_from_service_api(stats_this_year)
+        # aggregate stats_today and stats_this_year
+        for template_type in ["sms", "email"]:
+            stats_this_year[template_type] += stats_today[template_type]
+
+        return stats_this_year
+
+
+# TODO: consolidate this function and other functions that transform the results of template_statistics_client calls
+def _aggregate_notifications_stats(template_statistics):
+    template_statistics = _filter_out_cancelled_stats(template_statistics)
+    notifications = {"sms": 0, "email": 0}
+    for stat in template_statistics:
+        notifications[stat["template_type"]] += stat["count"]
+
+    return notifications
+
+
+def _filter_out_cancelled_stats(template_statistics):
+    return [s for s in template_statistics if s["status"] != "cancelled"]
+
+
+def _aggregate_stats_from_service_api(stats):
+    """Aggregate monthly notification stats excluding cancelled"""
+    total_stats = {"sms": {}, "email": {}}
+
+    for month_data in stats.values():
+        for msg_type in ["sms", "email"]:
+            if msg_type in month_data:
+                for status, count in month_data[msg_type].items():
+                    if status != "cancelled":
+                        if status not in total_stats[msg_type]:
+                            total_stats[msg_type][status] = 0
+                        total_stats[msg_type][status] += count
+
+    return {msg_type: sum(counts.values()) for msg_type, counts in total_stats.items()}
+
+
+notification_counts_client = NotificationCounts()

app/notify_client/template_statistics_api_client.py

@@ -1,4 +1,7 @@
+from itertools import groupby
+
 from app.notify_client import NotifyAdminAPIClient
+from app.utils import DELIVERED_STATUSES, FAILURE_STATUSES
 
 
 class TemplateStatisticsApiClient(NotifyAdminAPIClient):
@@ -17,3 +20,46 @@ def get_template_statistics_for_template(self, service_id, template_id):
 
 
 template_statistics_client = TemplateStatisticsApiClient()
+
+
+class TemplateStatistics:
+    def __init__(self, stats):
+        self.stats = stats
+
+    def as_aggregates(self):
+        template_statistics = self._filter_out_cancelled_stats(self.stats)
+        notifications = {
+            template_type: {status: 0 for status in ("requested", "delivered", "failed")} for template_type in ["sms", "email"]
+        }
+        for stat in template_statistics:
+            notifications[stat["template_type"]]["requested"] += stat["count"]
+            if stat["status"] in DELIVERED_STATUSES:
+                notifications[stat["template_type"]]["delivered"] += stat["count"]
+            elif stat["status"] in FAILURE_STATUSES:
+                notifications[stat["template_type"]]["failed"] += stat["count"]
+
+        return notifications
+
+    def as_template_usage(self, sort_key="count"):
+        template_statistics = self._filter_out_cancelled_stats(self.stats)
+        templates = []
+        for k, v in groupby(
+            sorted(template_statistics, key=lambda x: x["template_id"]),
+            key=lambda x: x["template_id"],
+        ):
+            template_stats = list(v)
+
+            templates.append(
+                {
+                    "template_id": k,
+                    "template_name": template_stats[0]["template_name"],
+                    "template_type": template_stats[0]["template_type"],
+                    "is_precompiled_letter": template_stats[0]["is_precompiled_letter"],
+                    "count": sum(s["count"] for s in template_stats),
+                }
+            )
+
+        return sorted(templates, key=lambda x: x[sort_key], reverse=True)
+
+    def _filter_out_cancelled_stats(self, template_statistics):
+        return [s for s in template_statistics if s["status"] != "cancelled"]

app/templates/partials/check/too-many-email-messages.html

@@ -1,9 +1,29 @@
+{% from "components/banner.html" import banner_wrapper %}
 {% from "components/links.html" import content_link %}
 
 <p>
   {%- if current_service.trial_mode %}
     {{ _("Your service is in trial mode. To send more messages, <a href='{}'>request to go live</a>").format(url_for('main.request_to_go_live', service_id=current_service.id)) }}
   {% else %}
-    {{ _("To request a daily limit above {} emails, {}").format(current_service.message_limit, content_link(_("contact us"), url_for('main.contact'), is_external_link=true)) }}
+    {% if send_exceeds_annual_limit %}
+
+      <p data-testid="exceeds-annual">{{ _('<strong>{}</strong> can only send <strong>{}</strong> more email messages until annual limit resets'.format(current_service.name, recipients_remaining_messages)) }}</p>
+      <p>
+        {{ _('To send some of these messages now, edit the spreadsheet to <strong>{}</strong> recipients maximum. '.format(recipients_remaining_messages)) }}
+        {{ _('To send to recipients you removed, wait until <strong>April 1, {}</strong> or contact them some other way.'.format(now().year)) }}
+      </p>
+
+    {% elif send_exceeds_daily_limit or recipients.more_rows_than_can_send %}
+      {% call banner_wrapper(type='dangerous') %}
+        {{ _("To request a daily limit above {} emails, {}").format(current_service.message_limit, content_link(_("contact us"), url_for('main.contact'), is_external_link=true)) }}
+      {% endcall %}
+
+      <h2 class="heading-medium">{{ _('You cannot send all these email messages today') }}</h2>
+      <p data-testid="exceeds-daily">
+        {{ _("You can try sending these messages after {} Eastern Time. Check {}.").format(time_to_reset[current_lang],
+        content_link(_("your current local time"), _('https://nrc.canada.ca/en/web-clock/'), is_external_link=true))}}
+      </p>
+    {% endif %}
+
   {%- endif -%}
 </p>

app/templates/partials/check/too-many-sms-message-parts.html

@@ -1,6 +1,6 @@
 {% from "components/links.html" import content_link %}
 
-<p>
+<p data-testid="exceeds-daily">
   {%- if current_service.trial_mode %}
     {{ _("Your service is in trial mode. To send more messages, <a href='{}'>request to go live</a>").format(url_for('main.request_to_go_live', service_id=current_service.id)) }}
   {% else %}

app/templates/views/check/column-errors.html

@@ -10,7 +10,9 @@
 {% set prefix_txt = _('a column called') %}
 {% set prefix_plural_txt = _('columns called') %}
 
-{% if send_exceeds_daily_limit and (sms_parts_remaining <= 0) %}
+{% if send_exceeds_annual_limit %}
+  {% set page_title = _('These messages exceed the annual limit') %}
+{% elif send_exceeds_daily_limit and (sms_parts_remaining <= 0) %}
   {% set page_title = _('These messages exceed your daily limit') %}
 {% elif send_exceeds_daily_limit or recipients.more_rows_than_can_send %}
   {% set page_title = _('These messages exceed your daily limit') %}
@@ -168,20 +170,9 @@ <h2 class="heading-medium">{{ _('You cannot send all these text messages today')
         {% call banner_wrapper(type='dangerous') %}
           {% include "partials/check/too-many-email-messages.html" %}
         {% endcall %}
-      {% elif recipients.more_rows_than_can_send %}
-          {% call banner_wrapper(type='dangerous') %}
-            {% include "partials/check/too-many-email-messages.html" %}
-          {% endcall %}
-          <h2 class="heading-medium">{{ _('You cannot send all these email messages today') }}</h2>
-          <p>
-            {{ _("You can try sending these messages after {} Eastern Time. Check {}.").format(time_to_reset[current_lang],
-            content_link(_("your current local time"), _('https://nrc.canada.ca/en/web-clock/'), is_external_link=true))}}
-          </p>
-
-
+      {% elif recipients.more_rows_than_can_send or  send_exceeds_annual_limit %}
+          {% include "partials/check/too-many-email-messages.html" %}
       {% endif %}
-
-
   </div>
 
   {% if not send_exceeds_daily_limit %}

app/translations/csv/fr.csv

@@ -2027,4 +2027,8 @@
 "Annual usage","Utilisation annuelle"
 "resets at 7pm Eastern Time","Réinitialisation à 19 h, heure de l’Est"
 "Visit usage report","Consulter le rapport d’utilisation"
-"Month by month totals","Totaux mensuels"
+"Month by month totals","Totaux mensuels"
+"<strong>{}</strong> can only send <strong>{}</strong> more email messages until annual limit resets","FR: <strong>{}</strong> can only send <strong>{}</strong> more email messages until annual limit resets"
+"To send some of these messages now, edit the spreadsheet to <strong>{}</strong> recipients maximum. ","FR: To send some of these messages now, edit the spreadsheet to <strong>{}</strong> recipients maximum. "
+"To send to recipients you removed, wait until <strong>April 1, {}</strong> or contact them some other way.","FR: To send to recipients you removed, wait until <strong>April 1, {}</strong> or contact them some other way"
+"These messages exceed the annual limit","FR: These messages exceed the annual limit"

gunicorn_config.py

@@ -51,6 +51,7 @@
 # Start timer for total running time
 start_time = time.time()
 
+
 def on_starting(server):
     server.log.info("Starting Notifications Admin")