Skip to content

Commit

Permalink
Annual limit sending validation + budget component on ready to send s…
Browse files Browse the repository at this point in the history
…creen (#2010)

* feat: validate annual limit

* fix: modify wrapper to work with older redis keys

* test: add tests!

* chore: `utcnow` is deprecated, use `now` instead

* feat(annual limits): show error if user is over annual limits

* fix: move daily limit message into the appropriate template

* feat: add a client to get daily/yearly sent stats using caching where possible

* add translations

* chore: formatting

* fix send to work without FF too

* feat: fix sending; write tests; 🙏

* fix: only check for `send_exceeds_annual_limit` when FF is on

* fix(tests): only run the new tests with the FF on (they cant pass with it off!)

* chore: remove unused imports

* fix: move secondary message outside banner

* fix: undo ruff change made by accident

* feat(error message): split annual into its own

* feat(send): refactor limit checker to use `notification_counts_client` class when sending

* feat(remaining messages summary): add heading mode; fix missing translations

* feat(ready to send): show RMS componont on page, prevent users from moving forward if service is at either limit

* feat(ready to send): update view template

* feat(annual error mesage email): refactor page a bit, add case for remaining = 0

* fix(notifications_count_client): cast bytes to int (why are they stored as bytes?)

* fix: refactor error message views to make them re-usable

* chore: translations

* tests: add tests for new send code, template code, and changes to notification_counts_client

* chore: fix translation

* chore: add missing translations

* chore: fix failing tests due to merge

* fix: remove commented out code

* test fixes

* chore: refactor a bit so other methods that render _template.html all have the info needed for budget component

* fix failing tests; remove letter test

* fix(test_templates): update mocks in tests

* fix(ready to send): always show the buttons when the FF is off; if the FF is on, show them as long as they have remaining cap today and this year

* fix(test_template): only run the test with the FF on

* fix(rms): update text only versions to use new signature

---------

Co-authored-by: William B <[email protected]>
  • Loading branch information
andrewleith and whabanks authored Dec 9, 2024
1 parent d7f02c0 commit 51b10a9
Show file tree
Hide file tree
Showing 16 changed files with 971 additions and 191 deletions.
38 changes: 32 additions & 6 deletions app/main/views/send.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
)
from app.main.views.dashboard import aggregate_notifications_stats
from app.models.user import Users
from app.notify_client.notification_counts_client import notification_counts_client
from app.s3_client.s3_csv_client import (
copy_bulk_send_file_to_uploads,
list_bulk_send_uploads,
Expand Down Expand Up @@ -649,8 +650,8 @@ def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_

sms_fragments_sent_today = daily_sms_fragment_count(service_id)
emails_sent_today = daily_email_count(service_id)
remaining_sms_message_fragments = current_service.sms_daily_limit - sms_fragments_sent_today
remaining_email_messages = current_service.message_limit - emails_sent_today
remaining_sms_message_fragments_today = current_service.sms_daily_limit - sms_fragments_sent_today
remaining_email_messages_today = current_service.message_limit - emails_sent_today

contents = s3download(service_id, upload_id)

Expand All @@ -659,7 +660,7 @@ def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_
email_reply_to = None
sms_sender = None
recipients_remaining_messages = (
remaining_email_messages if db_template["template_type"] == "email" else remaining_sms_message_fragments
remaining_email_messages_today if db_template["template_type"] == "email" else remaining_sms_message_fragments_today
)

if db_template["template_type"] == "email":
Expand Down Expand Up @@ -743,8 +744,8 @@ def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_
original_file_name=request.args.get("original_file_name", ""),
upload_id=upload_id,
form=CsvUploadForm(),
remaining_messages=remaining_email_messages,
remaining_sms_message_fragments=remaining_sms_message_fragments,
remaining_messages=remaining_email_messages_today,
remaining_sms_message_fragments=remaining_sms_message_fragments_today,
sms_parts_to_send=sms_parts_to_send,
is_sms_parts_estimated=is_sms_parts_estimated,
choose_time_form=choose_time_form,
Expand Down Expand Up @@ -783,7 +784,24 @@ def check_messages(service_id, template_id, upload_id, row_index=2):
data["original_file_name"] = SanitiseASCII.encode(data.get("original_file_name", ""))
data["sms_parts_requested"] = data["stats_daily"]["sms"]["requested"]
data["sms_parts_remaining"] = current_service.sms_daily_limit - daily_sms_fragment_count(service_id)
data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]

if current_app.config["FF_ANNUAL_LIMIT"]:
data["send_exceeds_annual_limit"] = False
data["send_exceeds_daily_limit"] = False
# determine the remaining sends for daily + annual
limit_stats = notification_counts_client.get_limit_stats(current_service)
remaining_annual = limit_stats[data["template"].template_type]["annual"]["remaining"]

if remaining_annual < data["count_of_recipients"]:
data["recipients_remaining_messages"] = remaining_annual
data["send_exceeds_annual_limit"] = True
else:
# if they arent over their limit, and its sms, check if they are over their daily limit
if data["template"].template_type == "sms":
data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]

else:
data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]

if (
data["recipients"].too_many_rows
Expand All @@ -804,6 +822,10 @@ def check_messages(service_id, template_id, upload_id, row_index=2):
if data["send_exceeds_daily_limit"]:
return render_template("views/check/column-errors.html", **data)

if current_app.config["FF_ANNUAL_LIMIT"]:
if data["send_exceeds_annual_limit"]:
return render_template("views/check/column-errors.html", **data)

metadata_kwargs = {
"notification_count": data["count_of_recipients"],
"template_id": str(template_id),
Expand Down Expand Up @@ -1085,6 +1107,10 @@ def get_template_error_dict(exception):
error = "too-many-sms-messages"
elif "Content for template has a character count greater than the limit of" in exception.message:
error = "message-too-long"
elif "Exceeded annual email sending limit" in exception.message:
error = "too-many-email-annual"
elif "Exceeded annual SMS sending limit" in exception.message:
error = "too-many-sms-annual"
else:
raise exception

Expand Down
29 changes: 29 additions & 0 deletions app/main/views/templates.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
TemplateList,
TemplateLists,
)
from app.notify_client.notification_counts_client import notification_counts_client
from app.template_previews import TemplatePreview, get_page_count_for_letter
from app.utils import (
email_or_sms_not_enabled,
Expand Down Expand Up @@ -125,6 +126,31 @@ def get_char_limit_error_msg():
return _("Too many characters")


def get_limit_stats(notification_type):
# get the limit stats for the current service
limit_stats = notification_counts_client.get_limit_stats(current_service)

# transform the stats into a format that can be used in the template
limit_stats = {
"dailyLimit": limit_stats[notification_type]["daily"]["limit"],
"dailyUsed": limit_stats[notification_type]["daily"]["sent"],
"dailyRemaining": limit_stats[notification_type]["daily"]["remaining"],
"yearlyLimit": limit_stats[notification_type]["annual"]["limit"],
"yearlyUsed": limit_stats[notification_type]["annual"]["sent"],
"yearlyRemaining": limit_stats[notification_type]["annual"]["remaining"],
"notification_type": notification_type,
"heading": _("Ready to send?"),
}

# determine ready to send heading
if limit_stats["yearlyRemaining"] == 0:
limit_stats["heading"] = _("Sending paused until annual limit resets")
elif limit_stats["dailyRemaining"] == 0:
limit_stats["heading"] = _("Sending paused until 7pm ET. You can schedule more messages to send later.")

return limit_stats


@main.route("/services/<service_id>/templates/<uuid:template_id>")
@user_has_permissions()
def view_template(service_id, template_id):
Expand All @@ -142,6 +168,7 @@ def view_template(service_id, template_id):
template=get_email_preview_template(template, template_id, service_id),
template_postage=template["postage"],
user_has_template_permission=user_has_template_permission,
**get_limit_stats(template["template_type"]),
)


Expand Down Expand Up @@ -1072,6 +1099,7 @@ def delete_service_template(service_id, template_id):
"views/templates/template.html",
template=get_email_preview_template(template, template["id"], service_id),
user_has_template_permission=True,
**get_limit_stats(template["template_type"]),
)


Expand All @@ -1085,6 +1113,7 @@ def confirm_redact_template(service_id, template_id):
template=get_email_preview_template(template, template["id"], service_id),
user_has_template_permission=True,
show_redaction_message=True,
**get_limit_stats(template["template_type"]),
)


Expand Down
149 changes: 149 additions & 0 deletions app/notify_client/notification_counts_client.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
from datetime import datetime

from notifications_utils.clients.redis import (
email_daily_count_cache_key,
sms_daily_count_cache_key,
)

from app import redis_client, service_api_client, template_statistics_client
from app.models.service import Service


class NotificationCounts:
def get_all_notification_counts_for_today(self, service_id):
# try to get today's stats from redis
todays_sms = redis_client.get(sms_daily_count_cache_key(service_id))
todays_sms = int(todays_sms) if todays_sms is not None else None

todays_email = redis_client.get(email_daily_count_cache_key(service_id))
todays_email = int(todays_email) if todays_email is not None else None

if todays_sms is not None and todays_email is not None:
return {"sms": todays_sms, "email": todays_email}
# fallback to the API if the stats are not in redis
else:
stats = template_statistics_client.get_template_statistics_for_service(service_id, limit_days=1)
transformed_stats = _aggregate_notifications_stats(stats)

return transformed_stats

def get_all_notification_counts_for_year(self, service_id, year):
"""
Get total number of notifications by type for the current service for the current year
Return value:
{
'sms': int,
'email': int
}
"""
stats_today = self.get_all_notification_counts_for_today(service_id)
stats_this_year = service_api_client.get_monthly_notification_stats(service_id, year)["data"]
stats_this_year = _aggregate_stats_from_service_api(stats_this_year)
# aggregate stats_today and stats_this_year
for template_type in ["sms", "email"]:
stats_this_year[template_type] += stats_today[template_type]

return stats_this_year

def get_limit_stats(self, service: Service):
"""
Get the limit stats for the current service, by notification type, including:
- how many notifications were sent today and this year
- the monthy and daily limits
- the number of notifications remaining today and this year
Returns:
dict: A dictionary containing the limit stats for email and SMS notifications. The structure is as follows:
{
"email": {
"annual": {
"limit": int, # The annual limit for email notifications
"sent": int, # The number of email notifications sent this year
"remaining": int, # The number of email notifications remaining this year
},
"daily": {
"limit": int, # The daily limit for email notifications
"sent": int, # The number of email notifications sent today
"remaining": int, # The number of email notifications remaining today
},
},
"sms": {
"annual": {
"limit": int, # The annual limit for SMS notifications
"sent": int, # The number of SMS notifications sent this year
"remaining": int, # The number of SMS notifications remaining this year
},
"daily": {
"limit": int, # The daily limit for SMS notifications
"sent": int, # The number of SMS notifications sent today
"remaining": int, # The number of SMS notifications remaining today
},
}
}
"""

sent_today = self.get_all_notification_counts_for_today(service.id)
sent_thisyear = self.get_all_notification_counts_for_year(service.id, datetime.now().year)

limit_stats = {
"email": {
"annual": {
"limit": service.email_annual_limit,
"sent": sent_thisyear["email"],
"remaining": service.email_annual_limit - sent_thisyear["email"],
},
"daily": {
"limit": service.message_limit,
"sent": sent_today["email"],
"remaining": service.message_limit - sent_today["email"],
},
},
"sms": {
"annual": {
"limit": service.sms_annual_limit,
"sent": sent_thisyear["sms"],
"remaining": service.sms_annual_limit - sent_thisyear["sms"],
},
"daily": {
"limit": service.sms_daily_limit,
"sent": sent_today["sms"],
"remaining": service.sms_daily_limit - sent_today["sms"],
},
},
}

return limit_stats


# TODO: consolidate this function and other functions that transform the results of template_statistics_client calls
def _aggregate_notifications_stats(template_statistics):
template_statistics = _filter_out_cancelled_stats(template_statistics)
notifications = {"sms": 0, "email": 0}
for stat in template_statistics:
notifications[stat["template_type"]] += stat["count"]

return notifications


def _filter_out_cancelled_stats(template_statistics):
return [s for s in template_statistics if s["status"] != "cancelled"]


def _aggregate_stats_from_service_api(stats):
"""Aggregate monthly notification stats excluding cancelled"""
total_stats = {"sms": {}, "email": {}}

for month_data in stats.values():
for msg_type in ["sms", "email"]:
if msg_type in month_data:
for status, count in month_data[msg_type].items():
if status != "cancelled":
if status not in total_stats[msg_type]:
total_stats[msg_type][status] = 0
total_stats[msg_type][status] += count

return {msg_type: sum(counts.values()) for msg_type, counts in total_stats.items()}


notification_counts_client = NotificationCounts()
12 changes: 6 additions & 6 deletions app/templates/components/remaining-messages-summary.html
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{% macro remaining_messages_summary(dailyLimit, dailyUsed, yearlyLimit, yearlyUsed, notification_type, textOnly=None) %}
{% macro remaining_messages_summary(dailyLimit, dailyUsed, yearlyLimit, yearlyUsed, notification_type, headingMode=False, textOnly=None) %}
<!-- Validate textOnly param -->
{% set textOnly_allowed_values = ['text', 'emoji'] %}
{% if textOnly not in textOnly_allowed_values %}
Expand Down Expand Up @@ -95,14 +95,14 @@
</div>
{% endif %}

{% if sections[0].skip %}
{% if not headingMode and sections[0].skip %}
<p class="mt-4 pl-10 py-4 border-l-4 border-gray-300" data-testid="yearly-sending-paused">
Sending paused until annual limit resets
{{ _('Sending paused until annual limit resets') }}
</p>
{% elif sections[0].remaining == "0" %}
{% elif not headingMode and sections[0].remaining == "0" %}
<p class="mt-4 pl-10 py-4 border-l-4 border-gray-300" data-testid="daily-sending-paused">
Sending paused until 7pm ET. You can schedule more messages to send later.
{{ _('Sending paused until 7pm ET. You can schedule more messages to send later.') }}
</p>
{% endif %}

{% endmacro %}
{% endmacro %}
2 changes: 1 addition & 1 deletion app/templates/partials/check/too-many-email-messages.html
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{% from "components/links.html" import content_link %}

<p>
<p data-testid="exceeds-daily">
{%- if current_service.trial_mode %}
{{ _("Your service is in trial mode. To send more messages, <a href='{}'>request to go live</a>").format(url_for('main.request_to_go_live', service_id=current_service.id)) }}
{% else %}
Expand Down
24 changes: 24 additions & 0 deletions app/templates/partials/check/too-many-messages-annual.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
{% from "components/links.html" import content_link %}

{% if template.template_type == 'email' %}
{% set units = _('email messages') %}
{% else %}
{% set units = _('text messages') %}
{% endif %}

<p data-testid="exceeds-annual">
{%- if current_service.trial_mode %}
{{ _("Your service is in trial mode. To send more messages, <a href='{}'>request to go live</a>").format(url_for('main.request_to_go_live', service_id=current_service.id)) }}
{% else %}
{% if recipients_remaining_messages > 0 %}
<p>{{ _('<strong>{}</strong> can only send <strong>{}</strong> more {} until annual limit resets'.format(current_service.name, recipients_remaining_messages, units)) }}</p>
<p>
{{ _('To send some of these messages now, edit the spreadsheet to <strong>{}</strong> recipients maximum. '.format(recipients_remaining_messages)) }}
{{ _('To send to recipients you removed, wait until <strong>April 1, {}</strong> or contact them some other way.'.format(now().year)) }}
</p>
{% else %}
<p>{{ _('<strong>{}</strong> cannot send any more {} until <strong>April 1, {}</strong>'.format(current_service.name, units, now().year)) }}</p>
<p>{{ _('For more information, visit the <a href={}>usage report for {}</a>.'.format(url_for('.monthly', service_id=current_service.id), current_service.name)) }}</p>
{% endif %}
{%- endif -%}
</p>
4 changes: 2 additions & 2 deletions app/templates/partials/check/too-many-sms-message-parts.html
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{% from "components/links.html" import content_link %}

<p>
<p data-testid="exceeds-daily">
{%- if current_service.trial_mode %}
{{ _("Your service is in trial mode. To send more messages, <a href='{}'>request to go live</a>").format(url_for('main.request_to_go_live', service_id=current_service.id)) }}
{% else %}
{{ _("To request a daily limit above {} text messages, {}").format(current_service.sms_daily_limit, content_link(_("contact us"), url_for('main.contact'), is_external_link=true)) }}
{%- endif -%}
</p>
</p>
Loading

0 comments on commit 51b10a9

Please sign in to comment.