Merge branch 'main' into chore/python-3.12

.devcontainer/Dockerfile

@@ -17,6 +17,7 @@ RUN apt-get update \
     emacs \
     exa \
     fd-find \
+    fzf \
     git \
     iputils-ping \
     iproute2 \

.devcontainer/scripts/installations.sh

@@ -13,6 +13,10 @@ echo -e "alias ll='exa -alh@ --git'" >> ~/.zshrc
 echo -e "alias lt='exa -al -T -L 2'" >> ~/.zshrc
 echo -e "alias poe='poetry run poe'" >> ~/.zshrc
 
+echo -e "# fzf key bindings and completion" >> ~/.zshrc
+echo -e "source /usr/share/doc/fzf/examples/key-bindings.zsh" >> ~/.zshrc
+echo -e "source /usr/share/doc/fzf/examples/completion.zsh" >> ~/.zshrc
+
 # Poetry autocomplete
 echo -e "fpath+=/.zfunc" >> ~/.zshrc
 echo -e "autoload -Uz compinit && compinit"

app/config.py

@@ -80,10 +80,10 @@ class Config(object):
     # FEATURE FLAGS
     FF_SALESFORCE_CONTACT = env.bool("FF_SALESFORCE_CONTACT", True)
     FF_RTL = env.bool("FF_RTL", True)
-    FF_ANNUAL_LIMIT = env.bool("FF_ANNUAL_LIMIT", True)
+    FF_ANNUAL_LIMIT = env.bool("FF_ANNUAL_LIMIT", False)
 
-    FREE_YEARLY_EMAIL_LIMIT = env.int("FREE_YEARLY_EMAIL_LIMIT", 10_000_000)
-    FREE_YEARLY_SMS_LIMIT = env.int("FREE_YEARLY_SMS_LIMIT", 25_000)
+    FREE_YEARLY_EMAIL_LIMIT = env.int("FREE_YEARLY_EMAIL_LIMIT", 20_000_000)
+    FREE_YEARLY_SMS_LIMIT = env.int("FREE_YEARLY_SMS_LIMIT", 100_000)
     GC_ARTICLES_API = os.environ.get("GC_ARTICLES_API", "articles.alpha.canada.ca/notification-gc-notify")
     GC_ARTICLES_API_AUTH_PASSWORD = os.environ.get("GC_ARTICLES_API_AUTH_PASSWORD")
     GC_ARTICLES_API_AUTH_USERNAME = os.environ.get("GC_ARTICLES_API_AUTH_USERNAME")
@@ -215,7 +215,7 @@ class Test(Development):
     NO_BRANDING_ID = "0af93cf1-2c49-485f-878f-f3e662e651ef"
 
     FF_RTL = True
-    FF_ANNUAL_LIMIT = False
+    FF_ANNUAL_LIMIT = True
 
 
 class ProductionFF(Config):

app/extensions.py

@@ -1,5 +1,6 @@
 from flask_caching import Cache
 from notifications_utils.clients.antivirus.antivirus_client import AntivirusClient
+from notifications_utils.clients.redis.annual_limit import RedisAnnualLimit
 from notifications_utils.clients.redis.bounce_rate import RedisBounceRate
 from notifications_utils.clients.redis.redis_client import RedisClient
 from notifications_utils.clients.statsd.statsd_client import StatsdClient
@@ -10,4 +11,6 @@
 zendesk_client = ZendeskClient()
 redis_client = RedisClient()
 bounce_rate_client = RedisBounceRate(redis_client)
+annual_limit_client = RedisAnnualLimit(redis_client)
+
 cache = Cache(config={"CACHE_TYPE": "simple"})  # TODO: pull config out to config.py later

app/main/views/dashboard.py

@@ -24,7 +24,7 @@
     service_api_client,
     template_statistics_client,
 )
-from app.extensions import bounce_rate_client
+from app.extensions import annual_limit_client, bounce_rate_client
 from app.main import main
 from app.models.enum.bounce_rate_status import BounceRateStatus
 from app.models.enum.notification_statuses import NotificationStatuses
@@ -229,16 +229,90 @@ def usage(service_id):
 @main.route("/services/<service_id>/monthly")
 @user_has_permissions("view_activity")
 def monthly(service_id):
+    def combine_daily_to_annual(daily, annual, mode):
+        if mode == "redis":
+            # the redis client omits properties if there are no counts yet, so account for this here\
+            daily_redis = {
+                field: daily.get(field, 0) for field in ["sms_delivered", "sms_failed", "email_delivered", "email_failed"]
+            }
+            annual["sms"] += daily_redis["sms_delivered"] + daily_redis["sms_failed"]
+            annual["email"] += daily_redis["email_delivered"] + daily_redis["email_failed"]
+        elif mode == "db":
+            annual["sms"] += daily["sms"]["requested"]
+            annual["email"] += daily["email"]["requested"]
+
+        return annual
+
+    def combine_daily_to_monthly(daily, monthly, mode):
+        if mode == "redis":
+            # the redis client omits properties if there are no counts yet, so account for this here\
+            daily_redis = {
+                field: daily.get(field, 0) for field in ["sms_delivered", "sms_failed", "email_delivered", "email_failed"]
+            }
+
+            monthly[0]["sms_counts"]["failed"] += daily_redis["sms_failed"]
+            monthly[0]["sms_counts"]["requested"] += daily_redis["sms_failed"] + daily_redis["sms_delivered"]
+            monthly[0]["email_counts"]["failed"] += daily_redis["email_failed"]
+            monthly[0]["email_counts"]["requested"] += daily_redis["email_failed"] + daily_redis["email_delivered"]
+        elif mode == "db":
+            monthly[0]["sms_counts"]["failed"] += daily["sms"]["failed"]
+            monthly[0]["sms_counts"]["requested"] += daily["sms"]["requested"]
+            monthly[0]["email_counts"]["failed"] += daily["email"]["failed"]
+            monthly[0]["email_counts"]["requested"] += daily["email"]["requested"]
+
+        return monthly
+
+    def aggregate_by_type(notification_data):
+        counts = {"sms": 0, "email": 0, "letter": 0}
+        for month_data in notification_data["data"].values():
+            for message_type, message_counts in month_data.items():
+                if isinstance(message_counts, dict):
+                    counts[message_type] += sum(message_counts.values())
+
+        # return the result
+        return counts
+
     year, current_financial_year = requested_and_current_financial_year(request)
+
+    # if FF_ANNUAL is on
+    if current_app.config["FF_ANNUAL_LIMIT"]:
+        monthly_data = service_api_client.get_monthly_notification_stats(service_id, year)
+        annual_data = aggregate_by_type(monthly_data)
+
+        todays_data = annual_limit_client.get_all_notification_counts(current_service.id)
+
+        # if redis is empty, query the db
+        if todays_data is None:
+            todays_data = service_api_client.get_service_statistics(service_id, limit_days=1, today_only=False)
+            annual_data_aggregate = combine_daily_to_annual(todays_data, annual_data, "db")
+
+            months = (format_monthly_stats_to_list(monthly_data["data"]),)
+            monthly_data_aggregate = combine_daily_to_monthly(todays_data, months[0], "db")
+        else:
+            # aggregate daily + annual
+            current_app.logger.info("todays data" + str(todays_data))
+            annual_data_aggregate = combine_daily_to_annual(todays_data, annual_data, "redis")
+
+            months = (format_monthly_stats_to_list(monthly_data["data"]),)
+            monthly_data_aggregate = combine_daily_to_monthly(todays_data, months[0], "redis")
+    else:
+        monthly_data_aggregate = (
+            format_monthly_stats_to_list(service_api_client.get_monthly_notification_stats(service_id, year)["data"]),
+        )
+        monthly_data_aggregate = monthly_data_aggregate[0]
+        annual_data_aggregate = None
+
     return render_template(
         "views/dashboard/monthly.html",
-        months=format_monthly_stats_to_list(service_api_client.get_monthly_notification_stats(service_id, year)["data"]),
+        months=monthly_data_aggregate,
         years=get_tuples_of_financial_years(
             partial_url=partial(url_for, ".monthly", service_id=service_id),
             start=current_financial_year - 2,
             end=current_financial_year,
         ),
+        annual_data=annual_data_aggregate,
         selected_year=year,
+        current_financial_year=current_financial_year,
     )
 
 
@@ -284,6 +358,21 @@ def aggregate_notifications_stats(template_statistics):
 
 
 def get_dashboard_partials(service_id):
+    def aggregate_by_type(data, daily_data):
+        counts = {"sms": 0, "email": 0, "letter": 0}
+        # flatten out this structure to match the above
+        for month_data in data["data"].values():
+            for message_type, message_counts in month_data.items():
+                if isinstance(message_counts, dict):
+                    counts[message_type] += sum(message_counts.values())
+
+        # add todays data to the annual data
+        counts = {
+            "sms": counts["sms"] + daily_data["sms"]["requested"],
+            "email": counts["email"] + daily_data["email"]["requested"],
+        }
+        return counts
+
     all_statistics_weekly = template_statistics_client.get_template_statistics_for_service(service_id, limit_days=7)
     template_statistics_weekly = aggregate_template_usage(all_statistics_weekly)
 
@@ -300,6 +389,10 @@ def get_dashboard_partials(service_id):
     dashboard_totals_weekly = (get_dashboard_totals(stats_weekly),)
     bounce_rate_data = get_bounce_rate_data_from_redis(service_id)
 
+    # get annual data from fact table (all data this year except today)
+    annual_data = service_api_client.get_monthly_notification_stats(service_id, year=get_current_financial_year())
+    annual_data = aggregate_by_type(annual_data, dashboard_totals_daily[0])
+
     return {
         "upcoming": render_template("views/dashboard/_upcoming.html", scheduled_jobs=scheduled_jobs),
         "daily_totals": render_template(
@@ -308,6 +401,13 @@ def get_dashboard_partials(service_id):
             statistics=dashboard_totals_daily[0],
             column_width=column_width,
         ),
+        "annual_totals": render_template(
+            "views/dashboard/_totals_annual.html",
+            service_id=service_id,
+            statistics=dashboard_totals_daily[0],
+            statistics_annual=annual_data,
+            column_width=column_width,
+        ),
         "weekly_totals": render_template(
             "views/dashboard/_totals.html",
             service_id=service_id,
@@ -329,6 +429,7 @@ def get_dashboard_partials(service_id):
 
 
 def _get_daily_stats(service_id):
+    # TODO: get from redis, else fallback to template_statistics_client.get_template_statistics_for_service
     all_statistics_daily = template_statistics_client.get_template_statistics_for_service(service_id, limit_days=1)
     stats_daily = aggregate_notifications_stats(all_statistics_daily)
     dashboard_totals_daily = (get_dashboard_totals(stats_daily),)

app/notify_client/service_api_client.py

@@ -1,5 +1,5 @@
 import json
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 
 from flask import current_app
 from flask_login import current_user
@@ -9,6 +9,12 @@
 from app.notify_client import NotifyAdminAPIClient, _attach_current_user, cache
 
 
+def _seconds_until_midnight():
+    now = datetime.now(timezone.utc)
+    midnight = datetime.combine(now + timedelta(days=1), datetime.min.time())
+    return int((midnight - now).total_seconds())
+
+
 class ServiceAPIClient(NotifyAdminAPIClient):
     @cache.delete("user-{user_id}")
     def create_service(
@@ -377,8 +383,15 @@ def is_service_email_from_unique(self, service_id, email_from):
     def get_service_history(self, service_id):
         return self.get("/service/{0}/history".format(service_id))
 
+    # TODO: cache this once the backend is updated to exlude data from the current day
+    # @flask_cache.memoize(timeout=_seconds_until_midnight())
     def get_monthly_notification_stats(self, service_id, year):
-        return self.get(url="/service/{}/notifications/monthly?year={}".format(service_id, year))
+        return self.get(
+            url="/service/{}/notifications/monthly?year={}".format(
+                service_id,
+                year,
+            )
+        )
 
     def get_safelist(self, service_id):
         return self.get(url="/service/{}/safelist".format(service_id))
@@ -622,5 +635,15 @@ def _use_case_data_name(self, service_id):
     def _tos_key_name(self, service_id):
         return f"tos-accepted-{service_id}"
 
+    def aggregate_by_type(self, notification_data):
+        counts = {"sms": 0, "email": 0, "letter": 0}
+        for month_data in notification_data["data"].values():
+            for message_type, message_counts in month_data.items():
+                if isinstance(message_counts, dict):
+                    counts[message_type] += sum(message_counts.values())
+
+        # return the result
+        return counts
+
 
 service_api_client = ServiceAPIClient()

app/templates/components/message-count-label.html

@@ -104,9 +104,9 @@
 
   {%- if session["userlang"] == "fr" -%}
     {%- if count <= 1 -%}
-      {{ _('addresse courriel problématique') }}
+      addresse courriel problématique
     {%- else -%}
-      {{ _('addresses courriel problématiques') }}
+      addresses courriel problématiques
     {%- endif %}
     {{" "}}
   {%- endif %}

app/templates/components/terms.html

@@ -53,7 +53,7 @@ <h2 class="heading-medium">{{ _(headings[1].title) }}</h2>
         <p><strong>{{ _('Daily limit per service:') }}</strong></p>
     </div>
     <div class="wp-block-column is-layout-flow wp-block-column-is-layout-flow">
-        <p>{{ _('10,000 emails<br />1000 text messages') }}</p>
+        <p>{{ _('10,000 emails<br />1,000 text messages') }}</p>
     </div>
     </div>
 
@@ -66,7 +66,7 @@ <h2 class="heading-medium">{{ _(headings[1].title) }}</h2>
         </p>
     </div>
     <div class="wp-block-column is-layout-flow wp-block-column-is-layout-flow">
-        <p>{{ _('10 million emails<br />25,000 text messages') }}</p>
+        <p>{{ _('20 million emails<br />100,000 text messages') }}</p>
     </div>
     </div>
 

app/templates/views/api/callbacks/delivery-status-callback.html

@@ -41,13 +41,13 @@ <h2 class="heading-small">
           autocomplete='new-password'
         ) }}
         {% set test_response_txt = _('Test response time')  if has_callback_config else None %}
-        {% set test_response_value = _('test_response_time') if has_callback_config else None %}
+        {% set test_response_value = 'test_response_time' if has_callback_config else None %}
         {% set display_footer = is_deleting if is_deleting else False %}
         {% set delete_link = url_for('.delete_delivery_status_callback', service_id=current_service.id) if has_callback_config else None%}
         {% if not display_footer %}
           {{ sticky_page_footer_two_submit_buttons_and_delete_link(
             button1_text=_('Save'),
-            button1_value=_('save'),
+            button1_value='save',
             button2_text=test_response_txt,
             button2_value=test_response_value,
             delete_link=delete_link,

app/templates/views/api/callbacks/received-text-messages-callback.html

@@ -36,14 +36,14 @@ <h2 class="heading-small">
           autocomplete='new-password'
         ) }}
         {% set test_response_txt = _('Test response time')  if has_callback_config else None %}
-        {% set test_response_value = _('test_response_time') if has_callback_config else None %}
+        {% set test_response_value = 'test_response_time' if has_callback_config else None %}
         {% set display_footer = is_deleting if is_deleting else False %}
         {% set delete_link = url_for('.delete_received_text_messages_callback', service_id=current_service.id) if has_callback_config else None%}
         {% set delete_link_text = _('Delete') if has_callback_config else None %}
         {% if not display_footer %}
           {{ sticky_page_footer_two_submit_buttons_and_delete_link(
             button1_text=_('Save'),
-            button1_value=_('save'),
+            button1_value='save',
             button2_text=test_response_txt,
             button2_value=test_response_value,
             delete_link=delete_link,

app/templates/views/check/column-errors.html

@@ -161,7 +161,8 @@ <h2 class='banner-title' data-module="track-error" data-error-type="Missing plac
         {% endcall %}
         <h2 class="heading-medium">{{ _('You cannot send all these text messages today') }}</h2>
           <p>
-            {{ _("You can try sending these messages after {} Eastern Time. Check {}.").format(time_to_reset[current_lang], content_link(_("your current local time"), 'https://nrc.canada.ca/en/web-clock/', is_external_link=true))}}
+            {{ _("You can try sending these messages after {} Eastern Time. Check {}.").format(time_to_reset[current_lang],
+            content_link(_("your current local time"), _('https://nrc.canada.ca/en/web-clock/'), is_external_link=true))}}
           </p>
       {% elif recipients.more_rows_than_can_send and false %}
         {% call banner_wrapper(type='dangerous') %}
@@ -173,7 +174,8 @@ <h2 class="heading-medium">{{ _('You cannot send all these text messages today')
           {% endcall %}
           <h2 class="heading-medium">{{ _('You cannot send all these email messages today') }}</h2>
           <p>
-            {{ _("You can try sending these messages after {} Eastern Time. Check {}.").format(time_to_reset[current_lang], content_link(_("your current local time"), 'https://nrc.canada.ca/en/web-clock/', is_external_link=true))}}
+            {{ _("You can try sending these messages after {} Eastern Time. Check {}.").format(time_to_reset[current_lang],
+            content_link(_("your current local time"), _('https://nrc.canada.ca/en/web-clock/'), is_external_link=true))}}
           </p>
 
 

app/templates/views/dashboard/_totals_annual.html

@@ -0,0 +1,25 @@
+{% from "components/big-number.html" import big_number %}
+{% from "components/message-count-label.html" import message_count_label %}
+{% from 'components/remaining-messages.html' import remaining_messages %}
+{% from "components/show-more.html" import show_more %}
+
+<div class="ajax-block-container" data-testid="annual-usage">
+    <h2 class="heading-medium mt-8">
+      {{ _('Annual usage') }}
+      <br />
+      <small class="text-gray-600 text-small font-normal" style="color: #5E6975">
+        {% set current_year = current_year or (now().year if now().month < 4 else now().year + 1) %}
+        {{ _('resets on April 1, ') ~ current_year }}
+      </small> 
+    </h2>
+    <div class="grid-row contain-floats mb-10">
+      <div class="{{column_width}}">
+        {{ remaining_messages(header=_('emails'), total=current_service.email_annual_limit, used=statistics_annual['email'], muted=true) }}
+      </div>
+      <div class="{{column_width}}">
+        {{ remaining_messages(header=_('text messages'), total=current_service.sms_annual_limit, used=statistics_annual['sms'], muted=true) }}
+      </div>
+    </div>
+    {{ show_more(url_for('.monthly', service_id=current_service.id), _('Visit usage report')) }}
+</div>
+