From 97f656e3fa67d558a9455fae93ea0e1709ba6e7b Mon Sep 17 00:00:00 2001 From: Michael Volo Date: Thu, 24 Oct 2024 15:10:52 -0500 Subject: [PATCH 1/8] use query_all to get all records --- salesforce/management/commands/update_opportunities.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salesforce/management/commands/update_opportunities.py b/salesforce/management/commands/update_opportunities.py index d888759e..4b67a6e2 100644 --- a/salesforce/management/commands/update_opportunities.py +++ b/salesforce/management/commands/update_opportunities.py @@ -39,7 +39,9 @@ def handle(self, *args, **options): "AND Confirmation_Type__c = 'OpenStax Confirmed Adoption' " "AND Opportunity__r.Contact__r.Adoption_Status__c != 'Current Adopter'").format(base_year) - response = sf.query(query) + # This generally returns more than 2,000 records (the SF limit) + # See simplate_salesforce documentation for query_all: https://github.com/simple-salesforce/simple-salesforce?tab=readme-ov-file#queries + response = sf.query_all(query) records = response['records'] for record in records: From af314f339d33d0b93099fe85ce1867ad73bb04cf Mon Sep 17 00:00:00 2001 From: Michael Volo Date: Thu, 24 Oct 2024 18:34:35 -0500 Subject: [PATCH 2/8] cast uuid field --- salesforce/management/commands/update_opportunities.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salesforce/management/commands/update_opportunities.py b/salesforce/management/commands/update_opportunities.py index 4b67a6e2..b415988a 100644 --- a/salesforce/management/commands/update_opportunities.py +++ b/salesforce/management/commands/update_opportunities.py @@ -1,4 +1,5 @@ import datetime +import uuid from django.core.management.base import BaseCommand from salesforce.models import AdoptionOpportunityRecord from salesforce.salesforce import Salesforce @@ -47,7 +48,7 @@ def handle(self, *args, **options): for record in records: opportunity, created = AdoptionOpportunityRecord.objects.update_or_create( opportunity_id=record['Id'], - defaults={'account_uuid': record['Opportunity__r']['Contact__r']['Accounts_UUID__c'], + defaults={'account_uuid': uuid.UUID(record['Opportunity__r']['Contact__r']['Accounts_UUID__c']), 'opportunity_stage': record['Opportunity__r']['StageName'], 'adoption_type': record['Adoption_Type__c'], 'base_year': record['Base_Year__c'], From 236d37efa353873b5075261022170b4929d11283 Mon Sep 17 00:00:00 2001 From: Michael Volo Date: Thu, 24 Oct 2024 18:39:50 -0500 Subject: [PATCH 3/8] throw some to-dos in for future self --- salesforce/management/commands/update_opportunities.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salesforce/management/commands/update_opportunities.py b/salesforce/management/commands/update_opportunities.py index b415988a..60c6971e 100644 --- a/salesforce/management/commands/update_opportunities.py +++ b/salesforce/management/commands/update_opportunities.py @@ -45,6 +45,7 @@ def handle(self, *args, **options): response = sf.query_all(query) records = response['records'] + # TODO: this doesn't need to be updating on the opp id, the info never changes for record in records: opportunity, created = AdoptionOpportunityRecord.objects.update_or_create( opportunity_id=record['Id'], @@ -61,3 +62,6 @@ def handle(self, *args, **options): } ) opportunity.save() + + # TODO: need to grab current adoptions and if the info has changed, update it so the user sees most recent adoption info + # re-submitting the form will update the current year adoption numbers, which the user might not expect From fe3aa8a2d0b3b29a6359aff36c18d3fa292a2ca3 Mon Sep 17 00:00:00 2001 From: Michael Volo Date: Mon, 28 Oct 2024 11:00:39 -0500 Subject: [PATCH 4/8] use bulk query method --- .../management/commands/update_opportunities.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/salesforce/management/commands/update_opportunities.py b/salesforce/management/commands/update_opportunities.py index 60c6971e..3fa0a69c 100644 --- a/salesforce/management/commands/update_opportunities.py +++ b/salesforce/management/commands/update_opportunities.py @@ -41,15 +41,15 @@ def handle(self, *args, **options): "AND Opportunity__r.Contact__r.Adoption_Status__c != 'Current Adopter'").format(base_year) # This generally returns more than 2,000 records (the SF limit) - # See simplate_salesforce documentation for query_all: https://github.com/simple-salesforce/simple-salesforce?tab=readme-ov-file#queries - response = sf.query_all(query) - records = response['records'] + # See simplate_salesforce documentation for query_all: https://github.com/simple-salesforce/simple-salesforce?tab=readme-ov-file#using-bulk + results = sf.bulk.Adoption__c.query(query) # TODO: this doesn't need to be updating on the opp id, the info never changes - for record in records: + for i, record in enumerate(results): opportunity, created = AdoptionOpportunityRecord.objects.update_or_create( - opportunity_id=record['Id'], - defaults={'account_uuid': uuid.UUID(record['Opportunity__r']['Contact__r']['Accounts_UUID__c']), + account_uuid=uuid.UUID(record['Opportunity__r']['Contact__r']['Accounts_UUID__c']), + book_name=record['Opportunity__r']['Book__r']['Name'], + defaults={'opportunity_id': record['Id'], 'opportunity_stage': record['Opportunity__r']['StageName'], 'adoption_type': record['Adoption_Type__c'], 'base_year': record['Base_Year__c'], @@ -57,8 +57,7 @@ def handle(self, *args, **options): 'confirmation_type': record['Confirmation_Type__c'], 'how_using': record['How_Using__c'], 'savings': record['Savings__c'], - 'students': record['Students__c'], - 'book_name': record['Opportunity__r']['Book__r']['Name'], + 'students': record['Students__c'] } ) opportunity.save() From f0ff7cb9bb9378b77909f7fc4030555e5bcbfc9f Mon Sep 17 00:00:00 2001 From: Michael Volo Date: Mon, 28 Oct 2024 11:53:18 -0500 Subject: [PATCH 5/8] handle badly formatted uuids from SF (and log them) --- .../commands/update_opportunities.py | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/salesforce/management/commands/update_opportunities.py b/salesforce/management/commands/update_opportunities.py index 3fa0a69c..0f3b39dc 100644 --- a/salesforce/management/commands/update_opportunities.py +++ b/salesforce/management/commands/update_opportunities.py @@ -3,7 +3,7 @@ from django.core.management.base import BaseCommand from salesforce.models import AdoptionOpportunityRecord from salesforce.salesforce import Salesforce - +import sentry_sdk class Command(BaseCommand): help = "update book adoptions from salesforce.com for getting adoptions by account uuid" @@ -41,26 +41,29 @@ def handle(self, *args, **options): "AND Opportunity__r.Contact__r.Adoption_Status__c != 'Current Adopter'").format(base_year) # This generally returns more than 2,000 records (the SF limit) - # See simplate_salesforce documentation for query_all: https://github.com/simple-salesforce/simple-salesforce?tab=readme-ov-file#using-bulk + # See simplate_salesforce documentation for bulk queries: https://github.com/simple-salesforce/simple-salesforce?tab=readme-ov-file#using-bulk results = sf.bulk.Adoption__c.query(query) # TODO: this doesn't need to be updating on the opp id, the info never changes for i, record in enumerate(results): - opportunity, created = AdoptionOpportunityRecord.objects.update_or_create( - account_uuid=uuid.UUID(record['Opportunity__r']['Contact__r']['Accounts_UUID__c']), - book_name=record['Opportunity__r']['Book__r']['Name'], - defaults={'opportunity_id': record['Id'], - 'opportunity_stage': record['Opportunity__r']['StageName'], - 'adoption_type': record['Adoption_Type__c'], - 'base_year': record['Base_Year__c'], - 'confirmation_date': record['Confirmation_Date__c'], - 'confirmation_type': record['Confirmation_Type__c'], - 'how_using': record['How_Using__c'], - 'savings': record['Savings__c'], - 'students': record['Students__c'] - } - ) - opportunity.save() + try: + opportunity, created = AdoptionOpportunityRecord.objects.update_or_create( + account_uuid=uuid.UUID(record['Opportunity__r']['Contact__r']['Accounts_UUID__c']), + book_name=record['Opportunity__r']['Book__r']['Name'], + defaults={'opportunity_id': record['Id'], + 'opportunity_stage': record['Opportunity__r']['StageName'], + 'adoption_type': record['Adoption_Type__c'], + 'base_year': record['Base_Year__c'], + 'confirmation_date': record['Confirmation_Date__c'], + 'confirmation_type': record['Confirmation_Type__c'], + 'how_using': record['How_Using__c'], + 'savings': record['Savings__c'], + 'students': record['Students__c'] + } + ) + opportunity.save() + except ValueError: + sentry_sdk.capture_message("Adoption {} has a badly formatted Account UUID: {}".format(record['Id'], record['Opportunity__r']['Contact__r']['Accounts_UUID__c'])) # TODO: need to grab current adoptions and if the info has changed, update it so the user sees most recent adoption info # re-submitting the form will update the current year adoption numbers, which the user might not expect From 8084e2754f3fd9d4089b425776fabf1a25e15e84 Mon Sep 17 00:00:00 2001 From: Michael Volo Date: Mon, 28 Oct 2024 12:33:23 -0500 Subject: [PATCH 6/8] handle blank book (and log) --- salesforce/management/commands/update_opportunities.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salesforce/management/commands/update_opportunities.py b/salesforce/management/commands/update_opportunities.py index 0f3b39dc..042b108c 100644 --- a/salesforce/management/commands/update_opportunities.py +++ b/salesforce/management/commands/update_opportunities.py @@ -64,6 +64,8 @@ def handle(self, *args, **options): opportunity.save() except ValueError: sentry_sdk.capture_message("Adoption {} has a badly formatted Account UUID: {}".format(record['Id'], record['Opportunity__r']['Contact__r']['Accounts_UUID__c'])) + except TypeError: + sentry_sdk.capture_message("Adoption {} exists without a book".format(record['Id'])) # TODO: need to grab current adoptions and if the info has changed, update it so the user sees most recent adoption info # re-submitting the form will update the current year adoption numbers, which the user might not expect From be4b51bf7c47d9c72380d0a8a1be32e7e5ed8084 Mon Sep 17 00:00:00 2001 From: Michael Volo Date: Mon, 28 Oct 2024 12:55:37 -0500 Subject: [PATCH 7/8] code cleanup, get current year to replace with new data --- .../commands/update_opportunities.py | 104 +++++++++--------- 1 file changed, 55 insertions(+), 49 deletions(-) diff --git a/salesforce/management/commands/update_opportunities.py b/salesforce/management/commands/update_opportunities.py index 042b108c..9a785884 100644 --- a/salesforce/management/commands/update_opportunities.py +++ b/salesforce/management/commands/update_opportunities.py @@ -5,67 +5,73 @@ from salesforce.salesforce import Salesforce import sentry_sdk + class Command(BaseCommand): help = "update book adoptions from salesforce.com for getting adoptions by account uuid" + def query_base_year(self, base_year, adoption_status): + query = ("SELECT Id, " + "Adoption_Type__c, " + "Base_Year__c, " + "Confirmation_Date__c, " + "Confirmation_Type__c, " + "How_Using__c, " + "Savings__c, " + "Students__c, " + "Opportunity__r.Book__r.Name, " + "Opportunity__r.StageName, " + "Opportunity__r.Contact__r.Accounts_UUID__c " + "FROM Adoption__c WHERE " + "Base_Year__c = {} AND Opportunity__r.Contact__r.Accounts_UUID__c != null " + "AND Confirmation_Type__c = 'OpenStax Confirmed Adoption' " + "AND Opportunity__r.Contact__r.Adoption_Status__c = '{}'").format(base_year, adoption_status) + + return query + + def process_results(self, results): + for i, record in enumerate(results): + try: + opportunity, created = AdoptionOpportunityRecord.objects.update_or_create( + account_uuid=uuid.UUID(record['Opportunity__r']['Contact__r']['Accounts_UUID__c']), + book_name=record['Opportunity__r']['Book__r']['Name'], + defaults={'opportunity_id': record['Id'], + 'opportunity_stage': record['Opportunity__r']['StageName'], + 'adoption_type': record['Adoption_Type__c'], + 'base_year': record['Base_Year__c'], + 'confirmation_date': record['Confirmation_Date__c'], + 'confirmation_type': record['Confirmation_Type__c'], + 'how_using': record['How_Using__c'], + 'savings': record['Savings__c'], + 'students': record['Students__c'] + } + ) + opportunity.save() + except ValueError: + sentry_sdk.capture_message("Adoption {} has a badly formatted Account UUID: {}".format(record['Id'], record['Opportunity__r']['Contact__r']['Accounts_UUID__c'])) + except TypeError: + sentry_sdk.capture_message("Adoption {} exists without a book".format(record['Id'])) + def handle(self, *args, **options): with Salesforce() as sf: now = datetime.datetime.now() base_year = now.year - if now.month < 7: # if it's before July, the base year is the previous year (4/1/2024 = base_year 2023) + if now.month < 7: # before july, current year minus 2 (4/1/2024, base year = 2023) base_year -= 2 - else: + else: # otherwise, current year minus 1 (10/1/2024, base year = 2023) base_year -= 1 - # TODO: I don't think this is needed - updating the records should be fine, and keeps something on the form - # TODO: for the user. Eventually, this should update CMS DB with updated data if they fill out the form - # truncate the table - # AdoptionOpportunityRecord.objects.all().delete() + # first, we get last year's records + # these people need to renew, show them what we know from last base year confirmed adoption + query = self.query_base_year(base_year, "Past Adopter") + results = sf.bulk.Adoption__c.query(query) - # then we will get any new records - query = ("SELECT Id, " - "Adoption_Type__c, " - "Base_Year__c, " - "Confirmation_Date__c, " - "Confirmation_Type__c, " - "How_Using__c, " - "Savings__c, " - "Students__c, " - "Opportunity__r.Book__r.Name, " - "Opportunity__r.StageName, " - "Opportunity__r.Contact__r.Accounts_UUID__c " - "FROM Adoption__c WHERE " - "Base_Year__c = {} AND Opportunity__r.Contact__r.Accounts_UUID__c != null " - "AND Confirmation_Type__c = 'OpenStax Confirmed Adoption' " - "AND Opportunity__r.Contact__r.Adoption_Status__c != 'Current Adopter'").format(base_year) + self.process_results(results) - # This generally returns more than 2,000 records (the SF limit) - # See simplate_salesforce documentation for bulk queries: https://github.com/simple-salesforce/simple-salesforce?tab=readme-ov-file#using-bulk - results = sf.bulk.Adoption__c.query(query) + # now get this year's records - this will ensure accurate data on the renewal form if already filled out + current_base_year = base_year + 1 + updated_records = self.query_base_year(current_base_year, "Current Adopter") - # TODO: this doesn't need to be updating on the opp id, the info never changes - for i, record in enumerate(results): - try: - opportunity, created = AdoptionOpportunityRecord.objects.update_or_create( - account_uuid=uuid.UUID(record['Opportunity__r']['Contact__r']['Accounts_UUID__c']), - book_name=record['Opportunity__r']['Book__r']['Name'], - defaults={'opportunity_id': record['Id'], - 'opportunity_stage': record['Opportunity__r']['StageName'], - 'adoption_type': record['Adoption_Type__c'], - 'base_year': record['Base_Year__c'], - 'confirmation_date': record['Confirmation_Date__c'], - 'confirmation_type': record['Confirmation_Type__c'], - 'how_using': record['How_Using__c'], - 'savings': record['Savings__c'], - 'students': record['Students__c'] - } - ) - opportunity.save() - except ValueError: - sentry_sdk.capture_message("Adoption {} has a badly formatted Account UUID: {}".format(record['Id'], record['Opportunity__r']['Contact__r']['Accounts_UUID__c'])) - except TypeError: - sentry_sdk.capture_message("Adoption {} exists without a book".format(record['Id'])) + results = sf.bulk.Adoption__c.query(updated_records) + self.process_results(results) - # TODO: need to grab current adoptions and if the info has changed, update it so the user sees most recent adoption info - # re-submitting the form will update the current year adoption numbers, which the user might not expect From 6da964bddf85348ef4443a6fed9697aad4232d62 Mon Sep 17 00:00:00 2001 From: Michael Volo Date: Mon, 28 Oct 2024 13:39:21 -0500 Subject: [PATCH 8/8] add sentry cron monitor - run daily --- openstax/settings/base.py | 2 +- salesforce/management/commands/update_opportunities.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/openstax/settings/base.py b/openstax/settings/base.py index cb633b69..09d03831 100644 --- a/openstax/settings/base.py +++ b/openstax/settings/base.py @@ -259,7 +259,7 @@ ('0 2 * * *', 'django.core.management.call_command', ['delete_resource_downloads']), ('0 6 * * *', 'django.core.management.call_command', ['update_resource_downloads']), ('0 0 8 * *', 'django.core.management.call_command', ['update_schools_and_mapbox']), - # ('0 0 1 * *', 'django.core.management.call_command', ['update_opportunities']), + ('0 0 * * *', 'django.core.management.call_command', ['update_opportunities']), ('0 10 * * *', 'django.core.management.call_command', ['update_partners']), ] diff --git a/salesforce/management/commands/update_opportunities.py b/salesforce/management/commands/update_opportunities.py index 9a785884..be6d0607 100644 --- a/salesforce/management/commands/update_opportunities.py +++ b/salesforce/management/commands/update_opportunities.py @@ -4,6 +4,7 @@ from salesforce.models import AdoptionOpportunityRecord from salesforce.salesforce import Salesforce import sentry_sdk +from sentry_sdk.crons import monitor class Command(BaseCommand): @@ -51,6 +52,7 @@ def process_results(self, results): except TypeError: sentry_sdk.capture_message("Adoption {} exists without a book".format(record['Id'])) + @monitor(monitor_slug='update-opportunities') def handle(self, *args, **options): with Salesforce() as sf: now = datetime.datetime.now() @@ -64,8 +66,8 @@ def handle(self, *args, **options): # first, we get last year's records # these people need to renew, show them what we know from last base year confirmed adoption query = self.query_base_year(base_year, "Past Adopter") - results = sf.bulk.Adoption__c.query(query) + results = sf.bulk.Adoption__c.query(query) self.process_results(results) # now get this year's records - this will ensure accurate data on the renewal form if already filled out