fix: updates the inputs to match the latest lambda payload schema (#395)

* fix: updates the inputs to match the latest lambda payload schema

* chore: refactor and add typing for the step-function payload creation function

* fix: typing and refactor fnctions

* chore: ensure default cors config file exists for developer experience improvement

* fix: ensure client is not responsible for trace-header setting. update to provide a unique execution name

* chore: add some notes on testing step functions locally and sample files to make it easier for devs

* docs: add info on accessing local postgresql databases via psql

* chore: add additional help for how the local step function can be used

* chore: remove unnecessary guard clause. This env-var must be set

* fix: update tests to reflect the new refactored functions

* fix: ensure aws module exports similar to other modules. Update the tests to reflect the new export-type.

* fix: updates tests reflect new payload structure for create_archive lambda

* fix: all instances where aws module is being imported incorrectly

* fix: payload value type

* fix: duplicate log

* fix: payload to match schema

* chore: add a separate directory explaining how to simulate treasury file generation locally. Note: this will be removed once the terraform can be run directly on localstack, but for now this is a good starting point to verify the work.

* chore: add documentation on how to use the newly added scripts for testing step functions locally

* fix: ensure that the step function input always contains the minimum required fields

* chore: add guard-clause when no information is supplied to lambda

* fix: update the step function definition to use execution-state rather than local state among states

* fix: remove unnecessary logs and add description

* fix: updates tests to reflect the new step-function input

.env.defaults

@@ -44,9 +44,5 @@ DD_RUM_TRACK_LONG_TASKS=true
 # Auth provider environment variables
 AUTH_PROVIDER=local
 
-# Treasury Report Generation
-# Note: this requires creating a step function in localstack with the name `GenerateTreasuryReport`
-# Example command to create the step function:
-# Create a definition file that contains the step function definition as defined in `treasury_generation_step_function.tf`
-# awslocal stepfunctions create-state-machine --name GenerateTreasuryReport --definition file://./step-functions/GenerateTreasuryReport.json --role-arn "arn:aws:iam::000000000000:role/stepfunctions-role"
+# See /scripts/local_treasury_file_generation/README.md for more information
 TREASURY_STEP_FUNCTION_ARN="arn:aws:states:us-west-2:000000000000:stateMachine:GenerateTreasuryReport"

README.md

@@ -82,6 +82,10 @@ To use Docker to run the pytest suite, run the following commands (after startin
 - `docker compose -f ./docker-compose.dev.yml run --rm -it python-console /bin/bash` - to go into the python container
   - `poetry run pytest` - to run the pytest suite
 
+To access the local `redwood` and `redwood_test` databases via [psql](https://www.postgresql.org/docs/current/app-psql.html):
+- Database used for local development `PGPASSWORD=redwood psql -h localhost -p 5432 -U redwood -d redwood`
+- Database used by test suites locally: `PGPASSWORD=redwood psql -h localhost -p 5432 -U redwood -d redwood_test`
+
 ## Development
 
 We recommend checking out the [Getting Started with RedwoodJS](./docs/redwood-introduction.md) guide to get familiar with our development patterns.

api/src/functions/processValidationJson/processValidationJson.ts

@@ -2,7 +2,7 @@ import { GetObjectCommand, DeleteObjectCommand } from '@aws-sdk/client-s3'
 import { Prisma, Version } from '@prisma/client'
 import { S3Handler, S3ObjectCreatedNotificationEvent } from 'aws-lambda'
 
-import aws from 'src/lib/aws'
+import { getS3Client, sendPutObjectToS3Bucket } from 'src/lib/aws'
 import { db, getPrismaClient } from 'src/lib/db'
 import { logger } from 'src/lib/logger'
 
@@ -56,7 +56,7 @@ type UploadValidationS3Client = {
 export const handler: S3Handler = async (
   event: S3ObjectCreatedNotificationEvent
 ): Promise<Response> => {
-  const s3 = aws.getS3Client()
+  const s3 = getS3Client()
   if (process.env.LOCALSTACK_HOSTNAME) {
     /*
     This allows us to easily test this function during local development by making a GET request as follows:
@@ -232,7 +232,7 @@ export const processRecord = async (
         const subrecipients = {
           subrecipients: subrecipientsWithUploads,
         }
-        await aws.sendPutObjectToS3Bucket(
+        await sendPutObjectToS3Bucket(
           bucket,
           subrecipientKey,
           JSON.stringify(subrecipients)

api/src/lib/auth.ts

@@ -11,7 +11,7 @@ import { logger } from 'src/lib/logger'
  * Represents the user attributes returned by the decoding the
  * Authentication provider's JWT together with an optional list of roles.
  */
-interface CurrentUser extends User {
+export interface CurrentUser extends User {
   roles?: string[]
 }
 /**

api/src/lib/aws.ts

@@ -66,7 +66,7 @@ function getS3Client() {
   return s3
 }
 
-async function sendPutObjectToS3Bucket(
+export async function sendPutObjectToS3Bucket(
   bucketName: string,
   key: string,
   body: StreamingBlobPayloadInputTypes
@@ -151,7 +151,7 @@ export async function deleteUploadFile(upload: Upload) {
   await s3DeleteObject(fileKey)
 }
 
-async function s3DeleteObject(key: string) {
+export async function s3DeleteObject(key: string) {
   const s3 = getS3Client()
   const baseParams: DeleteObjectCommandInput = {
     Bucket: REPORTING_DATA_BUCKET_NAME,
@@ -164,7 +164,7 @@ async function s3DeleteObject(key: string) {
  *  This function is a wrapper around the getSignedUrl function from the @aws-sdk/s3-request-presigner package.
  *  Exists to organize the imports and to make it easier to mock in tests.
  */
-async function getSignedUrl(upload: Upload): Promise<string> {
+export async function getSignedUrl(upload: Upload): Promise<string> {
   const key = getS3UploadFileKey(upload.agency.organizationId, upload)
   const s3 = getS3Client()
   const baseParams = { Bucket: REPORTING_DATA_BUCKET_NAME, Key: key }
@@ -178,7 +178,7 @@ const OUTPUT_TEMPLATE = {
   '1C': 'CPF1CMultiPurposeCommunityTemplate',
   Subrecipient: 'CPFSubrecipientTemplate',
 }
-async function getTreasurySignedUrl(
+export async function getTreasurySignedUrl(
   fileType: string,
   organization_id: number,
   current_reporting_period_id: string
@@ -206,7 +206,7 @@ function getSQSClient() {
   return sqs
 }
 
-async function sendSqsMessage(queueUrl: string, messageBody: unknown) {
+export async function sendSqsMessage(queueUrl: string, messageBody: unknown) {
   const sqs = getSQSClient()
   await sqs.send(
     new SendMessageCommand({
@@ -216,7 +216,7 @@ async function sendSqsMessage(queueUrl: string, messageBody: unknown) {
   )
 }
 
-async function receiveSqsMessage(queueUrl: string) {
+export async function receiveSqsMessage(queueUrl: string) {
   const sqs = getSQSClient()
   // const receiveResp = await sqs.send(new ReceiveMessageCommand({
   //   QueueUrl: process.env.TASK_QUEUE_URL, WaitTimeSeconds: 20, MaxNumberOfMessages: 1,
@@ -247,19 +247,18 @@ async function receiveSqsMessage(queueUrl: string) {
  * @param input
  * @param traceHeader
  */
-async function startStepFunctionExecution(
+export async function startStepFunctionExecution(
   arn: string,
   name?: string,
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  input?: any,
-  traceHeader?: string
+  input?: any
 ): Promise<StartExecutionCommandOutput> {
   let client: SFNClient
   const command = new StartExecutionCommand({
     stateMachineArn: arn,
     name,
     input: input ?? '{}',
-    traceHeader,
+    traceHeader: process.env._X_AMZN_TRACE_ID,
   })
   if (process.env.LOCALSTACK_HOSTNAME) {
     console.log('------------ USING LOCALSTACK FOR SFN ------------')
@@ -272,15 +271,3 @@ async function startStepFunctionExecution(
   }
   return await client.send(command)
 }
-
-export default {
-  sendPutObjectToS3Bucket,
-  sendHeadObjectToS3Bucket,
-  getSignedUrl,
-  sendSqsMessage,
-  receiveSqsMessage,
-  getS3Client,
-  startStepFunctionExecution,
-  s3DeleteObject,
-  getTreasurySignedUrl,
-}

api/src/services/organizations/organizations.test.ts

@@ -21,6 +21,7 @@ import type { StandardScenario } from './organizations.scenarios'
 // https://redwoodjs.com/docs/testing#jest-expect-type-considerations
 
 jest.mock('src/lib/aws', () => ({
+  ...jest.requireActual('src/lib/aws'),
   getTreasurySignedUrl: jest.fn(),
   startStepFunctionExecution: jest.fn(),
 }))

api/src/services/organizations/organizations.ts

@@ -6,7 +6,7 @@ import type {
   Agency,
 } from 'types/graphql'
 
-import aws from 'src/lib/aws'
+import { getTreasurySignedUrl, startStepFunctionExecution } from 'src/lib/aws'
 import { db } from 'src/lib/db'
 import { logger } from 'src/lib/logger'
 
@@ -92,7 +92,7 @@ export const downloadTreasuryFile: MutationResolvers['downloadTreasuryFile'] =
 
     // Retrieve the signed URL for the treasury file
     logger.info(`Downloading treasury file for ${fileType}`)
-    const url = await aws.getTreasurySignedUrl(
+    const url = await getTreasurySignedUrl(
       fileType,
       organization.id,
       organization.preferences['current_reporting_period_id']
@@ -124,7 +124,7 @@ export const kickOffTreasuryReportGeneration: MutationResolvers['kickOffTreasury
     logger.info(
       `Kicking off treasury report generation for ${organization.name}`
     )
-    const response = await aws.startStepFunctionExecution(
+    const response = await startStepFunctionExecution(
       process.env.TREASURY_STEP_FUNCTION_ARN,
       `manual-treasury-report-generation-${Date.now()}`,
       payload,

api/src/services/uploads/uploads.scenarios.ts

@@ -17,19 +17,46 @@ export const standard = defineScenario<
   | Prisma.ReportingPeriodCreateArgs
   | Prisma.UploadCreateArgs
 >({
-  organization: {
+  reportingPeriod: {
     one: {
       data: {
-        name: 'USDR',
-        preferences: {},
+        name: 'String',
+        startDate: '2024-01-26T15:11:27.688Z',
+        endDate: '2024-01-26T15:11:27.688Z',
+        inputTemplate: {
+          create: {
+            name: 'String',
+            version: 'String',
+            effectiveDate: '2024-01-26T15:11:27.688Z',
+          },
+        },
+        outputTemplate: {
+          create: {
+            name: 'String',
+            version: 'String',
+            effectiveDate: '2024-01-26T15:11:27.688Z',
+          },
+        },
       },
     },
-    two: {
+  },
+  organization: {
+    one: (scenario) => ({
+      data: {
+        name: 'USDR',
+        preferences: {
+          current_reporting_period_id: scenario.reportingPeriod.one.id,
+        },
+      },
+    }),
+    two: (scenario) => ({
       data: {
         name: 'Example Organization',
-        preferences: {},
+        preferences: {
+          current_reporting_period_id: scenario.reportingPeriod.one.id,
+        },
       },
-    },
+    }),
   },
   agency: {
     one: (scenario) => ({
@@ -99,29 +126,6 @@ export const standard = defineScenario<
       },
     }),
   },
-  reportingPeriod: {
-    one: () => ({
-      data: {
-        name: 'String',
-        startDate: '2024-01-26T15:11:27.688Z',
-        endDate: '2024-01-26T15:11:27.688Z',
-        inputTemplate: {
-          create: {
-            name: 'String',
-            version: 'String',
-            effectiveDate: '2024-01-26T15:11:27.688Z',
-          },
-        },
-        outputTemplate: {
-          create: {
-            name: 'String',
-            version: 'String',
-            effectiveDate: '2024-01-26T15:11:27.688Z',
-          },
-        },
-      },
-    }),
-  },
   expenditureCategory: {
     one: {
       data: {