chore: adding synthetic monitoring tests (#3407)

.github/workflows/scheduled-jobs.yml

@@ -0,0 +1,112 @@
+name: Synthetic monitoring with Tracetest
+
+on:
+  # allows the manual trigger
+  workflow_dispatch:
+
+  schedule:
+  - cron: '0 */1 * * *' # every hour
+
+jobs:
+  pokeshop-trace-based-tests:
+    name: Run trace based tests for Pokeshop
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install Tracetest CLI
+        run: |
+          curl -L https://raw.githubusercontent.com/kubeshop/tracetest/main/install-cli.sh | bash
+      - name: Configure Tracetest CLI
+        run: |
+          tracetest configure --token ${{secrets.TRACETEST_POKESHOP_TOKEN}}
+      - name: Run synthetic monitoring tests
+        run: |
+          tracetest run testsuite --file ./testing/synthetic-monitoring/pokeshop/_testsuite.yaml
+      - name: Send message on Slack in case of failure
+        if: ${{ failure() }}
+        uses: slackapi/[email protected]
+        with:
+          # check the block kit builder docs to understand how it works
+          # and how to modify it: https://api.slack.com/block-kit
+          payload: |
+            {
+              "blocks": [
+                {
+                  "type": "header",
+                  "text": {
+                    "type": "plain_text",
+                    "text": ":warning: Synthetic Monitoring Alert - Pokeshop Demo :warning:",
+                    "emoji": true
+                  }
+                },
+                {
+                  "type": "section",
+                  "fields": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Status:*\nFailed"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Pipeline:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View workflow>"
+                    }
+                  ]
+                }
+              ]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SYNTETIC_MONITORING_SLACK_WEBHOOK_URL }}
+          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
+  otel-demo-trace-based-tests:
+    name: Run trace based tests for Open Telemetry demo
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install Tracetest CLI
+        run: |
+          curl -L https://raw.githubusercontent.com/kubeshop/tracetest/main/install-cli.sh | bash
+      - name: Configure Tracetest CLI
+        run: |
+          tracetest configure --token ${{secrets.TRACETEST_OTELDEMO_TOKEN}}
+      - name: Run synthetic monitoring tests
+        run: |
+          tracetest run testsuite --file ./testing/synthetic-monitoring/otel-demo/_testsuite.yaml --vars ./testing/synthetic-monitoring/otel-demo/_variableset.yaml
+      - name: Send message on Slack in case of failure
+        if: ${{ failure() }}
+        uses: slackapi/[email protected]
+        with:
+          # check the block kit builder docs to understand how it works
+          # and how to modify it: https://api.slack.com/block-kit
+          payload: |
+            {
+              "blocks": [
+                {
+                  "type": "header",
+                  "text": {
+                    "type": "plain_text",
+                    "text": ":warning: Synthetic Monitoring Alert - OTel Demo :warning:",
+                    "emoji": true
+                  }
+                },
+                {
+                  "type": "section",
+                  "fields": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Status:*\nFailed"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Pipeline:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View workflow>"
+                    }
+                  ]
+                }
+              ]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SYNTETIC_MONITORING_SLACK_WEBHOOK_URL }}
+          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

testing/synthetic-monitoring/otel-demo/01-see-ads.yaml

@@ -0,0 +1,34 @@
+# test suite based on https://github.com/open-telemetry/opentelemetry-demo/tree/main/test/tracetesting/frontend-service
+
+type: Test
+spec:
+  id: frontend-see-adds
+  name: 'Frontend: See Ads'
+  description: Simulate the user seeing a ads on Astronomy Shop
+  trigger:
+    type: http
+    httpRequest:
+      url: http://${var:FRONTEND_ADDR}/api/data
+      method: GET
+      headers:
+      - key: Content-Type
+        value: application/json
+      body: |
+        {
+          "contextKeys": [
+            "binoculars",
+            "telescopes",
+            "accessories"
+          ]
+        }
+  specs:
+  - name: It called the frontend with success and got a valid redirectUrl for each ads
+    selector: span[tracetest.span.type="general" name="Tracetest trigger"]
+    assertions:
+    - attr:tracetest.response.status = 200
+    - attr:tracetest.response.body | json_path '$[0].redirectUrl' contains "/product/"
+    - attr:tracetest.response.body | json_path '$[1].redirectUrl' contains "/product/"
+  - name: It returns two ads
+    selector: span[tracetest.span.type="rpc" name="oteldemo.AdService/GetAds" rpc.system="grpc" rpc.method="GetAds" rpc.service="oteldemo.AdService"]
+    assertions:
+    - attr:app.ads.count = 2

testing/synthetic-monitoring/otel-demo/02-get-product-recommendation.yaml

@@ -0,0 +1,35 @@
+# test suite based on https://github.com/open-telemetry/opentelemetry-demo/tree/main/test/tracetesting/frontend-service
+
+type: Test
+spec:
+  id: frontend-get-recommendation
+  name: 'Frontend: Get recommendations'
+  description: Simulate the user seeing recomendations on Astronomy Shop
+  trigger:
+    type: http
+    httpRequest:
+      url: http://${var:FRONTEND_ADDR}/api/recommendations
+      method: GET
+      headers:
+      - key: Content-Type
+        value: application/json
+      body: |
+        {
+          "productIds":[
+            "0PUK6V6EV0",
+            "1YMWWN1N4O",
+            "2ZYFJ3GM2N",
+            "66VCHSJNUP",
+            "6E92ZMYYFZ"
+          ]
+        }
+  specs:
+  - name: It called the frontend with success
+    selector: span[tracetest.span.type="general" name="Tracetest trigger"]
+    assertions:
+    - attr:tracetest.response.status = 200
+  - name: It called ListRecommendations correctly and got 5 products
+    selector: span[tracetest.span.type="rpc" name="/oteldemo.RecommendationService/ListRecommendations" rpc.system="grpc" rpc.method="ListRecommendations" rpc.service="oteldemo.RecommendationService"]
+    assertions:
+    - attr:rpc.grpc.status_code = 0
+    - attr:app.products_recommended.count = 5

testing/synthetic-monitoring/otel-demo/03-browse-product.yaml

@@ -0,0 +1,30 @@
+# test suite based on https://github.com/open-telemetry/opentelemetry-demo/tree/main/test/tracetesting/frontend-service
+
+type: Test
+spec:
+  id: frontend-browse-product
+  name: 'Frontend: Browse products'
+  description: Simulate the user browsing products on Astronomy Shop
+  trigger:
+    type: http
+    httpRequest:
+      url: http://${var:FRONTEND_ADDR}/api/products/0PUK6V6EV0
+      method: GET
+      headers:
+      - key: Content-Type
+        value: application/json
+  specs:
+  - name: It called the frontend with success and got a product with valid attributes
+    selector: span[tracetest.span.type="general" name="Tracetest trigger"]
+    assertions:
+    - attr:tracetest.response.status = 200
+    - attr:tracetest.response.body | json_path '$.id' = "0PUK6V6EV0"
+    - attr:tracetest.response.body | json_path '$.description' != ""
+    - attr:tracetest.response.body | json_path '$.picture' != ""
+    - attr:tracetest.response.body | json_path '$.priceUsd' != "{}"
+    - attr:tracetest.response.body | json_path '$.categories' != "[]"
+  - name: It queried the product catalog correctly for a specific product
+    selector: span[tracetest.span.type="rpc" name="oteldemo.ProductCatalogService/GetProduct" rpc.system="grpc" rpc.method="GetProduct" rpc.service="oteldemo.ProductCatalogService"]
+    assertions:
+    - attr:rpc.grpc.status_code  =  0
+    - attr:app.product.id  =  "0PUK6V6EV0"

testing/synthetic-monitoring/otel-demo/04-add-product-to-cart.yaml

@@ -0,0 +1,37 @@
+# test suite based on https://github.com/open-telemetry/opentelemetry-demo/tree/main/test/tracetesting/frontend-service
+
+type: Test
+spec:
+  id: frontend-add-product
+  name: 'Frontend: Add product to the cart'
+  description: Simulate a user adding a selected product to the shopping cart
+  trigger:
+    type: http
+    httpRequest:
+      url: http://${var:FRONTEND_ADDR}/api/cart
+      method: POST
+      headers:
+      - key: Content-Type
+        value: application/json
+      body: |
+        {
+          "item": {
+            "productId": "0PUK6V6EV0",
+            "quantity": 2
+          },
+          "userId": "2491f868-88f1-4345-8836-d5d8511a9f83"
+        }
+  specs:
+  - name: It called the frontend with success
+    selector: span[tracetest.span.type="general" name="Tracetest trigger"]
+    assertions:
+    - attr:tracetest.response.status = 200
+  - name: It added an item correctly into the shopping cart
+    selector: span[name="oteldemo.CartService/AddItem"]
+    assertions:
+    - attr:rpc.grpc.status_code = 0
+    - attr:app.product.id = "0PUK6V6EV0"
+  - name: It set the cart item correctly on the database
+    selector: span[tracetest.span.type="database" name="HMSET" db.system="redis" db.redis.database_index="0"]
+    assertions:
+    - attr:db.statement = "HMSET 2491f868-88f1-4345-8836-d5d8511a9f83"

testing/synthetic-monitoring/otel-demo/05-view-cart.yaml

@@ -0,0 +1,24 @@
+# test suite based on https://github.com/open-telemetry/opentelemetry-demo/tree/main/test/tracetesting/frontend-service
+
+type: Test
+spec:
+  id: frontend-view-cart
+  name: 'Frontend: View cart'
+  description: Simulate a user viewing the shopping cart
+  trigger:
+    type: http
+    httpRequest:
+      url: http://${var:FRONTEND_ADDR}/api/cart?userId=2491f868-88f1-4345-8836-d5d8511a9f83
+      method: GET
+      headers:
+      - key: Content-Type
+        value: application/json
+  specs:
+  - name: It called the frontend with success
+    selector: span[tracetest.span.type="general" name="Tracetest trigger"]
+    assertions:
+    - attr:tracetest.response.status = 200
+  - name: It retrieved the cart items correctly
+    selector: span[name="oteldemo.CartService/GetCart"]
+    assertions:
+    - attr:rpc.grpc.status_code = 0

testing/synthetic-monitoring/otel-demo/06-checking-out-cart.yaml

@@ -0,0 +1,74 @@
+# test suite based on https://github.com/open-telemetry/opentelemetry-demo/tree/main/test/tracetesting/frontend-service
+
+type: Test
+spec:
+  id: frontend-checkout-shopping-cart
+  name: 'Frontend: Checking out shopping cart'
+  description: Simulate user checking out shopping cart
+  trigger:
+    type: http
+    httpRequest:
+      url: http://${var:FRONTEND_ADDR}/api/checkout
+      method: POST
+      headers:
+      - key: Content-Type
+        value: application/json
+      body: |
+        {
+          "userId": "2491f868-88f1-4345-8836-d5d8511a9f83",
+          "email": "[email protected]",
+          "address": {
+            "streetAddress": "1600 Amphitheatre Parkway",
+            "state": "CA",
+            "country": "United States",
+            "city": "Mountain View",
+            "zipCode": "94043"
+          },
+          "userCurrency": "USD",
+          "creditCard": {
+            "creditCardCvv": 672,
+            "creditCardExpirationMonth": 1,
+            "creditCardExpirationYear": 2030,
+            "creditCardNumber": "4432-8015-6152-0454"
+          }
+        }
+  specs:
+  - name: It called the frontend with success
+    selector: span[tracetest.span.type="general" name="Tracetest trigger"]
+    assertions:
+    - attr:tracetest.response.status = 200
+  - name: "The order was placed"
+    selector: span[tracetest.span.type="rpc" name="oteldemo.CheckoutService/PlaceOrder" rpc.system="grpc" rpc.method="PlaceOrder" rpc.service="oteldemo.CheckoutService"]
+    assertions:
+    - attr:app.user.id = "2491f868-88f1-4345-8836-d5d8511a9f83"
+    - attr:app.order.items.count = 1
+  - name: "The user was charged"
+    selector: span[tracetest.span.type="rpc" name="oteldemo.PaymentService/Charge" rpc.system="grpc" rpc.method="Charge" rpc.service="oteldemo.PaymentService"]
+    assertions:
+    - attr:rpc.grpc.status_code  =  0
+    - attr:tracetest.selected_spans.count >= 1
+  - name: "The product was shipped"
+    selector: span[tracetest.span.type="rpc" name="oteldemo.ShippingService/ShipOrder" rpc.system="grpc" rpc.method="ShipOrder" rpc.service="oteldemo.ShippingService"]
+    assertions:
+    - attr:rpc.grpc.status_code = 0
+    - attr:tracetest.selected_spans.count >= 1
+  - name: "The cart was emptied"
+    selector: span[tracetest.span.type="rpc" name="oteldemo.CartService/EmptyCart" rpc.system="grpc" rpc.method="EmptyCart" rpc.service="oteldemo.CartService"]
+    assertions:
+    - attr:rpc.grpc.status_code = 0
+    - attr:tracetest.selected_spans.count >= 1
+  - name: The order was sent to be processed asyncronously
+    selector: span[tracetest.span.type="messaging" name="orders publish" messaging.system="kafka" messaging.destination.name="orders" messaging.destination.kind="topic" messaging.operation="publish"]
+    assertions:
+    - attr:messaging.destination.name = "orders"
+  # TODO: Accountability service is not working in our Demo env, we need to understand why
+  # - name: The order was sent to accountability
+  #   # captures the span emitted by Kafka instrumentation for Go
+  #   selector: span[tracetest.span.type="messaging" name="orders receive" messaging.system="kafka" messaging.destination.name="orders" messaging.destination.kind="topic" messaging.operation="receive"]
+  #   assertions:
+  #   - attr:name = "orders receive"
+  - name: The order was sent to fraud detection team
+    # captures the span emitted by Kafka instrumentation for Kotlin
+    selector: span[tracetest.span.type="messaging" name="orders process" messaging.system="kafka" messaging.operation="process"]
+    assertions:
+    - attr:name = "orders process"

testing/synthetic-monitoring/otel-demo/_testsuite.yaml

@@ -0,0 +1,14 @@
+# test suite based on https://github.com/open-telemetry/opentelemetry-demo/tree/main/test/tracetesting/frontend-service
+
+type: TestSuite
+spec:
+  id: pokeshop-demo-test-suite
+  name: OTel Demo Synthetic tests
+  description: Run all Frontend tests enabled in sequence, simulating a process of a user purchasing products on Astronomy store
+  steps:
+  - ./01-see-ads.yaml
+  - ./02-get-product-recommendation.yaml
+  - ./03-browse-product.yaml
+  - ./04-add-product-to-cart.yaml
+  - ./05-view-cart.yaml
+  - ./06-checking-out-cart.yaml

testing/synthetic-monitoring/otel-demo/_variableset.yaml

@@ -0,0 +1,7 @@
+type: VariableSet
+spec:
+  id: tracetesting-vars
+  name: tracetesting-vars
+  values:
+    - key: FRONTEND_ADDR
+      value: demo-otel-frontend:8080