Implement glean ping tagging for "debug view"

-Validate pingTag using regex pattern matching
-Add config item to hold pingTag
-Update uploader to add header with pingTag
-Add server redirect to GCP endpoint for tagged pings
-Add documentation on how to use ping tagging
-Update unit tests to cover ping tagging functionality

components/service/glean/README.md

@@ -127,16 +127,24 @@ for the command line switches used to pass the extra keys. These are the current
 |key|type|description|
 |---|----|-----------|
 | logPings | boolean (--ez) | If set to `true`, glean dumps pings to logcat; defaults to `false` |
-| sendPing | string (--es) | Sends the ping with the given name immediately. |
+| sendPing | string (--es) | Sends the ping with the given name immediately |
+| tagPings | string (--es) | Tags all outgoing pings as debug pings to make them available for real-time validation. The value must match the pattern `[a-zA-Z0-9-]{1,20}` |
 
-For example, to direct the glean sample application to dump pings to logcat, and send the "metrics" ping immediately, the following command can be used:
+For example, to direct the glean sample application to (1) dump pings to logcat, (2) tag the ping 
+with the `test-metrics-ping` tag, and (3) send the "metrics" ping immediately, the following command
+can be used:
 
 ```
 adb shell am start -n org.mozilla.samples.glean/mozilla.components.service.glean.debug.GleanDebugActivity \
   --ez logPings true \
-  --es sendPing metrics
+  --es sendPing metrics \
+  --es tagPings test-metrics-ping
 ```
 
+### Important GleanDebugActivity note!
+
+Options that are set using the adb flags are not immediately reset and will persist until the application is closed or manually reset.
+
 ## Contact
 
 To contact us you can:

components/service/glean/src/main/java/mozilla/components/service/glean/config/Configuration.kt

@@ -20,13 +20,20 @@ import mozilla.components.service.glean.BuildConfig
  * @property maxEvents the number of events to store before the events ping is sent
  * @property logPings whether to log ping contents to the console.
  * @property httpClient The HTTP client implementation to use for uploading pings.
+ * @property pingTag String tag to be applied to headers when uploading pings for debug view
  */
 data class Configuration(
-    val serverEndpoint: String = "https://incoming.telemetry.mozilla.org",
+    val serverEndpoint: String = DEFAULT_TELEMETRY_ENDPOINT,
     val userAgent: String = "Glean/${BuildConfig.LIBRARY_VERSION} (Android)",
     val connectionTimeout: Long = 10000,
     val readTimeout: Long = 30000,
     val maxEvents: Int = 500,
     val logPings: Boolean = false,
-    val httpClient: Lazy<Client> = lazy { HttpURLConnectionClient() }
-)
+    val httpClient: Lazy<Client> = lazy { HttpURLConnectionClient() },
+    val pingTag: String? = null
+) {
+    companion object {
+        const val DEFAULT_TELEMETRY_ENDPOINT = "https://incoming.telemetry.mozilla.org"
+        const val DEFAULT_DEBUGVIEW_ENDPOINT = "https://stage.ingestion.nonprod.dataops.mozgcp.net"
+    }
+}

components/service/glean/src/main/java/mozilla/components/service/glean/debug/GleanDebugActivity.kt

@@ -22,6 +22,16 @@ import mozilla.components.support.base.log.logger.Logger
 class GleanDebugActivity : Activity() {
     private val logger = Logger("glean/GleanDebugActivity")
 
+    companion object {
+        // This is a list of the currently accepted commands
+        const val SEND_PING_EXTRA_KEY = "sendPing"
+        const val LOG_PINGS_EXTRA_KEY = "logPings"
+        const val TAG_DEBUG_VIEW_EXTRA_KEY = "tagPings"
+
+        // Regular expression filter for debugId
+        val pingTagPattern = "[a-zA-Z0-9-]{1,20}".toRegex()
+    }
+
     // IMPORTANT: These activities are unsecured, and may be triggered by
     // any other application on the device, including in release builds.
     // Therefore, care should be taken in selecting what features are
@@ -33,16 +43,29 @@ class GleanDebugActivity : Activity() {
 
         // Enable debugging options and start the application.
         intent.extras?.let {
+            // Check for ping debug view tag to apply to the X-Debug-ID header when uploading the
+            // ping to the endpoint
+            var pingTag: String? = intent.getStringExtra(TAG_DEBUG_VIEW_EXTRA_KEY)
+
+            // Validate the ping tag against the regex pattern
+            pingTag?.let {
+                if (!pingTagPattern.matches(it)) {
+                    logger.error("tagPings value $it does not match accepted pattern $pingTagPattern")
+                    pingTag = null
+                }
+            }
+
             val debugConfig = Glean.configuration.copy(
-                logPings = intent.getBooleanExtra("logPings", Glean.configuration.logPings)
+                logPings = intent.getBooleanExtra(LOG_PINGS_EXTRA_KEY, Glean.configuration.logPings),
+                pingTag = pingTag
             )
 
             // Finally set the default configuration before starting
             // the real product's activity.
             logger.info("Setting debug config $debugConfig")
             Glean.configuration = debugConfig
 
-            intent.getStringExtra("sendPing")?.let {
+            intent.getStringExtra(SEND_PING_EXTRA_KEY)?.let {
                 Glean.sendPings(listOf(it))
             }
         }

components/service/glean/src/main/java/mozilla/components/service/glean/net/HttpPingUploader.kt

@@ -76,25 +76,42 @@ internal class HttpPingUploader : PingUploader {
     }
 
     @VisibleForTesting(otherwise = PRIVATE)
-    internal fun buildRequest(path: String, data: String, config: Configuration) = Request(
-        url = config.serverEndpoint + path,
-        method = Request.Method.POST,
-        connectTimeout = Pair(config.connectionTimeout, TimeUnit.MILLISECONDS),
-        readTimeout = Pair(config.readTimeout, TimeUnit.MILLISECONDS),
-        headers = MutableHeaders(
+    internal fun buildRequest(path: String, data: String, config: Configuration): Request {
+        val headers = MutableHeaders(
             "Content-Type" to "application/json; charset=utf-8",
             "User-Agent" to config.userAgent,
             "Date" to createDateHeaderValue(),
             // Add headers for supporting the legacy pipeline.
             "X-Client-Type" to "Glean",
             "X-Client-Version" to BuildConfig.LIBRARY_VERSION
-        ),
-        // Make sure we are not sending cookies. Unfortunately, HttpURLConnection doesn't
-        // offer a better API to do that, so we nuke all cookies going to our telemetry
-        // endpoint.
-        cookiePolicy = Request.CookiePolicy.OMIT,
-        body = Request.Body.fromString(data)
-    )
+        )
+
+        var endpoint = config.serverEndpoint
+
+        // If there is a pingTag set, then this header needs to be added in order to flag pings
+        // for "debug view" use.
+        config.pingTag?.let {
+            headers.append("X-Debug-ID", it)
+
+            // NOTE: Tagged pings must be redirected to the GCP endpoint as the AWS endpoint isn't
+            // configured to handle them.  This may pose an issue with testing if a local server
+            // is used to capture pings.
+            endpoint = Configuration.DEFAULT_DEBUGVIEW_ENDPOINT
+        }
+
+        return Request(
+            url = endpoint + path,
+            method = Request.Method.POST,
+            connectTimeout = Pair(config.connectionTimeout, TimeUnit.MILLISECONDS),
+            readTimeout = Pair(config.readTimeout, TimeUnit.MILLISECONDS),
+            headers = headers,
+            // Make sure we are not sending cookies. Unfortunately, HttpURLConnection doesn't
+            // offer a better API to do that, so we nuke all cookies going to our telemetry
+            // endpoint.
+            cookiePolicy = Request.CookiePolicy.OMIT,
+            body = Request.Body.fromString(data)
+        )
+    }
 
     @Throws(IOException::class)
     internal fun performUpload(client: Client, request: Request): Boolean {

components/service/glean/src/test/java/mozilla/components/service/glean/TestUtil.kt

@@ -14,6 +14,11 @@ import androidx.work.testing.WorkManagerTestInitHelper
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.GlobalScope
 import kotlinx.coroutines.launch
+import mozilla.components.concept.fetch.Client
+import mozilla.components.concept.fetch.Headers
+import mozilla.components.concept.fetch.MutableHeaders
+import mozilla.components.concept.fetch.Request
+import mozilla.components.concept.fetch.Response
 import mozilla.components.service.glean.config.Configuration
 import mozilla.components.service.glean.firstrun.FileFirstRunDetector
 import mozilla.components.service.glean.ping.PingMaker
@@ -191,3 +196,28 @@ internal fun triggerWorkManager() {
         PingUploadWorker.uploadPings()
     }
 }
+
+/**
+ * This is a helper class to facilitate testing of ping tagging
+ */
+internal class TestPingTagClient(
+    private val responseUrl: String = Configuration.DEFAULT_DEBUGVIEW_ENDPOINT,
+    private val responseStatus: Int = 200,
+    private val responseHeaders: Headers = MutableHeaders(),
+    private val responseBody: Response.Body = Response.Body.empty(),
+    private val debugHeaderValue: String? = null
+) : Client() {
+    override fun fetch(request: Request): Response {
+        Assert.assertTrue("URL must be redirected for tagged pings",
+            request.url.startsWith(responseUrl))
+        Assert.assertEquals("Debug headers must match what the ping tag was set to",
+            debugHeaderValue, request.headers!!["X-Debug-ID"])
+
+        // Have to return a response here.
+        return Response(
+            responseUrl ?: request.url,
+            responseStatus,
+            request.headers ?: responseHeaders,
+            responseBody)
+    }
+}

components/service/glean/src/test/java/mozilla/components/service/glean/debug/GleanDebugActivityTest.kt

@@ -21,6 +21,7 @@ import mozilla.components.service.glean.BooleanMetricType
 import mozilla.components.service.glean.Lifetime
 import mozilla.components.service.glean.resetGlean
 import mozilla.components.service.glean.triggerWorkManager
+import mozilla.components.service.glean.TestPingTagClient
 import okhttp3.mockwebserver.MockResponse
 import okhttp3.mockwebserver.MockWebServer
 import org.robolectric.Shadows.shadowOf
@@ -81,7 +82,7 @@ class GleanDebugActivityTest {
         // Set the extra values and start the intent.
         val intent = Intent(ApplicationProvider.getApplicationContext<Context>(),
             GleanDebugActivity::class.java)
-        intent.putExtra("logPings", true)
+        intent.putExtra(GleanDebugActivity.LOG_PINGS_EXTRA_KEY, true)
         val activity = Robolectric.buildActivity(GleanDebugActivity::class.java, intent)
         activity.create().start().resume()
 
@@ -128,16 +129,116 @@ class GleanDebugActivityTest {
         // Set the extra values and start the intent.
         val intent = Intent(ApplicationProvider.getApplicationContext<Context>(),
         GleanDebugActivity::class.java)
-        intent.putExtra("sendPing", "store1")
+        intent.putExtra(GleanDebugActivity.SEND_PING_EXTRA_KEY, "store1")
         val activity = Robolectric.buildActivity(GleanDebugActivity::class.java, intent)
         activity.create().start().resume()
+
+        // Since we reset the serverEndpoint back to the default for untagged pings, we need to
+        // override it here so that the local server we created to intercept the pings will
+        // be the one that the ping is sent to.
+        Glean.configuration = Glean.configuration.copy(
+            serverEndpoint = "http://" + server.hostName + ":" + server.port
+        )
+
+        triggerWorkManager()
+        val request = server.takeRequest(10L, TimeUnit.SECONDS)
+
+        assertTrue(
+            request.requestUrl.encodedPath().startsWith("/submit/mozilla-components-service-glean/store1")
+        )
+
+        server.shutdown()
+    }
+
+    @Test
+    fun `tagPings filters ID's that don't match the pattern`() {
+        val server = MockWebServer()
+        server.enqueue(MockResponse().setBody("OK"))
+
+        resetGlean(config = Glean.configuration.copy(
+            serverEndpoint = "http://" + server.hostName + ":" + server.port
+        ))
+
+        // Put some metric data in the store, otherwise we won't get a ping out
+        // Define a 'booleanMetric' boolean metric, which will be stored in "store1"
+        val booleanMetric = BooleanMetricType(
+            disabled = false,
+            category = "telemetry",
+            lifetime = Lifetime.Application,
+            name = "boolean_metric",
+            sendInPings = listOf("store1")
+        )
+
+        booleanMetric.set(true)
+        assertTrue(booleanMetric.testHasValue())
+
+        // Set the extra values and start the intent.
+        val intent = Intent(ApplicationProvider.getApplicationContext<Context>(),
+            GleanDebugActivity::class.java)
+        intent.putExtra(GleanDebugActivity.SEND_PING_EXTRA_KEY, "store1")
+        intent.putExtra(GleanDebugActivity.TAG_DEBUG_VIEW_EXTRA_KEY, "inv@lid_id")
+        val activity = Robolectric.buildActivity(GleanDebugActivity::class.java, intent)
+        activity.create().start().resume()
+
+        // Since a bad tag ID results in resetting the endpoint to the default, verify that
+        // has happened.
+        assertEquals("Server endpoint must be reset if tag didn't pass regex",
+            "http://" + server.hostName + ":" + server.port, Glean.configuration.serverEndpoint)
+
         triggerWorkManager()
         val request = server.takeRequest(10L, TimeUnit.SECONDS)
 
         assertTrue(
+            "Request path must be correct",
             request.requestUrl.encodedPath().startsWith("/submit/mozilla-components-service-glean/store1")
         )
 
+        assertNull(
+            "Headers must not contain X-Debug-ID if passed a non matching pattern",
+            request.headers.get("X-Debug-ID")
+        )
+
         server.shutdown()
     }
+
+    @Test
+    fun `pings are correctly tagged using tagPings`() {
+        val pingTag = "test-debug-ID"
+
+        // The TestClient class found at the bottom of this file is used to intercept the request
+        // in order to check that the header has been added and the URL has been redirected.
+        val testClient = TestPingTagClient(
+            responseUrl = Configuration.DEFAULT_DEBUGVIEW_ENDPOINT,
+            debugHeaderValue = pingTag)
+
+        // Use the test client in the Glean configuration
+        resetGlean(config = Glean.configuration.copy(
+            httpClient = lazy { testClient }
+        ))
+
+        // Put some metric data in the store, otherwise we won't get a ping out
+        // Define a 'booleanMetric' boolean metric, which will be stored in "store1"
+        val booleanMetric = BooleanMetricType(
+            disabled = false,
+            category = "telemetry",
+            lifetime = Lifetime.Application,
+            name = "boolean_metric",
+            sendInPings = listOf("store1")
+        )
+
+        booleanMetric.set(true)
+        assertTrue(booleanMetric.testHasValue())
+
+        // Set the extra values and start the intent.
+        val intent = Intent(ApplicationProvider.getApplicationContext<Context>(),
+            GleanDebugActivity::class.java)
+        intent.putExtra(GleanDebugActivity.SEND_PING_EXTRA_KEY, "store1")
+        intent.putExtra(GleanDebugActivity.TAG_DEBUG_VIEW_EXTRA_KEY, pingTag)
+        val activity = Robolectric.buildActivity(GleanDebugActivity::class.java, intent)
+        activity.create().start().resume()
+
+        // This will trigger the call to `fetch()` in the TestPingTagClient which is where the
+        // test assertions will occur
+        triggerWorkManager()
+    }
 }

components/service/glean/src/test/java/mozilla/components/service/glean/net/HttpPingUploaderTest.kt

@@ -10,6 +10,7 @@ import mozilla.components.concept.fetch.Response
 import mozilla.components.lib.fetch.httpurlconnection.HttpURLConnectionClient
 import mozilla.components.lib.fetch.okhttp.OkHttpClient
 import mozilla.components.service.glean.BuildConfig
+import mozilla.components.service.glean.TestPingTagClient
 import mozilla.components.service.glean.config.Configuration
 import mozilla.components.support.test.any
 import mozilla.components.support.test.mock
@@ -276,4 +277,41 @@ class HttpPingUploaderTest {
         // return false in this case.
         assertFalse(uploader.upload("path", "ping", config))
     }
+
+    @Test
+    fun `X-Debug-ID header is correctly added when pingTag is not null`() {
+        val uploader = spy<HttpPingUploader>(HttpPingUploader())
+
+        val debugConfig = Configuration().copy(
+            userAgent = "Glean/Test 25.0.2",
+            connectionTimeout = 3050,
+            readTimeout = 7050,
+            pingTag = "this-ping-is-tagged"
+        )
+
+        val request = uploader.buildRequest(testPath, testPing, debugConfig)
+        assertEquals("this-ping-is-tagged", request.headers!!["X-Debug-ID"])
+    }
+
+    @Test
+    fun `server is correctly redirected when pings are tagged`() {
+        val pingTag = "this-ping-is-tagged"
+
+        // The TestClient class found at the bottom of this file is used to intercept the request
+        // in order to check that the header has been added and the URL has been redirected.
+        val testClient = TestPingTagClient(
+            responseUrl = Configuration.DEFAULT_DEBUGVIEW_ENDPOINT,
+            debugHeaderValue = pingTag)
+
+        // Use the test client in the Glean configuration
+        val testConfig = testDefaultConfig.copy(
+            httpClient = lazy { testClient },
+            pingTag = pingTag
+        )
+
+        // This should trigger the call to `fetch()` within the TestPingTagClient which will perform
+        // both a check against the url and that a header was added.
+        val client = HttpPingUploader()
+        assertTrue(client.upload(testPath, testPing, testConfig))
+    }
 }