add basic site analytics + check for which posts to update after bulk import

.gitignore

@@ -24,3 +24,4 @@ un_sdg/metadata/
 *.Rproj
 .Rproj.user
 venv
+credentials/

requirements.txt

@@ -15,4 +15,6 @@ black==21.8b0
 flake8==3.9.2
 xlrd==2.0.1
 pytest==6.2.5
-mypy==0.910
+mypy==0.910
+google-auth-oauthlib==0.4.6
+google-api-python-client==2.19.1

site_analytics/README.md

@@ -0,0 +1,49 @@
+# Site analytics
+
+Exposes simple site analytics, such as page views by slug URL on the [ourworldindata.org](https://ourworldindata.org) website. For internal use by OWID staff only.
+
+This service uses the [Google Analytics Reporting API v4](https://developers.google.com/analytics/devguides/reporting/core/v4). 
+
+## Setup
+
+In order to use this service, you must:
+
+1. Have a client secrets file located in `site_analytics/config/credentials/owid-analytics-client-secrets.json`. 
+2. Add the following variables to `.env` at the root of this repository:
+
+```bash
+GA_VIEW_ID = "{GA_VIEW_ID}"
+GA_ACCOUNT_ID = "{GA_ACCOUNT_ID}"
+GA_PROPERTY_ID = "{GA_PROPERTY_ID}"
+```
+
+Ask an OWID developer for these files/variables (available to OWID staff only).
+
+
+## Examples
+
+Execute a single simple [Google Analytics report request](https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet) with one metric and one dimension over one time range:
+
+```python
+from site_analytics.request_report import execute_report_request
+
+df = execute_report_request(
+  metric="pageviews",
+  dimension="pagePath",
+  start_date="7daysAgo",
+  end_date="yesterday",
+  filters_expression="ga:dimension1==0"  # excludes views of embedded pages
+)
+print(df.tail(5))
+#       start_date   end_date             pagePath  pageviews
+# 41932   7daysAgo  yesterday         /covid-cases    90053.0
+# 41933   7daysAgo  yesterday  /covid-vaccinations   123601.0
+# 41934   7daysAgo  yesterday         /coronavirus   167887.0
+# 41935   7daysAgo  yesterday                    /   176392.0
+# 41936   7daysAgo  yesterday  /covid-vaccinations   711477.0
+
+```
+
+For acceptable values of the `metric`, `dimension`, etc parameters, check out the [Google Analytics UA query explorer](https://ga-dev-tools.web.app/query-explorer/).
+
+> Note: set `filters_expression="ga:dimension1==0"` to exclude visits to embedded OWID pages.

site_analytics/__init__.py

@@ -0,0 +1,6 @@
+import os
+
+CURRENT_DIR = os.path.dirname(__file__).split("/")[-1]
+CONFIGPATH = os.path.join(CURRENT_DIR, "config")
+CREDSPATH = os.path.join(CONFIGPATH, "credentials")
+CLIENT_SECRETS_PATH = os.path.join(CREDSPATH, "owid-analytics-client-secrets.json")

site_analytics/request_report.py

@@ -0,0 +1,84 @@
+import os
+import pandas as pd
+from typing import Optional
+from dotenv import load_dotenv
+from apiclient.discovery import build
+from google.oauth2.credentials import Credentials
+
+from site_analytics.utils import google_analytics_authenticate
+
+load_dotenv()
+
+GA_ACCOUNT_ID = os.getenv("GA_ACCOUNT_ID")
+GA_PROPERTY_ID = os.getenv("GA_PROPERTY_ID")
+GA_VIEW_ID = os.getenv("GA_VIEW_ID")
+
+
+def execute_report_request(
+    metric: str,
+    dimension: str,
+    start_date: str,
+    end_date: str,
+    filters_expression: Optional[str] = None,
+    credentials: Optional[Credentials] = None,
+) -> pd.DataFrame:
+    """Request a single simple Google Analytics report with one metric and one
+    dimension in a single time range.
+
+    Uses the Google Analytics Reporting API v4. See
+    https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet
+    for valid values of the `metric`, `dimension`, `start_date`, etc parameters.
+    """
+    if not credentials:
+        credentials = google_analytics_authenticate()
+    analytics = build("analyticsreporting", "v4", credentials=credentials)
+    ga_rows = []
+    report_request = {
+        "viewId": f"ga:{GA_VIEW_ID}",
+        "dateRanges": [{"startDate": start_date, "endDate": end_date}],
+        "metrics": [{"expression": f"ga:{metric}"}],
+        "dimensions": [
+            {"name": f"ga:{dimension}"},
+        ],
+        "pageSize": "100000",
+        "orderBys": [
+            {
+                "fieldName": f"ga:{metric}",
+                "sortOrder": "DESCENDING",
+            }
+        ],
+    }
+    if filters_expression:
+        report_request["filtersExpression"] = filters_expression
+    response = (
+        analytics.reports()
+        .batchGet(body={"reportRequests": [report_request]})
+        .execute()
+    )
+    for row in response["reports"][0]["data"]["rows"]:
+        dims = row["dimensions"]
+        assert "(other)" not in dims
+        val = row["metrics"][0]["values"][0]
+        ga_rows.append([start_date, end_date] + dims + [val])
+    while response["reports"][0].get("nextPageToken"):
+        report_request["pageToken"] = response["reports"][0].get("nextPageToken")
+        response = (
+            analytics.reports()
+            .batchGet(body={"reportRequests": [report_request]})
+            .execute()
+        )
+        for row in response["reports"][0]["data"]["rows"]:
+            dims = row["dimensions"]
+            assert "(other)" not in dims
+            val = row["metrics"][0]["values"][0]
+            ga_rows.append([start_date, end_date] + dims + [val])
+    df = pd.DataFrame(
+        ga_rows,
+        columns=["start_date", "end_date", dimension, metric],
+    )
+    try:
+        df[metric] = df[metric].astype(float)
+    except:
+        pass
+    df = df.sort_values(by=metric).reset_index(drop=True)
+    return df

site_analytics/test_request_report.py

@@ -0,0 +1,56 @@
+import pytest
+
+import os
+import pandas as pd
+from apiclient.discovery import build
+
+from site_analytics.utils import google_analytics_authenticate
+from site_analytics.request_report import execute_report_request
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+GA_ACCOUNT_ID = os.getenv("GA_ACCOUNT_ID")
+GA_PROPERTY_ID = os.getenv("GA_PROPERTY_ID")
+
+
+@pytest.fixture(scope="module")
+def credentials():
+    yield google_analytics_authenticate()
+
+
+def test_dimension1_name(credentials):
+    nm = (
+        build("analytics", "v3", credentials=credentials)
+        .management()
+        .customDimensions()
+        .get(
+            accountId=GA_ACCOUNT_ID,
+            webPropertyId=GA_PROPERTY_ID,
+            customDimensionId="ga:dimension1",
+        )
+        .execute()["name"]
+    )
+    assert nm == "Page is embedded"
+
+
+@pytest.fixture(scope="module")
+def df_simple_request(credentials):
+    df = execute_report_request(
+        metric="pageviews",
+        dimension="pagePath",
+        start_date="yesterday",
+        end_date="yesterday",
+        filters_expression=None,
+        credentials=credentials,
+    )
+    yield df
+
+
+def test_is_frame(df_simple_request):
+    assert type(df_simple_request) == pd.DataFrame
+
+
+def test_gt_zero_rows(df_simple_request):
+    assert df_simple_request.shape[0] > 0

site_analytics/utils.py

@@ -0,0 +1,16 @@
+from google_auth_oauthlib.flow import InstalledAppFlow
+from site_analytics import CLIENT_SECRETS_PATH
+
+
+def google_analytics_authenticate():
+    flow = InstalledAppFlow.from_client_secrets_file(
+        CLIENT_SECRETS_PATH,
+        scopes=[
+            "openid",
+            "https://www.googleapis.com/auth/userinfo.email",
+            "https://www.googleapis.com/auth/analytics.readonly",
+        ],
+    )
+    flow.run_local_server()
+    credentials = flow.credentials
+    return credentials