Clean up the druid sync api. (#1101)

apache · Sep 14, 2016 · a871ee7 · a871ee7
1 parent 2e6b4b1
commit a871ee7
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 31 deletions.
diff --git a/caravel/views.py b/caravel/views.py
@@ -1388,9 +1388,10 @@ def sync_druid_source(self):
                 "dimensions": ["affiliate_id", "campaign", "first_seen"]
             }
         """
-        druid_config = json.loads(request.form.get('config'))
-        user_name = request.form.get('user')
-        cluster_name = request.form.get('cluster')
+        payload = request.get_json(force=True)
+        druid_config = payload['config']
+        user_name = payload['user']
+        cluster_name = payload['cluster']
 
         user = sm.find_user(username=user_name)
         if not user:

diff --git a/tests/core_tests.py b/tests/core_tests.py
@@ -218,22 +218,24 @@ def test_druid_sync_from_config(self):
         db.session.commit()
 
         cfg = {
-            "name": "test_click",
-            "dimensions": ["affiliate_id", "campaign", "first_seen"],
-            "metrics_spec": [{"type": "count", "name": "count"},
-                             {"type": "sum", "name": "sum"}],
-            "batch_ingestion": {
-                "sql": "SELECT * FROM clicks WHERE d='{{ ds }}'",
-                "ts_column": "d",
-                "sources": [{
-                    "table": "clicks",
-                    "partition": "d='{{ ds }}'"
-                }]
+            "user": "admin",
+            "cluster": "new_druid",
+            "config": {
+                "name": "test_click",
+                "dimensions": ["affiliate_id", "campaign", "first_seen"],
+                "metrics_spec": [{"type": "count", "name": "count"},
+                                 {"type": "sum", "name": "sum"}],
+                "batch_ingestion": {
+                    "sql": "SELECT * FROM clicks WHERE d='{{ ds }}'",
+                    "ts_column": "d",
+                    "sources": [{
+                        "table": "clicks",
+                        "partition": "d='{{ ds }}'"
+                    }]
+                }
             }
         }
-        resp = self.client.post(
-            '/caravel/sync_druid/', data=dict(
-                config=json.dumps(cfg), user="admin", cluster="new_druid"))
+        resp = self.client.post('/caravel/sync_druid/', data=json.dumps(cfg))
 
         druid_ds = db.session.query(DruidDatasource).filter_by(
             datasource_name="test_click").first()
@@ -243,10 +245,8 @@ def test_druid_sync_from_config(self):
             ["count", "sum"])
         assert resp.status_code == 201
 
-        # Datasource exists, not changes required
-        resp = self.client.post(
-            '/caravel/sync_druid/', data=dict(
-                config=json.dumps(cfg), user="admin", cluster="new_druid"))
+        # datasource exists, not changes required
+        resp = self.client.post('/caravel/sync_druid/', data=json.dumps(cfg))
         druid_ds = db.session.query(DruidDatasource).filter_by(
             datasource_name="test_click").first()
         assert set([c.column_name for c in druid_ds.columns]) == set(
@@ -255,18 +255,20 @@ def test_druid_sync_from_config(self):
             ["count", "sum"])
         assert resp.status_code == 201
 
-        # datasource exists, not changes required
+        # datasource exists, add new metrics and dimentions
         cfg = {
-            "name": "test_click",
-            "dimensions": ["affiliate_id", "second_seen"],
-            "metrics_spec": [
-                {"type": "bla", "name": "sum"},
-                {"type": "unique", "name": "unique"}
-            ],
+            "user": "admin",
+            "cluster": "new_druid",
+            "config": {
+                "name": "test_click",
+                "dimensions": ["affiliate_id", "second_seen"],
+                "metrics_spec": [
+                    {"type": "bla", "name": "sum"},
+                    {"type": "unique", "name": "unique"}
+                ],
+            }
         }
-        resp = self.client.post(
-            '/caravel/sync_druid/', data=dict(
-                config=json.dumps(cfg), user="admin", cluster="new_druid"))
+        resp = self.client.post('/caravel/sync_druid/', data=json.dumps(cfg))
         druid_ds = db.session.query(DruidDatasource).filter_by(
             datasource_name="test_click").first()
         # columns and metrics are not deleted if config is changed as