diff --git a/caravel/assets/javascripts/explore.js b/caravel/assets/javascripts/explore.js index 27faccbf0d7dd..2f7ded6b00fd2 100644 --- a/caravel/assets/javascripts/explore.js +++ b/caravel/assets/javascripts/explore.js @@ -23,35 +23,35 @@ require('../node_modules/bootstrap-toggle/css/bootstrap-toggle.min.css'); var slice; +var getPanelClass = function (fieldPrefix) { + return (fieldPrefix === "flt" ? "filter" : "having") + "_panel"; +}; + function prepForm() { - var i = 1; // Assigning the right id to form elements in filters - $("#filters > div").each(function () { - $(this).attr("id", function () { - return "flt_" + i; - }); - $(this).find("#flt_col_0") - .attr("id", function () { - return "flt_col_" + i; - }) - .attr("name", function () { - return "flt_col_" + i; - }); - $(this).find("#flt_op_0") - .attr("id", function () { - return "flt_op_" + i; - }) - .attr("name", function () { - return "flt_op_" + i; - }); - $(this).find("#flt_eq_0") - .attr("id", function () { - return "flt_eq_" + i; - }) - .attr("name", function () { - return "flt_eq_" + i; - }); - i++; + var fixId = function ($filter, fieldPrefix, i) { + $filter.attr("id", function () { + return fieldPrefix + "_" + i; + }); + + ["col", "op", "eq"].forEach(function (fieldMiddle) { + var fieldName = fieldPrefix + "_" + fieldMiddle; + $filter.find("#" + fieldName + "_0") + .attr("id", function () { + return fieldName + "_" + i; + }) + .attr("name", function () { + return fieldName + "_" + i; + }); + }); + }; + + ["flt", "having"].forEach(function (fieldPrefix) { + var i = 1; + $("#" + getPanelClass(fieldPrefix) + " #filters > div").each(function () { + fixId($(this), fieldPrefix, i); + i++; + }); }); } @@ -59,9 +59,6 @@ function query(force, pushState) { if (force === undefined) { force = false; } - if (pushState !== false) { - history.pushState({}, document.title, slice.querystring()); - } $('.query-and-save button').attr('disabled', 'disabled'); $('.btn-group.results span,a').attr('disabled', 'disabled'); if (force) { // Don't hide the alert message when the page is just loaded @@ -69,6 +66,10 @@ function query(force, pushState) { } $('#is_cached').hide(); prepForm(); + if (pushState !== false) { + // update the url after prepForm() fix the field ids + history.pushState({}, document.title, slice.querystring()); + } slice.render(force); } @@ -291,24 +292,26 @@ function initExploreView() { $(".ui-helper-hidden-accessible").remove(); // jQuery-ui 1.11+ creates a div for every tooltip function set_filters() { - for (var i = 1; i < 10; i++) { - var eq = px.getParam("flt_eq_" + i); - var col = px.getParam("flt_col_" + i); - if (eq !== '' && col !== '') { - add_filter(i); + ["flt", "having"].forEach(function (prefix) { + for (var i = 1; i < 10; i++) { + var eq = px.getParam(prefix + "_eq_" + i); + var col = px.getParam(prefix + "_col_" + i); + if (eq !== '' && col !== '') { + add_filter(i, prefix); + } } - } + }); } set_filters(); - function add_filter(i) { - var cp = $("#flt0").clone(); - $(cp).appendTo("#filters"); + function add_filter(i, fieldPrefix) { + var cp = $("#"+fieldPrefix+"0").clone(); + $(cp).appendTo("#" + getPanelClass(fieldPrefix) + " #filters"); $(cp).show(); if (i !== undefined) { - $(cp).find("#flt_eq_0").val(px.getParam("flt_eq_" + i)); - $(cp).find("#flt_op_0").val(px.getParam("flt_op_" + i)); - $(cp).find("#flt_col_0").val(px.getParam("flt_col_" + i)); + $(cp).find("#"+fieldPrefix+"_eq_0").val(px.getParam(fieldPrefix+"_eq_" + i)); + $(cp).find("#"+fieldPrefix+"_op_0").val(px.getParam(fieldPrefix+"_op_" + i)); + $(cp).find("#"+fieldPrefix+"_col_0").val(px.getParam(fieldPrefix+"_col_" + i)); } $(cp).find('select').select2(); $(cp).find('.remove').click(function () { @@ -324,7 +327,12 @@ function initExploreView() { returnLocation.reload(); }); - $("#plus").click(add_filter); + $("#filter_panel #plus").click(function () { + add_filter(undefined, "flt"); + }); + $("#having_panel #plus").click(function () { + add_filter(undefined, "having"); + }); $("#btn_save").click(function () { var slice_name = prompt("Name your slice!"); if (slice_name !== "" && slice_name !== null) { diff --git a/caravel/forms.py b/caravel/forms.py index cddd475e84937..6f5d76b76b550 100644 --- a/caravel/forms.py +++ b/caravel/forms.py @@ -846,6 +846,8 @@ def add_to_form(attrs): setattr(QueryForm, attr, self.field_dict[attr]) filter_choices = self.choicify(['in', 'not in']) + having_op_choices = [] + filter_prefixes = ['flt'] # datasource type specific form elements datasource_classname = viz.datasource.__class__.__name__ time_fields = None @@ -885,21 +887,31 @@ def add_to_form(attrs): field_css_classes['granularity'] = ['form-control', 'select2_freeform'] field_css_classes['druid_time_origin'] = ['form-control', 'select2_freeform'] filter_choices = self.choicify(['in', 'not in', 'regex']) + having_op_choices = self.choicify(['>', '<', '==']) + filter_prefixes += ['having'] add_to_form(('since', 'until')) - filter_cols = viz.datasource.filterable_column_names or [''] - for i in range(10): - setattr(QueryForm, 'flt_col_' + str(i), SelectField( - _('Filter 1'), - default=filter_cols[0], - choices=self.choicify(filter_cols))) - setattr(QueryForm, 'flt_op_' + str(i), SelectField( - _('Filter 1'), - default='in', - choices=filter_choices)) - setattr( - QueryForm, 'flt_eq_' + str(i), - TextField(_("Super"), default='')) + filter_cols = self.choicify( + viz.datasource.filterable_column_names or ['']) + having_cols = filter_cols + viz.datasource.metrics_combo + for field_prefix in filter_prefixes: + is_having_filter = field_prefix == 'having' + col_choices = filter_cols if not is_having_filter else having_cols + op_choices = filter_choices if not is_having_filter else \ + having_op_choices + for i in range(10): + setattr(QueryForm, field_prefix + '_col_' + str(i), + SelectField( + _('Filter 1'), + default=col_choices[0][0], + choices=col_choices)) + setattr(QueryForm, field_prefix + '_op_' + str(i), SelectField( + _('Filter 1'), + default=op_choices[0][0], + choices=op_choices)) + setattr( + QueryForm, field_prefix + '_eq_' + str(i), + TextField(_("Super"), default='')) if time_fields: QueryForm.fieldsets = ({ diff --git a/caravel/models.py b/caravel/models.py index dc62d511ee8f5..779605e69c1bd 100644 --- a/caravel/models.py +++ b/caravel/models.py @@ -28,6 +28,7 @@ from pydruid.client import PyDruid from pydruid.utils.filters import Dimension, Filter from pydruid.utils.postaggregator import Postaggregator +from pydruid.utils.having import Having, Aggregation from six import string_types from sqlalchemy import ( Column, Integer, String, ForeignKey, Text, Boolean, DateTime, Date, @@ -41,7 +42,7 @@ import caravel from caravel import app, db, get_session, utils, sm from caravel.viz import viz_types -from caravel.utils import flasher, MetricPermException +from caravel.utils import flasher, MetricPermException, DimSelector config = app.config @@ -1191,38 +1192,15 @@ def recursive_get_fields(_conf): post_aggregations=post_aggs, intervals=from_dttm.isoformat() + '/' + to_dttm.isoformat(), ) - filters = None - for col, op, eq in filter: - cond = None - if op == '==': - cond = Dimension(col) == eq - elif op == '!=': - cond = ~(Dimension(col) == eq) - elif op in ('in', 'not in'): - fields = [] - splitted = eq.split(',') - if len(splitted) > 1: - for s in eq.split(','): - s = s.strip() - fields.append(Dimension(col) == s) - cond = Filter(type="or", fields=fields) - else: - cond = Dimension(col) == eq - if op == 'not in': - cond = ~cond - elif op == 'regex': - cond = Filter(type="regex", pattern=eq, dimension=col) - if filters: - filters = Filter(type="and", fields=[ - cond, - filters - ]) - else: - filters = cond + filters = self.get_filters(filter) if filters: qry['filter'] = filters + having_filters = self.get_having_filters(extras.get('having')) + if having_filters: + qry['having'] = having_filters + client = self.cluster.get_pydruid_client() orig_filters = filters if timeseries_limit and is_timeseries: @@ -1303,6 +1281,62 @@ def recursive_get_fields(_conf): query=query_str, duration=datetime.now() - qry_start_dttm) + @staticmethod + def get_filters(raw_filters): + filters = None + for col, op, eq in raw_filters: + cond = None + if op == '==': + cond = Dimension(col) == eq + elif op == '!=': + cond = ~(Dimension(col) == eq) + elif op in ('in', 'not in'): + fields = [] + splitted = eq.split(',') + if len(splitted) > 1: + for s in eq.split(','): + s = s.strip() + fields.append(Dimension(col) == s) + cond = Filter(type="or", fields=fields) + else: + cond = Dimension(col) == eq + if op == 'not in': + cond = ~cond + elif op == 'regex': + cond = Filter(type="regex", pattern=eq, dimension=col) + if filters: + filters = Filter(type="and", fields=[ + cond, + filters + ]) + else: + filters = cond + return filters + + def get_having_filters(self, raw_filters): + filters = None + for col, op, eq in raw_filters: + cond = None + if op == '==': + if col in self.column_names: + cond = DimSelector(dimension=col, value=eq) + else: + cond = Aggregation(col) == eq + elif op == '!=': + cond = ~(Aggregation(col) == eq) + elif op == '>': + cond = Aggregation(col) > eq + elif op == '<': + cond = Aggregation(col) < eq + if filters: + filters = Filter(type="and", fields=[ + Having.build_having(cond), + Having.build_having(filters) + ]) + else: + filters = cond + return filters + class Log(Model): diff --git a/caravel/templates/caravel/explore.html b/caravel/templates/caravel/explore.html index 925d85620af1f..1f54db7cb52a7 100644 --- a/caravel/templates/caravel/explore.html +++ b/caravel/templates/caravel/explore.html @@ -132,7 +132,7 @@ {% endfor %} -
+
{{ _("Filters") }}
+ + + {% if form.having_col_0 %} +
+
+ Result Filters ("having" filters) + + [-] +
+
+ +
+ +
+
+ {% endif %} {{ form.slice_id() }} {{ form.slice_name() }} {{ form.collapsed_fieldsets() }} diff --git a/caravel/utils.py b/caravel/utils.py index 6ac360cbc5f99..871972c6d065b 100644 --- a/caravel/utils.py +++ b/caravel/utils.py @@ -19,6 +19,7 @@ from flask_appbuilder.security.sqla import models as ab_models from markdown import markdown as md from sqlalchemy.types import TypeDecorator, TEXT +from pydruid.utils.having import Having class CaravelException(Exception): @@ -77,6 +78,18 @@ def __get__(self, obj, objtype): return functools.partial(self.__call__, obj) +class DimSelector(Having): + def __init__(self, **args): + # Just a hack to prevent any exceptions + Having.__init__(self, type='equalTo', aggregation=None, value=None) + + self.having = {'having': { + 'type': 'dimSelector', + 'dimension': args['dimension'], + 'value': args['value'], + }} + + def list_minus(l, minus): """Returns l without what is in minus diff --git a/caravel/viz.py b/caravel/viz.py index 5ae423bcfc0e1..c735c6959dfd4 100644 --- a/caravel/viz.py +++ b/caravel/viz.py @@ -162,21 +162,22 @@ def form(self): def form_class(self): return FormFactory(self).get_form() - def query_filters(self): + def query_filters(self, is_having_filter=False): """Processes the filters for the query""" form_data = self.form_data # Building filters filters = [] + field_prefix = 'flt' if not is_having_filter else 'having' for i in range(1, 10): - col = form_data.get("flt_col_" + str(i)) - op = form_data.get("flt_op_" + str(i)) - eq = form_data.get("flt_eq_" + str(i)) + col = form_data.get(field_prefix + "_col_" + str(i)) + op = form_data.get(field_prefix + "_op_" + str(i)) + eq = form_data.get(field_prefix + "_eq_" + str(i)) if col and op and eq: filters.append((col, op, eq)) # Extra filters (coming from dashboard) extra_filters = form_data.get('extra_filters') - if extra_filters: + if extra_filters and not is_having_filter: extra_filters = json.loads(extra_filters) for slice_filters in extra_filters.values(): for col, vals in slice_filters.items(): @@ -208,7 +209,7 @@ def query_obj(self): # for instance the extra where clause that applies only to Tables extras = { 'where': form_data.get("where", ''), - 'having': form_data.get("having", ''), + 'having': self.query_filters(True) or form_data.get("having", ''), 'time_grain_sqla': form_data.get("time_grain_sqla", ''), 'druid_time_origin': form_data.get("druid_time_origin", ''), }