Skip to content

Commit

Permalink
Simplified index query values, updated finder and index tests
Browse files Browse the repository at this point in the history
  • Loading branch information
Dan committed Dec 15, 2016
1 parent 3e10b13 commit 93e8c91
Show file tree
Hide file tree
Showing 8 changed files with 1,575 additions and 1,258 deletions.
12 changes: 6 additions & 6 deletions influxgraph/classes/finder.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,17 +307,17 @@ def find_nodes(self, query):
:param query: Query to search for
:type query: :mod:`influxgraph.utils.Query`
"""
paths = self.index.query(query.pattern)
for path in paths:
if path['is_leaf']:
node_paths = self.index.query(query.pattern)
for path, node in node_paths:
if node.is_leaf():
# Set path on existing reader to avoid having to create
# new objects for each path which is expensive
# Reader is not used for queries when multi fetch is enabled
# regardless
self.reader.path = path['metric']
yield InfluxDBLeafNode(path['metric'], self.reader)
self.reader.path = path
yield InfluxDBLeafNode(path, self.reader)
else:
yield BranchNode(path['metric'])
yield BranchNode(path)

def _gen_aggregation_func(self, paths):
aggregation_funcs = list(set(get_aggregation_func(path, self.aggregation_functions)
Expand Down
6 changes: 1 addition & 5 deletions influxgraph/classes/tree.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,10 +90,6 @@ class NodeTreeIndex(object):
def __init__(self):
self.index = Node()

def insert_series(self, series):
for serie in series:
self.insert(serie)

def insert(self, metric_path):
"""Insert metric path into tree index"""
paths = deque([_encode_bytes(s) for s in metric_path.split('.')])
Expand All @@ -110,7 +106,7 @@ def clear(self):
def query(self, query):
"""Return nodes matching Graphite glob pattern query"""
nodes = sorted(self.search(self.index, query.split('.'), []))
return ({'metric': '.'.join(path), 'is_leaf': node.is_leaf()}
return (('.'.join(path), node,)
for path, node in nodes)

def _get_children_from_matched_paths(self, matched_paths, node):
Expand Down
2,682 changes: 1,498 additions & 1,184 deletions influxgraph/ext/classes/tree.c
View file

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion influxgraph/ext/classes/tree.pyx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ cdef class NodeTreeIndex:
"""Return nodes matching Graphite glob pattern query"""
cdef list nodes = sorted(self.search(self.index, query.split('.'), []))
cdef Node node
return ({'metric': '.'.join(path), 'is_leaf': node.is_leaf()}
return (('.'.join(path), node,)
for path, node in nodes)

def _get_children_from_matched_paths(self, list matched_paths, Node node):
Expand Down
45 changes: 23 additions & 22 deletions tests/test_c_index.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,44 +27,45 @@ def test_root_wildcard(self):
# Unicode query test
result = list(self.index.query(u'*'))
self.assertTrue(len(result) == 1)
self.assertTrue(result[0]['metric'] == 'b1')
self.assertTrue(result[0][0] == 'b1')
result = list(self.index.query('b1'))
self.assertTrue(result[0]['metric'] == 'b1')
self.assertTrue(result[0][0] == 'b1')
result = list(self.index.query('b1.*'))
self.assertTrue(len(result) == 1)
self.assertTrue(result[0]['metric'] == 'b1.b1')
self.assertTrue(result[0][0] == 'b1.b1')
result = list(self.index.query('b1.b1.*'))
self.assertTrue(len(result) == 2)
self.assertTrue(result[0]['metric'] == 'b1.b1.b1')
self.assertTrue(result[1]['metric'] == 'b1.b1.b2')
self.assertTrue(result[0][0] == 'b1.b1.b1')
self.assertTrue(result[1][0] == 'b1.b1.b2')
result = list(self.index.query('b1.b1.*.*'))
self.assertTrue(len(result) == 3)
self.assertTrue(result[0]['metric'] == 'b1.b1.b1.b1')
self.assertTrue(result[1]['metric'] == 'b1.b1.b1.b2')
self.assertTrue(result[2]['metric'] == 'b1.b1.b2.b2')
self.assertTrue(result[0][0] == 'b1.b1.b1.b1')
self.assertTrue(result[1][0] == 'b1.b1.b1.b2')
self.assertTrue(result[2][0] == 'b1.b1.b2.b2')
result = list(self.index.query('b1.b1.*.*.*'))
self.assertTrue(len(result) == 6)
self.assertTrue(result[0]['metric'] == 'b1.b1.b1.b1.leaf1')
self.assertTrue(result[1]['metric'] == 'b1.b1.b1.b1.leaf2')
self.assertTrue(result[2]['metric'] == 'b1.b1.b1.b2.leaf1')
self.assertTrue(result[3]['metric'] == 'b1.b1.b1.b2.leaf2')
self.assertTrue(result[4]['metric'] == 'b1.b1.b2.b2.leaf1')
self.assertTrue(result[5]['metric'] == 'b1.b1.b2.b2.leaf2')
self.assertTrue(result[0][0] == 'b1.b1.b1.b1.leaf1')
self.assertTrue(result[1][0] == 'b1.b1.b1.b1.leaf2')
self.assertTrue(result[2][0] == 'b1.b1.b1.b2.leaf1')
self.assertTrue(result[3][0] == 'b1.b1.b1.b2.leaf2')
self.assertTrue(result[4][0] == 'b1.b1.b2.b2.leaf1')
self.assertTrue(result[5][0] == 'b1.b1.b2.b2.leaf2')
result = list(self.index.query('b1.b1.*.*.{leaf1,leaf2}'))
self.assertTrue(len(result) == 6)
self.assertTrue(result[0]['metric'] == 'b1.b1.b1.b1.leaf1')
self.assertTrue(result[1]['metric'] == 'b1.b1.b1.b1.leaf2')
self.assertTrue(result[2]['metric'] == 'b1.b1.b1.b2.leaf1')
self.assertTrue(result[3]['metric'] == 'b1.b1.b1.b2.leaf2')
self.assertTrue(result[4]['metric'] == 'b1.b1.b2.b2.leaf1')
self.assertTrue(result[5]['metric'] == 'b1.b1.b2.b2.leaf2')
self.assertTrue(result[0][0] == 'b1.b1.b1.b1.leaf1')
self.assertTrue(result[1][0] == 'b1.b1.b1.b1.leaf2')
self.assertTrue(result[2][0] == 'b1.b1.b1.b2.leaf1')
self.assertTrue(result[3][0] == 'b1.b1.b1.b2.leaf2')
self.assertTrue(result[4][0] == 'b1.b1.b2.b2.leaf1')
self.assertTrue(result[5][0] == 'b1.b1.b2.b2.leaf2')
result = list(self.index.query('b1.b1.b1.b1.leaf1'))
self.assertTrue(len(result) == 1)
self.assertTrue(result[0]['metric'] == 'b1.b1.b1.b1.leaf1')
self.assertTrue(result[0][0] == 'b1.b1.b1.b1.leaf1')
result = list(self.index.query('fakey*'))
self.assertFalse(result)

def test_to_from_array(self):
index2 = NodeTreeIndex.from_array(self.index.to_array())
self.assertEqual(index2.to_array(), self.index.to_array())
self.assertEqual(list(self.index.query('*')), list(index2.query('*')))
self.assertEqual([path for (path, _) in self.index.query('*')],
[path for (path, _) in index2.query('*')])
30 changes: 15 additions & 15 deletions tests/test_c_templates.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,44 +42,44 @@ def test_template_parse(self):
["a.b.c.d.e.f.g.m.n.j.measurement.field* env=int,region=the_west"])
index = parse_series(series, fields, templates)
self.assertTrue(index is not None)
self.assertEqual([n['metric'] for n in index.query('*')], [u'1'])
self.assertEqual([n['metric'] for n in index.query('*.*')], [u'1.2'])
self.assertEqual([n['metric'] for n in index.query('*.*.*')], [u'1.2.3'])
self.assertEqual([n['metric'] for n in index.query('*.*.*.*')], [u'1.2.3.4'])
self.assertEqual([n['metric'] for n in index.query('*.*.*.*.*')], [u'1.2.3.4.5'])
self.assertEqual([n['metric'] for n in index.query('*.*.*.*.*.*')], [u'1.2.3.4.5.6'])
self.assertEqual([n['metric'] for n in index.query('*.*.*.*.*.*.*')], [u'1.2.3.4.5.6.7'])
self.assertEqual([n['metric'] for n in index.query('*.*.*.*.*.*.*.*')], [u'1.2.3.4.5.6.7.8'])
self.assertEqual([n['metric'] for n in index.query('*.*.*.*.*.*.*.*.*')], [u'1.2.3.4.5.6.7.8.9'])
self.assertEqual([n['metric'] for n in index.query('*.*.*.*.*.*.*.*.*.*')], [u'1.2.3.4.5.6.7.8.9.10'])
self.assertEqual(sorted([n['metric'] for n in index.query('*.*.*.*.*.*.*.*.*.*.%s.*' % (measurements[0],))]),
self.assertEqual([n[0] for n in index.query('*')], [u'1'])
self.assertEqual([n[0] for n in index.query('*.*')], [u'1.2'])
self.assertEqual([n[0] for n in index.query('*.*.*')], [u'1.2.3'])
self.assertEqual([n[0] for n in index.query('*.*.*.*')], [u'1.2.3.4'])
self.assertEqual([n[0] for n in index.query('*.*.*.*.*')], [u'1.2.3.4.5'])
self.assertEqual([n[0] for n in index.query('*.*.*.*.*.*')], [u'1.2.3.4.5.6'])
self.assertEqual([n[0] for n in index.query('*.*.*.*.*.*.*')], [u'1.2.3.4.5.6.7'])
self.assertEqual([n[0] for n in index.query('*.*.*.*.*.*.*.*')], [u'1.2.3.4.5.6.7.8'])
self.assertEqual([n[0] for n in index.query('*.*.*.*.*.*.*.*.*')], [u'1.2.3.4.5.6.7.8.9'])
self.assertEqual([n[0] for n in index.query('*.*.*.*.*.*.*.*.*.*')], [u'1.2.3.4.5.6.7.8.9.10'])
self.assertEqual(sorted([n[0] for n in index.query('*.*.*.*.*.*.*.*.*.*.%s.*' % (measurements[0],))]),
sorted([u'1.2.3.4.5.6.7.8.9.10.%s.%s' % (measurements[0], f,) for f in field_names]))

def test_templated_index_find(self):
index = parse_series(self.series, self.all_fields, self.templates)
query = '*'
nodes = [n['metric'] for n in index.query(query)]
nodes = [n[0] for n in index.query(query)]
expected = [self.metric_prefix]
self.assertEqual(nodes, expected,
msg="Got root branch query result %s - wanted %s" % (
nodes, expected,))
query = str("%s.*" % (self.metric_prefix,))
nodes = [n['metric'] for n in index.query(query)]
nodes = [n[0] for n in index.query(query)]
expected = ['.'.join([self.metric_prefix,
self.tags[self.paths[1]]])]
self.assertEqual(nodes, expected,
msg="Got sub branch query result %s - wanted %s" % (
nodes, expected,))
query = str("%s.%s.*" % (self.metric_prefix, self.tags[self.paths[1]]))
nodes = sorted([n['metric'] for n in index.query(query)])
nodes = sorted([n[0] for n in index.query(query)])
expected = sorted(['.'.join([self.metric_prefix,
self.tags[self.paths[1]],
m]) for m in self.measurements])
self.assertEqual(nodes, expected,
msg="Got sub branch query result %s - wanted %s" % (
nodes, expected,))
query = str("%s.%s.*.*" % (self.metric_prefix, self.tags[self.paths[1]],))
nodes = sorted([n['metric'] for n in index.query(query)])
nodes = sorted([n[0] for n in index.query(query)])
expected = sorted(['.'.join([self.metric_prefix,
self.tags[self.paths[1]],
m, f])
Expand Down
53 changes: 29 additions & 24 deletions tests/test_index.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,49 +13,54 @@ def setUp(self):
'b1.b1.b2.b2.leaf2'
]
self.index = NodeTreeIndex()
self.index.insert_series(all_series)

for serie in all_series:
self.index.insert(serie)

def test_root_wildcard(self):
result = list(self.index.query('*'))
self.assertTrue(len(result) == 1)
self.assertTrue(result[0]['metric'] == 'b1')
# Unicode query test
result = list(self.index.query(u'*'))
self.assertTrue(len(result) == 1)
self.assertTrue(result[0][0] == 'b1')
result = list(self.index.query('b1'))
self.assertTrue(result[0]['metric'] == 'b1')
self.assertTrue(result[0][0] == 'b1')
result = list(self.index.query('b1.*'))
self.assertTrue(len(result) == 1)
self.assertTrue(result[0]['metric'] == 'b1.b1')
self.assertTrue(result[0][0] == 'b1.b1')
result = list(self.index.query('b1.b1.*'))
self.assertTrue(len(result) == 2)
self.assertTrue(result[0]['metric'] == 'b1.b1.b1')
self.assertTrue(result[1]['metric'] == 'b1.b1.b2')
self.assertTrue(result[0][0] == 'b1.b1.b1')
self.assertTrue(result[1][0] == 'b1.b1.b2')
result = list(self.index.query('b1.b1.*.*'))
self.assertTrue(len(result) == 3)
self.assertTrue(result[0]['metric'] == 'b1.b1.b1.b1')
self.assertTrue(result[1]['metric'] == 'b1.b1.b1.b2')
self.assertTrue(result[2]['metric'] == 'b1.b1.b2.b2')
self.assertTrue(result[0][0] == 'b1.b1.b1.b1')
self.assertTrue(result[1][0] == 'b1.b1.b1.b2')
self.assertTrue(result[2][0] == 'b1.b1.b2.b2')
result = list(self.index.query('b1.b1.*.*.*'))
self.assertTrue(len(result) == 6)
self.assertTrue(result[0]['metric'] == 'b1.b1.b1.b1.leaf1')
self.assertTrue(result[1]['metric'] == 'b1.b1.b1.b1.leaf2')
self.assertTrue(result[2]['metric'] == 'b1.b1.b1.b2.leaf1')
self.assertTrue(result[3]['metric'] == 'b1.b1.b1.b2.leaf2')
self.assertTrue(result[4]['metric'] == 'b1.b1.b2.b2.leaf1')
self.assertTrue(result[5]['metric'] == 'b1.b1.b2.b2.leaf2')
self.assertTrue(result[0][0] == 'b1.b1.b1.b1.leaf1')
self.assertTrue(result[1][0] == 'b1.b1.b1.b1.leaf2')
self.assertTrue(result[2][0] == 'b1.b1.b1.b2.leaf1')
self.assertTrue(result[3][0] == 'b1.b1.b1.b2.leaf2')
self.assertTrue(result[4][0] == 'b1.b1.b2.b2.leaf1')
self.assertTrue(result[5][0] == 'b1.b1.b2.b2.leaf2')
result = list(self.index.query('b1.b1.*.*.{leaf1,leaf2}'))
self.assertTrue(len(result) == 6)
self.assertTrue(result[0]['metric'] == 'b1.b1.b1.b1.leaf1')
self.assertTrue(result[1]['metric'] == 'b1.b1.b1.b1.leaf2')
self.assertTrue(result[2]['metric'] == 'b1.b1.b1.b2.leaf1')
self.assertTrue(result[3]['metric'] == 'b1.b1.b1.b2.leaf2')
self.assertTrue(result[4]['metric'] == 'b1.b1.b2.b2.leaf1')
self.assertTrue(result[5]['metric'] == 'b1.b1.b2.b2.leaf2')
self.assertTrue(result[0][0] == 'b1.b1.b1.b1.leaf1')
self.assertTrue(result[1][0] == 'b1.b1.b1.b1.leaf2')
self.assertTrue(result[2][0] == 'b1.b1.b1.b2.leaf1')
self.assertTrue(result[3][0] == 'b1.b1.b1.b2.leaf2')
self.assertTrue(result[4][0] == 'b1.b1.b2.b2.leaf1')
self.assertTrue(result[5][0] == 'b1.b1.b2.b2.leaf2')
result = list(self.index.query('b1.b1.b1.b1.leaf1'))
self.assertTrue(len(result) == 1)
self.assertTrue(result[0]['metric'] == 'b1.b1.b1.b1.leaf1')
self.assertTrue(result[0][0] == 'b1.b1.b1.b1.leaf1')
result = list(self.index.query('fakey*'))
self.assertFalse(result)

def test_to_from_array(self):
index2 = NodeTreeIndex.from_array(self.index.to_array())
self.assertEqual(index2.to_array(), self.index.to_array())
self.assertEqual(list(self.index.query('*')), list(index2.query('*')))
self.assertEqual([path for (path, _) in self.index.query('*')],
[path for (path, _) in index2.query('*')])
3 changes: 2 additions & 1 deletion tests/test_influxdb_integration.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -835,7 +835,8 @@ def test_index_save_load(self):
index_fh.close()
self.assertTrue(index is not None)
for query in ['*', '*.*', '*.*.*', '*.*.*.*']:
self.assertEqual(list(index.query(query)), list(finder_index.query(query)))
self.assertEqual([path for (path, _) in index.query(query)],
[path for (path, _) in finder_index.query(query)])

def test_index_load_from_file(self):
values = [['carbon.relays.host.dispatcher1.wallTime_us'],
Expand Down

0 comments on commit 93e8c91

Please sign in to comment.