Allow all secondary config files to be reloaded, update comments to s…

…ay so

conf/blacklist.conf.example

@@ -1,4 +1,5 @@
 # This file takes a single regular expression per line
 # If USE_WHITELIST is set to True in carbon.conf, any metrics received which
 # match one of these expressions will be dropped
+# This file is reloaded automatically when changes are made
 ^some\.noisy\.metric\.prefix\..*

conf/storage-aggregation.conf.example

@@ -1,5 +1,5 @@
 # Aggregation methods for whisper files. Entries are scanned in order,
-# and first match wins.
+# and first match wins. This file is scanned for changes every 60 seconds
 #
 #  [name]
 #  pattern = <regex>

conf/storage-schemas.conf.example

@@ -1,5 +1,5 @@
-# Schema definitions for whisper files. Entries are scanned in order,
-# and first match wins.
+# Schema definitions for Whisper files. Entries are scanned in order,
+# and first match wins. This file is scanned for changes every 60 seconds.
 #
 #  [name]
 #  pattern = regex

conf/whitelist.conf.example

@@ -1,5 +1,6 @@
 # This file takes a single regular expression per line
 # If USE_WHITELIST is set to True in carbon.conf, only metrics received which
 # match one of these expressions will be persisted. If this file is empty or
-# missing, all metrics will pass through
+# missing, all metrics will pass through.
+# This file is reloaded automatically when changes are made
 .*

lib/carbon/aggregator/rules.py

@@ -13,19 +13,25 @@ def __init__(self):
     self.read_task = LoopingCall(self.read_rules)
     self.rules_last_read = 0.0
 
+  def clear(self):
+    self.rules = []
+
   def read_from(self, rules_file):
     self.rules_file = rules_file
     self.read_rules()
     self.read_task.start(10, now=False)
 
   def read_rules(self):
+    if not exists(self.rules_file):
+      self.clear()
+      return
+
     # Only read if the rules file has been modified
     try:
       mtime = getmtime(self.rules_file)
     except:
       log.err("Failed to get mtime of %s" % self.rules_file)
       return
-
     if mtime <= self.rules_last_read:
       return
 

lib/carbon/rewrite.py

@@ -1,17 +1,45 @@
+import time
 import re
+from os.path import exists, getmtime
+from twisted.internet.task import LoopingCall
+from carbon import log
 
 
 class RewriteRuleManager:
   def __init__(self):
     self.preRules = []
     self.postRules = []
+    self.read_task = LoopingCall(self.read_rules)
+    self.rules_last_read = 0.0
+
+  def clear(self):
+    self.preRules = []
+    self.postRules = []
+
+  def read_from(self, rules_file):
+    self.rules_file = rules_file
+    self.read_rules()
+    self.read_task.start(10, now=False)
+
+  def read_rules(self)
+    if not exists(self.rules_file):
+      self.clear()
+      return
+
+    # Only read if the rules file has been modified
+    try:
+      mtime = getmtime(self.rules_file)
+    except:
+      log.err("Failed to get mtime of %s" % self.rules_file)
+      return:
+    if mtime <= self.rules_last_read:
+      return
 
-  def read_from(self, path):
     pre = []
     post = []
 
     section = None
-    for line in open(path):
+    for line in open(self.rules_file):
       line = line.strip()
       if line.startswith('#') or not line:
         continue
@@ -31,6 +59,7 @@ def read_from(self, path):
 
     self.preRules = pre
     self.postRules = post
+    self.rules_last_read = mtime
 
 
 class RewriteRule:

lib/carbon/writer.py

@@ -170,14 +170,24 @@ def reloadStorageSchemas():
     log.msg("Failed to reload storage schemas")
     log.err()
 
+def reloadAggregationStorageSchemas():
+  global agg_schemas
+  try:
+    schemas = loadAggregationSchemas()
+  except:
+    log.msg("Failed to reload aggregation schemas")
+    log.err()
+
 
 class WriterService(Service):
 
     def __init__(self):
-        self.reload_task = LoopingCall(reloadStorageSchemas)
+        self.storage_reload_task = LoopingCall(reloadStorageSchemas)
+        self.aggregation_reload_task = LoopingCall(reloadAggregationSchemas)
 
     def startService(self):
-        self.reload_task.start(60, False)
+        self.storage_reload_task.start(60, False)
+        self.aggregation_reload_task.start(60, False)
         reactor.callInThread(writeForever)
         Service.startService(self)