diff --git a/.gitignore b/.gitignore index 23678671..ffd4c9a5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,10 @@ *.pyc *.swp *.log -settings.py dump.rdb nohup.out +/build +/dist +*.egg +/nosetests.xml +*.egg-info diff --git a/.travis.yml b/.travis.yml index 4da68121..7fbb4c36 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,10 +8,9 @@ install: - PYTHONPATH= PATH=/home/travis/anaconda/bin:$PATH pip install -r requirements.txt --use-mirrors - PYTHONPATH= PATH=/home/travis/anaconda/bin:$PATH pip install patsy --use-mirrors - PYTHONPATH= PATH=/home/travis/anaconda/bin:$PATH pip install msgpack_python --use-mirrors - - cp src/settings.py.example src/settings.py - pip install pep8 --use-mirrors script: - - PYTHONPATH= PATH=/home/travis/anaconda/bin:$PATH nosetests -v --nocapture + - PYTHONPATH= python setup.py test - pep8 --exclude=migrations --ignore=E501,E251,E265 ./ notifications: email: false diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..69a25ac8 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +recursive-include src/skyline/webapp/static * +recursive-include src/skyline/webapp/templates * diff --git a/bin/analyzer.d b/bin/analyzer.d index ba825f67..94249c1b 100755 --- a/bin/analyzer.d +++ b/bin/analyzer.d @@ -5,9 +5,17 @@ BASEDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.." RETVAL=0 +# Determine the script to run depending on whether this script is in the source +# tree or installed +SCRIPT="${BASEDIR}/bin/analyzer-agent" +if [ -e "${BASEDIR}/setup.py" ] +then + SCRIPT="${BASEDIR}/src/skyline/analyzer/agent.py" +fi + start () { rm -f $BASEDIR/src/analyzer/*.pyc - /usr/bin/env python $BASEDIR/src/analyzer/analyzer-agent.py start + /usr/bin/env python $SCRIPT start RETVAL=$? if [[ $RETVAL -eq 0 ]]; then echo "started analyzer-agent" @@ -20,7 +28,7 @@ start () { stop () { # TODO: write a real kill script ps aux | grep 'analyzer-agent.py start' | grep -v grep | awk '{print $2 }' | xargs sudo kill -9 - /usr/bin/env python $BASEDIR/src/analyzer/analyzer-agent.py stop + /usr/bin/env python $SCRIPT stop RETVAL=$? if [[ $RETVAL -eq 0 ]]; then echo "stopped analyzer-agent" @@ -32,7 +40,7 @@ stop () { run () { echo "running analyzer" - /usr/bin/env python $BASEDIR/src/analyzer/analyzer-agent.py run + /usr/bin/env python $SCRIPT run } # See how we were called. diff --git a/bin/horizon.d b/bin/horizon.d index 89e0afea..714fea3e 100755 --- a/bin/horizon.d +++ b/bin/horizon.d @@ -5,9 +5,17 @@ BASEDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.." RETVAL=0 +# Determine the script to run depending on whether this script is in the source +# tree or installed +SCRIPT="${BASEDIR}/bin/horizon-agent" +if [ -e "${BASEDIR}/setup.py" ] +then + SCRIPT="${BASEDIR}/src/skyline/horizon/agent.py" +fi + start () { rm $BASEDIR/src/horizon/*.pyc - /usr/bin/env python $BASEDIR/src/horizon/horizon-agent.py start + /usr/bin/env python $SCRIPT start RETVAL=$? if [[ $RETVAL -eq 0 ]]; then echo "started horizon-agent" @@ -20,7 +28,7 @@ start () { stop () { # TODO: write a real kill script ps aux | grep 'horizon-agent.py start' | grep -v grep | awk '{print $2 }' | xargs sudo kill -9 - /usr/bin/env python $BASEDIR/src/horizon/horizon-agent.py stop + /usr/bin/env python $SCRIPT stop RETVAL=$? if [[ $RETVAL -eq 0 ]]; then echo "stopped horizon-agent" @@ -32,7 +40,7 @@ stop () { run () { echo "running horizon" - /usr/bin/env python $BASEDIR/src/horizon/horizon-agent.py run + /usr/bin/env python $SCRIPT run } # See how we were called. diff --git a/bin/webapp.d b/bin/webapp.d index 117f980d..d130ba9d 100755 --- a/bin/webapp.d +++ b/bin/webapp.d @@ -5,9 +5,17 @@ BASEDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.." RETVAL=0 +# Determine the script to run depending on whether this script is in the source +# tree or installed +SCRIPT="${BASEDIR}/bin/skyline-webapp" +if [ -e "${BASEDIR}/setup.py" ] +then + SCRIPT="${BASEDIR}/src/skyline/webapp/webapp.py" +fi + start () { rm -f $BASEDIR/src/webapp/*.pyc - /usr/bin/env python $BASEDIR/src/webapp/webapp.py start + /usr/bin/env python $SCRIPT start RETVAL=$? if [[ $RETVAL -eq 0 ]]; then echo "started webapp" @@ -18,7 +26,7 @@ start () { } stop () { - /usr/bin/env python $BASEDIR/src/webapp/webapp.py stop + /usr/bin/env python $SCRIPT stop RETVAL=$? if [[ $RETVAL -eq 0 ]]; then echo "stopped webapp" @@ -30,7 +38,7 @@ stop () { restart () { rm -f $BASEDIR/src/webapp/*.pyc - /usr/bin/env python $BASEDIR/src/webapp/webapp.py restart + /usr/bin/env python $SCRIPT restart RETVAL=$? if [[ $RETVAL -eq 0 ]]; then echo "restarted webapp" @@ -42,7 +50,7 @@ restart () { run () { echo "running webapp" - /usr/bin/env python $BASEDIR/src/webapp/webapp.py run + /usr/bin/env python $SCRIPT run } # See how we were called. diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..19bb1cd6 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[nosetests] +tests = tests diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..1fd96727 --- /dev/null +++ b/setup.py @@ -0,0 +1,49 @@ +from setuptools import setup, find_packages + +# work around a nose test bug: http://bugs.python.org/issue15881#msg170215 +try: + import multiprocessing +except ImportError: + pass + +setup(name='skyline', + version='0.1.0', + description=''' + It'll detect your anomalies! + ''', + author='etsy', + author_email='', + license='MIT', + url='https://github.com/etsy/skyline', + keywords=['anomaly detection', 'timeseries', 'monitoring'], + classifiers=['Programming Language :: Python'], + + setup_requires=['nose>=1.0', 'unittest2', 'mock'], + install_requires=['redis==2.7.2', + 'hiredis==0.1.1', + 'python-daemon==1.6', + 'flask==0.9', + 'simplejson==2.0.9', + 'numpy', + 'scipy', + 'pandas', + 'patsy', + 'statsmodels', + 'msgpack_python'], + + packages=find_packages('src'), + package_dir={'': 'src'}, + include_package_data=True, + + entry_points={ + 'console_scripts': [ + 'analyzer-agent = skyline.analyzer.agent:run', + 'horizon-agent = skyline.horizon.agent:run', + 'skyline-webapp = skyline.webapp.webapp:run' + ] + }, + + scripts=['bin/analyzer.d', 'bin/horizon.d', 'bin/webapp.d'], + + test_suite='nose.collector', + ) diff --git a/src/skyline/__init__.py b/src/skyline/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/skyline/analyzer/__init__.py b/src/skyline/analyzer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/analyzer/analyzer-agent.py b/src/skyline/analyzer/agent.py similarity index 83% rename from src/analyzer/analyzer-agent.py rename to src/skyline/analyzer/agent.py index 67aac11a..47b9ae96 100644 --- a/src/analyzer/analyzer-agent.py +++ b/src/skyline/analyzer/agent.py @@ -2,15 +2,14 @@ import sys import traceback from os import getpid -from os.path import dirname, abspath, isdir +from os.path import isdir from daemon import runner from time import sleep, time -# add the shared settings file to namespace -sys.path.insert(0, dirname(dirname(abspath(__file__)))) -import settings +from skyline import settings +from skyline.analyzer.analyzer import Analyzer -from analyzer import Analyzer +logger = logging.getLogger("AnalyzerLog") class AnalyzerAgent(): @@ -28,7 +27,8 @@ def run(self): while 1: sleep(100) -if __name__ == "__main__": + +def run(): """ Start the Analyzer agent. """ @@ -42,9 +42,9 @@ def run(self): # Make sure we can run all the algorithms try: - from algorithms import * + from skyline.analyzer import algorithms timeseries = map(list, zip(map(float, range(int(time()) - 86400, int(time()) + 1)), [1] * 86401)) - ensemble = [globals()[algorithm](timeseries) for algorithm in settings.ALGORITHMS] + ensemble = [getattr(algorithms, algorithm)(timeseries) for algorithm in settings.ALGORITHMS] except KeyError as e: print "Algorithm %s deprecated or not defined; check settings.ALGORITHMS" % e sys.exit(1) @@ -55,7 +55,6 @@ def run(self): analyzer = AnalyzerAgent() - logger = logging.getLogger("AnalyzerLog") logger.setLevel(logging.DEBUG) formatter = logging.Formatter("%(asctime)s :: %(process)s :: %(message)s", datefmt="%Y-%m-%d %H:%M:%S") handler = logging.FileHandler(settings.LOG_PATH + '/analyzer.log') @@ -68,3 +67,6 @@ def run(self): daemon_runner = runner.DaemonRunner(analyzer) daemon_runner.daemon_context.files_preserve = [handler.stream] daemon_runner.do_action() + +if __name__ == "__main__": + run() diff --git a/src/analyzer/alerters.py b/src/skyline/analyzer/alerters.py similarity index 98% rename from src/analyzer/alerters.py rename to src/skyline/analyzer/alerters.py index 04f42096..74ecb3ec 100644 --- a/src/analyzer/alerters.py +++ b/src/skyline/analyzer/alerters.py @@ -3,7 +3,7 @@ from email.MIMEImage import MIMEImage from smtplib import SMTP import alerters -import settings +from skyline import settings """ diff --git a/src/analyzer/algorithm_exceptions.py b/src/skyline/analyzer/algorithm_exceptions.py similarity index 100% rename from src/analyzer/algorithm_exceptions.py rename to src/skyline/analyzer/algorithm_exceptions.py diff --git a/src/analyzer/algorithms.py b/src/skyline/analyzer/algorithms.py similarity index 99% rename from src/analyzer/algorithms.py rename to src/skyline/analyzer/algorithms.py index 8ee5c80f..a857fc5a 100644 --- a/src/analyzer/algorithms.py +++ b/src/skyline/analyzer/algorithms.py @@ -8,7 +8,7 @@ from msgpack import unpackb, packb from redis import StrictRedis -from settings import ( +from skyline.settings import ( ALGORITHMS, CONSENSUS, FULL_DURATION, diff --git a/src/analyzer/analyzer.py b/src/skyline/analyzer/analyzer.py similarity index 99% rename from src/analyzer/analyzer.py rename to src/skyline/analyzer/analyzer.py index 51bf4d95..26bba86b 100644 --- a/src/analyzer/analyzer.py +++ b/src/skyline/analyzer/analyzer.py @@ -11,7 +11,7 @@ import traceback import operator import socket -import settings +from skyline import settings from alerters import trigger_alert from algorithms import run_selected_algorithm diff --git a/src/skyline/horizon/__init__.py b/src/skyline/horizon/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/horizon/horizon-agent.py b/src/skyline/horizon/agent.py similarity index 94% rename from src/horizon/horizon-agent.py rename to src/skyline/horizon/agent.py index 65c88a4e..975c3485 100644 --- a/src/horizon/horizon-agent.py +++ b/src/skyline/horizon/agent.py @@ -6,9 +6,7 @@ from multiprocessing import Queue from daemon import runner -# add the shared settings file to namespace -sys.path.insert(0, dirname(dirname(abspath(__file__)))) -import settings +from skyline import settings from listen import Listen from roomba import Roomba @@ -16,6 +14,7 @@ # TODO: http://stackoverflow.com/questions/6728236/exception-thrown-in-multiprocessing-pool-not-detected +logger = logging.getLogger("HorizonLog") class Horizon(): def __init__(self): @@ -60,7 +59,8 @@ def run(self): while 1: time.sleep(100) -if __name__ == "__main__": + +def run(): """ Start the Horizon agent. """ @@ -74,7 +74,6 @@ def run(self): horizon = Horizon() - logger = logging.getLogger("HorizonLog") logger.setLevel(logging.DEBUG) formatter = logging.Formatter("%(asctime)s :: %(process)s :: %(message)s", datefmt="%Y-%m-%d %H:%M:%S") handler = logging.FileHandler(settings.LOG_PATH + '/horizon.log') @@ -87,3 +86,6 @@ def run(self): daemon_runner = runner.DaemonRunner(horizon) daemon_runner.daemon_context.files_preserve = [handler.stream] daemon_runner.do_action() + +if __name__ == "__main__": + run() diff --git a/src/horizon/listen.py b/src/skyline/horizon/listen.py similarity index 98% rename from src/horizon/listen.py rename to src/skyline/horizon/listen.py index cf4afd2c..c9e88642 100644 --- a/src/horizon/listen.py +++ b/src/skyline/horizon/listen.py @@ -6,7 +6,7 @@ from msgpack import unpackb import logging -import settings +from skyline import settings logger = logging.getLogger("HorizonLog") @@ -18,6 +18,11 @@ import pickle USING_CPICKLE = False +try: + from cStringIO import StringIO +except: + from StringIO import StringIO + # This whole song & dance is due to pickle being insecure # yet performance critical for carbon. We leave the insecure # mode (which is faster) as an option (USE_INSECURE_UNPICKLER). diff --git a/src/horizon/roomba.py b/src/skyline/horizon/roomba.py similarity index 99% rename from src/horizon/roomba.py rename to src/skyline/horizon/roomba.py index 3c5e466e..9fffe2ab 100644 --- a/src/horizon/roomba.py +++ b/src/skyline/horizon/roomba.py @@ -7,7 +7,7 @@ from time import time, sleep import logging -import settings +from skyline import settings logger = logging.getLogger("HorizonLog") diff --git a/src/horizon/worker.py b/src/skyline/horizon/worker.py similarity index 99% rename from src/horizon/worker.py rename to src/skyline/horizon/worker.py index 6bdd44ff..dcc41ce3 100644 --- a/src/horizon/worker.py +++ b/src/skyline/horizon/worker.py @@ -7,7 +7,7 @@ import logging import socket -import settings +from skyline import settings logger = logging.getLogger("HorizonLog") diff --git a/src/settings.py.example b/src/skyline/settings.py similarity index 100% rename from src/settings.py.example rename to src/skyline/settings.py diff --git a/src/skyline/webapp/__init__.py b/src/skyline/webapp/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/webapp/static/css/bootstrap.min.css b/src/skyline/webapp/static/css/bootstrap.min.css similarity index 100% rename from src/webapp/static/css/bootstrap.min.css rename to src/skyline/webapp/static/css/bootstrap.min.css diff --git a/src/webapp/static/css/font-awesome.min.css b/src/skyline/webapp/static/css/font-awesome.min.css similarity index 100% rename from src/webapp/static/css/font-awesome.min.css rename to src/skyline/webapp/static/css/font-awesome.min.css diff --git a/src/webapp/static/css/skyline.css b/src/skyline/webapp/static/css/skyline.css similarity index 100% rename from src/webapp/static/css/skyline.css rename to src/skyline/webapp/static/css/skyline.css diff --git a/src/webapp/static/dump/.gitignore b/src/skyline/webapp/static/dump/.gitignore similarity index 100% rename from src/webapp/static/dump/.gitignore rename to src/skyline/webapp/static/dump/.gitignore diff --git a/src/webapp/static/font/FontAwesome.otf b/src/skyline/webapp/static/font/FontAwesome.otf similarity index 100% rename from src/webapp/static/font/FontAwesome.otf rename to src/skyline/webapp/static/font/FontAwesome.otf diff --git a/src/webapp/static/font/fontawesome-webfont.eot b/src/skyline/webapp/static/font/fontawesome-webfont.eot similarity index 100% rename from src/webapp/static/font/fontawesome-webfont.eot rename to src/skyline/webapp/static/font/fontawesome-webfont.eot diff --git a/src/webapp/static/font/fontawesome-webfont.svg b/src/skyline/webapp/static/font/fontawesome-webfont.svg similarity index 100% rename from src/webapp/static/font/fontawesome-webfont.svg rename to src/skyline/webapp/static/font/fontawesome-webfont.svg diff --git a/src/webapp/static/font/fontawesome-webfont.ttf b/src/skyline/webapp/static/font/fontawesome-webfont.ttf similarity index 100% rename from src/webapp/static/font/fontawesome-webfont.ttf rename to src/skyline/webapp/static/font/fontawesome-webfont.ttf diff --git a/src/webapp/static/font/fontawesome-webfont.woff b/src/skyline/webapp/static/font/fontawesome-webfont.woff similarity index 100% rename from src/webapp/static/font/fontawesome-webfont.woff rename to src/skyline/webapp/static/font/fontawesome-webfont.woff diff --git a/src/webapp/static/js/dygraph-combined.js b/src/skyline/webapp/static/js/dygraph-combined.js similarity index 100% rename from src/webapp/static/js/dygraph-combined.js rename to src/skyline/webapp/static/js/dygraph-combined.js diff --git a/src/webapp/static/js/jquery.min.js b/src/skyline/webapp/static/js/jquery.min.js similarity index 100% rename from src/webapp/static/js/jquery.min.js rename to src/skyline/webapp/static/js/jquery.min.js diff --git a/src/webapp/static/js/mousetrap.min.js b/src/skyline/webapp/static/js/mousetrap.min.js similarity index 100% rename from src/webapp/static/js/mousetrap.min.js rename to src/skyline/webapp/static/js/mousetrap.min.js diff --git a/src/webapp/static/js/skyline.js b/src/skyline/webapp/static/js/skyline.js similarity index 99% rename from src/webapp/static/js/skyline.js rename to src/skyline/webapp/static/js/skyline.js index 2c3ca93e..d01765c0 100644 --- a/src/webapp/static/js/skyline.js +++ b/src/skyline/webapp/static/js/skyline.js @@ -38,7 +38,7 @@ var handle_data = function(data) { // The callback to this function is handle_data() var pull_data = function() { $.ajax({ - url: "/static/dump/anomalies.json", + url: "/anomalies.json", dataType: 'jsonp' }); } diff --git a/src/webapp/templates/index.html b/src/skyline/webapp/templates/index.html similarity index 100% rename from src/webapp/templates/index.html rename to src/skyline/webapp/templates/index.html diff --git a/src/webapp/webapp.py b/src/skyline/webapp/webapp.py similarity index 92% rename from src/webapp/webapp.py rename to src/skyline/webapp/webapp.py index 6751d761..33b2b0d9 100644 --- a/src/webapp/webapp.py +++ b/src/skyline/webapp/webapp.py @@ -7,21 +7,27 @@ from daemon import runner from os.path import dirname, abspath -# add the shared settings file to namespace -sys.path.insert(0, dirname(dirname(abspath(__file__)))) -import settings +from skyline import settings REDIS_CONN = redis.StrictRedis(unix_socket_path=settings.REDIS_SOCKET_PATH) app = Flask(__name__) app.config['PROPAGATE_EXCEPTIONS'] = True +logger = logging.getLogger("AppLog") + @app.route("/") def index(): return render_template('index.html'), 200 +@app.route("/anomalies.json") +def anomalies(): + with open(settings.ANOMALY_DUMP, 'r') as f: + return f.read(), 200 + + @app.route("/app_settings") def app_settings(): @@ -70,14 +76,14 @@ def run(self): app.run(settings.WEBAPP_IP, settings.WEBAPP_PORT) -if __name__ == "__main__": + +def run(): """ Start the server """ webapp = App() - logger = logging.getLogger("AppLog") logger.setLevel(logging.DEBUG) formatter = logging.Formatter("%(asctime)s :: %(message)s", datefmt="%Y-%m-%d %H:%M:%S") handler = logging.FileHandler(settings.LOG_PATH + '/webapp.log') @@ -90,3 +96,6 @@ def run(self): daemon_runner = runner.DaemonRunner(webapp) daemon_runner.daemon_context.files_preserve = [handler.stream] daemon_runner.do_action() + +if __name__ == "__main__": + run() diff --git a/tests/algorithms_test.py b/tests/algorithms_test.py index 30b37b4c..2487892b 100644 --- a/tests/algorithms_test.py +++ b/tests/algorithms_test.py @@ -2,14 +2,8 @@ from mock import Mock, patch from time import time -import sys -from os.path import dirname, abspath - -sys.path.insert(0, dirname(dirname(abspath(__file__))) + '/src') -sys.path.insert(0, dirname(dirname(abspath(__file__))) + '/src/analyzer') - -import algorithms -import settings +from skyline.analyzer import algorithms +from skyline import settings class TestAlgorithms(unittest.TestCase): diff --git a/utils/continuity.py b/utils/continuity.py index c25ef12c..99b5ff5e 100644 --- a/utils/continuity.py +++ b/utils/continuity.py @@ -6,7 +6,7 @@ # add the shared settings file to namespace sys.path.insert(0, ''.join((dirname(dirname(abspath(__file__))), "/src"))) -import settings +from skyline import settings metric = 'horizon.test.udp' diff --git a/utils/seed_data.py b/utils/seed_data.py index 087e39a1..19341d53 100755 --- a/utils/seed_data.py +++ b/utils/seed_data.py @@ -19,7 +19,7 @@ # Add the shared settings file to namespace. sys.path.insert(0, join(__location__, '..', 'src')) -import settings +from skyline import settings class NoDataException(Exception): diff --git a/utils/verify_alerts.py b/utils/verify_alerts.py index cc631b82..6a8774f8 100644 --- a/utils/verify_alerts.py +++ b/utils/verify_alerts.py @@ -11,11 +11,10 @@ # Add the shared settings file to namespace. sys.path.insert(0, join(__location__, '..', 'src')) -import settings +from skyline import settings # Add the analyzer file to namespace. -sys.path.insert(0, join(__location__, '..', 'src', 'analyzer')) -from alerters import trigger_alert +from skyline.analyzer.alerters import trigger_alert parser = OptionParser() parser.add_option("-t", "--trigger", dest="trigger", default=False,