Merge pull request #2 from s4w3d0ff/dev

v0.0.3

donnie/poloapi.py

@@ -1,12 +1,16 @@
 import poloniex
-from .tools import getDatabase, getLogger, TA
+from .tools import (getDatabase, getLogger, zoomOHLC, addIndicators,
+                    getChartDataFrame, updateChartData, getLastEntry,
+                    UTCstr2epoch, epoch2UTCstr)
 
 logger = getLogger(__name__)
 
 
 class Poloniex(poloniex.PoloniexSocketed):
     def __init__(self, *args, **kwargs):
         super(Poloniex, self).__init__(*args, **kwargs)
+        if not 'jsonNums' in kwargs:
+            self.jsonNums = float
         self.db = getDatabase('poloniex')
         # holds stop orders
         self.stopOrders = {}
@@ -124,3 +128,74 @@ def cbck(self, id):
         Example callback for stop orders
         """
         print(self.stopOrders[id])
+
+
+    def chartDataFrame(self, pair, frame=172800, zoom=False, indica=False):
+        """ returns chart data in a dataframe from mongodb, updates/fills the
+        data, the date column is the '_id' of each candle entry, and
+        the date column has been removed. Use 'frame' to restrict the amount
+        of data returned.
+        Example: 'frame=self.YEAR' will return last years data
+        """
+        dbcolName = pair.upper() + '-chart'
+
+        # get db collection
+        db = self.db[dbcolName]
+
+        # get last candle data
+        last = getLastEntry(db)
+
+        # no entrys found, get all 5min data from poloniex
+        if not last:
+            self.logger.warning('%s collection is empty!', dbcolName)
+            last = {
+                '_id': UTCstr2epoch("2015-01-01", fmat="%Y-%m-%d")
+                }
+
+        stop = int(last['_id'])
+        start = poloniex.time()
+        end = poloniex.time()
+        flag = True
+        while not int(stop) == int(start) and flag:
+            #
+            start = start - self.MONTH * 3
+
+            # dont go past 'stop'
+            if start < stop:
+                start = stop
+
+            # get needed data
+            self.logger.debug('Getting %s - %s %s candles from Poloniex...',
+                              epoch2UTCstr(start), epoch2UTCstr(end), pair)
+            new = self.returnChartData(pair,
+                                       period=60 * 5,
+                                       start=start,
+                                       end=end)
+
+            # stop if data has stopped comming in
+            if len(new) == 1:
+                flag = False
+
+            # add new candles
+            self.logger.debug(
+                'Updating %s database with %s entrys...', pair, str(len(new))
+                )
+
+            updateChartData(db, new)
+
+            # make new end the old start
+            end = start
+
+        # make dataframe
+        self.logger.debug('Getting %s chart data from db', pair)
+        df = getChartDataFrame(db, poloniex.time() - frame)
+
+        # adjust candle period 'zoom'
+        if zoom:
+            df = zoomOHLC(df, zoom)
+
+        # add TA indicators
+        if indica:
+            df = addIndicators(df, indica)
+
+        return df

donnie/tools.py

@@ -36,6 +36,7 @@
 import pandas as pd
 import pymongo
 from finta import TA
+import tqdm
 
 getLogger = logging.getLogger
 
@@ -117,11 +118,61 @@ def shuffleDataFrame(df):
     del df['index']
     return df.reindex(np.random.permutation(df.index))
 
+def addIndicators(df, conf={}):
+    """ Adds indicators to a ohlc df using 'finta.TA' """
+    avail = dir(TA)
+    for ind in conf:
+        if ind in avail:
+            df = pd.concat(
+                [getattr(TA, ind)(ohlc=df, **conf[ind]), df],
+                axis=1
+                )
+    return df
+
+def zoomOHLC(df, zoom):
+    """ Resamples a ohlc df """
+    df.reset_index(inplace=True)
+    df.set_index('date', inplace=True)
+    df = df.resample(rule=zoom,
+                     closed='left',
+                     label='left').apply({'_id': 'first',
+                                          'open': 'first',
+                                          'high': 'max',
+                                          'low': 'min',
+                                          'close': 'last',
+                                          'quoteVolume': 'sum',
+                                          'volume': 'sum',
+                                          'weightedAverage': 'mean'})
+    df.reset_index(inplace=True)
+    return df.set_index('_id')
 
 def getDatabase(db):
     """ Returns a mongodb database """
     return DB[db]
 
+def getLastEntry(db):
+    """ Get the last entry of a collection """
+    return db.find_one(sort=[('_id', pymongo.DESCENDING)])
+
+def updateChartData(db, data):
+    """ Upserts chart data into db with a tqdm wrapper. """
+    for i in tqdm.trange(len(data)):
+        db.update_one({'_id': data[i]['date']}, {
+                      "$set": data[i]}, upsert=True)
+
+def getChartDataFrame(db, start):
+    """
+    Gets the last collection entrys starting from 'start' and puts them in a df
+    """
+    try:
+        df = pd.DataFrame(list(db.find({"_id": {"$gt": start}})))
+        # set date column to datetime
+        df['date'] = pd.to_datetime(df["_id"], unit='s')
+        df.set_index('_id', inplace=True)
+        return df
+    except Exception as e:
+        logger.exception(e)
+        return False
 
 def wait(i=10):
     """ Wraps 'time.sleep()' with logger output """

setup.py

@@ -26,7 +26,7 @@
 read_md = lambda f: open(f, 'r').read()
 
 setup(name='donnie',
-      version='0.0.2',
+      version='0.0.3',
       description='Poloniex tradebot toolkit',
       long_description=read_md('README.md'),
       long_description_content_type='text/markdown',
@@ -35,7 +35,7 @@
       license='GPL v3',
       packages=['donnie'],
       setup_requires=['pandas', 'numpy', 'scipy'],
-      install_requires=['pymongo', 'poloniexapi', 'finta'],
+      install_requires=['pymongo', 'poloniexapi', 'finta', 'tqdm'],
       zip_safe=False,
       keywords=['poloniex', 'poloniexapi', 'exchange',
                 'api', 'tradebot', 'framework'],