Modified data analyzer and implemented shared_config file to avoid duplication

bot/market_data_analyzer.py

@@ -1,16 +1,64 @@
 #!/usr/bin/python
 from . import market_data_crawler
+from bot.shared_config import *
 
 import time
+import sys
 import operator
+import pprint
 
+def calculate_arbitrage_opportunities(exchanges):
+    market_data = market_data_crawler.update_market_data_for_symbol_and_exchange(exchanges)
+    sorted_market_data = {}
+
+    for exchange_name, order_books in market_data.items():
+        for order_book in order_books:
+            symbol = order_book['symbol']
+            new_dictionary = {symbol:
+                                  {exchange_name:
+                                       {"bids": order_book['bids'][:5],
+                                        "asks": order_book['asks'][:5],
+                                        "timestamp": order_book['timestamp']}}}
+            if symbol not in sorted_market_data.keys():
+                sorted_market_data.update(new_dictionary)
+            else:
+                sorted_market_data[symbol].update(new_dictionary[symbol])
+
+    dump(green(str(len(sorted_market_data))), "possible symbols found in total:", ' '.join(sorted_market_data.keys()))
 
-def calculate_arbitrage_opportunities(basecoin):
-    market_data = market_data_crawler.market_data
-    print("Total currency pairs: {0}".format(len(market_data)))
     market_opport = {}
+    for symbol, exchanges in sorted_market_data.items():
+        lowest_ask = None
+        highest_bid = None
+        market_opport.update({symbol: {}})
+        for exchange_name, order_book in exchanges.items():
+
+            """""""""
+            TODO: This is wrong - you have to calculate the biggest spread!!!
+            """""""""
+
+            if lowest_ask is None or lowest_ask['value'] < order_book['asks'][0]:
+                lowest_ask = {"exchange_name":exchange_name,
+                               "value":order_book['asks'][0],
+                               "order_book": order_book['asks'][:3]}
+
+            if highest_bid is None or highest_bid['value'] > order_book['bids'][0]:
+                highest_bid = {"exchange_name": exchange_name,
+                               "value": order_book['bids'][0],
+                               "order_book": order_book['bids'][:3]}
+
+        spread = float(highest_bid['value'][0]) - float(lowest_ask['value'][0])
+
+        market_opport[symbol].update({"highest_bid": highest_bid,
+                                      "lowest_ask": lowest_ask,
+                                      "spread": spread})
+
+    with open("market_analyzation.txt", "w") as file:
+        pprint.pprint(market_opport, stream=file)
+
+    return
+
 
-    print("--- Calculating arbitrage oportunities now ---")
     start_time = time.time()
 
     for pair in market_data:
@@ -26,7 +74,7 @@ def calculate_arbitrage_opportunities(basecoin):
             spread_perc = round((spread / sorted_x[0][1]) * 100, 2)
 
             market_opport[pair] = {"highest_market": sorted_x[0][0],
-                                   "highest_price:": sorted_x[0][1],
+                                   "highest_price": sorted_x[0][1],
                                    "lowest_market": sorted_x[-1][0],
                                    "lowest_price": sorted_x[-1][1],
                                    "spread": spread,
@@ -38,3 +86,6 @@ def calculate_arbitrage_opportunities(basecoin):
     print("--- Arbitrage oportunities calculated in {0:.3f}ms ---".format((time.time() - start_time)*100))
 
     return market_opport
+
+if __name__ == '__main__':
+    calculate_arbitrage_opportunities(sys.argv[1:])

bot/market_data_crawler.py

@@ -1,7 +1,6 @@
 #!/usr/bin/python
-
-from tornado import gen
 from collections import defaultdict
+from bot.shared_config import *
 
 import sys
 import time
@@ -11,49 +10,7 @@ import ccxt.async as ccxt
 market_data = defaultdict(list)
 
 
-def style(s, style):
+def update_market_data_for_symbol_and_exchange(exchanges):
-    return style + s + '\033[0m'
-
-
-def green(s):
-    return style(s, '\033[92m')
-
-
-def blue(s):
-    return style(s, '\033[94m')
-
-
-def yellow(s):
-    return style(s, '\033[93m')
-
-
-def red(s):
-    return style(s, '\033[91m')
-
-
-def pink(s):
-    return style(s, '\033[95m')
-
-
-def bold(s):
-    return style(s, '\033[1m')
-
-
-def underline(s):
-    return style(s, '\033[4m')
-
-
-def dump(*args):
-    print(' '.join([str(arg) for arg in args]))
-
-proxies = [
-    '',  # no proxy by default
-    'https://cors-anywhere.herokuapp.com/',
-
-]
-
-
-def update_market_data_for_symbol_and_exchange(allowed_symbols, exchanges):
     if len(exchanges) > 1:
         start_time = time.time()
         ids = list(exchanges)
@@ -77,16 +34,19 @@ def update_market_data_for_symbol_and_exchange(allowed_symbols, exchanges):
         # filter out symbols that are not present on at least two exchanges
         arbitrableSymbols = sorted([symbol for symbol in uniqueSymbols if allSymbols.count(symbol) > 1])
 
+        # filter out symbols which have a different basecoin
+        arbitrableSymbols = sorted([symbol for symbol in arbitrableSymbols if '/'+basecoin in symbol])
+
         dump(yellow('Loading'), 'order books for following exchanges:', ' '.join(ids))
         exchanges = fetch_all_order_books(exchanges, arbitrableSymbols)
 
         dump(green('Finished!'), 'Responsetime:', red("{:.2f}ms".format((time.time() - start_time) * 100)))
 
         with open("market_data.txt", "w") as file:
-            for key, value in market_data.items():
+            for exchange_name, order_books in market_data.items():
-                file.write("\nMarket: {}".format(key))
+                file.write("\nMarket: {}".format(exchange_name))
 
-                for order_book in value:
+                for order_book in order_books:
                     file.write("\n    Order Book: {0}".format(order_book))
 
         return market_data
@@ -138,9 +98,9 @@ def fetch_all_order_books(exchanges, arbitrableSymbols):
         market_data[exchange.id] = order_books
 
     async_executor = []
-    for key, value in exchanges.items():
+    for exchange_name, exchange in exchanges.items():
         # add future to list
-        async_executor.append(asyncio.ensure_future(fetch_single_order_books(exchanges[key], arbitrableSymbols)))
+        async_executor.append(asyncio.ensure_future(fetch_single_order_books(exchange, arbitrableSymbols)))
 
     # wait till all futures in list completed
     asyncio.get_event_loop().run_until_complete(asyncio.gather(*async_executor))
@@ -183,9 +143,9 @@ def fetch_all_markets(exchanges):
         dump(' ', green(exchange.id), 'loaded', green(str(len(exchange.symbols))), 'markets')
 
     async_executor = []
-    for key, value in exchanges.items():
+    for exchange_name, exchange in exchanges.items():
         # add future to list
-        async_executor.append(asyncio.ensure_future(fetch_single_market(exchanges[key])))
+        async_executor.append(asyncio.ensure_future(fetch_single_market(exchange)))
 
     # wait till all futures in list completed
     asyncio.get_event_loop().run_until_complete(asyncio.gather(*async_executor))
@@ -196,4 +156,4 @@ def fetch_all_markets(exchanges):
 
 
 if __name__ == '__main__':
-    update_market_data_for_symbol_and_exchange(None, sys.argv[1:])
+    update_market_data_for_symbol_and_exchange(sys.argv[1:])

bot/shared_config.py

@@ -0,0 +1,42 @@
+#!/usr/bin/python
+def style(s, style):
+    return style + s + '\033[0m'
+
+
+def green(s):
+    return style(s, '\033[92m')
+
+
+def blue(s):
+    return style(s, '\033[94m')
+
+
+def yellow(s):
+    return style(s, '\033[93m')
+
+
+def red(s):
+    return style(s, '\033[91m')
+
+
+def pink(s):
+    return style(s, '\033[95m')
+
+
+def bold(s):
+    return style(s, '\033[1m')
+
+
+def underline(s):
+    return style(s, '\033[4m')
+
+
+def dump(*args):
+    print(' '.join([str(arg) for arg in args]))
+
+proxies = [
+    '',  # no proxy by default
+    'https://cors-anywhere.herokuapp.com/',
+]
+
+basecoin = "ETH"