From ccd1d53715a8981b23ed56c335dd26e71bd3e1ac Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Wed, 18 Oct 2017 16:27:16 +0200 Subject: [PATCH] Modified data analyzer and implemented shared_config file to avoid duplication --- bot/market_data_analyzer.py | 61 +++++++++++++++++++++++++++++++--- bot/market_data_crawler.py | 66 ++++++++----------------------------- bot/shared_config.py | 42 +++++++++++++++++++++++ 3 files changed, 111 insertions(+), 58 deletions(-) create mode 100644 bot/shared_config.py diff --git a/bot/market_data_analyzer.py b/bot/market_data_analyzer.py index 97bba07..4538480 100644 --- a/bot/market_data_analyzer.py +++ b/bot/market_data_analyzer.py @@ -1,16 +1,64 @@ #!/usr/bin/python from . import market_data_crawler +from bot.shared_config import * import time +import sys import operator +import pprint +def calculate_arbitrage_opportunities(exchanges): + market_data = market_data_crawler.update_market_data_for_symbol_and_exchange(exchanges) + sorted_market_data = {} + + for exchange_name, order_books in market_data.items(): + for order_book in order_books: + symbol = order_book['symbol'] + new_dictionary = {symbol: + {exchange_name: + {"bids": order_book['bids'][:5], + "asks": order_book['asks'][:5], + "timestamp": order_book['timestamp']}}} + if symbol not in sorted_market_data.keys(): + sorted_market_data.update(new_dictionary) + else: + sorted_market_data[symbol].update(new_dictionary[symbol]) + + dump(green(str(len(sorted_market_data))), "possible symbols found in total:", ' '.join(sorted_market_data.keys())) -def calculate_arbitrage_opportunities(basecoin): - market_data = market_data_crawler.market_data - print("Total currency pairs: {0}".format(len(market_data))) market_opport = {} + for symbol, exchanges in sorted_market_data.items(): + lowest_ask = None + highest_bid = None + market_opport.update({symbol: {}}) + for exchange_name, order_book in exchanges.items(): + + """"""""" + TODO: This is wrong - you have to calculate the biggest spread!!! + """"""""" + + if lowest_ask is None or lowest_ask['value'] < order_book['asks'][0]: + lowest_ask = {"exchange_name":exchange_name, + "value":order_book['asks'][0], + "order_book": order_book['asks'][:3]} + + if highest_bid is None or highest_bid['value'] > order_book['bids'][0]: + highest_bid = {"exchange_name": exchange_name, + "value": order_book['bids'][0], + "order_book": order_book['bids'][:3]} + + spread = float(highest_bid['value'][0]) - float(lowest_ask['value'][0]) + + market_opport[symbol].update({"highest_bid": highest_bid, + "lowest_ask": lowest_ask, + "spread": spread}) + + with open("market_analyzation.txt", "w") as file: + pprint.pprint(market_opport, stream=file) + + return + - print("--- Calculating arbitrage oportunities now ---") start_time = time.time() for pair in market_data: @@ -26,7 +74,7 @@ def calculate_arbitrage_opportunities(basecoin): spread_perc = round((spread / sorted_x[0][1]) * 100, 2) market_opport[pair] = {"highest_market": sorted_x[0][0], - "highest_price:": sorted_x[0][1], + "highest_price": sorted_x[0][1], "lowest_market": sorted_x[-1][0], "lowest_price": sorted_x[-1][1], "spread": spread, @@ -38,3 +86,6 @@ def calculate_arbitrage_opportunities(basecoin): print("--- Arbitrage oportunities calculated in {0:.3f}ms ---".format((time.time() - start_time)*100)) return market_opport + +if __name__ == '__main__': + calculate_arbitrage_opportunities(sys.argv[1:]) \ No newline at end of file diff --git a/bot/market_data_crawler.py b/bot/market_data_crawler.py index ecb7b79..e68e9a1 100644 --- a/bot/market_data_crawler.py +++ b/bot/market_data_crawler.py @@ -1,7 +1,6 @@ #!/usr/bin/python - -from tornado import gen from collections import defaultdict +from bot.shared_config import * import sys import time @@ -11,49 +10,7 @@ import ccxt.async as ccxt market_data = defaultdict(list) -def style(s, style): - return style + s + '\033[0m' - - -def green(s): - return style(s, '\033[92m') - - -def blue(s): - return style(s, '\033[94m') - - -def yellow(s): - return style(s, '\033[93m') - - -def red(s): - return style(s, '\033[91m') - - -def pink(s): - return style(s, '\033[95m') - - -def bold(s): - return style(s, '\033[1m') - - -def underline(s): - return style(s, '\033[4m') - - -def dump(*args): - print(' '.join([str(arg) for arg in args])) - -proxies = [ - '', # no proxy by default - 'https://cors-anywhere.herokuapp.com/', - -] - - -def update_market_data_for_symbol_and_exchange(allowed_symbols, exchanges): +def update_market_data_for_symbol_and_exchange(exchanges): if len(exchanges) > 1: start_time = time.time() ids = list(exchanges) @@ -77,16 +34,19 @@ def update_market_data_for_symbol_and_exchange(allowed_symbols, exchanges): # filter out symbols that are not present on at least two exchanges arbitrableSymbols = sorted([symbol for symbol in uniqueSymbols if allSymbols.count(symbol) > 1]) + # filter out symbols which have a different basecoin + arbitrableSymbols = sorted([symbol for symbol in arbitrableSymbols if '/'+basecoin in symbol]) + dump(yellow('Loading'), 'order books for following exchanges:', ' '.join(ids)) exchanges = fetch_all_order_books(exchanges, arbitrableSymbols) dump(green('Finished!'), 'Responsetime:', red("{:.2f}ms".format((time.time() - start_time) * 100))) with open("market_data.txt", "w") as file: - for key, value in market_data.items(): - file.write("\nMarket: {}".format(key)) + for exchange_name, order_books in market_data.items(): + file.write("\nMarket: {}".format(exchange_name)) - for order_book in value: + for order_book in order_books: file.write("\n Order Book: {0}".format(order_book)) return market_data @@ -138,9 +98,9 @@ def fetch_all_order_books(exchanges, arbitrableSymbols): market_data[exchange.id] = order_books async_executor = [] - for key, value in exchanges.items(): + for exchange_name, exchange in exchanges.items(): # add future to list - async_executor.append(asyncio.ensure_future(fetch_single_order_books(exchanges[key], arbitrableSymbols))) + async_executor.append(asyncio.ensure_future(fetch_single_order_books(exchange, arbitrableSymbols))) # wait till all futures in list completed asyncio.get_event_loop().run_until_complete(asyncio.gather(*async_executor)) @@ -183,9 +143,9 @@ def fetch_all_markets(exchanges): dump(' ', green(exchange.id), 'loaded', green(str(len(exchange.symbols))), 'markets') async_executor = [] - for key, value in exchanges.items(): + for exchange_name, exchange in exchanges.items(): # add future to list - async_executor.append(asyncio.ensure_future(fetch_single_market(exchanges[key]))) + async_executor.append(asyncio.ensure_future(fetch_single_market(exchange))) # wait till all futures in list completed asyncio.get_event_loop().run_until_complete(asyncio.gather(*async_executor)) @@ -196,4 +156,4 @@ def fetch_all_markets(exchanges): if __name__ == '__main__': - update_market_data_for_symbol_and_exchange(None, sys.argv[1:]) + update_market_data_for_symbol_and_exchange(sys.argv[1:]) diff --git a/bot/shared_config.py b/bot/shared_config.py new file mode 100644 index 0000000..b1012e5 --- /dev/null +++ b/bot/shared_config.py @@ -0,0 +1,42 @@ +#!/usr/bin/python +def style(s, style): + return style + s + '\033[0m' + + +def green(s): + return style(s, '\033[92m') + + +def blue(s): + return style(s, '\033[94m') + + +def yellow(s): + return style(s, '\033[93m') + + +def red(s): + return style(s, '\033[91m') + + +def pink(s): + return style(s, '\033[95m') + + +def bold(s): + return style(s, '\033[1m') + + +def underline(s): + return style(s, '\033[4m') + + +def dump(*args): + print(' '.join([str(arg) for arg in args])) + +proxies = [ + '', # no proxy by default + 'https://cors-anywhere.herokuapp.com/', +] + +basecoin = "ETH"