From ae8d75da25abd51b2d38731dd350484c2e93d93a Mon Sep 17 00:00:00 2001
From: Dennis Thiessen <det@vizrt.com>
Date: Tue, 10 Oct 2017 15:59:25 +0200
Subject: [PATCH] Implemented parallel async API calls and implemented support
 for bittrex and poloniex. Added missing files to commit

---
 .idea/workspace.xml        | 133 +++++++++++++++++++++----------------
 bot/core.py                |  26 ++++++--
 bot/market_data_crawler.py |  73 +++++++++++++++-----
 3 files changed, 150 insertions(+), 82 deletions(-)

diff --git a/.idea/workspace.xml b/.idea/workspace.xml
index d42df62..093d036 100644
--- a/.idea/workspace.xml
+++ b/.idea/workspace.xml
@@ -2,14 +2,10 @@
 <project version="4">
   <component name="ChangeListManager">
     <list default="true" id="6f37b47a-0ca7-46ea-a0fe-3607d36927b6" name="Default" comment="">
-      <change type="DELETED" beforePath="$PROJECT_DIR$/.idea/crypto-arbitrage-bot.iml" afterPath="" />
-      <change type="DELETED" beforePath="$PROJECT_DIR$/.idea/misc.xml" afterPath="" />
-      <change type="DELETED" beforePath="$PROJECT_DIR$/.idea/modules.xml" afterPath="" />
-      <change type="DELETED" beforePath="$PROJECT_DIR$/.idea/vcs.xml" afterPath="" />
-      <change type="DELETED" beforePath="$PROJECT_DIR$/README.md" afterPath="" />
-      <change type="DELETED" beforePath="$PROJECT_DIR$/__init__.py" afterPath="" />
-      <change type="DELETED" beforePath="$PROJECT_DIR$/__main__.py" afterPath="" />
       <change type="MODIFICATION" beforePath="$PROJECT_DIR$/.idea/workspace.xml" afterPath="$PROJECT_DIR$/.idea/workspace.xml" />
+      <change type="MODIFICATION" beforePath="$PROJECT_DIR$/bot/core.py" afterPath="$PROJECT_DIR$/bot/core.py" />
+      <change type="MODIFICATION" beforePath="$PROJECT_DIR$/bot/market_data_crawler.py" afterPath="$PROJECT_DIR$/bot/market_data_crawler.py" />
+      <change type="MODIFICATION" beforePath="$PROJECT_DIR$/requirements.txt" afterPath="$PROJECT_DIR$/requirements.txt" />
     </list>
     <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
     <option name="TRACKING_ENABLED" value="true" />
@@ -20,16 +16,6 @@
   </component>
   <component name="FileEditorManager">
     <leaf>
-      <file leaf-file-name="MANIFEST.in" pinned="false" current-in-tab="false">
-        <entry file="file://$PROJECT_DIR$/MANIFEST.in">
-          <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="0">
-              <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
-              <folding />
-            </state>
-          </provider>
-        </entry>
-      </file>
       <file leaf-file-name="Makefile" pinned="false" current-in-tab="false">
         <entry file="file://$PROJECT_DIR$/Makefile">
           <provider selected="true" editor-type-id="text-editor">
@@ -85,9 +71,11 @@
       <file leaf-file-name="core.py" pinned="false" current-in-tab="false">
         <entry file="file://$PROJECT_DIR$/bot/core.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="799">
-              <caret line="53" column="0" lean-forward="false" selection-start-line="53" selection-start-column="0" selection-end-line="53" selection-end-column="0" />
-              <folding />
+            <state relative-caret-position="0">
+              <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="17" />
+              <folding>
+                <element signature="e#18#51#0" expanded="true" />
+              </folding>
             </state>
           </provider>
         </entry>
@@ -102,11 +90,11 @@
           </provider>
         </entry>
       </file>
-      <file leaf-file-name="test_advanced.py" pinned="false" current-in-tab="true">
+      <file leaf-file-name="test_advanced.py" pinned="false" current-in-tab="false">
         <entry file="file://$PROJECT_DIR$/tests/test_advanced.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="102">
-              <caret line="6" column="0" lean-forward="true" selection-start-line="6" selection-start-column="0" selection-end-line="6" selection-end-column="0" />
+            <state relative-caret-position="272">
+              <caret line="16" column="0" lean-forward="false" selection-start-line="16" selection-start-column="0" selection-end-line="16" selection-end-column="0" />
               <folding>
                 <element signature="e#25#49#0" expanded="true" />
               </folding>
@@ -114,6 +102,28 @@
           </provider>
         </entry>
       </file>
+      <file leaf-file-name="market_data_crawler.py" pinned="false" current-in-tab="true">
+        <entry file="file://$PROJECT_DIR$/bot/market_data_crawler.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="269">
+              <caret line="22" column="11" lean-forward="true" selection-start-line="22" selection-start-column="11" selection-end-line="22" selection-end-column="11" />
+              <folding>
+                <element signature="e#19#60#0" expanded="true" />
+              </folding>
+            </state>
+          </provider>
+        </entry>
+      </file>
+      <file leaf-file-name="requirements.txt" pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/requirements.txt">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="34">
+              <caret line="2" column="7" lean-forward="true" selection-start-line="2" selection-start-column="7" selection-end-line="2" selection-end-column="7" />
+              <folding />
+            </state>
+          </provider>
+        </entry>
+      </file>
     </leaf>
   </component>
   <component name="FileTemplateManagerImpl">
@@ -134,10 +144,13 @@
         <option value="$PROJECT_DIR$/__main__.py" />
         <option value="$PROJECT_DIR$/README.rst" />
         <option value="$PROJECT_DIR$/LICENSE" />
-        <option value="$PROJECT_DIR$/bot/core.py" />
         <option value="$PROJECT_DIR$/bot/__init__.py" />
         <option value="$PROJECT_DIR$/tests/context.py" />
         <option value="$PROJECT_DIR$/tests/test_advanced.py" />
+        <option value="$PROJECT_DIR$/bot/market_data.py" />
+        <option value="$PROJECT_DIR$/requirements.txt" />
+        <option value="$PROJECT_DIR$/bot/core.py" />
+        <option value="$PROJECT_DIR$/bot/market_data_crawler.py" />
       </list>
     </option>
   </component>
@@ -175,11 +188,6 @@
               <item name="crypto-arbitrage-bot" type="462c0819:PsiDirectoryNode" />
               <item name="bot" type="462c0819:PsiDirectoryNode" />
             </path>
-            <path>
-              <item name="crypto-arbitrage-bot" type="b2602c69:ProjectViewProjectNode" />
-              <item name="crypto-arbitrage-bot" type="462c0819:PsiDirectoryNode" />
-              <item name="tests" type="462c0819:PsiDirectoryNode" />
-            </path>
           </expand>
           <select />
         </subPane>
@@ -214,9 +222,8 @@
   </component>
   <component name="ToolWindowManager">
     <frame x="-8" y="-8" width="2576" height="1416" extended-state="6" />
-    <editor active="true" />
     <layout>
-      <window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.13554688" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
+      <window_info id="Project" active="true" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.13554688" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
       <window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
       <window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="true" content_ui="tabs" />
       <window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
@@ -276,14 +283,6 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/requirements.txt">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="0">
-          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
-          <folding />
-        </state>
-      </provider>
-    </entry>
     <entry file="file://$PROJECT_DIR$/README.rst">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="102">
@@ -300,14 +299,6 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/Makefile">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="0">
-          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
-          <folding />
-        </state>
-      </provider>
-    </entry>
     <entry file="file://$PROJECT_DIR$/LICENSE">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="0">
@@ -326,14 +317,6 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/bot/core.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="799">
-          <caret line="53" column="0" lean-forward="false" selection-start-line="53" selection-start-column="0" selection-end-line="53" selection-end-column="0" />
-          <folding />
-        </state>
-      </provider>
-    </entry>
     <entry file="file://$PROJECT_DIR$/bot/__init__.py">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="0">
@@ -368,13 +351,49 @@
     </entry>
     <entry file="file://$PROJECT_DIR$/tests/test_advanced.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="102">
-          <caret line="6" column="0" lean-forward="true" selection-start-line="6" selection-start-column="0" selection-end-line="6" selection-end-column="0" />
+        <state relative-caret-position="272">
+          <caret line="16" column="0" lean-forward="false" selection-start-line="16" selection-start-column="0" selection-end-line="16" selection-end-column="0" />
           <folding>
             <element signature="e#25#49#0" expanded="true" />
           </folding>
         </state>
       </provider>
     </entry>
+    <entry file="file://$PROJECT_DIR$/bot/core.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="0">
+          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="17" />
+          <folding>
+            <element signature="e#18#51#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/requirements.txt">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="34">
+          <caret line="2" column="7" lean-forward="true" selection-start-line="2" selection-start-column="7" selection-end-line="2" selection-end-column="7" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/Makefile">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="0">
+          <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/bot/market_data_crawler.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="269">
+          <caret line="22" column="11" lean-forward="true" selection-start-line="22" selection-start-column="11" selection-end-line="22" selection-end-column="11" />
+          <folding>
+            <element signature="e#19#60#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
   </component>
 </project>
\ No newline at end of file
diff --git a/bot/core.py b/bot/core.py
index a521140..7a9e6ea 100644
--- a/bot/core.py
+++ b/bot/core.py
@@ -1,13 +1,15 @@
 #!/usr/bin/python
-from . import helpers
+from . import market_data_crawler
 
-from tornado import httpserver
 from tornado import gen
 from tornado.ioloop import IOLoop
 import tornado.web
 import json
+import sys
+
 
 class MainHandler(tornado.web.RequestHandler):
+    @gen.coroutine
     def post(self):
         print("POST received from IP {0}".format(self.request.remote_ip))
 
@@ -21,10 +23,19 @@ class MainHandler(tornado.web.RequestHandler):
 
         if "command" in request:
             print("Command received: {0}".format(request["command"]))
-            response["msg"] = "All good - command received"
+
+            if request["command"] == "init_market_data":
+                yield self.update_market_data(request, response)
 
         self.write(json.dumps(response))
 
+    @gen.coroutine
+    def update_market_data(self, request, response):
+        yield market_data_crawler.update_market_data_for_basecoin(request["basecoin"])
+        response["msg"] = "Market Data initialized"
+        response["data"] = market_data_crawler.market_data
+
+    @gen.coroutine
     def delete(self):
         print("Stopping server...")
 
@@ -42,9 +53,9 @@ class Application(tornado.web.Application):
         tornado.web.Application.__init__(self, handlers)
 
 
-def main():
+def main(port):
     app = Application()
-    app.listen(666)
+    app.listen(port)
     IOLoop.instance().start()
 
 
@@ -53,5 +64,6 @@ def exists():
 
 
 if __name__ == '__main__':
-        print("Starting arbitrage bot...")
-        main()
\ No newline at end of file
+        port = int(sys.argv[1])
+        print("Starting arbitrage bot on port {0}...".format(port))
+        main(port)
\ No newline at end of file
diff --git a/bot/market_data_crawler.py b/bot/market_data_crawler.py
index 9cb0fe6..182b11b 100644
--- a/bot/market_data_crawler.py
+++ b/bot/market_data_crawler.py
@@ -1,37 +1,74 @@
-import tornado.escape, tornado.httpclient
+#!/usr/bin/python
 
+import tornado.escape, tornado.httpclient
+import tornado.httpclient
+from tornado import gen
+import time
+from collections import defaultdict
+
+
+@gen.coroutine
 def update_market_data_for_basecoin(basecoin):
     global market_data
-    market_data = {}
+    market_data = defaultdict(list)
     market_requests = []
 
-    """Bittrex"""
+    @gen.coroutine
+    def call_market_data(url, response_handler):
+        http_client = tornado.httpclient.AsyncHTTPClient()
+        response = yield http_client.fetch(url)
+        return response_handler(response)
 
+    """
+    Bittrex
+    """
     def handle_response_bittrex(response):
         if response.error:
             print("Error: %s" % response.error)
+            return False
+
         else:
+            print("Response received from Bittrex - handling now!")
             response_data = tornado.escape.json_decode(response.body)
 
             for market in response_data["result"]:
                 base, target = market["MarketName"].split("-")
                 if base == basecoin:
-                    market_data.update({
-                        target: {
-                            "Bittrex": market["Last"]
-                        }
-                    })
+                    market_data[target].append({"Bittrex": market["Last"]})
+
+            return True
+
+    market_requests.append(
+        call_market_data(
+            "https://bittrex.com/api/v1.1/public/getmarketsummaries",
+            handle_response_bittrex))
+
+    """
+    Poloniex
+    """
+    def handle_response_poloniex(response):
+        if response.error:
+            print("Error: %s" % response.error)
+            return False
+
+        else:
+            print("Response received from Poloniex - handling now!")
+            response_data = tornado.escape.json_decode(response.body)
+
+            for market in response_data:
+                base, target = market.split("_")
+                if base == basecoin:
+                    market_data[target].append({"Poloniex": response_data[market]["last"]})
 
 
-    bittrex = {
-        "url": "https://bittrex.com/api/v1.1/public/getmarketsummaries",
-        "response_handler": handle_response_bittrex
-    }
+            return True
 
-    market_requests.append(bittrex)
-
-    http_client = tornado.httpclient.AsyncHTTPClient()
-    for request in market_requests:
-        print("Doing request to {0}".format(request["url"]))
-        http_client.fetch(request["url"], request["response_handler"])
+    market_requests.append(
+        call_market_data(
+            "https://poloniex.com/public?command=returnTicker",
+            handle_response_poloniex))
 
+    print("--- Retrieve market data now ---")
+    start_time = time.time()
+    response_dict = yield market_requests
+    print("--- Marked data updated in {0} seconds. Responses: {1} ---".format(time.time() - start_time, response_dict))