chore(qd): fix qd.py failure

qd.py

@@ -5,14 +5,16 @@
 #         http://binux.me
 # Created on 2014-08-18 12:17:21
 
-import functools
+import asyncio
 import json
 import sys
 
-from tornado.ioloop import IOLoop
+import config
 
+config.display_import_warning = False
 from libs.fetcher import Fetcher
 from libs.log import Log
+from run import start_server
 
 logger_QD = Log('QD').getlogger()
 
@@ -51,27 +53,61 @@ def usage():
         except Exception as e:
             logger_QD.error(e)
             usage()
-    if 'variables' not in env or not isinstance(env['variables'], dict)\
+    if 'variables' not in env or not isinstance(env['variables'], dict) \
             or 'session' not in env:
         env = {
                 'variables': env,
                 'session': [],
                 }
     env['variables'].update(variables)
 
-    # do fetch
-    ioloop = IOLoop.instance()
-    def ioloop_stop(x):
-        ioloop.stop()
+    # 判断 端口 是否被占用
+    import re
+    import socket
+    def check_port(port):
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        try:
+            s.connect(('127.0.0.1', port))
+            s.shutdown(2)
+            logger_QD.debug('Port %s is used' % port)
+            return False
+        except:
+            logger_QD.debug('Port %s is available' % port)
+            return True
+    manual_start = check_port(config.port)
+    if manual_start:
+        logger_QD.info('QD service is not running on port %s' % config.port)
+        logger_QD.info('QD service will be started on port %s' % config.port)
+        # 创建新进程, 以执行 run 中的 main 异步函数
+        import multiprocessing
+        p = multiprocessing.Process(target=start_server)
+        p.start()
+        # 循环检测端口是否被占用, 如果被占用, 则继续下一步
+        while True:
+            if not check_port(config.port):
+                break
+            else:
+                import time
+                time.sleep(1)
+    else:
+        logger_QD.info('QD service is running on port %s' % config.port)
 
-    fetcher = Fetcher()
-    result = fetcher.do_fetch(tpl, env)
-    ioloop.add_future(result, ioloop_stop)
-    ioloop.start()
+    # do fetch
+    ioloop = asyncio.new_event_loop()
+    asyncio.set_event_loop(ioloop)
+    result:asyncio.Task = asyncio.ensure_future(Fetcher().do_fetch(tpl, env), loop=ioloop)
+    logger_QD.info('QD start to do fetch: %s' % tpl_file)
+    ioloop.run_until_complete(result)
+    ioloop.stop()
 
     try:
         result = result.result()
     except Exception as e:
         print('QD failed!', e)
     else:
-        print('QD success!', result.get('variables', {}).get('__log__', '').replace('\\r\\n','\r\n'))
+        print('QD success! Results:\n', result.get('variables', {}).get('__log__', '').replace('\\r\\n','\r\n'))
+
+    if manual_start:
+        p.terminate()
+        p.join()
+        logger_QD.info('QD service is ended. ')

run.py

@@ -2,19 +2,27 @@
 # vim:fenc=utf-8
 #
 # Copyright © 2016 Binux <[email protected]>
+
 import asyncio
 import json
+import logging
 import os
 import platform
 import sys
 
 import tornado.log
+from tornado.httpserver import HTTPServer
+from tornado.ioloop import IOLoop, PeriodicCallback
 
 import config
-from db import db_converter
+from db import DB, db_converter
+from db.basedb import engine
 from libs.log import Log
+from web.app import Application
+from worker import BatchWorker, QueueWorker
 
-if __name__ == "__main__":
+
+def start_server():
     if sys.getdefaultencoding() != 'utf-8':
         import importlib
         importlib.reload(sys)
@@ -23,7 +31,6 @@
     logger_QD = Log('QD.Run').getlogger()
 
     if config.debug:
-        import logging
         channel = logging.StreamHandler(sys.stderr)
         channel.setFormatter(tornado.log.LogFormatter())
         channel.setLevel(logging.WARNING)
@@ -46,18 +53,13 @@
         config.autoreload = False
 
     try:
-        from db import DB
-        from db.basedb import engine
         database = DB()
         converter = db_converter.DBconverter(database)
         loop = asyncio.new_event_loop()
         asyncio.set_event_loop(loop)
         run = asyncio.ensure_future(converter.ConvertNewType(database) , loop=loop)
         loop.run_until_complete(run)
 
-        from tornado.httpserver import HTTPServer
-
-        from web.app import Application
         default_version = json.load(open(os.path.join(os.path.dirname(__file__), 'version.json'),'r', encoding='utf-8'))['version']
         App= Application(database, default_version)
         http_server = HTTPServer(App, xheaders=True)
@@ -67,9 +69,6 @@
         else:
             http_server.start()
 
-        from tornado.ioloop import IOLoop, PeriodicCallback
-
-        from worker import BatchWorker, QueueWorker
         io_loop = IOLoop.instance()
         try:
             if config.worker_method.upper() == 'QUEUE':
@@ -94,3 +93,6 @@
         loop.run_until_complete(run)
         logger_QD.info("Http Server is ended. ")
 
+
+if __name__ == "__main__":
+    start_server()