add log file

.gitignore

@@ -8,5 +8,5 @@ pages/depts/*
 pages/retry.json
 pages/index.html
 
-mongo-connector.log
-oplog.timestamp
+*.log
+*.timestamp

pages/get_all.py

@@ -18,7 +18,6 @@
 def download(dept):
     global depts
     url = "http://course-query.acad.ncku.edu.tw/qry/qry001.php?dept_no=" + dept
-    print("Request sent: "+dept)
 
     req = requests.Session()
     res = req.get(url)
@@ -36,9 +35,19 @@ def download(dept):
     json.dump(depts, f)
     f.close()
     print(dept+".html updated. Remain: "+str(len(depts)))
-    if(len(depts)==0):
-        print("All done.")
     lock.release()
 
+threads = []
 for dept in depts:
-    threading.Thread(target=download, args=(dept,)).start()
+    t = threading.Thread(target=download, args=(dept,))
+    threads.append(t)
+    print("Thread "+dept+" established.")
+
+for t in threads:
+    t.start()
+
+for t in threads:
+    t.join()
+
+if(len(depts)==0):
+    print("All done.")

pages/update_pages.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
-timeout 1m python get_all.py
+timeout 1m python -u get_all.py
 while [ "$(<retry.json)" != "[]" ]; do
   echo "Execution timeout, retry ..."
-  timeout 1m python get_all.py retry
+  timeout 1m python -u get_all.py retry
 done

predate.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+while read line ; do
+  echo "[$(date '+%Y-%m-%d %H:%M')] ${line}"
+done

update.js

@@ -39,7 +39,7 @@ function updateCourseList(dept) {
     if (courses<0) {
       yield new Error(dept_no+": timeout");
     } else {
-      console.log(dept_no+": "+courses.length);
+      // console.log(dept_no+": "+courses.length);
       if (courses.length > 0) {
         var db = yield MongoClient.connect(url);
         var collection = db.collection('courses');
@@ -84,6 +84,7 @@ function update(updateDept) {
     var colleges = yield collection.find({}).toArray();
     db.close();
 
+    console.log("Start to update course list ...")
     for (let i=0; i<colleges.length; i++) {
       let depts = colleges[i].depts;
       for (let j=0; j<depts.length; j++) {

update_db.sh

@@ -1,12 +1,24 @@
 #!/bin/bash
 
-echo "Starting mongo-connector"
-mongo-connector -m localhost:27017 -t localhost:9200 -d elastic2_doc_manager --admin-username ccns --password CCNSccns &
+cd $(dirname "$0")
+BASEDIR=$(pwd)
+LOGDIR=${1:-$BASEDIR/log}
+LOGFILE=$LOGDIR/update.log
+
+echo "Starting mongo-connector" | $BASEDIR/predate.sh >> $LOGFILE
+mongo-connector -m localhost:27017 -t localhost:9200 -d elastic2_doc_manager\
+  -a ccns -p CCNSccns\
+  --logfile $LOGDIR/mongo-connector.log --oplog-ts $LOGDIR/oplog.timestamp &
 MCPID=$!
-echo "Mongo-connector started. PID=$MCPID"
-BASEDIR=$(dirname "$0")
-(cd $BASEDIR/pages && ./update_pages.sh && cd ..)
-node run
-echo "Killing mongo-connector. PID=$MCPID"
-kill $MCPID
-echo "Success"
+echo "Mongo-connector started. PID=$MCPID" | $BASEDIR/predate.sh >> $LOGFILE
+
+(time bash -c "cd $BASEDIR/pages && ./update_pages.sh | $BASEDIR/predate.sh >> $LOGFILE && cd ..") 2>&1 | $BASEDIR/predate.sh >> $LOGFILE
+node run | $BASEDIR/predate.sh >> $LOGFILE
+
+echo "Killing mongo-connector. PID=$MCPID" | $BASEDIR/predate.sh >> $LOGFILE
+kill $MCPID 2>&1 | $BASEDIR/predate.sh >> $LOGFILE
+if [ $PIPESTATUS -eq 0 ]; then
+  echo "All success." | $BASEDIR/predate.sh >> $LOGFILE
+else
+  echo "Kill mongo-connector failed." | $BASEDIR/predate.sh >> $LOGFILE
+fi