modify update_db.sh

ccns · Feb 10, 2017 · 22d0f93 · 22d0f93
1 parent 17350d3
commit 22d0f93
Show file tree

Hide file tree

Showing 6 changed files with 13 additions and 20 deletions.
diff --git a/config.json → backups/config.json b/config.json → backups/config.json
diff --git a/backups/mongo-connector-cmd.backup b/backups/mongo-connector-cmd.backup
@@ -0,0 +1,3 @@
+mongo-connector -m localhost:27017 -t localhost:9200 -d elastic2_doc_manager\
+  -a ccns -p CCNSccns\
+  --logfile $LOGDIR/mongo-connector.log --oplog-ts $LOGDIR/oplog.timestamp &
diff --git a/package.json b/package.json
@@ -4,7 +4,8 @@
   "description": "a crawler",
   "main": "run.js",
   "scripts": {
-    "start": "node run.js"
+    "start": "node run.js",
+    "updateDept": "node -e 'require(\"./update.js\").updateDeptList()'"
   },
   "repository": {
     "type": "git",

diff --git a/run.js b/run.js
@@ -1,3 +1,3 @@
 var update = require("./update.js");
 
-update(false);
+update.update(false);
diff --git a/update.js b/update.js
@@ -39,7 +39,7 @@ function updateCourseList(dept) {
     if (courses<0) {
       yield new Error(dept_no+": timeout");
     } else {
-      // console.log(dept_no+": "+courses.length);
+      console.log(dept_no+": "+courses.length);
       if (courses.length > 0) {
         var db = yield MongoClient.connect(url);
         var collection = db.collection('courses');
@@ -106,4 +106,8 @@ function update(updateDept) {
   go(gen.next());
 }
 
-module.exports = update
+module.exports = {
+  update: update,
+  updateDeptList: updateDeptList,
+  updateCourseList: updateCourseList
+}
diff --git a/update_db.sh b/update_db.sh
@@ -7,23 +7,8 @@ BASEDIR=$(pwd)
 LOGDIR=${1:-$BASEDIR/log}
 LOGFILE=$LOGDIR/update.log
 
-echo "Starting mongo-connector" | $BASEDIR/predate.sh >> $LOGFILE
-mongo-connector -m localhost:27017 -t localhost:9200 -d elastic2_doc_manager\
-  -a ccns -p CCNSccns\
-  --logfile $LOGDIR/mongo-connector.log --oplog-ts $LOGDIR/oplog.timestamp &
-MCPID=$!
-echo "Mongo-connector started. PID=$MCPID" | $BASEDIR/predate.sh >> $LOGFILE
-
 (time bash -c "cd $BASEDIR/pages && ./update_pages.sh | $BASEDIR/predate.sh >> $LOGFILE && cd ..") 2>&1 | $BASEDIR/predate.sh >> $LOGFILE
 node run | $BASEDIR/predate.sh >> $LOGFILE
 
-echo "Killing mongo-connector. PID=$MCPID" | $BASEDIR/predate.sh >> $LOGFILE
-kill $MCPID 2>&1 | $BASEDIR/predate.sh >> $LOGFILE
-if [ $PIPESTATUS -eq 0 ]; then
-  echo "All success." | $BASEDIR/predate.sh >> $LOGFILE
-else
-  echo "Kill mongo-connector failed." | $BASEDIR/predate.sh >> $LOGFILE
-fi
-
 TOC=$((`date +%s`-TIC))
-echo "Total excution time: $TOC" | $BASEDIR/predate.sh >> $LOGFILE
+echo "Finished. Total excution time: $TOC" | $BASEDIR/predate.sh >> $LOGFILE