fetch_tb_nb_products.js

// Generated by CoffeeScript 1.3.3
(function() {
  var fetch_products, grap_done, index, intervalId, jsdom, operators, page_size, pageno, parse_product_li, previousGrap, products, request;

  jsdom = require("jsdom");

  request = require("request");

  operators = [
    {
      "title": "来趣",
      "user_num_id": "867510228",
      "shop_url": "http://lqwlshfw.tmall.com"
    }, {
      "title": "瑞淘",
      "user_num_id": "732279177",
      "shop_url": "http://rtdzshfw.tmall.com"
    }, {
      "title": "心怡",
      "user_num_id": "717301792",
      "shop_url": "http://xyshfw.tmall.com"
    }, {
      "title": "完美",
      "user_num_id": "792448220",
      "shop_url": "http://wmzxshfw.tmall.com"
    }, {
      "title": "应邀",
      "user_num_id": "698513518",
      "shop_url": "http://yyshfw.tmall.com"
    }
  ];

  grap_done = false;

  previousGrap = true;

  pageno = 1;

  page_size = 20;

  products = [];

  index = 0;

  fetch_products = function(operators, callback) {
    var fetch_url;
    if (previousGrap === false) {
      return;
    }
    previousGrap = false;
    fetch_url = operators[index].shop_url + "/search.htm?search=y&viewType=grid&orderType=_newOn&pageNum=" + pageno;
    return request({
      url: fetch_url,
      headers: {
        "User-Agent": "Mozilla/5.0 (X11; U; Linux i686; zh-CN; rv:1.9.1.2) Gecko/20090803 Fedora/3.5.2-2.fc11 Firefox/3.5.2"
      },
      encoding: "gbk"
    }, function(error, response, body) {
      var document, product, product_li, products_ul, xPathResult, _i, _len, _ref;
      if (!error && response.statusCode === 200) {
        document = jsdom.jsdom(body);
        xPathResult = document.evaluate('//*[@id="J_ShopSearchResult"]/div/div[2]/ul', document, null, 4, null);
        products_ul = xPathResult.iterateNext();
        if (products_ul === null && index === operators.length - 1) {
          clearInterval(intervalId);
          callback(null, products);
          console.log("done fetching");
          return;
        } else if (products_ul === null) {
          console.log("done fetch " + operators[index]['title'] + "'s products!");
          console.log("current fetched product length is" + products.length);
          pageno = 1;
          index++;
          previousGrap = true;
          console.log("begin fetch " + operators[index]['title'] + "'s products!");
          return;
        }
        _ref = products_ul.children;
        for (_i = 0, _len = _ref.length; _i < _len; _i++) {
          product_li = _ref[_i];
          product = parse_product_li(product_li);
          product.operator = operators[index]['title'];
          product.operator_id = operators[index]['user_num_id'];
          products.push(product);
        }
        pageno++;
        if (products_ul.length < page_size && index === operators.length - 1) {
          clearInterval(intervalId);
          callback(null, products);
          console.log("done fetching");
        } else if (products_ul.length < page_size) {
          console.log("done fetch " + operators[index]['title'] + "'s products!");
          console.log("current fetched product length is" + products.length);
          pageno = 1;
          index++;
          console.log("begin fetch " + operators[index]['title'] + "'s products!");
        }
      } else {
        clearInterval(intervalId);
        callback(error);
      }
      return previousGrap = true;
    });
  };

  parse_product_li = function(product_li) {
    var $, desc, href, price, price_text, product, product_id, product_jq, rate, rate_text, sales_amount, url;
    $ = require('jQuery');
    url = require("url");
    product_jq = $(product_li);
    product = {};
    href = product_jq.find(".item .pic a").attr("href");
    product_id = href.substr(href.indexOf("id=") + 3, 11);
    desc = product_jq.find(".desc a").text();
    price_text = product_jq.find(".price strong").text();
    price = parseInt(price_text.substring(0, price_text.indexOf("元") - 1), 10);
    sales_amount = parseInt(product_jq.find(".sales-amount em").text(), 10);
    rate_text = product_jq.find(".rating span:nth-child(1)").attr("title");
    rate = rate_text.substring(0, rate_text.indexOf("分"));
    product.href = href;
    product.product_id = product_id;
    product.title = desc.trim();
    product.price = price;
    product.sales_amount = sales_amount;
    product.rate = rate;
    return product;
  };

  console.log("begin fetch " + operators[0]['title'] + "'s products!");

  intervalId = setInterval(fetch_products, 1000, operators, function(error, products) {
    var products_ju_nb;
    if (error) {
      return console.log(error);
    } else {
      products_ju_nb = require('./modules/products_ju_nb');
      products_ju_nb.empty_table(function(result) {
        if (result === true) {
          return console.log("done empty table products_ju_nb");
        } else {
          console.log("fail to empty table:" + result);
        }
      });
      return products_ju_nb.insert_products(products, function(result) {
        if (result === true) {
          return console.log("Insert products done!");
        } else {
          return console.log("Insert products error:" + result);
        }
      });
    }
  });

}).call(this);