20200428_Python百度文库爬虫终极版/Python百度文库爬虫之xls文件.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Python百度文库爬虫之xls文件"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "对于文件的所有类型，我都会用一篇文章进行说明，链接："
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. [Python百度文库爬虫之txt文件](https://blog.csdn.net/V_lq6h/article/details/105794291)\n",
    "2. [Python百度文库爬虫之doc文件](https://blog.csdn.net/V_lq6h/article/details/105800177)\n",
    "3. Python百度文库爬虫之pdf文件\n",
    "4. Python百度文库爬虫之ppt文件\n",
    "5. Python百度文库爬虫之xls文件"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "xls文件的爬取与pdf类型，不懂直接看前面pdf文件分析"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "请输入下载的文件URL地址： https://wenku.baidu.com/view/62c7ce0115791711cc7931b765ce0508763275bd.html?fr=search\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "import json\n",
    "import re\n",
    "import json\n",
    "\n",
    "session=requests.session()\n",
    "\n",
    "url=input(\"请输入下载的文件URL地址：\")\n",
    "\n",
    "content=session.get(url).content.decode('gbk')\n",
    "doc_id=re.findall('view/(.*?).html',url)[0]\n",
    "types=re.findall(r\"docType.*?:.*?'(.*?)'\",content)[0]\n",
    "title=re.findall(r\"title.*?:.*?'(.*?)'\",content)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "url_list=re.findall(r'(https.*?0.json.*?)\\\\x22}',content)\n",
    "url_list=[addr.replace(\"\\\\\\\\\\\\/\",\"/\") for addr in url_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda\\envs\\mypython\\lib\\site-packages\\ipykernel_launcher.py:16: DeprecationWarning: invalid escape sequence '\\/'\n",
      "  app.launch_new_instance()\n"
     ]
    }
   ],
   "source": [
    "result=\"\"\n",
    "\n",
    "for url in set(url_list):\n",
    "    content=session.get(url).content.decode('gbk')\n",
    "    \n",
    "    y=0\n",
    "    txtlists=re.findall(r'\"c\":\"(.*?)\".*?\"y\":(.*?),',content)\n",
    "    for item in txtlists:\n",
    "        # 当item[1]的值与前面不同时，代表要换行了\n",
    "        if not y==item[1]:\n",
    "            y=item[1]\n",
    "            n='\\n'\n",
    "        else:\n",
    "            n=''\n",
    "        result+=n\n",
    "        result+=item[0].encode('utf-8').decode('unicode_escape','ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "result=result.replace(\"\\udb80\",\"\").replace(\"\\udc00\",\"\")\n",
    "filename=\"./Files/\"+title+'.txt'\n",
    "\n",
    "with open(filename,'w',encoding='utf-8') as f:\n",
    "    f.write(result)\n",
    "    \n",
    "f.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 代码"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "完整程序，下载所有类型['txt','doc','pdf','ppt','xls']。文件下载不提供排版"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "欢迎来到百度文库文件下载：\n",
      "-----------------------\n",
      "\n",
      "1.doc \n",
      " 2.txt \n",
      " 3.ppt \n",
      " 4.xls\n",
      " 5.ppt\n",
      "\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "请输入需要下载文件的格式(0退出)： aaa\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "抱歉文件格式错误，请重新输入!\n",
      "1.doc \n",
      " 2.txt \n",
      " 3.ppt \n",
      " 4.xls\n",
      " 5.ppt\n",
      "\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "请输入需要下载文件的格式(0退出)： xls\n",
      "请输入下载的文库URL地址： https://wenku.baidu.com/view/c527b5d668eae009581b6bd97f1922791688beec.html?fr=search\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "文件南京信息工程大学2017级开题安排保存成功\n",
      "1.doc \n",
      " 2.txt \n",
      " 3.ppt \n",
      " 4.xls\n",
      " 5.ppt\n",
      "\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "请输入需要下载文件的格式(0退出)： xls\n",
      "请输入下载的文库URL地址： https://wenku.baidu.com/view/68d81664ddccda38376baffa.html?fr=search\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "文件接口技术类型保存成功\n",
      "1.doc \n",
      " 2.txt \n",
      " 3.ppt \n",
      " 4.xls\n",
      " 5.ppt\n",
      "\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "请输入需要下载文件的格式(0退出)： 0\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "import json\n",
    "import re\n",
    "import os\n",
    "\n",
    "session=requests.session()\n",
    "\n",
    "path=\"F:\\\\桌面\\\\Files\"\n",
    "\n",
    "if not os.path.exists(path):\n",
    "    os.mkdir(path)\n",
    "\n",
    "def parse_txt1(code,doc_id):\n",
    "    \n",
    "    content_url='https://wenku.baidu.com/api/doc/getdocinfo?callback=cb&doc_id='+doc_id\n",
    "\n",
    "    content=session.get(content_url).content.decode(code)\n",
    "    md5sum=re.findall('\"md5sum\":\"(.*?)\",',content)[0]\n",
    "    rsign=re.findall('\"rsign\":\"(.*?)\"',content)[0]\n",
    "    pn=re.findall('\"totalPageNum\":\"(.*?)\"',content)[0]\n",
    "    \n",
    "    content_url='https://wkretype.bdimg.com/retype/text/'+doc_id+'?rn='+pn+'&type=txt'+md5sum+'&rsign='+rsign\n",
    "    content=json.loads(session.get(content_url).content.decode('gbk'))\n",
    "    \n",
    "    result=''\n",
    "\n",
    "    for item in content:\n",
    "        for i in item['parags']:\n",
    "            result+=i['c']\n",
    "            \n",
    "    return result\n",
    "\n",
    "def parse_txt2(content,code,doc_id):\n",
    "    md5sum=re.findall('\"md5sum\":\"(.*?)\",',content)[0]\n",
    "    rsign=re.findall('\"rsign\":\"(.*?)\"',content)[0]\n",
    "    pn=re.findall('\"show_page\":\"(.*?)\"',content)[0]\n",
    "    \n",
    "    content_url='https://wkretype.bdimg.com/retype/text/'+doc_id+'?rn='+pn+'&type=txt'+md5sum+'&rsign='+rsign\n",
    "    content=json.loads(session.get(content_url).content.decode('utf-8'))\n",
    "    \n",
    "    result=''\n",
    "\n",
    "    for item in content:\n",
    "        for i in item['parags']:\n",
    "            result+=i['c']\n",
    "            \n",
    "    return result\n",
    "\n",
    "def parse_doc(content):\n",
    "    \n",
    "    url_list=re.findall(r'(https.*?0.json.*?)\\\\x22}',content)\n",
    "    url_list=[addr.replace(\"\\\\\\\\\\\\/\",\"/\") for addr in url_list]\n",
    "    \n",
    "    result=\"\"\n",
    "\n",
    "    for url in set(url_list):\n",
    "        content=session.get(url).content.decode('gbk')\n",
    "\n",
    "        y=0\n",
    "        txtlists=re.findall(r'\"c\":\"(.*?)\".*?\"y\":(.*?),',content)\n",
    "        for item in txtlists:\n",
    "            # 当item[1]的值与前面不同时，代表要换行了\n",
    "            if not y==item[1]:\n",
    "                y=item[1]\n",
    "                n='\\n'\n",
    "            else:\n",
    "                n=''\n",
    "            result+=n\n",
    "            result+=item[0].encode('utf-8').decode('unicode_escape','ignore')\n",
    "    \n",
    "    return result\n",
    "\n",
    "def parse_pdf(content):\n",
    "    \n",
    "    url_list=re.findall(r'(https.*?0.json.*?)\\\\x22}',content)\n",
    "    url_list=[addr.replace(\"\\\\\\\\\\\\/\",\"/\") for addr in url_list]\n",
    "    \n",
    "    result=\"\"\n",
    "\n",
    "    for url in set(url_list):\n",
    "        content=session.get(url).content.decode('gbk')\n",
    "\n",
    "        y=0\n",
    "        txtlists=re.findall(r'\"c\":\"(.*?)\".*?\"y\":(.*?),',content)\n",
    "        for item in txtlists:\n",
    "            # 当item[1]的值与前面不同时，代表要换行了\n",
    "            if not y==item[1]:\n",
    "                y=item[1]\n",
    "                n='\\n'\n",
    "            else:\n",
    "                n=''\n",
    "            result+=n\n",
    "            result+=item[0].encode('utf-8').decode('unicode_escape','ignore')\n",
    "    \n",
    "    return result\n",
    "\n",
    "\n",
    "def parse_ppt(doc_id,title):\n",
    "    \n",
    "    content_url='https://wenku.baidu.com/browse/getbcsurl?doc_id='+doc_id+'&pn=1&rn=9999&type=ppt'\n",
    "    content=session.get(content_url).content.decode('gbk')\n",
    "    \n",
    "    url_list=re.findall('{\"zoom\":\"(.*?)\",\"page\"',content)\n",
    "    url_list=[addr.replace('\\\\','') for addr in url_list]\n",
    "    \n",
    "    path=\"F:\\\\桌面\\\\Files\"+\"\\\\\"+title\n",
    "\n",
    "    if not os.path.exists(path):\n",
    "        os.mkdir(path)\n",
    "    \n",
    "    for index,url in enumerate(url_list):\n",
    "        content=session.get(url).content\n",
    "        paths=os.path.join(path,str(index)+'.jpg')\n",
    "    \n",
    "        with open(paths,'wb') as f:\n",
    "            f.write(content)\n",
    "    print(\"图片保存在\"+title+\"文件夹\")\n",
    "    \n",
    "    \n",
    "def parse_xls(content):\n",
    "    \n",
    "    url_list=re.findall(r'(https.*?0.json.*?)\\\\x22}',content)\n",
    "    url_list=[addr.replace(\"\\\\\\\\\\\\/\",\"/\") for addr in url_list]\n",
    "    \n",
    "    result=\"\"\n",
    "\n",
    "    for url in set(url_list):\n",
    "        content=session.get(url).content.decode('gbk')\n",
    "\n",
    "        y=0\n",
    "        txtlists=re.findall(r'\"c\":\"(.*?)\".*?\"y\":(.*?),',content)\n",
    "        for item in txtlists:\n",
    "            # 当item[1]的值与前面不同时，代表要换行了\n",
    "            if not y==item[1]:\n",
    "                y=item[1]\n",
    "                n='\\n'\n",
    "            else:\n",
    "                n=''\n",
    "            result+=n\n",
    "            result+=item[0].encode('utf-8').decode('unicode_escape','ignore')\n",
    "            \n",
    "    result=result.replace(\"\\udb80\",\"\").replace(\"\\udc00\",\"\")\n",
    "    \n",
    "    return result\n",
    "    \n",
    "def save_file(title,filename,content):\n",
    "    \n",
    "    with open(filename,'w',encoding='utf-8') as f:\n",
    "        f.write(content)\n",
    "        print(\"文件\"+title+\"保存成功\")\n",
    "    f.close()\n",
    "    \n",
    "\n",
    "def main():\n",
    "    \n",
    "    print(\"欢迎来到百度文库文件下载：\")\n",
    "    print(\"-----------------------\\r\\n\")\n",
    "     \n",
    "    \n",
    "    while True:\n",
    "        try:\n",
    "            print(\"1.doc \\n 2.txt \\n 3.ppt \\n 4.xls\\n 5.ppt\\n\")\n",
    "            types=input(\"请输入需要下载文件的格式(0退出)：\")\n",
    "\n",
    "            if types==\"0\":\n",
    "                break\n",
    "\n",
    "            if types not in ['txt','doc','pdf','ppt','xls']:\n",
    "                print(\"抱歉文件格式错误，请重新输入!\")\n",
    "                continue\n",
    "\n",
    "\n",
    "            url=input(\"请输入下载的文库URL地址：\")\n",
    "\n",
    "            # 网页内容\n",
    "            response=session.get(url)\n",
    "\n",
    "            code=re.findall('charset=(.*?)\"',response.text)[0]\n",
    "\n",
    "            if code.lower()!='utf-8':\n",
    "                code='gbk'\n",
    "\n",
    "            content=response.content.decode(code)\n",
    "\n",
    "            # 文件id\n",
    "            doc_id=re.findall('view/(.*?).html',url)[0]\n",
    "            # 文件类型\n",
    "            #types=re.findall(r\"docType.*?:.*?'(.*?)'\",content)[0]\n",
    "            # 文件主题\n",
    "            #title=re.findall(r\"title.*?:.*?'(.*?)'\",content)[0]\n",
    "\n",
    "            if types=='txt':\n",
    "                md5sum=re.findall('\"md5sum\":\"(.*?)\",',content)\n",
    "                if md5sum!=[]:\n",
    "                    result=parse_txt2(content,code,doc_id)\n",
    "                    title=re.findall(r'<title>(.*?). ',content)[0]\n",
    "                    #filename=os.getcwd()+\"\\\\Files\\\\\"+title+'.txt'\n",
    "                    filename=path+\"\\\\\"+title+\".txt\"\n",
    "                    save_file(title,filename,result)\n",
    "                else: \n",
    "                    result=parse_txt1(code,doc_id)\n",
    "                    title=re.findall(r\"title.*?:.*?'(.*?)'\",content)[0]\n",
    "                    #filename=os.getcwd()+\"\\\\Files\\\\\"+title+'.txt'\n",
    "                    filename=path+\"\\\\\"+title+\".txt\"\n",
    "                    save_file(title,filename,result)\n",
    "            elif types=='doc':\n",
    "                title=re.findall(r\"title.*?:.*?'(.*?)'\",content)[0]\n",
    "                result=parse_doc(content)\n",
    "                filename=path+\"\\\\\"+title+\".doc\"\n",
    "                save_file(title,filename,result)\n",
    "            elif types=='pdf':\n",
    "                title=re.findall(r\"title.*?:.*?'(.*?)'\",content)[0]\n",
    "                result=parse_pdf(content)\n",
    "                filename=path+\"\\\\\"+title+\".txt\"\n",
    "                save_file(title,filename,result)\n",
    "            elif types=='ppt':\n",
    "                title=re.findall(r\"title.*?:.*?'(.*?)'\",content)[0]\n",
    "                parse_ppt(doc_id,title)\n",
    "            elif types=='xls':\n",
    "                title=re.findall(r\"title.*?:.*?'(.*?)'\",content)[0]\n",
    "                result=parse_xls(content)\n",
    "                filename=path+\"\\\\\"+title+\".txt\"\n",
    "                save_file(title,filename,result)\n",
    "                \n",
    "                \n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "\n",
    "\n",
    "if __name__=='__main__':\n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}