(╬ Ò ‸ Ó)_重啟碩博AAAAAA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "60fdde5e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "搜尋時間： 22/12/01 15:42:11\n",
      "共1054筆\n",
      "在第1頁第270筆出錯\n",
      "Restart\n",
      "連回第270筆資料\n",
      "準備前往第271筆資料\n",
      "在第1頁第778筆出錯\n",
      "Restart\n",
      "連回第778筆資料\n",
      "準備前往第779筆資料\n",
      "爬蟲成功\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Temp\\TEMP\\ipykernel_12784\\3233028154.py:440: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  res_all = res_all.append(result_df)\n"
     ]
    }
   ],
   "source": [
    "#!/usr/bin/env python\n",
    "# coding: utf-8\n",
    "\n",
    "# In[1]:\n",
    "\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.service import Service\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import Select\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from glob import glob\n",
    "import time\n",
    "import random\n",
    "import requests\n",
    "import os\n",
    "import glob\n",
    "from html.parser import HTMLParser\n",
    "import re\n",
    "\n",
    "#設定chromedriver的路徑\n",
    "browser = webdriver.Chrome(\n",
    "    executable_path=r'C:\\Users\\user\\Desktop\\py\\Driver\\chromedriver.exe')\n",
    "browser.get('https://ndltd.ncl.edu.tw/')\n",
    "#cookie = re.findall(r'ccd=(.*?)/', browser.current_url)[0]\n",
    "WebDriverWait(browser, 30).until(  #等待網頁讀取 直到出現 \"指令查詢\"\n",
    "    EC.presence_of_element_located((By.LINK_TEXT, \"指令查詢\")))\n",
    "browser.maximize_window()\n",
    "\n",
    "requests.adapters.DEFAULT_RETRIES = 5  #增加重連次數\n",
    "rs = requests.session()\n",
    "i = 1\n",
    "#url = \"'https://ndltd.ncl.edu.tw/cgi-bin/gs32/gsweb.cgi/ccd={}/record?r1={}&h1=0'\".format(cookie, i)\n",
    "url = \"https://ndltd.ncl.edu.tw/cgi-bin/gs32/gsweb.cgi/ccd=ZqKe9e/webmge?mode=basic\"\n",
    "#s.keep_alive = False  #關閉多餘的連接\n",
    "\n",
    "##各計畫參數區\n",
    "\n",
    "lst1 = [\"surveyall\",\n",
    "    \"survey1\", \"survey2\", \"survey3\", \"survey4\", \"survey5\", \"survey6\",\n",
    "    \"survey7\", \"survey8\", \"survey9\", \"survey10\", \"survey11\", \"survey12\",\n",
    "    \"survey13\", \"survey14\", \"survey15\", \"survey16\", \"survey17\", \"survey18\",\n",
    "    \"survey19\", \"survey20\", \"survey21\", \"survey22\", \"survey23\", \"survey24\",\n",
    "    \"survey25\", \"survey26\", \"survey27\", \"survey28\", \"survey29\", \"survey30\",\n",
    "    \"survey31\", \"survey32\", \"survey33\", \"survey34\", \"survey35\", \"survey36\",\n",
    "    \"survey37\", \"survey38\"\n",
    "]\n",
    "\n",
    "lst2 = pd.Series([\"全搜\",\n",
    "    \"TSCS\", \"PSFD\", \"TEPS\", \"SNELS\", \"SFIE\", \"TWVS\", \"TYP\", \"TEDS\", \"TEPS-B\",\n",
    "    \"MPS\", \"MUS\", \"WMFES\", \"DDS\", \"TDCYP\", \"TCS\", \"ISSP\", \"AS032\", \"GS068\",\n",
    "    \"GS067\", \"AS003\", \"AS004\", \"AS030\", \"AS024\", \"AS014\", \"AS025\", \"NCDR\",\n",
    "    \"新住民\", \"外籍與大陸配偶\", \"新住民數位機會現況與需求調查\", \"農林漁牧業普查\", \"攤販經營概況調查\", \"受僱員工薪資調查\",\n",
    "    \"工作期望調查\", \"失業狀況調查\", \"住宅狀況調查\", \"時間運用調查\", \"國內遷徙調查\", \"台灣貧窮兒少資料庫\"\n",
    "],\n",
    "                 index=lst1)\n",
    "\n",
    "# 1.台灣社會變遷基本調查(TSCS)\n",
    "survey1 = 'TSCS'\n",
    "code1 = \"('台灣'.ab and '社會變遷'.ab and '調查'.ab and '資料'.ab or '樣本'.ab) or ('台灣社會變遷調查'.ti,kw or 'Taiwan Social Change Survey'.ti,kw or 'TSCS'.ti,kw)\"\n",
    "\n",
    "# 2.華人家庭動態資料庫(PSFD)\n",
    "survey2 = 'PSFD'\n",
    "code2 = \"('華人'.ab and '家庭'.ab and '動態'.ab and '資料'.ab or '樣本'.ab) or ('華人家庭動態'.ti,kw or '家庭動態資料'.ti,kw or 'Panel Study of Family Dynamics'.ti,kw or 'PSFD'.ti,kw)\"\n",
    "\n",
    "# 3.台灣教育長期追蹤(TEPS)\n",
    "survey3 = 'TEPS'\n",
    "code3 = \"('台灣教育'.ab and '追蹤'.ab and '資料'.ab or '樣本'.ab) or ('台灣教育長期追蹤'.ti,kw or 'Taiwan Education Panel Survey'.ti,kw or 'TEPS'.ti,kw)\"\n",
    "\n",
    "# 4.特殊教育長期追蹤資料庫(SNELS)\n",
    "survey4 = 'SNELS'\n",
    "code4 = \"('特殊教育'.ab and '追蹤'.ab and '資料'.ab or '樣本'.ab) or ('特殊教育長期追蹤'.ti,kw or 'Special Needs Education Longitudinal Study'.ti,kw or 'SNELS'.ti,kw)\"\n",
    "\n",
    "# 5.家庭收支調查(SFIE)\n",
    "survey5 = 'SFIE'\n",
    "code5 = \"('家庭'.ab and '收支'.ab and '調查'.ab and '資料'.ab) or ('家庭收支調查'.ti,kw or 'Survey of Family Income and Expenditure'.ti,kw or 'SFIE'.ti,kw)\"\n",
    "\n",
    "# 6.台灣世界價值觀調查(TWVS)\n",
    "survey6 = 'TWVS'\n",
    "code6 = \"('世界價值觀'.ab and '調查'.ab and '資料'.ab or '樣本'.ab) or ('台灣世界價值觀調查'.ti,kw or 'Taiwan World Value Survey'.ti,kw or 'TWVS'.ti,kw)\"\n",
    "\n",
    "# 7.台灣青少年成長歷程研究(TYP)\n",
    "survey7 = 'TYP'\n",
    "code7 = \"('青少年'.ab and '成長'.ab and '調查'.ab and '資料'.ab or '樣本'.ab) or ('台灣青少年成長歷程研究'.ti,kw or 'Taiwan Youth Project'.ti,kw or 'TYP'.ti,kw)\"\n",
    "\n",
    "# 8.台灣選舉調查資料(TEDS)\n",
    "survey8 = 'TEDS'\n",
    "code8 = \"('選舉'.ab and '調查'.ab and '資料'.ab or '樣本'.ab) or ('台灣選舉與民主化調查'.ti,kw or 'Taiwan lection and Democratization Study'.ti,kw or 'TEDS'.ti,kw)\"\n",
    "\n",
    "# 9.「台灣教育長期追蹤資料庫」後續調查(TEPS-B)\n",
    "survey9 = 'TEPS-B'\n",
    "code9 = \"('台灣教育'.ab and '追蹤'.ab and '資料'.ab and '後續'.ab and '調查'.ab or '樣本'.ab) or (('台灣教育長期追蹤資料庫'.ti,kw and '後續調查'.ti,kw) or 'Taiwan Education Panel Survey and Beyond'.ti,kw or 'TEPS-B'.ti,kw)\"\n",
    "\n",
    "# 10.人力資源調查(MPS)\n",
    "survey10 = 'MPS'\n",
    "code10 = \"('人力資源調查'.ab) or ('人力資源調查'.ti,kw or 'Manpower Survey'.ti,kw)\"\n",
    "\n",
    "# 11.人力運用調查(MUS)\n",
    "survey11 = 'MUS'\n",
    "code11 = \"('人力運用調查'.ab) or ('人力運用調查'.ti,kw or 'Manpower Utilization Survey'.ti,kw)\"\n",
    "\n",
    "# 12.婦女婚育與就業調查(WMFES)\n",
    "survey12 = 'WMFES'\n",
    "code12 = \"('婦女婚育'.ab and '就業'.ab) or ('女婚育與就業調查'.ti,kw)\"\n",
    "\n",
    "# 13.個人家戶數位機會(落差)(數位落差)(DDS)\n",
    "survey13 = 'DDS'\n",
    "code13 = \"('個人'.ab and '家戶'.ab and '數位'.ab and '機會'.ab and '調查'.ab or '落差'.ab) or ('個人家戶數位機會調查'.ti,kw or '個人家戶數位落差調查'.ti,kw or 'Digital Divide Survey'.ti,kw)\"\n",
    "\n",
    "# 14.台灣貧窮兒少資料庫(TDCYP)\n",
    "survey14 = 'TDCYP'\n",
    "code14 = \"('貧窮'.ab and '兒少'.ab and '資料'.ab) or ('貧窮兒少'.ti,kw or '家扶'.ti,kw or 'Database of Children and Youth in Poverty'.ti,kw or 'TDCYP'.ti,kw)\"\n",
    "\n",
    "# 15.台灣傳播調查資料庫(TCS)\n",
    "survey15 = 'TCS'\n",
    "code15 = \"('台灣'.ab and '傳播'.ab and '調查'.ab and '資料'.ab) or ('台灣傳播調查資料庫'.ti,kw or '傳播調查資料庫'.ti,kw or 'Taiwan Communication Survey'.ti,kw or 'TCS'.ti,kw)\"\n",
    "\n",
    "# 16.International Social Survey Programme (ISSP)\n",
    "survey16 = 'ISSP'\n",
    "code16 = \"('International Social Survey Programme'.ab or '國際社會調查計畫'.ab) or ('International Social Survey Programme'.ti,kw or '國際社會調查計畫'.ti,kw or 'ISSP'.ti,kw)\"\n",
    "\n",
    "# 17.AS032\n",
    "survey17 = 'AS032'\n",
    "code17 = \"('幼兒發展'.ab and '調查'.ab and '資料'.ab or '樣本'.ab) or ('幼兒發展調查'.ti,kw,rf or ('National Longitudinal Study of Child Development'.ti,kw,rf AND 'Care'.ti,kw,rf) or ('KIT'.ti,kw AND 'Taiwan'.ti,kw))\"\n",
    "\n",
    "# 18.GS068\n",
    "survey18 = 'GS068'\n",
    "code18 = \"('國人旅遊狀況'.ab and '調查'.ab and '資料'.ab or '樣本'.ab) or ('國人旅遊狀況調查'.ti,kw,rf or ('Survey of Travel'.ti,kw and 'R.O.C. Citizens'.ti,kw))\"\n",
    "\n",
    "# 19.GS067\n",
    "survey19 = 'GS067'\n",
    "code19 = \"(('來臺旅客'.ab or '來台旅客'.ab) and '消費'.ab and '動向'.ab and '調查'.ab and ('資料'.ab or '樣本'.ab)) or ('來臺旅客消費及動向調查'.ti,kw,rf or '來台旅客消費及動向調查'.ti,kw,rf or ('Annual Survey'.ti,kw and 'Visitors Expenditure and Trends'.ti,kw and 'Taiwan'.ti,kw))\"\n",
    "\n",
    "# 20.AS003 ✔️\n",
    "survey20 = 'AS003'\n",
    "code20 = \"('社會意向'.ab and '調查'.ab and '資料'.ab or '樣本'.ab) or ('社會意向調查'.ti,kw,rf or 'Taiwan Social Image Survey'.ti,kw,rf)\"\n",
    "\n",
    "# 21.AS004\n",
    "survey21 = 'AS004'\n",
    "code21 = \"(('台灣'.ab or '臺灣'.ab) and '營養健康狀況變遷'.ab and '調查'.ab and '資料'.ab or '樣本'.ab) or ('台灣營養健康狀況變遷調查'.ti,kw,rf or '臺灣營養健康狀況變遷調查'.ti,kw,rf or ('Nutrition And Health Survey'.ti,kw and 'Taiwan'.ti,kw) or 'NAHSIT'.ti,kw)\"\n",
    "\n",
    "# 22.AS030\n",
    "survey22 = 'AS030'\n",
    "code22 = \"('我國大學生政治社會化'.ab and '定群追蹤研究'.ab and '資料'.ab or '樣本'.ab) or ('我國大學生政治社會化的定群追蹤研究'.ti,kw,rf or ('Political Socialization of University Students in Taiwan'.ti,kw and 'Panel Studies'.ti,kw))\"\n",
    "\n",
    "# 23.AS024\n",
    "survey23 = 'AS024'\n",
    "code23 = \"((('台灣'.ab or '臺灣'.ab) and '政府文官'.ab and '調查'.ab) and ('資料'.ab or '樣本'.ab)) or ('台灣政府文官調查'.ti,kw or '公務人員風險偏好、服務動機、與決策類型之關係'.ti,kw) or ('民主治理下的政府效能'.ti,kw and '公共服務動機、繁文縟節、與政務/事務關係'.ti,kw) or ('台灣民主治理機制鞏固之研究'.ti,kw or '臺灣民主治理機制鞏固之研究'.ti,kw)\"\n",
    "\n",
    "# 24.AS014\n",
    "survey24 = 'AS014'\n",
    "code24 = \"('青少年藥物濫用'.ab and '起因'.ab and '資料'.ab or '樣本'.ab) or ('青少年藥物濫用之起因'.ti,kw or ('Etiology'.ti,kw and 'Adolescent'.ti,kw and 'Substance Abuse'.ti,kw))\"\n",
    "\n",
    "# 25.AS025\n",
    "survey25 = 'AS025'\n",
    "code25 = \"(('法實證研究資料庫'.ab,rf or '法律與社會變遷'.kw,ab,rf) or ('台灣法實證研究'.ti,kw or 'Taiwan Database for Empirical Studies'.ti,kw or 'TaDELS'.kw,ab,rf or 'tcsd.lib'.kw,ab,rf) and yr='98-110')\"\n",
    "\n",
    "# 26.國家災害防救科技中心\n",
    "survey26 = '國家災害防救科技中心'\n",
    "code26 = \"(國家災害防救科技中心.kw  or NCDR.kw or 災防中心.kw) or (國家災害防救科技中心.ab  or 災防中心.ab) or (國家災害防救科技中心.rf  or NCDR.rf)\"\n",
    "\n",
    "# 27.新住民相關\n",
    "survey27 = '新住民相關'\n",
    "code27 = \"(新住民數位發展現況與需求調查.kw  or 新住民生活需求調查.kw or 新住民.kw or 新住民配偶.kw or 新移民.kw or 新移民子女.kw) or (新住民數位發展現況與需求調查.ab  or 新住民生活需求調查.ab) or (新住民數位發展現況與需求調查.rf  or 新住民生活需求調查.rf)\"\n",
    "\n",
    "# 28.外籍與大陸配偶生活狀況調查\n",
    "survey28 = '外籍與大陸配偶'\n",
    "code28 = \"(外籍與大陸配偶生活狀況調查.kw  or 外籍與大陸配偶.kw or 外籍配偶.kw or 大陸配偶.kw) or (外籍與大陸配偶生活狀況調查.ab  or 外籍與大陸配偶.ab) or (外籍與大陸配偶生活狀況調查.rf)\"\n",
    "\n",
    "# 29.數位機會現況與需求調查\n",
    "survey29 = '新住民數位機會現況與需求調查'\n",
    "code29 = \"(新住民數位機會現況與需求調查.kw  or 新住民數位機會.kw ) or (新住民數位機會現況與需求調查.ab  or 新住民數位機會.ab) or (新住民數位機會現況與需求調查.rf)\"\n",
    "\n",
    "# 30.農林漁牧業普查\n",
    "survey30 = '農林漁牧業普查'\n",
    "code30 = \"(農林漁牧業普查.kw  or 農林漁牧業.kw ) or (農林漁牧業普查.ab) or (農林漁牧業普查.rf)\"\n",
    "\n",
    "# 31.攤販經營概況調查\n",
    "survey31 = '攤販經營概況調查'\n",
    "code31 = \"(攤販經營概況調查.kw  or 攤販經營概況.kw ) or (攤販經營概況調查.ab) or (攤販經營概況調查.rf)\"\n",
    "\n",
    "# 32.受僱員工薪資調查\n",
    "survey32 = '受僱員工薪資調查'\n",
    "code32 = \"(受僱員工薪資調查.kw  or 受僱員工薪資.kw ) or (受僱員工薪資調查.ab) or (受僱員工薪資調查.rf)\"\n",
    "\n",
    "# 33.工作期望調查\n",
    "survey33 = '工作期望調查'\n",
    "code33 = \"(工作期望調查.kw  or 工作期望.kw ) or (工作期望調查.ab) or (工作期望調查.rf)\"\n",
    "\n",
    "# 34.失業狀況調查\n",
    "survey34 = '工作期望調查'\n",
    "code34 = \"(失業狀況調查.kw) or (失業狀況調查.ab) or (失業狀況調查.rf)\"\n",
    "\n",
    "# 35.住宅狀況調查\n",
    "survey35 = '住宅狀況調查'\n",
    "code35 = \"(住宅狀況調查.kw  or 住宅狀況.kw ) or (住宅狀況調查.ab) or (住宅狀況調查.rf)\"\n",
    "\n",
    "# 36.時間運用調查\n",
    "survey36 = '時間運用調查'\n",
    "code36 = \"(時間運用調查.kw  or 時間運用.kw ) or (時間運用調查.ab) or (時間運用調查.rf)\"\n",
    "\n",
    "# 37.國內遷徙調查\n",
    "survey37 = '國內遷徙調查'\n",
    "code37 = \"(國內遷徙調查.kw  or 國內遷徙.kw ) or (國內遷徙調查.ab) or (國內遷徙調查.rf)\"\n",
    "\n",
    "# 38.台灣貧窮兒少資料庫\n",
    "survey38 = '台灣貧窮兒少資料庫'\n",
    "code38 = \"(貧窮兒少資料庫.kw  or 弱勢兒少.kw ) or (貧窮兒少資料庫.ab or 弱勢兒少資料庫.ab) or (貧窮兒少資料庫.rf or 弱勢兒少資料庫.rf)\"\n",
    "\n",
    "surveyall = '全搜'\n",
    "ult = \"('SRDA'.ti) or ('學術調查研究資料庫' or '調查研究專題中心'.rf)\"\n",
    "\n",
    "\n",
    "browser.find_element(By.LINK_TEXT, \"指令查詢\").click()\n",
    "\n",
    "WebDriverWait(browser, 30).until(  #等待網頁讀取 直到出現 \"ysearchinput0\"\n",
    "    EC.presence_of_element_located((By.ID, \"ysearchinput0\")))\n",
    "\n",
    "gg = ult  #code__       請參照上方計畫參數輸入\n",
    "browser.find_element(By.ID, \"ysearchinput0\").send_keys(gg)\n",
    "cluster = lst2['surveyall']  # () 請輸入計畫號碼   範例 code2  survey2 數字要一樣\n",
    "system = 'ndltd'\n",
    "browser.find_element(By.ID, \"gs32search\").click()\n",
    "\n",
    "WebDriverWait(browser, 30).until(  #等待網頁讀取 直到出現 \"jpsize\"\n",
    "    EC.presence_of_element_located((By.ID, \"jpsize\")))\n",
    "\n",
    "# 設定顯示頁數\n",
    "browser.find_element(By.ID, 'jpsize').click()\n",
    "showpage = 100\n",
    "browser.find_element(By.XPATH, '//option[@value=\"%s\"]' % showpage).click()\n",
    "# browser.find_element(By.XPATH,'//option[@value=\"100\"]').click()\n",
    "\n",
    "#此處選擇性使用\n",
    "#\n",
    "\n",
    "#選擇排序(建議每次都固定)\n",
    "#Select(browser.find_element(By.ID, \"sortby\")).select_by_value(\"-yr\")\n",
    "#選擇跳至第幾頁 (可從上次出錯的頁數開始)\n",
    "#起始頁\n",
    "startpage = 1\n",
    "browser.find_element(By.XPATH, \"//*[@id='jmpage']\").clear()\n",
    "browser.find_element(By.XPATH, \"//*[@id='jmpage']\").send_keys(startpage)\n",
    "browser.find_element(By.NAME, 'jumpfmt1page').click()\n",
    "time.sleep(5.33)\n",
    "\n",
    "#解析html架構並儲存成資料\n",
    "soup = BeautifulSoup(browser.page_source, 'html.parser')\n",
    "# print(soup.prettify())\n",
    "print('搜尋時間： ' + time.strftime(\"%y/%m/%d\") + ' ' + time.strftime(\"%H:%M:%S\"))\n",
    "\n",
    "# 確認頁數\n",
    "#pagevalue = soup.find_all(\"option\", {'selected': True})[2].get(\"value\")\n",
    "# print(type(pagevalue))\n",
    "page_str = soup.find_all(\"span\", class_='etd_e')[1].string\n",
    "page_int = int(page_str)  #筆數\n",
    "page_last = page_int % 100\n",
    "# page_all = (page_int // 100) + 1 #頁數\n",
    "if page_int % 100 == 0:\n",
    "    page_all = page_int // 100\n",
    "else:\n",
    "    page_all = (page_int // 100) + 1  #頁數\n",
    "print('共{}筆'.format(page_int))\n",
    "#print('有{}頁'.format(page_all))\n",
    "# print('最後一頁剩 '+ str(page_last) +' 筆')\n",
    "page = 1\n",
    "\n",
    "res_all = pd.DataFrame()\n",
    "\n",
    "\n",
    "def strip_tags(html):\n",
    "    html = html.strip()\n",
    "    html = html.strip('\\n')\n",
    "    result = []\n",
    "    parser = HTMLParser()\n",
    "    parser.handle_data = result.append\n",
    "    parser.feed(html)\n",
    "    parser.close()\n",
    "    result = ''.join(result)\n",
    "    result = result.replace('\\n', '')\n",
    "    return result\n",
    "\n",
    "\n",
    "def extract_chinese(txt):\n",
    "    pattern = re.compile('[\\u4e00-\\u9fa5]')\n",
    "    return ''.join(pattern.findall(txt))\n",
    "\n",
    "\n",
    "def get_info(soup):\n",
    "    doi = soup.find(\"input\", id=\"fe_text1\").get(\"value\")\n",
    "\n",
    "    #防止論文名稱跳出英文導致錯誤\n",
    "    try:\n",
    "        title = soup.find(\"th\", string=\"論文名稱:\").find_next_sibling(\"td\").text\n",
    "    except:\n",
    "        title = ''\n",
    "    try:\n",
    "        title2 = soup.find(\"th\",\n",
    "                           string=\"論文名稱(外文):\").find_next_sibling(\"td\").text\n",
    "    except:\n",
    "        title2 = ''\n",
    "    #學位類別:碩士或博士\n",
    "    masdoc = soup.find(\"th\", string=\"學位類別:\").find_next_sibling(\"td\").text\n",
    "    # 研究生           (找th='__'  底下的td)\n",
    "    author = soup.find(\"th\", string=\"研究生:\").find_next_sibling(\"td\").text\n",
    "    # 校院名稱\n",
    "    school = soup.find(\"th\", string=\"校院名稱:\").find_next_sibling(\"td\").text\n",
    "    # 系所名稱\n",
    "    info1 = soup.find(\"th\", string=\"系所名稱:\").find_next_sibling(\"td\").text\n",
    "    # 學門\n",
    "    try:\n",
    "        info2 = soup.find(\"th\", string=\"學門:\").find_next_sibling(\"td\").text\n",
    "    except:\n",
    "        info2 = ''\n",
    "    # 學類\n",
    "    try:\n",
    "        info3 = soup.find(\"th\", string=\"學類:\").find_next_sibling(\"td\").text\n",
    "    except:\n",
    "        info3 = ''\n",
    "    #論文出版年\n",
    "    try:\n",
    "        year = soup.find(\"th\", string=\"論文出版年:\").find_next_sibling(\"td\").text\n",
    "    except:\n",
    "        year = ''\n",
    "    #論文種類\n",
    "    typee = soup.find(\"th\", string=\"論文種類:\").find_next_sibling(\"td\").text\n",
    "\n",
    "    WebDriverWait(browser, 30).until(  #等待網頁讀取 直到出現 \"摘要\"\n",
    "        EC.presence_of_element_located(\n",
    "            (By.CSS_SELECTOR, \"#gs32_levelrecord li:nth-child(2) em\")))\n",
    "    #點擊摘要↓\n",
    "    try:\n",
    "        browser.find_element(By.LINK_TEXT, \"摘要\").click()\n",
    "        # 摘要\n",
    "        abst = soup.find('td', {'class': 'stdncl2'}).text\n",
    "    except:\n",
    "        abst = ''\n",
    "    #點擊參考文獻↓\n",
    "    try:\n",
    "        browser.find_element(By.LINK_TEXT, \"參考文獻\").click()\n",
    "        get_cite = soup.find_all(id=\"format0_disparea\")\n",
    "        cite = strip_tags(str(get_cite[4]))\n",
    "        cite = extract_chinese(cite)\n",
    "    except:\n",
    "        cite = ''\n",
    "    result_list.append([\n",
    "        title, school + info1 + info2 + info3, title, doi, cluster, system,\n",
    "        typee, masdoc, year, author, title2, abst, cite\n",
    "    ])\n",
    "    return result_list\n",
    "\n",
    "\n",
    "result_list = []\n",
    "i = 1\n",
    "WebDriverWait(browser, 30).until(  #等待網頁讀取 直到出現 \"XPATH\"\n",
    "    EC.presence_of_element_located((By.XPATH, '//a[@class=\"slink\"]')))\n",
    "browser.find_elements(By.XPATH, '//a[@class=\"slink\"]')[0].click()\n",
    "time.sleep(3.3)\n",
    "WebDriverWait(browser, 30).until(  #等待網頁讀取 直到出現主要表格\n",
    "    EC.presence_of_element_located((\n",
    "        By.XPATH,\n",
    "        \"//*[@id='bodyid']/form[1]/div/table/tbody/tr[1]/td[2]/table/tbody/tr[4]/td/div[1]\"\n",
    "    )))\n",
    "while i <= page_int:\n",
    "    try:\n",
    "        soup = BeautifulSoup(browser.page_source, 'html.parser')\n",
    "        get_info(soup)\n",
    "        i = i + 1\n",
    "        browser.find_element(By.ID, \"jumpage\").clear()\n",
    "        browser.find_element(By.ID, \"jumpage\").send_keys(i)\n",
    "        browser.find_element(By.NAME, \"jumpfmt0page\").click()\n",
    "        time.sleep(3.3)\n",
    "    except:  #出錯會重啟，並且重回頁面\n",
    "        print('在第' + str(int(page)) + '頁' + '第' + str(i) + '筆出錯')\n",
    "        print('Restart')\n",
    "        browser = webdriver.Chrome(\n",
    "            executable_path=r'C:\\Users\\user\\Desktop\\py\\Driver\\chromedriver.exe'\n",
    "        )\n",
    "        browser.get('https://ndltd.ncl.edu.tw/')\n",
    "        browser.maximize_window()\n",
    "        WebDriverWait(browser, 30).until(\n",
    "            EC.presence_of_element_located((By.XPATH, '//a[@title=\"指令查詢\"]')))\n",
    "        browser.find_element(By.XPATH, '//a[@title=\"指令查詢\"]').click()\n",
    "        WebDriverWait(browser, 30).until(\n",
    "            EC.presence_of_element_located((By.ID, 'ysearchinput0')))\n",
    "        browser.find_element(By.ID, 'ysearchinput0').send_keys(gg)\n",
    "        time.sleep(0.5)\n",
    "        browser.find_element(By.ID, 'gs32search').click()\n",
    "        time.sleep(2)\n",
    "        cookie = re.findall(r'ccd=(.*?)/', browser.current_url)[0]\n",
    "        headers = {\n",
    "            'User-Agent':\n",
    "            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36',\n",
    "            'Cookie': 'ccd={}'.format(cookie)\n",
    "        }\n",
    "\n",
    "        payload = {\n",
    "            'qs0': '{}'.format(gg),\n",
    "            'qf0': '_hist_',\n",
    "            'gs32search.x': '27',\n",
    "            'gs32search.y': '9',\n",
    "            'displayonerecdisable': '1',\n",
    "            'dbcode': 'nclcdr',\n",
    "            'action': '',\n",
    "            'op': '',\n",
    "            'h': '',\n",
    "            'histlist': '',\n",
    "            'opt': 'm',\n",
    "            '_status_': 'search__v2'\n",
    "        }\n",
    "        rs = requests.session()\n",
    "        res = rs.post(\n",
    "            'https://ndltd.ncl.edu.tw/cgi-bin/gs32/gsweb.cgi/ccd={}/search'.\n",
    "            format(cookie),\n",
    "            data=payload,\n",
    "            headers=headers)\n",
    "        WebDriverWait(browser, 30).until(  #等待網頁讀取 直到出現 \"XPATH\"\n",
    "        EC.presence_of_element_located((By.XPATH, '//a[@class=\"slink\"]')))\n",
    "        browser.find_elements(By.XPATH, '//a[@class=\"slink\"]')[0].click()\n",
    "        time.sleep(3.3)\n",
    "        print('連回第{}筆資料'.format(i))\n",
    "        browser.find_element(By.ID, \"jumpage\").clear()\n",
    "        browser.find_element(By.ID, \"jumpage\").send_keys(i)\n",
    "        browser.find_element(By.NAME, \"jumpfmt0page\").click()\n",
    "        time.sleep(3.3)\n",
    "        soup = BeautifulSoup(browser.page_source, 'html.parser')\n",
    "        get_info(soup)\n",
    "        i = i + 1\n",
    "        print('準備前往第{}筆資料'.format(i))\n",
    "        browser.find_element(By.ID, \"jumpage\").clear()\n",
    "        browser.find_element(By.ID, \"jumpage\").send_keys(i)\n",
    "        browser.find_element(By.NAME, \"jumpfmt0page\").click()\n",
    "        time.sleep(3.3)\n",
    "        continue\n",
    "\n",
    "result_df = pd.DataFrame(result_list)\n",
    "res_all = res_all.append(result_df)\n",
    "\n",
    "#儲存結果\n",
    "\n",
    "SaveFile_Path = r'C:\\Users\\user\\Desktop\\py\\TEST'  #要儲存檔案的路徑\n",
    "SaveFile_Name = r'碩博'  #儲存的檔名\n",
    "\n",
    "try:\n",
    "    res_all.to_csv(SaveFile_Path + '\\\\' + SaveFile_Name + \"_\" + cluster + \"_\" +\n",
    "                   time.strftime(\"%m%d\") + \"_\" + time.strftime(\"%H%M\") +\n",
    "                   \".csv\",\n",
    "                   encoding=\"utf_8_sig\",\n",
    "                   index=False)  #儲存結果\n",
    "    print(\"爬蟲成功\")  #最終檔名會加上 1.計畫名稱 2.月份 3.日期 4.時間(時分)\n",
    "except:\n",
    "    print(\"爬蟲失敗，請檢察結果檔是否正在開啟，或處於無法被寫入的狀態。\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f378660",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36c22bf1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "963f21c6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}