ψ(｀∇´)ψ_PROQ_EXPORT.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d31983a0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Temp\\TEMP\\ipykernel_20848\\1392897550.py:63: DeprecationWarning: executable_path has been deprecated, please pass in a Service object\n",
      "  browser = webdriver.Chrome(path, chrome_options=chrome_options)\n",
      "C:\\Temp\\TEMP\\ipykernel_20848\\1392897550.py:63: DeprecationWarning: use options instead of chrome_options\n",
      "  browser = webdriver.Chrome(path, chrome_options=chrome_options)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "搜尋時間： 22/11/15 09:04:25\n",
      "共 13 筆\n",
      "有 1 頁\n",
      "最後一頁剩 13 筆\n",
      "此為最後一頁\n",
      "成功爬取PROQ-台灣貧窮兒少資料庫(TDCYP)\n"
     ]
    }
   ],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.service import Service\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import Select\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.webdriver.common.action_chains import ActionChains\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "import selenium.webdriver.support.ui as ui# 必要每五秒檢元素是否存在\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from glob import glob\n",
    "import time\n",
    "import random\n",
    "import requests\n",
    "import os\n",
    "import glob\n",
    "from html.parser import HTMLParser\n",
    "import re\n",
    "\n",
    "# 1. 台灣社會變遷基本調查(TSCS)\n",
    "# 2. 華人家庭動態資料庫(PSFD)\n",
    "# 3. 台灣教育長期追蹤(TEPS)\n",
    "# 4. 特殊教育長期追蹤資料庫(SNELS)\n",
    "# 5. 家庭收支調查(SFIE)\n",
    "# 6. 台灣世界價值觀調查(TWVS)                \n",
    "# 7. 台灣青少年成長歷程研究(TYP)\n",
    "# 8. 台灣選舉調查資料(TEDS)\n",
    "# 9. 「台灣教育長期追蹤資料庫」後續調查(TEPS-B)\n",
    "# 10. 人力資源調查(MPS)\n",
    "# 11. 人力運用調查(MUS)\n",
    "# 12. 婦女婚育與就業調查(WMFES)\n",
    "# 13. 個人家戶數位機會(落差)(數位落差)(DDS)\n",
    "# 14. 台灣貧窮兒少資料庫(TDCYP)\n",
    "# 15. 台灣傳播調查資料庫(TCS)\n",
    "# 16. International Social Survey Programme (ISSP)\n",
    "# 17. 幼兒發展調查資料庫建置計畫\n",
    "# 18. 國人旅遊狀況調查\n",
    "# 19. 來臺旅客消費及動向調查\n",
    "# 20. 台灣社會意向調查\n",
    "# 21. 台灣營養健康狀況變遷調查\n",
    "# 22. 我國大學生政治社會化的定群追蹤研究\n",
    "# 23. 台灣政府文官調查\n",
    "# 24. 青少年藥物濫用之起因\n",
    "\n",
    "gg = 14  #輸入想要的叢集\n",
    "\n",
    "download_path = r'C:\\Users\\user\\Desktop\\py\\衍伸書目\\proquest'\n",
    "code_path = r'C:\\Users\\user\\Desktop\\py\\指令整理.xlsx'\n",
    "code = pd.read_excel(code_path)\n",
    "keys = code[96:120]['Final'].iloc[gg - 1]\n",
    "dataset = code[96:120]['Dataset'].iloc[gg - 1]\n",
    "SaveFile_Path = r'C:\\Users\\user\\Desktop\\py\\衍伸書目\\proquest'  #檔案下載位置\n",
    "SaveFile_Name = code[96:120]['Dataset'].iloc[gg - 1]\n",
    "\n",
    "# 開啟chromedriver，連到目標網頁\n",
    "chrome_options = webdriver.ChromeOptions()\n",
    "prefs = {'download.default_directory': str(download_path)}\n",
    "chrome_options.add_experimental_option('prefs', prefs)\n",
    "download_path = r'C:\\Users\\user\\Desktop\\py\\衍伸書目\\proquest'\n",
    "path = r'C:\\Users\\user\\Desktop\\py\\Driver\\chromedriver.exe'\n",
    "browser = webdriver.Chrome(path, chrome_options=chrome_options)\n",
    "browser.get('https://search.proquest.com/index')\n",
    "time.sleep(5)\n",
    "browser.find_element(By.CSS_SELECTOR, \"#mrDropdown > .uxf-icon\").click()\n",
    "time.sleep(5)\n",
    "browser.find_element(By.LINK_TEXT, \"登入 [我的檢索]\").click()\n",
    "time.sleep(5)\n",
    "browser.find_element(By.ID, \"1-email\").send_keys(\"zowieergotech@gmail.com\")\n",
    "browser.find_element(By.NAME, \"password\").send_keys(\"@St760cso\")\n",
    "browser.find_element(By.NAME, \"submit\").click()\n",
    "time.sleep(5)\n",
    "browser.find_element(By.CSS_SELECTOR, \".pq-header-logo\").click()\n",
    "# 模擬操作：指令查詢 ＞ 輸入指令 ＞ Search ＞ 每頁顯示100筆\n",
    "time.sleep(5)\n",
    "browser.find_element(By.ID, \"queryTermField\").send_keys(keys)\n",
    "browser.find_element(By.ID, \"searchToResultPage\").click()\n",
    "time.sleep(5)\n",
    "# 解析第一頁，取得總頁數，計算要點幾次，以及最後一次剩幾筆\n",
    "soup = BeautifulSoup(browser.page_source, 'html.parser')\n",
    "\n",
    "# 日期\n",
    "print('搜尋時間： ' + time.strftime(\"%y/%m/%d\") + ' ' + time.strftime(\"%H:%M:%S\"))\n",
    "\n",
    "#頁數\n",
    "\n",
    "page_str = soup.find('h1', {'id': 'pqResultsCount'})\n",
    "page_text = page_str.text\n",
    "pattern = \"[0-9]+\"\n",
    "page_re = re.findall(pattern, page_text)[0]  #利用正規表達式抓出文字串的數字\n",
    "page_int = int(page_re)\n",
    "page_all = (page_int // 100) + 1\n",
    "page_last = page_int % 100\n",
    "print('共 ' + str(page_int) + ' 筆')\n",
    "print('有 ' + str(page_all) + ' 頁')\n",
    "print('最後一頁剩 ' + str(page_last) + ' 筆')\n",
    "\n",
    "def download_file():\n",
    "    soup = BeautifulSoup(browser.page_source, 'html.parser')\n",
    "    browser.find_element(By.CSS_SELECTOR, \"#mlcbAll\").click()\n",
    "    browser.find_element(By.CSS_SELECTOR, \".uxf-dot-three\").click()\n",
    "    time.sleep(10)\n",
    "    browser.find_element(By.XPATH, \"/html/body/div[4]/div[1]/div/div[7]/div[7]/div[3]/div[1]/div[2]/div[1]/div/div/div/div[3]/div/div[5]/div[1]/div[2]/div/div[2]/div[3]/div[3]/a/span[1]/span\").click()\n",
    "    time.sleep(10)\n",
    "    window_before = browser.window_handles[0]\n",
    "    time.sleep(16)\n",
    "    browser.find_element(By.XPATH, \"/html/body/div[4]/div[1]/div/div[7]/div[7]/div[3]/div[1]/div[2]/div[1]/div/div/div/div[3]/div/div[6]/div/div[2]/div/div[2]/div/div/div[2]/div[1]/form/div[1]/div[1]/div[3]/input\").click()\n",
    "    browser.find_element(By.XPATH, \"/html/body/div[4]/div[1]/div/div[7]/div[7]/div[3]/div[1]/div[2]/div[1]/div/div/div/div[3]/div/div[6]/div/div[2]/div/div[2]/div/div/div[2]/div[1]/form/div[3]/div[2]/a\").click()\n",
    "    time.sleep(20)\n",
    "    window_after = browser.window_handles[1]\n",
    "    time.sleep(20)\n",
    "    browser.switch_to.window(window_after)\n",
    "    browser.close()\n",
    "    browser.switch_to.window(window_before)\n",
    "\n",
    "def close_driver():\n",
    "    browser.quit()\n",
    "    print('成功爬取PROQ-'+ dataset)    \n",
    "\n",
    "for page in range(1, page_all+1):\n",
    "    if page == page_all:\n",
    "        print('此為最後一頁')\n",
    "        download_file()\n",
    "        break\n",
    "    else:\n",
    "        download_file()\n",
    "        browser.find_element(By.XPATH, \"//*[@id='updateForm']/nav/ul/li[9]/a\").click()\n",
    "        time.sleep(5)\n",
    "        print('爬取第{}頁'.format(page))\n",
    "        continue    \n",
    "close_driver()\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab1c6431",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0cd6ba4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc294c89",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22af24dd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}