Crawler.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:3423089d372502a3466747db7e1d83a812560126547c67cd75238ff60aeac95d"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import searchengine"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "crawler = searchengine.crawler(\"test.db\")\n",
      "crawler.createindextables()\n",
      "crawler.importdata(\"data.csv\")\n",
      "crawler.indexurl()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Indexing http://indiatoday.intoday.in/story/obama-pledges-federal-coordination-on-ebola-to-dallas-mayor/1/393946.html\n",
        "Indexing http://indianexpress.com/article/cities/pune/borgaon-village-mourns-death-of-its-hero/"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Indexing http://indianexpress.com/article/cities/pune/game-up-for-kalmadi-his-cong-friends-keep-him-at-arms-length/"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Could not open http://economictimes.indiatimes.com/city/meerut/85-BJP-men-jailed-for-violence-in-Moradabad-to-be-felicitated/articleshow/44131057.cms"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Could not open http://economictimes.indiatimes.com/india/Prime-Minister-Narendra-Modi-kicks-off-Swachh-Bharat-campaign-from-police-station/articleshow/44163572.cms"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Could not open http://economictimes.indiatimes.com/sports/tournaments/asian-games-2014/india-at-incheon/Asian-Games-Boxer-Sarita-Devi-tenders-unconditional-apology-for-rejecting-Asiad-medal/articleshow/44163522.cms"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Could not open http://economictimes.indiatimes.com/city/bangalore/For-rich-Bangalore-kids-its-football-career-over-school/articleshow/44162206.cms"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Indexing http://economictimes.indiatimes.com/markets/stocks/news/fii-flows-hit-7-month-low-on-us-federal-reserve-rate-hike-jitters/articleshow/44143437.cms"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Indexing http://economictimes.indiatimes.com/markets/investors-cut-back-on-stock-market-positions-as-they-wait-for-volatile-end-to-2014/articleshow/44155778.cms"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Indexing http://economictimes.indiatimes.com/markets/stocks/news/dont-be-tempted-by-low-rated-ncds-offering-high-interest-say-experts/articleshow/44144903.cms"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Indexing http://timesofindia.indiatimes.com/india/Prime-Minister-Narendra-Modi-kicks-off-Swachh-Bharat-campaign-from-police-station/articleshow/44163572.cms"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Indexing http://timesofindia.indiatimes.com/articleshow/44162118.cms"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Could not open http://money.cnn.com/2014/10/02/world/asia/china-hong-kong-protests/index.html?hpt=hp_t1"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Could not open http://money.cnn.com/video/data/2.0/video/world/2014/10/02/lklv-ripley-hong-kong-deadline.cnn.html?hpt=hp_t1"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Indexing http://bleacherreport.com/articles/2218862-minnesota-vikings-vs-green-bay-packers-live-score-highlights-and-analysis?utm_source=cnn.com&utm_medium=referral&utm_campaign=cnn-wsport-bin&hpt=isp_bn11"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Could not open http://www.ft.com/cms/s/0/961a31fa-4a7a-11e4-b8bc-00144feab7de.html?ftcamp=published_links%2Frss%2Fcompanies%2Ffeed%2F%2Fproduct&hpt=ibu_bn7"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Indexing http://news-briefs.ew.com/2014/10/02/giudice-sentenced-to-jail/"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Could not open http://entertainthis.usatoday.com/story/news/world/2014/10/02/joshua-wong-hong-kong-protests/16597653/"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Could not open http://entertainthis.usatoday.com/story/news/world/2014/10/02/hong-kong-pro-democracy-protests-china-deadline/16572959/"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Could not open http://entertainthis.usatoday.com/story/news/world/2014/10/02/kurdish-peshmerga-defeat-islamic-state-in-rabia/16610563/"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Could not open http://entertainthis.usatoday.com/story/tech/2014/10/02/home-depot-data-breach-credit-card-fast-food/16435337/"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Could not open http://entertainthis.usatoday.com/story/money/business/2014/10/02/mms-mm-crispy-mars-chocolate-concessions/16604411/"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Could not open http://entertainthis.usatoday.com/story/tech/2014/10/02/home-depot-data-breach-credit-card-fast-food/16435337/"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Indexing http://www.usatoday.com/story/sports/nfl/colts/2014/10/02/colts--dwayne-allen-shares-experience-witnessing-domestic-violence/16614639/"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Could not open http://usat.ly/story/money/cars/2014/10/02/cars-worry-free-index/16604075/"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Indexing http://www.bbc.co.uk/news/business-29470381"
       ]
      }
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "reload(searchengine)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 7,
       "text": [
        "<module 'searchengine' from 'searchengine.pyc'>"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "searcher = searchengine.searcher(\"searchengindex.db\")\n",
      "searcher.query(\"ebola\")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "select w0.urlid,w0.location from wordlocation  w0 where w0.wordid=21\n",
        "2.244186\thttp://abcnews.go.com/Health/texas-hospital-blames-records-flaw-reason-ebola-patient/story?id=25912405\n",
        "2.186047\thttp://abcnews.go.com/Health/family-identifies-american-journalist-tested-positive-ebola/story?id=25930807\n",
        "2.186047\thttp://abcnews.go.com/Health/american-journalist-liberia-tests-positive-ebola/story?id=25930807\n",
        "2.162791\thttp://abcnews.go.com/Health/man-thought-died-ebola-awakes-burial-team-arrives/story?id=25915800\n",
        "2.081395\thttp://abcnews.go.com/Health/wireStory/nbc-cameraman-tests-positive-ebola-liberia-25931383\n",
        "2.034884\thttp://abcnews.go.com/International/wireStory/australian-parliament-restricts-veiled-visitors-25910149\n",
        "2.034884\thttp://abcnews.go.com/Health/wireStory/fewer-texas-abortion-providers-court-ruling-25934479\n",
        "2.034884\thttp://abcnews.go.com/Sports/wireStory/royals-angels-head-extras-alds-opener-25934044\n",
        "2.034884\thttp://abcnews.go.com/International/wireStory/vietnam-welcomes-us-partial-lift-arms-sales-ban-25934480\n",
        "2.034884\thttp://abcnews.go.com/US/wireStory/judge-tosses-state-suit-california-egg-law-25933710\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 14,
       "text": [
        "([21], [318, 581, 25, 319, 182, 908, 907, 906, 905, 904])"
       ]
      }
     ],
     "prompt_number": 14
    }
   ],
   "metadata": {}
  }
 ]
}