Savannah Usernames for Contextual Stream.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json, sys\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'peachy92,zone6nova,SamGlizzy,RealSavyJenny,SMNJuliaMuller,MrShields2,ancrum_jordan,savannahdammit ,Courtneyyy_2030,kristensnett,MrRADRC,cjb715,charmantmaries,dannyrolltribe,rb5235,ovavanilla,stephmsloan,C00PTR00P,savannahxr0se,Kerky_,SOLDinSavannah,_MarimamaKiki,JoeyBlair8,ShebbyBarks,_dezuhray,SoberDOGS,UmaineMark72,AddieMae08,Karlinishere,DatBoyNique12,giannnarosee,KindofABigSeal,imaginativemad,aimspdf,emma_sersich,Rhedivivus,charmantmaries,YeahThatsJosh ,yanajones,justintlittle,nmavs_,JessicaOlliff,simplysavvvv,jallyw11,PatrickBergen,rhibloui,savannahseymour,dcbertagnolli01,Everett54R,VV2r101,KitSeaton,euphemiisms,geandraimani,DevotedDeanGirl,occasionallyzen,BrennaCeDria,2percent_wilk,The5H31LA,jasonanavitarte,sarahlbourque,ashtaaelisabeth,eli_wally,sa_dreww,findmarcusm,JHGrove3,KevAnderson10,ambiedextris,druidpup,kmartSMN,jessebushkar,amarieproctor,meggreg7,rice_taylor,JRBisMe,Jaylee023,bloodless_coup,cportsownmara,baseball_girly2,quasigrownup,LowKeyyWill305,brittanytaylor3,BiancaModelChic,LTCjRet,BeauNurr,TBirdInn,halle_michelle,Cipher_J,dl1288,KSullivanLingle,sa__murphy,WednesdayElf,Tre_Gorgeous,ParisB_6,ajaye_doe,88furiae,RintaroNow,RKJackson75,garrulousbanter,VanRJohnson,geeeali,SelenaShines,cmitch,Sahernn,KatherinegWalsh,BitterSongSC,haydensanders19,_wxnderlvst_,DVBaxter,BIG_BAUM_,__Rezzz,MariaLum9,cindy_jeans,katie_allen__,s0uthernvibes,rolyaat_,SammyKeener,predsgal,Carl_noWinslow,LLCoolLucy,Rorymarieee,KingChadwic,Z_Ztopp,NephthysPhoenix,Swolo912,crystal_cassels,p_cucumbers,jaya_ballard,julieetangoo,_LilMamaSims,macanales99,MeederNick,_samlopolito,DixonCindy,haleighhickman,QuakesBlogger,_AmnPakIsh,OnlyTechnophile,ErikReagan,tabitha1211,Tika_Boss4U,MaryGraceSasser,_natks,carokaay,MarriedJen,DashiellColeman,yelvington,espirituzander,ShipShippity,tcjsavannah,wellseasondlife,monigheandonn3,timothycookjr1,burkvest,inamerate,EmilySeigenberg,MaryLandersSMN,BrookeManolis,kate_grubbs,terinsimone,drealities,_shakiradeshon,traci008,nose_inthee_air,dwhiteonc,Kyradan,ShezBourgie,HHI_FnB,peck0nthecheek,MiiranAway,Thebittersteel,kevinklink,joshuagrotheer,IanBrown1229,brittaniwest,Loraleilanier,xerchisha,sugarfoot85,ThatDam_Keisha,dishonesstly,Juztmx,TheNamesBrad,Vashti_Bester,caileenage,chauntilena,c_neighbors,BennettMab,Ady_Solo,JPennsy,LaurenPetracca,GACoastPatriot,JignaPatel,hgmeyers,Troopa8W,WarrenC46rose,DeLaSelis,JuanCavsFan,faraway0326,brandischroeder,davekartunen,jasonluckydog,Kaps_Locked,lcaitlyn_wilson,golfrnsavh,speciaaalK_,KennyRoofing,Southerngirl829,SheilaTheBold1,Lauray427,Frost_Scope,layinpipebombs,jking065,BlondzDoHave,JoshAids,AlisonFredKirk,CedricSmithArt,duzins,sean319,mgasav,Rae_sharkey8,Generators_Fast,heartsdesire456,arrington_alyse,lisasolod,Emily_Marie_,Kyle__Lanier,Steelersdepot,amieflanagan,GravyFarce,bonsha9,hal_mccarthy,theGEGEeffect,alexeller99,DavidWithun,iAM_Buchanan,upfront_blunt,sirGR8NESS,afnewton,bonniegodbee,_AndreStafford_,BMPGA99,mswhystress11,im_shakur_,prem1ertime,VinayHarpalani,rb5235,_RLTRA_,TatlyfeStudios,pamela1101,ty_stunning,KLTheEverChosen,ShayBananas,Lowcountrydawg,NoellBarnidge,ramani_sid,savsandy,_blvckpearl,quisie3bryant,RaRa07Star,MollyBozeman,AzulayRomond,eecwilson,ninjaj03,nancymccarley,QueenBishJay,jackietherobot,emilylecomte,ejreports,JMSim70,larsen_ken,SavannahLodging,ShotsofTequila_,fritztheyodeler,EnochaEdenfield,PatriciaBynes,Ammar6557,Emulhh,Rachel_Thomps,linaberry,aprilinoctober,PostGradMoves,RavnosNouveau,Bicyclebythesea,beghead,mbwoody02,onlymonae,VEMA_Super7,MeganCanady,heavenslilangl,Tinkfan79,MeghannCalhoun,MorganElektra,I_am_Jr_Myers,_stephencurry,Craneman51M,risunokairu,MrsCaudill,FitzTrubey,LizaDiMarco,CJFoster2,thehoneybeeeee,myles_rmitchell,bullett420,biggyswoles,KF4LMT,LuValW,CodyHausherr,_Ashley_Simone,e_br0wnn,teratoaster,EVSavannahGA,dancrodev,BeauTnBargains,AsukaKojou,BaseballDesigns,whiskeyforone,MsMelanieGrace,MusicSoul_Child,roughcopydotcom,RyoTenArmy,jordan_dominy'"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "userstring = 'peachy92,zone6nova,SamGlizzy,RealSavyJenny,SMNJuliaMuller,MrShields2,ancrum_jordan,savannahdammit ,Courtneyyy_2030,kristensnett,MrRADRC,cjb715,charmantmaries,dannyrolltribe,rb5235,ovavanilla,stephmsloan,C00PTR00P,savannahxr0se,Kerky_,SOLDinSavannah,_MarimamaKiki,JoeyBlair8,ShebbyBarks,_dezuhray,SoberDOGS,UmaineMark72,AddieMae08,Karlinishere,DatBoyNique12,giannnarosee,KindofABigSeal,imaginativemad,aimspdf,emma_sersich,Rhedivivus,charmantmaries,YeahThatsJosh ,yanajones,justintlittle,nmavs_,JessicaOlliff,simplysavvvv,jallyw11,PatrickBergen,rhibloui,savannahseymour,dcbertagnolli01,Everett54R,VV2r101,KitSeaton,euphemiisms,geandraimani,DevotedDeanGirl,occasionallyzen,BrennaCeDria,2percent_wilk,The5H31LA,jasonanavitarte,sarahlbourque,ashtaaelisabeth,eli_wally,sa_dreww,findmarcusm,JHGrove3,KevAnderson10,ambiedextris,druidpup,kmartSMN,jessebushkar,amarieproctor,meggreg7,rice_taylor,JRBisMe,Jaylee023,bloodless_coup,cportsownmara,baseball_girly2,quasigrownup,LowKeyyWill305,brittanytaylor3,BiancaModelChic,LTCjRet,BeauNurr,TBirdInn,halle_michelle,Cipher_J,dl1288,KSullivanLingle,sa__murphy,WednesdayElf,Tre_Gorgeous,ParisB_6,ajaye_doe,88furiae,RintaroNow,RKJackson75,garrulousbanter,VanRJohnson,geeeali,SelenaShines,cmitch,Sahernn,KatherinegWalsh,BitterSongSC,haydensanders19,_wxnderlvst_,DVBaxter,BIG_BAUM_,__Rezzz,MariaLum9,cindy_jeans,katie_allen__,s0uthernvibes,rolyaat_,SammyKeener,predsgal,Carl_noWinslow,LLCoolLucy,Rorymarieee,KingChadwic,Z_Ztopp,NephthysPhoenix,Swolo912,crystal_cassels,p_cucumbers,jaya_ballard,julieetangoo,_LilMamaSims,macanales99,MeederNick,_samlopolito,DixonCindy,haleighhickman,QuakesBlogger,_AmnPakIsh,OnlyTechnophile,ErikReagan,tabitha1211,Tika_Boss4U,MaryGraceSasser,_natks,carokaay,MarriedJen,DashiellColeman,yelvington,espirituzander,ShipShippity,tcjsavannah,wellseasondlife,monigheandonn3,timothycookjr1,burkvest,inamerate,EmilySeigenberg,MaryLandersSMN,BrookeManolis,kate_grubbs,terinsimone,drealities,_shakiradeshon,traci008,nose_inthee_air,dwhiteonc,Kyradan,ShezBourgie,HHI_FnB,peck0nthecheek,MiiranAway,Thebittersteel,kevinklink,joshuagrotheer,IanBrown1229,brittaniwest,Loraleilanier,xerchisha,sugarfoot85,ThatDam_Keisha,dishonesstly,Juztmx,TheNamesBrad,Vashti_Bester,caileenage,chauntilena,c_neighbors,BennettMab,Ady_Solo,JPennsy,LaurenPetracca,GACoastPatriot,JignaPatel,hgmeyers,Troopa8W,WarrenC46rose,DeLaSelis,JuanCavsFan,faraway0326,brandischroeder,davekartunen,jasonluckydog,Kaps_Locked,lcaitlyn_wilson,golfrnsavh,speciaaalK_,KennyRoofing,Southerngirl829,SheilaTheBold1,Lauray427,Frost_Scope,layinpipebombs,jking065,BlondzDoHave,JoshAids,AlisonFredKirk,CedricSmithArt,duzins,sean319,mgasav,Rae_sharkey8,Generators_Fast,heartsdesire456,arrington_alyse,lisasolod,Emily_Marie_,Kyle__Lanier,Steelersdepot,amieflanagan,GravyFarce,bonsha9,hal_mccarthy,theGEGEeffect,alexeller99,DavidWithun,iAM_Buchanan,upfront_blunt,sirGR8NESS,afnewton,bonniegodbee,_AndreStafford_,BMPGA99,mswhystress11,im_shakur_,prem1ertime,VinayHarpalani,rb5235,_RLTRA_,TatlyfeStudios,pamela1101,ty_stunning,KLTheEverChosen,ShayBananas,Lowcountrydawg,NoellBarnidge,ramani_sid,savsandy,_blvckpearl,quisie3bryant,RaRa07Star,MollyBozeman,AzulayRomond,eecwilson,ninjaj03,nancymccarley,QueenBishJay,jackietherobot,emilylecomte,ejreports,JMSim70,larsen_ken,SavannahLodging,ShotsofTequila_,fritztheyodeler,EnochaEdenfield,PatriciaBynes,Ammar6557,Emulhh,Rachel_Thomps,linaberry,aprilinoctober,PostGradMoves,RavnosNouveau,Bicyclebythesea,beghead,mbwoody02,onlymonae,VEMA_Super7,MeganCanady,heavenslilangl,Tinkfan79,MeghannCalhoun,MorganElektra,I_am_Jr_Myers,_stephencurry,Craneman51M,risunokairu,MrsCaudill,FitzTrubey,LizaDiMarco,CJFoster2,thehoneybeeeee,myles_rmitchell,bullett420,biggyswoles,KF4LMT,LuValW,CodyHausherr,_Ashley_Simone,e_br0wnn,teratoaster,EVSavannahGA,dancrodev,BeauTnBargains,AsukaKojou,BaseballDesigns,whiskeyforone,MsMelanieGrace,MusicSoul_Child,roughcopydotcom,RyoTenArmy,jordan_dominy'\n",
    "userstring"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "320"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "userNames = userstring.split(\",\")\n",
    "len(userNames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LOOKING FOR 318 users\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " Processed 13830000 tweets, 0 users left to findd"
     ]
    }
   ],
   "source": [
    "count = 0;\n",
    "users = { user.lower().strip(): {} for user in userNames }\n",
    "print(\"LOOKING FOR {0} users\".format(len(users)))\n",
    "# user_tuples = []\n",
    "with open(\"savannah_usernames_for_contextual_consumption_4.txt\",'w') as oFile:\n",
    "    for line in open('/data/chime/matthew/keyword_tweets_full_gnip_format.jsonl'):\n",
    "        t = json.loads(line.strip())\n",
    "        count += 1\n",
    "        if (t['actor']['preferredUsername'].lower()) in users:\n",
    "            oFile.write(\"{0},{1}\\n\".format(t['actor']['preferredUsername'],t['actor']['id'].split(\":\")[2]))\n",
    "            oFile.flush()\n",
    "            del users[t['actor']['preferredUsername'].lower()]\n",
    "        if count%10000==0:\n",
    "            sys.stderr.write(\"\\r Processed {0} tweets, {1} users left to find\".format(count, len(users)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "in_users = set(users.keys())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Debugging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "id_users = []\n",
    "with open('savannah_usernames_for_contextual_consumption3.txt') as inFile:\n",
    "    for line in inFile:\n",
    "        id_users.append(line.split(\",\")[0].lower())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'_amnpaklsh',\n",
       " 'ashtaeelisabeth',\n",
       " 'justinlittle',\n",
       " 'sa_murphy',\n",
       " 'savannahdammit ',\n",
       " 'shawybananas',\n",
       " 'shotsoftequila',\n",
       " 'snebbybarks',\n",
       " 'yeahthatsjosh '}"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "in_users - set(id_users)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}