CRF model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pycrfsuite\n",
    "import numpy as np\n",
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"finalData.tsv\", 'r') as fp:\n",
    "\tdata = fp.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(data)):\n",
    "\tdata[i] = data[i].strip('\\n')\n",
    "\tdata[i] = data[i].split('\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1489\n"
     ]
    }
   ],
   "source": [
    "tweets = []\n",
    "currPoint = []\n",
    "\n",
    "for token in data:\n",
    "\tif token[0] == '':\n",
    "\t\tif len(currPoint) > 0:\n",
    "\t\t\ttweets.append(currPoint)\n",
    "\t\t\tcurrPoint = []\n",
    "\telse:\n",
    "\t\tcurrPoint.append(token)\n",
    "print len(tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets = np.array(tweets)\n",
    "np.random.seed(52)\n",
    "np.random.shuffle(tweets)\n",
    "tweets = tweets.tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def asciiPercentage(s):\n",
    "\tcount = 0.\n",
    "\tfor char in s:\n",
    "\t\tif ord(char) < 128:\n",
    "\t\t\tcount += 1\n",
    "\treturn count/len(s)\n",
    "\n",
    "def vowelPercentage(s):\n",
    "\tvowels = \"aeiou\"\n",
    "\tcount = 0.\n",
    "\tfor char in s:\n",
    "\t\tif char in vowels:\n",
    "\t\t\tcount += 1\n",
    "\treturn count/len(s)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature extractor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def word2features(sent, i):\n",
    "\n",
    "\t# feature vector\n",
    "\t# word, pos, lang\n",
    "\n",
    "    word = sent[i][0]\n",
    "    wordClean = ''.join([ch for ch in word if ch in 'asdfghjklqwertyuiopzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM']).lower()\n",
    "    normalizedWord = wordClean.lower()\n",
    "    \n",
    "    anyCap = any(char.isupper() for char in word)\n",
    "    allCap = all(char.isupper() for char in word)\n",
    "    hasSpecial = any(ord(char) > 32 and ord(char) < 65 for char in word)\n",
    "    lang = sent[i][1]\n",
    "    \n",
    "    hashTag = word[0] == '#'\n",
    "    mention = word[0] == '@'\n",
    "    \n",
    "    \n",
    "    features = {'word' : word, 'wordClean' : wordClean, 'normalizedWord' : normalizedWord, \\\n",
    "                'lang' : lang,\n",
    "                'isTitle' : word.istitle(), 'wordLength' : len(word), \\\n",
    "                'anyCap' : anyCap, 'allCap' : word.isupper(),\n",
    "                'hasSpecial' : hasSpecial, 'asciiPer' : asciiPercentage(word)}\n",
    "    \n",
    "    \n",
    "#     features['suffix5'] = word[-5:]\n",
    "#     features['prefix5'] = word[:5]\n",
    "#     features['suffix4'] = word[-4:]\n",
    "#     features['prefix4'] = word[:4]\n",
    "    features['suffix3'] = word[-3:]\n",
    "    features['prefix3'] = word[:3]\n",
    "    features['suffix2'] = word[-2:]\n",
    "    features['prefix2'] = word[:2]\n",
    "    features['suffix1'] = word[-1:]\n",
    "    features['prefix1'] = word[:1]  \n",
    "    \n",
    "    return features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extracting features from Sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sent2features(sent):\n",
    "\tfeatures = []\n",
    "\n",
    "\tfor i in range(len(sent)):\n",
    "\t\tfeatures.append(word2features(sent, i))\n",
    "\n",
    "\treturn features\n",
    "\n",
    "def sent2labels(sent):\n",
    "\tallLabels = []\n",
    "\n",
    "\tfor i in sent:\n",
    "\t\tallLabels.append(i[2])\n",
    "\n",
    "\treturn allLabels\n",
    "\n",
    "def sent2tokens(sent):\n",
    "\n",
    "\tallTokens = []\n",
    "\n",
    "\tfor i in sent:\n",
    "\t\tallTokens.append(i[0])\n",
    "\n",
    "\treturn allTokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Params; obtained from Grid Search\n",
    "\n",
    "c1 = 0.0001\n",
    "c2 = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweetsVal = tweets[int(len(tweets) * 0.8):]\n",
    "tweets = tweets[:int(len(tweets) * 0.8)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(298, 1191)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tweetsVal), len(tweets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cross validation 0 for c1 : 0.0001 c2 : 0.1\n",
      "--> Extracting Train Set ...\n",
      "--> Extracting Test Set ...\n",
      "--> Loading CRF module ...\n",
      "Training ...\n",
      "Testing ...\n",
      "cross validation 1 for c1 : 0.0001 c2 : 0.1\n",
      "--> Extracting Train Set ...\n",
      "--> Extracting Test Set ...\n",
      "--> Loading CRF module ...\n",
      "Training ...\n",
      "Testing ...\n",
      "cross validation 2 for c1 : 0.0001 c2 : 0.1\n",
      "--> Extracting Train Set ...\n",
      "--> Extracting Test Set ...\n",
      "--> Loading CRF module ...\n",
      "Training ...\n",
      "Testing ...\n",
      "cross validation 3 for c1 : 0.0001 c2 : 0.1\n",
      "--> Extracting Train Set ...\n",
      "--> Extracting Test Set ...\n",
      "--> Loading CRF module ...\n",
      "Training ...\n",
      "Testing ...\n",
      "cross validation 4 for c1 : 0.0001 c2 : 0.1\n",
      "--> Extracting Train Set ...\n",
      "--> Extracting Test Set ...\n",
      "--> Loading CRF module ...\n",
      "Training ...\n",
      "Testing ...\n",
      " CRF Classification\n",
      "c1 : 0.0001 c2 : 0.1\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        ADJ     0.7218    0.6756    0.6979      1233\n",
      "        ADP     0.9273    0.9273    0.9273      2379\n",
      "        ADV     0.8472    0.7861    0.8155       832\n",
      "       CONJ     0.9517    0.8992    0.9247       635\n",
      "        DET     0.9213    0.8919    0.9064       879\n",
      "       NOUN     0.8052    0.8409    0.8227      3997\n",
      "        NUM     0.9590    0.8836    0.9198       318\n",
      "       PART     0.8023    0.6888    0.7412      1131\n",
      "   PART_NEG     0.9601    0.9837    0.9717       367\n",
      "       PRON     0.9028    0.8598    0.8808      1177\n",
      "    PRON_WH     0.9362    0.9139    0.9249       337\n",
      "      PROPN     0.9072    0.9185    0.9128      2171\n",
      "       VERB     0.8430    0.8928    0.8672      4822\n",
      "          X     0.9927    0.9850    0.9888      6074\n",
      "\n",
      "avg / total     0.8895    0.8892    0.8888     26352\n",
      "\n"
     ]
    }
   ],
   "source": [
    "k = 5\n",
    "\n",
    "chunk = len(tweets) / k\n",
    "results = []\n",
    "\n",
    "allTestPredictions = []\n",
    "allTestGroundTruth = []\n",
    "\n",
    "for i in range(k):\n",
    "\n",
    "    print \"cross validation\", i, 'for', 'c1 :', c1, 'c2 :', c2\n",
    "\n",
    "    test_sents = tweets[i * chunk : (i + 1) * chunk]\n",
    "    train_sents = tweets[:i * chunk] + tweets[(i + 1) * chunk:]\n",
    "\n",
    "    print \"--> Extracting Train Set ...\"\n",
    "    X_train = [sent2features(s) for s in train_sents]\n",
    "    y_train = [sent2labels(s) for s in train_sents]\n",
    "\n",
    "    print \"--> Extracting Test Set ...\"\n",
    "    X_test = [sent2features(s) for s in test_sents]\n",
    "    y_test = [sent2labels(s) for s in test_sents]\n",
    "\n",
    "    print \"--> Loading CRF module ...\"\n",
    "    trainer = pycrfsuite.Trainer(verbose=False)\n",
    "\n",
    "    for xseq, yseq in zip(X_train, y_train):\n",
    "        trainer.append(xseq, yseq)\n",
    "\n",
    "    trainer.set_params({\n",
    "        'c1': c1,   # coefficient for L1 penalty\n",
    "        'c2': c2,  # coefficient for L2 penalty\n",
    "        'max_iterations': 1000,  # stop earlier\n",
    "\n",
    "        # include transitions that are possible, but not observed\n",
    "        'feature.possible_transitions': True,\n",
    "        'feature.possible_states' : True\n",
    "    })\n",
    "\n",
    "    print \"Training ...\"\n",
    "    trainer.train('pos_crf_' + str(i))\n",
    "\n",
    "    print \"Testing ...\"\n",
    "    tagger = pycrfsuite.Tagger()\n",
    "    tagger.open('pos_crf_' + str(i))\n",
    "\n",
    "    y_pred = []\n",
    "\n",
    "    for xseq in X_test:\n",
    "        y_pred.append(tagger.tag(xseq))\n",
    "\n",
    "    \"\"\" CRF based classification \"\"\"\n",
    "\n",
    "    predictedLabels = []\n",
    "    correctLabels = []\n",
    "\n",
    "    for i in y_pred:\n",
    "        for j in i:\n",
    "            predictedLabels.append(j)\n",
    "            allTestPredictions.append(j)\n",
    "\n",
    "    for i in y_test:\n",
    "        for j in i:\n",
    "            correctLabels.append(j)\n",
    "            allTestGroundTruth.append(j)\n",
    "\n",
    "print \"\"\" CRF Classification\"\"\"\n",
    "print 'c1 :', c1, 'c2 :', c2\n",
    "print classification_report(allTestGroundTruth, allTestPredictions, digits = 4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--> Extracting Train Set ...\n",
      "--> Loading CRF module ...\n",
      "Training ...\n",
      "Testing ...\n",
      " CRF Classification\n",
      "c1 : 0.0001 c2 : 0.1\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        ADJ     0.7492    0.7492    0.7492       303\n",
      "        ADP     0.9283    0.9468    0.9375       602\n",
      "        ADV     0.8653    0.7877    0.8247       212\n",
      "       CONJ     0.9542    0.9182    0.9359       159\n",
      "        DET     0.9091    0.9009    0.9050       222\n",
      "       NOUN     0.8242    0.8581    0.8408       994\n",
      "        NUM     0.9710    0.8481    0.9054        79\n",
      "       PART     0.8249    0.7465    0.7837       284\n",
      "   PART_NEG     0.9659    1.0000    0.9827        85\n",
      "       PRON     0.9412    0.8481    0.8922       283\n",
      "    PRON_WH     0.9500    0.9620    0.9560        79\n",
      "      PROPN     0.9265    0.9180    0.9222       549\n",
      "       VERB     0.8581    0.8997    0.8784      1156\n",
      "          X     0.9930    0.9831    0.9880      1593\n",
      "\n",
      "avg / total     0.9029    0.9020    0.9020      6600\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print \"--> Extracting Train Set ...\"\n",
    "X_train = [sent2features(s) for s in tweets]\n",
    "y_train = [sent2labels(s) for s in tweets]\n",
    "\n",
    "X_test = [sent2features(s) for s in tweetsVal]\n",
    "y_test = [sent2labels(s) for s in tweetsVal]\n",
    "\n",
    "print \"--> Loading CRF module ...\"\n",
    "trainer = pycrfsuite.Trainer(verbose=False)\n",
    "for xseq, yseq in zip(X_train, y_train):\n",
    "    trainer.append(xseq, yseq)\n",
    "\n",
    "trainer.set_params({\n",
    "    'c1': c1,   # coefficient for L1 penalty\n",
    "    'c2': c2,  # coefficient for L2 penalty\n",
    "    'max_iterations': 1000,  # stop earlier\n",
    "\n",
    "    # include transitions that are possible, but not observed\n",
    "    'feature.possible_transitions': True,\n",
    "    'feature.possible_states' : True\n",
    "})\n",
    "\n",
    "print \"Training ...\"\n",
    "trainer.train('pos_crf')\n",
    "print \"Testing ...\"\n",
    "tagger = pycrfsuite.Tagger()\n",
    "tagger.open('pos_crf')\n",
    "\n",
    "allTestPredictions = []\n",
    "allTestGroundTruth = []\n",
    "y_pred = []\n",
    "\n",
    "for xseq in X_test:\n",
    "    y_pred.append(tagger.tag(xseq))\n",
    "    \n",
    "predictedLabels = []\n",
    "correctLabels = []\n",
    "\n",
    "for i in y_pred:\n",
    "    for j in i:\n",
    "        predictedLabels.append(j)\n",
    "        allTestPredictions.append(j)\n",
    "\n",
    "for i in y_test:\n",
    "    for j in i:\n",
    "        correctLabels.append(j)\n",
    "        allTestGroundTruth.append(j)\n",
    "\n",
    "print \"\"\" CRF Classification\"\"\"\n",
    "print 'c1 :', c1, 'c2 :', c2\n",
    "print classification_report(allTestGroundTruth, allTestPredictions, digits = 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}