1.one_hot情感分类.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "batch_size = 200\n",
    "word_count = 8945"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "首先加载数据\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y</th>\n",
       "      <th>x</th>\n",
       "      <th>part</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>23800</th>\n",
       "      <td>0</td>\n",
       "      <td>102,129,374,21,150,-1,-1,-1,-1,54,169,28,1130,...</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23801</th>\n",
       "      <td>0</td>\n",
       "      <td>29,470,12,144,6675,957,14,663,178,117,1383,303...</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23802</th>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23803</th>\n",
       "      <td>0</td>\n",
       "      <td>1,29,470,12,-1,117,1684,839,14,3026,44,1356,17...</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23804</th>\n",
       "      <td>0</td>\n",
       "      <td>144,842,153,-1,78,4036,311,78,3453,3544,2249,-...</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55995</th>\n",
       "      <td>1</td>\n",
       "      <td>1,589,42,1014,1667,105,2936,6115,-1,42,3113,11...</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55996</th>\n",
       "      <td>1</td>\n",
       "      <td>19,1017,78,1,71,1269,578,86,-1,78,6206,12,852,...</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55997</th>\n",
       "      <td>1</td>\n",
       "      <td>173,178,35,1198,696,44,2468,3305,1883,135,8939...</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55998</th>\n",
       "      <td>1</td>\n",
       "      <td>861,1401,93,12,512,189,-1,281,785,-1,281,304,-...</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55999</th>\n",
       "      <td>1</td>\n",
       "      <td>1,104,470,171,12,74,178,1279,18,216,569,134,78...</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8400 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       y                                                  x  part\n",
       "23800  0  102,129,374,21,150,-1,-1,-1,-1,54,169,28,1130,...  test\n",
       "23801  0  29,470,12,144,6675,957,14,663,178,117,1383,303...  test\n",
       "23802  0                                                  6  test\n",
       "23803  0  1,29,470,12,-1,117,1684,839,14,3026,44,1356,17...  test\n",
       "23804  0  144,842,153,-1,78,4036,311,78,3453,3544,2249,-...  test\n",
       "...   ..                                                ...   ...\n",
       "55995  1  1,589,42,1014,1667,105,2936,6115,-1,42,3113,11...  test\n",
       "55996  1  19,1017,78,1,71,1269,578,86,-1,78,6206,12,852,...  test\n",
       "55997  1  173,178,35,1198,696,44,2468,3305,1883,135,8939...  test\n",
       "55998  1  861,1401,93,12,512,189,-1,281,785,-1,281,304,-...  test\n",
       "55999  1  1,104,470,171,12,74,178,1279,18,216,569,134,78...  test\n",
       "\n",
       "[8400 rows x 3 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('首先加载数据')\n",
    "\n",
    "data = pd.read_csv('./data/yelp/数字化数据.csv')\n",
    "\n",
    "train_data = data[data.part == 'train']\n",
    "val_data = data[data.part == 'val']\n",
    "test_data = data[data.part == 'test']\n",
    "\n",
    "test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[0., 1., 0.,  ..., 0., 0., 0.],\n",
       "         [0., 1., 0.,  ..., 0., 0., 0.],\n",
       "         [0., 1., 0.,  ..., 0., 0., 0.],\n",
       "         ...,\n",
       "         [0., 1., 1.,  ..., 0., 0., 0.],\n",
       "         [0., 1., 0.,  ..., 0., 0., 0.],\n",
       "         [0., 1., 1.,  ..., 0., 0., 0.]]),\n",
       " tensor([0., 0., 0., 0., 1., 0., 1., 0., 1., 1., 1., 0., 0., 1., 0., 1., 0., 1.,\n",
       "         1., 0., 0., 1., 0., 1., 0., 1., 0., 0., 1., 0., 0., 1., 0., 1., 0., 0.,\n",
       "         0., 1., 0., 0., 1., 0., 1., 0., 0., 1., 1., 1., 0., 1., 1., 1., 1., 1.,\n",
       "         0., 1., 0., 1., 0., 1., 0., 0., 0., 1., 1., 1., 0., 0., 1., 1., 0., 0.,\n",
       "         1., 1., 0., 1., 1., 1., 0., 0., 0., 1., 1., 0., 0., 0., 0., 0., 0., 0.,\n",
       "         0., 0., 1., 0., 1., 1., 0., 0., 0., 0., 0., 0., 1., 1., 0., 1., 1., 1.,\n",
       "         1., 1., 0., 0., 1., 1., 1., 0., 1., 1., 0., 1., 1., 1., 0., 1., 1., 1.,\n",
       "         0., 1., 1., 1., 0., 0., 1., 1., 1., 1., 0., 0., 0., 0., 1., 0., 1., 0.,\n",
       "         1., 0., 0., 1., 0., 1., 0., 1., 0., 1., 0., 1., 1., 0., 1., 0., 0., 0.,\n",
       "         1., 1., 1., 1., 0., 0., 0., 1., 1., 0., 1., 1., 0., 0., 1., 1., 0., 1.,\n",
       "         1., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 0., 1., 1., 0.,\n",
       "         1., 0.]))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "#获取一批数据\n",
    "\n",
    "\n",
    "def get_batch_data(data):\n",
    "    x_batch_data = torch.empty(batch_size, word_count)\n",
    "    y_batch_data = torch.empty(batch_size)\n",
    "\n",
    "    for i in range(batch_size):\n",
    "        r = random.randint(0, len(data) - 1)\n",
    "        line = data.iloc[r]\n",
    "\n",
    "        y_batch_data[i] = int(line.y)\n",
    "\n",
    "        line = [int(i) for i in line.x.split(',')]\n",
    "\n",
    "        one_hot_line = [0] * word_count\n",
    "\n",
    "        for j in line:\n",
    "            if j == -1:\n",
    "                continue\n",
    "            one_hot_line[j] = 1\n",
    "\n",
    "        x_batch_data[i] = torch.Tensor(one_hot_line)\n",
    "\n",
    "    return x_batch_data, y_batch_data\n",
    "\n",
    "\n",
    "get_batch_data(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "定义网络结构\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([-0.1498,  0.2177, -0.7149, -0.7198,  1.2622],\n",
       "       grad_fn=<SqueezeBackward0>)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('定义网络结构')\n",
    "\n",
    "class Classifier(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Classifier, self).__init__()\n",
    "        self.fc = nn.Linear(in_features=word_count, out_features=1)\n",
    "\n",
    "    def forward(self, x):\n",
    "        y_out = self.fc(x).squeeze()\n",
    "        return y_out\n",
    "\n",
    "\n",
    "Classifier()(torch.randn(5, word_count))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "定义工具对象\n"
     ]
    }
   ],
   "source": [
    "print('定义工具对象')\n",
    "\n",
    "classifier = Classifier()\n",
    "loss_func = nn.BCEWithLogitsLoss()\n",
    "optimizer = optim.Adam(classifier.parameters(), lr=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "测试\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.535"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('测试')\n",
    "\n",
    "\n",
    "def test(data):\n",
    "    classifier.eval()\n",
    "    with torch.no_grad():\n",
    "        x_batch_data, y_batch_data = get_batch_data(test_data)\n",
    "        pred = classifier(x_batch_data)\n",
    "\n",
    "        pred[pred > 0.5] = 1\n",
    "        pred[pred <= 0.5] = 0\n",
    "        accurecy = (pred == y_batch_data).sum().item() / len(x_batch_data)\n",
    "\n",
    "        return accurecy\n",
    "\n",
    "\n",
    "test(val_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "训练\n",
      "0 0.6931234002113342 0.55\n",
      "100 0.4894689619541168 0.72\n",
      "200 0.4142768979072571 0.8\n",
      "300 0.3728046417236328 0.86\n",
      "400 0.2889450192451477 0.875\n",
      "500 0.29901936650276184 0.86\n",
      "600 0.2745330035686493 0.885\n",
      "700 0.24842305481433868 0.885\n",
      "800 0.22090713679790497 0.895\n",
      "900 0.24673797190189362 0.93\n",
      "1000 0.23263780772686005 0.915\n",
      "1100 0.2328919768333435 0.89\n",
      "1200 0.24049343168735504 0.92\n",
      "1300 0.22512060403823853 0.87\n",
      "1400 0.1858808398246765 0.905\n",
      "1500 0.19333171844482422 0.895\n",
      "1600 0.18710917234420776 0.895\n",
      "1700 0.1727665364742279 0.895\n",
      "1800 0.18362097442150116 0.93\n",
      "1900 0.1785171926021576 0.895\n"
     ]
    }
   ],
   "source": [
    "print('训练')\n",
    "for epoch in range(2000):\n",
    "    x_batch_data, y_batch_data = get_batch_data(train_data)\n",
    "\n",
    "    pred = classifier(x_batch_data)\n",
    "    loss = loss_func(pred, y_batch_data)\n",
    "\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "\n",
    "    if epoch % 100 == 0:\n",
    "\n",
    "        #测试\n",
    "        accurecy = test(val_data)\n",
    "        print(epoch, loss.item(), accurecy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.895"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#最终测试\n",
    "test(test_data)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "156px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": "5",
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}