data_process.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "print(os.getcwd())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "WORKING_PATH = \"\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data():\n",
    "    data_set=dict()\n",
    "    train_set = []\n",
    "    val_set = []\n",
    "    test_set = []\n",
    "    for dataset in [\"train\"]:\n",
    "        file=open(os.path.join(WORKING_PATH,\"dataset_text/text_data/\",dataset+\".txt\"),\"rb\")\n",
    "        for line in file:\n",
    "            content=eval(line)\n",
    "            image=content[0]\n",
    "            sentence=content[1]\n",
    "            group=content[2]\n",
    "            if os.path.isfile(os.path.join(WORKING_PATH,\"dataset_image\",image+\".jpg\")):\n",
    "                train_set.append([content[0],content[1],content[2]])\n",
    "    for dataset in [\"valid\"]:\n",
    "        file=open(os.path.join(WORKING_PATH,\"dataset_text/text_data/\",dataset+\".txt\"),\"rb\")\n",
    "        for line in file:\n",
    "            content=eval(line)\n",
    "            image=content[0]\n",
    "            sentence=content[1]\n",
    "            group=content[3] #2 hashtag\n",
    "            if os.path.isfile(os.path.join(WORKING_PATH,\"dataset_image\",image+\".jpg\")):\n",
    "                val_set.append([content[0],content[1],content[3],content[2]])\n",
    "    for dataset in [\"test\"]:\n",
    "        file=open(os.path.join(WORKING_PATH,\"dataset_text/text_data/\",dataset+\".txt\"),\"rb\")\n",
    "        for line in file:\n",
    "            content=eval(line)\n",
    "            image=content[0]\n",
    "            sentence=content[1]\n",
    "            group=content[3] #2 is the \n",
    "            if os.path.isfile(os.path.join(WORKING_PATH,\"dataset_image\",image+\".jpg\")):\n",
    "                test_set.append([content[0],content[1],content[3],content[2]])\n",
    "    return train_set, val_set, test_set\n",
    "\n",
    "def read_json(path):\n",
    "    with open(path,\"r\",encoding = 'utf-8') as f:\n",
    "        data = json.load(f)\n",
    "    return data\n",
    "def write_json(path,data):\n",
    "    with open(path,\"w\",encoding = 'utf-8') as f:\n",
    "        json.dump(data,f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt # plt 用于显示图片\n",
    "import os\n",
    "from PIL import Image\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set, val_set, test_set = load_data()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean(dataset):\n",
    "    dataset_ = []\n",
    "    for sample in dataset:\n",
    "        line = sample[1].strip()\n",
    "        if \"sarcasm\" in line:\n",
    "            continue\n",
    "        if \"sarcastic\" in line:\n",
    "            continue\n",
    "        if \"reposting\" in line:\n",
    "            continue\n",
    "        if \"<url>\" in line:\n",
    "            continue\n",
    "        if \"joke\" in line:\n",
    "            continue\n",
    "        if \"humour\" in line:\n",
    "            continue\n",
    "        if \"humor\" in line:\n",
    "            continue\n",
    "        if \"jokes\" in line:\n",
    "            continue\n",
    "        if \"irony\" in line:\n",
    "            continue\n",
    "        if \"ironic\" in line:\n",
    "            continue\n",
    "        if \"exgag\" in line:\n",
    "            continue\n",
    "        sample[1] = line\n",
    "        dataset_.append(sample)\n",
    "    return dataset_\n",
    "\n",
    "trainset = clean(train_set)\n",
    "valset = clean(val_set)\n",
    "testset = clean(test_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the final paths\n",
    "write_json(\"./sarcasm/twitter/dataset_text/train.json\", trainset)\n",
    "write_json(\"./sarcasm/twitter/dataset_text/val.json\", valset)\n",
    "write_json(\"./sarcasm/twitter/dataset_text/test.json\", testset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualization for twitter-image pair"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# i= random.randint(1,2000)\n",
    "# print(423)\n",
    "i=1604 \n",
    "print(val[i][1])\n",
    "print(val[i][2],val[i][3])\n",
    "print(val[i][-3])\n",
    "display_img(test[i][0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "import os\n",
    "import cv2\n",
    "def display_img(flag):\n",
    "    path = \"./sarcasm/twitter/dataset_image\"\n",
    "    path = os.path.join(path,flag+\".jpg\")\n",
    "    im=Image.open(path)\n",
    "    im.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(train_set),len(val_set),len(test_set))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dataset[423]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "img_switch = Image.open(\"./sarcasm/demo.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "img_switch = img_switch.resize((224,224)).convert('RGBA')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "b.paste(img_switch,(0,0,224,224),mask=a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r, g, b, a = img_switch.split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = Image.new('RGB', (224, 224), (0,0,0)).convert('RGBA')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "w = img_switch.size[0]               #获取图片宽度\n",
    "h = img_switch.size[1]               #获取图片高度\n",
    "img_1 = img_switch.crop([2*w/3, 2*h/3, w, h])       #获取左上1/4的图片\n",
    "img_1.save('demo_images/' + '8' + '.png')          #保存在本地图片命名为1.jpg\n",
    "# img_2 = img.crop([w/2, 0, w, h/2])       #获得右上1/4的图片\n",
    "# img_2.save('./' + '2' + '.jpg')          #保存在本地图片命名为2.jpg\n",
    "# img_3 = img.crop([0, h/2, w/2, h])       #获取左下1/4的图片\n",
    "# img_3.save('./' + '3' + '.jpg')          #保存在本地图片命名为3.jpg\n",
    "# img_4 = img.crop([w/2, h/2, w, h])       #获取右下1/4的图片\n",
    "# img_4.save('./' + '4' + '.jpg') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "b=img_switch .load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 改变图片被背景色\n",
    "b=img_switch .load()\n",
    "for y in range(224):\n",
    "    for x in range(224):\n",
    "        if all(b[x,y][i]>220 for i in range(4)):\n",
    "            b[x,y] = 238,233,233\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "img_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "img_switch.save(\"demo.png\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 图像特征抽取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "import torch\n",
    "from torchvision import transforms\n",
    "from pytorc import ViT"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## VIT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 224 7*7 32*32\n",
    "# use pre_trained model B_32 L_32 \n",
    "model_name = 'B_32_imagenet1k'\n",
    "model = ViT(model_name, pretrained=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def B_32_embedding(dataset, img_path):\n",
    "    model_name = 'B_32'\n",
    "    model = ViT(model_name, pretrained=True)\n",
    "    tfms = transforms.Compose([transforms.Resize(model.image_size), transforms.ToTensor(), \n",
    "                               transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5]),])\n",
    "    print(model.image_size)\n",
    "    embedding = []\n",
    "    model.__delattr__(\"fc\")\n",
    "    model.cuda()\n",
    "    model.eval()\n",
    "    for sample in dataset:\n",
    "        img = os.path.join(img_path,sample[0]+\".jpg\")\n",
    "        img = Image.open(img)\n",
    "        img = tfms(img).unsqueeze(0)\n",
    "        with torch.no_grad():\n",
    "            # remove class token\n",
    "            img = model(img.cuda())[0,:-1,:]\n",
    "        embedding.append(img.cpu())\n",
    "    return embedding\n",
    "\n",
    "def L_32_embedding(dataset, img_path):\n",
    "    model_name = 'L_32'\n",
    "    model = ViT(model_name, pretrained=True)\n",
    "    tfms = transforms.Compose([transforms.Resize(model.image_size), transforms.ToTensor(), \n",
    "                               transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5]),])\n",
    "    print(model.image_size)\n",
    "    embedding = []\n",
    "    model.__delattr__(\"fc\")\n",
    "    model.cuda()\n",
    "    model.eval()\n",
    "    for sample in dataset:\n",
    "        img = os.path.join(img_path,sample[0]+\".jpg\")\n",
    "        img = Image.open(img)\n",
    "        img = tfms(img).unsqueeze(0)\n",
    "        with torch.no_grad():\n",
    "            # remove class token\n",
    "            img = model(img.cuda())[0,:-1,:]\n",
    "        embedding.append(img.cpu())\n",
    "    return embedding\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# B_train_emb = B_32_embedding(trainset, img_path = \"./sarcasm/twitter/dataset_image\")\n",
    "B_val_emb = B_32_embedding(valset, img_path = \"./sarcasm/twitter/dataset_image\")\n",
    "B_test_emb = B_32_embedding(testset, img_path = \"./sarcasm/twitter/dataset_image\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "L_train_emb = L_32_embedding(trainset, img_path = \"./sarcasm/twitter/dataset_image\")\n",
    "L_val_emb = L_32_embedding(valset, img_path = \"./sarcasm/twitter/dataset_image\")\n",
    "L_test_emb = L_32_embedding(testset, img_path = \"./sarcasm/twitter/dataset_image\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(L_train_emb, \"./sarcasm/twitter/img_emb/train_L32.pt\")\n",
    "torch.save(L_val_emb, \"./sarcasm/twitter/img_emb/val_L32.pt\")\n",
    "torch.save(L_test_emb, \"./sarcasm/twitter/img_emb/test_L32.pt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Resnet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torchvision.models as models\n",
    "resnet152 = models.resnet152(pretrained=True, progress = True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Resnet152_embedding(dataset, img_path):\n",
    "    resnet152 = models.resnet152(pretrained=True, progress = True)\n",
    "    tfms = transforms.Compose([transforms.Resize((224,224)), transforms.ToTensor(), \n",
    "                               transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])\n",
    "    w=int(224/7)\n",
    "    h=int(224/7)\n",
    "\n",
    "    embedding = []\n",
    "    resnet152.fc = torch.nn.Identity()\n",
    "    resnet152.cuda()\n",
    "    resnet152.eval()\n",
    "    for sample in dataset:\n",
    "        img = os.path.join(img_path,sample[0]+\".jpg\")\n",
    "        img = Image.open(img)\n",
    "        img = tfms(img).unsqueeze(0)\n",
    "        patch = []\n",
    "        for row in range(7):\n",
    "            for col in range(7):\n",
    "                patch.append(img[:,:,row*w:row*w+w,col*h:col*h+h])\n",
    "        patches = torch.cat(patch, dim=0)\n",
    "        with torch.no_grad():\n",
    "            embedding.append(resnet152(patches.cuda()).cpu())\n",
    "    return embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_res152 = Resnet152_embedding(trainset,img_path='./sarcasm/twitter/dataset_image' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(train_res152,\"./sarcasm/twitter/img_emb/train_152.pt\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_res152 = Resnet152_embedding(valset,img_path='./sarcasm/twitter/dataset_image' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(val_res152,\"./sarcasm/twitter/img_emb/val_152.pt\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_res152 = Resnet152_embedding(testset,img_path='./sarcasm/twitter/dataset_image' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(test_res152,\"./sarcasm/twitter/img_emb/test_152.pt\" )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 依赖生成"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def re_organize_caption(tokens,noun_phrases):\n",
    "    # tokens_smaple token+np\n",
    "    # token_map token_map[a] a 是\n",
    "    tokens_sample = []\n",
    "    chunk_index = 0\n",
    "    chunk_len = len(noun_phrases)\n",
    "    i = 0\n",
    "    token_map = []\n",
    "    while (i<len(tokens)):\n",
    "        if chunk_index<chunk_len:\n",
    "            if i<noun_phrases[chunk_index][1]:\n",
    "                tokens_sample.append(tokens[i][0])\n",
    "                token_map.append(len(tokens_sample)-1)\n",
    "                i = i+1\n",
    "            else:\n",
    "                tokens_sample.append(noun_phrases[chunk_index][0])\n",
    "                for a in range(i,noun_phrases[chunk_index][2]):\n",
    "                    token_map.append(len(tokens_sample)-1)\n",
    "                i = noun_phrases[chunk_index][2]\n",
    "                chunk_index = chunk_index+1\n",
    "        else:\n",
    "            tokens_sample.append(tokens[i][0])\n",
    "            token_map.append(len(tokens_sample)-1)\n",
    "            i = i+1\n",
    "    return tokens_sample, token_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "nlp = spacy.load(\"en_core_web_trf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = \"Men never were shown in a negative light\" \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc = nlp(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[(token.text.lower(),token.i,token.head.i, token.is_punct) for token in doc]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# token dependency\n",
    "def token_dependency(dataset):\n",
    "    for i,sample in enumerate(dataset):\n",
    "        dataset_article = {}\n",
    "        dataset_article_a = {}\n",
    "        doc = nlp(sample[1])\n",
    "        dataset_article[\"token_caption\"] = [(token.text.lower(),token.i,token.head.i, token.is_punct) for token in doc]\n",
    "        dataset_article[\"chunk\"] =  [(chunk.text.lower(),chunk.start,chunk.end) for chunk in doc.noun_chunks]\n",
    "        token_sample,token_map = re_organize_caption(dataset_article[\"token_caption\"],dataset_article[\"chunk\"])\n",
    "        dependency = [(token_map[t[1]],token_map[t[2]]) for t in dataset_article[\"token_caption\"] if (not (token_map[t[1]]) == token_map[t[2]]) and \n",
    "                      (not t[3])]\n",
    "        \n",
    "        dataset_article_a[\"chunk_cap\"] = token_sample\n",
    "        dataset_article_a[\"token_cap\"] = [t[0] for t in dataset_article[\"token_caption\"]]\n",
    "        \n",
    "        dataset_article_a[\"token_dep\"] = [(t[1],t[2]) for t in dataset_article[\"token_caption\"] if (not t[1] == t[2]) and (not t[3]) and t[0]!=\" \" \n",
    "                                          and dataset_article[\"token_caption\"][t[2]][0]!= \" \"]\n",
    "        dataset_article_a[\"chunk_dep\"] = dependency\n",
    "        \n",
    "        dataset_article_a[\"chunk\"] = [temp[0] for temp in dataset_article[\"chunk\"]]\n",
    "        dataset_article_a[\"chunk_index\"] = [token_map[temp[1]] for temp in dataset_article[\"chunk\"]]\n",
    "        \n",
    "        dataset[i].append(dataset_article_a)\n",
    "    return dataset\n",
    "        \n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainset = read_json(\"./sarcasm/twitter/dataset_text/train.json\")\n",
    "valset = read_json(\"./sarcasm/twitter/dataset_text/val.json\")\n",
    "testset = read_json(\"./sarcasm/twitter/dataset_text/test.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dep = token_dependency(testset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_dep = token_dependency(valset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dep = token_dependency(trainset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_json(\"./sarcasm/twitter/dataset_text/traindep.json\", train_dep)\n",
    "write_json(\"./sarcasm/twitter/dataset_text/valdep.json\", val_dep)\n",
    "write_json(\"./sarcasm/twitter/dataset_text/testdep.json\", test_dep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(train_dep)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Knowledge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train的caption有一定的重复"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_attr = \"./sarcasm/twitter/dataset_text/multilabel_database/img_to_five_words.txt\"\n",
    "cap_test = \"./sarcasm/twitter/dataset_text/gecaption/testcap.json\"\n",
    "cap_val = \"./sarcasm/twitter/dataset_text/gecaption/valcap.json\"\n",
    "cap_train = \"./sarcasm/twitter/dataset_text/gecaption/traincap.json\"\n",
    "train_dep = \"./sarcasm/twitter/dataset_text/traindep.json\"\n",
    "val_dep = \"./sarcasm/twitter/dataset_text/valdep.json\"\n",
    "test_dep = \"./sarcasm/twitter/dataset_text/testdep.json\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "def read_json(path):\n",
    "    with open(path,\"r\",encoding = 'utf-8') as f:\n",
    "        data = json.load(f)\n",
    "    return data\n",
    "def write_json(path,data):\n",
    "    with open(path,\"w\",encoding = 'utf-8') as f:\n",
    "        json.dump(data,f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dep = read_json(train_dep)\n",
    "val_dep = read_json(val_dep)\n",
    "test_dep = read_json(test_dep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cap_test = read_json(cap_test)\n",
    "cap_val = read_json(cap_val)\n",
    "cap_train = read_json(cap_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(train_dep), len(val_dep), len(test_dep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(cap_train), len(cap_val), len(cap_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_attr=dict()\n",
    "file=open(\"./sarcasm/twitter/dataset_text/multilabel_database/img_to_five_words.txt\",\"rb\")\n",
    "a=0\n",
    "for line in file:\n",
    "    content=eval(line)\n",
    "    data_attr[content[0]]=content[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_txt\n",
    "file_path = \"train.txt\"\n",
    "with open(file_path,\"w\") as f:\n",
    "    for sample in train_dataset:\n",
    "        f.write(\"dataset_image/\"+sample[0]+\".jpg\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path = \"test.txt\"\n",
    "with open(file_path,\"w\") as f:\n",
    "    for sample in test_dataset:\n",
    "        f.write(\"dataset_image/\"+sample[0]+\".jpg\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path = \"val.txt\"\n",
    "with open(file_path,\"w\") as f:\n",
    "    for sample in val_dataset:\n",
    "        f.write(\"dataset_image/\"+sample[0]+\".jpg\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add anp\n",
    "test_anp= read_json(\"./sarcasm/twitter/dataset_text/anp/test.json\")\n",
    "val_anp= read_json(\"./sarcasm/twitter/dataset_text/anp/val.json\")\n",
    "train_anp= read_json(\"./sarcasm/twitter/dataset_text/anp/train.json\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_anp_set = []\n",
    "for sample in test_anp['images']:\n",
    "    anps = sample['bi-concepts']\n",
    "    test_anp_set.append([a.replace('_', \" \") for a in list(anps.keys())])  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_anp_set = []\n",
    "for sample in val_anp['images']:\n",
    "    anps = sample['bi-concepts']\n",
    "    val_anp_set.append([a.replace('_', \" \") for a in list(anps.keys())])  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_anp_set = []\n",
    "for sample in train_anp['images']:\n",
    "    anps = sample['bi-concepts']\n",
    "    train_anp_set.append([a.replace('_', \" \") for a in list(anps.keys())])  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(train_anp_set), len(val_anp_set), len(test_anp_set))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = []\n",
    "for i, sample in enumerate(train_dep):\n",
    "    id_attr = sample[0]\n",
    "    sample.append(cap_train[20:][i])\n",
    "    attr = data_attr[id_attr]\n",
    "    sample.append(attr)\n",
    "    sample.append(train_anp_set[i])\n",
    "    train_dataset.append(sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_dataset = []\n",
    "for i, sample in enumerate(val_dep):\n",
    "    id_attr = sample[0]\n",
    "    sample.append(cap_val[i])\n",
    "    attr = data_attr[id_attr]\n",
    "    sample.append(attr)\n",
    "    sample.append(val_anp_set[i])\n",
    "    val_dataset.append(sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dataset = []\n",
    "for i, sample in enumerate(test_dep):\n",
    "    id_attr = sample[0]\n",
    "    sample.append(cap_test[i])\n",
    "    attr = data_attr[id_attr]\n",
    "    sample.append(attr)\n",
    "    sample.append(test_anp_set[i])\n",
    "    test_dataset.append(sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_json(\"./sarcasm/twitter/dataset_text/trainknow.json\", train_dataset)\n",
    "write_json(\"./twitter/dataset_text/valknow.json\", val_dataset)\n",
    "write_json(\"./sarcasm/twitter/dataset_text/testknow.json\", test_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = read_json(\"./sarcasm/twitter/dataset_text/trainknow.json\")\n",
    "val = read_json(\"./sarcasm/twitter/dataset_text/valknow.json\")\n",
    "test = read_json(\"./sarcasm/twitter/dataset_text/testknow.json\")\n",
    "# *."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "test[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i,sample in enumerate(val):\n",
    "    print (i, sample[-3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "isinstance(test_dataset[0][-1], list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "nlp = spacy.load(\"en_core_web_trf\")\n",
    "def re_organize_caption(tokens,noun_phrases):\n",
    "    # tokens_smaple token+np\n",
    "    # token_map token_map[a] a 是\n",
    "    tokens_sample = []\n",
    "    chunk_index = 0\n",
    "    chunk_len = len(noun_phrases)\n",
    "    i = 0\n",
    "    token_map = []\n",
    "    while (i<len(tokens)):\n",
    "        if chunk_index<chunk_len:\n",
    "            if i<noun_phrases[chunk_index][1]:\n",
    "                tokens_sample.append(tokens[i][0])\n",
    "                token_map.append(len(tokens_sample)-1)\n",
    "                i = i+1\n",
    "            else:\n",
    "                tokens_sample.append(noun_phrases[chunk_index][0])\n",
    "                for a in range(i,noun_phrases[chunk_index][2]):\n",
    "                    token_map.append(len(tokens_sample)-1)\n",
    "                i = noun_phrases[chunk_index][2]\n",
    "                chunk_index = chunk_index+1\n",
    "        else:\n",
    "            tokens_sample.append(tokens[i][0])\n",
    "            token_map.append(len(tokens_sample)-1)\n",
    "            i = i+1\n",
    "    return tokens_sample, token_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# token dependency\n",
    "def token_dependency(dataset):\n",
    "    for i,sample in enumerate(dataset):\n",
    "        dataset_article = {}\n",
    "        dataset_article_a = {}\n",
    "        # the caption index is -3 \n",
    "        doc = nlp(sample[-3].strip())\n",
    "        dataset_article[\"token_caption\"] = [(token.text.lower(),token.i,token.head.i, token.is_punct) for token in doc]\n",
    "        dataset_article[\"chunk\"] =  [(chunk.text.lower(),chunk.start,chunk.end) for chunk in doc.noun_chunks]\n",
    "        token_sample,token_map = re_organize_caption(dataset_article[\"token_caption\"],dataset_article[\"chunk\"])\n",
    "        dependency = [(token_map[t[1]],token_map[t[2]]) for t in dataset_article[\"token_caption\"] if (not (token_map[t[1]]) == token_map[t[2]]) and \n",
    "                      (not t[3])]\n",
    "        \n",
    "        dataset_article_a[\"token_cap\"] = [t[0] for t in dataset_article[\"token_caption\"]]\n",
    "        \n",
    "        dataset_article_a[\"token_dep\"] = [(t[1],t[2]) for t in dataset_article[\"token_caption\"] if (not t[1] == t[2]) and (not t[3]) and t[0]!=\" \" \n",
    "                                          and dataset_article[\"token_caption\"][t[2]][0]!= \" \"]\n",
    "        \n",
    "        dataset[i].append(dataset_article_a)\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dep = token_dependency(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_dep = token_dependency(val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dep = token_dependency(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write_json(\"./sarcasm/twitter/dataset_text/trainknow_dep.json\", train_dep)\n",
    "write_json(\"./sarcasm/twitter/dataset_text/valknow_dep.json\", val_dep)\n",
    "# write_json(\"./sarcasm/twitter/dataset_text/testknow_dep.json\", test_dep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_dep = read_json(\"./sarcasm/twitter/dataset_text/trainknow_dep.json\")\n",
    "val_dep = read_json(\"./sarcasm/twitter/dataset_text/valknow_dep.json\")\n",
    "# test_dep = read_json(\"./sarcasm/twitter/dataset_text/testknow_dep.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "length = []\n",
    "for smaple in val_dep:\n",
    "    length.append(len(smaple[-1]['token_dep'])) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "min(length)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}