Fuzzy_extractor.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from wikidata_concept_finder import naive_embedd, wikify, make_embedding\n",
    "import pandas as pd\n",
    "import time\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets = [\"LIAR_PANTS\"]#, \"AAAI2021_COVID19_fake_news\", \"LIAR\", \"FakeNewsNet\"] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_kgs(uniq_names, kg):\n",
    "    present_terms = wikify(uniq_names, kg)\n",
    "    doc2terms = []\n",
    "    for l in uniq_names:\n",
    "        term_in_dect = [l]\n",
    "        doc2terms.append(term_in_dect)\n",
    "    print(\"Embedding started.\")\n",
    "    doc_kg_embs = []\n",
    "    final_terms = []\n",
    "    for doc in doc2terms:\n",
    "        term2vec = {}\n",
    "        for term in doc:\n",
    "            if term in present_terms:\n",
    "                term2vec[term] = present_terms[term]\n",
    "        doc_kg_emb = make_embedding(term2vec)\n",
    "        doc_kg_embs.append(doc_kg_emb)\n",
    "        final_terms.append(list(term2vec.keys()))\n",
    "    print(\"Embedding ended.\")\n",
    "    outputs = zip(final_terms, doc_kg_embs)\n",
    "    return outputs "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_split_2(split):\n",
    "    cols = ['id', 'label', 'statement', 'subject', 'speaker', 'job_speaker', 'state', 'party', 'barely true', 'false', 'half true', 'mostly true', 'pants on', 'location_or_venue']\n",
    "    split_path = \"data/raw/liar_pants/\"+split+\".tsv\"\n",
    "    df = pd.read_csv(split_path, delimiter='\\t', names=cols)\n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['', 'ken mercer', 'independence usa pac', 'elfant', 'vaught', 'george will', 'vanden', 'freedom religion foundation', 'kanavas', 'founding partner']\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "3410\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "10240 10240\n",
      "DATA DROPPED @  kg_emb_dump/LIAR_PANTS/train_complex_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "3410\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "10240 10240\n",
      "DATA DROPPED @  kg_emb_dump/LIAR_PANTS/train_transe_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "3410\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "10240 10240\n",
      "DATA DROPPED @  kg_emb_dump/LIAR_PANTS/train_quate_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "3410\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "10240 10240\n",
      "DATA DROPPED @  kg_emb_dump/LIAR_PANTS/train_simple_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "3410\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "10240 10240\n",
      "DATA DROPPED @  kg_emb_dump/LIAR_PANTS/train_rotate_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "3410\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "10240 10240\n",
      "DATA DROPPED @  kg_emb_dump/LIAR_PANTS/train_distmult_speakers.pkl\n",
      "['', 'congressional', 'ventura', 'action', 'jackson', 'southerland', 'george will', 'portland schools superintendent', 'speaker of the house of representatives', 'democratic party chairman']\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "1061\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1267 1267\n",
      "DATA DROPPED @  kg_emb_dump/LIAR_PANTS/test_complex_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "1061\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1267 1267\n",
      "DATA DROPPED @  kg_emb_dump/LIAR_PANTS/test_transe_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "1061\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1267 1267\n",
      "DATA DROPPED @  kg_emb_dump/LIAR_PANTS/test_quate_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "1061\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1267 1267\n",
      "DATA DROPPED @  kg_emb_dump/LIAR_PANTS/test_simple_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "1061\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1267 1267\n",
      "DATA DROPPED @  kg_emb_dump/LIAR_PANTS/test_rotate_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "1061\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1267 1267\n",
      "DATA DROPPED @  kg_emb_dump/LIAR_PANTS/test_distmult_speakers.pkl\n",
      "['', 'congressional', 'action', 'jackson', 'southerland', 'wentworth', 'george will', 'speaker of the house of representatives', 'chiusolo', 'stephen martin']\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "1066\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1284 1284\n",
      "DATA DROPPED @  kg_emb_dump/LIAR_PANTS/dev_complex_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "1066\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1284 1284\n",
      "DATA DROPPED @  kg_emb_dump/LIAR_PANTS/dev_transe_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "1066\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1284 1284\n",
      "DATA DROPPED @  kg_emb_dump/LIAR_PANTS/dev_quate_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "1066\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1284 1284\n",
      "DATA DROPPED @  kg_emb_dump/LIAR_PANTS/dev_simple_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "1066\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1284 1284\n",
      "DATA DROPPED @  kg_emb_dump/LIAR_PANTS/dev_rotate_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "1066\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1284 1284\n",
      "DATA DROPPED @  kg_emb_dump/LIAR_PANTS/dev_distmult_speakers.pkl\n",
      "['ken mercer', 'independence usa pac', 'veronica escobar', 'Ileana Ros', 'david sweetser', 'george will', 'brian treece', 'rod smith', 'thomas sgouros jr.', 'arnold schwarzenegger']\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "2196\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "15212 15212\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/train_complex_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "2196\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "15212 15212\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/train_transe_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "2196\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "15212 15212\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/train_quate_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "2196\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "15212 15212\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/train_simple_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "2196\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "15212 15212\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/train_rotate_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "2196\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "15212 15212\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/train_distmult_speakers.pkl\n",
      "['national right to life committee', 'Ileana Ros', 'thenewyorkevening.com', 'george will', 'rachel maddow', 'louise slaughter', 'antiwar.com', 'al gore', \"lawrence o'donnell\", 'the breaking dawn']\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "204\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1054 1054\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/test_complex_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "204\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1054 1054\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/test_transe_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "204\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1054 1054\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/test_quate_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "204\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1054 1054\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/test_simple_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "204\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1054 1054\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/test_rotate_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "204\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1054 1054\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/test_distmult_speakers.pkl\n",
      "['steve munisteri', 'kurt schrader', 'george will', 'rod smith', 'bobby  scott', 'hakeem jeffries', 'bill nelson', 'george allen', 'rachel maddow', 'chris abele']\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "425\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1058 1058\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/dev_complex_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "425\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1058 1058\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/dev_transe_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "425\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1058 1058\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/dev_quate_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "425\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1058 1058\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/dev_simple_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "425\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1058 1058\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/dev_rotate_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "425\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1058 1058\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/dev_distmult_speakers.pkl\n"
     ]
    }
   ],
   "source": [
    "for dataset in [\"LIAR_PANTS\", \"FakeNewsNet\"]: #\"LIAR_PANTS\",\n",
    "    for t in [\"train\", \"test\", \"dev\"]:       \n",
    "        prefix_path = \"data/final/\" + dataset + \"/\" + t + \".csv\"\n",
    "        df = pd.read_csv(prefix_path)   \n",
    "        unq = set()\n",
    "        for c in df['entity']:\n",
    "            tmp = c.split(',')\n",
    "            for xs in tmp:\n",
    "                if '-' in xs:                    \n",
    "                    parts = xs.split('-')\n",
    "                    for p in parts:\n",
    "                        unq.add(p)\n",
    "                    joined = \" \".join(parts)\n",
    "                    unq.add(joined)\n",
    "                else:\n",
    "                    unq.add(xs.lower())\n",
    "        unq = list(unq)\n",
    "        print(unq[:10])\n",
    "        for kg in [\"complex\", \"transe\", \"quate\", \"simple\", \"rotate\", \"distmult\"]: \n",
    "            present_terms = wikify(unq, kg)\n",
    "            print(len(present_terms))\n",
    "            print(\"Embedding started.\")\n",
    "            doc_kg_embs = []\n",
    "            final_terms = []\n",
    "            for doc in df['entity']:\n",
    "                term2vec = {}\n",
    "                for term in doc.split(','):\n",
    "                    term = term.lower()\n",
    "                    if term in present_terms:\n",
    "                        term2vec[term] = present_terms[term]\n",
    "                doc_kg_emb = make_embedding(term2vec)\n",
    "                doc_kg_embs.append(doc_kg_emb)\n",
    "                final_terms.append(list(term2vec.keys()))\n",
    "            print(\"Embedding ended.\")\n",
    "            ys = zip(final_terms, doc_kg_embs)\n",
    "            out_path = \"kg_emb_dump/\"+dataset+\"/\"+t+\"_\"+kg+\"_speakers.pkl\"\n",
    "            print(len(final_terms),len(df))\n",
    "            with open(out_path, \"wb\") as f:\n",
    "                pickle.dump(ys, f)\n",
    "            print(\"DATA DROPPED @ \", out_path)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "15212 15212\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/train_complex_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "15212 15212\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/train_transe_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "15212 15212\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/train_quate_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "15212 15212\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/train_simple_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "15212 15212\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/train_rotate_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "15212 15212\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/train_distmult_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1054 1054\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/test_complex_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1054 1054\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/test_transe_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1054 1054\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/test_quate_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1054 1054\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/test_simple_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1054 1054\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/test_rotate_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1054 1054\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/test_distmult_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1058 1058\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/dev_complex_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1058 1058\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/dev_transe_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1058 1058\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/dev_quate_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1058 1058\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/dev_simple_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1058 1058\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/dev_rotate_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1058 1058\n",
      "DATA DROPPED @  kg_emb_dump/FakeNewsNet/dev_distmult_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "15052 15052\n",
      "DATA DROPPED @  kg_emb_dump/LIAR/train_complex_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "15052 15052\n",
      "DATA DROPPED @  kg_emb_dump/LIAR/train_transe_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "15052 15052\n",
      "DATA DROPPED @  kg_emb_dump/LIAR/train_quate_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "15052 15052\n",
      "DATA DROPPED @  kg_emb_dump/LIAR/train_simple_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "15052 15052\n",
      "DATA DROPPED @  kg_emb_dump/LIAR/train_rotate_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "15052 15052\n",
      "DATA DROPPED @  kg_emb_dump/LIAR/train_distmult_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1266 1266\n",
      "DATA DROPPED @  kg_emb_dump/LIAR/test_complex_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1266 1266\n",
      "DATA DROPPED @  kg_emb_dump/LIAR/test_transe_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1266 1266\n",
      "DATA DROPPED @  kg_emb_dump/LIAR/test_quate_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1266 1266\n",
      "DATA DROPPED @  kg_emb_dump/LIAR/test_simple_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1266 1266\n",
      "DATA DROPPED @  kg_emb_dump/LIAR/test_rotate_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1266 1266\n",
      "DATA DROPPED @  kg_emb_dump/LIAR/test_distmult_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1265 1265\n",
      "DATA DROPPED @  kg_emb_dump/LIAR/dev_complex_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1265 1265\n",
      "DATA DROPPED @  kg_emb_dump/LIAR/dev_transe_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1265 1265\n",
      "DATA DROPPED @  kg_emb_dump/LIAR/dev_quate_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1265 1265\n",
      "DATA DROPPED @  kg_emb_dump/LIAR/dev_simple_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1265 1265\n",
      "DATA DROPPED @  kg_emb_dump/LIAR/dev_rotate_speakers.pkl\n",
      "Wikifying started.\n",
      "Wikifying finished.\n",
      "Embedding started.\n",
      "Embedding ended.\n",
      "1265 1265\n",
      "DATA DROPPED @  kg_emb_dump/LIAR/dev_distmult_speakers.pkl\n"
     ]
    }
   ],
   "source": [
    "paths = {\"FakeNewsNet\" : \"data/fnn.pkl\"}\n",
    "for dataset in paths:\n",
    "    for t in [\"train\", \"test\", \"dev\"]:       \n",
    "        with open(paths[dataset], \"rb\") as f:\n",
    "            unq = pickle.load(f)\n",
    "        prefix_path = \"data/final/\" + dataset + \"/\" + t + \".csv\"\n",
    "        df = pd.read_csv(prefix_path)\n",
    "        for kg in [\"complex\", \"transe\", \"quate\", \"simple\", \"rotate\", \"distmult\"]: \n",
    "            present_terms = wikify(unq, kg)\n",
    "            print(\"Embedding started.\")\n",
    "            doc_kg_embs = []\n",
    "            final_terms = []\n",
    "            for doc in df['entity']:\n",
    "                term = doc.lower()\n",
    "                term2vec = {}\n",
    "                if term in present_terms:\n",
    "                    term2vec[term] = present_terms[term]\n",
    "                doc_kg_emb = make_embedding(term2vec)\n",
    "                doc_kg_embs.append(doc_kg_emb)\n",
    "                final_terms.append(list(term2vec.keys()))\n",
    "            print(\"Embedding ended.\")\n",
    "            ys = zip(final_terms, doc_kg_embs)\n",
    "            out_path = \"kg_emb_dump/\"+dataset+\"/\"+t+\"_\"+kg+\"_speakers.pkl\"\n",
    "            print(len(final_terms),len(df))\n",
    "            with open(out_path, \"wb\") as f:\n",
    "                pickle.dump(ys, f)\n",
    "            print(\"DATA DROPPED @ \", out_path)\n",
    "        "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}