local_eval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "import pandas as pd\n",
    "from transformers import pipeline\n",
    "import numpy as np\n",
    "from tqdm.notebook import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import_df = pd.read_csv(\"bertTesting.csv\")\n",
    "test_files = []\n",
    "for tf in list(import_df[\"0\"]):\n",
    "    test_files.append(tf[:-4])\n",
    "test_files = test_files[:1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_word_indices(full_text, discourse_start, discourse_end):\n",
    "    start_index = len(full_text[:discourse_start].split())\n",
    "    token_len = len(full_text[discourse_start:discourse_end].split())\n",
    "    output = list(range(start_index, start_index + token_len))\n",
    "    if output[-1] >= len(full_text.split()):\n",
    "        output = list(range(start_index, start_index + token_len-1))\n",
    "    return output\n",
    "\n",
    "def predict_and_format(fileName, predict_df, predictAcc_df):\n",
    "    data = \"\"\n",
    "    with open('train/' + fileName + \".txt\", 'r') as file:\n",
    "        data = file.read().replace('\\n', '')\n",
    "        data = data.replace(\"Â\\xa0\", '')\n",
    "    \n",
    "    predicts = token_classifier(data)\n",
    "\n",
    "    for p in predicts:\n",
    "        word_Indices_Array = calc_word_indices(data, p[\"start\"], p[\"end\"])\n",
    "        word_Indices_String = \" \".join(str(x) for x in word_Indices_Array)\n",
    "        word_class = p[\"entity_group\"]\n",
    "        word_id = fileName\n",
    "        predictAcc_df.loc[len(predictAcc_df.index)] = [word_id, word_class, word_Indices_String]\n",
    "        if len(word_Indices_Array) >= 3:\n",
    "            predict_df.loc[len(predict_df.index)] = [word_id, word_class, word_Indices_String]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# code by ROB MULLA\n",
    "# source: https://www.kaggle.com/robikscube/student-writing-competition-twitch#Competition-Metric-Code\n",
    "\n",
    "def calc_overlap(row):\n",
    "    \"\"\"\n",
    "    Calculates the overlap between prediction and\n",
    "    ground truth and overlap percentages used for determining\n",
    "    true positives.\n",
    "    \"\"\"\n",
    "    set_pred = set(row.predictionstring_pred.split(\" \"))\n",
    "    set_gt = set(row.predictionstring_gt.split(\" \"))\n",
    "    # Length of each and intersection\n",
    "    len_gt = len(set_gt)\n",
    "    len_pred = len(set_pred)\n",
    "    inter = len(set_gt.intersection(set_pred))\n",
    "    overlap_1 = inter / len_gt\n",
    "    overlap_2 = inter / len_pred\n",
    "    return [overlap_1, overlap_2]\n",
    "\n",
    "\n",
    "def score_feedback_comp_micro(pred_df, gt_df):\n",
    "    \"\"\"\n",
    "    A function that scores for the kaggle\n",
    "        Student Writing Competition\n",
    "\n",
    "    Uses the steps in the evaluation page here:\n",
    "        https://www.kaggle.com/c/feedback-prize-2021/overview/evaluation\n",
    "    \"\"\"\n",
    "    gt_df = (\n",
    "        gt_df[[\"id\", \"discourse_type\", \"predictionstring\"]]\n",
    "        .reset_index(drop=True)\n",
    "        .copy()\n",
    "    )\n",
    "    pred_df = pred_df[[\"id\", \"class\", \"predictionstring\"]].reset_index(drop=True).copy()\n",
    "    pred_df[\"pred_id\"] = pred_df.index\n",
    "    gt_df[\"gt_id\"] = gt_df.index\n",
    "    # Step 1. all ground truths and predictions for a given class are compared.\n",
    "    joined = pred_df.merge(\n",
    "        gt_df,\n",
    "        left_on=[\"id\", \"class\"],\n",
    "        right_on=[\"id\", \"discourse_type\"],\n",
    "        how=\"outer\",\n",
    "        suffixes=(\"_pred\", \"_gt\"),\n",
    "    )\n",
    "    joined[\"predictionstring_gt\"] = joined[\"predictionstring_gt\"].fillna(\" \")\n",
    "    joined[\"predictionstring_pred\"] = joined[\"predictionstring_pred\"].fillna(\" \")\n",
    "\n",
    "    joined[\"overlaps\"] = joined.apply(calc_overlap, axis=1)\n",
    "\n",
    "    # 2. If the overlap between the ground truth and prediction is >= 0.5,\n",
    "    # and the overlap between the prediction and the ground truth >= 0.5,\n",
    "    # the prediction is a match and considered a true positive.\n",
    "    # If multiple matches exist, the match with the highest pair of overlaps is taken.\n",
    "    joined[\"overlap1\"] = joined[\"overlaps\"].apply(lambda x: eval(str(x))[0])\n",
    "    joined[\"overlap2\"] = joined[\"overlaps\"].apply(lambda x: eval(str(x))[1])\n",
    "\n",
    "    joined[\"potential_TP\"] = (joined[\"overlap1\"] >= 0.5) & (joined[\"overlap2\"] >= 0.5)\n",
    "    joined[\"max_overlap\"] = joined[[\"overlap1\", \"overlap2\"]].max(axis=1)\n",
    "    tp_pred_ids = (\n",
    "        joined.query(\"potential_TP\")\n",
    "        .sort_values(\"max_overlap\", ascending=False)\n",
    "        .groupby([\"id\", \"predictionstring_gt\"])\n",
    "        .first()[\"pred_id\"]\n",
    "        .values\n",
    "    )\n",
    "\n",
    "    # 3. Any unmatched ground truths are false negatives\n",
    "    # and any unmatched predictions are false positives.\n",
    "    fp_pred_ids = [p for p in joined[\"pred_id\"].unique() if p not in tp_pred_ids]\n",
    "\n",
    "    matched_gt_ids = joined.query(\"potential_TP\")[\"gt_id\"].unique()\n",
    "    unmatched_gt_ids = [c for c in joined[\"gt_id\"].unique() if c not in matched_gt_ids]\n",
    "\n",
    "    # Get numbers of each type\n",
    "    TP = len(tp_pred_ids)\n",
    "    FP = len(fp_pred_ids)\n",
    "    FN = len(unmatched_gt_ids)\n",
    "    # calc microf1\n",
    "    my_f1_score = TP / (TP + 0.5 * (FP + FN))\n",
    "    return my_f1_score\n",
    "\n",
    "\n",
    "def score_feedback_comp(pred_df, gt_df, return_class_scores=False):\n",
    "    class_scores = {}\n",
    "    pred_df = pred_df[[\"id\", \"class\", \"predictionstring\"]].reset_index(drop=True).copy()\n",
    "    for discourse_type, gt_subset in gt_df.groupby(\"discourse_type\"):\n",
    "        pred_subset = (\n",
    "            pred_df.loc[pred_df[\"class\"] == discourse_type]\n",
    "            .reset_index(drop=True)\n",
    "            .copy()\n",
    "        )\n",
    "        class_score = score_feedback_comp_micro(pred_subset, gt_subset)\n",
    "        class_scores[discourse_type] = class_score\n",
    "    f1 = np.mean([v for v in class_scores.values()])\n",
    "    if return_class_scores:\n",
    "        return f1, class_scores\n",
    "    return f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_predictions(test_files):\n",
    "    dict = {'id':[],\n",
    "        'class': [],\n",
    "        'predictionstring':[]}\n",
    "    predict_df = pd.DataFrame(dict)\n",
    "    predictAcc_df = pd.DataFrame(dict)\n",
    "    error = 0\n",
    "    success = 0\n",
    "    for file in tqdm(test_files):\n",
    "        try:\n",
    "            predict_and_format(file, predict_df, predictAcc_df)\n",
    "            success = success + 1\n",
    "        except:\n",
    "            error = error + 1\n",
    "    print(success)\n",
    "    print(error)\n",
    "    return predict_df, predictAcc_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ground_truth(test_files):\n",
    "    full_df = pd.read_csv(\"train.csv\")\n",
    "    full_df = full_df.set_index(\"id\")\n",
    "    train_df = full_df.loc[test_files]\n",
    "    train_df = train_df.reset_index()\n",
    "    return train_df\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_checkpoint = \"brad1141/Bert_v5\"\n",
    "longformer_checkpoint = \"brad1141/Longformer_v5\"\n",
    "gpt2_checkpoint = \"brad1141/GPT2_v5\"\n",
    "\n",
    "#change model to the model api that you wish to use\n",
    "#the three models mentioned in the paper are listed above\n",
    "model = longformer_checkpoint\n",
    "# model = \"OldData_bertv2\"\n",
    "token_classifier = pipeline(\n",
    "    \"token-classification\", model=model, aggregation_strategy=\"simple\", tokenizer=model\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2890997b6aa34de6974764718ad30772",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "results_df, resultsAcc_df = make_predictions(test_files)\n",
    "gt_df = ground_truth(test_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "brad1141/Longformer_v5\n",
      "0.5666666666666667\n",
      "{'Claim': 0.0, 'Concluding Statement': 1.0, 'Counterclaim': 1.0, 'Evidence': 0.4, 'Position': 0.0, 'Rebuttal': 1.0}\n"
     ]
    }
   ],
   "source": [
    "f1_score, cs = score_feedback_comp(results_df, gt_df, return_class_scores=True)\n",
    "print(model)\n",
    "print(f1_score)\n",
    "print(cs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "avg_acc = []\n",
    "labels = []\n",
    "predictions = []\n",
    "tokenDict  = {\"LABEL_0\": 'Lead', \n",
    "    \"LABEL_1\": 'Position', \n",
    "    \"LABEL_2\": 'Evidence', \n",
    "    \"LABEL_3\": 'Claim', \n",
    "    \"LABEL_4\": 'Concluding Statement', \n",
    "    \"LABEL_5\": 'Counterclaim', \n",
    "    \"LABEL_6\": 'Rebuttal', \n",
    "    \"LABEL_7\": 'No Class',\n",
    "}\n",
    "for index, row in resultsAcc_df.iterrows():\n",
    "    wordCount = len(row[\"predictionstring\"].split())\n",
    "    for w in range(wordCount):\n",
    "        predictions.append(row[\"class\"])\n",
    "for index, row in gt_df.iterrows():\n",
    "    wordCount = len(row[\"predictionstring\"].split())\n",
    "    for w in range(wordCount):\n",
    "        labels.append(row[\"discourse_type\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "261\n",
      "251\n"
     ]
    }
   ],
   "source": [
    "print(len(predictions))\n",
    "print(len(labels))\n",
    "predictions = predictions[:len(labels)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_metric\n",
    "\n",
    "metric = load_metric(\"seqeval\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Brad\\anaconda3\\lib\\site-packages\\seqeval\\metrics\\v1.py:57: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'ead': {'precision': 0.0, 'recall': 0.0, 'f1': 0.0, 'number': 0},\n",
       " 'ebuttal': {'precision': 0.0, 'recall': 0.0, 'f1': 0.0, 'number': 1},\n",
       " 'laim': {'precision': 0.0, 'recall': 0.0, 'f1': 0.0, 'number': 1},\n",
       " 'oncluding Statement': {'precision': 0.0,\n",
       "  'recall': 0.0,\n",
       "  'f1': 0.0,\n",
       "  'number': 1},\n",
       " 'osition': {'precision': 0.0, 'recall': 0.0, 'f1': 0.0, 'number': 1},\n",
       " 'ounterclaim': {'precision': 0.0, 'recall': 0.0, 'f1': 0.0, 'number': 1},\n",
       " 'vidence': {'precision': 0.36619718309859156,\n",
       "  'recall': 0.3170731707317073,\n",
       "  'f1': 0.33986928104575165,\n",
       "  'number': 82},\n",
       " 'overall_precision': 0.2857142857142857,\n",
       " 'overall_recall': 0.2988505747126437,\n",
       " 'overall_f1': 0.2921348314606742,\n",
       " 'overall_accuracy': 0.46613545816733065}"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metric.compute(predictions=[predictions], references=[labels])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "c931ba401747e1100110d99c7b2e1195adf3961a7e00160e720e39c4d164b397"
  },
  "kernelspec": {
   "display_name": "Python 3.8.5 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}