Prompt_Engineering-Nepali-NER.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17cbb198-c010-48fa-b4aa-6789c93bc1b3",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Install dependencies\n",
    "!pip install -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92b59ea4-f098-4b45-a4c4-b48d57af25f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import requests\n",
    "import sklearn\n",
    "import numpy as np\n",
    "import scipy\n",
    "\n",
    "print(f\"pandas version: {pd.__version__}\")\n",
    "print(f\"requests version: {requests.__version__}\")\n",
    "print(f\"scikit-learn version: {sklearn.__version__}\")\n",
    "print(f\"numpy version: {np.__version__}\")\n",
    "print(f\"scipy version: {scipy.__version__}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff7dd1e2-6590-498e-95b9-06cee274d4c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the Nepali data from \"https://github.com/oya163/nepali-ner\", divide data in train, test set and put it in DATA_BASE_PATH\n",
    "\n",
    "# Define base path for data and output\n",
    "BASE_PATH = Path('/base/folder/path/')\n",
    "DATA_BASE_PATH = BASE_PATH / 'nepali-data'\n",
    "DATA_PREPROCESSED_PATH = DATA_BASE_PATH / 'processed'\n",
    "OUTPUT_BASE_PATH = BASE_PATH / 'results'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f6e85ca-a44a-44cc-9ece-d9c7057e4610",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "from collections import Counter\n",
    "\n",
    "# Read an NER-tagged file, split sentences, and count total characters\n",
    "def read_sentences_with_tags(input_path):\n",
    "    with open(input_path, 'r', encoding='utf-8') as file:\n",
    "        lines = file.readlines()\n",
    "\n",
    "    sentences, current_sentence = [], []\n",
    "    total_characters = sum(len(line) for line in lines)\n",
    "\n",
    "    for line in lines:\n",
    "        if line.strip():\n",
    "            current_sentence.append(line.strip())\n",
    "        else:\n",
    "            if current_sentence:\n",
    "                sentences.append(current_sentence)\n",
    "                current_sentence = []\n",
    "    if current_sentence:\n",
    "        sentences.append(current_sentence)\n",
    "\n",
    "    print(f\"Total sentences in {input_path}: {len(sentences)}\")\n",
    "    print(f\"Total characters in {input_path}: {total_characters}\")\n",
    "    return sentences\n",
    "\n",
    "# Check if a sentence contains all required NER tags\n",
    "def contains_all_required_tags(sentence, required_tags):\n",
    "    tag_set = {line.split()[1] for line in sentence if len(line.split()) > 1}\n",
    "    return required_tags.issubset(tag_set)\n",
    "\n",
    "# Count NER tags and measure tag diversity in a sentence\n",
    "def count_and_diversify_tags_in_sentence(sentence, ner_tags):\n",
    "    tag_types = {line.split()[1] for line in sentence if len(line.split()) > 1 and line.split()[1] in ner_tags}\n",
    "    return len(tag_types), len(tag_types)\n",
    "\n",
    "# Select sentences with all required tags and high tag diversity\n",
    "def get_sentences_with_all_tags_and_max_diversity(sentences, required_tags, ner_tags, top_n=5):\n",
    "    sentences_with_all_tags, sentences_with_diversity = [], []\n",
    "\n",
    "    for sentence in sentences:\n",
    "        tag_diversity, tag_count = count_and_diversify_tags_in_sentence(sentence, ner_tags)\n",
    "        if contains_all_required_tags(sentence, required_tags):\n",
    "            sentences_with_all_tags.append((tag_count, sentence))\n",
    "        else:\n",
    "            sentences_with_diversity.append((tag_diversity, tag_count, sentence))\n",
    "\n",
    "    print(f\"Number of sentences with all required tags: {len(sentences_with_all_tags)}\")\n",
    "    sentences_with_all_tags.sort(reverse=True, key=lambda x: x[0])\n",
    "    sentences_with_diversity.sort(reverse=True, key=lambda x: (x[0], x[1]))\n",
    "\n",
    "    selected_sentences = sentences_with_all_tags[:top_n]\n",
    "    selected_sentence_set = set(map(lambda x: tuple(x[1]), selected_sentences))\n",
    "\n",
    "    if len(selected_sentences) < top_n:\n",
    "        remaining_slots = top_n - len(selected_sentences)\n",
    "        diversity_candidates = [(tc, s) for _, tc, s in sentences_with_diversity if tuple(s) not in selected_sentence_set]\n",
    "        selected_sentences += diversity_candidates[:remaining_slots]\n",
    "\n",
    "    return selected_sentences\n",
    "\n",
    "# Convert selected sentences to a DataFrame\n",
    "def convert_sentences_to_dataframe(selected_sentences):\n",
    "    words, tags = [], []\n",
    "    for _, sentence in selected_sentences:\n",
    "        for line in sentence:\n",
    "            word, tag = line.split()\n",
    "            words.append(word)\n",
    "            tags.append(tag)\n",
    "        words.append('')  # Separator for sentences\n",
    "        tags.append('')\n",
    "    return pd.DataFrame({'words': words, 'tags': tags})\n",
    "\n",
    "# Adjust sentence endings: replace '.' with '।', move '।' to new lines, and remove duplicates\n",
    "def move_sentence_end(df):\n",
    "    new_rows = []\n",
    "    for _, row in df.iterrows():\n",
    "        word, tag = row['words'], row['tags']\n",
    "        if pd.isna(word) or word.strip() == '':\n",
    "            new_rows.append([word, tag])\n",
    "        elif isinstance(word, str):\n",
    "            if word.endswith('.'):\n",
    "                word = word.rstrip('.') + '।'\n",
    "            if '।' in word and word != '।':\n",
    "                word_without_end = word.replace('।', '')\n",
    "                if word_without_end:\n",
    "                    new_rows.append([word_without_end, tag])\n",
    "                new_rows.append(['।', 'O'])\n",
    "            else:\n",
    "                new_rows.append([word, tag])\n",
    "        else:\n",
    "            new_rows.append([word, tag])\n",
    "\n",
    "    cleaned_rows = []\n",
    "    for i in range(len(new_rows)):\n",
    "        word, tag = new_rows[i]\n",
    "        if i > 0 and word == '।' and new_rows[i - 1][0] == '।':\n",
    "            continue\n",
    "        cleaned_rows.append([word, tag])\n",
    "\n",
    "    return pd.DataFrame(cleaned_rows, columns=['words', 'tags'])\n",
    "\n",
    "# List of NER tags\n",
    "ner_tags = {'B-LOC', 'I-LOC', 'B-ORG', 'I-ORG', 'B-PER', 'I-PER'}\n",
    "required_tags = set(ner_tags)\n",
    "\n",
    "print(\"Processing 'nepali-data'\")\n",
    "all_sentences = read_sentences_with_tags(DATA_BASE_PATH / 'train.txt')\n",
    "\n",
    "# Process data for each top_n value and apply sentence-end adjustments\n",
    "top_n_values = [50, 100, 150, 200, 250, 300, 350, 400, 450, 500, 750, 1000, 1500, 2000, 2796]\n",
    "DATA_PREPROCESSED_PATH.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "for top_n in top_n_values:\n",
    "    print(f\"\\nProcessing top {top_n} sentences:\")\n",
    "    top_sentences = get_sentences_with_all_tags_and_max_diversity(all_sentences, required_tags, ner_tags, top_n)\n",
    "    df = convert_sentences_to_dataframe(top_sentences)\n",
    "    df_final = move_sentence_end(df)\n",
    "    \n",
    "    # Save final output\n",
    "    output_file = DATA_PREPROCESSED_PATH / f'{top_n}_selected_sentences.csv'\n",
    "    df_final.to_csv(output_file, index=False)\n",
    "    print(f\"Saved processed output to {output_file}\")\n",
    "\n",
    "# Process and save the test file with sentence-end adjustments\n",
    "test_input_file = DATA_BASE_PATH / 'test.txt'\n",
    "test_sentences = read_sentences_with_tags(test_input_file)\n",
    "\n",
    "df_test = convert_sentences_to_dataframe([(0, s) for s in test_sentences])  # Count not needed here\n",
    "df_test_final = move_sentence_end(df_test)\n",
    "\n",
    "# Save final test output\n",
    "test_output_file = DATA_PREPROCESSED_PATH / 'test_final.csv'\n",
    "df_test_final.to_csv(test_output_file, index=False)\n",
    "print(f\"Processed test file saved to {test_output_file}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c754251f-7a28-49b1-abb9-3a32727cc928",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "\n",
    "def post_request_to_api(api_url, model_name, tokens, prompt):\n",
    "    data = {\n",
    "        \"model\": model_name,  # Include the model name here\n",
    "        \"messages\": [\n",
    "            {\n",
    "                \"role\": \"system\",\n",
    "                \"content\": \"You are a helpful assistant\"\n",
    "            },\n",
    "            {\n",
    "                \"role\": \"user\",\n",
    "                \"content\": f\"{prompt}\",\n",
    "            }\n",
    "        ],\n",
    "        \"max_tokens\": tokens\n",
    "    }\n",
    "    response = requests.post(api_url, json=data, headers={\"Content-Type\": \"application/json\"})\n",
    "    response = response.json()\n",
    "    return response\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5963640d-04de-488e-a12f-1a4d860d2217",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Base class for all NER experiments\n",
    "class NERExperiment:\n",
    "    BIO_TAG_DESCRIPTION = \"\"\"\n",
    "    Each word is tagged as one of the following: B-LOC, I-LOC, B-ORG, I-ORG, B-PER, I-PER, or O.\n",
    "    - 'B-' marks the start of an entity.\n",
    "    - 'I-' marks the continuation of the same entity.\n",
    "    - Each 'I-' tag must follow a matching 'B-' tag (e.g., 'I-LOC' after 'B-LOC').\n",
    "    - Correct any cases where an 'I-' tag appears without a preceding 'B-' tag.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, train_examples, test_sentence):\n",
    "        self.train_examples = train_examples\n",
    "        self.test_sentence = test_sentence\n",
    "\n",
    "    def format_training_examples(self):\n",
    "        # Format all training examples for inclusion in the prompt\n",
    "        return \"\\n\\n\".join([format_sentence(s) for s in self.train_examples.values()])\n",
    "\n",
    "    def generate_prompt(self):\n",
    "        # To be implemented by subclasses\n",
    "        raise NotImplementedError(\"Subclasses must implement generate_prompt.\")\n",
    "\n",
    "    def get_llm_response(self, api_url, model_name, tokens, prompt):\n",
    "        # Send prompt to LLM API and retrieve response\n",
    "        return post_request_to_api(api_url, model_name, tokens, prompt)['choices'][0]['message']['content']\n",
    "\n",
    "\n",
    "# Experiment 1: NER with no training examples\n",
    "class Experiment1(NERExperiment):\n",
    "    def __init__(self, test_sentence):\n",
    "        super().__init__(None, test_sentence)\n",
    "\n",
    "    def generate_prompt(self):\n",
    "        prompt = f\"\"\"\n",
    "<context>You are an expert in identifying named entities in Nepali text, including persons, locations, and organizations.</context>\n",
    "<description>{self.BIO_TAG_DESCRIPTION}</description>\n",
    "<task>\n",
    "<test_sentence>{self.test_sentence}</test_sentence>\n",
    "Analyze the sentence and ensure each word is tagged correctly. Confirm that each 'I-' tag follows a 'B-' tag.\n",
    "</task>\n",
    "<output_formatting>\n",
    "<tagged_output>\n",
    "<pair><word>WORD</word><pred_tag>TAG</pred_tag></pair>\n",
    "...\n",
    "</tagged_output>\n",
    "Ensure no extra text outside the tags.\n",
    "</output_formatting>\n",
    "\"\"\"\n",
    "        return prompt\n",
    "\n",
    "\n",
    "# Experiment 2: NER with training examples\n",
    "class Experiment2(NERExperiment):\n",
    "    def __init__(self, train_examples, test_sentence):\n",
    "        super().__init__(train_examples, test_sentence)\n",
    "\n",
    "    def generate_prompt(self):\n",
    "        formatted_train_examples = self.format_training_examples()\n",
    "        prompt = f\"\"\"\n",
    "<context>You are an expert in identifying named entities in Nepali text. Below are training examples, followed by a test sentence.</context>\n",
    "<training_examples>{formatted_train_examples}</training_examples>\n",
    "<description>{self.BIO_TAG_DESCRIPTION}</description>\n",
    "<task>\n",
    "<test_sentence>{self.test_sentence}</test_sentence>\n",
    "Analyze the sentence and ensure each word is tagged correctly. Confirm that each 'I-' tag has a preceding 'B-' tag.\n",
    "</task>\n",
    "<output_formatting>\n",
    "<tagged_output>\n",
    "<pair><word>WORD</word><pred_tag>TAG</pred_tag></pair>\n",
    "...\n",
    "</tagged_output>\n",
    "Ensure no extra text outside the tags.\n",
    "</output_formatting>\n",
    "\"\"\"\n",
    "        return prompt\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db18b700-322e-4c30-a5b7-6836b3d718f9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1c61542-dcb4-402b-b3b6-448a49e83f7e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "import re\n",
    "import requests\n",
    "\n",
    "# Allowed NER tags for validation\n",
    "ALLOWED_TAGS = ['B-LOC', 'I-LOC', 'B-ORG', 'I-ORG', 'B-PER', 'I-PER', 'O']\n",
    "\n",
    "# Parse LLM response to extract word-tag pairs\n",
    "def parse_tagged_output(response):\n",
    "    tagged_output_match = re.search(r\"<tagged_output>(.*?)</tagged_output>\", response, re.DOTALL)\n",
    "    if tagged_output_match:\n",
    "        tagged_output_content = tagged_output_match.group(1)\n",
    "        word_tag_dict = {}\n",
    "        pair_matches = re.findall(r\"<pair>(.*?)</pair>\", tagged_output_content, re.DOTALL)\n",
    "        for pair in pair_matches:\n",
    "            word_match = re.search(r\"<word>(.*?)</word>\", pair, re.DOTALL)\n",
    "            pred_tag_match = re.search(r\"<pred_tag>(.*?)</pred_tag>\", pair, re.DOTALL)\n",
    "            if word_match and pred_tag_match:\n",
    "                word = word_match.group(1)\n",
    "                pred_tag = pred_tag_match.group(1)\n",
    "                word_tag_dict[word] = pred_tag\n",
    "        return word_tag_dict\n",
    "    else:\n",
    "        print(\"No <tagged_output> found in the response.\")\n",
    "        return {}\n",
    "\n",
    "# Check if all tags in the output are valid\n",
    "def are_tags_valid(tagged_output):\n",
    "    return all(tag in ALLOWED_TAGS for tag in tagged_output.values())\n",
    "\n",
    "# Send prompt to LLM with retry mechanism in case of missing or invalid output\n",
    "def prompt_with_retry(api_url, model_name, tokens, prompt, max_attempts=3):\n",
    "    attempt_count = 0\n",
    "    while attempt_count < max_attempts:\n",
    "        response = post_request_to_api(api_url, model_name, tokens, prompt)['choices'][0]['message']['content']\n",
    "        tagged_output = parse_tagged_output(response)\n",
    "        if tagged_output and are_tags_valid(tagged_output):\n",
    "            return tagged_output, attempt_count + 1, response\n",
    "        attempt_count += 1\n",
    "    # Log failed responses\n",
    "    with open(\"failed_responses.txt\", \"a\") as f:\n",
    "        f.write(f\"Prompt:\\n{prompt}\\nResponse:\\n{response}\\n\\n\")\n",
    "    return None, attempt_count, response\n",
    "\n",
    "# Format test sentence to contain only words (for LLM input)\n",
    "def format_test_sentence(sentence):\n",
    "    return \"\\n\".join([f\"<pair><word>{pair.split('<word>')[1].split('</word>')[0]}</word></pair>\" for pair in sentence])\n",
    "\n",
    "# Format sentence with NER tags for LLM training examples\n",
    "def format_sentence(sentence):\n",
    "    return \"<ner_tagged_sentence>\\n\" + \"\\n\".join(sentence) + \"\\n</ner_tagged_sentence>\"\n",
    "\n",
    "# Run an experiment and save NER predictions to a CSV file\n",
    "def run_experiment_and_save_predictions(experiment_class, train_examples, test_sentence, api_url, model_name, tokens, output_csv_file, index):\n",
    "    formatted_test_sentence = format_test_sentence(test_sentence)\n",
    "    if experiment_class in [Experiment1, Experiment2]:\n",
    "        experiment = experiment_class(formatted_test_sentence)\n",
    "    else:\n",
    "        experiment = experiment_class(train_examples, formatted_test_sentence)\n",
    "\n",
    "    prompt = experiment.generate_prompt()\n",
    "    tagged_output, attempt_count, response = prompt_with_retry(api_url, model_name, tokens, prompt)\n",
    "\n",
    "    # Create dictionary of words and predicted tags\n",
    "    if tagged_output is None:\n",
    "        word_tag_dict = {word: \"\" for word in re.findall(r\"<word>(.*?)</word>\", formatted_test_sentence, re.DOTALL)}\n",
    "    else:\n",
    "        word_tag_dict = tagged_output\n",
    "\n",
    "    # Convert word-tag dictionary to DataFrame and save\n",
    "    word_tag_df = pd.DataFrame(list(word_tag_dict.items()), columns=['words', 'pred_tags'])\n",
    "    word_tag_df['sentence_index'] = index\n",
    "    word_tag_df['llm_prompt_count'] = attempt_count\n",
    "    word_tag_df['llm_response'] = \"\"\n",
    "    word_tag_df.at[0, 'llm_response'] = response  # Save response in the first row\n",
    "\n",
    "    empty_row = pd.DataFrame([[\"\", \"\", \"\", \"\", \"\"]], columns=['words', 'pred_tags', 'sentence_index', 'llm_prompt_count', 'llm_response'])\n",
    "\n",
    "    # Append results and an empty row to the CSV file\n",
    "    if not output_csv_file.exists():\n",
    "        word_tag_df.to_csv(output_csv_file, index=False, mode='w', header=True)\n",
    "        empty_row.to_csv(output_csv_file, index=False, mode='a', header=False)\n",
    "    else:\n",
    "        word_tag_df.to_csv(output_csv_file, index=False, mode='a', header=False)\n",
    "        empty_row.to_csv(output_csv_file, index=False, mode='a', header=False)\n",
    "    print(f'Saved output for sentence index {index}')\n",
    "\n",
    "# Extract sentences from DataFrame for LLM input\n",
    "def extract_sentences_from_dataframe(df):\n",
    "    sentences = {}\n",
    "    current_sentence = []\n",
    "    sentence_index = 0\n",
    "\n",
    "    for _, row in df.iterrows():\n",
    "        word = row['words']\n",
    "        tag = row['tags']\n",
    "\n",
    "        if pd.isna(word):\n",
    "            if current_sentence:\n",
    "                sentences[sentence_index] = current_sentence\n",
    "                current_sentence = []\n",
    "                sentence_index += 1\n",
    "        else:\n",
    "            current_sentence.append(f\"<pair><word>{word}</word><pred_tag>{tag}</pred_tag></pair>\")\n",
    "\n",
    "    if current_sentence:\n",
    "        sentences[sentence_index] = current_sentence\n",
    "            \n",
    "    return sentences\n",
    "\n",
    "# Training data sizes to use in experiments\n",
    "training_data_sizes = [50, 100, 150, 200, 250, 300, 350, 400, 450, 500, 750, 1000, 1500, 2000, 2796]\n",
    "\n",
    "# Run experiments with specified training data sizes\n",
    "def main():\n",
    "    api_url = \"model's api\"\n",
    "    model_name = \"your model name\"\n",
    "    tokens = 3000\n",
    "\n",
    "    for num_of_training_data in training_data_sizes:\n",
    "        print(f\"Running experiments with {num_of_training_data} training examples.\")\n",
    "\n",
    "        # Load train and test data for Nepali NER\n",
    "        nepali_train_df = pd.read_csv(DATA_PREPROCESSED_PATH / f'{num_of_training_data}_selected_sentences_final.csv')\n",
    "        nepali_test_df = pd.read_csv(DATA_PREPROCESSED_PATH / 'test_final.csv')\n",
    "\n",
    "        # Extract sentences for experiment input\n",
    "        nepali_train_sentences = extract_sentences_from_dataframe(nepali_train_df)\n",
    "        nepali_test_sentences = extract_sentences_from_dataframe(nepali_test_df)\n",
    "\n",
    "        # Experiment configurations\n",
    "        experiment_data = {\n",
    "            1: {\n",
    "                'train_examples': None,  # No training data required\n",
    "                'test_examples': nepali_test_sentences,\n",
    "                'experiment_class': Experiment1,\n",
    "                'output_file': OUTPUT_BASE_PATH / f'experiment_1_predictions_{num_of_training_data}.csv'\n",
    "            },\n",
    "            2: {\n",
    "                'train_examples': nepali_train_sentences,\n",
    "                'test_examples': nepali_test_sentences,\n",
    "                'experiment_class': Experiment2,\n",
    "                'output_file': OUTPUT_BASE_PATH / f'experiment_2_predictions_{num_of_training_data}.csv'\n",
    "            }\n",
    "        }\n",
    "\n",
    "        # Run each experiment\n",
    "        for exp_num, exp_data in experiment_data.items():\n",
    "            train_examples = exp_data['train_examples']\n",
    "            test_sentences = exp_data['test_examples']\n",
    "            experiment_class = exp_data['experiment_class']\n",
    "            output_csv_file = exp_data['output_file']\n",
    "\n",
    "            total_sentences = len(test_sentences)\n",
    "            print(f\"Total test sentences for Experiment {exp_num}: {total_sentences}\")\n",
    "\n",
    "            # Save predictions for each sentence\n",
    "            for index, test_sentence in test_sentences.items():\n",
    "                if test_sentence:\n",
    "                    run_experiment_and_save_predictions(\n",
    "                        experiment_class, train_examples, test_sentence,\n",
    "                        api_url, model_name, tokens, output_csv_file, index\n",
    "                    )\n",
    "                else:\n",
    "                    print(f\"Sentence index {index} not found in test sentences.\")\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7485ac8f-8d3c-4849-9294-584c65583fc8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c835858-b571-4bc5-8efc-a8557b399f89",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "# Define paths for output directory\n",
    "output_folder = os.path.join(OUTPUT_BASE_PATH, 'empty_pred_tags_results')\n",
    "\n",
    "# Create the output directory if it doesn't exist\n",
    "os.makedirs(output_folder, exist_ok=True)\n",
    "\n",
    "# Process each CSV file in the results folder\n",
    "for filename in os.listdir(OUTPUT_BASE_PATH):\n",
    "    if filename.endswith('.csv'):\n",
    "        # Load CSV data into a DataFrame\n",
    "        file_path = os.path.join(OUTPUT_BASE_PATH, filename)\n",
    "        df = pd.read_csv(file_path)\n",
    "\n",
    "        print(f\"Processing file: '{filename}'\")\n",
    "\n",
    "        # Fill NaN values with empty strings for consistent handling\n",
    "        df = df.fillna('')\n",
    "\n",
    "        # Track the current sentence index for clearing repeated 'llm_prompt_count' values\n",
    "        current_sentence_index = None\n",
    "\n",
    "        # Clear repeated 'llm_prompt_count' values within the same sentence\n",
    "        for index, row in df.iterrows():\n",
    "            # Identify new sentence boundaries\n",
    "            if row['sentence_index'] != current_sentence_index:\n",
    "                current_sentence_index = row['sentence_index']\n",
    "            else:\n",
    "                # Set 'llm_prompt_count' to empty for rows within the same sentence\n",
    "                df.at[index, 'llm_prompt_count'] = ''\n",
    "\n",
    "        # Save the modified DataFrame to a new CSV file\n",
    "        modified_output_path = os.path.join(output_folder, f'modified_{filename}')\n",
    "        df.to_csv(modified_output_path, index=False)\n",
    "        print(f\"Processed file saved as '{modified_output_path}'.\")\n",
    "\n",
    "        # Count unique values in 'llm_prompt_count' after removing empty entries\n",
    "        non_empty_counts = df['llm_prompt_count'][df['llm_prompt_count'] != '']\n",
    "        value_counts = non_empty_counts.value_counts()\n",
    "        print(\"Occurrences of each unique 'llm_prompt_count' value:\")\n",
    "        print(value_counts)\n",
    "\n",
    "        # Save the counts to a CSV file\n",
    "        output_counts_path = os.path.join(output_folder, f'llm_prompt_count_repeats_{filename}')\n",
    "        value_counts.to_csv(output_counts_path, header=['Count'])\n",
    "        print(f\"Counts of 'llm_prompt_count' values saved as '{output_counts_path}'.\")\n",
    "\n",
    "        # Filter rows where 'llm_prompt_count' is 3 and 'pred_tags' is empty\n",
    "        prompt_3_df = df[df['llm_prompt_count'] == 3]\n",
    "        empty_pred_tags_df = prompt_3_df[(prompt_3_df['pred_tags'].isna()) | (prompt_3_df['pred_tags'] == '')]\n",
    "\n",
    "        # Identify unique sentence indices with empty 'pred_tags' where 'llm_prompt_count' is 3\n",
    "        sentence_indices_with_empty_pred_tags = empty_pred_tags_df['sentence_index'].unique()\n",
    "\n",
    "        # Extract all rows for sentences with empty 'pred_tags' and save to a new file\n",
    "        full_sentences_df = df[df['sentence_index'].isin(sentence_indices_with_empty_pred_tags)]\n",
    "        empty_pred_tags_count = len(empty_pred_tags_df)\n",
    "        print(f\"Number of times 'pred_tags' is empty for 'llm_prompt_count' = 3: {empty_pred_tags_count}\")\n",
    "\n",
    "        # Save sentences with empty 'pred_tags' to a new CSV\n",
    "        output_empty_pred_tags_path = os.path.join(output_folder, f'empty_pred_tags_sentences_{filename}')\n",
    "        full_sentences_df.to_csv(output_empty_pred_tags_path, index=False)\n",
    "        print(f\"Saved sentences with 'llm_prompt_count' = 3 and empty 'pred_tags' to '{output_empty_pred_tags_path}'.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ba9e335-c42f-4017-9cfa-28c03296c44b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0bd1b80e-814e-4f94-83f1-a4b6dd1a7137",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import math\n",
    "import difflib\n",
    "import csv\n",
    "\n",
    "# Allowed NER tags for validation\n",
    "ALLOWED_TAGS = ['B-LOC', 'I-LOC', 'B-ORG', 'I-ORG', 'B-PER', 'I-PER', 'O']\n",
    "\n",
    "# Check if all tags in a list are valid\n",
    "def tags_are_allowed(tags):\n",
    "    return all(tag in ALLOWED_TAGS for tag in tags)\n",
    "\n",
    "# Extract sentences and tags from DataFrame columns\n",
    "def extract_sentences_and_tags(df, words_column, tags_column):\n",
    "    sentences, tags, current_sentence, current_tag = [], [], [], []\n",
    "    \n",
    "    for _, row in df.iterrows():\n",
    "        if isinstance(row[words_column], float) and math.isnan(row[words_column]):  # NaN separates sentences\n",
    "            if current_sentence:\n",
    "                sentences.append(current_sentence)\n",
    "                tags.append(current_tag)\n",
    "                current_sentence, current_tag = [], []  # Reset for next sentence\n",
    "        else:\n",
    "            current_sentence.append(row[words_column])\n",
    "            current_tag.append(row[tags_column])\n",
    "    \n",
    "    # Append the last sentence if present\n",
    "    if current_sentence:\n",
    "        sentences.append(current_sentence)\n",
    "        tags.append(current_tag)\n",
    "    \n",
    "    return sentences, tags\n",
    "\n",
    "# Calculate similarity between two sentences based on word sequence\n",
    "def sentence_similarity(sent1, sent2):\n",
    "    return difflib.SequenceMatcher(None, sent1, sent2).ratio()\n",
    "\n",
    "# Align sentences and save to CSV with both tags and predicted tags columns\n",
    "def save_aligned_sentences_to_csv(sentences_test, tags_test, sentences_pred, tags_pred, threshold=0.8, output_file='aligned_sentences.csv'):\n",
    "    aligned_count = 0\n",
    "    unmatched_test, unmatched_tags_test = sentences_test.copy(), tags_test.copy()\n",
    "    unmatched_pred, unmatched_tags_pred = sentences_pred.copy(), tags_pred.copy()\n",
    "\n",
    "    with open(output_file, mode='w', newline='', encoding='utf-8') as file:\n",
    "        writer = csv.writer(file)\n",
    "        writer.writerow(['Test Sentence', 'Predicted Sentence', 'Test Tags', 'Predicted Tags'])\n",
    "        \n",
    "        # Align test sentences with predicted sentences based on similarity threshold\n",
    "        for i, test_sent in enumerate(sentences_test):\n",
    "            for j, pred_sent in enumerate(sentences_pred):\n",
    "                similarity = sentence_similarity(test_sent, pred_sent)\n",
    "                \n",
    "                # Save aligned pairs if similarity threshold and tag validation are met\n",
    "                if similarity >= threshold and tags_are_allowed(tags_pred[j]):\n",
    "                    aligned_test, aligned_pred, aligned_tags_test, aligned_tags_pred = align_sentences_tags(\n",
    "                        test_sent, pred_sent, tags_test[i], tags_pred[j])\n",
    "                    \n",
    "                    for t_word, p_word, t_tag, p_tag in zip(aligned_test, aligned_pred, aligned_tags_test, aligned_tags_pred):\n",
    "                        writer.writerow([t_word, p_word, t_tag, p_tag])\n",
    "                    writer.writerow([])  # Empty row between aligned pairs\n",
    "\n",
    "                    unmatched_test.remove(test_sent)\n",
    "                    unmatched_tags_test.remove(tags_test[i])\n",
    "                    unmatched_pred.remove(pred_sent)\n",
    "                    unmatched_tags_pred.remove(tags_pred[j])\n",
    "\n",
    "                    aligned_count += 1\n",
    "                    break  # Move to next test sentence after alignment\n",
    "                elif not tags_are_allowed(tags_pred[j]):\n",
    "                    unaligned_output = output_file.replace('aligned', 'unaligned')\n",
    "                    save_unaligned_sentence_to_csv(test_sent, tags_test[i], pred_sent, tags_pred[j], unaligned_output)\n",
    "    \n",
    "    print(f\"Aligned sentences: {aligned_count}\")\n",
    "    print(f\"Unmatched sentences in test set: {len(unmatched_test)}\")\n",
    "    print(f\"Unmatched sentences in predictions: {len(unmatched_pred)}\")\n",
    "\n",
    "    # Save unaligned sentences in order of similarity\n",
    "    save_unaligned_analysis(unmatched_test, unmatched_tags_test, unmatched_pred, unmatched_tags_pred, output_file=output_file.replace('aligned', 'unaligned'))\n",
    "\n",
    "# Save unaligned sentences to a CSV file\n",
    "def save_unaligned_sentence_to_csv(test_sent, test_tags, pred_sent, pred_tags, output_file):\n",
    "    with open(output_file, mode='a', newline='', encoding='utf-8') as file:\n",
    "        writer = csv.writer(file)\n",
    "        writer.writerow(['Test Sentence', 'Predicted Sentence', 'Test Tags', 'Predicted Tags'])\n",
    "        for t_word, p_word, t_tag, p_tag in zip(test_sent, pred_sent, test_tags, pred_tags):\n",
    "            writer.writerow([t_word, p_word, t_tag, p_tag])\n",
    "        writer.writerow([])\n",
    "\n",
    "# Align sentences and tags by matching words, filling unmatched words with blanks\n",
    "def align_sentences_tags(sent1, sent2, tags1, tags2):\n",
    "    matcher = difflib.SequenceMatcher(None, sent1, sent2)\n",
    "    aligned_sent1, aligned_sent2, aligned_tags1, aligned_tags2 = [], [], [], []\n",
    "    \n",
    "    for opcode, i1, i2, j1, j2 in matcher.get_opcodes():\n",
    "        if opcode == 'equal':\n",
    "            aligned_sent1.extend(sent1[i1:i2])\n",
    "            aligned_sent2.extend(sent2[j1:j2])\n",
    "            aligned_tags1.extend(tags1[i1:i2])\n",
    "            aligned_tags2.extend(tags2[j1:j2])\n",
    "        elif opcode == 'insert':\n",
    "            aligned_sent1.extend([''] * (j2 - j1))\n",
    "            aligned_sent2.extend(sent2[j1:j2])\n",
    "            aligned_tags1.extend([''] * (j2 - j1))\n",
    "            aligned_tags2.extend(tags2[j1:j2])\n",
    "        elif opcode == 'delete':\n",
    "            aligned_sent1.extend(sent1[i1:i2])\n",
    "            aligned_sent2.extend([''] * (i2 - i1))\n",
    "            aligned_tags1.extend(tags1[i1:i2])\n",
    "            aligned_tags2.extend([''] * (i2 - i1))\n",
    "        elif opcode == 'replace':\n",
    "            aligned_sent1.extend(sent1[i1:i2])\n",
    "            aligned_sent2.extend(sent2[j1:j2])\n",
    "            aligned_tags1.extend(tags1[i1:i2])\n",
    "            aligned_tags2.extend(tags2[j1:j2])\n",
    "    \n",
    "    return aligned_sent1, aligned_sent2, aligned_tags1, aligned_tags2\n",
    "\n",
    "# Analyze and save unaligned sentences by descending similarity\n",
    "def save_unaligned_analysis(unmatched_test, unmatched_tags_test, unmatched_pred, unmatched_tags_pred, output_file='unaligned_sentences.csv'):\n",
    "    unaligned_pairs = []\n",
    "    \n",
    "    # Calculate similarity for unaligned sentences\n",
    "    for i, test_sent in enumerate(unmatched_test):\n",
    "        for j, pred_sent in enumerate(unmatched_pred):\n",
    "            if i >= len(unmatched_tags_test) or j >= len(unmatched_tags_pred):\n",
    "                continue\n",
    "            similarity = sentence_similarity(test_sent, pred_sent)\n",
    "            if similarity > 0:\n",
    "                unaligned_pairs.append((test_sent, pred_sent, unmatched_tags_test[i], unmatched_tags_pred[j], similarity))\n",
    "    \n",
    "    # Sort pairs by similarity and save to CSV\n",
    "    unaligned_pairs.sort(key=lambda x: x[4], reverse=True)\n",
    "\n",
    "    with open(output_file, mode='w', newline='', encoding='utf-8') as file:\n",
    "        writer = csv.writer(file)\n",
    "        writer.writerow(['Test Sentence', 'Predicted Sentence', 'Test Tags', 'Predicted Tags'])\n",
    "        \n",
    "        for test_sent, pred_sent, test_tags, pred_tags, similarity in unaligned_pairs:\n",
    "            aligned_test, aligned_pred, aligned_tags_test, aligned_tags_pred = align_sentences_tags(test_sent, pred_sent, test_tags, pred_tags)\n",
    "            for t_word, p_word, t_tag, p_tag in zip(aligned_test, aligned_pred, aligned_tags_test, aligned_tags_pred):\n",
    "                writer.writerow([t_word, p_word, t_tag, p_tag])\n",
    "            writer.writerow([])\n",
    "\n",
    "    print(f\"Unaligned sentences sorted by similarity saved to {output_file}.\")\n",
    "\n",
    "# Folder paths for aligned and unaligned output\n",
    "aligned_folder = os.path.join(OUTPUT_BASE_PATH, 'aligned/')\n",
    "unaligned_folder = os.path.join(OUTPUT_BASE_PATH, 'unaligned/')\n",
    "\n",
    "# Create output folders if they don't exist\n",
    "os.makedirs(aligned_folder, exist_ok=True)\n",
    "os.makedirs(unaligned_folder, exist_ok=True)\n",
    "\n",
    "# Load test file and extract sentences and tags\n",
    "test_file = pd.read_csv(\"/home/jovyan/transfer_learning/data/nepali-data/test_mapped_updated.csv\")\n",
    "sentences_test, tags_test = extract_sentences_and_tags(test_file, 'words', 'tags')\n",
    "\n",
    "# Process each results file in the results folder\n",
    "for file_name in os.listdir(OUTPUT_BASE_PATH):\n",
    "    if file_name.endswith('.csv') and ('experiment_4' in file_name or 'experiment_2' in file_name):\n",
    "        results_path = os.path.join(OUTPUT_BASE_PATH, file_name)\n",
    "        results = pd.read_csv(results_path)\n",
    "        \n",
    "        # Extract sentences and predicted tags from results\n",
    "        sentences_pred, tags_pred = extract_sentences_and_tags(results, 'words', 'pred_tags')\n",
    "\n",
    "        # Save aligned and unaligned sentences\n",
    "        aligned_output = os.path.join(aligned_folder, f'aligned_{file_name}')\n",
    "        unaligned_output = os.path.join(unaligned_folder, f'unaligned_{file_name}')\n",
    "        save_aligned_sentences_to_csv(sentences_test, tags_test, sentences_pred, tags_pred, threshold=0.8, output_file=aligned_output)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64edf426-94d6-49c3-877a-52b416bb2479",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from sklearn.metrics import precision_recall_fscore_support, classification_report, confusion_matrix\n",
    "import numpy as np\n",
    "\n",
    "# Function to compute and print evaluation metrics for each CSV file in a folder\n",
    "def compute_metrics_for_each_file(folder_path):\n",
    "    # Process each CSV file in the specified folder\n",
    "    for file_name in os.listdir(folder_path):\n",
    "        if file_name.endswith('.csv'):\n",
    "            file_path = os.path.join(folder_path, file_name)\n",
    "            df = pd.read_csv(file_path)\n",
    "            \n",
    "            # Remove rows with missing values in 'Predicted Tags' or 'Test Tags'\n",
    "            df = df.dropna(subset=['Predicted Tags', 'Test Tags'])\n",
    "\n",
    "            # Extract true and predicted tags from the DataFrame\n",
    "            true_tags = df['Test Tags'].values\n",
    "            pred_tags = df['Predicted Tags'].values\n",
    "\n",
    "            # Exclude 'O' tags (non-entity tokens) from evaluation\n",
    "            mask = true_tags != 'O'\n",
    "            true_tags = true_tags[mask]\n",
    "            pred_tags = pred_tags[mask]\n",
    "\n",
    "            # Calculate weighted precision, recall, and F1 score with zero division handling\n",
    "            precision, recall, f1, _ = precision_recall_fscore_support(true_tags, pred_tags, average='weighted', zero_division=0)\n",
    "            conf_matrix = confusion_matrix(true_tags, pred_tags, labels=np.unique(true_tags))\n",
    "\n",
    "            # Display metrics for the current file\n",
    "            print(f\"Metrics for {file_name}:\")\n",
    "            print(\"Classification Report:\")\n",
    "            print(classification_report(true_tags, pred_tags, labels=np.unique(true_tags), zero_division=0))\n",
    "            print(\"\\nConfusion Matrix:\")\n",
    "            print(conf_matrix)\n",
    "            print(f\"\\nOverall Precision: {precision:.4f}\")\n",
    "            print(f\"Overall Recall: {recall:.4f}\")\n",
    "            print(f\"Overall F1 Score: {f1:.4f}\\n\")\n",
    "            print(\"=\" * 50)  # Divider for readability\n",
    "\n",
    "# Run the metric computation function on the specified folder\n",
    "compute_metrics_for_each_file(aligned_folder)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60e72ce0-45e2-45fa-a3b5-2abf15871139",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd\n",
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "from scipy.stats import kruskal\n",
    "\n",
    "# Initialize lists to store results for each metric across all files\n",
    "all_f1_scores = []\n",
    "file_names = []\n",
    "\n",
    "# Function to load and process each file, compute metrics\n",
    "def compute_metrics_for_each_file(folder_path):\n",
    "    # Iterate over each CSV file in the folder\n",
    "    for file_name in os.listdir(folder_path):\n",
    "        if file_name.endswith('.csv'):\n",
    "            file_path = os.path.join(folder_path, file_name)\n",
    "            df = pd.read_csv(file_path)\n",
    "            \n",
    "            # Filter out rows with missing 'Predicted Tags' or 'Test Tags'\n",
    "            df = df.dropna(subset=['Predicted Tags', 'Test Tags'])\n",
    "\n",
    "            # Extract true and predicted tags for the file\n",
    "            true_tags = df['Test Tags'].values\n",
    "            pred_tags = df['Predicted Tags'].values\n",
    "\n",
    "            # Filter out 'O' tags\n",
    "            mask = true_tags != 'O'\n",
    "            true_tags = true_tags[mask]\n",
    "            pred_tags = pred_tags[mask]\n",
    "\n",
    "            # Calculate precision, recall, F1 for each unique tag, excluding 'O'\n",
    "            labels = np.unique(true_tags)\n",
    "            _, _, f1, _ = precision_recall_fscore_support(true_tags, pred_tags, labels=labels, average='weighted',zero_division=0)\n",
    "\n",
    "            # Append the list of scores per tag for each file (multiple values per group for Kruskal-Wallis)\n",
    "            all_f1_scores.append(f1)\n",
    "            file_names.append(file_name)\n",
    "\n",
    "# Run the function on the specified folder\n",
    "compute_metrics_for_each_file(aligned_folder)\n",
    "\n",
    "# Check if there are at least two groups for the Kruskal-Wallis test\n",
    "if len(all_precisions) < 2 or len(all_recalls) < 2 or len(all_f1_scores) < 2:\n",
    "    print(\"Not enough groups to perform Kruskal-Wallis test. Ensure at least two CSV files are in the specified folder.\")\n",
    "else:\n",
    "    \n",
    "    # Flatten the lists to compare values across files\n",
    "    f1_test = kruskal(*all_f1_scores)\n",
    "\n",
    "    # Display the results\n",
    "    print(\"Kruskal-Wallis Test Results:\")\n",
    "    print(f\"F1 Score - p-value: {f1_test.pvalue:.4f}\")\n",
    "\n",
    "    # Interpret results\n",
    "    if f1_test.pvalue < 0.05:\n",
    "        print(\"\\nThere is a statistically significant difference in F1 scores among the models.\")\n",
    "    else:\n",
    "        print(\"\\nNo statistically significant difference in F1 scores among the models.\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}