user_study_analysis/mmlu_study.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9acc6a8-6211-43d2-9baa-6ae5d8e5d164",
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import math\n",
    "import os\n",
    "import pickle\n",
    "import random\n",
    "import sys\n",
    "from random import randint\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import requests\n",
    "import statsmodels\n",
    "from PIL import Image\n",
    "from scipy import stats\n",
    "from scipy.stats import ttest_ind, ttest_rel\n",
    "from sklearn.metrics import classification_report\n",
    "from statsmodels.stats import multitest\n",
    "from tabulate import tabulate\n",
    "from tqdm import tqdm\n",
    "\n",
    "\n",
    "def get_each_time_taken(response, time_string):\n",
    "    raw_times = response[time_string]\n",
    "    clean_times = []  # first time will be dropped (unless we know what time they)\n",
    "    i = 1\n",
    "    while i < len(raw_times):\n",
    "        clean_time = raw_times[i] - raw_times[i - 1]\n",
    "        # print(clean_time_noai)\n",
    "        # print(clean_time_noai)\n",
    "        if (\n",
    "            clean_time < 120000\n",
    "        ):  # if people leave mid-task, then comeback (if more than 2minutes spent on 1 task, don't add to clean time)\n",
    "            clean_times.append(clean_time)\n",
    "        i += 1\n",
    "    # print(clean_times_noai)\n",
    "    return clean_times\n",
    "\n",
    "\n",
    "def time_add_to_table(times, col_num):\n",
    "    # print(times)\n",
    "    s_times = [x / 1000 for x in times]\n",
    "    mean, std = np.mean(s_times), np.std(s_times) / np.sqrt(len(s_times))\n",
    "    r_mean = round(mean, 3)\n",
    "    r_std = round(std, 3)\n",
    "    time_row[col_num] = (r_mean, r_std)\n",
    "\n",
    "\n",
    "def ai_rel_add_to_table(ai_rel, col_num):\n",
    "    mean, std = np.mean(ai_rel), np.std(ai_rel) / np.sqrt(len(ai_rel))\n",
    "    r_mean = round(mean, 3)\n",
    "    r_std = round(std, 3)\n",
    "    ai_reliance_row[col_num] = (r_mean, r_std)\n",
    "\n",
    "\n",
    "def mean_std_add_to_table(accuracies, col_num):\n",
    "    mean, std = np.mean(accuracies), np.std(accuracies) / np.sqrt(len(accuracies))\n",
    "    r_mean = round(mean, 3)\n",
    "    r_std = round(std, 3)\n",
    "    mean_row[col_num] = (r_mean, r_std)\n",
    "\n",
    "\n",
    "def p_t_add_to_table(accuracies, col_num):\n",
    "    t_stat, p_val = ttest_ind(accuracies, accuracies_hum_ai_1)\n",
    "    r_t_stat = round(t_stat, 3)\n",
    "    r_p_val = round(p_val, 3)\n",
    "    p_row[col_num] = (r_p_val, r_t_stat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af5ece7b-70ad-4d2f-b2e6-eddb04f4cc89",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pickle.load(open(\"../data/mmlu_study_data.pkl\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba8563ad-c182-4076-86e4-dbdb87d9a3d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracies_ai_all = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e286968-6828-4a8f-bd4f-281c8a6baf18",
   "metadata": {},
   "outputs": [],
   "source": [
    "tasks = data['tasks']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e407a554-731e-4f36-af68-f64387dc43f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "tasks['task_1exp_condition_2']['testing_withai_ai_answers']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5eb31e6-c9b1-4277-b7b5-94b923a1b291",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Human+AI baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7280bd33-44e3-43a5-bd88-29884253580a",
   "metadata": {},
   "outputs": [],
   "source": [
    "responses = data['human+ai baseline'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d751bddc-91be-45e8-bfaf-a267de92aaf0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "accuracies_hum_no_ai = []\n",
    "accuracies_hum_ai = []\n",
    "accuracies_ai_with_ai = []\n",
    "accuracies_ai_no_ai = []\n",
    "failed_attention_check = []\n",
    "ai_reliance = []\n",
    "responses_no_ai = {}\n",
    "responses_with_ai = {}\n",
    "# time variables\n",
    "ai_reliance = []\n",
    "times_hum_no_ai = []\n",
    "times_hum_with_ai = []\n",
    "\n",
    "for i in range(len(responses)):\n",
    "# find the matching task\n",
    "    id_task = responses[i]['id'].split('-')[0]\n",
    "    task = tasks[id_task]\n",
    "    user_answers_no_ai_raw = responses[i]['testing_answers_noai']\n",
    "    user_answers_ai_raw = responses[i]['testing_answers_withai']\n",
    "    # task\n",
    "    labels_no_ai = task['testing_withairec_labels']\n",
    "    labels_ai = task['testing_withai_label']\n",
    "    ai_no_ai = task['testing_withairec_ai_answers_raw']\n",
    "    ai_ai = task['testing_withai_ai_answers_raw']\n",
    "    att_check_no_ai = task['testing_attentioncheck']\n",
    "    att_check_ai = task['testing_withai_attentioncheck']\n",
    "    response = responses[i]\n",
    "    clean_times_noai = get_each_time_taken(response,'testing_times_noai') # [example1 time, example2 time, ...]\n",
    "    clean_times_withai = get_each_time_taken(response,'testing_times_withai')\n",
    "    times_hum_no_ai.append(sum(clean_times_noai)/len(clean_times_noai))  # [avg time per example by person1, by person2, ...]\n",
    "    times_hum_with_ai.append(sum(clean_times_withai)/len(clean_times_withai))\n",
    "\n",
    "    user_answers_no_ai =user_answers_no_ai_raw\n",
    "    user_answers_ai = []\n",
    "    reliance_ai = 0\n",
    "\n",
    "    ids_no_ai = task['testing_ids']\n",
    "    ids_ai = task['testing_withai_ids']\n",
    "\n",
    "\n",
    "    for j in range(len(user_answers_ai_raw)):\n",
    "        if user_answers_ai_raw[j] == 'ai':\n",
    "            user_answers_ai.append(ai_ai[j])\n",
    "            reliance_ai += 1\n",
    "        else:\n",
    "            user_answers_ai.append(user_answers_ai_raw[j])\n",
    "\n",
    "\n",
    "    # check if the attention check is correct\n",
    "    failed_checks = 0\n",
    "\n",
    "    for j in range(len(user_answers_no_ai)):\n",
    "        if att_check_no_ai[j] == 1:\n",
    "            if user_answers_no_ai[j] != labels_no_ai[j]:\n",
    "                failed_checks += 1\n",
    "    if failed_checks >= np.sum(att_check_no_ai):\n",
    "        continue\n",
    "    failed_checks = 0\n",
    "\n",
    "    for j in range(len(user_answers_ai)):\n",
    "        if att_check_ai[j] == 1:\n",
    "            if user_answers_ai[j] != labels_ai[j]:\n",
    "                failed_checks += 1\n",
    "    failed_attention_check.append(failed_checks)\n",
    "    if failed_checks >= np.sum(att_check_ai):\n",
    "        continue\n",
    "    for j in range(len(user_answers_no_ai_raw)):\n",
    "        responses_no_ai[ids_no_ai[j]] = user_answers_no_ai[j]\n",
    "\n",
    "    for j in range(len(user_answers_ai_raw)):\n",
    "        ai_reliancej = 0 \n",
    "        if user_answers_ai_raw[j] == 'ai':\n",
    "            ai_reliancej = 1\n",
    "        responses_with_ai[ids_ai[j]] = 1.0 * ai_reliancej     \n",
    "\n",
    "    \n",
    "\n",
    "    try:\n",
    "        accuracies_hum_no_ai.append(accuracy_score(labels_no_ai, user_answers_no_ai))\n",
    "        accuracies_ai_no_ai.append(accuracy_score(labels_no_ai, ai_no_ai))\n",
    "\n",
    "    except:\n",
    "        accuracies_hum_no_ai.append(accuracy_score(labels_no_ai[:14], user_answers_no_ai[:14]))\n",
    "        accuracies_hum_ai.append(accuracy_score(labels_ai[:14], user_answers_ai[:14]))\n",
    "\n",
    "    try:\n",
    "\n",
    "        accuracies_hum_ai.append(accuracy_score(labels_ai, user_answers_ai))\n",
    "        accuracies_ai_with_ai.append(accuracy_score(labels_ai, ai_ai))\n",
    "        ai_reliance.append(reliance_ai/len(user_answers_ai))\n",
    "\n",
    "    except:\n",
    "        accuracies_ai_with_ai.append(accuracy_score(labels_ai[:14], ai_ai[:14]))\n",
    "        accuracies_ai_no_ai.append(accuracy_score(labels_no_ai[:14], ai_no_ai[:14]))\n",
    "        ai_reliance.append(reliance_ai/len(user_answers_ai))\n",
    "\n",
    "print(len(accuracies_hum_no_ai))\n",
    "accuracies_hum_no_ai_1 = accuracies_hum_no_ai.copy()\n",
    "accuracies_hum_ai_1 = accuracies_hum_ai.copy()\n",
    "ai_reliance_1 = ai_reliance.copy()\n",
    "accuracies_ai_no_ai_1 = accuracies_ai_no_ai.copy()\n",
    "accuracies_ai_all.append(accuracies_ai_no_ai_1)\n",
    "\n",
    "# print(times_hum_no_ai)\n",
    "print(len(accuracies_hum_ai_1))\n",
    "print(f'Human only accuracy: {np.mean(accuracies_hum_no_ai_1)}')\n",
    "print(f'Human + AI accuracy: {np.mean(accuracies_hum_ai_1)}')\n",
    "print(f' AI (no ai) accuracy: {np.mean(accuracies_ai_no_ai_1)}')\n",
    "print(f' AI (with AI) accuracy: {np.mean(accuracies_ai_with_ai)}')\n",
    "\n",
    "print(f'AI reliance: {np.mean(ai_reliance_1)}')\n",
    "print(f'Failed attention checks: {np.mean(failed_attention_check)}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10a5d05e-4c0c-4b4a-95be-999c7495309c",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# initialize table\n",
    "\n",
    "num_col = 6\n",
    "empty_row = [None]*num_col\n",
    "\n",
    "mean_row = [\"Accuracy (mean, std dev)\"]+empty_row\n",
    "p_row = [\"Test with Human+AI (p-value, t-value)\"]+empty_row\n",
    "ai_reliance_row = [\"AI reliance\"]+empty_row\n",
    "time_row = [\"Time per example\"]+empty_row\n",
    "ai_acc_row = [\"Accuracy when using AI\"]+empty_row\n",
    "no_ai_acc_row = [\"Accuracy when not using AI\"]+empty_row\n",
    "\n",
    "table = [[\"Metric\",\"Human\",\"Human+AI\",\"Human+AI+Teach+Rec\",\"Human+AI+Teach\", \"Human+AI+Rec\",\"AI only\"],\n",
    "        mean_row,\n",
    "        p_row,\n",
    "        ai_reliance_row,\n",
    "        time_row,\n",
    "        ai_acc_row,\n",
    "        no_ai_acc_row\n",
    "        ]\n",
    "\n",
    "headers = table.pop(0)\n",
    "\n",
    "print(tabulate(table, headers=headers))\n",
    "# print(tabulate(table, headers=headers, tablefmt=\"latex\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "03735d2c-f6c1-4c83-8d37-382532add096",
   "metadata": {},
   "outputs": [],
   "source": [
    "# populate table for human and human_ai columns\n",
    "mean_std_add_to_table(accuracies_hum_no_ai_1,1)\n",
    "mean_std_add_to_table(accuracies_hum_ai_1,2)\n",
    "\n",
    "ai_reliance_row[1] = \"N/A\"\n",
    "ai_rel_add_to_table(ai_reliance_1, 2)\n",
    "\n",
    "time_add_to_table(times_hum_no_ai,1)\n",
    "time_add_to_table(times_hum_with_ai,2)\n",
    "\n",
    "# human and human_ai pairwise t test\n",
    "\n",
    "t_stat1, p_val1 = ttest_ind(accuracies_hum_no_ai_1, accuracies_hum_ai_1)\n",
    "r_t_stat1 = round(t_stat1,3)\n",
    "r_p_val1 = round(p_val1,3)\n",
    "p_row[1] = (r_p_val1,r_t_stat1)\n",
    "p_row[2] = (r_p_val1,r_t_stat1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e445722-ec1e-45ad-b9ff-d7c88393b9c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(tabulate(table, headers=headers))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3875f6ce-18ff-4854-8d31-b36f03ecb160",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# Human + AI + Teach (ours)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dad3ee77-0344-49e7-874f-670e46308877",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "responses = data['human+ai+teach(ours)'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d79f266a-8524-4929-8131-327946b4a4d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "accuracies_hum_no_ai = []\n",
    "accuracies_hum_ai = []\n",
    "accuracies_ai_with_ai = []\n",
    "accuracies_ai_no_ai = []\n",
    "failed_attention_check = []\n",
    "ai_reliance_3 = []\n",
    "ai_reliance_4 = []\n",
    "\n",
    "# time\n",
    "times_hum_no_ai = []\n",
    "times_hum_with_ai = []\n",
    "\n",
    "for i in range(len(responses)):\n",
    "    # find the matching task\n",
    "    id_task = responses[i]['id'].split('-')[0]\n",
    "    task = tasks[id_task]\n",
    "    user_answers_no_ai_raw = responses[i]['testing_answers_withairec']\n",
    "    user_answers_ai_raw = responses[i]['testing_answers_withai']\n",
    "    # task\n",
    "    labels_no_ai = task['testing_withairec_labels']\n",
    "    labels_ai = task['testing_withai_label']\n",
    "    ai_no_ai = task['testing_withairec_ai_answers_raw']\n",
    "    ai_ai = task['testing_withai_ai_answers_raw']\n",
    "    att_check_no_ai = task['testing_attentioncheck']\n",
    "    att_check_ai = task['testing_withai_attentioncheck']\n",
    "    user_answers_no_ai = []\n",
    "    user_answers_ai = []\n",
    "    reliance_ai_3 = 0\n",
    "    reliance_ai_4 = 0\n",
    "\n",
    "    # time\n",
    "    response = responses[i]\n",
    "    clean_times_noai = get_each_time_taken(response, 'testing_times_withairec') # [example1 time, example2 time, ...]\n",
    "    clean_times_withai = get_each_time_taken(response, 'testing_times_withai')\n",
    "    times_hum_no_ai.append(sum(clean_times_noai)/len(clean_times_noai))  # [avg time per example by person1, by person2, ...]\n",
    "    times_hum_with_ai.append(sum(clean_times_withai)/len(clean_times_withai))\n",
    "\n",
    "\n",
    "            \n",
    "    for j in range(len(user_answers_ai_raw)):\n",
    "        if user_answers_ai_raw[j] == 'ai':\n",
    "            reliance_ai_4 += 1\n",
    "\n",
    "            user_answers_ai.append(ai_ai[j])\n",
    "            #reliance_ai += 1\n",
    "        else:\n",
    "            user_answers_ai.append(user_answers_ai_raw[j])\n",
    "\n",
    "    for j in range(len(user_answers_no_ai_raw)):\n",
    "        if user_answers_no_ai_raw[j] == 'ai':\n",
    "            user_answers_no_ai.append(ai_no_ai[j])\n",
    "            reliance_ai += 1\n",
    "            reliance_ai_3 += 1\n",
    "\n",
    "        else:\n",
    "            user_answers_no_ai.append(user_answers_no_ai_raw[j])\n",
    "\n",
    "\n",
    "\n",
    "            \n",
    "            \n",
    "    # check if the attention check is correct\n",
    "    failed_checks = 0\n",
    "\n",
    "    for j in range(len(user_answers_no_ai)-1):\n",
    "        if att_check_no_ai[j] == 1:\n",
    "            if user_answers_no_ai[j] != labels_no_ai[j]:\n",
    "                failed_checks += 1\n",
    "    if failed_checks >= np.sum(att_check_no_ai):\n",
    "        continue\n",
    "    failed_checks = 0\n",
    "    for j in range(len(user_answers_ai)):\n",
    "        if att_check_ai[j] == 1:\n",
    "            if user_answers_ai[j] != labels_ai[j]:\n",
    "                failed_checks += 1\n",
    "    if failed_checks >= np.sum(att_check_ai):\n",
    "        continue\n",
    "    failed_attention_check.append(failed_checks)\n",
    "\n",
    "\n",
    "\n",
    "    try:\n",
    "        accuracies_hum_no_ai.append(accuracy_score(labels_no_ai, user_answers_no_ai))\n",
    "        accuracies_ai_no_ai.append(accuracy_score(labels_no_ai, ai_no_ai))\n",
    "        ai_reliance_3.append(reliance_ai_3/len(user_answers_no_ai))\n",
    "\n",
    "    except:\n",
    "        accuracies_hum_no_ai.append(accuracy_score(labels_no_ai[:14], user_answers_no_ai[:14]))\n",
    "        accuracies_hum_ai.append(accuracy_score(labels_ai[:14], user_answers_ai[:14]))\n",
    "\n",
    "    try:\n",
    "\n",
    "        accuracies_hum_ai.append(accuracy_score(labels_ai, user_answers_ai))\n",
    "        accuracies_ai_with_ai.append(accuracy_score(labels_ai, ai_ai))\n",
    "        ai_reliance.append(reliance_ai/len(user_answers_ai))\n",
    "        ai_reliance_4.append(reliance_ai_4/len(user_answers_ai))\n",
    "\n",
    "    except:\n",
    "        accuracies_ai_with_ai.append(accuracy_score(labels_ai[:14], ai_ai[:14]))\n",
    "        accuracies_ai_no_ai.append(accuracy_score(labels_no_ai[:14], ai_no_ai[:14]))\n",
    "        ai_reliance.append(reliance_ai/len(user_answers_ai))    \n",
    "\n",
    "   \n",
    "\n",
    "accuracies_hum_no_ai_teach_rec = accuracies_hum_no_ai.copy()\n",
    "accuracies_hum_no_ai_teach = accuracies_hum_ai.copy()\n",
    "accuracies_ai_no_ai_2 = accuracies_ai_no_ai.copy()\n",
    "accuracies_ai_with_ai_2 = accuracies_ai_with_ai.copy()\n",
    "accuracies_ai_all.append(accuracies_ai_with_ai_2)\n",
    "print(len(accuracies_hum_no_ai))\n",
    "print(f'Human + AI + Teach + rec accuracy: {np.mean(accuracies_hum_no_ai_teach_rec)}')\n",
    "print(f'Human + AI + Teach accuracy: {np.mean(accuracies_hum_no_ai_teach)}')\n",
    "print(f' AI (with rec) accuracy: {np.mean(accuracies_ai_no_ai_2)}')\n",
    "print(f' AI (no rec) accuracy: {np.mean(accuracies_ai_with_ai_2)}')\n",
    "\n",
    "print(f'AI reliance: {np.mean(ai_reliance_3)}')\n",
    "print(f'AI reliance: {np.mean(ai_reliance_4)}')\n",
    "print(f'Failed attention checks: {np.mean(failed_attention_check)}')\n",
    "# show the distribution of accuracies\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1dcffbd1-5caf-49aa-8309-ff7ee239e775",
   "metadata": {},
   "outputs": [],
   "source": [
    "# populate table for 3: human+ai+teach+rec t-test, 4: human+ai+teach\n",
    "mean_std_add_to_table(accuracies_hum_no_ai_teach_rec,3)\n",
    "p_t_add_to_table(accuracies_hum_no_ai_teach_rec,3)\n",
    "\n",
    "mean_std_add_to_table(accuracies_hum_no_ai_teach,4)\n",
    "p_t_add_to_table(accuracies_hum_no_ai_teach,4)\n",
    "\n",
    "ai_rel_add_to_table(ai_reliance_3, 3)\n",
    "ai_rel_add_to_table(ai_reliance_4, 4)\n",
    "\n",
    "time_add_to_table(times_hum_no_ai,3)\n",
    "time_add_to_table(times_hum_with_ai,4)\n",
    "\n",
    "print(tabulate(table, headers=headers))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a24634f-0482-4e1a-807e-83a1529f4ced",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# Human + AI + REC "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a9b9f61-c03a-4085-82d2-e4a7ce5e4809",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "responses = data['human+ai rec'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f88b87da-5e7a-4c6b-ab4e-6f636b2bc5a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "accuracies_hum_no_ai = []\n",
    "accuracies_hum_ai = []\n",
    "accuracies_ai_with_ai = []\n",
    "accuracies_ai_no_ai = []\n",
    "failed_attention_check = []\n",
    "ai_reliance = []\n",
    "# time\n",
    "times_hum_ai = []\n",
    "\n",
    "for i in range(len(responses)):\n",
    "    # find the matching task\n",
    "    id_task = responses[i]['id'].split('-')[0]\n",
    "    task = tasks[id_task]\n",
    "    user_answers_ai_raw = responses[i]['testing_answers_withairec']\n",
    "    # task\n",
    "    labels_no_ai = task['testing_withairec_labels']\n",
    "    labels_ai = task['testing_withai_label']\n",
    "    ai_no_ai = task['testing_withairec_ai_answers_raw']\n",
    "    ai_ai = task['testing_withai_ai_answers_raw']\n",
    "    att_check_no_ai = task['testing_attentioncheck']\n",
    "    att_check_ai = task['testing_withai_attentioncheck']\n",
    "    user_answers_no_ai = []\n",
    "    user_answers_ai = []\n",
    "    reliance_ai = 0\n",
    "\n",
    "    # time\n",
    "    response = responses[i]\n",
    "    clean_times_withai = get_each_time_taken(response, 'testing_times_withairec')\n",
    "    times_hum_ai.append(sum(clean_times_withai)/len(clean_times_withai))\n",
    "\n",
    "    for j in range(len(user_answers_ai_raw)):\n",
    "        if user_answers_ai_raw[j] == 'ai':\n",
    "            user_answers_ai.append(ai_no_ai[j])\n",
    "            reliance_ai += 1\n",
    "        else:\n",
    "            user_answers_ai.append(user_answers_ai_raw[j])            \n",
    "            \n",
    "    # check if the attention check is correct\n",
    "    failed_checks = 0\n",
    "\n",
    "    for j in range(len(att_check_ai)-1):\n",
    "        if att_check_ai[j] == 1:\n",
    "            if user_answers_ai[j] != labels_no_ai[j]:\n",
    "                failed_checks += 1\n",
    "    if failed_checks >= np.sum(att_check_ai):\n",
    "        continue\n",
    "    failed_attention_check.append(failed_checks)\n",
    "    try:\n",
    "        accuracies_hum_ai.append(accuracy_score(labels_no_ai, user_answers_ai))\n",
    "        accuracies_ai_with_ai.append(accuracy_score(labels_no_ai, ai_no_ai))\n",
    "        if accuracy_score(labels_ai, user_answers_ai) == 0:\n",
    "            print(user_answers_ai)\n",
    "            print(labels_ai)\n",
    "        ai_reliance.append(reliance_ai/len(user_answers_ai))\n",
    "    except:\n",
    "        print(\"some error\")\n",
    "        continue\n",
    "\n",
    "accuracies_hum_ai_rec = accuracies_hum_ai.copy()\n",
    "accuracies_ai_with_ai4 = accuracies_ai_with_ai.copy()\n",
    "accuracies_ai_all.append(accuracies_ai_with_ai4)\n",
    "print(len(accuracies_hum_ai_rec))\n",
    "print(f'Human + AI + REC Basline: {np.mean(accuracies_hum_ai_rec)}') # just rec (no teaching)\n",
    "print(f' AI (no rec) accuracy: {np.mean(accuracies_ai_with_ai4)}')\n",
    "\n",
    "print(f'AI reliance: {np.mean(ai_reliance)}')\n",
    "print(f'Failed attention checks: {np.mean(failed_attention_check)}')\n",
    "# show the distribution of accuracies\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ac4698f-634a-47c9-9e23-2ca3131fedfe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 6: human+ai+teach baseline\n",
    "mean_std_add_to_table(accuracies_hum_ai_rec,5)\n",
    "p_t_add_to_table(accuracies_hum_ai_rec,5)\n",
    "ai_rel_add_to_table(ai_reliance, 5)\n",
    "time_add_to_table(times_hum_ai,5)\n",
    "print(tabulate(table, headers=headers))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89c17179-8bf6-4804-ac15-362afecb30d2",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# SUMMARY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84741a1d-2772-4b00-ad08-ed77e15e3d9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracies_ai_all_flat = [item for nested_list in accuracies_ai_all for item in nested_list]\n",
    "data['ai only'] = accuracies_ai_all_flat\n",
    "# 7: AI only\n",
    "mean_std_add_to_table(accuracies_ai_all_flat, 6)\n",
    "p_t_add_to_table(accuracies_ai_all_flat, 6)\n",
    "print(tabulate(table, headers=headers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b60bb000-2e2c-4f38-b9de-3ba6df2d0f7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(tabulate(table, headers=headers))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0caa4f1d-49c8-4821-99ac-ff7d855d842e",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# List of array names for reference\n",
    "array_names = [\n",
    "    \"accuracies_hum_ai_rec\",\n",
    "    \"accuracies_ai_all_flat\",\n",
    "    \"accuracies_hum_no_ai_teach_rec\",\n",
    "    \"accuracies_hum_no_ai_teach\",\n",
    "    \"accuracies_hum_no_ai_1\",\n",
    "    \"accuracies_hum_ai_1\"]\n",
    "names_array = [\n",
    "    \"Rec\",\n",
    "    \"AI only\",\n",
    "    \"Onboard(ours)+Rec\",\n",
    "    \"Onboard(ours)\",\n",
    "    \"Human\",\n",
    "    \"Human-AI\"]\n",
    "\n",
    "\n",
    "\n",
    "# Create a list to hold the results\n",
    "results = []\n",
    "\n",
    "j = -1\n",
    "# Perform pairwise t-tests\n",
    "for i in range(len(array_names)-1):\n",
    "    p_value = stats.ttest_ind(eval(array_names[i]), eval(array_names[j])).pvalue\n",
    "    results.append([names_array[i], names_array[j], p_value])\n",
    "\n",
    "\n",
    "tests_to_add = []\n",
    "\n",
    "for test in tests_to_add:\n",
    "    i = test[0]\n",
    "    j = test[1]\n",
    "    p_value = stats.ttest_ind(eval(array_names[i]), eval(array_names[j])).pvalue\n",
    "    results.append([names_array[i], names_array[j], p_value])\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# Create a pandas DataFrame to organize the results\n",
    "results_df = pd.DataFrame(results, columns=[\"Array 1\", \"Array 2\", \"p-value\"])\n",
    "p_values = results_df['p-value'].values\n",
    "\n",
    "# Perform FDR correction using Benjamini-Hochberg method\n",
    "rejected, adjusted_p_values = multitest.fdrcorrection(p_values, alpha=0.05, method='indep')\n",
    "\n",
    "# Add adjusted p-values to the DataFrame\n",
    "results_df['adjusted_p-value'] = adjusted_p_values\n",
    "# Display the results\n",
    "print(results_df)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:teach_vision] *",
   "language": "python",
   "name": "conda-env-teach_vision-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}