notebooks/evaluation_user_study.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4f72d14fe64e74a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:33:44.934193136Z",
     "start_time": "2023-10-20T00:33:42.412045392Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from math import nan, isnan\n",
    "\n",
    "from pandas import read_excel, DataFrame, to_datetime, concat, Categorical, crosstab, Series\n",
    "from scipy.stats import pearsonr, chi2_contingency\n",
    "from scipy.stats.contingency import Chi2ContingencyResult\n",
    "from seaborn import set_theme, displot, move_legend"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:33:59.624028506Z",
     "start_time": "2023-10-20T00:33:42.451618180Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df: DataFrame = read_excel(\"../data/06-study.xlsx\", sheet_name=\"07 Survey\", header=0)\n",
    "df.rename(columns={\n",
    "    \"V1\": \"survey_id\",\n",
    "    \"Zeitstempel\": \"datetime\",\n",
    "    \"FrageNr\": \"qid\",\n",
    "    \"Fachlichpersönlich\": \"background\",\n",
    "    \"ObjektA\": \"object1\",\n",
    "    \"ObjektB\": \"object2\",\n",
    "    \"HastdudazuVorkenntnisse\": \"prior_knowledge\",\n",
    "    \"WiewürdestdudichohneweitergehendeRechercheentscheiden\": \"decision_before_search_raw\",\n",
    "    \"Wiesicherbistdudirdamit\": \"confidence_before_search\",\n",
    "    \"HastdudiesesDokumentgelesen\": \"has_read_document1\",\n",
    "    \"HastdudiesesDokumentgelesen_A\": \"has_read_document2\",\n",
    "    \"HastdudiesesDokumentgelesen_B\": \"has_read_document3\",\n",
    "    \"HastdudiesesDokumentgelesen_C\": \"has_read_document4\",\n",
    "    \"WieentscheidestdudichnachdemdudieseDokumentegelesenhast\": \"decision_after_search_raw\",\n",
    "    \"Wiesicherbistdudirjetztdamit\": \"confidence_after_search\",\n",
    "    \"WelchesDokumentehathabendeineEntscheidungbeeinflusst\": \"influencing_documents\",\n",
    "    \"WelchedieserAussagentrifftzu\": \"statements\",\n",
    "    \"Relevanz_Dok1\": \"relevance_document1\",\n",
    "    \"Relevanz_Dok2\": \"relevance_document2\",\n",
    "    \"Relevanz_Dok3\": \"relevance_document3\",\n",
    "    \"Relevanz_Dok4\": \"relevance_document4\",\n",
    "    \"Haltung_Dok1\": \"stance_document1\",\n",
    "    \"Haltung_Dok2\": \"stance_document2\",\n",
    "    \"Haltung_Dok3\": \"stance_document3\",\n",
    "    \"Haltung_Dok4\": \"stance_document4\",\n",
    "    \"Qualität_Dok1\": \"quality_score_document1\",\n",
    "    \"Qualität_Dok2\": \"quality_score_document2\",\n",
    "    \"Qualität_Dok3\": \"quality_score_document3\",\n",
    "    \"Qualität_Dok4\": \"quality_score_document4\",\n",
    "    \"Qualität_Gesamt\": \"quality_score\",\n",
    "    \"Entscheidung1A\": \"decision_before_search_object1\",\n",
    "    \"Entscheidung1B\": \"decision_before_search_object2\",\n",
    "    \"Entscheidung2A\": \"decision_after_search_object1\",\n",
    "    \"Entscheidung2B\": \"decision_after_search_object2\",\n",
    "    \"Beeinflussung1\": \"influence_document1\",\n",
    "    \"Beeinflussung2\": \"influence_document2\",\n",
    "    \"Beeinflussung3\": \"influence_document3\",\n",
    "    \"Beeinflussung4\": \"influence_document4\",\n",
    "    \"AussageDieRecherchehatmichinmeinerMeinungbestätigt\": \"statement_research_confirmed\",\n",
    "    \"AussageIchkannmichmithilfedieserSuchergebnissebesserentscheiden\": \"statement_research_helped\",\n",
    "    \"AussageDurchdieRecherchehabeichetwasNeueszudemThemagelernt\": \"statement_research_learned\",\n",
    "    \"AussageDieSuchergebnissehabenmirnichtgeholfen\": \"statement_research_not_helped\",\n",
    "    \"AussageIchwürdeandieserStellenochweiterrecherchieren\": \"statement_research_continue\",\n",
    "    \"EntscheidungVorher\": \"stance_before_search\",\n",
    "    \"EntscheidungVorherA\": \"stance_before_search_object1\",\n",
    "    \"EntscheidungVorherB\": \"stance_before_search_object2\",\n",
    "    \"EntscheidungNachher\": \"stance_after_search\",\n",
    "    \"EntscheidungNachherA\": \"stance_after_search_object1\",\n",
    "    \"EntscheidungNachherB\": \"stance_after_search_object2\",\n",
    "    \"EntscheidungVorherX\": \"stance_before_search_none\",\n",
    "    \"EntscheidungNachherX\": \"stance_after_search_none\",\n",
    "}, inplace=True)\n",
    "df[\"background\"] = df[\"background\"].map({\n",
    "    \"fachlich\": \"factual\",\n",
    "    \"persönlich\": \"subjective\",\n",
    "}).astype(\"category\")\n",
    "for doc in [1, 2, 3, 4]:\n",
    "    df[f\"has_partially_read_document{doc}\"] = df[f\"has_read_document{doc}\"].map({\n",
    "        \"Ja\": True,\n",
    "        \"Überflogen\": True,\n",
    "        \"Nein\": False,\n",
    "    })\n",
    "    df[f\"has_fully_read_document{doc}\"] = df[f\"has_read_document{doc}\"].map({\n",
    "        \"Ja\": True,\n",
    "        \"Überflogen\": False,\n",
    "        \"Nein\": False,\n",
    "    })\n",
    "    df[f\"stance_direction_document{doc}\"] = df[f\"stance_document{doc}\"].map({\n",
    "        1: \"object1\",\n",
    "        2: \"object1\",\n",
    "        3: \"neutral\",\n",
    "        4: \"object2\",\n",
    "        5: \"object2\",\n",
    "        6: nan,\n",
    "        nan: nan,\n",
    "    }, na_action=\"ignore\")\n",
    "    df[f\"stance_strength_document{doc}\"] = df[f\"stance_document{doc}\"].map({\n",
    "        1: 2,\n",
    "        2: 1,\n",
    "        3: 0,\n",
    "        4: 1,\n",
    "        5: 2,\n",
    "        6: nan,\n",
    "        nan: nan,\n",
    "    }, na_action=\"ignore\")\n",
    "df[\"prior_knowledge\"] = df[\"prior_knowledge\"].map({\n",
    "    \"Ja\": True,\n",
    "    \"Nein\": False,\n",
    "})\n",
    "df[\"datetime\"] = to_datetime(df[\"datetime\"], format=\"%d.%m.%Y %H:%M:%S\", utc=True)\n",
    "df.drop(columns=[\n",
    "    \"datetime\",\n",
    "    \"object1\",\n",
    "    \"object2\",\n",
    "    \"decision_before_search_raw\",\n",
    "    \"decision_after_search_raw\",\n",
    "    \"influencing_documents\",\n",
    "    \"statements\",\n",
    "    \"stance_before_search\",\n",
    "    \"stance_before_search_object1\",\n",
    "    \"stance_before_search_object2\",\n",
    "    \"stance_before_search_none\",\n",
    "    \"stance_after_search\",\n",
    "    \"stance_after_search_object1\",\n",
    "    \"stance_after_search_object2\",\n",
    "    \"stance_after_search_none\",\n",
    "    \"has_read_document1\",\n",
    "    \"has_read_document2\",\n",
    "    \"has_read_document3\",\n",
    "    \"has_read_document4\",\n",
    "], inplace=True)\n",
    "for col in [\n",
    "    \"decision_before_search_object1\",\n",
    "    \"decision_before_search_object2\",\n",
    "    \"decision_after_search_object1\",\n",
    "    \"decision_after_search_object2\",\n",
    "    \"influence_document1\",\n",
    "    \"influence_document2\",\n",
    "    \"influence_document3\",\n",
    "    \"influence_document4\",\n",
    "    \"statement_research_confirmed\",\n",
    "    \"statement_research_helped\",\n",
    "    \"statement_research_learned\",\n",
    "    \"statement_research_not_helped\",\n",
    "    \"statement_research_continue\",\n",
    "    \"relevance_document1\",\n",
    "    \"relevance_document2\",\n",
    "    \"relevance_document3\",\n",
    "    \"relevance_document4\",\n",
    "]:\n",
    "    df[col] = df[col].astype(bool)\n",
    "df[\"decision_before_search_none\"] = ~df[\"decision_before_search_object1\"] & ~df[\"decision_before_search_object2\"]\n",
    "df[\"decision_after_search_none\"] = ~df[\"decision_after_search_object1\"] & ~df[\"decision_after_search_object2\"]\n",
    "df[\"confidence_delta\"] = df[\"confidence_after_search\"] - df[\"confidence_before_search\"]\n",
    "df[\"decision_unchanged\"] = (\n",
    "        df[\"decision_before_search_object1\"].eq(df[\"decision_after_search_object1\"]) &\n",
    "        df[\"decision_before_search_object2\"].eq(df[\"decision_after_search_object2\"]) &\n",
    "        df[\"decision_before_search_none\"].eq(df[\"decision_after_search_none\"])\n",
    ")\n",
    "df[\"decision_changed\"] = ~df[\"decision_unchanged\"]\n",
    "df[\"num_influencing_documents\"] = df[[\n",
    "    \"influence_document1\",\n",
    "    \"influence_document2\",\n",
    "    \"influence_document3\",\n",
    "    \"influence_document4\",\n",
    "]].sum(axis=\"columns\")\n",
    "df[\"relevance\"] = df[[\n",
    "    \"relevance_document1\",\n",
    "    \"relevance_document2\",\n",
    "    \"relevance_document3\",\n",
    "    \"relevance_document4\",\n",
    "]].mean(axis=\"columns\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6547f4bcbd6fae8e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:33:59.647745216Z",
     "start_time": "2023-10-20T00:33:45.266900002Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_doc = concat([\n",
    "    DataFrame([\n",
    "        {\n",
    "            \"survey_id\": row[\"survey_id\"],\n",
    "            \"qid\": row[\"qid\"],\n",
    "            \"background\": row[\"background\"],\n",
    "            \"rank\": rank,\n",
    "            \"decision_before_search_object1\": row[\"decision_before_search_object1\"],\n",
    "            \"decision_before_search_object2\": row[\"decision_before_search_object2\"],\n",
    "            \"decision_before_search_none\": row[\"decision_before_search_none\"],\n",
    "            \"confidence_before_search\": row[\"confidence_before_search\"],\n",
    "            \"decision_after_search_object1\": row[\"decision_after_search_object1\"],\n",
    "            \"decision_after_search_object2\": row[\"decision_after_search_object2\"],\n",
    "            \"decision_after_search_none\": row[\"decision_after_search_none\"],\n",
    "            \"confidence_after_search\": row[\"confidence_after_search\"],\n",
    "            \"decision_unchanged\": row[\"decision_unchanged\"],\n",
    "            \"decision_changed\": row[\"decision_changed\"],\n",
    "            \"confidence_delta\": row[\"confidence_delta\"],\n",
    "            \"has_fully_read_document\": row[f\"has_fully_read_document{rank}\"],\n",
    "            \"has_partially_read_document\": row[f\"has_partially_read_document{rank}\"],\n",
    "            \"influence_document\": row[f\"influence_document{rank}\"],\n",
    "            \"prior_knowledge\": row[\"prior_knowledge\"],\n",
    "            \"quality_score\": row[\"quality_score\"],\n",
    "            \"quality_score_document\": row[f\"quality_score_document{rank}\"],\n",
    "            \"relevance_document\": row[f\"relevance_document{rank}\"],\n",
    "            \"stance_document\": row[f\"stance_document{rank}\"],\n",
    "            \"stance_direction_document\": row[f\"stance_direction_document{rank}\"],\n",
    "            \"stance_strength_document\": row[f\"stance_strength_document{rank}\"],\n",
    "            \"statement_research_confirmed\": row[\"statement_research_confirmed\"],\n",
    "            \"statement_research_continue\": row[\"statement_research_continue\"],\n",
    "            \"statement_research_helped\": row[\"statement_research_helped\"],\n",
    "            \"statement_research_learned\": row[\"statement_research_learned\"],\n",
    "            \"statement_research_not_helped\": row[\"statement_research_not_helped\"],\n",
    "        }\n",
    "        for rank in [1, 2, 3, 4]\n",
    "    ])\n",
    "    for _, row in df.iterrows()\n",
    "])\n",
    "df_doc"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aea50df6ced4ca7",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "## Descriptive statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c740f857aeb4bad5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:33:59.692427502Z",
     "start_time": "2023-10-20T00:33:49.883528879Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_doc[\"background\"].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd48ee5cfd868895",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:33:59.698663922Z",
     "start_time": "2023-10-20T00:33:49.926836959Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_doc[[\"has_fully_read_document\", \"has_partially_read_document\"]].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e79ccc7505bb8966",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:33:59.701489993Z",
     "start_time": "2023-10-20T00:33:49.971548847Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_doc[\n",
    "    [\"decision_before_search_none\", \"decision_before_search_object1\", \"decision_before_search_object2\"]].value_counts(\n",
    "    dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64d9abef7dca827b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:33:59.709031933Z",
     "start_time": "2023-10-20T00:33:50.014895234Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_doc[[\"decision_after_search_none\", \"decision_after_search_object1\", \"decision_after_search_object2\"]].value_counts(\n",
    "    dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4f73f6f3a5c80dc",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:33:59.712575288Z",
     "start_time": "2023-10-20T00:33:50.033894092Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_doc[[\"decision_before_search_none\", \"decision_after_search_none\"]].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ad5d131dfcb082b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:33:59.716080267Z",
     "start_time": "2023-10-20T00:33:50.079905011Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_doc[\"decision_changed\"].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fe1c26df90e65155",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "## Histograms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6686bc66f65b775c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:33:59.719065253Z",
     "start_time": "2023-10-20T00:33:50.166503321Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "set_theme(\n",
    "    style=\"ticks\",\n",
    "    palette=\"colorblind\",\n",
    "    font_scale=1.0,\n",
    "    font=\"sans-serif\",\n",
    ")\n",
    "df_plot = df.copy()\n",
    "df_plot = df_plot[df_plot[\"confidence_delta\"].notna()]\n",
    "df_plot[\"confidence_changed\"] = df_plot[\"confidence_delta\"].map(\n",
    "    lambda x: \"no change\" if x == 0 else \"more confident\" if x > 0 else \"less confident\")\n",
    "df_plot[\"confidence_changed\"] = Categorical(\n",
    "    df_plot[\"confidence_changed\"],\n",
    "    categories=[\"less confident\", \"no change\", \"more confident\"],\n",
    "    ordered=True,\n",
    ")\n",
    "df_plot[\"background\"] = Categorical(\n",
    "    df_plot[\"background\"],\n",
    "    categories=[\"factual\", \"subjective\"],\n",
    "    ordered=True,\n",
    ")\n",
    "df_plot.sort_values([\"confidence_changed\", \"background\"], inplace=True)\n",
    "plot = displot(\n",
    "    df_plot,\n",
    "    x=\"background\",\n",
    "    hue=\"confidence_changed\",\n",
    "    multiple=\"dodge\",\n",
    "    aspect=1.7,\n",
    "    height=2.5,\n",
    "    legend=False,\n",
    "    shrink=0.9,\n",
    ")\n",
    "plot.set_xlabels(label=\"Background\")\n",
    "plot.savefig(\"../data/figures/histogram-confidence-change-background.pdf\")\n",
    "plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d77c8e8eca746b81",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:33:59.722540118Z",
     "start_time": "2023-10-20T00:33:51.759780407Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "set_theme(\n",
    "    style=\"ticks\",\n",
    "    palette=\"colorblind\",\n",
    "    font_scale=1.0,\n",
    "    font=\"sans-serif\",\n",
    ")\n",
    "df_plot = df.copy()\n",
    "df_plot = df_plot[df_plot[\"confidence_delta\"].notna()]\n",
    "df_plot[\"confidence_changed\"] = df_plot[\"confidence_delta\"].map(\n",
    "    lambda x: \"no change\" if x == 0 else \"more confident\" if x > 0 else \"less confident\")\n",
    "df_plot[\"confidence_changed\"] = Categorical(\n",
    "    df_plot[\"confidence_changed\"],\n",
    "    categories=[\"less confident\", \"no change\", \"more confident\"],\n",
    "    ordered=True,\n",
    ")\n",
    "df_plot[\"decision_changed\"] = df_plot[\"decision_changed\"].map(\n",
    "    lambda x: \"change\" if x else \"no change\")\n",
    "df_plot[\"decision_changed\"] = Categorical(\n",
    "    df_plot[\"decision_changed\"],\n",
    "    categories=[\"no change\", \"change\"],\n",
    "    ordered=True,\n",
    ")\n",
    "df_plot.sort_values([\"decision_changed\", \"confidence_changed\"], inplace=True)\n",
    "plot = displot(\n",
    "    df_plot,\n",
    "    x=\"decision_changed\",\n",
    "    hue=\"confidence_changed\",\n",
    "    multiple=\"dodge\",\n",
    "    aspect=1.7,\n",
    "    height=2.5,\n",
    "    shrink=0.9,\n",
    ")\n",
    "move_legend(\n",
    "    plot,\n",
    "    title=\"Confidence\",\n",
    "    loc=\"upper right\",\n",
    "    bbox_to_anchor=(0.69, 1),\n",
    ")\n",
    "plot.set_xlabels(label=\"Decision\")\n",
    "plot.savefig(\"../data/figures/histogram-confidence-change-decision-change.pdf\")\n",
    "plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98ed88a28aff97a8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:33:59.726107676Z",
     "start_time": "2023-10-20T00:33:53.731746488Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "set_theme(\n",
    "    style=\"ticks\",\n",
    "    palette=\"colorblind\",\n",
    "    font_scale=1.0,\n",
    "    font=\"sans-serif\",\n",
    ")\n",
    "df_plot = df_doc.copy()\n",
    "plot = displot(\n",
    "    df_plot,\n",
    "    x=\"quality_score_document\",\n",
    "    hue=\"influence_document\",\n",
    "    multiple=\"fill\",\n",
    ")\n",
    "plot.set_xlabels(\"Quality\")\n",
    "plot.set_ylabels(\"Proportion\")\n",
    "plot.legend.set_title(\"Influence\")\n",
    "plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dbb402a2b7de0d01",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:04.142700720Z",
     "start_time": "2023-10-20T00:33:58.774947908Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "set_theme(\n",
    "    style=\"ticks\",\n",
    "    palette=\"colorblind\",\n",
    "    font_scale=1.0,\n",
    "    font=\"sans-serif\",\n",
    ")\n",
    "df_plot = df_doc.copy()\n",
    "df_plot = df_plot[df_plot[\"stance_document\"].notna()]\n",
    "plot = displot(\n",
    "    df_plot,\n",
    "    x=\"stance_document\",\n",
    "    hue=\"influence_document\",\n",
    "    multiple=\"fill\",\n",
    ")\n",
    "plot.set_xlabels(\"Stance strength\")\n",
    "plot.set_ylabels(\"Proportion\")\n",
    "plot.legend.set_title(\"Influence\")\n",
    "plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb778cf29291b215",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:09.588285367Z",
     "start_time": "2023-10-20T00:34:01.803573688Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "set_theme(\n",
    "    style=\"ticks\",\n",
    "    palette=\"colorblind\",\n",
    "    font_scale=1.0,\n",
    "    font=\"sans-serif\",\n",
    ")\n",
    "df_plot = df_doc.copy()\n",
    "# df_plot[\"relevance_document\"] = df_plot[\"relevance_document\"].map(\n",
    "#     lambda x: \"relevant\" if x else \"irrelevant\")\n",
    "plot = displot(\n",
    "    df_plot,\n",
    "    x=\"relevance_document\",\n",
    "    hue=\"influence_document\",\n",
    "    multiple=\"fill\",\n",
    ")\n",
    "plot.set_xlabels(\"Relevance\")\n",
    "plot.set_ylabels(\"Proportion\")\n",
    "plot.legend.set_title(\"Influence\")\n",
    "plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c6557fb0fd9c6dd",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:12.500113351Z",
     "start_time": "2023-10-20T00:34:09.377167518Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "set_theme(\n",
    "    style=\"ticks\",\n",
    "    palette=\"colorblind\",\n",
    "    font_scale=1.0,\n",
    "    font=\"sans-serif\",\n",
    ")\n",
    "df_plot = df.copy()\n",
    "df_plot.sort_values(\"num_influencing_documents\", inplace=True)\n",
    "df_plot[\"num_influencing_documents\"] = df_plot[\"num_influencing_documents\"].astype(str)\n",
    "plot = displot(\n",
    "    df_plot,\n",
    "    x=\"num_influencing_documents\",\n",
    "    hue=\"background\",\n",
    "    multiple=\"stack\",\n",
    "    aspect=2,\n",
    "    height=2.5,\n",
    "    shrink=0.7,\n",
    ")\n",
    "plot.set_xlabels(\"# Influencing documents\")\n",
    "move_legend(\n",
    "    plot,\n",
    "    title=\"Background\",\n",
    "    loc=\"upper right\",\n",
    "    bbox_to_anchor=(0.77, 0.9),\n",
    ")\n",
    "plot.savefig(\"../data/figures/histogram-num-influencing-documents.pdf\")\n",
    "plot"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0b6bdbf28ec887f",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "## Significance tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51d4359c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def chi2_to_tex(chi2_result: Chi2ContingencyResult, alpha: float, p_value_digits: int = 3) -> str:\n",
    "    statistic, p_value, dof, _ = chi2_result\n",
    "    is_significant = p_value < alpha\n",
    "    statistic_tex = r\"\\chi^2(\" + f\"{dof}\" + r\")\" + f\"={statistic:.2f}\"\n",
    "    min_p_value = pow(10, -p_value_digits)\n",
    "    p_value_tex = format(max(p_value, min_p_value), f\".{p_value_digits}f\")\n",
    "    significance_tex = r\"p\" + (\"=\" if p_value >= min_p_value else \"<\") + p_value_tex\n",
    "    if is_significant:\n",
    "        statistic_tex = r\"\\bm{\" + statistic_tex + r\"}\"\n",
    "        significance_tex = r\"\\bm{\" + significance_tex + r\"}\"\n",
    "    tex = r\"\\(\" + statistic_tex + r\"\\), \\(\" + significance_tex + r\"\\)\"\n",
    "    if is_significant:\n",
    "        tex = r\"\\textbfn{\" + tex + r\"}\"\n",
    "    return tex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c37af30a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def diff_expected_freq_tex(row: Series, col: str, index: str, i: int, chi2_result: Chi2ContingencyResult) -> str:\n",
    "    if index == r\"\\(\\sum\\)\":\n",
    "        return \"\"\n",
    "    col_index = row.index.get_loc(col)\n",
    "    diff_expected_freq = (row[col] - chi2_result.expected_freq[i,col_index]).round()\n",
    "    diff_expected_freq_sign = r\"\\(+\\)\" if diff_expected_freq > 0 else r\"--\" if diff_expected_freq < 0 else r\"\\(\\pm\\)\"\n",
    "    diff_expected_freq_tex = f\"{diff_expected_freq_sign}{abs(diff_expected_freq):.0f}\"\n",
    "    diff_expected_freq_tex = r\"\\secondary{(\" + diff_expected_freq_tex + r\")}\"\n",
    "    return diff_expected_freq_tex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "598a9e631623cf3e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:12.683101121Z",
     "start_time": "2023-10-20T00:34:12.367569301Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Copy-pasted from the quality assessments evaluation notebook (median).\n",
    "quality_threshold = 0.56969696969697\n",
    "label_low_quality = r\"low quality~\\secondary{(\\(<\" + f\"{quality_threshold:.2f}\" + r\"\\))}\"\n",
    "label_high_quality = r\"high quality~\\secondary{(\\(\\geq\" + f\"{quality_threshold:.2f}\" + r\"\\))}\"\n",
    "quality_threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8a680c01dd7f59c7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:12.684763097Z",
     "start_time": "2023-10-20T00:34:12.419005581Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "initial_confidence_threshold = df[\"confidence_before_search\"].median()\n",
    "label_low_initial_confidence = r\"low confidence~\\secondary{(\\(<\" + f\"{initial_confidence_threshold:.0f}\" + r\"\\))}\"\n",
    "label_high_initial_confidence = r\"high confidence~\\secondary{(\\(\\geq\" + f\"{initial_confidence_threshold:.0f}\" + r\"\\))}\"\n",
    "initial_confidence_threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8acd92faddded1f8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:12.790778817Z",
     "start_time": "2023-10-20T00:34:12.462913867Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "final_confidence_threshold = df[\"confidence_after_search\"].median()\n",
    "label_low_final_confidence = r\"low confidence~\\secondary{(\\(<\" + f\"{final_confidence_threshold:.0f}\" + r\"\\))}\"\n",
    "label_high_final_confidence = r\"high confidence~\\secondary{(\\(\\geq\" + f\"{final_confidence_threshold:.0f}\" + r\"\\))}\"\n",
    "final_confidence_threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc029c749a145951",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:14.098576928Z",
     "start_time": "2023-10-20T00:34:12.507098626Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_test = df_doc.copy()\n",
    "df_test = df_test[df_test[\"has_partially_read_document\"]]\n",
    "df_test[\"influence_document\"] = df_test[\"influence_document\"].map(\n",
    "    lambda x: \"influence\" if x else \"no influence\")\n",
    "df_test[\"influence_document\"] = Categorical(\n",
    "    df_test[\"influence_document\"],\n",
    "    categories=[\"no influence\", \"influence\"],\n",
    "    ordered=True,\n",
    ")\n",
    "df_test[\"quality_document\"] = df_test[\"quality_score_document\"].map(\n",
    "    lambda x: label_high_quality if x >= quality_threshold else label_low_quality)\n",
    "df_test[\"quality_document\"] = Categorical(\n",
    "    df_test[\"quality_document\"],\n",
    "    categories=[label_low_quality, label_high_quality],\n",
    "    ordered=True,\n",
    ")\n",
    "df_test[\"relevance_document\"] = df_test[\"relevance_document\"].map(\n",
    "    lambda x: \"relevant\" if x else \"not relevant\")\n",
    "df_test[\"relevance_document\"] = Categorical(\n",
    "    df_test[\"relevance_document\"],\n",
    "    categories=[\"not relevant\", \"relevant\"],\n",
    "    ordered=True,\n",
    ")\n",
    "df_test[\"relevance_quality_document\"] = [\n",
    "    row[\"relevance_document\"] + \", \" + row[\"quality_document\"]\n",
    "    for _, row in df_test.iterrows()\n",
    "]\n",
    "df_test[\"relevance_quality_document\"] = Categorical(\n",
    "    df_test[\"relevance_quality_document\"],\n",
    "    categories=[f\"not relevant, {label_low_quality}\", f\"not relevant, {label_high_quality}\", f\"relevant, {label_low_quality}\", f\"relevant, {label_high_quality}\"],\n",
    "    ordered=True,\n",
    ")\n",
    "df_test[\"stance_strength_document\"] = df_test[\"stance_strength_document\"].map({\n",
    "    0: \"no stance\",\n",
    "    1: \"weak stance\",\n",
    "    2: \"strong stance\",\n",
    "})\n",
    "df_test[\"stance_strength_document\"] = Categorical(\n",
    "    df_test[\"stance_strength_document\"],\n",
    "    categories=[\"no stance\", \"weak stance\", \"strong stance\"],\n",
    "    ordered=True,\n",
    ")\n",
    "df_test[\"initial_confidence\"] = df_test[\"confidence_before_search\"].map(\n",
    "    lambda x: label_high_initial_confidence if x >= initial_confidence_threshold else label_low_initial_confidence)\n",
    "df_test[\"initial_confidence\"] = Categorical(\n",
    "    df_test[\"initial_confidence\"],\n",
    "    categories=[label_low_initial_confidence, label_high_initial_confidence],\n",
    "    ordered=True,\n",
    ")\n",
    "df_test[\"final_confidence\"] = df_test[\"confidence_after_search\"].map(\n",
    "    lambda x: label_high_final_confidence if x >= final_confidence_threshold else label_low_final_confidence)\n",
    "df_test[\"final_confidence\"] = Categorical(\n",
    "    df_test[\"final_confidence\"],\n",
    "    categories=[label_low_final_confidence, label_high_final_confidence],\n",
    "    ordered=True,\n",
    ")\n",
    "df_test[\"initial_confidence_low_quality\"] = df_test[\"initial_confidence\"].where(\n",
    "    df_test[\"quality_document\"] == label_low_quality)\n",
    "df_test[\"rank\"] = \"rank~\" + df_test[\"rank\"].astype(str)\n",
    "df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7086e52aae86f01e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:14.254056045Z",
     "start_time": "2023-10-20T00:34:12.748352081Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "contingency_quality = crosstab(\n",
    "    index=df_test[\"quality_document\"],\n",
    "    columns=df_test[\"influence_document\"],\n",
    ")\n",
    "contingency_full_quality = crosstab(\n",
    "    index=df_test[\"quality_document\"],\n",
    "    columns=df_test[\"influence_document\"],\n",
    "    margins=True,\n",
    ")\n",
    "contingency_full_quality"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e54cdfac40ec0b32",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:14.255604324Z",
     "start_time": "2023-10-20T00:34:13.017228871Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chi2_quality = chi2_contingency(contingency_full_quality)\n",
    "chi2_quality"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c49658f01a9ca29a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:14.276373292Z",
     "start_time": "2023-10-20T00:34:13.158934084Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "contingency_relevance = crosstab(\n",
    "    index=df_test[\"relevance_document\"],\n",
    "    columns=df_test[\"influence_document\"],\n",
    ")\n",
    "contingency_full_relevance = crosstab(\n",
    "    index=df_test[\"relevance_document\"],\n",
    "    columns=df_test[\"influence_document\"],\n",
    "    margins=True,\n",
    ")\n",
    "contingency_full_relevance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73cbecd5007a0c27",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:14.279198959Z",
     "start_time": "2023-10-20T00:34:13.461888403Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chi2_relevance = chi2_contingency(contingency_relevance)\n",
    "chi2_relevance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3619484521d5aef1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:14.300293434Z",
     "start_time": "2023-10-20T00:34:13.670688296Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "contingency_quality_quality_relevance = crosstab(\n",
    "    index=df_test[\"relevance_quality_document\"],\n",
    "    columns=df_test[\"influence_document\"],\n",
    ")\n",
    "contingency_full_quality_relevance = crosstab(\n",
    "    index=df_test[\"relevance_quality_document\"],\n",
    "    columns=df_test[\"influence_document\"],\n",
    "    margins=True,\n",
    ")\n",
    "contingency_full_quality_relevance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31ec985de692108b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:14.303157466Z",
     "start_time": "2023-10-20T00:34:13.980288839Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chi2_quality_relevance = chi2_contingency(contingency_full_quality_relevance)\n",
    "chi2_quality_relevance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f9d6a6260f74b116",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:14.896404184Z",
     "start_time": "2023-10-20T00:34:14.015505328Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "contingency_stance_strength = crosstab(\n",
    "    index=df_test[\"stance_strength_document\"],\n",
    "    columns=df_test[\"influence_document\"],\n",
    ")\n",
    "contingency_full_stance_strength = crosstab(\n",
    "    index=df_test[\"stance_strength_document\"],\n",
    "    columns=df_test[\"influence_document\"],\n",
    "    margins=True,\n",
    ")\n",
    "contingency_full_stance_strength"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3cf1e599247e0bd5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:14.986197015Z",
     "start_time": "2023-10-20T00:34:14.191172969Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chi2_stance_strength = chi2_contingency(contingency_stance_strength)\n",
    "chi2_stance_strength"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d253165ece96953f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:16.898164776Z",
     "start_time": "2023-10-20T00:34:14.502309670Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "contingency_initial_confidence_low_quality = crosstab(\n",
    "    index=df_test[\"initial_confidence_low_quality\"],\n",
    "    columns=df_test[\"influence_document\"],\n",
    ")\n",
    "contingency_full_initial_confidence_low_quality = crosstab(\n",
    "    index=df_test[\"initial_confidence_low_quality\"],\n",
    "    columns=df_test[\"influence_document\"],\n",
    "    margins=True,\n",
    ")\n",
    "contingency_full_initial_confidence_low_quality"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "56b78a4facd48f77",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:16.900920226Z",
     "start_time": "2023-10-20T00:34:14.736762302Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chi2_initial_confidence_low_quality = chi2_contingency(contingency_initial_confidence_low_quality)\n",
    "chi2_initial_confidence_low_quality"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d0e231700f5a22f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:17.440734884Z",
     "start_time": "2023-10-20T00:34:14.778963228Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "contingency_rank = crosstab(\n",
    "    index=df_test[\"rank\"],\n",
    "    columns=df_test[\"influence_document\"],\n",
    ")\n",
    "contingency_full_rank = crosstab(\n",
    "    index=df_test[\"rank\"],\n",
    "    columns=df_test[\"influence_document\"],\n",
    "    margins=True,\n",
    ")\n",
    "contingency_full_rank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a4ba07862579b42",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:17.947328115Z",
     "start_time": "2023-10-20T00:34:14.989168739Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chi2_rank = chi2_contingency(contingency_rank)\n",
    "chi2_rank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e01a611d59189e75",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:17.951998802Z",
     "start_time": "2023-10-20T00:34:15.030609293Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tests = [\n",
    "    (r\"Quality\", contingency_full_quality, chi2_quality),\n",
    "    (r\"Relevance\", contingency_full_relevance, chi2_relevance),\n",
    "    (r\"Quality\\,\\(\\times\\)\\,relevance\", contingency_full_quality_relevance, chi2_quality_relevance),\n",
    "    (r\"Stance strength\", contingency_full_stance_strength, chi2_stance_strength),\n",
    "    (r\"Initial confidence (for low-quality docs.)\", contingency_full_initial_confidence_low_quality, chi2_initial_confidence_low_quality),\n",
    "    (r\"Ranking position\", contingency_full_rank, chi2_rank),\n",
    "]\n",
    "alpha = 0.05\n",
    "correction = 1\n",
    "alpha_corrected = alpha / correction  # Bonferroni correction across outcomes\n",
    "# TODO: Indicate change compared to expected frequencies.\n",
    "print(r\"\\begin{tabular}{@{}l@{\\hspace{\\largetabcolsep}}r@{ }lr@{ }lr@{}}\")\n",
    "print(r\"  \\toprule\")\n",
    "print(r\"  \\textbf{Predictor} & \\multicolumn{5}{@{}c@{}}{\\textbf{Document influence}} \\\\\")\n",
    "print(r\"  & \\multicolumn{2}{@{}l}{No influence} & \\multicolumn{2}{l}{Influence} & \\(\\sum\\) \\\\\")\n",
    "for variable_name, contingency, chi2_result in tests:\n",
    "    print(r\"  \\midrule\")\n",
    "    print(r\"  \\textit{\" + variable_name + r\"} & \\multicolumn{5}{@{}c@{}}{\" +\n",
    "          chi2_to_tex(chi2_result, alpha_corrected) + r\"} \\\\\")\n",
    "    print(r\"  \\midrule\")\n",
    "    for i, (index, row) in enumerate(contingency.iterrows()):\n",
    "        if isinstance(index, tuple):\n",
    "            index = \", \".join(index)\n",
    "        elif \"All\" in index:\n",
    "            index = r\"\\(\\sum\\)\"\n",
    "        columns = [\n",
    "            index,\n",
    "            f\"{row['no influence']}\",\n",
    "            diff_expected_freq_tex(row, \"no influence\", index, i, chi2_result),\n",
    "            f\"{row['influence']}\",\n",
    "            diff_expected_freq_tex(row, \"influence\", index, i, chi2_result),\n",
    "            f\"{row['All']}\",\n",
    "        ]\n",
    "        print(r\"  \" + r\" & \".join(columns) + r\" \\\\\")\n",
    "print(r\"  \\bottomrule\")\n",
    "print(r\"\\end{tabular}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce9efd13827264a9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:17.961320694Z",
     "start_time": "2023-10-20T00:34:15.075133401Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_test2 = df.copy()\n",
    "df_test2[\"decision_changed\"] = df_test2[\"decision_changed\"].map(\n",
    "    lambda x: \"changed decision\" if x else \"same decision\")\n",
    "df_test2[\"decision_changed\"] = Categorical(\n",
    "    df_test2[\"decision_changed\"],\n",
    "    categories=[\"same decision\", \"changed decision\"],\n",
    "    ordered=True,\n",
    ")\n",
    "df_test2[\"final_confidence\"] = df_test2[\"confidence_after_search\"].map(\n",
    "    lambda x: label_high_final_confidence if x >= final_confidence_threshold else label_low_final_confidence)\n",
    "df_test2[\"final_confidence\"] = Categorical(\n",
    "    df_test2[\"final_confidence\"],\n",
    "    categories=[label_low_final_confidence, label_high_final_confidence],\n",
    "    ordered=True,\n",
    ")\n",
    "df_test2[\"confidence_changed\"] = df_test2[\"confidence_delta\"].map(\n",
    "    lambda x: nan if isnan(x) else \"less confident\" if x < 0 else \"more confident\" if x > 0 else \"same confidence\")\n",
    "df_test2[\"confidence_changed\"] = Categorical(\n",
    "    df_test2[\"confidence_changed\"],\n",
    "    categories=[\"less confident\", \"same confidence\", \"more confident\"],\n",
    "    ordered=True,\n",
    ")\n",
    "df_test2[\"quality\"] = df_test2[\"quality_score\"].map(\n",
    "    lambda x: label_high_quality if x >= quality_threshold else label_low_quality)\n",
    "df_test2[\"quality\"] = Categorical(\n",
    "    df_test2[\"quality\"],\n",
    "    categories=[label_low_quality, label_high_quality],\n",
    "    ordered=True,\n",
    ")\n",
    "df_test2[\"background\"] = Categorical(\n",
    "    df_test2[\"background\"],\n",
    "    categories=[\"factual\", \"subjective\"],\n",
    "    ordered=True,\n",
    ")\n",
    "for col in (\n",
    "        \"statement_research_confirmed\",\n",
    "        \"statement_research_helped\",\n",
    "        \"statement_research_not_helped\",\n",
    "        \"statement_research_learned\",\n",
    "        \"statement_research_continue\",\n",
    "            ):\n",
    "    df_test2[col]=    df_test2[col].map({True: \"yes\", False: \"no\"})\n",
    "    df_test2[col] = Categorical(\n",
    "        df_test2[col],\n",
    "        categories=[\"no\", \"yes\"],\n",
    "        ordered=True,\n",
    "    )\n",
    "df_test2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "157bb7208aa9a410",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:17.963698196Z",
     "start_time": "2023-10-20T00:34:15.274659709Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "contingency_background_decision_change = crosstab(\n",
    "    index=df_test2[\"background\"],\n",
    "    columns=df_test2[\"decision_changed\"],\n",
    ")\n",
    "contingency_full_background_decision_change = crosstab(\n",
    "    index=df_test2[\"background\"],\n",
    "    columns=df_test2[\"decision_changed\"],\n",
    "    margins=True,\n",
    ")\n",
    "contingency_full_background_decision_change"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6975e33c538a8b3c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:17.965938064Z",
     "start_time": "2023-10-20T00:34:15.450090616Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chi2_background_decision_change = chi2_contingency(contingency_background_decision_change)\n",
    "chi2_background_decision_change"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b118f4a697b25d34",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:18.036149509Z",
     "start_time": "2023-10-20T00:34:15.472418873Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "contingency_quality_decision_change = crosstab(\n",
    "    index=df_test2[\"quality\"],\n",
    "    columns=df_test2[\"decision_changed\"],\n",
    ")\n",
    "contingency_full_quality_decision_change = crosstab(\n",
    "    index=df_test2[\"quality\"],\n",
    "    columns=df_test2[\"decision_changed\"],\n",
    "    margins=True,\n",
    ")\n",
    "contingency_full_quality_decision_change"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7684bb57a6637a2f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:18.062482308Z",
     "start_time": "2023-10-20T00:34:15.722740146Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chi2_quality_decision_change = chi2_contingency(contingency_quality_decision_change)\n",
    "chi2_quality_decision_change"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aca06ffc497e37a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:18.421173251Z",
     "start_time": "2023-10-20T00:34:15.854252690Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "contingency_background_final_confidence = crosstab(\n",
    "    index=df_test2[\"background\"],\n",
    "    columns=df_test2[\"final_confidence\"],\n",
    ")\n",
    "contingency_full_background_final_confidence = crosstab(\n",
    "    index=df_test2[\"background\"],\n",
    "    columns=df_test2[\"final_confidence\"],\n",
    "    margins=True,\n",
    ")\n",
    "contingency_full_background_final_confidence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0e78c4d16d2c900",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:18.422332965Z",
     "start_time": "2023-10-20T00:34:16.183663109Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chi2_background_final_confidence = chi2_contingency(contingency_background_final_confidence)\n",
    "chi2_background_final_confidence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "342ef717615c624f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:19.259793975Z",
     "start_time": "2023-10-20T00:34:16.238538356Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "contingency_quality_final_confidence = crosstab(\n",
    "    index=df_test2[\"quality\"],\n",
    "    columns=df_test2[\"final_confidence\"],\n",
    ")\n",
    "contingency_full_quality_final_confidence = crosstab(\n",
    "    index=df_test2[\"quality\"],\n",
    "    columns=df_test2[\"final_confidence\"],\n",
    "    margins=True,\n",
    ")\n",
    "contingency_full_quality_final_confidence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8a160198bebbaee",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:19.501254760Z",
     "start_time": "2023-10-20T00:34:16.420265462Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chi2_quality_final_confidence = chi2_contingency(contingency_quality_final_confidence)\n",
    "chi2_quality_final_confidence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52ce420b4ad1afda",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:19.766347116Z",
     "start_time": "2023-10-20T00:34:16.518446838Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "contingency_background_confidence_change = crosstab(\n",
    "    index=df_test2[\"background\"],\n",
    "    columns=df_test2[\"confidence_changed\"],\n",
    ")\n",
    "contingency_full_background_confidence_change = crosstab(\n",
    "    index=df_test2[\"background\"],\n",
    "    columns=df_test2[\"confidence_changed\"],\n",
    "    margins=True,\n",
    ")\n",
    "contingency_full_background_confidence_change"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d66f4bfa1ba892b7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:20.754172337Z",
     "start_time": "2023-10-20T00:34:16.716230978Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chi2_background_confidence_change = chi2_contingency(contingency_background_confidence_change)\n",
    "chi2_background_confidence_change"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39056d139780ac18",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:21.611437696Z",
     "start_time": "2023-10-20T00:34:16.762568475Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "contingency_quality_confidence_change = crosstab(\n",
    "    index=df_test2[\"quality\"],\n",
    "    columns=df_test2[\"confidence_changed\"],\n",
    ")\n",
    "contingency_full_quality_confidence_change = crosstab(\n",
    "    index=df_test2[\"quality\"],\n",
    "    columns=df_test2[\"confidence_changed\"],\n",
    "    margins=True,\n",
    ")\n",
    "contingency_full_quality_confidence_change"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5a8461da8f3d9f1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:21.616415265Z",
     "start_time": "2023-10-20T00:34:16.935850856Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chi2_quality_confidence_change = chi2_contingency(contingency_quality_confidence_change)\n",
    "chi2_quality_confidence_change"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3f67c9b703f9f0e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:21.862519187Z",
     "start_time": "2023-10-20T00:34:16.978853565Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "contingency_decision_confidence_change = crosstab(\n",
    "    index=df_test2[\"decision_changed\"],\n",
    "    columns=df_test2[\"confidence_changed\"],\n",
    ")\n",
    "contingency_full_decision_confidence_change = crosstab(\n",
    "    index=df_test2[\"decision_changed\"],\n",
    "    columns=df_test2[\"confidence_changed\"],\n",
    "    margins=True,\n",
    ")\n",
    "contingency_full_decision_confidence_change"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f55e895b0e3fd850",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:22.378021700Z",
     "start_time": "2023-10-20T00:34:17.194907975Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chi2_decision_confidence_change = chi2_contingency(contingency_decision_confidence_change)\n",
    "chi2_decision_confidence_change"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7251c507989f173b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:22.629647055Z",
     "start_time": "2023-10-20T00:34:17.243062152Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tests = [\n",
    "    (r\"Background\", contingency_full_background_decision_change, chi2_background_decision_change, contingency_full_background_final_confidence, chi2_background_final_confidence, contingency_full_background_confidence_change, chi2_background_confidence_change),\n",
    "    (r\"Quality\", contingency_full_quality_decision_change, chi2_quality_decision_change, contingency_full_quality_final_confidence, chi2_quality_final_confidence, contingency_full_quality_confidence_change, chi2_quality_confidence_change),\n",
    "]\n",
    "alpha = 0.05\n",
    "correction = 3\n",
    "alpha_corrected = alpha / correction  # Bonferroni correction across outcomes\n",
    "# TODO: Indicate change compared to expected frequencies.\n",
    "# print(r\"\\newlength{\\largetabcolsep}\")\n",
    "# print(r\"\\setlength{\\largetabcolsep}{3\\tabcolsep}\")\n",
    "print(r\"\\begin{tabular}{@{}l@{\\hspace{\\largetabcolsep}}r@{ }lr@{ }lr@{\\hspace{\\largetabcolsep}}r@{ }lr@{ }lr@{\\hspace{\\largetabcolsep}}r@{ }lr@{ }lr@{ }lr@{}}\")\n",
    "print(r\"  \\toprule\")\n",
    "print(r\"  \\textbf{Predictor} & \\multicolumn{5}{@{}c@{\\hspace{\\largetabcolsep}}}{\\textbf{Decision change}} & \\multicolumn{5}{@{}c@{\\hspace{\\largetabcolsep}}}{\\textbf{Final decision confidence}} & \\multicolumn{7}{@{}c@{}}{\\textbf{Decision confidence change}} \\\\\")\n",
    "print(r\"  & \\multicolumn{2}{@{}l}{Unchanged} & \\multicolumn{2}{l}{Changed} & \\(\\sum\\) & \\multicolumn{2}{@{}l}{Low~\\secondary{(\\(<\"+f\"{final_confidence_threshold:.2f}\"+r\"\\))}} & \\multicolumn{2}{l}{High~\\secondary{(\\(<\"+f\"{final_confidence_threshold:.2f}\"+r\"\\))}} & \\(\\sum\\) & \\multicolumn{2}{@{}l}{Decreased} & \\multicolumn{2}{l}{Unchanged} & \\multicolumn{2}{l}{Increased} & \\(\\sum\\) \\\\\")\n",
    "for variable_name, contingency_decision_change, chi2_result_decision_change, contingency_final_confidence, chi2_result_final_confidence, contingency_confidence_change, chi2_result_confidence_change in tests:\n",
    "    print(r\"  \\midrule\")\n",
    "    print(r\"  \\textit{\" + variable_name + r\"} & \\multicolumn{5}{@{}c@{\\hspace{\\largetabcolsep}}}{\" + chi2_to_tex(chi2_result_decision_change, alpha_corrected) + r\"} & \\multicolumn{5}{@{}c@{\\hspace{\\largetabcolsep}}}{\" + chi2_to_tex(chi2_result_final_confidence, alpha_corrected) + r\"} & \\multicolumn{7}{@{}c@{}}{\" + chi2_to_tex(chi2_result_confidence_change, alpha_corrected) + r\"} \\\\\")\n",
    "    print(r\"  \\midrule\")\n",
    "    for i, ((index_decision_change, row_decision_change), (index_final_confidence, row_final_confidence), (index_confidence_change, row_confidence_change)) in enumerate(zip(contingency_decision_change.iterrows(), contingency_final_confidence.iterrows(), contingency_confidence_change.iterrows())):\n",
    "        assert index_decision_change == index_final_confidence == index_confidence_change\n",
    "        index = index_confidence_change\n",
    "        if \"All\" in index:\n",
    "            index = r\"\\(\\sum\\)\"\n",
    "        columns = [\n",
    "            index,\n",
    "            f\"{row_decision_change['same decision']}\",\n",
    "            diff_expected_freq_tex(row_decision_change, \"same decision\", index, i, chi2_result_decision_change),\n",
    "            f\"{row_decision_change['changed decision']}\",\n",
    "            diff_expected_freq_tex(row_decision_change, \"changed decision\", index, i, chi2_result_decision_change),\n",
    "            f\"{row_decision_change['All']}\",\n",
    "            f\"{row_final_confidence[label_low_final_confidence]}\",\n",
    "            diff_expected_freq_tex(row_final_confidence, label_low_final_confidence, index, i, chi2_result_final_confidence),\n",
    "            f\"{row_final_confidence[label_high_final_confidence]}\",\n",
    "            diff_expected_freq_tex(row_final_confidence, label_high_final_confidence, index, i, chi2_result_final_confidence),\n",
    "            f\"{row_final_confidence['All']}\",\n",
    "            f\"{row_confidence_change['less confident']}\",\n",
    "            diff_expected_freq_tex(row_confidence_change, \"less confident\", index, i, chi2_result_confidence_change),\n",
    "            f\"{row_confidence_change['same confidence']}\",\n",
    "            diff_expected_freq_tex(row_confidence_change, \"same confidence\", index, i, chi2_result_confidence_change),\n",
    "            f\"{row_confidence_change['more confident']}\",\n",
    "            diff_expected_freq_tex(row_confidence_change, \"more confident\", index, i, chi2_result_confidence_change),\n",
    "            f\"{row_confidence_change['All']}\",\n",
    "        ]\n",
    "        print(r\"  \" + r\" & \".join(columns) + r\" \\\\\")\n",
    "print(r\"  \\bottomrule\")\n",
    "print(r\"\\end{tabular}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7611220d7684288",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:24.806609235Z",
     "start_time": "2023-10-20T00:34:17.416809445Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "contingency_statement_research_confirmed = crosstab(\n",
    "    index=df_test2[\"quality\"],\n",
    "    columns=df_test2[\"statement_research_confirmed\"],\n",
    ")\n",
    "contingency_full_statement_research_confirmed = crosstab(\n",
    "    index=df_test2[\"quality\"],\n",
    "    columns=df_test2[\"statement_research_confirmed\"],\n",
    "    margins=True,\n",
    ")\n",
    "contingency_full_statement_research_confirmed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c55d2c9e107d0061",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:25.191929230Z",
     "start_time": "2023-10-20T00:34:17.630816788Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chi2_statement_research_confirmed = chi2_contingency(contingency_statement_research_confirmed)\n",
    "chi2_statement_research_confirmed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9dd3d54c76db7ca",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:25.263527660Z",
     "start_time": "2023-10-20T00:34:17.820864298Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "contingency_statement_research_helped = crosstab(\n",
    "    index=df_test2[\"quality\"],\n",
    "    columns=df_test2[\"statement_research_helped\"],\n",
    ")\n",
    "contingency_full_statement_research_helped = crosstab(\n",
    "    index=df_test2[\"quality\"],\n",
    "    columns=df_test2[\"statement_research_helped\"],\n",
    "    margins=True,\n",
    ")\n",
    "contingency_full_statement_research_helped"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8bf7c25f27a4792a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:25.657408262Z",
     "start_time": "2023-10-20T00:34:17.933134820Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chi2_statement_research_helped = chi2_contingency(contingency_statement_research_helped)\n",
    "chi2_statement_research_helped"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "225d0014b9caa1cb",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:27.036285027Z",
     "start_time": "2023-10-20T00:34:17.953319980Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "contingency_statement_research_not_helped = crosstab(\n",
    "    index=df_test2[\"quality\"],\n",
    "    columns=df_test2[\"statement_research_not_helped\"],\n",
    ")\n",
    "contingency_full_statement_research_not_helped = crosstab(\n",
    "    index=df_test2[\"quality\"],\n",
    "    columns=df_test2[\"statement_research_not_helped\"],\n",
    "    margins=True,\n",
    ")\n",
    "contingency_full_statement_research_not_helped"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88a0cefae3aaaa17",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:27.120535650Z",
     "start_time": "2023-10-20T00:34:18.186491583Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chi2_statement_research_not_helped = chi2_contingency(contingency_statement_research_not_helped)\n",
    "chi2_statement_research_not_helped"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15e41b4bd24a64b2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:27.148204792Z",
     "start_time": "2023-10-20T00:34:18.242898235Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "contingency_statement_research_learned = crosstab(\n",
    "    index=df_test2[\"quality\"],\n",
    "    columns=df_test2[\"statement_research_learned\"],\n",
    ")\n",
    "contingency_full_statement_research_learned = crosstab(\n",
    "    index=df_test2[\"quality\"],\n",
    "    columns=df_test2[\"statement_research_learned\"],\n",
    "    margins=True,\n",
    ")\n",
    "contingency_full_statement_research_learned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48b11cfe7866eb8c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:27.150156454Z",
     "start_time": "2023-10-20T00:34:18.366668906Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chi2_statement_research_learned = chi2_contingency(contingency_statement_research_learned)\n",
    "chi2_statement_research_learned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76dc35704e29c52",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:27.277358275Z",
     "start_time": "2023-10-20T00:34:18.414948798Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "contingency_statement_research_continue = crosstab(\n",
    "    index=df_test2[\"quality\"],\n",
    "    columns=df_test2[\"statement_research_continue\"],\n",
    ")\n",
    "contingency_full_statement_research_continue = crosstab(\n",
    "    index=df_test2[\"quality\"],\n",
    "    columns=df_test2[\"statement_research_continue\"],\n",
    "    margins=True,\n",
    ")\n",
    "contingency_full_statement_research_continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7122c9b96ec24aef",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:27.356054588Z",
     "start_time": "2023-10-20T00:34:18.650419463Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chi2_statement_research_continue = chi2_contingency(contingency_statement_research_continue)\n",
    "chi2_statement_research_continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97940cbd929350c3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:27.468649110Z",
     "start_time": "2023-10-20T00:34:18.774762432Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "contingency_statement_research_confirmed_background = crosstab(\n",
    "    index=df_test2[\"background\"],\n",
    "    columns=df_test2[\"statement_research_confirmed\"],\n",
    ")\n",
    "contingency_full_statement_research_confirmed_background = crosstab(\n",
    "    index=df_test2[\"background\"],\n",
    "    columns=df_test2[\"statement_research_confirmed\"],\n",
    "    margins=True,\n",
    ")\n",
    "contingency_full_statement_research_confirmed_background"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41cee742e3906f04",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:27.470988948Z",
     "start_time": "2023-10-20T00:34:19.020738116Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chi2_statement_research_confirmed_background = chi2_contingency(contingency_statement_research_confirmed_background)\n",
    "chi2_statement_research_confirmed_background"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb31df613f7441cf",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:27.602509807Z",
     "start_time": "2023-10-20T00:34:19.069789357Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "contingency_statement_research_helped_background = crosstab(\n",
    "    index=df_test2[\"background\"],\n",
    "    columns=df_test2[\"statement_research_helped\"],\n",
    ")\n",
    "contingency_full_statement_research_helped_background = crosstab(\n",
    "    index=df_test2[\"background\"],\n",
    "    columns=df_test2[\"statement_research_helped\"],\n",
    "    margins=True,\n",
    ")\n",
    "contingency_full_statement_research_helped_background"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fef02918158c7caa",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:27.604217837Z",
     "start_time": "2023-10-20T00:34:19.366912481Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chi2_statement_research_helped_background = chi2_contingency(contingency_statement_research_helped_background)\n",
    "chi2_statement_research_helped_background"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5a0f49a570efc93",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:27.804170090Z",
     "start_time": "2023-10-20T00:34:19.410996555Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "contingency_statement_research_not_helped_background = crosstab(\n",
    "    index=df_test2[\"background\"],\n",
    "    columns=df_test2[\"statement_research_not_helped\"],\n",
    ")\n",
    "contingency_full_statement_research_not_helped_background = crosstab(\n",
    "    index=df_test2[\"background\"],\n",
    "    columns=df_test2[\"statement_research_not_helped\"],\n",
    "    margins=True,\n",
    ")\n",
    "contingency_full_statement_research_not_helped_background"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7bcc2d23fe67dd3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:27.809100367Z",
     "start_time": "2023-10-20T00:34:19.667321692Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chi2_statement_research_not_helped_background = chi2_contingency(contingency_statement_research_not_helped_background)\n",
    "chi2_statement_research_not_helped_background"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c2678426313387d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:28.010868279Z",
     "start_time": "2023-10-20T00:34:19.758624551Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "contingency_statement_research_learned_background = crosstab(\n",
    "    index=df_test2[\"background\"],\n",
    "    columns=df_test2[\"statement_research_learned\"],\n",
    ")\n",
    "contingency_full_statement_research_learned_background = crosstab(\n",
    "    index=df_test2[\"background\"],\n",
    "    columns=df_test2[\"statement_research_learned\"],\n",
    "    margins=True,\n",
    ")\n",
    "contingency_full_statement_research_learned_background"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "72f8210b1888fcf6",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:28.014608510Z",
     "start_time": "2023-10-20T00:34:20.027128290Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chi2_statement_research_learned_background = chi2_contingency(contingency_statement_research_learned_background)\n",
    "chi2_statement_research_learned_background"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3af7643e3e3231df",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:28.019095498Z",
     "start_time": "2023-10-20T00:34:20.077736499Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "contingency_statement_research_continue_background = crosstab(\n",
    "    index=df_test2[\"background\"],\n",
    "    columns=df_test2[\"statement_research_continue\"],\n",
    ")\n",
    "contingency_full_statement_research_continue_background = crosstab(\n",
    "    index=df_test2[\"background\"],\n",
    "    columns=df_test2[\"statement_research_continue\"],\n",
    "    margins=True,\n",
    ")\n",
    "contingency_full_statement_research_continue_background"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9cffd8a6fc7650aa",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:28.022588603Z",
     "start_time": "2023-10-20T00:34:20.350940629Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chi2_statement_research_continue_background = chi2_contingency(contingency_statement_research_continue_background)\n",
    "chi2_statement_research_continue_background"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0a465368f4cbace",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-20T00:34:27.386806066Z",
     "start_time": "2023-10-20T00:34:18.688662076Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tests = [\n",
    "    (r\"Background\", contingency_full_statement_research_confirmed_background, chi2_statement_research_confirmed_background, contingency_full_statement_research_helped_background, chi2_statement_research_helped_background, contingency_full_statement_research_not_helped_background, chi2_statement_research_not_helped_background, contingency_full_statement_research_learned_background, chi2_statement_research_learned_background, contingency_full_statement_research_continue_background, chi2_statement_research_continue_background),\n",
    "    (r\"Quality\", contingency_full_statement_research_confirmed, chi2_statement_research_confirmed, contingency_full_statement_research_helped, chi2_statement_research_helped, contingency_full_statement_research_not_helped, chi2_statement_research_not_helped, contingency_full_statement_research_learned, chi2_statement_research_learned, contingency_full_statement_research_continue, chi2_statement_research_continue),\n",
    "]\n",
    "alpha = 0.05\n",
    "correction = 5\n",
    "alpha_corrected = alpha / correction  # Bonferroni correction across outcomes\n",
    "# TODO: Indicate change compared to expected frequencies.\n",
    "# print(r\"\\newlength{\\largetabcolsep}\")\n",
    "# print(r\"\\setlength{\\largetabcolsep}{3\\tabcolsep}\")\n",
    "print(r\"\\begin{tabular}{@{}l@{\\hspace{\\largetabcolsep}}r@{ }lr@{ }lr@{\\hspace{\\largetabcolsep}}r@{ }lr@{ }lr@{\\hspace{\\largetabcolsep}}r@{ }lr@{ }lr@{\\hspace{\\largetabcolsep}}r@{ }lr@{ }lr@{\\hspace{\\largetabcolsep}}r@{ }lr@{ }lr@{}}\")\n",
    "print(r\"  \\toprule\")\n",
    "print(r\"  \\textbf{Predictor} & \\multicolumn{5}{@{}c@{\\hspace{\\largetabcolsep}}}{\\textbf{Confirm.\\ prev.\\ opinion}} & \\multicolumn{5}{@{}c@{\\hspace{\\largetabcolsep}}}{\\textbf{Could make better decis.}} & \\multicolumn{5}{@{}c@{\\hspace{\\largetabcolsep}}}{\\textbf{Results did not help}} & \\multicolumn{5}{@{}c@{\\hspace{\\largetabcolsep}}}{\\textbf{Learned something new}} & \\multicolumn{5}{@{}c@{}}{\\textbf{Would do further search}} \\\\\")\n",
    "print(r\"  & \\multicolumn{2}{@{}l}{No} & \\multicolumn{2}{l}{Yes} & \\(\\sum\\) & \\multicolumn{2}{@{}l}{No} & \\multicolumn{2}{l}{Yes} & \\(\\sum\\) & \\multicolumn{2}{@{}l}{No} & \\multicolumn{2}{l}{Yes} & \\(\\sum\\) & \\multicolumn{2}{@{}l}{No} & \\multicolumn{2}{l}{Yes} & \\(\\sum\\) & \\multicolumn{2}{@{}l}{No} & \\multicolumn{2}{l}{Yes} & \\(\\sum\\) \\\\\")\n",
    "for variable_name, contingency_1, chi2_result_1, contingency_2, chi2_result_2, contingency_3, chi2_result_3, contingency_4, chi2_result_4, contingency_5, chi2_result_5 in tests:\n",
    "    statistic_1, p_value_1, dof_1, _ = chi2_result_1\n",
    "    statistic_2, p_value_2, dof_2, _ = chi2_result_2\n",
    "    statistic_3, p_value_3, dof_3, _ = chi2_result_3\n",
    "    statistic_4, p_value_4, dof_4, _ = chi2_result_4\n",
    "    statistic_5, p_value_5, dof_5, _ = chi2_result_5\n",
    "    print(r\"  \\midrule\")\n",
    "    print(r\"  \\textit{\" + variable_name + r\"} & \\multicolumn{5}{@{}c@{\\hspace{\\largetabcolsep}}}{\" + chi2_to_tex(chi2_result_1, alpha_corrected) + r\"} & \\multicolumn{5}{@{}c@{\\hspace{\\largetabcolsep}}}{\" + chi2_to_tex(chi2_result_2, alpha_corrected) + r\"} & \\multicolumn{5}{@{}c@{\\hspace{\\largetabcolsep}}}{\" + chi2_to_tex(chi2_result_3, alpha_corrected) + r\"} & \\multicolumn{5}{@{}c@{\\hspace{\\largetabcolsep}}}{\" + chi2_to_tex(chi2_result_4, alpha_corrected) + r\"} & \\multicolumn{5}{@{}c@{}}{\" + chi2_to_tex(chi2_result_5, alpha_corrected) + r\"} \\\\\")\n",
    "    print(r\"  \\midrule\")\n",
    "    for i, ((index_1, row_1), (index_2, row_2), (index_3, row_3), (index_4, row_4), (index_5, row_5)) in enumerate(zip(contingency_1.iterrows(), contingency_2.iterrows(), contingency_3.iterrows(), contingency_4.iterrows(), contingency_5.iterrows())):\n",
    "        assert index_1 == index_2 == index_3 == index_4 == index_5\n",
    "        index = index_1\n",
    "        if \"All\" in index:\n",
    "            index = r\"\\(\\sum\\)\"\n",
    "        columns = [\n",
    "            index,\n",
    "            f\"{row_1['no']}\",\n",
    "            diff_expected_freq_tex(row_1, \"no\", index, i, chi2_result_1),\n",
    "            f\"{row_1['yes']}\",\n",
    "            diff_expected_freq_tex(row_1, \"yes\", index, i, chi2_result_1),\n",
    "            f\"{row_1['All']}\",\n",
    "            f\"{row_2['no']}\",\n",
    "            diff_expected_freq_tex(row_2, \"no\", index, i, chi2_result_2),\n",
    "            f\"{row_2['yes']}\",\n",
    "            diff_expected_freq_tex(row_2, \"yes\", index, i, chi2_result_2),\n",
    "            f\"{row_2['All']}\",\n",
    "            f\"{row_3['no']}\",\n",
    "            diff_expected_freq_tex(row_3, \"no\", index, i, chi2_result_3),\n",
    "            f\"{row_3['yes']}\",\n",
    "            diff_expected_freq_tex(row_3, \"yes\", index, i, chi2_result_3),\n",
    "            f\"{row_3['All']}\",\n",
    "            f\"{row_4['no']}\",\n",
    "            diff_expected_freq_tex(row_4, \"no\", index, i, chi2_result_4),\n",
    "            f\"{row_4['yes']}\",\n",
    "            diff_expected_freq_tex(row_4, \"yes\", index, i, chi2_result_4),\n",
    "            f\"{row_4['All']}\",\n",
    "            f\"{row_5['no']}\",\n",
    "            diff_expected_freq_tex(row_5, \"no\", index, i, chi2_result_5),\n",
    "            f\"{row_5['yes']}\",\n",
    "            diff_expected_freq_tex(row_5, \"yes\", index, i, chi2_result_5),\n",
    "            f\"{row_5['All']}\",\n",
    "        ]\n",
    "        print(r\"  \" + r\" & \".join(columns) + r\" \\\\\")\n",
    "print(r\"  \\bottomrule\")\n",
    "print(r\"\\end{tabular}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea1115f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "alpha_corrected"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a61159915424bb7e",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "## Correlations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1a8d2962ab4d906",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-10-20T00:34:26.316878968Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pearsonr(df_doc[\"rank\"], df_doc[\"has_fully_read_document\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77b95a14c125fba6",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-10-20T00:34:26.381427268Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pearsonr(df_doc[\"rank\"], df_doc[\"has_partially_read_document\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85831cf4955a095f",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-10-20T00:34:26.447282744Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pearsonr(df_doc[\"rank\"], df_doc[\"influence_document\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b6e8c25e32f8aef",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-10-20T00:34:26.514925244Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pearsonr(df_doc[\"rank\"], df_doc[\"quality_score_document\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f23d6ad07030c3d2",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-10-20T00:34:26.515897801Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pearsonr(df_doc[\"rank\"], df_doc[\"relevance_document\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66ab9a14453e4ddf",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-10-20T00:34:26.559890449Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pearsonr(df_doc[\"relevance_document\"], df_doc[\"quality_score_document\"])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}