Chapter 13.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "78f36a51",
   "metadata": {},
   "source": [
    "# Chapter 13"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cda5b72c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import statsmodels.api as sm\n",
    "from ISLP import load_data\n",
    "from ISLP.models import (ModelSpec as MS, summarize)\n",
    "from scipy.stats import \\\n",
    "    (ttest_1samp,\n",
    "     ttest_rel,\n",
    "     ttest_ind,\n",
    "     t as t_dbn)\n",
    "from statsmodels.stats.multicomp import \\\n",
    "     pairwise_tukeyhsd\n",
    "from statsmodels.stats.multitest import \\\n",
    "     multipletests as mult_test"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6151b0fa",
   "metadata": {},
   "source": [
    "## Exercise 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "02cf8848",
   "metadata": {},
   "outputs": [],
   "source": [
    "Carseats=load_data('Carseats')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8e4dd5e",
   "metadata": {},
   "source": [
    "### (a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "065db109",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sales</th>\n",
       "      <th>CompPrice</th>\n",
       "      <th>Income</th>\n",
       "      <th>Advertising</th>\n",
       "      <th>Population</th>\n",
       "      <th>Price</th>\n",
       "      <th>ShelveLoc</th>\n",
       "      <th>Age</th>\n",
       "      <th>Education</th>\n",
       "      <th>Urban</th>\n",
       "      <th>US</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9.50</td>\n",
       "      <td>138</td>\n",
       "      <td>73</td>\n",
       "      <td>11</td>\n",
       "      <td>276</td>\n",
       "      <td>120</td>\n",
       "      <td>Bad</td>\n",
       "      <td>42</td>\n",
       "      <td>17</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11.22</td>\n",
       "      <td>111</td>\n",
       "      <td>48</td>\n",
       "      <td>16</td>\n",
       "      <td>260</td>\n",
       "      <td>83</td>\n",
       "      <td>Good</td>\n",
       "      <td>65</td>\n",
       "      <td>10</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10.06</td>\n",
       "      <td>113</td>\n",
       "      <td>35</td>\n",
       "      <td>10</td>\n",
       "      <td>269</td>\n",
       "      <td>80</td>\n",
       "      <td>Medium</td>\n",
       "      <td>59</td>\n",
       "      <td>12</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7.40</td>\n",
       "      <td>117</td>\n",
       "      <td>100</td>\n",
       "      <td>4</td>\n",
       "      <td>466</td>\n",
       "      <td>97</td>\n",
       "      <td>Medium</td>\n",
       "      <td>55</td>\n",
       "      <td>14</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.15</td>\n",
       "      <td>141</td>\n",
       "      <td>64</td>\n",
       "      <td>3</td>\n",
       "      <td>340</td>\n",
       "      <td>128</td>\n",
       "      <td>Bad</td>\n",
       "      <td>38</td>\n",
       "      <td>13</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Sales  CompPrice  Income  Advertising  Population  Price ShelveLoc  Age  \\\n",
       "0   9.50        138      73           11         276    120       Bad   42   \n",
       "1  11.22        111      48           16         260     83      Good   65   \n",
       "2  10.06        113      35           10         269     80    Medium   59   \n",
       "3   7.40        117     100            4         466     97    Medium   55   \n",
       "4   4.15        141      64            3         340    128       Bad   38   \n",
       "\n",
       "   Education Urban   US  \n",
       "0         17   Yes  Yes  \n",
       "1         10   Yes  Yes  \n",
       "2         12   Yes  Yes  \n",
       "3         14   Yes  Yes  \n",
       "4         13   Yes   No  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Carseats.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ef76df0a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 400 entries, 0 to 399\n",
      "Data columns (total 11 columns):\n",
      " #   Column       Non-Null Count  Dtype   \n",
      "---  ------       --------------  -----   \n",
      " 0   Sales        400 non-null    float64 \n",
      " 1   CompPrice    400 non-null    int64   \n",
      " 2   Income       400 non-null    int64   \n",
      " 3   Advertising  400 non-null    int64   \n",
      " 4   Population   400 non-null    int64   \n",
      " 5   Price        400 non-null    int64   \n",
      " 6   ShelveLoc    400 non-null    category\n",
      " 7   Age          400 non-null    int64   \n",
      " 8   Education    400 non-null    int64   \n",
      " 9   Urban        400 non-null    category\n",
      " 10  US           400 non-null    category\n",
      "dtypes: category(3), float64(1), int64(7)\n",
      "memory usage: 26.7 KB\n"
     ]
    }
   ],
   "source": [
    "Carseats.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "017a6ac4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>intercept</th>\n",
       "      <th>CompPrice</th>\n",
       "      <th>Income</th>\n",
       "      <th>Advertising</th>\n",
       "      <th>Population</th>\n",
       "      <th>Price</th>\n",
       "      <th>Age</th>\n",
       "      <th>Education</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>138</td>\n",
       "      <td>73</td>\n",
       "      <td>11</td>\n",
       "      <td>276</td>\n",
       "      <td>120</td>\n",
       "      <td>42</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>111</td>\n",
       "      <td>48</td>\n",
       "      <td>16</td>\n",
       "      <td>260</td>\n",
       "      <td>83</td>\n",
       "      <td>65</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>113</td>\n",
       "      <td>35</td>\n",
       "      <td>10</td>\n",
       "      <td>269</td>\n",
       "      <td>80</td>\n",
       "      <td>59</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>117</td>\n",
       "      <td>100</td>\n",
       "      <td>4</td>\n",
       "      <td>466</td>\n",
       "      <td>97</td>\n",
       "      <td>55</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>141</td>\n",
       "      <td>64</td>\n",
       "      <td>3</td>\n",
       "      <td>340</td>\n",
       "      <td>128</td>\n",
       "      <td>38</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>395</th>\n",
       "      <td>1.0</td>\n",
       "      <td>138</td>\n",
       "      <td>108</td>\n",
       "      <td>17</td>\n",
       "      <td>203</td>\n",
       "      <td>128</td>\n",
       "      <td>33</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>396</th>\n",
       "      <td>1.0</td>\n",
       "      <td>139</td>\n",
       "      <td>23</td>\n",
       "      <td>3</td>\n",
       "      <td>37</td>\n",
       "      <td>120</td>\n",
       "      <td>55</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>397</th>\n",
       "      <td>1.0</td>\n",
       "      <td>162</td>\n",
       "      <td>26</td>\n",
       "      <td>12</td>\n",
       "      <td>368</td>\n",
       "      <td>159</td>\n",
       "      <td>40</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>398</th>\n",
       "      <td>1.0</td>\n",
       "      <td>100</td>\n",
       "      <td>79</td>\n",
       "      <td>7</td>\n",
       "      <td>284</td>\n",
       "      <td>95</td>\n",
       "      <td>50</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>399</th>\n",
       "      <td>1.0</td>\n",
       "      <td>134</td>\n",
       "      <td>37</td>\n",
       "      <td>0</td>\n",
       "      <td>27</td>\n",
       "      <td>120</td>\n",
       "      <td>49</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>400 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     intercept  CompPrice  Income  Advertising  Population  Price  Age  \\\n",
       "0          1.0        138      73           11         276    120   42   \n",
       "1          1.0        111      48           16         260     83   65   \n",
       "2          1.0        113      35           10         269     80   59   \n",
       "3          1.0        117     100            4         466     97   55   \n",
       "4          1.0        141      64            3         340    128   38   \n",
       "..         ...        ...     ...          ...         ...    ...  ...   \n",
       "395        1.0        138     108           17         203    128   33   \n",
       "396        1.0        139      23            3          37    120   55   \n",
       "397        1.0        162      26           12         368    159   40   \n",
       "398        1.0        100      79            7         284     95   50   \n",
       "399        1.0        134      37            0          27    120   49   \n",
       "\n",
       "     Education  \n",
       "0           17  \n",
       "1           10  \n",
       "2           12  \n",
       "3           14  \n",
       "4           13  \n",
       "..         ...  \n",
       "395         14  \n",
       "396         11  \n",
       "397         18  \n",
       "398         12  \n",
       "399         16  \n",
       "\n",
       "[400 rows x 8 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "design=MS(Carseats.columns.drop(['Sales','ShelveLoc','Urban','US']))\n",
    "y=Carseats['Sales']\n",
    "X=design.fit_transform(Carseats)\n",
    "X\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "f9344d4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "Xs=[]\n",
    "results=[]\n",
    "p_values=[]\n",
    "for i in range(1,X.shape[1]):\n",
    "    Xs.append(X.iloc[:,[0,i]])\n",
    "for i in range(len(Xs)):\n",
    "    results.append(sm.OLS(y,Xs[i]).fit())\n",
    "for i in range(len(results)):\n",
    "    p_values.append(results[i].pvalues[1])#This way you will prevent pvalues form being rounded\n",
    "df=pd.DataFrame({'Variables':X.columns.drop(['intercept']),'p_values':p_values})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "4e78ce32",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Variables</th>\n",
       "      <th>p_values</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CompPrice</td>\n",
       "      <td>2.009398e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Income</td>\n",
       "      <td>2.309670e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Advertising</td>\n",
       "      <td>4.377677e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Population</td>\n",
       "      <td>3.139816e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Price</td>\n",
       "      <td>7.618187e-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Age</td>\n",
       "      <td>2.788950e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Education</td>\n",
       "      <td>2.999442e-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Variables      p_values\n",
       "0    CompPrice  2.009398e-01\n",
       "1       Income  2.309670e-03\n",
       "2  Advertising  4.377677e-08\n",
       "3   Population  3.139816e-01\n",
       "4        Price  7.618187e-21\n",
       "5          Age  2.788950e-06\n",
       "6    Education  2.999442e-01"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "121f3629",
   "metadata": {},
   "source": [
    "### (b)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b2666e5",
   "metadata": {},
   "source": [
    "If we control the type I error for each the seven models fitted in the previous question at level 5%, then we reject the null hypotheses of the form $H_{0,j}: \\beta_{1,j}=0$, associated with variables 'CompPrice', 'Population' and 'Education'."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98255375",
   "metadata": {},
   "source": [
    "### (c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "5801dea0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#We use the Holm's step down approach \n",
    "alpha=0.05\n",
    "m=df.shape[0]\n",
    "df_sorted=df.sort_values(by=['p_values'])\n",
    "#Step 4 of Holm's step down method\n",
    "condition=[]\n",
    "for i in range(df_sorted.shape[0]):\n",
    "    if df_sorted['p_values'].iloc[i]>alpha/(m+2-i):\n",
    "        condition.append(i)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "77f802d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "L=min(condition)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "369f1cab",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sorted['Reject']=np.where(df_sorted['p_values']<df_sorted['p_values'].iloc[L],'Yes','No')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "f7b6073e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Variables</th>\n",
       "      <th>p_values</th>\n",
       "      <th>Reject</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Price</td>\n",
       "      <td>7.618187e-21</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Advertising</td>\n",
       "      <td>4.377677e-08</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Age</td>\n",
       "      <td>2.788950e-06</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Income</td>\n",
       "      <td>2.309670e-03</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CompPrice</td>\n",
       "      <td>2.009398e-01</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Education</td>\n",
       "      <td>2.999442e-01</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Population</td>\n",
       "      <td>3.139816e-01</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Variables      p_values Reject\n",
       "4        Price  7.618187e-21    Yes\n",
       "2  Advertising  4.377677e-08    Yes\n",
       "5          Age  2.788950e-06    Yes\n",
       "1       Income  2.309670e-03    Yes\n",
       "0    CompPrice  2.009398e-01     No\n",
       "6    Education  2.999442e-01     No\n",
       "3   Population  3.139816e-01     No"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_sorted"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "108b1e15",
   "metadata": {},
   "source": [
    "### (d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "833670c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "qvalues = mult_test(df['p_values'],alpha=0.2, method = \"fdr_bh\")[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "c2b25c9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Reject_FDR_bh']=np.where(qvalues==True,'Yes','No')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "ae961c2d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Variables</th>\n",
       "      <th>p_values</th>\n",
       "      <th>Reject_FDR_bh</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CompPrice</td>\n",
       "      <td>2.009398e-01</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Income</td>\n",
       "      <td>2.309670e-03</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Advertising</td>\n",
       "      <td>4.377677e-08</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Population</td>\n",
       "      <td>3.139816e-01</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Price</td>\n",
       "      <td>7.618187e-21</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Age</td>\n",
       "      <td>2.788950e-06</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Education</td>\n",
       "      <td>2.999442e-01</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Variables      p_values Reject_FDR_bh\n",
       "0    CompPrice  2.009398e-01            No\n",
       "1       Income  2.309670e-03           Yes\n",
       "2  Advertising  4.377677e-08           Yes\n",
       "3   Population  3.139816e-01            No\n",
       "4        Price  7.618187e-21           Yes\n",
       "5          Age  2.788950e-06           Yes\n",
       "6    Education  2.999442e-01            No"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7f03b565",
   "metadata": {},
   "source": [
    "## Exercise 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9eb0c014",
   "metadata": {},
   "outputs": [],
   "source": [
    "rng = np.random.default_rng (1)\n",
    "n, m = 20, 100\n",
    "X = rng.normal(size=(n, m))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1b26032a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20, 100)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98a83524",
   "metadata": {},
   "source": [
    "### (a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bc409d02",
   "metadata": {},
   "outputs": [],
   "source": [
    "results = ttest_1samp(X, 0).pvalue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "93e4943b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100,)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e0fa78df",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAj0AAAHFCAYAAAAZuEjoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA4o0lEQVR4nO3deVxVdf7H8fdF8OIGKiqCImIpYo5LkGuOW1K4tP90xgU3phxcUrR+kpXLWE42OZS5tKhMkxq/Uhun/Gm0uOTSiGJa0uYGKmRoAm6IcH5/+OD+uoEKCBwv5/V8PM6jzvd+v/d8zteb993Zrs0wDEMAAABVnJvZBQAAAFQGQg8AALAEQg8AALAEQg8AALAEQg8AALAEQg8AALAEQg8AALAEQg8AALAEQg8AALAEQg9QSeLj42Wz2ZSUlFTs6wMHDlTz5s2d2po3b65Ro0aVajs7duzQrFmzdPbs2bIVakEJCQm64447VKNGDdlsNu3bt8/skkqsLJ8RwKoIPcAtbN26dXr22WdLNWbHjh2aPXs2oaeEfv75Z40YMUK33XabNm7cqJ07d6pVq1ZmlwWgAribXQCAa+vYsaPZJZRaXl6ebDab3N1d46+X77//Xnl5eRo+fLh69uxpdjkAKhBHeoBb2G9PXRQUFGju3LkKDg5WjRo1VLduXbVr106vvPKKJGnWrFl68sknJUlBQUGy2Wyy2WzavHmzY/z8+fPVunVr2e12NWrUSJGRkTp+/LjTdg3D0AsvvKDAwEB5enoqLCxMiYmJ6tWrl3r16uXot3nzZtlsNv3zn//U1KlT1aRJE9ntdv3444/6+eefFR0drTZt2qh27dpq1KiR+vTpo23btjlt6+jRo7LZbHrppZf04osvqnnz5qpRo4Z69erlCCTTp0+Xv7+/vL299dBDD+nUqVMlmr/169era9euqlmzpurUqaN+/fpp586djtdHjRqlu+++W5I0ZMgQ2Ww2p/37rcJTlImJiRo9erTq16+vWrVqadCgQTp8+PB1a8nLy1OjRo00YsSIIq+dPXtWNWrUUExMjCTp0qVLmjp1qjp06CBvb2/Vr19fXbt21b/+9a8b7nNhjUePHnVqL/yzKvwsFPrkk0/Ut29feXl5qWbNmurevbs+/fRTpz4///yzHnvsMQUEBMhut6thw4bq3r27PvnkkxvWA9xKXON/xYAqJD8/X1euXCnSbhjGDcfOnz9fs2bN0jPPPKPf//73ysvL07fffus4lRUVFaUzZ85o4cKFWrt2rfz8/CRJbdq0kST9+c9/1htvvKEJEyZo4MCBOnr0qJ599llt3rxZe/fuVYMGDSRJM2bM0Lx58/TYY4/p4YcfVlpamqKiopSXl1fsqZ/Y2Fh17dpVS5culZubmxo1aqSff/5ZkjRz5kw1btxY586d07p169SrVy99+umnRcLFokWL1K5dOy1atEhnz57V1KlTNWjQIHXu3FkeHh5avny5jh07pmnTpikqKkrr16+/7lytWrVKw4YNU3h4uFavXq3c3FzNnz/fsf27775bzz77rDp16qTx48frhRdeUO/eveXl5XXDP4exY8eqX79+WrVqldLS0vTMM8+oV69e2r9/v+rWrVvsGA8PDw0fPlxLly7VokWLnLazevVqXbp0SaNHj5Yk5ebm6syZM5o2bZqaNGmiy5cv65NPPtHDDz+sFStWKDIy8oY1lsQ777yjyMhIPfDAA/rHP/4hDw8Pvf7667r33nu1adMm9e3bV5I0YsQI7d27V88//7xatWqls2fPau/evTp9+nS51AFUGgNApVixYoUh6bpLYGCg05jAwEBj5MiRjvWBAwcaHTp0uO52XnrpJUOSceTIEaf2lJQUQ5IRHR3t1P7ll18akoynn37aMAzDOHPmjGG3240hQ4Y49du5c6chyejZs6ej7fPPPzckGb///e9vuP9Xrlwx8vLyjL59+xoPPfSQo/3IkSOGJKN9+/ZGfn6+oz0uLs6QZNx///1O7zN58mRDkpGVlXXNbeXn5xv+/v7G7373O6f3zMnJMRo1amR069atyD689957N9yHwj/DX9dvGIaxfft2Q5Ixd+7c647fv3+/Icl44403nNo7depkhIaGXnNc4dyNHTvW6Nixo9Nrv/2MFNb42z//wv38/PPPDcMwjPPnzxv169c3Bg0a5NQvPz/faN++vdGpUydHW+3atY3Jkydfd98AV8DpLaCSvf3229q9e3eRpfA0y/V06tRJX331laKjo7Vp0yZlZ2eXeLuff/65JBW506dTp04KCQlxnNLYtWuXcnNzNXjwYKd+Xbp0KXJ3WaFHHnmk2PalS5fqzjvvlKenp9zd3eXh4aFPP/1UKSkpRfr2799fbm7//1dSSEiIJGnAgAFO/QrbU1NTr7Gn0nfffaeTJ09qxIgRTu9Zu3ZtPfLII9q1a5cuXLhwzfE3MmzYMKf1bt26KTAw0DHHhmHoypUrTosk/e53v1NoaKhWrFjhGJuSkqL//Oc/GjNmjNN7vvfee+revbtq167tmLtly5YVO3dlsWPHDp05c0YjR450qrOgoED33Xefdu/erfPnz0u6+hmJj4/X3LlztWvXLuXl5ZVLDUBlI/QAlSwkJERhYWFFFm9v7xuOjY2N1d/+9jft2rVLERER8vHxUd++fa95G/yvFZ6KKDzl9Wv+/v6O1wv/6evrW6RfcW3Xes8FCxboz3/+szp37qw1a9Zo165d2r17t+677z5dvHixSP/69es7rVevXv267ZcuXSq2ll/vw7X2taCgQL/88ss1x99I48aNi20r3G7hqaJfL4XGjBmjnTt36ttvv5UkrVixQna7XX/84x8dfdauXavBgwerSZMmeuedd7Rz507t3r1bY8aMue5+l8ZPP/0kSXr00UeL1Priiy/KMAydOXNG0tVb+keOHKm33npLXbt2Vf369RUZGamMjIxyqQWoLFzTA7gQd3d3xcTEKCYmRmfPntUnn3yip59+Wvfee6/S0tJUs2bNa4718fGRJKWnp6tp06ZOr508edJxPU9hv8IvxV/LyMgo9miPzWYr0vbOO++oV69eWrJkiVN7Tk7O9XeyHPx6X3/r5MmTcnNzU7169cr8/sV92WdkZOj222+XJA0aNEi7d+8uduwf//hHxcTEKD4+Xs8//7z++c9/6sEHH3Sq55133lFQUJASEhKc5jY3N/eGtXl6ehbbNzMz02m98M974cKF6tKlS7HvVRhyGzRooLi4OMXFxSk1NVXr16/X9OnTderUKW3cuPGGNQG3Co70AC6qbt26evTRRzV+/HidOXPGcbeO3W6XpCJHU/r06SPp6hfqr+3evVspKSmOi1Y7d+4su92uhIQEp367du3SsWPHSlyfzWZz1FJo//79TndPVZTg4GA1adJEq1atcrpA/Pz581qzZo3jjq6yWrlypdP6jh07dOzYMcfF2T4+PkWO5BWqV6+eHnzwQb399tv68MMPlZGRUeTUls1mU/Xq1Z0CT0ZGRonu3ioMpfv373dq/+2F3927d1fdunV18ODBYo88hoWFOY6q/VqzZs00YcIE9evXT3v37r1hPcCthCM9gAsZNGiQ2rZtq7CwMDVs2FDHjh1TXFycAgMD1bJlS0lXrxuRpFdeeUUjR46Uh4eHgoODFRwcrMcee0wLFy6Um5ubIiIiHHdvBQQEaMqUKZKunk6KiYnRvHnzVK9ePT300EM6fvy4Zs+eLT8/P6drZK5n4MCB+stf/qKZM2eqZ8+e+u677zRnzhwFBQUVe/daeXJzc9P8+fM1bNgwDRw4UI8//rhyc3P10ksv6ezZs/rrX/96U++flJSkqKgo/dd//ZfS0tI0Y8YMNWnSRNHR0SUaP2bMGCUkJGjChAlq2rSp7rnnHqfXBw4cqLVr1yo6OlqPPvqo0tLS9Je//EV+fn764Ycfrvved911l4KDgzVt2jRduXJF9erV07p16/TFF1849atdu7YWLlyokSNH6syZM3r00Ucdd9199dVX+vnnn7VkyRJlZWWpd+/eGjp0qFq3bq06depo9+7d2rhxox5++OHSTRxgNpMvpAYso/Cumt27dxf7+oABA25499bLL79sdOvWzWjQoIFRvXp1o1mzZsbYsWONo0ePOo2LjY01/P39DTc3N6c7dvLz840XX3zRaNWqleHh4WE0aNDAGD58uJGWluY0vqCgwJg7d67RtGlTo3r16ka7du2MDz/80Gjfvr3TnUvXu/MpNzfXmDZtmtGkSRPD09PTuPPOO40PPvjAGDlypNN+Ft699dJLLzmNv9Z732gef+2DDz4wOnfubHh6ehq1atUy+vbta2zfvr1E2ylO4bY//vhjY8SIEUbdunWNGjVqGP379zd++OGHG44vlJ+fbwQEBBiSjBkzZhTb569//avRvHlzw263GyEhIcabb75pzJw50/jtX9u//YwYhmF8//33Rnh4uOHl5WU0bNjQmDhxovHRRx85fRYKbdmyxRgwYIBRv359w8PDw2jSpIkxYMAAx3xcunTJGDdunNGuXTvDy8vLqFGjhhEcHGzMnDnTOH/+fIn3GbgV2AyjBA8HAWB5R44cUevWrTVz5kw9/fTTZpdjivj4eI0ePVq7d+92OmUFwDVwegtAEV999ZVWr16tbt26ycvLS999953mz58vLy8vjR071uzyAKBMCD0AiqhVq5aSkpK0bNkynT17Vt7e3urVq5eef/75a962DgC3Ok5vAQAAS+CWdQAAYAmEHgAAYAmEHgAAYAmWu5C5oKBAJ0+eVJ06dYp9dD4AALj1GIahnJwc+fv7l/ghqb9ludBz8uRJBQQEmF0GAAAog7S0tCK/H1hSpoaerVu36qWXXtKePXuUnp6udevW6cEHHyzR2O3bt6tnz55q27at9u3bV+Jt1qlTR9LVSfPy8ipD1QAAoLJlZ2crICDA8T1eFqaGnvPnz6t9+/YaPXq0HnnkkRKPy8rKUmRkpPr27VvsL0FfT+EpLS8vL0IPAAAu5mYuTTE19ERERCgiIqLU4x5//HENHTpU1apV0wcffFD+hQEAgCrH5e7eWrFihQ4dOqSZM2eWqH9ubq6ys7OdFgAAYD0uFXp++OEHTZ8+XStXrpS7e8kOUs2bN0/e3t6OhYuYAQCwJpcJPfn5+Ro6dKhmz56tVq1alXhcbGyssrKyHEtaWloFVgkAAG5VLnPLek5OjpKSkpScnKwJEyZIuvrMHcMw5O7uro8//lh9+vQpMs5ut8tut1d2uQAA4BbjMqHHy8tLBw4ccGpbvHixPvvsM73//vsKCgoyqTIAAOAKTA09586d048//uhYP3LkiPbt26f69eurWbNmio2N1YkTJ/T222/Lzc1Nbdu2dRrfqFEjeXp6FmkHAAD4LVNDT1JSknr37u1Yj4mJkSSNHDlS8fHxSk9PV2pqqlnlAQCAKsRmGIZhdhGVKTs7W97e3srKyuLhhAAAuIjy+P52mbu3AAAAbgahBwAAWAKhBwAAWAKhBwAAWAKhBwAAWILLPJwQAEoiNTVVmZmZZpdRKg0aNFCzZs3MLgO3KD7T5YfQA6DKSE1NVXDrEF26eMHsUkrFs0ZNffdtyi35JQFz8ZkuX4QeAFVGZmamLl28IJ+BU+XhE2B2OSWSdzpNpz98WZmZmbfcFwTMx2e6fBF6AFQ5Hj4Bsje+3ewygHLDZ7p8cCEzAACwBEIPAACwBEIPAACwBEIPAACwBEIPAACwBEIPAACwBEIPAACwBEIPAACwBEIPAACwBEIPAACwBEIPAACwBEIPAACwBEIPAACwBEIPAACwBEIPAACwBEIPAACwBEIPAACwBEIPAACwBEIPAACwBEIPAACwBEIPAACwBEIPAACwBEIPAACwBHezC4D5UlNTlZmZaXYZpdKgQQM1a9bM7DJKhXkGAHMReiwuNTVVwa1DdOniBbNLKRXPGjX13bcpLvOFzDwDgPkIPRaXmZmpSxcvyGfgVHn4BJhdTonknU7T6Q9fVmZmpst8GTPPAGA+Qg8kSR4+AbI3vt3sMqo85hkAzMOFzAAAwBIIPQAAwBIIPQAAwBIIPQAAwBIIPQAAwBJMDT1bt27VoEGD5O/vL5vNpg8++OC6/deuXat+/fqpYcOG8vLyUteuXbVp06bKKRYAALg0U0PP+fPn1b59e7322msl6r9161b169dPGzZs0J49e9S7d28NGjRIycnJFVwpAABwdaY+pyciIkIREREl7h8XF+e0/sILL+hf//qX/v3vf6tjx47lXB0AAKhKXPrhhAUFBcrJyVH9+vWv2Sc3N1e5ubmO9ezs7MooDQAA3GJc+kLml19+WefPn9fgwYOv2WfevHny9vZ2LAEBrvETAAAAoHy5bOhZvXq1Zs2apYSEBDVq1Oia/WJjY5WVleVY0tLSKrFKAABwq3DJ01sJCQkaO3as3nvvPd1zzz3X7Wu322W32yupMgAAcKtyuSM9q1ev1qhRo7Rq1SoNGDDA7HIAAICLMPVIz7lz5/Tjjz861o8cOaJ9+/apfv36atasmWJjY3XixAm9/fbbkq4GnsjISL3yyivq0qWLMjIyJEk1atSQt7e3KfsAAABcg6lHepKSktSxY0fH7eYxMTHq2LGjnnvuOUlSenq6UlNTHf1ff/11XblyRePHj5efn59jeeKJJ0ypHwAAuA5Tj/T06tVLhmFc8/X4+Hin9c2bN1dsQQAAoMpyuWt6AAAAyoLQAwAALIHQAwAALIHQAwAALIHQAwAALIHQAwAALIHQAwAALMElf3vrVpaamqrMzEyzyyixlJQUs0sAAKBSEHrKUWpqqoJbh+jSxQtmlwIAAH6D0FOOMjMzdeniBfkMnCoPnwCzyymRi4eTlLXtHbPLAACgwhF6KoCHT4DsjW83u4wSyTudZnYJAABUCi5kBgAAlkDoAQAAlkDoAQAAlkDoAQAAlkDoAQAAlkDoAQAAlkDoAQAAlkDoAQAAlkDoAQAAlkDoAQAAlkDoAQAAlkDoAQAAlkDoAQAAlkDoAQAAlkDoAQAAlkDoAQAAlkDoAQAAlkDoAQAAlkDoAQAAlkDoAQAAlkDoAQAAlkDoAQAAlkDoAQAAluBudgFAWaWkpJhdQom5Uq0AUFUReuBy8s/9ItlsGj58uNmlAABcCKEHLqcg95xkGPIZOFUePgFml1MiFw8nKWvbO2aXAQCWRuiBy/LwCZC98e1ml1EieafTzC4BACyPC5kBAIAlEHoAAIAlEHoAAIAlmBp6tm7dqkGDBsnf3182m00ffPDBDcds2bJFoaGh8vT0VIsWLbR06dKKLxQAALg8U0PP+fPn1b59e7322msl6n/kyBH1799fPXr0UHJysp5++mlNmjRJa9asqeBKAQCAqzP17q2IiAhFRESUuP/SpUvVrFkzxcXFSZJCQkKUlJSkv/3tb3rkkUcqqEoAAFAVuNQ1PTt37lR4eLhT27333qukpCTl5eUVOyY3N1fZ2dlOCwAAsB6XCj0ZGRny9fV1avP19dWVK1eUmZlZ7Jh58+bJ29vbsQQEuMbD7AAAQPlyqdAjSTabzWndMIxi2wvFxsYqKyvLsaSl8ZA4AACsyKWeyNy4cWNlZGQ4tZ06dUru7u7y8fEpdozdbpfdbq+M8gAAwC3MpY70dO3aVYmJiU5tH3/8scLCwuTh4WFSVQAAwBWYGnrOnTunffv2ad++fZKu3pK+b98+paamSrp6aioyMtLRf9y4cTp27JhiYmKUkpKi5cuXa9myZZo2bZoZ5QMAABdi6umtpKQk9e7d27EeExMjSRo5cqTi4+OVnp7uCECSFBQUpA0bNmjKlClatGiR/P399eqrr3K7OgAAuCFTQ0+vXr0cFyIXJz4+vkhbz549tXfv3gqsCgAAVEUudU0PAABAWRF6AACAJRB6AACAJRB6AACAJRB6AACAJRB6AACAJRB6AACAJRB6AACAJRB6AACAJbjUr6wDqHwpKSlml1BirlQrgMpH6AFQrPxzv0g2m4YPH252KQBQLgg9AIpVkHtOMgz5DJwqD58As8spkYuHk5S17R2zywBwiyL0ALguD58A2RvfbnYZJZJ3Os3sEgDcwriQGQAAWAKhBwAAWAKhBwAAWAKhBwAAWAKhBwAAWAKhBwAAWAKhBwAAWAKhBwAAWAKhBwAAWAKhBwAAWAKhBwAAWAKhBwAAWAKhBwAAWAKhBwAAWAKhBwAAWAKhBwAAWAKhBwAAWAKhBwAAWAKhBwAAWAKhBwAAWAKhBwAAWAKhBwAAWAKhBwAAWIK72QUAAKSUlBSzSyi1Bg0aqFmzZmaXAZQYoQcATJR/7hfJZtPw4cPNLqXUPGvU1HffphB84DIIPQBgooLcc5JhyGfgVHn4BJhdTonlnU7T6Q9fVmZmJqEHLoPQAwC3AA+fANkb3252GUCVxoXMAADAEkwPPYsXL1ZQUJA8PT0VGhqqbdu2Xbf/ypUr1b59e9WsWVN+fn4aPXq0Tp8+XUnVAgAAV2Vq6ElISNDkyZM1Y8YMJScnq0ePHoqIiFBqamqx/b/44gtFRkZq7Nix+uabb/Tee+9p9+7dioqKquTKAQCAqzE19CxYsEBjx45VVFSUQkJCFBcXp4CAAC1ZsqTY/rt27VLz5s01adIkBQUF6e6779bjjz+upKSkSq4cAAC4GtNCz+XLl7Vnzx6Fh4c7tYeHh2vHjh3FjunWrZuOHz+uDRs2yDAM/fTTT3r//fc1YMCAa24nNzdX2dnZTgsAALAe00JPZmam8vPz5evr69Tu6+urjIyMYsd069ZNK1eu1JAhQ1S9enU1btxYdevW1cKFC6+5nXnz5snb29uxBAS4zi2hAACg/Jh+IbPNZnNaNwyjSFuhgwcPatKkSXruuee0Z88ebdy4UUeOHNG4ceOu+f6xsbHKyspyLGlpaeVaPwAAcA2mPaenQYMGqlatWpGjOqdOnSpy9KfQvHnz1L17dz355JOSpHbt2qlWrVrq0aOH5s6dKz8/vyJj7Ha77HZ7+e8AAABwKaYd6alevbpCQ0OVmJjo1J6YmKhu3boVO+bChQtyc3MuuVq1apKuHiECAAC4FlNPb8XExOitt97S8uXLlZKSoilTpig1NdVxuio2NlaRkZGO/oMGDdLatWu1ZMkSHT58WNu3b9ekSZPUqVMn+fv7m7UbAADABZj6MxRDhgzR6dOnNWfOHKWnp6tt27basGGDAgMDJUnp6elOz+wZNWqUcnJy9Nprr2nq1KmqW7eu+vTpoxdffNGsXQAAAC6iTKGnRYsW2r17t3x8fJzaz549qzvvvFOHDx8u8XtFR0crOjq62Nfi4+OLtE2cOFETJ04sVb0AAABlOr119OhR5efnF2nPzc3ViRMnbrooAACA8laqIz3r1693/PumTZvk7e3tWM/Pz9enn36q5s2bl1txAAAA5aVUoefBBx+UdPXZOiNHjnR6zcPDQ82bN9fLL79cbsUBAACUl1KFnoKCAklSUFCQdu/erQYNGlRIUQAAAOWtTBcyHzlypLzrAAAAqFBlvmX9008/1aeffqpTp045jgAVWr58+U0XBgAAUJ7KFHpmz56tOXPmKCwsTH5+ftf8rSwAAIBbRZlCz9KlSxUfH68RI0aUdz0AAAAVokzP6bl8+fI1fx8LAADgVlSm0BMVFaVVq1aVdy0AAAAVpkynty5duqQ33nhDn3zyidq1aycPDw+n1xcsWFAuxQEAAJSXMoWe/fv3q0OHDpKkr7/+2uk1LmoGANyqUlNTlZmZaXYZJZaSkmJ2CVVKmULP559/Xt51AABQoVJTUxXcOkSXLl4wuxSYpMzP6QEAwJVkZmbq0sUL8hk4VR4+AWaXUyIXDycpa9s7ZpdRZZQp9PTu3fu6p7E+++yzMhcEAEBF8vAJkL3x7WaXUSJ5p9PMLqFKKVPoKbyep1BeXp727dunr7/+usgPkQIAANwKyhR6/v73vxfbPmvWLJ07d+6mCgIAAKgIZXpOz7UMHz6c390CAAC3pHINPTt37pSnp2d5viUAAEC5KNPprYcffthp3TAMpaenKykpSc8++2y5FAYAAFCeyhR6vL29ndbd3NwUHBysOXPmKDw8vFwKAwAAKE9lCj0rVqwo7zoAAAAq1E09nHDPnj1KSUmRzWZTmzZt1LFjx/KqCwAAoFyVKfScOnVKf/jDH7R582bVrVtXhmEoKytLvXv31rvvvquGDRuWd50AAAA3pUx3b02cOFHZ2dn65ptvdObMGf3yyy/6+uuvlZ2drUmTJpV3jQAAADetTEd6Nm7cqE8++UQhISGOtjZt2mjRokVcyAwAAG5JZTrSU1BQIA8PjyLtHh4eKigouOmiAAAAyluZQk+fPn30xBNP6OTJk462EydOaMqUKerbt2+5FQcAAFBeyhR6XnvtNeXk5Kh58+a67bbbdPvttysoKEg5OTlauHBhedcIAABw08p0TU9AQID27t2rxMREffvttzIMQ23atNE999xT3vUBAACUi1Id6fnss8/Upk0bZWdnS5L69euniRMnatKkSbrrrrt0xx13aNu2bRVSKAAAwM0oVeiJi4vTn/70J3l5eRV5zdvbW48//rgWLFhQbsUBAACUl1KFnq+++kr33XffNV8PDw/Xnj17brooAACA8laq0PPTTz8Ve6t6IXd3d/388883XRQAAEB5K1XoadKkiQ4cOHDN1/fv3y8/P7+bLgoAAKC8lSr09O/fX88995wuXbpU5LWLFy9q5syZGjhwYLkVBwAAUF5Kdcv6M888o7Vr16pVq1aaMGGCgoODZbPZlJKSokWLFik/P18zZsyoqFoBAADKrFShx9fXVzt27NCf//xnxcbGyjAMSZLNZtO9996rxYsXy9fXt0IKBQAAuBmlfjhhYGCgNmzYoF9++UU//vijDMNQy5YtVa9evYqoDwAAoFyU6YnMklSvXj3ddddd5VkLAABAhSnTb28BAAC4GtNDz+LFixUUFCRPT0+Fhobe8GcscnNzNWPGDAUGBsput+u2227T8uXLK6laAADgqsp8eqs8JCQkaPLkyVq8eLG6d++u119/XRERETp48KCaNWtW7JjBgwfrp59+0rJly3T77bfr1KlTunLlSiVXDgAAXI2poWfBggUaO3asoqKiJF39ba9NmzZpyZIlmjdvXpH+Gzdu1JYtW3T48GHVr19fktS8efPKLBkAALgo005vXb58WXv27FF4eLhTe3h4uHbs2FHsmPXr1yssLEzz589XkyZN1KpVK02bNk0XL1685nZyc3OVnZ3ttAAAAOsx7UhPZmam8vPzizzXx9fXVxkZGcWOOXz4sL744gt5enpq3bp1yszMVHR0tM6cOXPN63rmzZun2bNnl3v9AADAtZh+IbPNZnNaNwyjSFuhgoIC2Ww2rVy5Up06dVL//v21YMECxcfHX/NoT2xsrLKyshxLWlpaue8DAAC49Zl2pKdBgwaqVq1akaM6p06duuZTnf38/NSkSRN5e3s72kJCQmQYho4fP66WLVsWGWO322W328u3eAAA4HJMO9JTvXp1hYaGKjEx0ak9MTFR3bp1K3ZM9+7ddfLkSZ07d87R9v3338vNzU1Nmzat0HoBAIBrM/X0VkxMjN566y0tX75cKSkpmjJlilJTUzVu3DhJV09NRUZGOvoPHTpUPj4+Gj16tA4ePKitW7fqySef1JgxY1SjRg2zdgMAALgAU29ZHzJkiE6fPq05c+YoPT1dbdu21YYNGxQYGChJSk9PV2pqqqN/7dq1lZiYqIkTJyosLEw+Pj4aPHiw5s6da9YuAAAAF2Fq6JGk6OhoRUdHF/tafHx8kbbWrVsXOSUGAABwI6bfvQUAAFAZCD0AAMASCD0AAMASCD0AAMASCD0AAMASCD0AAMASCD0AAMASCD0AAMASCD0AAMASCD0AAMASCD0AAMASCD0AAMASCD0AAMASTP+VdQCA60pJSTG7hBJzpVpRMQg9AIBSyz/3i2Szafjw4WaXApQYoQcAUGoFueckw5DPwKny8Akwu5wSuXg4SVnb3jG7DJiI0AMAKDMPnwDZG99udhklknc6zewSYDIuZAYAAJZA6AEAAJZA6AEAAJZA6AEAAJZA6AEAAJZA6AEAAJZA6AEAAJZA6AEAAJZA6AEAAJZA6AEAAJZA6AEAAJZA6AEAAJZA6AEAAJZA6AEAAJZA6AEAAJZA6AEAAJZA6AEAAJZA6AEAAJZA6AEAAJZA6AEAAJZA6AEAAJZA6AEAAJZA6AEAAJZA6AEAAJZgeuhZvHixgoKC5OnpqdDQUG3btq1E47Zv3y53d3d16NChYgsEAABVgqmhJyEhQZMnT9aMGTOUnJysHj16KCIiQqmpqdcdl5WVpcjISPXt27eSKgUAAK7O1NCzYMECjR07VlFRUQoJCVFcXJwCAgK0ZMmS6457/PHHNXToUHXt2rWSKgUAAK7OtNBz+fJl7dmzR+Hh4U7t4eHh2rFjxzXHrVixQocOHdLMmTMrukQAAFCFuJu14czMTOXn58vX19ep3dfXVxkZGcWO+eGHHzR9+nRt27ZN7u4lKz03N1e5ubmO9ezs7LIXDQAAXJbpFzLbbDandcMwirRJUn5+voYOHarZs2erVatWJX7/efPmydvb27EEBATcdM0AAMD1mBZ6GjRooGrVqhU5qnPq1KkiR38kKScnR0lJSZowYYLc3d3l7u6uOXPm6KuvvpK7u7s+++yzYrcTGxurrKwsx5KWllYh+wMAAG5tpp3eql69ukJDQ5WYmKiHHnrI0Z6YmKgHHnigSH8vLy8dOHDAqW3x4sX67LPP9P777ysoKKjY7djtdtnt9vItHgAAuBzTQo8kxcTEaMSIEQoLC1PXrl31xhtvKDU1VePGjZN09SjNiRMn9Pbbb8vNzU1t27Z1Gt+oUSN5enoWaQcAAPgtU0PPkCFDdPr0ac2ZM0fp6elq27atNmzYoMDAQElSenr6DZ/ZAwAAUBKmhh5Jio6OVnR0dLGvxcfHX3fsrFmzNGvWrPIvCgAAVDmm370FAABQGQg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEgg9AADAEkwPPYsXL1ZQUJA8PT0VGhqqbdu2XbPv2rVr1a9fPzVs2FBeXl7q2rWrNm3aVInVAgAAV2Vq6ElISNDkyZM1Y8YMJScnq0ePHoqIiFBqamqx/bdu3ap+/fppw4YN2rNnj3r37q1BgwYpOTm5kisHAACuxtTQs2DBAo0dO1ZRUVEKCQlRXFycAgICtGTJkmL7x8XF6amnntJdd92lli1b6oUXXlDLli3173//u5IrBwAArsa00HP58mXt2bNH4eHhTu3h4eHasWNHid6joKBAOTk5ql+/fkWUCAAAqhB3szacmZmp/Px8+fr6OrX7+voqIyOjRO/x8ssv6/z58xo8ePA1++Tm5io3N9exnp2dXbaCAQCASzP9Qmabzea0bhhGkbbirF69WrNmzVJCQoIaNWp0zX7z5s2Tt7e3YwkICLjpmgEAgOsxLfQ0aNBA1apVK3JU59SpU0WO/vxWQkKCxo4dq//5n//RPffcc92+sbGxysrKcixpaWk3XTsAAHA9poWe6tWrKzQ0VImJiU7tiYmJ6tat2zXHrV69WqNGjdKqVas0YMCAG27HbrfLy8vLaQEAANZj2jU9khQTE6MRI0YoLCxMXbt21RtvvKHU1FSNGzdO0tWjNCdOnNDbb78t6WrgiYyM1CuvvKIuXbo4jhLVqFFD3t7epu0HAAC49ZkaeoYMGaLTp09rzpw5Sk9PV9u2bbVhwwYFBgZKktLT052e2fP666/rypUrGj9+vMaPH+9oHzlypOLj4yu7fAAA4EJMDT2SFB0drejo6GJf+22Q2bx5c8UXBAAAqiTT794CAACoDIQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCYQeAABgCaaHnsWLFysoKEienp4KDQ3Vtm3brtt/y5YtCg0Nlaenp1q0aKGlS5dWUqUAAMCVmRp6EhISNHnyZM2YMUPJycnq0aOHIiIilJqaWmz/I0eOqH///urRo4eSk5P19NNPa9KkSVqzZk0lVw4AAFyNqaFnwYIFGjt2rKKiohQSEqK4uDgFBARoyZIlxfZfunSpmjVrpri4OIWEhCgqKkpjxozR3/72t0quHAAAuBrTQs/ly5e1Z88ehYeHO7WHh4drx44dxY7ZuXNnkf733nuvkpKSlJeXV2G1AgAA1+du1oYzMzOVn58vX19fp3ZfX19lZGQUOyYjI6PY/leuXFFmZqb8/PyKjMnNzVVubq5jPSsrS5KUnZ19s7tQxLlz565uM+NHFVy+VO7vXxHyTqdJouaKRs2Vg5orjyvWTc2VI+/McUlXvxPL87u28L0Mwyj7mxgmOXHihCHJ2LFjh1P73LlzjeDg4GLHtGzZ0njhhRec2r744gtDkpGenl7smJkzZxqSWFhYWFhYWKrAkpaWVubsYdqRngYNGqhatWpFjuqcOnWqyNGcQo0bNy62v7u7u3x8fIodExsbq5iYGMd6QUGBzpw5Ix8fH9lstpvcC2fZ2dkKCAhQWlqavLy8yvW9UTzm3BzMuzmYd3Mw75WvuDk3DEM5OTny9/cv8/uaFnqqV6+u0NBQJSYm6qGHHnK0JyYm6oEHHih2TNeuXfXvf//bqe3jjz9WWFiYPDw8ih1jt9tlt9ud2urWrXtzxd+Al5cX/2FUMubcHMy7OZh3czDvle+3c+7t7X1T72fq3VsxMTF66623tHz5cqWkpGjKlClKTU3VuHHjJF09ShMZGenoP27cOB07dkwxMTFKSUnR8uXLtWzZMk2bNs2sXQAAAC7CtCM9kjRkyBCdPn1ac+bMUXp6utq2basNGzYoMDBQkpSenu70zJ6goCBt2LBBU6ZM0aJFi+Tv769XX31VjzzyiFm7AAAAXISpoUeSoqOjFR0dXexr8fHxRdp69uypvXv3VnBVZWO32zVz5swip9NQcZhzczDv5mDezcG8V76KmnObYdzMvV8AAACuwfTf3gIAAKgMhB4AAGAJhB4AAGAJhB4AAGAJhJ5SWLx4sYKCguTp6anQ0FBt27btuv23bNmi0NBQeXp6qkWLFlq6dGklVVq1lGbe165dq379+qlhw4by8vJS165dtWnTpkqstuoo7ee90Pbt2+Xu7q4OHTpUbIFVVGnnPTc3VzNmzFBgYKDsdrtuu+02LV++vJKqrRpKO+crV65U+/btVbNmTfn5+Wn06NE6ffp0JVVbNWzdulWDBg2Sv7+/bDabPvjggxuOKZfv1DL/gIXFvPvuu4aHh4fx5ptvGgcPHjSeeOIJo1atWsaxY8eK7X/48GGjZs2axhNPPGEcPHjQePPNNw0PDw/j/fffr+TKXVtp5/2JJ54wXnzxReM///mP8f333xuxsbGGh4eHsXfv3kqu3LWVdt4LnT171mjRooURHh5utG/fvnKKrULKMu/333+/0blzZyMxMdE4cuSI8eWXXxrbt2+vxKpdW2nnfNu2bYabm5vxyiuvGIcPHza2bdtm3HHHHcaDDz5YyZW7tg0bNhgzZsww1qxZY0gy1q1bd93+5fWdSugpoU6dOhnjxo1zamvdurUxffr0Yvs/9dRTRuvWrZ3aHn/8caNLly4VVmNVVNp5L06bNm2M2bNnl3dpVVpZ533IkCHGM888Y8ycOZPQUwalnff//d//Nby9vY3Tp09XRnlVUmnn/KWXXjJatGjh1Pbqq68aTZs2rbAaq7qShJ7y+k7l9FYJXL58WXv27FF4eLhTe3h4uHbs2FHsmJ07dxbpf++99yopKUl5eXkVVmtVUpZ5/62CggLl5OSofv36FVFilVTWeV+xYoUOHTqkmTNnVnSJVVJZ5n39+vUKCwvT/Pnz1aRJE7Vq1UrTpk3TxYsXK6Nkl1eWOe/WrZuOHz+uDRs2yDAM/fTTT3r//fc1YMCAyijZssrrO9X0JzK7gszMTOXn5xf59XdfX98iv/peKCMjo9j+V65cUWZmpvz8/Cqs3qqiLPP+Wy+//LLOnz+vwYMHV0SJVVJZ5v2HH37Q9OnTtW3bNrm789dKWZRl3g8fPqwvvvhCnp6eWrdunTIzMxUdHa0zZ85wXU8JlGXOu3XrppUrV2rIkCG6dOmSrly5ovvvv18LFy6sjJItq7y+UznSUwo2m81p3TCMIm036l9cO66vtPNeaPXq1Zo1a5YSEhLUqFGjiiqvyirpvOfn52vo0KGaPXu2WrVqVVnlVVml+bwXFBTIZrNp5cqV6tSpk/r3768FCxYoPj6eoz2lUJo5P3jwoCZNmqTnnntOe/bs0caNG3XkyBHHD2Wj4pTHdyr/S1YCDRo0ULVq1Yok/1OnThVJnoUaN25cbH93d3f5+PhUWK1VSVnmvVBCQoLGjh2r9957T/fcc09FllnllHbec3JylJSUpOTkZE2YMEHS1S9jwzDk7u6ujz/+WH369KmU2l1ZWT7vfn5+atKkiby9vR1tISEhMgxDx48fV8uWLSu0ZldXljmfN2+eunfvrieffFKS1K5dO9WqVUs9evTQ3LlzOYpfQcrrO5UjPSVQvXp1hYaGKjEx0ak9MTFR3bp1K3ZM165di/T/+OOPFRYWJg8PjwqrtSopy7xLV4/wjBo1SqtWreI8exmUdt69vLx04MAB7du3z7GMGzdOwcHB2rdvnzp37lxZpbu0snzeu3fvrpMnT+rcuXOOtu+//15ubm5q2rRphdZbFZRlzi9cuCA3N+evzmrVqkn6/yMPKH/l9p1aqsueLazwtsZly5YZBw8eNCZPnmzUqlXLOHr0qGEYhjF9+nRjxIgRjv6Ft9dNmTLFOHjwoLFs2TJuWS+D0s77qlWrDHd3d2PRokVGenq6Yzl79qxZu+CSSjvvv8XdW2VT2nnPyckxmjZtajz66KPGN998Y2zZssVo2bKlERUVZdYuuJzSzvmKFSsMd3d3Y/HixcahQ4eML774wggLCzM6depk1i64pJycHCM5OdlITk42JBkLFiwwkpOTHY8KqKjvVEJPKSxatMgIDAw0qlevbtx5553Gli1bHK+NHDnS6Nmzp1P/zZs3Gx07djSqV69uNG/e3FiyZEklV1w1lGbee/bsaUgqsowcObLyC3dxpf28/xqhp+xKO+8pKSnGPffcY9SoUcNo2rSpERMTY1y4cKGSq3ZtpZ3zV1991WjTpo1Ro0YNw8/Pzxg2bJhx/PjxSq7atX3++efX/bu6or5TbYbB8TgAAFD1cU0PAACwBEIPAACwBEIPAACwBEIPAACwBEIPAACwBEIPAACwBEIPAACwBEIPgCpn1qxZ6tChg9llALjFEHoAAIAlEHoAAIAlEHoAVKhevXppwoQJmjBhgurWrSsfHx8988wzxf4idVZWlmrUqKGNGzc6ta9du1a1atVy/Jr4f//3f6tVq1aqWbOmWrRooWeffVZ5eXnXrWHy5MlObQ8++KBGjRrlWL98+bKeeuopNWnSRLVq1VLnzp21efNmx+vHjh3ToEGDVK9ePdWqVUt33HGHNmzYUPoJAWAad7MLAFD1/eMf/9DYsWP15ZdfKikpSY899pgCAwP1pz/9yamft7e3BgwYoJUrV+q+++5ztK9atUoPPPCAateuLUmqU6eO4uPj5e/vrwMHDuhPf/qT6tSpo6eeeqrMNY4ePVpHjx7Vu+++K39/f61bt0733XefDhw4oJYtW2r8+PG6fPmytm7dqlq1aungwYOOegC4BkIPgAoXEBCgv//977LZbAoODtaBAwf097//vUjokaRhw4YpMjJSFy5cUM2aNZWdna2PPvpIa9ascfR55plnHP/evHlzTZ06VQkJCWUOPYcOHdLq1at1/Phx+fv7S5KmTZumjRs3asWKFXrhhReUmpqqRx55RL/73e8kSS1atCjTtgCYh9NbACpcly5dZLPZHOtdu3bVDz/8oOeff161a9d2LKmpqRowYIDc3d21fv16SdKaNWtUp04dhYeHO8a///77uvvuu9W4cWPVrl1bzz77rFJTU8tc3969e2UYhlq1auVUz5YtW3To0CFJ0qRJkzR37lx1795dM2fO1P79+8u8PQDmIPQAMM24ceO0b98+x+Lv76/q1avr0Ucf1apVqyRdPbU1ZMgQubtfPTC9a9cu/eEPf1BERIQ+/PBDJScna8aMGbp8+fI1t+Pm5lbkGqJfXwNUUFCgatWqac+ePU71pKSk6JVXXpEkRUVF6fDhwxoxYoQOHDigsLAwLVy4sLynBEAF4vQWgAq3a9euIustW7aUj4+PfHx8ivQfNmyYwsPD9c033+jzzz/XX/7yF8dr27dvV2BgoGbMmOFoO3bs2HW337BhQ6WnpzvW8/Pz9fXXX6t3796SpI4dOyo/P1+nTp1Sjx49rvk+AQEBGjdunMaNG6fY2Fi9+eabmjhx4vV3HsAtgyM9ACpcWlqaYmJi9N1332n16tVauHChnnjiiWv279mzp3x9fTVs2DA1b95cXbp0cbx2++23KzU1Ve+++64OHTqkV199VevWrbvu9vv06aOPPvpIH330kb799ltFR0fr7NmzjtdbtWrluJZo7dq1OnLkiHbv3q0XX3zRcYfW5MmTtWnTJh05ckR79+7VZ599ppCQkJubGACVitADoMJFRkbq4sWL6tSpk8aPH6+JEyfqscceu2Z/m82mP/7xj/rqq680bNgwp9ceeOABTZkyRRMmTFCHDh20Y8cOPfvss9fd/pgxYzRy5EhFRkaqZ8+eCgoKchzlKbRixQpFRkZq6tSpCg4O1v33368vv/xSAQEBkq4eHRo/frxCQkJ03333KTg4WIsXLy7jjAAwg80o7mEZAFBOevXqpQ4dOiguLs7sUgBYHEd6AACAJRB6AACAJXB6CwAAWAJHegAAgCUQegAAgCUQegAAgCUQegAAgCUQegAAgCUQegAAgCUQegAAgCUQegAAgCUQegAAgCX8H2RX+8vAqyEyAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "kwargs = dict(histtype='bar', density=True, ec=\"k\")\n",
    "plt.hist(results,**kwargs)\n",
    "plt.title('Histogram of p-values')\n",
    "plt.xlabel('p-values')\n",
    "plt.ylabel('Count');"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "24bf2fb9",
   "metadata": {},
   "source": [
    "The histogram show a relatively flat distribution. This stems from the fact that all null hypotheses are true.<br>\n",
    "It also invokes that controlling Type I error for each null hypothesis using a unified significance level, as prescribed in the following question, is a bad idea. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60bb8452",
   "metadata": {},
   "source": [
    "### (b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1777c19a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(results<0.05).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e71d97c5",
   "metadata": {},
   "source": [
    "We reject 4 null hypotheses."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84b0fbae",
   "metadata": {},
   "source": [
    "### (c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7aaef697",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False, False, False, False, False, False, False, False, False,\n",
       "       False, False, False, False, False, False, False, False, False,\n",
       "       False, False, False, False, False, False, False, False, False,\n",
       "       False, False, False, False, False, False, False, False, False,\n",
       "       False, False, False, False, False, False, False, False, False,\n",
       "       False, False, False, False, False, False, False, False, False,\n",
       "       False, False, False, False, False, False, False, False, False,\n",
       "       False, False, False, False, False, False, False, False, False,\n",
       "       False, False, False, False, False, False, False, False, False,\n",
       "       False, False, False, False, False, False, False, False, False,\n",
       "       False, False, False, False, False, False, False, False, False,\n",
       "       False])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reject, holm = mult_test(results, method = \"holm\")[:2]\n",
    "reject"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f02bc91c",
   "metadata": {},
   "source": [
    "By using Holm step down method, we have failed to reject all null hypotheses."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "69a011c9",
   "metadata": {},
   "source": [
    "### (d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f8d489ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False, False, False, False, False, False, False, False, False,\n",
       "       False, False, False, False, False, False, False, False, False,\n",
       "       False, False, False, False, False, False, False, False, False,\n",
       "       False, False, False, False, False, False, False, False, False,\n",
       "       False, False, False, False, False, False, False, False, False,\n",
       "       False, False, False, False, False, False, False, False, False,\n",
       "       False, False, False, False, False, False, False, False, False,\n",
       "       False, False, False, False, False, False, False, False, False,\n",
       "       False, False, False, False, False, False, False, False, False,\n",
       "       False, False, False, False, False, False, False, False, False,\n",
       "       False, False, False, False, False, False, False, False, False,\n",
       "       False])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reject_fdr=mult_test(results, method = \"fdr_bh\")[0]\n",
    "reject_fdr"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d512ad7d",
   "metadata": {},
   "source": [
    "As the previous question, we have failed to reject all null hypotheses."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "576417c5",
   "metadata": {},
   "source": [
    "### (e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "03f235af",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Picking the 10 managers who perform well. We consider that performant managers are those whose \n",
    "# whose returns are not due to chance; i.e: those with the smallest p-values \n",
    "performance=[]\n",
    "for i in range(X.shape[1]):\n",
    "    performance.append((results[i].sum(),i))\n",
    "performance.sort(key=lambda x: x[0], reverse=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ddcee70c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Top 10 fund managers\n",
    "top_10=performance[:10]\n",
    "top_10_pvals=[]\n",
    "for i in range(10):\n",
    "    top_10_pvals.append(results[top_10[i][1]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "dcbed434",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.0008066133772248244, 14),\n",
       " (0.009551210237857455, 44),\n",
       " (0.022933282067325957, 39),\n",
       " (0.035539479614030645, 27),\n",
       " (0.05026563447042849, 72),\n",
       " (0.05388012638125132, 0),\n",
       " (0.06564566802881348, 50),\n",
       " (0.07550062589870506, 20),\n",
       " (0.07795962307812401, 9),\n",
       " (0.0879382731757003, 35)]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "03da3ef6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ True, False, False, False, False, False, False, False, False,\n",
       "        False]),\n",
       " array([0.00806613, 0.08596089, 0.18346626, 0.24877636, 0.30159381,\n",
       "        0.30159381, 0.30159381, 0.30159381, 0.30159381, 0.30159381]))"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Controlling FWER of the top 10 managers\n",
    "reject_cp= mult_test(top_10_pvals, method = \"holm\")[:2]\n",
    "reject_cp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "18d6fd1d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ True,  True, False, False, False, False, False, False, False,\n",
       "        False]),\n",
       " array([0.00806613, 0.04775605, 0.07644427, 0.0866218 , 0.0866218 ,\n",
       "        0.0866218 , 0.0866218 , 0.0866218 , 0.0866218 , 0.08793827]))"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Controlling FDR of the top 10 managers\n",
    "reject_cp_fdr= mult_test(top_10_pvals, method = \"fdr_bh\")[:2]\n",
    "reject_cp_fdr"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7676cdf4",
   "metadata": {},
   "source": [
    "We reject the null hypothesis corresponding to manager no 14 (Indexes start from 0 to 99), when controlling each of FWER and FDR."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "65848154",
   "metadata": {},
   "source": [
    "### (f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "86b24484",
   "metadata": {},
   "source": [
    "The approach of \"cherry-picking\" when controlling FDR or FWER is misleading.<br><br>\n",
    "This is because control algorithms (e.g: Holm's step down method or Benjamini-Hochberg) depend on the number of null hypotheses to determine the level of significance.<br><br>\n",
    "When we cherry-pick we tend to only show what we want, which is in fact a dangerous cognitive bias. For example, here we have falsely showed that the best performant manager owes his achieved returns only to chance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f818b080",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}