cvaccents-v9.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "747d78b5",
   "metadata": {},
   "source": [
    "# Get demographic details from an MCV dataset \n",
    "\n",
    "The purpose of this script is to get demographic details from an MCV downloaded dataset. \n",
    "This informs decision making around, for example, how much of the data in a particular language, has demographic details, and if so, what they are. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9e2024b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports go here \n",
    "\n",
    "# io \n",
    "import io\n",
    "\n",
    "# pandas \n",
    "import pandas as pd\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fd75b9e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# specify the path to the TSV file - this should be `validated.tsv` from the MCV download \n",
    "#filePath = '/media/kathyreid/Seagate Backup Plus Drive/cv-datasets/ru/validated.tsv'\n",
    "#filePath = '/media/kathyreid/Seagate Backup Plus Drive/cv-datasets/fr/validated.tsv'\n",
    "#filePath = '/media/kathyreid/Elements/de/validated.tsv'\n",
    "#filePath = '/media/kathyreid/Elements/es/validated.tsv'\n",
    "#filePath = '/media/kathyreid/Elements/en/validated.tsv'\n",
    "filePath = '/media/kathyreid/Seagate Backup Plus Drive/cv-datasets/en-v9/validated.tsv'\n",
    "\n",
    "# put it into a DataFrame \n",
    "df = pd.read_csv(filePath, sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e810b905",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method DataFrame.value_counts of                                                  client_id  \\\n",
       "0        000abb3006b78ea4c1144e55d9d158f05a9db011016051...   \n",
       "1        0013037a1d45cc33460806cc3f8ecee9d536c45639ba4c...   \n",
       "2        0014c5a3e5715a54855257779b89c2bb498d470b225866...   \n",
       "3        001509f4624a7dee75247f6a8b642c4a0d09f8be3eeea6...   \n",
       "4        001519f234e04528a2b36158c205dbe61c8da45ab0242f...   \n",
       "...                                                    ...   \n",
       "1556249  372293e65cdab88771e028a4351651ab2eff64438ddafc...   \n",
       "1556250  372293e65cdab88771e028a4351651ab2eff64438ddafc...   \n",
       "1556251  372293e65cdab88771e028a4351651ab2eff64438ddafc...   \n",
       "1556252  372293e65cdab88771e028a4351651ab2eff64438ddafc...   \n",
       "1556253  372293e65cdab88771e028a4351651ab2eff64438ddafc...   \n",
       "\n",
       "                                 path  \\\n",
       "0        common_voice_en_27710027.mp3   \n",
       "1          common_voice_en_699711.mp3   \n",
       "2        common_voice_en_21953345.mp3   \n",
       "3        common_voice_en_18132047.mp3   \n",
       "4        common_voice_en_27340672.mp3   \n",
       "...                               ...   \n",
       "1556249  common_voice_en_31747485.mp3   \n",
       "1556250  common_voice_en_31747533.mp3   \n",
       "1556251  common_voice_en_31747636.mp3   \n",
       "1556252  common_voice_en_31747771.mp3   \n",
       "1556253  common_voice_en_31747838.mp3   \n",
       "\n",
       "                                                  sentence  up_votes  \\\n",
       "0        Joe Keaton disapproved of films, and Buster al...         3   \n",
       "1                                     She'll be all right.         2   \n",
       "2                                                      six         3   \n",
       "3                               All's well that ends well.         2   \n",
       "4        It is a busy market town that serves a large s...         2   \n",
       "...                                                    ...       ...   \n",
       "1556249  According to The Guardian he lives in the Caym...         2   \n",
       "1556250  It is spatially distributed as either scattere...         2   \n",
       "1556251                 The exhibition had two main parts.         2   \n",
       "1556252                 The story is told in two segments.         2   \n",
       "1556253  Credits are adapted from Tidal and Tower Recor...         2   \n",
       "\n",
       "         down_votes      age gender                            accents locale  \\\n",
       "0                 1      NaN    NaN                                NaN     en   \n",
       "1                 1      NaN    NaN                                NaN     en   \n",
       "2                 2      NaN    NaN                                NaN     en   \n",
       "3                 0      NaN    NaN                                NaN     en   \n",
       "4                 0      NaN    NaN                                NaN     en   \n",
       "...             ...      ...    ...                                ...    ...   \n",
       "1556249           0  fifties   male  German English,Non native speaker     en   \n",
       "1556250           0  fifties   male  German English,Non native speaker     en   \n",
       "1556251           0  fifties   male  German English,Non native speaker     en   \n",
       "1556252           0  fifties   male  German English,Non native speaker     en   \n",
       "1556253           0  fifties   male  German English,Non native speaker     en   \n",
       "\n",
       "           segment  \n",
       "0              NaN  \n",
       "1              NaN  \n",
       "2        Benchmark  \n",
       "3              NaN  \n",
       "4              NaN  \n",
       "...            ...  \n",
       "1556249        NaN  \n",
       "1556250        NaN  \n",
       "1556251        NaN  \n",
       "1556252        NaN  \n",
       "1556253        NaN  \n",
       "\n",
       "[1556254 rows x 10 columns]>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# summary data \n",
    "df.value_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "74e3fefb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "69656"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# unique contributors \n",
    "len(df['client_id'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f8e90d46",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "999855"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# rows that have metadata \n",
    "len(df[df['age'].notna()])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4156a03e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, 'twenties', 'thirties', 'fourties', 'seventies', 'teens',\n",
       "       'sixties', 'fifties', 'eighties', 'nineties'], dtype=object)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get all the age ranges \n",
    "df['age'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "9769ef76",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "teens:  83073\n",
      "twenties:  337944\n",
      "thirties:  191258\n",
      "fourties:  139187\n",
      "fifties:  72327\n",
      "sixties:  70423\n",
      "seventies:  12364\n",
      "eighties:  1120\n",
      "nineties:  95\n",
      "NaN:  517993\n"
     ]
    }
   ],
   "source": [
    "# age ranges\n",
    "\n",
    "print('teens: ', len(df.loc[df['age'] == 'teens']))\n",
    "print('twenties: ', len(df.loc[df['age'] == 'twenties']))\n",
    "print('thirties: ', len(df.loc[df['age'] == 'thirties']))\n",
    "print('fourties: ', len(df.loc[df['age'] == 'fourties']))\n",
    "print('fifties: ', len(df.loc[df['age'] == 'fifties']))\n",
    "print('sixties: ', len(df.loc[df['age'] == 'sixties']))\n",
    "print('seventies: ', len(df.loc[df['age'] == 'seventies']))\n",
    "print('eighties: ', len(df.loc[df['age'] == 'eighties']))\n",
    "print('nineties: ', len(df.loc[df['age'] == 'nineties']))\n",
    "\n",
    "print('NaN: ', len(df.loc[df['age'].isna()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5e8954f6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, 'male', 'female', 'other'], dtype=object)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get the genders \n",
    "\n",
    "df['gender'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "9ee23ac9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "female:  216797\n",
      "male:  666819\n",
      "other:  27243\n",
      "NaN:  514925\n"
     ]
    }
   ],
   "source": [
    "# genders\n",
    "\n",
    "print('female: ', len(df.loc[df['gender'] == 'female']))\n",
    "print('male: ', len(df.loc[df['gender'] == 'male']))\n",
    "print('other: ', len(df.loc[df['gender'] == 'other']))\n",
    "\n",
    "print('NaN: ', len(df.loc[df['gender'].isna()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "59f0583c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, 'England English,United States English', 'Hong Kong English',\n",
       "       'England English', 'United States English',\n",
       "       'United States English,wolof', 'Australian English',\n",
       "       'Southern African (South Africa, Zimbabwe, Namibia)',\n",
       "       'India and South Asia (India, Pakistan, Sri Lanka)',\n",
       "       'England English,Hong Kong English',\n",
       "       'India and South Asia (India, Pakistan, Sri Lanka),Basic',\n",
       "       'East London ', 'Canadian English', 'Eastern European English',\n",
       "       'German', 'Scottish English', 'Filipino',\n",
       "       'England English,yorkshire', 'Singaporean English',\n",
       "       'United States English,England English',\n",
       "       'United States English,Variable',\n",
       "       'West Indies and Bermuda (Bahamas, Bermuda, Jamaica, Trinidad)',\n",
       "       'New Zealand English', 'Malaysian English', 'Slavic',\n",
       "       'United States English,England English,Hong Kong English',\n",
       "       'Irish English',\n",
       "       'India and South Asia (India, Pakistan, Sri Lanka),United States English',\n",
       "       'United States English,India and South Asia (India, Pakistan, Sri Lanka)',\n",
       "       'India and South Asia (India, Pakistan, Sri Lanka),Indian with a tinge of an RP accent',\n",
       "       'Catalan', 'United States English,southern United States',\n",
       "       \"Israeli's accent \",\n",
       "       'United States English,Southern Appalachian English,Southern United States English',\n",
       "       'England English,London English', 'Bangladeshi,Bangladesh English',\n",
       "       'Southern Texas Accent,United States English', 'polish accent',\n",
       "       'Russian',\n",
       "       'United States English,United States English. people say I sound like a surffer dude.',\n",
       "       'Colombian Accent',\n",
       "       'United States English,little bit classy little bit sassy and add some city.....thats me',\n",
       "       'New Zealand English,United States English', 'Welsh English',\n",
       "       'Norwegian', 'Hong Kong English,Scottish English',\n",
       "       'Japanese English', 'United States English,Canadian English',\n",
       "       'United States English,Hmong-American',\n",
       "       'United States English,Australian English,England English,New Zealand English,India and South Asia (India, Pakistan, Sri Lanka),international,Slovak',\n",
       "       'United States English,Australian English',\n",
       "       'South Atlantic (Falkland Islands, Saint Helena)', '2nd Language ',\n",
       "       'United States English,midwest',\n",
       "       \"England English,Porto des dels 3 anys aprenent anglès a l'escola i actualment m'estic preparant per a l'examen del B2.\",\n",
       "       'Slavic,Russian', 'United States English,Cool',\n",
       "       'Singaporean English,England English', 'German Accent',\n",
       "       'German English', 'fluent,ESL,European',\n",
       "       'A variety of Texan English with some German influence that has undergone the cot-caught merger',\n",
       "       'serbian,United States English', 'Bangladeshi English',\n",
       "       'Dutch English', 'Canadian English,Irish English',\n",
       "       'Australian English,England English',\n",
       "       'Singaporean English,Malaysian English',\n",
       "       'West Indies and Bermuda (Bahamas, Bermuda, Jamaica, Trinidad),England English,Scottish English,United States English',\n",
       "       'England English,Canadian English', 'polish',\n",
       "       'England English,south German / Swiss accent', 'Nigerian English',\n",
       "       'Midatlantic,England English',\n",
       "       'United States English,Californian Accent', 'minor french accent',\n",
       "       'bangladesh,India and South Asia (India, Pakistan, Sri Lanka)',\n",
       "       'Ukrainian',\n",
       "       'United States English,Some time spent in Scotland, mostly affecting inflection, I think',\n",
       "       'Austrian', 'United States English,Midwestern States (Michigan)',\n",
       "       'United States English,Upper Midwestern',\n",
       "       'England English,Northern English', 'Spanish,Foreign,Non-native',\n",
       "       'United States English,Pacific Northwest ',\n",
       "       'Javanese,Indonesian English,Malaysian English',\n",
       "       'European,German,Foreign,Non-native',\n",
       "       'England English,southern english,sussex',\n",
       "       'southern United States,United States English',\n",
       "       'New Zealand English,England English', 'German,south-west German',\n",
       "       'United States English,Southern Californian', 'Dutch',\n",
       "       'England English,South London', 'Northern England,England English',\n",
       "       'United States English,Midwestern,Minnesotan',\n",
       "       'Northumbrian British English',\n",
       "       'England English,New Zealand English', 'Northern Irish',\n",
       "       'German English,Non native speaker'], dtype=object)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get the accents \n",
    "\n",
    "# df['accent'].unique()\n",
    "# in CV9, this key changed to `accents`\n",
    "df['accents'].unique()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "403ef647",
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "nunique() got an unexpected keyword argument 'axis'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-19-583afc45ec1f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accents'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnunique\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m: nunique() got an unexpected keyword argument 'axis'"
     ]
    }
   ],
   "source": [
    "df['accents'].nunique(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "577bb6d9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nan :  0\n",
      "canada :  8682\n",
      "france :  347436\n",
      "belgium :  9679\n",
      "cote_d_ivoire :  104\n",
      "senegal :  41\n",
      "algeria :  354\n",
      "burundi :  1\n",
      "united_kingdom :  259\n",
      "cameroon :  56\n",
      "united_states :  751\n",
      "reunion :  1053\n",
      "germany :  274\n",
      "romania :  121\n",
      "morocco :  59\n",
      "tunisia :  16\n",
      "switzerland :  3989\n",
      "martinique :  68\n",
      "other :  43\n",
      "guadeloupe :  133\n",
      "new_caledonia :  144\n",
      "benin :  996\n",
      "congo_brazzaville :  5\n",
      "monaco :  109\n",
      "gabon :  5\n",
      "luxembourg :  14\n",
      "st_pierre_et_miquelon :  6\n",
      "mayotte :  6\n",
      "italy :  57\n",
      "congo_kinshasa :  10\n",
      "ireland :  14\n",
      "haiti :  34\n",
      "madagascar :  162\n",
      "portugal :  17\n",
      "netherlands :  81\n",
      "french_guiana :  115\n",
      "NaN:  167394\n"
     ]
    }
   ],
   "source": [
    "# french accents \n",
    "\n",
    "french_accents = df['accent'].unique()\n",
    "for accent in french_accents:\n",
    "    accent_total = len(df.loc[df['accent'] == accent])\n",
    "    print(accent, ': ', accent_total)\n",
    "\n",
    "print('NaN: ', len(df.loc[df['accent'].isna()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "76b2343b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nan :  0\n",
      "russia :  940\n",
      "germany :  437712\n",
      "france :  1405\n",
      "switzerland :  8602\n",
      "austria :  20659\n",
      "bulgaria :  1\n",
      "netherlands :  75\n",
      "denmark :  1\n",
      "poland :  103\n",
      "turkey :  24\n",
      "united_kingdom :  148\n",
      "czechia :  37\n",
      "united_states :  268\n",
      "greece :  120\n",
      "hungary :  151\n",
      "other :  184\n",
      "belgium :  9\n",
      "slovakia :  62\n",
      "lithuania :  5\n",
      "luxembourg :  57\n",
      "canada :  98\n",
      "liechtenstein :  62\n",
      "slovenia :  10\n",
      "brazil :  12\n",
      "italy :  978\n",
      "finland :  31\n",
      "NaN:  213040\n"
     ]
    }
   ],
   "source": [
    "# german accents \n",
    "\n",
    "german_accents = df['accent'].unique()\n",
    "for accent in german_accents:\n",
    "    accent_total = len(df.loc[df['accent'] == accent])\n",
    "    print(accent, ': ', accent_total)\n",
    "\n",
    "print('NaN: ', len(df.loc[df['accent'].isna()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "1a83a43e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mexicano :  16924\n",
      "nan :  0\n",
      "americacentral :  5532\n",
      "andino :  13729\n",
      "caribe :  8235\n",
      "centrosurpeninsular :  8713\n",
      "rioplatense :  12102\n",
      "chileno :  5316\n",
      "surpeninsular :  32622\n",
      "nortepeninsular :  35345\n",
      "canario :  952\n",
      "filipinas :  342\n",
      "NaN:  131198\n"
     ]
    }
   ],
   "source": [
    "# spanish accents \n",
    "\n",
    "spanish_accents = df['accent'].unique()\n",
    "for accent in spanish_accents:\n",
    "    accent_total = len(df.loc[df['accent'] == accent])\n",
    "    print(accent, ': ', accent_total)\n",
    "\n",
    "print('NaN: ', len(df.loc[df['accent'].isna()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "f39044ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nan :  0\n",
      "hongkong :  2750\n",
      "us :  351472\n",
      "england :  118401\n",
      "african :  8066\n",
      "indian :  73030\n",
      "other :  10505\n",
      "australia :  46951\n",
      "canada :  48453\n",
      "scotland :  12676\n",
      "philippines :  4158\n",
      "singapore :  2967\n",
      "bermuda :  643\n",
      "newzealand :  11281\n",
      "malaysia :  1685\n",
      "ireland :  9233\n",
      "wales :  1550\n",
      "southatlandtic :  203\n",
      "NaN:  721760\n"
     ]
    }
   ],
   "source": [
    "# english accents \n",
    "\n",
    "english_accents = df['accent'].unique()\n",
    "for accent in english_accents:\n",
    "    accent_total = len(df.loc[df['accent'] == accent])\n",
    "    print(accent, ': ', accent_total)\n",
    "\n",
    "print('NaN: ', len(df.loc[df['accent'].isna()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "286af20f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}