OpenSubtitles.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Open Subtitles"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Acquiring the Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "\n",
    "cd /home/datasets/srl/opensubtitles/\n",
    "mkdir dataset\n",
    "cd dataset\n",
    "\n",
    "wget http://opus.lingfil.uu.se/OpenSubtitles2016/en-he.txt.zip\n",
    "wget http://opus.lingfil.uu.se/OpenSubtitles2016/en.raw.tar.gz\n",
    "wget http://opus.lingfil.uu.se/OpenSubtitles2016/he.raw.tar.gz\n",
    "\n",
    "unzip en-he.txt.zip\n",
    "tar -xzvf en.raw.tar.gz\n",
    "tar -xzvf he.raw.tar.gz\n",
    "\n",
    "rm -f en-he.txt.zip en.raw.tar.gz he.raw.tar.gz"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove symmetric difference between English and Hebrew"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%python\n",
    "\n",
    "import os\n",
    "\n",
    "from collections import defaultdict\n",
    "\n",
    "os.chdir('./OpenSubtitles2016/raw/')\n",
    "\n",
    "L = defaultdict(set)\n",
    "with open('../../OpenSubtitles2016.en-he.ids', encoding='utf-8') as f:\n",
    "    for line in f:\n",
    "        en, he, _, _ = line.strip().split('\\t')\n",
    "        L['en'].add(en)\n",
    "        L['he'].add(he)\n",
    "\n",
    "for lang in ('en', 'he'):\n",
    "    for dirpath, dirnames, filenames in os.walk(lang):\n",
    "        for f in filenames:\n",
    "            if os.path.join(dirpath, f) not in L[lang]:\n",
    "                os.remove(os.path.join(dirpath, f))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove empty directories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "\n",
    "find OpenSubtitles2016/raw -depth -empty -delete"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import gzip\n",
    "import json\n",
    "import os\n",
    "import pickle\n",
    "import regex as re\n",
    "\n",
    "from collections import Counter, defaultdict, namedtuple\n",
    "from functools import partial\n",
    "from glob import glob\n",
    "from multiprocessing import cpu_count\n",
    "from operator import attrgetter, itemgetter\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import spacy\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "from cytoolz import compose\n",
    "from joblib import Parallel, delayed\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "ROOT = r'/home/datasets/srl/opensubtitles/'\n",
    "\n",
    "DATASET = os.path.join(ROOT, 'dataset')\n",
    "ARTEFACTS = os.path.join(ROOT, 'artefacts')\n",
    "XML_PAIRS = os.path.join(ARTEFACTS, 'en2he.json')\n",
    "ID_ALIGNMENTS = os.path.join(ARTEFACTS, 'alignments.pkl')\n",
    "ALIGNED_SENTS = os.path.join(ARTEFACTS, '01_aligned_sents')\n",
    "ENGLISH_SENTS = os.path.join(ARTEFACTS, '02_english_sents')\n",
    "TOKENIZED_SENTS = os.path.join(ARTEFACTS, '02_tokenized_sents')\n",
    "YAP_INPUTS = os.path.join(ARTEFACTS, '03_hebrew_processing', 'inputs')\n",
    "YAP_MA = os.path.join(ARTEFACTS, '03_hebrew_processing', 'ma')\n",
    "YAP_MD = os.path.join(ARTEFACTS, '03_hebrew_processing', 'md')\n",
    "YAP_OUTPUTS = os.path.join(ARTEFACTS, '03_hebrew_processing', 'outputs')\n",
    "PREPROCESSED_SENTS = os.path.join(ARTEFACTS, '04_preprocessed_sents')\n",
    "FASTALIGN_INPUTS = os.path.join(ARTEFACTS, '05_aligner_inputs', 'fastalign')\n",
    "EFLOMAL_INPUTS = os.path.join(ARTEFACTS, '05_aligner_inputs', 'eflomal')\n",
    "EFMARAL_INPUTS = os.path.join(ARTEFACTS, '05_aligner_inputs', 'efmaral')\n",
    "ALIGNERS_OUTPUT = os.path.join(ARTEFACTS, '06_aligners_output')\n",
    "ALIGNERS_ALIGNMENT = os.path.join(ARTEFACTS, '06_aligners_output_words')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing\n",
    "\n",
    "We begin by using the ID alignments file `OpenSubtitles2016.en-he.ids`.\n",
    "\n",
    "The file is formatted as `<English file>\\t<Hebrew file>\\t<English sentence IDs>\\t<Hebrew sentence IDs>`. We use this data to create a mapping from English files to Hebrew files, and a mapping from English files to pairs of English-Hebrew IDs. In this dataset, the mapping is always one-to-many or many-to-one, so there are no cases where several sentences in English are aligned to several sentences in Hebrew."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {},
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "def create_en_he_alignment_ids(ids_file):\n",
    "    en2he = {}\n",
    "    alignments = defaultdict(list)\n",
    "    with open(ids_file, encoding='utf-8') as ids:\n",
    "        for line in tqdm(ids):\n",
    "            en, he, en_sents, he_sents = line.strip().split('\\t')\n",
    "            en_split = [int(x) for x in en_sents.split()]\n",
    "            he_split = [int(x) for x in he_sents.split()]\n",
    "            alignments[en].append((en_split, he_split))\n",
    "            en2he[en] = he\n",
    "    with open(XML_PAIRS, 'w', encoding='utf-8') as f:\n",
    "        json.dump(en2he, f, ensure_ascii=False, indent=4)\n",
    "    with open(ID_ALIGNMENTS, 'wb') as f:\n",
    "        pickle.dump(alignments, f, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    del en2he, alignments\n",
    "\n",
    "create_en_he_alignment_ids(os.path.join(DATASET, 'OpenSubtitles2016.en-he.ids'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Each subtitle file is an XML file where each `s` tag has one sentence, and has an `id` attribute.\n",
    "\n",
    "Upon establishing which files are subtitles for the same movie, we replace the IDs with the corresponding sentence. At this point, only three problematic characters are removed during this step: newlines and tabs, since there's no reason to have those in the subtitles, and the dash (`-`) character, which is used very often to signify a different person is talking."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def read_id_alignments_from_file(en2he_file, alignments_file):\n",
    "    with open(en2he_file, encoding='utf-8') as f:\n",
    "        en2he = json.load(f)\n",
    "    with open(alignments_file, 'rb') as f:\n",
    "        alignments = pickle.load(f)\n",
    "    return en2he, alignments\n",
    "\n",
    "\n",
    "def id2sent(xml_root):\n",
    "    remove_symbols = partial(re.sub, r'[\\n\\t-]', '')\n",
    "    return {int(s['id']): remove_symbols(s.text).strip()\n",
    "            for s in BeautifulSoup(xml_root, \"xml\").find_all('s')}\n",
    "\n",
    "\n",
    "def create_sentence_pairs(en_xml, he_xml, alignment):\n",
    "    path = os.path.join(DATASET, 'OpenSubtitles2016', 'raw')    \n",
    "    with gzip.open(os.path.join(path, en_xml)) as e, gzip.open(os.path.join(path, he_xml)) as h:\n",
    "        en_sents = id2sent(e)\n",
    "        he_sents = id2sent(h)\n",
    "    subs = []\n",
    "    for en_ids, he_ids in alignment:\n",
    "        subs.append({\n",
    "            'en': ' '.join(en_sents[i] for i in en_ids).strip(),\n",
    "            'he': ' '.join(he_sents[i] for i in he_ids).strip()\n",
    "        })\n",
    "    new_file = '_'.join(en_xml[:en_xml.index('.')].split('/')) + '.json'\n",
    "    with open(os.path.join(ALIGNED_SENTS, new_file), 'w', encoding='utf-8') as f:\n",
    "        json.dump(subs, f, ensure_ascii=False, indent=4)\n",
    "    del en_sents, he_sents, subs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {},
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "en2he, alignments = read_id_alignments_from_file(XML_PAIRS, ID_ALIGNMENTS)\n",
    "Parallel(n_jobs=6)(delayed(create_sentence_pairs)(en_xml, he_xml, alignments[en_xml]) for en_xml, he_xml in tqdm(en2he.items()))\n",
    "del en2he, alignments"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenization, POS Tagging, Dependency Parsing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_english_sents(sub):\n",
    "    with open(os.path.join(ALIGNED_SENTS, sub), encoding='utf-8') as f:\n",
    "        sents = json.load(f)\n",
    "    with open(os.path.join(ENGLISH_SENTS, sub[:sub.index('.')] + '.txt'), 'w', encoding='utf-8') as f:\n",
    "        for sent in sents:\n",
    "            en_sent = sent['en']\n",
    "            if en_sent:\n",
    "                print(en_sent, file=f)\n",
    "\n",
    "\n",
    "Parallel(n_jobs=cpu_count())(delayed(extract_english_sents)(sub) for sub in tqdm(os.listdir(ALIGNED_SENTS)));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[93m    Warning: no model found for 'he'\u001b[0m\n",
      "\n",
      "    Only loading the 'he' tokenizer.\n",
      "\n"
     ]
    },
    {
     "data": {},
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "SPACY_KWARGS = {n_threads: cpu_count()}\n",
    "TOKEN_FIELDS = ('id', 'text', 'pos', 'iob', 'head', 'deprel')\n",
    "\n",
    "normalize_spaces = compose(' '.join, str.split)\n",
    "he_pipeline = spacy.load('he')\n",
    "en_pipeline = spacy.load('en_core_web_md')\n",
    "\n",
    "get_attrs_of = attrgetter('i', 'text', 'pos_', 'ent_iob_', 'head.i', 'dep_')\n",
    "\n",
    "def doc_to_dict(doc):\n",
    "    return [dict(zip(TOKEN_FIELDS, get_attrs_of(t))) for t in doc]\n",
    "\n",
    "def extract_subtitle_to_json(sub):\n",
    "    with open(os.path.join(ALIGNED_SENTS, sub), encoding='utf-8') as f:\n",
    "        sents = json.load(f)\n",
    "    sub_tuples = [(sent['en'], sent['he']) for sent in sents]\n",
    "    sub_tuples = ((normalize_spaces(en), normalize_spaces(he)) for en, he in sub_tuples if len(en) > 0 and len(he) > 0)\n",
    "    ens, hes = zip(*sub_tuples)\n",
    "    del sub_tuples\n",
    "    en_tokens = map(doc_to_dict, en_pipeline.pipe(ens, **SPACY_KWARGS))\n",
    "    he_tokens = map(doc_to_dict, he_pipeline.pipe(hes, **SPACY_KWARGS))\n",
    "    del ens, hes\n",
    "    with open(os.path.join(TOKENIZED_SENTS, sub), 'w', encoding='utf-8') as t:\n",
    "        json.dump({\n",
    "            'file': sub,\n",
    "            'en': list(en_tokens),\n",
    "            'he': list(he_tokens)\n",
    "        }, t, ensure_ascii=False, indent=4)\n",
    "    del en_tokens, he_tokens\n",
    "\n",
    "Parallel(n_jobs=cpu_count())(delayed(extract_subtitle_to_json)(sub) for sub in tqdm(os.listdir(ALIGNED_SENTS)));\n",
    "del en_pipeline, he_pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hebrew Processing\n",
    "\n",
    "We use [YAP](https://github.com/habeanf/yap) for Hebrew morphological analysis, disambiguation, and dependency parsing. First, we create input files in the format expected by YAP."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {},
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "def create_input_for_yap(sub):\n",
    "    with open(os.path.join(TOKENIZED_SENTS, sub), encoding='utf-8') as fsub:\n",
    "        hebrew_subs = json.load(fsub)['he']\n",
    "    with open(os.path.join(YAP_INPUTS, sub[:sub.index('.json')]), 'w', encoding='utf-8') as fyap:\n",
    "        for line in hebrew_subs:\n",
    "            print(*[t['text'] for t in line], sep='\\n', end='\\n\\n', file=fyap)\n",
    "\n",
    "Parallel(n_jobs=cpu_count())(delayed(create_input_for_yap)(sub) for sub in tqdm(os.listdir(TOKENIZED_SENTS)));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then we run YAP, saving all intermediate results, just in case.\n",
    "\n",
    "**Note**: This took one month on a 24-core computer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%file /home/datasets/srl/opensubtitles/artefacts/step2/run_yap.py\n",
    "#!/usr/bin/env python\n",
    "\n",
    "import os\n",
    "import subprocess\n",
    "\n",
    "from joblib import delayed, Parallel\n",
    "from tqdm import tqdm\n",
    "\n",
    "ROOT = r'/home/datasets/srl/opensubtitles/artefacts/step2'\n",
    "INPUTS = os.path.join(ROOT, 'inputs')\n",
    "MA_DIR = os.path.join(ROOT, 'ma')\n",
    "MD_DIR = os.path.join(ROOT, 'md')\n",
    "OUTPUTS = os.path.join(ROOT, 'outputs')\n",
    "\n",
    "os.chdir(os.path.join(os.getenv('GOPATH'), 'src', 'yap'))\n",
    "\n",
    "\n",
    "def ma(filename):\n",
    "    subprocess.Popen(['./yap', 'hebma',\n",
    "                      '-prefix', 'data/bgulex/bgupreflex_withdef.utf8.hr',\n",
    "                      '-lexicon', \"data/bgulex/bgulex.utf8.hr\",\n",
    "                      '-raw', os.path.join(INPUTS, filename),\n",
    "                      '-out', os.path.join(MA_DIR, filename)],\n",
    "                     stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT).wait()\n",
    "\n",
    "\n",
    "def md(filename):\n",
    "    subprocess.Popen(['./yap', 'md',\n",
    "                      '-m', 'data/hebmd',\n",
    "                      '-f', 'conf/standalone.md.yaml',\n",
    "                      '-b', '32',\n",
    "                      '-in', os.path.join(MA_DIR, filename),\n",
    "                      '-om', os.path.join(MD_DIR, filename)],\n",
    "                     stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT).wait()\n",
    "\n",
    "def dp(filename):\n",
    "    subprocess.Popen(['./yap', 'dep',\n",
    "                      '-m', 'data/dep',\n",
    "                      '-f', 'conf/zhangnivre2011.yaml',\n",
    "                      '-l', 'conf/hebtb.labels.conf',\n",
    "                      '-inl', os.path.join(MD_DIR, filename),\n",
    "                      '-oc', os.path.join(OUTPUTS, filename)],\n",
    "                      stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT).wait()\n",
    "\n",
    "\n",
    "Parallel(n_jobs=8)(delayed(ma)(filename) for filename in tqdm(os.listdir(INPUTS)))\n",
    "Parallel(n_jobs=8)(delayed(md)(filename) for filename in tqdm(os.listdir(MA_DIR)))\n",
    "Parallel(n_jobs=8)(delayed(dp)(filename) for filename in tqdm(os.listdir(MD_DIR)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Upon completion, we add the newly segmented Hebrew subtitles to our JSON files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {},
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def filename2segmented_hebrew(of):\n",
    "    df = pd.read_csv(os.path.join(YAP_OUTPUTS, of),\n",
    "                     sep='\\t', header=None,\n",
    "                     na_filter=False, quoting=csv.QUOTE_NONE)\n",
    "    df = df[[0, 1, 4, 5, 6, 7]]\n",
    "    df.columns = ('id', 'form', 'postag', 'feats', 'head', 'deprel')\n",
    "#     df = df.set_index('ID')\n",
    "    sub, sent = [], []\n",
    "    for row in df.itertuples():\n",
    "        if row.id == 1:\n",
    "            sub.append(sent)\n",
    "            sent = []\n",
    "        sent.append({\n",
    "            'id': int(row.id - 1),\n",
    "            'text': row.form,\n",
    "            'pos': row.postag,\n",
    "            'iob': '',\n",
    "            'head': int(row.head - 1),\n",
    "            'deprel': row.deprel\n",
    "        })\n",
    "    sub.append(sent)\n",
    "    return sub[1:]\n",
    "\n",
    "\n",
    "def add_segmented_hebrew_to_json(sub, heb_sents):\n",
    "    with open(os.path.join(TOKENIZED_SENTS, sub), encoding='utf-8') as fsub:\n",
    "        json_subs = json.load(fsub)\n",
    "    json_subs['seghe'] = heb_sents\n",
    "    with open(os.path.join(PREPROCESSED_SENTS, sub), 'w', encoding='utf-8') as fseg:\n",
    "        json.dump(json_subs, fseg, ensure_ascii=False, indent=4)\n",
    "\n",
    "\n",
    "def process(f):\n",
    "    seghe = filename2segmented_hebrew(f[:f.index('.')])\n",
    "    add_segmented_hebrew_to_json(f, seghe)\n",
    "\n",
    "\n",
    "Parallel(n_jobs=cpu_count())(delayed(process)(sub) for sub in tqdm(os.listdir(TOKENIZED_SENTS)));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Alignment\n",
    "\n",
    "Now that we have both English and Hebrew tokens aligned by sentence, we want to align them by token.\n",
    "We used three aligners, and inspected their outputs manually. `fast_align` gave the best results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {},
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def create_input_for_aligners(sub):\n",
    "    with open(os.path.join(PREPROCESSED_SENTS, sub), encoding='utf-8') as fjson:\n",
    "        d = json.load(fjson)\n",
    "    en, seghe = d['en'], d['seghe']\n",
    "    get_text = itemgetter('text')\n",
    "    en = [map(get_text, sent) for sent in en]\n",
    "    seghe = [map(get_text, sent) for sent in seghe]\n",
    "    fn = os.path.splitext(sub)[0]\n",
    "    with open(os.path.join(FASTALIGN_INPUTS, fn) + '.fa', 'w', encoding='utf-8') as ffa, open(os.path.join(EFMARAL_INPUTS, fn) + '.fa', 'w', encoding='utf-8') as fef:\n",
    "        for e, h in zip(en, seghe):\n",
    "            s = '{} ||| {}'.format(' '.join(e), ' '.join(h))\n",
    "            print(s, file=ffa)\n",
    "            print(s, file=fef)\n",
    "    with open(os.path.join(EFLOMAL_INPUTS, fn) + '.en', 'w', encoding='utf-8') as fen, open(os.path.join(EFLOMAL_INPUTS, fn) + '.he', 'w', encoding='utf-8') as fhe:\n",
    "        for i, e in enumerate(en):\n",
    "            print('<s snum={i}>{sent}</s>'.format(i=i, sent=' '.join(e)), file=fen)\n",
    "        for i, h in enumerate(seghe):\n",
    "            print('<s snum={i}>{sent}</s>'.format(i=i, sent=' '.join(h)), file=fhe)\n",
    "\n",
    "Parallel(n_jobs=cpu_count())(delayed(create_input_for_aligners)(sub) for sub in tqdm(os.listdir(PREPROCESSED_SENTS)));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting /home/datasets/srl/opensubtitles/artefacts/step4/run_aligners.py\n"
     ]
    }
   ],
   "source": [
    "%%file /home/datasets/srl/opensubtitles/artefacts/step4/run_aligners.py\n",
    "#!/usr/bin/env python\n",
    "\n",
    "import os\n",
    "import subprocess\n",
    "\n",
    "from glob import glob\n",
    "from multiprocessing import cpu_count\n",
    "\n",
    "from joblib import Parallel, delayed\n",
    "\n",
    "ROOT = r'/home/datasets/srl/tools/aligners/'\n",
    "FASTALIGN = os.path.join(ROOT, 'fast_align/build/fast_align')\n",
    "EFLOMAL = os.path.join(ROOT, 'eflomal/align.py')\n",
    "EFMARAL = os.path.join(ROOT, 'efmaral/align.py')\n",
    "\n",
    "\n",
    "def fastalign(fn):\n",
    "    with open(fn + '.fwd', 'w', encoding='utf-8') as fwd:\n",
    "        subprocess.Popen([FASTALIGN,\n",
    "                          '-i', fn,\n",
    "                          '-d', '-o', '-v'],\n",
    "                         stdout=fwd).wait()\n",
    "    with open(fn + '.rev', 'w', encoding='utf-8') as rev:\n",
    "        subprocess.Popen([FASTALIGN,\n",
    "                          '-i', fn,\n",
    "                          '-d', '-o', '-v', '-r'],\n",
    "                         stdout=rev).wait()\n",
    "\n",
    "\n",
    "def efmaral(fn):\n",
    "    with open(fn + '.fwd', 'w', encoding='utf-8') as fwd:\n",
    "        subprocess.Popen([EFMARAL,\n",
    "                          '-i', fn], stdout=fwd).wait()\n",
    "    with open(fn + '.rev', 'w', encoding='utf-8') as rev:\n",
    "        subprocess.Popen([EFMARAL,\n",
    "                          '-i', fn,\n",
    "                          '-r'], stdout=rev).wait()\n",
    "\n",
    "\n",
    "def eflomal(en, he):\n",
    "    fn = en[:-3]\n",
    "    subprocess.Popen([EFLOMAL,\n",
    "                      '-s', en,\n",
    "                      '-t', he,\n",
    "                      '-f', fn + '.fwd',\n",
    "                      '-r', fn + '.rev']).wait()\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    os.chdir(r'/home/datasets/srl/opensubtitles/artefacts/step4/fastalign')\n",
    "    print('Running fast_align...', end='')\n",
    "    Parallel(n_jobs=cpu_count())(delayed(fastalign)(fn) for fn in os.listdir('.'))\n",
    "    print('Done!')\n",
    "\n",
    "    os.chdir(r'/home/datasets/srl/tools/aligners/eflomal/')\n",
    "    ens = glob('/home/datasets/srl/opensubtitles/artefacts/step4/eflomal/*.en')\n",
    "    hes = glob('/home/datasets/srl/opensubtitles/artefacts/step4/eflomal/*.he')\n",
    "    print('Running eflomal...', end='')\n",
    "    Parallel(n_jobs=cpu_count())(delayed(eflomal)(en, he) for en, he in zip(sorted(ens), sorted(hes)))\n",
    "    print('Done!')\n",
    "\n",
    "    os.chdir(r'/home/datasets/srl/opensubtitles/artefacts/step4/efmaral/')\n",
    "    print('Running efmaral...', end='')\n",
    "    Parallel(n_jobs=cpu_count())(delayed(efmaral)(fn) for fn in os.listdir('.'))\n",
    "    print('Done!')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SRL with Semafor\n",
    "\n",
    "We now have at our disposal English tokens, Hebrew tokens (segmented), and alignments between them. We used the parsed English tokens as input to Semafor and get SRL for all sentences."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Final Dataset\n",
    "\n",
    "After all the work done in this notebook (+SRL as per Semafor's instructions), we have four folders which comprise the entire processed dataset:\n",
    "\n",
    "* `english_parsed` - Parsed English sentences\n",
    "* `hebrew_parsed` - Parsed and segmented Hebrew sentences\n",
    "* `english_srl` - SRL for each English sentence\n",
    "* `fastalign_outputs` - The English-Hebrew token alignment"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}