translation/opus/gnome.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://object.pouta.csc.fi/OPUS-GNOME/v1/raw/en.zip\n",
    "# !wget https://object.pouta.csc.fi/OPUS-GNOME/v1/raw/ms.zip\n",
    "# !unzip -o en.zip\n",
    "# !unzip -o ms.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://object.pouta.csc.fi/OPUS-GNOME/v1/xml/en-ms.xml.gz\n",
    "# !gzip -d en-ms.xml.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 306166/306166 [01:11<00:00, 4267.58it/s]\n"
     ]
    }
   ],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "from tqdm import tqdm\n",
    "\n",
    "with open('en-ms.xml') as fopen:\n",
    "    xml = fopen.read().split('\\n')\n",
    "    \n",
    "mapping, ids = {}, []\n",
    "enable = False\n",
    "for row in tqdm(xml):\n",
    "    if '<linkGrp targType' in row:\n",
    "        s = BeautifulSoup(row)\n",
    "        r = 'GNOME/raw/' + s.findAll('linkgrp')[0].get('fromdoc').replace('.gz','')\n",
    "        l = 'GNOME/raw/' + s.findAll('linkgrp')[0].get('todoc').replace('.gz','')\n",
    "        enable = True\n",
    "    if '<link id' in row and enable:\n",
    "        s = BeautifulSoup(row)\n",
    "        t = s.findAll('link')[0].get('xtargets')\n",
    "        t = list(filter(None, t.split(';')))\n",
    "        if len(t) != 2:\n",
    "            continue\n",
    "        ids.append([t[0].split(), t[1].split()])\n",
    "    if '</linkGrp>' in row:\n",
    "        label = f'{l} <> {r}'\n",
    "        mapping[label] = ids\n",
    "        enable = False\n",
    "        ids = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[['s4'], ['s4']],\n",
       " [['s5'], ['s5']],\n",
       " [['s6'], ['s6']],\n",
       " [['s12'], ['s12']],\n",
       " [['s19'], ['s19']],\n",
       " [['s22'], ['s22']],\n",
       " [['s23'], ['s23']],\n",
       " [['s24'], ['s24']],\n",
       " [['s25'], ['s25']],\n",
       " [['s26'], ['s26']],\n",
       " [['s27'], ['s27']],\n",
       " [['s28'], ['s28']],\n",
       " [['s29'], ['s29']],\n",
       " [['s31'], ['s31']],\n",
       " [['s32'], ['s32']],\n",
       " [['s33'], ['s33']],\n",
       " [['s34'], ['s34']],\n",
       " [['s35'], ['s35']],\n",
       " [['s36'], ['s36']],\n",
       " [['s37'], ['s37']],\n",
       " [['s38'], ['s38']],\n",
       " [['s39'], ['s39']],\n",
       " [['s40'], ['s40']],\n",
       " [['s41'], ['s41']],\n",
       " [['s42'], ['s42']],\n",
       " [['s43'], ['s43']],\n",
       " [['s44'], ['s44']],\n",
       " [['s45'], ['s45']],\n",
       " [['s46'], ['s46']],\n",
       " [['s47'], ['s47']],\n",
       " [['s48'], ['s48']],\n",
       " [['s50'], ['s50']],\n",
       " [['s53'], ['s53']],\n",
       " [['s54'], ['s54']],\n",
       " [['s56'], ['s56']],\n",
       " [['s57'], ['s57']],\n",
       " [['s58'], ['s58']],\n",
       " [['s60'], ['s60']],\n",
       " [['s62'], ['s62']],\n",
       " [['s64'], ['s64']],\n",
       " [['s66'], ['s66']],\n",
       " [['s67'], ['s67']],\n",
       " [['s68'], ['s68']],\n",
       " [['s69'], ['s69']],\n",
       " [['s70'], ['s70']],\n",
       " [['s75'], ['s75']],\n",
       " [['s76'], ['s76']],\n",
       " [['s77'], ['s77']],\n",
       " [['s78'], ['s78']],\n",
       " [['s79'], ['s79']],\n",
       " [['s82'], ['s82']],\n",
       " [['s83'], ['s83']],\n",
       " [['s85'], ['s85']],\n",
       " [['s87'], ['s87']],\n",
       " [['s88'], ['s88']],\n",
       " [['s90'], ['s90']],\n",
       " [['s91'], ['s91']],\n",
       " [['s92'], ['s92']],\n",
       " [['s93'], ['s93']],\n",
       " [['s94'], ['s94']],\n",
       " [['s95'], ['s95']],\n",
       " [['s98'], ['s98']],\n",
       " [['s99'], ['s99']],\n",
       " [['s100'], ['s100']],\n",
       " [['s102'], ['s102']],\n",
       " [['s103'], ['s103']],\n",
       " [['s104'], ['s104']],\n",
       " [['s106'], ['s106']],\n",
       " [['s107'], ['s107']],\n",
       " [['s110'], ['s110']],\n",
       " [['s112'], ['s112']],\n",
       " [['s113'], ['s113']],\n",
       " [['s114'], ['s114']],\n",
       " [['s120'], ['s120']],\n",
       " [['s121'], ['s121']],\n",
       " [['s122'], ['s122']],\n",
       " [['s126'], ['s126']],\n",
       " [['s129'], ['s129']],\n",
       " [['s130'], ['s130']],\n",
       " [['s134'], ['s134']],\n",
       " [['s143'], ['s143']],\n",
       " [['s144'], ['s144']],\n",
       " [['s145'], ['s145']],\n",
       " [['s146'], ['s146']],\n",
       " [['s147'], ['s147']],\n",
       " [['s148'], ['s148']],\n",
       " [['s149'], ['s149']],\n",
       " [['s150'], ['s150']],\n",
       " [['s151'], ['s151']],\n",
       " [['s152'], ['s152']],\n",
       " [['s153'], ['s153']],\n",
       " [['s154'], ['s154']],\n",
       " [['s155'], ['s155']],\n",
       " [['s156'], ['s156']],\n",
       " [['s157'], ['s157']],\n",
       " [['s158'], ['s158']],\n",
       " [['s159'], ['s159']],\n",
       " [['s160'], ['s160']],\n",
       " [['s161'], ['s161']],\n",
       " [['s162'], ['s162']],\n",
       " [['s163'], ['s163']],\n",
       " [['s164'], ['s164']],\n",
       " [['s165'], ['s165']],\n",
       " [['s166'], ['s166']],\n",
       " [['s167'], ['s167']],\n",
       " [['s168'], ['s168']],\n",
       " [['s169'], ['s169']],\n",
       " [['s170'], ['s170']],\n",
       " [['s171'], ['s171']],\n",
       " [['s172'], ['s172']],\n",
       " [['s173'], ['s173']],\n",
       " [['s174'], ['s174']],\n",
       " [['s175'], ['s175']],\n",
       " [['s176'], ['s176']],\n",
       " [['s177'], ['s177']],\n",
       " [['s178'], ['s178']],\n",
       " [['s179'], ['s179']],\n",
       " [['s180'], ['s180']],\n",
       " [['s181'], ['s181']],\n",
       " [['s182'], ['s182']],\n",
       " [['s183'], ['s183']],\n",
       " [['s184'], ['s184']],\n",
       " [['s186'], ['s186']],\n",
       " [['s187'], ['s187']]]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mapping[label]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 898/898 [01:08<00:00, 13.09it/s]\n"
     ]
    }
   ],
   "source": [
    "results = []\n",
    "for file in tqdm(mapping.keys()):\n",
    "    l, r = file.split(' <> ')\n",
    "    ids = mapping[file]\n",
    "    try:\n",
    "        with open(l) as fopen:\n",
    "            left = fopen.read()\n",
    "        soup_left = BeautifulSoup(left, 'html.parser')\n",
    "        with open(r) as fopen:\n",
    "            right = fopen.read()\n",
    "        soup_right = BeautifulSoup(right, 'html.parser')\n",
    "        left_dict = {}\n",
    "        for s in soup_left.find_all('s'):\n",
    "            i = s.get('id')\n",
    "            text = s.get_text()\n",
    "            left_dict[i] = text\n",
    "        right_dict = {}\n",
    "        for s in soup_right.find_all('s'):\n",
    "            i = s.get('id')\n",
    "            text = s.get_text()\n",
    "            right_dict[i] = text\n",
    "        combined = []\n",
    "        for i in ids:\n",
    "            try:\n",
    "                r = ' '.join([right_dict[k] for k in i[0]])\n",
    "                l = ' '.join([left_dict[k] for k in i[1]])\n",
    "                combined.append((l, r))\n",
    "            except:\n",
    "                pass\n",
    "        results.extend(combined)\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "302567"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('GNOME', 'GNOME'),\n",
       " ('Tema GNOME Default', 'Default GNOME Theme'),\n",
       " ('OK', 'OK'),\n",
       " ('Seni', 'Art'),\n",
       " ('Kamera', 'Camera'),\n",
       " ('Diperakui', 'Certified'),\n",
       " ('Sejuk', 'Cool'),\n",
       " ('Bahaya', 'Danger'),\n",
       " ('Desktop', 'Desktop'),\n",
       " ('Unggul', 'Distinguished'),\n",
       " ('Draf', 'Draft'),\n",
       " ('Jabat tangan', 'Handshake'),\n",
       " ('Pemasaran', 'Marketing'),\n",
       " ('Wang', 'Money'),\n",
       " ('Multimedia', 'Multimedia'),\n",
       " ('Nota', 'Note'),\n",
       " ('Alamak!', 'Oh no!'),\n",
       " ('Orang', 'People'),\n",
       " ('Peribadi', 'Personal'),\n",
       " ('Gambar', 'Pictures')]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('gnome-ms-en.json', 'w') as fopen:\n",
    "    json.dump(results, fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf GNOME\n",
    "!rm -rf ms.zip en.zip en-ms.xml.gz en-ms.xml"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}