From 055aa7e51fd56f5a6302799ee172b865b03aabfd Mon Sep 17 00:00:00 2001 From: Thamme Gowda Date: Mon, 10 Mar 2025 05:39:07 -0700 Subject: [PATCH 01/13] huggingface datasets; index subcommand; WIP docid meta --- .gitignore | 1 + mtdata/cache.py | 32 +++- mtdata/entry.py | 8 +- mtdata/index/__init__.py | 1 + mtdata/index/huggingface.py | 206 +++++++++++++++++++++ mtdata/main.py | 46 ++++- mtdata/parser.py | 26 ++- mtdata/resource/huggingface-datasets.jsonl | 2 + setup.py | 4 + tests/__init__.py | 5 +- tests/test_huggingface.py | 15 ++ 11 files changed, 327 insertions(+), 19 deletions(-) create mode 100644 mtdata/index/huggingface.py create mode 100644 mtdata/resource/huggingface-datasets.jsonl create mode 100644 tests/test_huggingface.py diff --git a/.gitignore b/.gitignore index 3f9ebb7..992f3a6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.egg-info *.pytest_cache +mtdata/resource/huggingface-datasets.all.jsonl # Byte-compiled / optimized / DLL files /tmp diff --git a/mtdata/cache.py b/mtdata/cache.py index 53fdf59..3504034 100644 --- a/mtdata/cache.py +++ b/mtdata/cache.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Author: Thamme Gowda [tg (at) isi (dot) edu] +# Author: Thamme Gowda [tg (at) isi (dot) edu] # Created: 4/4/20 import zipfile import tarfile @@ -45,7 +45,7 @@ def get_entry(self, entry: Entry, fix_missing=True) -> Union[Path, List[Path]]: else: assert isinstance(entry.url, str) local = self.get_local_path(entry.url, filename=entry.filename, fix_missing=fix_missing, entry=entry) - if zipfile.is_zipfile(local) or tarfile.is_tarfile(local): + if isinstance(local, Path) and (zipfile.is_zipfile(local) or tarfile.is_tarfile(local)): # look inside the archives and get the desired files local = self.get_local_in_paths(path=local, entry=entry) return local @@ -133,6 +133,11 @@ def get_flag_file(self, file: Path): def get_local_path(self, url, filename=None, fix_missing=True, entry=None): hostname = urlparse(url).hostname or 'nohost' + if hostname == "huggingface.co": + # HF is special cased by deligating the task to huggingface sdk + # I have considered not adding the dependency but there are a lot of file formats and + # some are sharded datasets which require custom logic, and my custom code might not be future proof + return self.get_hf_dataset(url, entry=entry) filename = filename or url.split('/')[-1] assert hostname and filename mdf5_sum = md5(url.encode('utf-8')).hexdigest() @@ -145,6 +150,29 @@ def get_local_path(self, url, filename=None, fix_missing=True, entry=None): raise return local + def get_hf_dataset(self, url: str, entry=None): + # dataset lib has a lot of transient dependencies, so lazily load it + # and only when needed + try: + from datasets import load_dataset + except ImportError as e: + raise MTDataException(f"huggingface datasets library is required to access {entry.did}, but it is missing. " + f"Run: 'pip install datasets' and try again") from e + hf_id = entry.meta["orig_id"] + config = entry.meta.get("config", None) + split = entry.meta.get("split", None) + cache_dir = self.root / 'huggingface' / 'datasets' + args = dict( + name=config, + split=split, + cache_dir=cache_dir, + streaming=False, + trust_remote_code=False, + ) + log.info(f"Loading dataset {hf_id} with args: {args}") + ds = load_dataset(hf_id, **args) + return ds + @classmethod def match_globs(cls, names, globs, meta=''): result = [] diff --git a/mtdata/entry.py b/mtdata/entry.py index ddba5b8..0d5c72d 100644 --- a/mtdata/entry.py +++ b/mtdata/entry.py @@ -80,7 +80,7 @@ def parse(cls, string, delim=DID_DELIM) -> 'DatasetId': class Entry: - __slots__ = ('did', 'url', 'filename', 'ext', 'in_paths', 'in_ext', 'cite', 'cols', 'is_archive') + __slots__ = ('did', 'url', 'filename', 'ext', 'in_paths', 'in_ext', 'cite', 'cols', 'is_archive', 'meta') def __init__(self, did: Union[str, DatasetId], url: Union[str, Tuple[str, str]], @@ -89,13 +89,15 @@ def __init__(self, did: Union[str, DatasetId], in_paths: Optional[List[str]] = None, in_ext: Optional[str] = None, cite: Optional[Tuple[str]] = None, - cols: Optional[Tuple[int, int]] = None): + cols: Optional[Tuple[int, int]] = None, + meta: Optional[dict] = None): if not isinstance(did, DatasetId): did = DatasetId.parse(did) self.did = did self.url = url self.filename = filename self.ext = ext + self.meta = meta if not self.ext: from mtdata.parser import detect_extension assert isinstance(self.url, str), '"ext" attribute must be explicitely set for multi-URL entries' @@ -109,7 +111,7 @@ def __init__(self, did: Union[str, DatasetId], self.cols = cols # column index starts from zero assert not self.ext.startswith("."), f'{did} :: ext {self.ext} should not start with a dot (.)' - self.is_archive = self.ext in ('zip', 'tar', 'tar.gz', 'tgz') + self.is_archive = self.ext in ('zip', 'tar', 'tar.gz', 'tgz', 'hfds') if self.is_archive: assert self.in_paths and len(self.in_paths) > 0, 'Archive entries must have in_paths' if not self.in_ext: diff --git a/mtdata/index/__init__.py b/mtdata/index/__init__.py index 2d2f297..4169888 100644 --- a/mtdata/index/__init__.py +++ b/mtdata/index/__init__.py @@ -90,6 +90,7 @@ def load_all(self): ".opus.opus_index", ".opus.opus100", ".leipzig", + ".huggingface", ] # modules from CWD for p in Path('.').glob('mtdata*.py'): diff --git a/mtdata/index/huggingface.py b/mtdata/index/huggingface.py new file mode 100644 index 0000000..87cefac --- /dev/null +++ b/mtdata/index/huggingface.py @@ -0,0 +1,206 @@ +import argparse +import json +from pathlib import Path +import time +from collections import defaultdict +from typing import List, Dict, Any + +import requests + +from mtdata import resource_dir, log, yaml +from mtdata.index import Index, DatasetId, Entry + +QUERY_URL = "https://huggingface.co/datasets-json" +README_URL = "https://huggingface.co/datasets/{repo_id}/raw/main/README.md" +RESOURCE_FILE = resource_dir / 'huggingface-datasets.jsonl' +QPARAMS = { + "task_categories": "task_categories:translation", + "sort": "created", + "withCount": "true", + "p": 0, +} +HF_EXT = "hfds" # huggingface dataset + +# To refresh the data_file from huggingface: +# python -m mtdata.index.huggingface --refresh + +def load_all(index: Index): + meta_file = RESOURCE_FILE + assert meta_file.exists(), f"{meta_file} does not exist" + assert meta_file.stat().st_size > 0, f"{meta_file} is empty" + with meta_file.open() as lines: + for line in lines: + line = line.strip() + if not line or line.startswith("#") or line.startswith("//"): + continue + data = json.loads(line) + if data['id'] != "google/wmt24pp": + continue # TODO: support other datasets + + id_parts = data['id'].split('/') + assert len(id_parts) == 2, f"Invalid dataset id: {data['id']}" + group, name = id_parts + group = group.title() + for config in data['configs']: + split_name = name + if config["split"] != "train": + # assume train is the default + split_name += f"_{config['split']}" + orig_langs = config["name"] + langs = tuple(orig_langs.split("-")) + assert len(langs) in (1, 2), f"Invalid langs: {langs}" + data_id = DatasetId(group=group, name=split_name, version="1", langs=langs) + if data_id in index: + log.warning(f"Duplicate dataset id: {data_id}") + continue + params = dict(langs=orig_langs, split=config["split"]) + url = "https://huggingface.co/datasets/" + data['id'] + meta = dict(config=config['name'], orig_id=data['id'], split=config["split"]) + in_paths = config["paths"] + cite = None + entry = Entry(did=data_id, url=url, in_paths=in_paths, cite=cite, ext=HF_EXT, in_ext=HF_EXT, meta=meta) + index.add_entry(entry) + + +def query_datasets(page_num=0): + params = QPARAMS.copy() + last_request_time = 0 + request_interval = 1 # seconds + while True: + ### Fair crawling: dont overwhelm the server + now = time.time() + assert now >= last_request_time + interval = now - last_request_time + if interval < request_interval: + time.sleep(request_interval - interval) + last_request_time = time.time() + ### + params["p"] = page_num + url_with_parms = f"{QUERY_URL}?{requests.compat.urlencode(params)}" + log.info(f"GET {url_with_parms}") + response = requests.get(url_with_parms) + if response.status_code != 200: + log.warning(f"Failed to fetch data: {response.status_code}; text:\n{response.text}") + raise Exception(f"Failed to fetch data: {response.status_code}") + data = response.json() + if "datasets" not in data or not data["datasets"]: + log.info("No more datasets found.") + break + for d in data["datasets"]: + yield d + page_num += 1 + # uncomment for testing on a subset of results + #if page_num >= 10: break + +def enrich_metadata(meta): + """ + Enrich the dataset's metadata by parsing YAML config from README the repository. + """ + # use API https://huggingface.co/docs/datasets/load_hub#configurations + configs = meta.get("config", []) + readme_url = README_URL.format(repo_id=meta["id"]) + log.info(f"GET {readme_url}") + readme_text = requests.get(readme_url).text + yaml_config_text = "" + parts = readme_text.split("---") + if len(parts) < 3: + log.warning(f"Failed to find YAML config in {readme_url}") + return False + try: + # find the text between the first and second "---" + yaml_config_text = readme_text.split("---", 2)[1] + readme_config = yaml.load(yaml_config_text) + my_config = [] + for config in readme_config.get("configs", []): + splits = defaultdict(list) + for split in config.get("data_files", []): + splits[split["split"]].append(split["path"]) + for name, paths in splits.items(): + my_config.append(dict( + name= config.get("config_name"), + split=name, + paths=paths, + #langs=None #FIXME: need to infer langs. often in config name but not consistent + )) + if my_config: + meta["configs"] = my_config + return True + except Exception as e: + log.warning(f"Failed to parse YAML config in {readme_url}: {e}") + return False + + +def enrich_metadata_lite(meta): + """ + Enrich the dataset's metadata by parsing YAML config from README the repository. + """ + # NOTE: this didnt work. code froze + import datasets as hfds + configs = hfds.get_dataset_config_names(meta["id"]) + my_config = [] + for config_name in configs: + splits = hfds.get_dataset_split_names(meta["id"], config_name) + for split_name in splits: + my_config.append(dict( + name=config_name, + split=split_name + #langs=None #FIXME: need to infer langs. often in config name but not consistent + )) + if my_config: + meta["configs"] = my_config + return True + return False + + +def update_resource_file(out_file, refresh=False): + all_out_file = out_file.with_name(out_file.name.replace(".jsonl", "") + ".all.jsonl") + if refresh or not out_file.exists(): + log.info(f"Fetching datasets from {QUERY_URL}. This might take a while") + datasets = query_datasets() + datasets = list(sorted(datasets, key=lambda x: x["id"].lower())) + with all_out_file.open("w", encoding="utf-8") as all_f: + for d in datasets: + line = json.dumps(d, ensure_ascii=False, indent=None) + all_f.write(line + "\n") + else: + log.info(f"Loading datasets from {all_out_file}") + # load existing datasets from the resource file + datasets = [] + with all_out_file.open("r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#") or line.startswith("//"): + continue + data = json.loads(line) + datasets.append(data) + if not datasets: + log.warning("No datasets found.") + return + + # focus on a selected few datasets for now before expanding to others + # enrich_metadata is slow and we need to be careful about the number of requests + select_datasets = {"google/wmt24pp", "ai4bharat/samanantar"} + log.info(f"Writing selected datasets to {out_file} and all datasets to {all_out_file}") + out_file.parent.mkdir(parents=True, exist_ok=True) + with open(out_file, "w", encoding="utf-8") as f: + for d in datasets: + if select_datasets and d["id"] not in select_datasets: + continue + # enrich metadata for a selected few datasets + enrich_metadata(d) + line = json.dumps(d, ensure_ascii=False, indent=None) + f.write(line + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description="Download datasets from Hugging Face") + parser.add_argument("-o", "--out", dest='out_file', type=str, default=RESOURCE_FILE, + help="Output file to save the downloaded datasets",) + parser.add_argument("-r", "--refresh", dest='refresh', action='store_true', + help="Refresh the datasets from Hugging Face") + + args = parser.parse_args() + out_file = Path(args.out_file).expanduser() + update_resource_file(out_file, refresh=args.refresh) diff --git a/mtdata/main.py b/mtdata/main.py index eee0ba5..2085f7e 100644 --- a/mtdata/main.py +++ b/mtdata/main.py @@ -72,10 +72,17 @@ def echo_data(did:DatasetId, delim='\t'): parser = Parser(path, ext=entry.in_ext or None, ent=entry) count = 0 all_segs = parser.read_segs() - for rec in all_segs: - if isinstance(rec, (list, tuple)): - rec = (col.replace(delim, ' ').replace('\n', ' ') for col in rec) + for row in all_segs: + if isinstance(row, str): + rec = row + elif isinstance(row, (list, tuple)): + rec = [col.replace(delim, ' ').replace('\n', ' ') for col in row[:2]] + if len(row) == 3: + meta = json.dumps(row[2], indent=None, ensure_ascii=False) + rec.append(meta) rec = delim.join(rec) + else: + raise ValueError(f'Unknown row type: {type(row)}. Expected str or list/tuple') print(rec) count += 1 log.info(f'Total rows={count:,}') @@ -125,7 +132,8 @@ def list_recipes(id_only=False, delim='\t', format='plain'): raise f'{format} not supported' -def get_recipe(recipe_id, out_dir: Path, compress=False, drop_dupes=False, drop_tests=False, fail_on_error=False, +def get_recipe(recipe_id, out_dir: Path, compress=False, drop_dupes=False, + drop_tests=False, fail_on_error=False, n_jobs=DEF_N_JOBS, merge_train=True, **kwargs): if kwargs: log.warning(f"Args are ignored: {kwargs}") @@ -180,6 +188,18 @@ def cache_datasets(recipes:List[str]=None, dids:List[DatasetId]=None, n_jobs=DEF log.info(f"Going to cache {len(entries)} entries at {cache.root}; n_jobs={n_jobs}") Dataset.parallel_download(entries, cache=cache, n_jobs=n_jobs) +def index_datasets(): + """ + Create or update the dataset index. This deletes action {cached_index_file} only and not the downloaded files. + Use this if you've modified the mtdata source code and you want to force refresh the dataset index. + """ + if cached_index_file.exists(): + bak_file = cached_index_file.with_suffix(".bak") + log.info(f"Invalidate index: {cached_index_file} -> {bak_file}") + cached_index_file.rename(bak_file) + # importing the index will recreate the index + from mtdata.index import INDEX as index + class MyFormatter(argparse.ArgumentDefaultsHelpFormatter): def _split_lines(self, text, width: int): @@ -206,8 +226,7 @@ def parse_args(): choices=log_levels, help=f'Set log level. Choices={log_levels}') p.add_argument('-ri', '--reindex', action='store_true', help=f"Invalidate index of entries and recreate it. This deletes" - f" {cached_index_file} only and not the downloaded files. " - f"Use this if you've modifying mtdata source code and want to force reload.") + f" Deprecated: use 'index' subcommand instead. ") grp = p.add_mutually_exclusive_group() grp.add_argument('-pb', '--pbar', action='store_true', dest='progressbar', help=f"Show progressbar", default=True) @@ -216,6 +235,7 @@ def parse_args(): sub_ps = p.add_subparsers(required=True, dest='task', metavar='', help='''R| +"index" - create or update the dataset index. "list" - List the available entries "get" - Downloads the entry files and prepares them for experiment "echo" - Print contents of a dataset into STDOUT. @@ -224,6 +244,10 @@ def parse_args(): "stats" - Get stats of dataset" "cache" - download datasets into cache ''') + index_p = sub_ps.add_parser( + 'index', formatter_class=MyFormatter, + help=f"Create or update the dataset index. This deletes action {cached_index_file} only and not the downloaded files. " + f"Use this if you've modified the mtdata source code and you want to force refresh the dataset index.") list_p = sub_ps.add_parser('list', formatter_class=MyFormatter) list_p.add_argument('-l', '--langs', metavar='L1/L1-L2', type=Langs, @@ -315,10 +339,12 @@ def main(): args = parse_args() if args.reindex and cached_index_file.exists(): - bak_file = cached_index_file.with_suffix(".bak") - log.info(f"Invalidate index: {cached_index_file} -> {bak_file}") - cached_index_file.rename(bak_file) - if args.task == 'list': + log.warning(f"--reindex flag is deprecated and will be removed in the future. Use 'index' subcommand instead") + index_datasets() + + if args.task == 'index': + index_datasets() + elif args.task == 'list': list_data(args.langs, args.names, not_names=args.not_names, full=args.full, groups=args.groups, not_groups=args.not_groups, id_only=args.id) elif args.task == 'get': diff --git a/mtdata/parser.py b/mtdata/parser.py index aa815e5..5fb293c 100644 --- a/mtdata/parser.py +++ b/mtdata/parser.py @@ -13,6 +13,8 @@ from mtdata.utils import IO COMPRESS_EXT = ['gz', 'bz2', 'xz'] +HF_EXT = 'hfds' # huggingface dataset +WMT21XML = 'wmt21xml' def detect_extension(name: Union[str, Path]): @@ -40,7 +42,9 @@ def __post_init__(self): if not isinstance(self.paths, list): self.paths = [self.paths] for p in self.paths: - assert p.exists(), f'{p} not exists' + if isinstance(p, Path): + assert p.exists(), f'{p} not exists' + # skip cheks on HF datasets if not self.ext: exts = [detect_extension(p.name) for p in self.paths] @@ -81,9 +85,11 @@ def read_segs(self): elif 'sgm' in self.ext: from mtdata.sgm import read_sgm readers.append(read_sgm(p)) - elif 'wmt21xml' in self.ext: + elif WMT21XML in self.ext: from mtdata.sgm import read_wmt21_xml readers.append(read_wmt21_xml(p)) + elif HF_EXT in self.ext: + readers.append(self.read_hfds(p)) else: raise Exception(f'Not supported {self.ext} : {p}') @@ -93,7 +99,8 @@ def read_segs(self): data = (rec for reader in readers for rec in reader) # flatten all readers elif len(readers) == 2: def _zip_n_check(): - for seg1, seg2 in zip_longest(*readers): + for row in zip_longest(*readers): + seg1, seg2 = row[:2] if seg1 is None or seg2 is None: raise Exception(f'{self.paths} have unequal number of segments') yield seg1, seg2 @@ -132,3 +139,16 @@ def read_tsv(self, path, delim='\t', cols=None, skipheader=False): if cols: row = [row[idx] for idx in cols] yield row + + def read_hfds(self, ds): + """ Read data from huggingface Dataset + :param ds: huggingface dataset + :return: generator of segments + """ + for row in ds: + src = row['source'] + tgt = row['target'] + #doc_id = row.get('document_id') + #seg_id = row.get('segment_id') + meta = dict(doc_id=row.get('document_id'), seg_id=row.get('segment_id'), domain=row.get('domain')) + yield src, tgt, meta \ No newline at end of file diff --git a/mtdata/resource/huggingface-datasets.jsonl b/mtdata/resource/huggingface-datasets.jsonl new file mode 100644 index 0000000..7e3ebe7 --- /dev/null +++ b/mtdata/resource/huggingface-datasets.jsonl @@ -0,0 +1,2 @@ +{"author": "ai4bharat", "downloads": 2265, "gated": false, "id": "ai4bharat/samanantar", "lastModified": "2024-12-24T08:13:04.000Z", "datasetsServerInfo": {"viewer": "viewer", "numRows": 49774246, "libraries": ["datasets", "dask", "mlcroissant", "polars"], "formats": ["parquet"], "modalities": ["text"]}, "private": false, "repoType": "dataset", "likes": 25, "isLikedByUser": false, "configs": [{"name": "as", "split": "train", "paths": ["as/train-*"]}, {"name": "bn", "split": "train", "paths": ["bn/train-*"]}, {"name": "gu", "split": "train", "paths": ["gu/train-*"]}, {"name": "hi", "split": "train", "paths": ["hi/train-*"]}, {"name": "kn", "split": "train", "paths": ["kn/train-*"]}, {"name": "ml", "split": "train", "paths": ["ml/train-*"]}, {"name": "mr", "split": "train", "paths": ["mr/train-*"]}, {"name": "or", "split": "train", "paths": ["or/train-*"]}, {"name": "pa", "split": "train", "paths": ["pa/train-*"]}, {"name": "ta", "split": "train", "paths": ["ta/train-*"]}, {"name": "te", "split": "train", "paths": ["te/train-*"]}]} +{"author": "google", "downloads": 3177, "gated": false, "id": "google/wmt24pp", "lastModified": "2025-02-19T16:37:15.000Z", "datasetsServerInfo": {"viewer": "viewer", "numRows": 54890, "libraries": ["datasets", "pandas", "mlcroissant", "polars"], "formats": ["json"], "modalities": ["text"]}, "private": false, "repoType": "dataset", "likes": 28, "isLikedByUser": false, "configs": [{"name": "en-ar_EG", "split": "train", "paths": ["en-ar_EG.jsonl"]}, {"name": "en-ar_SA", "split": "train", "paths": ["en-ar_SA.jsonl"]}, {"name": "en-bg_BG", "split": "train", "paths": ["en-bg_BG.jsonl"]}, {"name": "en-bn_IN", "split": "train", "paths": ["en-bn_IN.jsonl"]}, {"name": "en-ca_ES", "split": "train", "paths": ["en-ca_ES.jsonl"]}, {"name": "en-cs_CZ", "split": "train", "paths": ["en-cs_CZ.jsonl"]}, {"name": "en-da_DK", "split": "train", "paths": ["en-da_DK.jsonl"]}, {"name": "en-de_DE", "split": "train", "paths": ["en-de_DE.jsonl"]}, {"name": "en-el_GR", "split": "train", "paths": ["en-el_GR.jsonl"]}, {"name": "en-es_MX", "split": "train", "paths": ["en-es_MX.jsonl"]}, {"name": "en-et_EE", "split": "train", "paths": ["en-et_EE.jsonl"]}, {"name": "en-fa_IR", "split": "train", "paths": ["en-fa_IR.jsonl"]}, {"name": "en-fi_FI", "split": "train", "paths": ["en-fi_FI.jsonl"]}, {"name": "en-fil_PH", "split": "train", "paths": ["en-fil_PH.jsonl"]}, {"name": "en-fr_CA", "split": "train", "paths": ["en-fr_CA.jsonl"]}, {"name": "en-fr_FR", "split": "train", "paths": ["en-fr_FR.jsonl"]}, {"name": "en-gu_IN", "split": "train", "paths": ["en-gu_IN.jsonl"]}, {"name": "en-he_IL", "split": "train", "paths": ["en-he_IL.jsonl"]}, {"name": "en-hi_IN", "split": "train", "paths": ["en-hi_IN.jsonl"]}, {"name": "en-hr_HR", "split": "train", "paths": ["en-hr_HR.jsonl"]}, {"name": "en-hu_HU", "split": "train", "paths": ["en-hu_HU.jsonl"]}, {"name": "en-id_ID", "split": "train", "paths": ["en-id_ID.jsonl"]}, {"name": "en-is_IS", "split": "train", "paths": ["en-is_IS.jsonl"]}, {"name": "en-it_IT", "split": "train", "paths": ["en-it_IT.jsonl"]}, {"name": "en-ja_JP", "split": "train", "paths": ["en-ja_JP.jsonl"]}, {"name": "en-kn_IN", "split": "train", "paths": ["en-kn_IN.jsonl"]}, {"name": "en-ko_KR", "split": "train", "paths": ["en-ko_KR.jsonl"]}, {"name": "en-lt_LT", "split": "train", "paths": ["en-lt_LT.jsonl"]}, {"name": "en-lv_LV", "split": "train", "paths": ["en-lv_LV.jsonl"]}, {"name": "en-ml_IN", "split": "train", "paths": ["en-ml_IN.jsonl"]}, {"name": "en-mr_IN", "split": "train", "paths": ["en-mr_IN.jsonl"]}, {"name": "en-nl_NL", "split": "train", "paths": ["en-nl_NL.jsonl"]}, {"name": "en-no_NO", "split": "train", "paths": ["en-no_NO.jsonl"]}, {"name": "en-pa_IN", "split": "train", "paths": ["en-pa_IN.jsonl"]}, {"name": "en-pl_PL", "split": "train", "paths": ["en-pl_PL.jsonl"]}, {"name": "en-pt_BR", "split": "train", "paths": ["en-pt_BR.jsonl"]}, {"name": "en-pt_PT", "split": "train", "paths": ["en-pt_PT.jsonl"]}, {"name": "en-ro_RO", "split": "train", "paths": ["en-ro_RO.jsonl"]}, {"name": "en-ru_RU", "split": "train", "paths": ["en-ru_RU.jsonl"]}, {"name": "en-sk_SK", "split": "train", "paths": ["en-sk_SK.jsonl"]}, {"name": "en-sl_SI", "split": "train", "paths": ["en-sl_SI.jsonl"]}, {"name": "en-sr_RS", "split": "train", "paths": ["en-sr_RS.jsonl"]}, {"name": "en-sv_SE", "split": "train", "paths": ["en-sv_SE.jsonl"]}, {"name": "en-sw_KE", "split": "train", "paths": ["en-sw_KE.jsonl"]}, {"name": "en-sw_TZ", "split": "train", "paths": ["en-sw_TZ.jsonl"]}, {"name": "en-ta_IN", "split": "train", "paths": ["en-ta_IN.jsonl"]}, {"name": "en-te_IN", "split": "train", "paths": ["en-te_IN.jsonl"]}, {"name": "en-th_TH", "split": "train", "paths": ["en-th_TH.jsonl"]}, {"name": "en-tr_TR", "split": "train", "paths": ["en-tr_TR.jsonl"]}, {"name": "en-uk_UA", "split": "train", "paths": ["en-uk_UA.jsonl"]}, {"name": "en-ur_PK", "split": "train", "paths": ["en-ur_PK.jsonl"]}, {"name": "en-vi_VN", "split": "train", "paths": ["en-vi_VN.jsonl"]}, {"name": "en-zh_CN", "split": "train", "paths": ["en-zh_CN.jsonl"]}, {"name": "en-zh_TW", "split": "train", "paths": ["en-zh_TW.jsonl"]}, {"name": "en-zu_ZA", "split": "train", "paths": ["en-zu_ZA.jsonl"]}]} diff --git a/setup.py b/setup.py index 9209cd8..a74dc14 100644 --- a/setup.py +++ b/setup.py @@ -62,6 +62,10 @@ 'pybtex==0.24.0', 'ruamel.yaml >= 0.17.10', ], + extras_require={ + "hf": ["datasets>=3.3.0",], + }, + include_package_data=True, zip_safe=False, ) diff --git a/tests/__init__.py b/tests/__init__.py index ca0e7db..f0fefbe 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,4 +1,7 @@ #!/usr/bin/env python # -# Author: Thamme Gowda [tg (at) isi (dot) edu] +# Author: Thamme Gowda [tg (at) isi (dot) edu] # Created: 4/23/20 + +import sys +MTDATA_CMD = f"{sys.executable} -m mtdata" \ No newline at end of file diff --git a/tests/test_huggingface.py b/tests/test_huggingface.py new file mode 100644 index 0000000..d54de9a --- /dev/null +++ b/tests/test_huggingface.py @@ -0,0 +1,15 @@ + +from . import MTDATA_CMD + +import subprocess as sp + + +def test_hf_echo(): + """Test the hf echo command.""" + data_id = "Google-wmt24pp-1-eng-zho_TW" # an example dataset from HF + expected_lines = 998 + cmd = f"{MTDATA_CMD} echo {data_id}" + result = sp.run(cmd, shell=True, capture_output=True, text=True) + assert result.returncode == 0, f"Command failed with error: {result.stderr}" + out_lines = result.stdout.strip().splitlines() + assert len(out_lines) == expected_lines, f"Expected {expected_lines} lines, but got {len(out_lines)} lines." \ No newline at end of file From f3d34d19149a5d4e6eb2281943f17a922d307929 Mon Sep 17 00:00:00 2001 From: Thamme Gowda Date: Mon, 10 Mar 2025 09:00:43 -0700 Subject: [PATCH 02/13] setup.py -> pyproject.toml; huggingface datasets as optional dependency --- .github/workflows/python-build-test.yml | 2 +- pyproject.toml | 63 ++++++++++++++++++++++ setup.py | 71 ------------------------- 3 files changed, 64 insertions(+), 72 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.py diff --git a/.github/workflows/python-build-test.yml b/.github/workflows/python-build-test.yml index 9085bce..33140f4 100644 --- a/.github/workflows/python-build-test.yml +++ b/.github/workflows/python-build-test.yml @@ -33,7 +33,7 @@ jobs: pip install setuptools==61.2 flake8 pytest-cov - name: Install module run: | - python3 setup.py install + pip install .[hf] - name: Test with pytest run: | python3 -m pytest diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..a1d7189 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,63 @@ +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "mtdata" +dynamic = ["version"] +description = "mtdata is a tool to download datasets for machine translation" +readme = "README.md" +license = { file = "LICENSE" } +requires-python = ">=3.7" +authors = [ + { name = "Thamme Gowda", email = "tgowdan@gmail.com" } +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Topic :: Utilities", + "Topic :: Text Processing", + "Topic :: Text Processing :: General", + "Topic :: Text Processing :: Filters", + "Topic :: Text Processing :: Linguistic", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3 :: Only" +] +keywords = [ + "machine translation", + "datasets", + "NLP", + "natural language processing", + "computational linguistics" +] +dependencies = [ + "requests>=2.31.0", + "enlighten==1.10.1", + "portalocker==2.3.0", + "pybtex==0.24.0", + "ruamel.yaml >= 0.17.10" +] + +[project.urls] +homepage = "https://github.com/thammegowda/mtdata" +documentation = "https://github.com/thammegowda/mtdata" +repository = "https://github.com/thammegowda/mtdata" +#changelog = "" + +[tool.setuptools.packages.find] +include = ["mtdata*"] # ["*"] by default +exclude = ["tests*", "tmp*", "build*", "dist*", "crawler*"] + + +[project.optional-dependencies] +hf = ["datasets>=3.3.0"] +test = [ "pytest", "pytest-cov[all]", "black", "isort", "mypy"] + +[project.scripts] +mtdata = "mtdata.__main__:main" +mtdata-iso = "mtdata.iso.__main__:main" +mtdata-bcp47 = "mtdata.iso.bcp47:main" + + +[tool.setuptools.dynamic] +version = {attr = "mtdata.__version__"} \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index a74dc14..0000000 --- a/setup.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python -# -# Author: Thamme Gowda [tg (at) isi (dot) edu] -# Created: 4/6/20 - -import re -from pathlib import Path - -from setuptools import setup, find_namespace_packages - -long_description = Path('README.md').read_text(encoding='utf-8', errors='ignore') - -classifiers = [ # copied from https://pypi.org/classifiers/ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Topic :: Utilities', - 'Topic :: Text Processing', - 'Topic :: Text Processing :: General', - 'Topic :: Text Processing :: Filters', - 'Topic :: Text Processing :: Linguistic', - 'License :: OSI Approved :: Apache Software License', - 'Programming Language :: Python :: 3 :: Only', -] - -init_file = Path(__file__).parent / 'mtdata' / '__init__.py' -init_txt = init_file.read_text() -version_re = re.compile(r'''__version__ = ['"]([0-9.]+(-dev)?)['"]''') -__version__ = version_re.search(init_txt).group(1) -desc_re = re.compile(r'''__description__ = ['"](.*)['"]''') -__description__ = desc_re.search(init_txt).group(1) -assert __version__ -assert __description__ - - -setup( - name='mtdata', - version=__version__, - description=__description__, - long_description=long_description, - long_description_content_type='text/markdown', - classifiers=classifiers, - python_requires='>=3.7', - url='https://github.com/thammegowda/mtdata', - download_url='https://github.com/thammegowda/mtdata', - platforms=['any'], - author='Thamme Gowda', - author_email='tgowdan@gmail.com', - packages=find_namespace_packages(include=['mtdata*']), - keywords=['machine translation', 'datasets', 'NLP', 'natural language processing,' - 'computational linguistics'], - entry_points={ - 'console_scripts': [ - 'mtdata=mtdata.__main__:main', - 'mtdata-iso=mtdata.iso.__main__:main', - 'mtdata-bcp47=mtdata.iso.bcp47:main', - ], - }, - install_requires=[ - 'requests==2.31.0', - 'enlighten==1.10.1', - 'portalocker==2.3.0', - 'pybtex==0.24.0', - 'ruamel.yaml >= 0.17.10', - ], - extras_require={ - "hf": ["datasets>=3.3.0",], - }, - - include_package_data=True, - zip_safe=False, -) From e78e7564cf5d34c6ae0a9dbace664759fa6b5687 Mon Sep 17 00:00:00 2001 From: Thamme Gowda Date: Mon, 10 Mar 2025 09:11:07 -0700 Subject: [PATCH 03/13] set version 0.4.3-dev and update changelog --- CHANGELOG.md | 11 +++++++++++ mtdata/__init__.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f97101..d71251b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Change Log +## 0.4.3 - WIP +* Add preliminary support for huggingface datasets; currently wmt24++ is the only supported dataset +* Update setup.py -> pyproject.toml; hf datasets is optional dependency +* Add mtdata index subcommand. deprecate `mtdata --reindex ` +* Add meta, a dictionary field to Entry to store arbitrary key-vals which maybe useful for downloading and parsing datasets. +* Preliminary support for document id , (currently, one among the many in meta fields) + + +## v0.4.2 +- minor fixes + ## v0.4.1 - 20240425 * Better parallelization: parallel and mono data are scheduled at once (previously it was one after the other) * `mtdata cache` added. Improves concurrency by supporting multiple recipes diff --git a/mtdata/__init__.py b/mtdata/__init__.py index abaabdc..6c1ed92 100644 --- a/mtdata/__init__.py +++ b/mtdata/__init__.py @@ -4,7 +4,7 @@ # Created: 4/4/20 -__version__ = '0.4.2' +__version__ = '0.4.3-dev' __description__ = 'mtdata is a tool to download datasets for machine translation' __author__ = 'Thamme Gowda' From 885eaf0832c5489aa20454eb5a514da876182f1c Mon Sep 17 00:00:00 2001 From: Thamme Gowda Date: Tue, 11 Mar 2025 19:08:55 -0700 Subject: [PATCH 04/13] add wmt24 and wmt25 constrained recipes --- recipes/mtdata.recipes.wmt24-constrained.yml | 471 +++++++++++++++ recipes/mtdata.recipes.wmt25-constrained.yml | 599 +++++++++++++++++++ 2 files changed, 1070 insertions(+) create mode 100644 recipes/mtdata.recipes.wmt24-constrained.yml create mode 100644 recipes/mtdata.recipes.wmt25-constrained.yml diff --git a/recipes/mtdata.recipes.wmt24-constrained.yml b/recipes/mtdata.recipes.wmt24-constrained.yml new file mode 100644 index 0000000..2979846 --- /dev/null +++ b/recipes/mtdata.recipes.wmt24-constrained.yml @@ -0,0 +1,471 @@ +# Setup: pip install mtdata==0.4.0 +# To list/view all available datasets: +# mtdata list -id -l - # parallel +# mtdata list -id -l # monolingual +# To get a dataset +# mtdata echo + +- id: wmt24-eng-ces + langs: eng-ces + train: + - Statmt-europarl-10-ces-eng + - ParaCrawl-paracrawl-9-eng-ces + - Statmt-commoncrawl_wmt13-1-ces-eng + - Statmt-news_commentary-18.1-ces-eng + - Statmt-wikititles-3-ces-eng + - Facebook-wikimatrix-1-ces-eng + - Tilde-eesc-2017-ces-eng + - Tilde-ema-2016-ces-eng + - Tilde-ecb-2017-ces-eng + - Tilde-rapid-2019-ces-eng + # TODO: manually download bracktranslated news and CzEng2.0 + mono_train: + - Statmt-news_crawl-2023-ces + - Statmt-europarl-10-ces + - Statmt-news_commentary-18.1-ces + - Statmt-commoncrawl-wmt22-ces + - Leipzig-news-2022_1m-ces + - Leipzig-newscrawl-2019_1m-ces + - Leipzig-wikipedia-2021_1m-ces + - Leipzig-web_public-2019_1m-ces_CZ + # TODO: extended common crawl (too big) https://data.statmt.org/wmt21/translation-task/cc-mono/ + +- id: wmt24-eng-zho + langs: eng-zho + train: + - ParaCrawl-paracrawl-1_bonus-eng-zho + - Statmt-news_commentary-18.1-eng-zho + - Statmt-wikititles-3-zho-eng + - OPUS-unpc-v1.0-eng-zho + - Facebook-wikimatrix-1-eng-zho + - Statmt-backtrans_enzh-wmt20-eng-zho # English translated to Chinese. Double check if you want this + mono_train: &mono_zho + - Statmt-news_crawl-2023-zho + - Statmt-news_commentary-18.1-zho + - Statmt-commoncrawl-wmt22-zho + - Leipzig-wikipedia-2018_1m-zho + - Leipzig-web-2016_1m-zho_MO + - Leipzig-tradnewscrawl-2011_1m-zho + - Leipzig-news-2020_300k-zho + # TODO: extended common crawl (too big) https://data.statmt.org/wmt21/translation-task/cc-mono/ + +- id: wmt24-eng-deu + langs: eng-deu + train: + - Statmt-europarl-10-deu-eng + - ParaCrawl-paracrawl-9-eng-deu + - Statmt-commoncrawl_wmt13-1-deu-eng + - Statmt-news_commentary-18.1-deu-eng + - Statmt-wikititles-3-deu-eng + - Facebook-wikimatrix-1-deu-eng + - Tilde-eesc-2017-deu-eng + - Tilde-ema-2016-deu-eng + - Tilde-airbaltic-1-deu-eng + - Tilde-czechtourism-1-deu-eng + - Tilde-ecb-2017-deu-eng + - Tilde-rapid-2016-deu-eng + - Tilde-rapid-2019-deu-eng + mono_train: + - Statmt-news_crawl-2023-deu + - Statmt-europarl-10-deu + - Statmt-news_commentary-18.1-deu + - Statmt-commoncrawl-wmt22-deu + - Leipzig-wikipedia-2021_1m-deu + - Leipzig-comweb-2021_1m-deu + - Leipzig-mixed_typical-2011_1m-deu + - Leipzig-news-2022_30k-deu + - Leipzig-newscrawl-2020_1m-deu + - Leipzig-web-2021_100k-deu_DE + # TODO: extended common crawl +- id: wmt24-eng-spa + langs: eng-spa + train: + - Statmt-commoncrawl_wmt13-1-spa-eng + - Statmt-europarl-7-spa-eng + - Statmt-news_commentary-18.1-eng-spa + - Statmt-ccaligned-1-eng-spa + - ParaCrawl-paracrawl-9-eng-spa + - Tilde-eesc-2017-eng-spa + - Tilde-ema-2016-eng-spa + - Tilde-czechtourism-1-eng-spa + - Tilde-ecb-2017-eng-spa + - Tilde-rapid-2016-eng-spa + - Tilde-worldbank-1-eng-spa + - Facebook-wikimatrix-1-eng-spa + - EU-ecdc-1-eng-spa + - EU-eac_forms-1-eng-spa + - EU-eac_reference-1-eng-spa + - EU-dcep-1-eng-spa + - LinguaTools-wikititles-2014-eng-spa + - Neulab-tedtalks_train-1-eng-spa + - OPUS-books-v1-eng-spa + - OPUS-dgt-v2019-eng-spa + - OPUS-dgt-v4-eng-spa + - OPUS-ecb-v1-eng-spa + - OPUS-elitr_eca-v1-eng-spa + - OPUS-elra_w0147-v1-eng-spa + - OPUS-elra_w0305-v1-eng-spa + - OPUS-elrc_1076_euipo_law-v1-eng-spa + - OPUS-elrc_1082_cnio-v1-eng-spa + - OPUS-elrc_1083_aecosan-v1-eng-spa + - OPUS-elrc_1084_agencia_tributaria-v1-eng-spa + - OPUS-elrc_1096_euipo_list-v1-eng-spa + - OPUS-elrc_1125_cordis_news-v1-eng-spa + - OPUS-elrc_1126_cordis_results_brief-v1-eng-spa + - OPUS-elrc_2015_euipo_2017-v1-eng-spa + - OPUS-elrc_2410_portal_oficial_turis-v1-eng-spa + - OPUS-elrc_2478_glossrio_pt_en-v1-eng-spa + - OPUS-elrc_2479_lei_orgnica_2-v1-eng-spa + - OPUS-elrc_2480_estatuto_dos_deputad-v1-eng-spa + - OPUS-elrc_2481_constituio_da_repbli-v1-eng-spa + - OPUS-elrc_2498_plan_nacional_e-v1-eng-spa + - OPUS-elrc_2502_termitur-v1-eng-spa + - OPUS-elrc_2503_descripciones_vulner-v1-eng-spa + - OPUS-elrc_2536_estatuto_da_vtima-v1-eng-spa + - OPUS-elrc_2538_lei_25_2009-v1-eng-spa + - OPUS-elrc_2543_inteliterm-v1-eng-spa + - OPUS-elrc_2558_government_websites_-v1-eng-spa + - OPUS-elrc_2612_artigos_visitportuga-v1-eng-spa + - OPUS-elrc_2614_localidades_2007-v1-eng-spa + - OPUS-elrc_2616_museus_2007-v1-eng-spa + - OPUS-elrc_2622_arquitectura_2007-v1-eng-spa + - OPUS-elrc_2623_patrimnio_aores_2006-v1-eng-spa + - OPUS-elrc_2638_monumentos_2007-v1-eng-spa + - OPUS-elrc_2639_parques_e_reservas-v1-eng-spa + - OPUS-elrc_2641_praias_2007-v1-eng-spa + - OPUS-elrc_2722_emea-v1-eng-spa + - OPUS-elrc_2738_vaccination-v1-eng-spa + - OPUS-elrc_2881_eu_publications_medi-v1-eng-spa + - OPUS-elrc_3077_wikipedia_health-v1-eng-spa + - OPUS-elrc_3210_antibiotic-v1-eng-spa + - OPUS-elrc_3299_europarl_covid-v1-eng-spa + - OPUS-elrc_3470_ec_europa_covid-v1-eng-spa + - OPUS-elrc_3571_eur_lex_covid-v1-eng-spa + - OPUS-elrc_3612_presscorner_covid-v1-eng-spa + - OPUS-elrc_3852_development_funds_re-v1-eng-spa + - OPUS-elrc_401_swedish_labour_part2-v1-eng-spa + - OPUS-elrc_406_swedish_labour_part1-v1-eng-spa + - OPUS-elrc_416_swedish_social_secur-v1-eng-spa + - OPUS-elrc_417_swedish_work_environ-v1-eng-spa + - OPUS-elrc_436_swedish_food-v1-eng-spa + - OPUS-elrc_4992_customer_support_mt-v1-eng-spa + - OPUS-elrc_5067_scipar-v1-eng-spa + - OPUS-elrc_5190_cyber_mt_test-v1-eng-spa + - OPUS-elrc_637_sip-v1-eng-spa + - OPUS-elrc_832_charter_values_citiz-v1-eng-spa + - OPUS-elrc_844_paesi__administratio-v1-eng-spa + - OPUS-elrc_863_government_websites_-v1-eng-spa + - OPUS-elrc_arquitectura_2007-v1-eng-spa + - OPUS-elrc_artigos_visitportuga-v1-eng-spa + - OPUS-elrc_cordis_news-v1-eng-spa + - OPUS-elrc_cordis_results-v1-eng-spa + - OPUS-elrc_ec_europa-v1-eng-spa + - OPUS-elrc_emea-v1-eng-spa + - OPUS-elrc_euipo_2017-v1-eng-spa + - OPUS-elrc_euipo_law-v1-eng-spa + - OPUS-elrc_euipo_list-v1-eng-spa + - OPUS-elrc_europarl_covid-v1-eng-spa + - OPUS-elrc_eur_lex-v1-eng-spa + - OPUS-elrc_eu_publications-v1-eng-spa + - OPUS-elrc_localidades_2007-v1-eng-spa + - OPUS-elrc_museus_2007-v1-eng-spa + - OPUS-elrc_parques_e-v1-eng-spa + - OPUS-elrc_patrimnio_aores-v1-eng-spa + - OPUS-elrc_praias_2007-v1-eng-spa + - OPUS-elrc_swedish_labour-v1-eng-spa + - OPUS-elrc_termitur-v1-eng-spa + - OPUS-elrc_antibiotic-v1-eng-spa + - OPUS-elrc_government_websites-v1-eng-spa + - OPUS-elrc_presscorner_covid-v1-eng-spa + - OPUS-elrc_vaccination-v1-eng-spa + - OPUS-elrc_wikipedia_health-v1-eng-spa + - OPUS-elrc_2682-v1-eng-spa + - OPUS-elrc_2922-v1-eng-spa + - OPUS-elrc_2923-v1-eng-spa + - OPUS-elrc_3382-v1-eng-spa + - OPUS-emea-v3-eng-spa + - OPUS-eubookshop-v2-eng-spa + - OPUS-euconst-v1-eng-spa + - OPUS-europat-v3-eng-spa + - OPUS-europarl-v8-eng-spa + - OPUS-globalvoices-v2018q4-eng-spa + - OPUS-multiccaligned-v1-eng-spa + - OPUS-multiun-v1-eng-spa + - OPUS-unpc-v1.0-eng-spa + - OPUS-wikimatrix-v1-eng-spa + - OPUS-wikipedia-v1.0-eng-spa + - OPUS-xlent-v1.1-eng-spa + - OPUS-infopankki-v1-eng-spa + - OPUS-tico_19-v20201028-eng-spa + - OPUS-wikimedia-v20210402-eng-spa + mono_train: + - Statmt-europarl-10-spa + - Statmt-news_commentary-18.1-spa + - Statmt-news_crawl-2023-spa + - Leipzig-news-2022_1m-spa + - Leipzig-newscrawl_public-2019_1m-spa + - Leipzig-web-2016_1m-spa + - Leipzig-web-2016_1m-spa_AR + - Leipzig-web-2016_1m-spa_PA + - Leipzig-web-2016_1m-spa_PE + - Leipzig-web-2016_1m-spa_PY + - Leipzig-web-2016_1m-spa_SV + - Leipzig-web-2016_1m-spa_UY + - Leipzig-web-2016_1m-spa_VE + - Leipzig-wikipedia-2021_1m-spa + + +- id: wmt24-eng-jpn + langs: eng-jpn + train: + - Statmt-news_commentary-18.1-eng-jpn + - KECL-paracrawl-3-eng-jpn + - Statmt-wikititles-3-jpn-eng + - Facebook-wikimatrix-1-eng-jpn + - Statmt-ted-wmt20-eng-jpn + - StanfordNLP-jesc_train-1-eng-jpn + - Phontron-kftt_train-1-eng-jpn + mono_train: &mono_jpn + - Statmt-news_crawl-2023-jpn + - Statmt-news_commentary-18.1-jpn + - Statmt-commoncrawl-wmt22-jpn + - Leipzig-web-2020_1m-jpn_JP + - Leipzig-comweb-2018_1m-jpn + - Leipzig-web_public-2019_1m-jpn_JP + - Leipzig-news-2020_100k-jpn + - Leipzig-newscrawl-2019_1m-jpn + - Leipzig-wikipedia-2021_1m-jpn + # TODO: Extended Common Crawl + +- id: wmt24-eng-rus + langs: eng-rus + train: + - ParaCrawl-paracrawl-1_bonus-eng-rus + - Statmt-commoncrawl_wmt13-1-rus-eng + - Statmt-news_commentary-18.1-eng-rus + - Statmt-yandex-wmt22-eng-rus + - Statmt-wikititles-3-rus-eng + - OPUS-unpc-v1.0-eng-rus + - Facebook-wikimatrix-1-eng-rus + - Tilde-airbaltic-1-eng-rus + - Tilde-czechtourism-1-eng-rus + - Tilde-worldbank-1-eng-rus + - Statmt-backtrans_ruen-wmt20-rus-eng # russian translated to english + mono_train: &mono_rus + - Statmt-news_crawl-2023-rus + - Statmt-news_commentary-18.1-rus + - Statmt-commoncrawl-wmt22-rus + - Leipzig-news-2022_1m-rus + - Leipzig-newscrawl_public-2018_1m-rus + - Leipzig-web-2017_1m-rus_GE + - Leipzig-wikipedia-2021_1m-rus + +- id: wmt24-ces-ukr + langs: ces-ukr + train: + - Facebook-wikimatrix-1-ces-ukr + - ELRC-acts_ukrainian-1-ces-ukr + - OPUS-ccmatrix-v1-ces-ukr + - OPUS-elrc_5179_acts_ukrainian-v1-ces-ukr + - OPUS-elrc_wikipedia_health-v1-ces-ukr + - OPUS-eubookshop-v2-ces-ukr + - OPUS-gnome-v1-ces-ukr + - OPUS-kde4-v2-ces-ukr + - OPUS-multiccaligned-v1.1-ces-ukr + - OPUS-multiparacrawl-v9b-ces-ukr + - OPUS-opensubtitles-v2018-ces-ukr + - OPUS-qed-v2.0a-ces-ukr + - OPUS-ted2020-v1-ces-ukr + - OPUS-tatoeba-v20220303-ces-ukr + - OPUS-ubuntu-v14.10-ces-ukr + - OPUS-xlent-v1.1-ces-ukr + - OPUS-bible_uedin-v1-ces-ukr + - OPUS-wikimedia-v20210402-ces-ukr + mono_train: &mono_ukr + - Statmt-news_crawl-2023-ukr + - LangUk-news-1-ukr + - LangUk-wiki_dump-1-ukr + - LangUk-fiction-1-ukr + - LangUk-ubercorpus-1-ukr + - LangUk-laws-1-ukr + - Leipzig-news-2022_1m-ukr + - Leipzig-newscrawl-2018_1m-ukr + - Leipzig-web-2019_1m-ukr_UA + - Leipzig-wikipedia-2021_1m-ukr + +- id: wmt24-eng-ukr + langs: eng-ukr + train: ¶_eng_ukr + - ParaCrawl-paracrawl-1_bonus-eng-ukr + - Tilde-worldbank-1-eng-ukr + - Facebook-wikimatrix-1-eng-ukr + - ELRC-acts_ukrainian-1-eng-ukr + - Statmt-ccaligned-1-eng-ukr_UA + + mono_train: *mono_ukr + +- id: wmt24-eng-hin + langs: eng-hin + train: + - Statmt-news_commentary-18.1-eng-hin + - Statmt-pmindia-1-eng-hin + - Statmt-ccaligned-1-eng-hin_IN + - JoshuaDec-indian_training-1-hin-eng + - Facebook-wikimatrix-1-eng-hin + - IITB-hien_train-1.5-hin-eng + - Neulab-tedtalks_train-1-eng-hin + - ELRC-wikipedia_health-1-eng-hin + - AI4Bharath-samananthar-0.2-eng-hin + - Anuvaad-internal_judicial_2021-v1-eng-hin + - Anuvaad-legal_terms_2021-v1-eng-hin + - Anuvaad-pib_2017-2020-eng-hin + - Anuvaad-pibarchives_2009-2016-eng-hin + - Anuvaad-wikipedia-20210201-eng-hin + - Anuvaad-drivespark-20210303-eng-hin + - Anuvaad-nativeplanet-20210315-eng-hin + - Anuvaad-catchnews-20210320-eng-hin + - Anuvaad-dwnews_2008-2020-eng-hin + - Anuvaad-oneindia-20210320-eng-hin + - Anuvaad-mk-20210320-eng-hin + - Anuvaad-goodreturns-20210320-eng-hin + - Anuvaad-ie_sports-20210320-eng-hin + - Anuvaad-ie_tech-20210320-eng-hin + - Anuvaad-ie_news-20210320-eng-hin + - Anuvaad-ie_lifestyle-20210320-eng-hin + - Anuvaad-ie_general-20210320-eng-hin + - Anuvaad-ie_entertainment-20210320-eng-hin + - Anuvaad-ie_education-20210320-eng-hin + - Anuvaad-ie_business-20210320-eng-hin + - Anuvaad-fin_express-20210320-eng-hin + - Anuvaad-thewire-20210320-eng-hin + - Anuvaad-tribune-20210320-eng-hin + - Anuvaad-zeebiz-20210320-eng-hin + - Anuvaad-pa_govt-20210320-eng-hin + - Anuvaad-betterindia-20210320-eng-hin + - Anuvaad-jagran_news-20210320-eng-hin + - Anuvaad-jagran_tech-20210320-eng-hin + - Anuvaad-jagran_education-20210320-eng-hin + - Anuvaad-jagran_entertainment-20210320-eng-hin + - Anuvaad-jagran_business-20210320-eng-hin + - Anuvaad-jagran_sports-20210320-eng-hin + - Anuvaad-jagran_lifestyle-20210320-eng-hin + - Anuvaad-asianetnews-20210320-eng-hin + - Anuvaad-business_standard-20210320-eng-hin + - Anuvaad-pranabmukherjee-20210320-eng-hin + - Anuvaad-lokmat_entertainment-20210501-eng-hin + - Anuvaad-lokmat_news-20210501-eng-hin + - Anuvaad-lokmat_lifestyle-20210501-eng-hin + - Anuvaad-lokmat_sports-20210501-eng-hin + - Anuvaad-lokmat_tech-20210501-eng-hin + - Anuvaad-lokmat_financial-20210501-eng-hin + - Anuvaad-lokmat_healthcare-20210501-eng-hin + - AllenAi-nllb-1-eng-hin + - OPUS-elrc_wikipedia_health-v1-eng-hin + - OPUS-elrc_2922-v1-eng-hin + - OPUS-globalvoices-v2018q4-eng-hin + - OPUS-iitb-v2.0-eng-hin + - OPUS-multiccaligned-v1-eng-hin + - OPUS-opensubtitles-v2018-eng-hin + - OPUS-ted2020-v1-eng-hin + - OPUS-tanzil-v1-eng-hin + - OPUS-tatoeba-v20220303-eng-hin + - OPUS-ubuntu-v14.10-eng-hin + - OPUS-xlent-v1.1-eng-hin + - OPUS-tico_19-v20201028-eng-hin + - OPUS-wikimedia-v20210402-eng-hin + + mono_train: + - Statmt-news_crawl-2023-hin + - Statmt-news_commentary-18.1-hin + - Leipzig-web-2015_1m-hin_IN + - Leipzig-mixed-2019_1m-hin + - Leipzig-news-2020_1m-hin + - Leipzig-newscrawl-2017_1m-hin + - Leipzig-wikipedia-2021_1m-hin + +- id: wmt24-eng-isl + langs: eng-isl + train: + - Statmt-wikititles-3-isl-eng + - Statmt-ccaligned-1-eng-isl_IS + - ParaCrawl-paracrawl-9-eng-isl + - Tilde-eesc-2017-eng-isl + - Tilde-ema-2016-eng-isl + - Tilde-rapid-2016-eng-isl + - Facebook-wikimatrix-1-eng-isl + - ParIce-eea_train-20.05-eng-isl + - ParIce-ema_train-20.05-eng-isl + - EU-ecdc-1-eng-isl + - EU-eac_forms-1-eng-isl + - EU-eac_reference-1-eng-isl + - OPUS-ccmatrix-v1-eng-isl + - OPUS-elrc_2718_emea-v1-eng-isl + - OPUS-elrc_3206_antibiotic-v1-eng-isl + - OPUS-elrc_4295_www.malfong.is-v1-eng-isl + - OPUS-elrc_4324_government_offices_i-v1-eng-isl + - OPUS-elrc_4327_government_offices_i-v1-eng-isl + - OPUS-elrc_4334_rkiskaup_2020-v1-eng-isl + - OPUS-elrc_4338_university_iceland-v1-eng-isl + - OPUS-elrc_502_icelandic_financial_-v1-eng-isl + - OPUS-elrc_504_www.iceida.is-v1-eng-isl + - OPUS-elrc_505_www.pfs.is-v1-eng-isl + - OPUS-elrc_506_www.lanamal.is-v1-eng-isl + - OPUS-elrc_5067_scipar-v1-eng-isl + - OPUS-elrc_508_tilde_statistics_ice-v1-eng-isl + - OPUS-elrc_509_gallery_iceland-v1-eng-isl + - OPUS-elrc_510_harpa_reykjavik_conc-v1-eng-isl + - OPUS-elrc_511_bokmenntaborgin_is-v1-eng-isl + - OPUS-elrc_516_icelandic_medicines-v1-eng-isl + - OPUS-elrc_517_icelandic_directorat-v1-eng-isl + - OPUS-elrc_597_www.nordisketax.net-v1-eng-isl + - OPUS-elrc_718_statistics_iceland-v1-eng-isl + - OPUS-elrc_728_www.norden.org-v1-eng-isl + - OPUS-elrc_emea-v1-eng-isl + - OPUS-elrc_antibiotic-v1-eng-isl + - OPUS-elrc_www.norden.org-v1-eng-isl + - OPUS-elrc_www.nordisketax.net-v1-eng-isl + - OPUS-eubookshop-v2-eng-isl + - OPUS-multiccaligned-v1-eng-isl + - OPUS-multiparacrawl-v7.1-eng-isl + - OPUS-opensubtitles-v2018-eng-isl + - OPUS-ted2020-v1-eng-isl + - OPUS-tatoeba-v20220303-eng-isl + - OPUS-ubuntu-v14.10-eng-isl + - OPUS-wikimatrix-v1-eng-isl + - OPUS-wikititles-v3-eng-isl + - OPUS-xlent-v1.1-eng-isl + - OPUS-wikimedia-v20210402-eng-isl + mono_train: + - Statmt-news_crawl-2023-isl + - Leipzig-web-2020_1m-isl_IS + - Leipzig-web_public-2019_1m-isl_IS + - Leipzig-news-2020_30k-isl + - Leipzig-newscrawl-2019_300k-isl + - Leipzig-wikipedia-2021_100k-isl + +- id: wmt24-jpn-zho + langs: jpn-zho + train: + - Statmt-news_commentary-18.1-jpn-zho + - KECL-paracrawl-2-zho-jpn + - KECL-paracrawl-2wmt24-zho-jpn + - Facebook-wikimatrix-1-jpn-zho + - Neulab-tedtalks_train-1-jpn-zho + - LinguaTools-wikititles-2014-jpn-zho + - OPUS-ccmatrix-v1-jpn-zho + - OPUS-gnome-v1-jpn-zho_CN + - OPUS-kde4-v2-jpn-zho_CN + - OPUS-multiccaligned-v1-jpn-zho_CN + - OPUS-openoffice-v3-jpn-zho_CN + - OPUS-opensubtitles-v2018-jpn-zho_CN + - OPUS-php-v1-jpn-zho + - OPUS-qed-v2.0a-jpn-zho + - OPUS-ted2020-v1-jpn-zho + - OPUS-tanzil-v1-jpn-zho + - OPUS-ubuntu-v14.10-jpn-zho + - OPUS-ubuntu-v14.10-jpn-zho_CN + - OPUS-xlent-v1.1-jpn-zho + - OPUS-bible_uedin-v1-jpn-zho + - OPUS-wikimedia-v20210402-jpn-zho + + mono_train: *mono_zho \ No newline at end of file diff --git a/recipes/mtdata.recipes.wmt25-constrained.yml b/recipes/mtdata.recipes.wmt25-constrained.yml new file mode 100644 index 0000000..744a7b5 --- /dev/null +++ b/recipes/mtdata.recipes.wmt25-constrained.yml @@ -0,0 +1,599 @@ +# Setup: pip install mtdata==0.4.2 +# To list all the available datasets, use the following commands +# mtdata list -id -l - # parallel +# mtdata list -id -l # monolingual +# To get a dataset +# mtdata echo +########## CES-UKR ######### +- id: wmt25-ces-ukr + langs: ces-ukr + train: + - Facebook-wikimatrix-1-ces-ukr + - ELRC-acts_ukrainian-1-ces-ukr + - OPUS-ccmatrix-v1-ces-ukr + - OPUS-elrc_5179_acts_ukrainian-v1-ces-ukr + - OPUS-elrc_wikipedia_health-v1-ces-ukr + - OPUS-eubookshop-v2-ces-ukr + - OPUS-gnome-v1-ces-ukr + - OPUS-kde4-v2-ces-ukr + - OPUS-multiccaligned-v1.1-ces-ukr + - OPUS-multiparacrawl-v9b-ces-ukr + - OPUS-opensubtitles-v2018-ces-ukr + - OPUS-qed-v2.0a-ces-ukr + - OPUS-ted2020-v1-ces-ukr + - OPUS-tatoeba-v20220303-ces-ukr + - OPUS-ubuntu-v14.10-ces-ukr + - OPUS-xlent-v1.1-ces-ukr + - OPUS-bible_uedin-v1-ces-ukr + - OPUS-wikimedia-v20210402-ces-ukr + mono_train: &mono_ukr + - Statmt-news_crawl-2023-ukr + - LangUk-news-1-ukr + - LangUk-wiki_dump-1-ukr + - LangUk-fiction-1-ukr + - LangUk-ubercorpus-1-ukr + - LangUk-laws-1-ukr + - Leipzig-news-2022_1m-ukr + - Leipzig-newscrawl-2018_1m-ukr + - Leipzig-web-2019_1m-ukr_UA + - Leipzig-wikipedia-2021_1m-ukr + +###############CES_DEU######################## +- id: wmt25-ces-deu + langs: ces-deu + train: + - Statmt-news_commentary-18.1-ces-deu + - Tilde-eesc-2017-ces-deu + - Tilde-ema-2016-ces-deu + - Tilde-ecb-2017-ces-deu + - Tilde-rapid-2016-ces-deu + - Facebook-wikimatrix-1-ces-deu + - LinguaTools-wikititles-2014-ces-deu + - OPUS-ccmatrix-v1-ces-deu + - OPUS-dgt-v2019-ces-deu + - OPUS-dgt-v4-ces-deu + - OPUS-ecb-v1-ces-deu + - OPUS-ecdc-v20160316-ces-deu + - OPUS-elitr_eca-v1-ces-deu + - OPUS-elrc_417_swedish_work_environ-v1-ces-deu + - OPUS-elrc_ec_europa-v1-ces-deu + - OPUS-elrc_emea-v1-ces-deu + - OPUS-elrc_euipo_2017-v1-ces-deu + - OPUS-elrc_europarl_covid-v1-ces-deu + - OPUS-elrc_eur_lex-v1-ces-deu + - OPUS-elrc_eu_publications-v1-ces-deu + - OPUS-elrc_information_portal-v1-ces-deu + - OPUS-elrc_antibiotic-v1-ces-deu + - OPUS-elrc_presscorner_covid-v1-ces-deu + - OPUS-elrc_vaccination-v1-ces-deu + - OPUS-elrc_wikipedia_health-v1-ces-deu + - OPUS-emea-v3-ces-deu + - OPUS-eubookshop-v2-ces-deu + - OPUS-euconst-v1-ces-deu + - OPUS-europarl-v8-ces-deu + - OPUS-gnome-v1-ces-deu + - OPUS-globalvoices-v2018q4-ces-deu + - OPUS-jrc_acquis-v3.0-ces-deu + - OPUS-kde4-v2-ces-deu + - OPUS-linguatools_wikititles-v2014-ces-deu + - OPUS-multiccaligned-v1.1-ces-deu + - OPUS-multiparacrawl-v9b-ces-deu + - OPUS-nllb-v1-ces-deu + - OPUS-neulab_tedtalks-v1-ces-deu + - OPUS-news_commentary-v16-ces-deu + - OPUS-news_commentary-v9.1-ces-deu + - OPUS-opensubtitles-v2018-ces-deu + - OPUS-php-v1-ces-deu + - OPUS-qed-v2.0a-ces-deu + - OPUS-ted2020-v1-ces-deu + - OPUS-tanzil-v1-ces-deu + - OPUS-tatoeba-v20230412-ces-deu + - OPUS-tildemodel-v2018-ces-deu + - OPUS-ubuntu-v14.10-ces-deu + #- OPUS-wikimatrix-v1-ces-deu # already added from source + - OPUS-xlent-v1.2-ces-deu + - OPUS-bible_uedin-v1-ces-deu + - OPUS-wikimedia-v20230407-ces-deu + mono_train: &mono_deu + - Statmt-news_crawl-2023-deu + - Statmt-europarl-10-deu + - Statmt-news_commentary-18.1-deu + - Statmt-commoncrawl-wmt22-deu + - Leipzig-wikipedia-2021_1m-deu + - Leipzig-comweb-2021_1m-deu + - Leipzig-mixed_typical-2011_1m-deu + - Leipzig-news-2022_30k-deu + - Leipzig-newscrawl-2020_1m-deu + - Leipzig-web-2021_100k-deu_DE + # TODO: extended common crawl + +########## JPN-ZHO ########## +- id: wmt25-jpn-zho + langs: jpn-zho + train: + - Statmt-news_commentary-18.1-jpn-zho + - KECL-paracrawl-2-zho-jpn + - KECL-paracrawl-2wmt24-zho-jpn + - Facebook-wikimatrix-1-jpn-zho + - Neulab-tedtalks_train-1-jpn-zho + - LinguaTools-wikititles-2014-jpn-zho + - OPUS-ccmatrix-v1-jpn-zho + - OPUS-gnome-v1-jpn-zho_CN + - OPUS-kde4-v2-jpn-zho_CN + - OPUS-multiccaligned-v1-jpn-zho_CN + - OPUS-openoffice-v3-jpn-zho_CN + - OPUS-opensubtitles-v2018-jpn-zho_CN + - OPUS-php-v1-jpn-zho + - OPUS-qed-v2.0a-jpn-zho + - OPUS-ted2020-v1-jpn-zho + - OPUS-tanzil-v1-jpn-zho + - OPUS-ubuntu-v14.10-jpn-zho + - OPUS-ubuntu-v14.10-jpn-zho_CN + - OPUS-xlent-v1.1-jpn-zho + - OPUS-bible_uedin-v1-jpn-zho + - OPUS-wikimedia-v20210402-jpn-zho + + mono_train: &mono_zho + - Statmt-news_crawl-2023-zho + - Statmt-news_commentary-18.1-zho + - Statmt-commoncrawl-wmt22-zho + - Leipzig-wikipedia-2018_1m-zho + - Leipzig-web-2016_1m-zho_MO + - Leipzig-tradnewscrawl-2011_1m-zho + - Leipzig-news-2020_300k-zho + # TODO: extended common crawl (too big) https://data.statmt.org/wmt21/translation-task/cc-mono/ + +######### BHO-ENG ########### +- id: wmt25-bho-eng + langs: bho-eng + train: + - OPUS-nllb-v1-bho-eng + - OPUS-tatoeba-v20230412-bho-eng + - OPUS-ubuntu-v14.10-bho-eng + - OPUS-wikimedia-v20230407-bho-eng + mono_train: &mono_eng + - Statmt-commoncrawl-wmt22-eng + - Statmt-europarl-10-eng + - Statmt-news_commentary-18.1-eng + - Statmt-news_crawl-2023-eng + - Statmt-news_discussions-2019-eng + +######### MAS-ENG ########### +# TODO: could not find parallel data found for mas-eng in mtdata and OPUS +# - id: wmt25-mas-eng +# langs: mas-eng +# train: + +######### ENG-ARA ########### +- id: wmt25-eng-ara + langs: eng-ara + train: + - Statmt-news_commentary-18.1-ara-eng + - Statmt-tedtalks-2_clean-eng-ara + - Statmt-ccaligned-1-ara_AR-eng + - Facebook-wikimatrix-1-ara-eng + - LinguaTools-wikititles-2014-ara-eng + - OPUS-ccaligned-v1-ara-eng + - OPUS-ccmatrix-v1-ara-eng + - OPUS-elrc_3083_wikipedia_health-v1-ara-eng + - OPUS-elrc_wikipedia_health-v1-ara-eng + - OPUS-elrc_2922-v1-ara-eng + - OPUS-eubookshop-v2-ara-eng + - OPUS-gnome-v1-ara-eng + - OPUS-globalvoices-v2018q4-ara-eng + - OPUS-hplt-v1.1-ara-eng + - OPUS-kde4-v2-ara-eng + - OPUS-linguatools_wikititles-v2014-ara-eng + - OPUS-multiccaligned-v1-ara-eng + - OPUS-multihplt-v1.1-ara-eng + - OPUS-multiun-v1-ara-eng + - OPUS-nllb-v1-ara-eng + - OPUS-opensubtitles-v2018-ara-eng + - OPUS-qed-v2.0a-ara-eng + - OPUS-ted2020-v1-ara-eng + - OPUS-tatoeba-v20230412-ara-eng + - OPUS-unpc-v1.0-ara-eng + - OPUS-unpc-v20090831-ara-eng + - OPUS-ubuntu-v14.10-ara-eng + - OPUS-wikimatrix-v1-ara-eng + - OPUS-wikipedia-v1.0-ara-eng + - OPUS-xlent-v1.2-ara-eng + - OPUS-bible_uedin-v1-ara-eng + - OPUS-infopankki-v1-ara-eng + - OPUS-tico_19-v20201028-ara-eng + - OPUS-tldr_pages-v20230829-ara-eng + - OPUS-wikimedia-v20230407-ara-eng + mono_train: + - Statmt-news_crawl-2023-ara + - Statmt-news_commentary-18.1-ara + - Leipzig-news-2020_1m-ara + - Leipzig-wikipedia-2021_1m-ara + +######### ENG-ZHO ########### +- id: wmt25-eng-zho + langs: eng-zho + train: # TODO: add all public data + - ParaCrawl-paracrawl-1_bonus-eng-zho + - Statmt-news_commentary-18.1-eng-zho + - Statmt-wikititles-3-zho-eng + - OPUS-unpc-v1.0-eng-zho + - Facebook-wikimatrix-1-eng-zho + - Statmt-backtrans_enzh-wmt20-eng-zho # English translated to Chinese. Double check if you want this + mono_train: *mono_zho + +######### ENG-CES ########### +- id: wmt25-eng-ces + langs: eng-ces + train: # TODO: add all public data + - Statmt-europarl-10-ces-eng + - ParaCrawl-paracrawl-9-eng-ces + - Statmt-commoncrawl_wmt13-1-ces-eng + - Statmt-news_commentary-18.1-ces-eng + - Statmt-wikititles-3-ces-eng + - Facebook-wikimatrix-1-ces-eng + - Tilde-eesc-2017-ces-eng + - Tilde-ema-2016-ces-eng + - Tilde-ecb-2017-ces-eng + - Tilde-rapid-2019-ces-eng + # TODO: manually download bracktranslated news and CzEng2.0 + mono_train: + - Statmt-news_crawl-2023-ces + - Statmt-europarl-10-ces + - Statmt-news_commentary-18.1-ces + - Statmt-commoncrawl-wmt22-ces + - Leipzig-news-2022_1m-ces + - Leipzig-newscrawl-2019_1m-ces + - Leipzig-wikipedia-2021_1m-ces + - Leipzig-web_public-2019_1m-ces_CZ + # TODO: extended common crawl (too big) https://data.statmt.org/wmt21/translation-task/cc-mono/ + +######### ENG-EST ########### +- id: wmt25-eng-est + langs: eng-est + train: + - Statmt-europarl-7-est-eng + - Statmt-ccaligned-1-eng-est_EE + - ParaCrawl-paracrawl-9-eng-est + - Tilde-eesc-2017-eng-est + - Tilde-ema-2016-eng-est + - Tilde-airbaltic-1-eng-est + - Tilde-ecb-2017-eng-est + - Tilde-rapid-2016-eng-est + - Facebook-wikimatrix-1-eng-est + - Neulab-tedtalks_train-1-eng-est + - ELRC-estonian_cabinet_ministers-1-eng-est + - ELRC-bank_estonia-1-eng-est + - ELRC-legal_estonian_justice-1-eng-est + - ELRC-estonian_foreign_affairs-1-eng-est + - ELRC-parliament_estonia-1-eng-est + - ELRC-finnish_information_bank-1-eng-est + - ELRC-national_security_defence-1-eng-est + - ELRC-akadeemia.ee-1-eng-est + - ELRC-vp1992_2001.president.ee-1-eng-est + - ELRC-vp2001_2006.president.ee-1-eng-est + - ELRC-vp2006_2016.president.ee-1-eng-est + - ELRC-president.ee-1-eng-est + - ELRC-www.visitestonia.com-1-eng-est + - ELRC-euipo_2017-1-eng-est + - ELRC-estonian_classification_economic_activities-1-eng-est + - ELRC-press_releases_foreign_affairs_estonia-1-eng-est + - ELRC-emea-1-eng-est + - ELRC-vaccination-1-eng-est + - ELRC-eu_publications_medical_v2-1-eng-est + - ELRC-wikipedia_health-1-eng-est + - ELRC-antibiotic-1-eng-est + - ELRC-europarl_covid-1-eng-est + - ELRC-ec_europa_covid-1-eng-est + - ELRC-www.kriis.ee-1-eng-est + - ELRC-eur_lex_covid-1-eng-est + - ELRC-presscorner_covid-1-eng-est + - ELRC-nteu_tiera-1-eng-est + - ELRC-nteu_tierb-1-eng-est + - ELRC-scipar-1-eng-est + - ELRC-web_acquired_data_related_to_scientific_research-1-eng-est + - EU-ecdc-1-eng-est + - EU-eac_forms-1-eng-est + - EU-eac_reference-1-eng-est + - EU-dcep-1-eng-est + - OPUS-ccaligned-v1-eng-est + - OPUS-ccmatrix-v1-eng-est + - OPUS-dgt-v2019-eng-est + - OPUS-dgt-v4-eng-est + - OPUS-ecb-v1-eng-est + - OPUS-ecdc-v20160316-eng-est + - OPUS-elitr_eca-v1-eng-est + - OPUS-elra_w0154-v1-eng-est + - OPUS-elra_w0167-v1-eng-est + - OPUS-elra_w0168-v1-eng-est + - OPUS-elra_w0215-v1-eng-est + - OPUS-elra_w0218-v1-eng-est + - OPUS-elra_w0265-v1-eng-est + - OPUS-elrc_1129_www.visitestonia.com-v1-eng-est + - OPUS-elrc_2016_euipo_2017-v1-eng-est + - OPUS-elrc_2457_estonian_classificat-v1-eng-est + - OPUS-elrc_2461_press_releases_forei-v1-eng-est + - OPUS-elrc_2723_emea-v1-eng-est + - OPUS-elrc_2751_vaccination-v1-eng-est + - OPUS-elrc_2882_eu_publications_medi-v1-eng-est + - OPUS-elrc_3079_wikipedia_health-v1-eng-est + - OPUS-elrc_3211_antibiotic-v1-eng-est + - OPUS-elrc_3300_europarl_covid-v1-eng-est + - OPUS-elrc_3471_ec_europa_covid-v1-eng-est + - OPUS-elrc_3554_www.kriis.ee-v1-eng-est + - OPUS-elrc_3572_eur_lex_covid-v1-eng-est + - OPUS-elrc_3613_presscorner_covid-v1-eng-est + - OPUS-elrc_393_estonian_cabinet_min-v1-eng-est + - OPUS-elrc_411_bank_estonia-v1-eng-est + - OPUS-elrc_4271_nteu_tiera-v1-eng-est + - OPUS-elrc_429_legal_estonian_justi-v1-eng-est + - OPUS-elrc_431_estonian_foreign_aff-v1-eng-est + - OPUS-elrc_5067_scipar-v1-eng-est + - OPUS-elrc_714_parliament_estonia-v1-eng-est + - OPUS-elrc_717_finnish_information_-v1-eng-est + - OPUS-elrc_770_national_security_de-v1-eng-est + - OPUS-elrc_919_akadeemia.ee-v1-eng-est + - OPUS-elrc_937_vp1992_2001.presiden-v1-eng-est + - OPUS-elrc_938_vp2001_2006.presiden-v1-eng-est + - OPUS-elrc_939_vp2006_2016.presiden-v1-eng-est + - OPUS-elrc_940_president.ee-v1-eng-est + - OPUS-elrc_ec_europa-v1-eng-est + - OPUS-elrc_emea-v1-eng-est + - OPUS-elrc_euipo_2017-v1-eng-est + - OPUS-elrc_europarl_covid-v1-eng-est + - OPUS-elrc_eur_lex-v1-eng-est + - OPUS-elrc_eu_publications-v1-eng-est + - OPUS-elrc_finnish_information-v1-eng-est + - OPUS-elrc_antibiotic-v1-eng-est + - OPUS-elrc_presscorner_covid-v1-eng-est + - OPUS-elrc_vaccination-v1-eng-est + - OPUS-elrc_wikipedia_health-v1-eng-est + - OPUS-elrc_www.visitestonia.com-v1-eng-est + - OPUS-elrc_2682-v1-eng-est + - OPUS-elrc_2922-v1-eng-est + - OPUS-elrc_2923-v1-eng-est + - OPUS-elrc_3382-v1-eng-est + - OPUS-emea-v3-eng-est + - OPUS-eopc-v2022-eng-est + - OPUS-eubookshop-v2-eng-est + - OPUS-euconst-v1-eng-est + - OPUS-europarl-v8-eng-est + - OPUS-gnome-v1-eng-est + - OPUS-hplt-v1-eng-est + - OPUS-hplt-v1.1-eng-est + - OPUS-jrc_acquis-v3.0-eng-est + - OPUS-kde4-v2-eng-est + - OPUS-kdedoc-v1-eng_GB-est + - OPUS-multiccaligned-v1-eng-est + - OPUS-multihplt-v1.1-eng-est + - OPUS-multiparacrawl-v7.1-eng-est + - OPUS-nllb-v1-eng-est + - OPUS-neulab_tedtalks-v1-eng-est + - OPUS-opensubtitles-v2018-eng-est + - OPUS-paracrawl-v9-eng-est + - OPUS-qed-v2.0a-eng-est + - OPUS-ted2020-v1-eng-est + - OPUS-tatoeba-v20230412-eng-est + - OPUS-tildemodel-v2018-eng-est + - OPUS-ubuntu-v14.10-eng-est + - OPUS-wmt_news-v2019-eng-est + - OPUS-wikimatrix-v1-eng-est + - OPUS-xlent-v1.2-eng-est + - OPUS-bible_uedin-v1-eng-est + - OPUS-infopankki-v1-eng-est + - OPUS-wikimedia-v20230407-eng-est + mono_train: + - Statmt-news_crawl-2023-est + - Leipzig-web-2015_1m-est_EE + - Leipzig-news-2020_300k-est + - Leipzig-newscrawl-2017_1m-est + + +######### ENG-ISL ########### +- id: wmt25-eng-isl + langs: eng-isl + train: + - Statmt-wikititles-3-isl-eng + - Statmt-ccaligned-1-eng-isl_IS + - ParaCrawl-paracrawl-9-eng-isl + - Tilde-eesc-2017-eng-isl + - Tilde-ema-2016-eng-isl + - Tilde-rapid-2016-eng-isl + - Facebook-wikimatrix-1-eng-isl + - ParIce-eea_train-20.05-eng-isl + - ParIce-ema_train-20.05-eng-isl + - EU-ecdc-1-eng-isl + - EU-eac_forms-1-eng-isl + - EU-eac_reference-1-eng-isl + - OPUS-ccmatrix-v1-eng-isl + - OPUS-elrc_2718_emea-v1-eng-isl + - OPUS-elrc_3206_antibiotic-v1-eng-isl + - OPUS-elrc_4295_www.malfong.is-v1-eng-isl + - OPUS-elrc_4324_government_offices_i-v1-eng-isl + - OPUS-elrc_4327_government_offices_i-v1-eng-isl + - OPUS-elrc_4334_rkiskaup_2020-v1-eng-isl + - OPUS-elrc_4338_university_iceland-v1-eng-isl + - OPUS-elrc_502_icelandic_financial_-v1-eng-isl + - OPUS-elrc_504_www.iceida.is-v1-eng-isl + - OPUS-elrc_505_www.pfs.is-v1-eng-isl + - OPUS-elrc_506_www.lanamal.is-v1-eng-isl + - OPUS-elrc_5067_scipar-v1-eng-isl + - OPUS-elrc_508_tilde_statistics_ice-v1-eng-isl + - OPUS-elrc_509_gallery_iceland-v1-eng-isl + - OPUS-elrc_510_harpa_reykjavik_conc-v1-eng-isl + - OPUS-elrc_511_bokmenntaborgin_is-v1-eng-isl + - OPUS-elrc_516_icelandic_medicines-v1-eng-isl + - OPUS-elrc_517_icelandic_directorat-v1-eng-isl + - OPUS-elrc_597_www.nordisketax.net-v1-eng-isl + - OPUS-elrc_718_statistics_iceland-v1-eng-isl + - OPUS-elrc_728_www.norden.org-v1-eng-isl + - OPUS-elrc_emea-v1-eng-isl + - OPUS-elrc_antibiotic-v1-eng-isl + - OPUS-elrc_www.norden.org-v1-eng-isl + - OPUS-elrc_www.nordisketax.net-v1-eng-isl + - OPUS-eubookshop-v2-eng-isl + - OPUS-multiccaligned-v1-eng-isl + - OPUS-multiparacrawl-v7.1-eng-isl + - OPUS-opensubtitles-v2018-eng-isl + - OPUS-ted2020-v1-eng-isl + - OPUS-tatoeba-v20220303-eng-isl + - OPUS-ubuntu-v14.10-eng-isl + - OPUS-wikimatrix-v1-eng-isl + - OPUS-wikititles-v3-eng-isl + - OPUS-xlent-v1.1-eng-isl + - OPUS-wikimedia-v20210402-eng-isl + mono_train: + - Statmt-news_crawl-2023-isl + - Leipzig-web-2020_1m-isl_IS + - Leipzig-web_public-2019_1m-isl_IS + - Leipzig-news-2020_30k-isl + - Leipzig-newscrawl-2019_300k-isl + - Leipzig-wikipedia-2021_100k-isl + +######### ENG-JPN ########### +- id: wmt24-eng-jpn + langs: eng-jpn + train: # TODO: add all public data + - Statmt-news_commentary-18.1-eng-jpn + - KECL-paracrawl-3-eng-jpn + - Statmt-wikititles-3-jpn-eng + - Facebook-wikimatrix-1-eng-jpn + - Statmt-ted-wmt20-eng-jpn + - StanfordNLP-jesc_train-1-eng-jpn + - Phontron-kftt_train-1-eng-jpn + mono_train: &mono_jpn + - Statmt-news_crawl-2023-jpn + - Statmt-news_commentary-18.1-jpn + - Statmt-commoncrawl-wmt22-jpn + - Leipzig-web-2020_1m-jpn_JP + - Leipzig-comweb-2018_1m-jpn + - Leipzig-web_public-2019_1m-jpn_JP + - Leipzig-news-2020_100k-jpn + - Leipzig-newscrawl-2019_1m-jpn + - Leipzig-wikipedia-2021_1m-jpn + # TODO: Extended Common Crawl + +######### ENG-KOR ########### +- id: wmt25-eng-kor + langs: eng-kor + train: + - Statmt-ccaligned-1-eng-kor_KR + - ParaCrawl-paracrawl-1_bonus-eng-kor + - Facebook-wikimatrix-1-eng-kor + - Neulab-tedtalks_train-1-eng-kor + - ELRC-wikipedia_health-1-eng-kor + - ELRC-hrw_dataset_v1-1-eng-kor + - LinguaTools-wikititles-2014-eng-kor + - OPUS-ccaligned-v1-eng-kor + - OPUS-ccmatrix-v1-eng-kor + - OPUS-elrc_3070_wikipedia_health-v1-eng-kor + - OPUS-elrc_wikipedia_health-v1-eng-kor + - OPUS-elrc_2922-v1-eng-kor + - OPUS-gnome-v1-eng-kor + - OPUS-globalvoices-v2018q4-eng-kor + - OPUS-kde4-v2-eng-kor + - OPUS-linguatools_wikititles-v2014-eng-kor + - OPUS-mdn_web_docs-v20230925-eng-kor + - OPUS-multiccaligned-v1-eng-kor + - OPUS-nllb-v1-eng-kor + - OPUS-neulab_tedtalks-v1-eng-kor + - OPUS-opensubtitles-v2018-eng-kor + - OPUS-php-v1-eng-kor + - OPUS-paracrawl-v9-eng-kor + - OPUS-qed-v2.0a-eng-kor + - OPUS-ted2020-v1-eng-kor + - OPUS-tanzil-v1-eng-kor + - OPUS-tatoeba-v20230412-eng-kor + - OPUS-ubuntu-v14.10-eng-kor + - OPUS-wikimatrix-v1-eng-kor + - OPUS-xlent-v1.2-eng-kor + - OPUS-bible_uedin-v1-eng-kor + - OPUS-tldr_pages-v20230829-eng-kor + - OPUS-wikimedia-v20230407-eng-kor + mono_train: + - Statmt-news_crawl-2023-kor + - Leipzig-web-2020_1m-kor_KR + - Leipzig-news-2020_1m-kor + - Leipzig-wikipedia-2021_1m-kor + +######### ENG-RUS ########### +- id: wmt25-eng-rus + langs: eng-rus + train: # TODO: add all public data + - ParaCrawl-paracrawl-1_bonus-eng-rus + - Statmt-commoncrawl_wmt13-1-rus-eng + - Statmt-news_commentary-18.1-eng-rus + - Statmt-yandex-wmt22-eng-rus + - Statmt-wikititles-3-rus-eng + - OPUS-unpc-v1.0-eng-rus + - Facebook-wikimatrix-1-eng-rus + - Tilde-airbaltic-1-eng-rus + - Tilde-czechtourism-1-eng-rus + - Tilde-worldbank-1-eng-rus + - Statmt-backtrans_ruen-wmt20-rus-eng # russian translated to english + mono_train: &mono_rus + - Statmt-news_crawl-2023-rus + - Statmt-news_commentary-18.1-rus + - Statmt-commoncrawl-wmt22-rus + - Leipzig-news-2022_1m-rus + - Leipzig-newscrawl_public-2018_1m-rus + - Leipzig-web-2017_1m-rus_GE + - Leipzig-wikipedia-2021_1m-rus + + +######### ENG-SRP ########### +- id : wmt25-eng-srp + langs: eng-srp + train: + - Statmt-ccaligned-1-eng-srp_RS + - Tilde-worldbank-1-eng-srp + - Facebook-wikimatrix-1-eng-srp + - Neulab-tedtalks_train-1-eng-srp + - ELRC-swedish_social_security-1-eng-srp + - ELRC-wikipedia_health-1-eng-srp + - OPUS-ccaligned-v1-eng-srp + - OPUS-ccmatrix-v1-eng-srp + - OPUS-elrc_3041_wikipedia_health-v1-eng-srp + - OPUS-elrc_416_swedish_social_secur-v1-eng-srp + - OPUS-elrc_wikipedia_health-v1-eng-srp + - OPUS-elrc_2922-v1-eng-srp + - OPUS-eubookshop-v2-eng-srp + - OPUS-gnome-v1-eng-srp + - OPUS-globalvoices-v2018q4-eng-srp + - OPUS-gourmet-v2-eng-srp + - OPUS-hplt-v1.1-eng-srp + - OPUS-kde4-v2-eng-srp + - OPUS-kdedoc-v1-eng_GB-srp + - OPUS-multiccaligned-v1-eng-srp + - OPUS-multihplt-v1.1-eng-srp + - OPUS-nllb-v1-eng-srp + - OPUS-neulab_tedtalks-v1-eng-srp + - OPUS-opensubtitles-v2018-eng-srp + - OPUS-qed-v2.0a-eng-srp + - OPUS-setimes-v2-eng-srp + - OPUS-tatoeba-v20230412-eng-srp + - OPUS-tildemodel-v2018-eng-srp + - OPUS-ubuntu-v14.10-eng-srp + - OPUS-wikimatrix-v1-eng-srp + - OPUS-xlent-v1.2-eng-srp + - OPUS-bible_uedin-v1-eng-srp + - OPUS-tldr_pages-v20230829-eng-srp + - OPUS-wikimedia-v20230407-eng-srp + mono_train: + - Statmt-news_crawl-2023-srp + # TODO: verify if _ME and _RS country codes are safe to mix + - Leipzig-web-2016_300k-srp_ME + - Leipzig-web-2016_1m-srp_RS + - Leipzig-news-2019_30k-srp + - Leipzig-wikipedia-2021_1m-srp + +######### ENG-UKR ########### +- id: wmt25-eng-ukr + langs: eng-ukr + train: ¶_eng_ukr #TODO: add all public data + - ParaCrawl-paracrawl-1_bonus-eng-ukr + - Tilde-worldbank-1-eng-ukr + - Facebook-wikimatrix-1-eng-ukr + - ELRC-acts_ukrainian-1-eng-ukr + - Statmt-ccaligned-1-eng-ukr_UA + + mono_train: *mono_ukr From 0e4a85fb9dedc15181f2d00707dcf84e96ae0760 Mon Sep 17 00:00:00 2001 From: Thamme Gowda Date: Tue, 11 Mar 2025 19:15:06 -0700 Subject: [PATCH 05/13] fix python versions in GH workflow; add 3.11 and 3.12 --- .github/workflows/python-build-test.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-build-test.yml b/.github/workflows/python-build-test.yml index 33140f4..5a18070 100644 --- a/.github/workflows/python-build-test.yml +++ b/.github/workflows/python-build-test.yml @@ -20,17 +20,18 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest] # windows-latest - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 - - name: Set up Python 3.10 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 with: - python-version: "3.10" + python-version: "${{ matrix.python-version }}" - name: Install dependencies run: | python -m pip install --upgrade pip - pip install setuptools==61.2 flake8 pytest-cov + python --version + pip --version - name: Install module run: | pip install .[hf] From 1605d0f82c8d87f9410380e73b7b69cd31a2b0ac Mon Sep 17 00:00:00 2001 From: Thamme Gowda Date: Wed, 12 Mar 2025 01:11:16 -0700 Subject: [PATCH 06/13] upgrade GH actions python versions (#167) --- .github/workflows/python-build-test.yml | 20 ++++++++++++++------ .travis.yml.bak | 21 --------------------- pyproject.toml | 6 ++++-- 3 files changed, 18 insertions(+), 29 deletions(-) delete mode 100644 .travis.yml.bak diff --git a/.github/workflows/python-build-test.yml b/.github/workflows/python-build-test.yml index 13a2ab7..a3fb99a 100644 --- a/.github/workflows/python-build-test.yml +++ b/.github/workflows/python-build-test.yml @@ -13,28 +13,36 @@ permissions: env: PYTHONUTF8: "1" - + jobs: build: runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] # windows-latest + os: [ubuntu-latest, macos-latest] # windows-latest python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + exclude: + - os: macos-latest + python-version: '3.7' + - os: ubuntu-latest + python-version: '3.7' + # os x py versions here: https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: "${{ matrix.python-version }}" - name: Install dependencies run: | - python -m pip install --upgrade pip + pip install --upgrade pip + pip install setuptools==61.2 flake8 python --version pip --version - name: Install module run: | - pip install .[hf] + pip install .[hf,test] - name: Test with pytest run: | python3 -m pytest diff --git a/.travis.yml.bak b/.travis.yml.bak deleted file mode 100644 index 21329bc..0000000 --- a/.travis.yml.bak +++ /dev/null @@ -1,21 +0,0 @@ -dist: jammy # focal -# see all versions at https://hub.docker.com/_/ubuntu -sudo: false -language: python -python: - #- "3.6" # dataclasses came in 3.7, so 3.6 isnt supported - - "3.7" - - "3.8" - - "3.9" - -before_install: - - sudo apt-get -y update - - pip install setuptools==61.2 - - python3 setup.py install - -install: - - pip install pytest-cov - - pip install . - -script: - - python3 -m pytest diff --git a/pyproject.toml b/pyproject.toml index a1d7189..467cf3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ dynamic = ["version"] description = "mtdata is a tool to download datasets for machine translation" readme = "README.md" license = { file = "LICENSE" } -requires-python = ">=3.7" +requires-python = ">=3.8" authors = [ { name = "Thamme Gowda", email = "tgowdan@gmail.com" } ] @@ -50,8 +50,10 @@ exclude = ["tests*", "tmp*", "build*", "dist*", "crawler*"] [project.optional-dependencies] -hf = ["datasets>=3.3.0"] +hf = ["datasets>=3.1.0"] test = [ "pytest", "pytest-cov[all]", "black", "isort", "mypy"] +# Note: hf datasets>=3.2 onwards doesnot support python 3.8 + [project.scripts] mtdata = "mtdata.__main__:main" From 3e8dc78902a41b5fa63b9a1c5e0e1be62984c379 Mon Sep 17 00:00:00 2001 From: Thamme Gowda Date: Wed, 12 Mar 2025 09:59:12 -0700 Subject: [PATCH 07/13] metadata/ docid creation for "get" and "get-recipe" commands --- mtdata/cache.py | 2 +- mtdata/data.py | 75 ++++++++++++++++++++++++++----------- mtdata/index/huggingface.py | 4 +- mtdata/index/statmt.py | 15 +++++--- mtdata/main.py | 10 +++-- mtdata/parser.py | 40 ++++++++++++++------ mtdata/utils.py | 2 +- tests/test_cli.py | 36 ++++++++++++++---- 8 files changed, 130 insertions(+), 54 deletions(-) diff --git a/mtdata/cache.py b/mtdata/cache.py index 3504034..0419093 100644 --- a/mtdata/cache.py +++ b/mtdata/cache.py @@ -169,7 +169,7 @@ def get_hf_dataset(self, url: str, entry=None): streaming=False, trust_remote_code=False, ) - log.info(f"Loading dataset {hf_id} with args: {args}") + log.debug(f"Loading dataset {hf_id} with args: {args}") ds = load_dataset(hf_id, **args) return ds diff --git a/mtdata/data.py b/mtdata/data.py index 45a4af1..144f96c 100644 --- a/mtdata/data.py +++ b/mtdata/data.py @@ -137,7 +137,7 @@ def prepare(cls, langs, out_dir: Path, dataset_ids=Dict[str, List[DatasetId]], if drop_tests: pair_files = [] for ent in dev_entries + test_entries: - p1, p2 = dataset.get_paths(dataset.tests_dir, ent) + p1, p2, _ = dataset.get_paths(dataset.tests_dir, ent) if BCP47Tag.check_compat_swap(langs, ent.did.langs, fail_on_incompat=True)[1]: p1, p2 = p2, p1 # swap pair_files.append((p1, p2)) @@ -152,7 +152,7 @@ def prepare(cls, langs, out_dir: Path, dataset_ids=Dict[str, List[DatasetId]], dirpath.mkdir(exist_ok=True) entries = cls.resolve_entries(dataset_ids[key]) for entry in entries: - dataset.add_mono_entry(dirpath, entry, compress=compress) + dataset.add_mono_entry(dirpath, entry, compress=compress) # citations refs_file = out_dir / 'references.bib' @@ -193,11 +193,11 @@ def add_train_entries(self, entries, merge_train=False, compress=False, drop_has # paired_files = self.find_bitext_pairs(self.train_parts_dir, lang1, lang2) paired_files = {} for ent in entries: - e1, e2 = self.get_paths(self.train_parts_dir, ent, compress=compress) + e1, e2, e3 = self.get_paths(self.train_parts_dir, ent, compress=compress) _, swapped = BCP47Tag.check_compat_swap(self.langs, ent.did.langs, fail_on_incompat=True) if swapped: e1, e2 = e2, e1 - paired_files[str(ent.did)] = e1, e2 + paired_files[str(ent.did)] = e1, e2, e3 log.info(f"Going to merge {len(paired_files)} files as one train file") compress_ext = f'.{DEF_COMPRESS}' if compress else '' @@ -205,7 +205,7 @@ def add_train_entries(self, entries, merge_train=False, compress=False, drop_has l2_ext = f'{lang2}{compress_ext}' of1 = self.dir / f'train.{l1_ext}' of2 = self.dir / f'train.{l2_ext}' - of3 = self.dir / f'train.meta.{DEF_COMPRESS}' + of3 = self.dir / f'train.meta.jsonl.{DEF_COMPRESS}' counts = dict(total=coll.defaultdict(int), dupes_skips=coll.defaultdict(int), @@ -216,8 +216,9 @@ def add_train_entries(self, entries, merge_train=False, compress=False, drop_has with IO.writer(of1) as w1, IO.writer(of2) as w2, IO.writer(of3) as w3: with pbar_man.counter(color='green', total=len(paired_files), unit='it', desc="Merging", leave=False, min_delta=Defaults.PBAR_REFRESH_INTERVAL, autorefresh=True) as pbar: - for name, (if1, if2) in paired_files.items(): - for seg1, seg2 in self.read_parallel(if1, if2): + for name, (if1, if2, if3) in paired_files.items(): + for fields in self.read_parallel(if1, if2): + seg1, seg2 = fields[:2] counts['total'][name] += 1 if self.drop_dupes or self.drop_tests: hash_val = hash((seg1, seg2)) @@ -253,14 +254,18 @@ def add_mono_entry(self, dirpath, entry: Entry, compress=False): return -1, -1 cache_path = self.cache.get_entry(entry) parser = Parser(cache_path, ext=entry.in_ext or None, ent=entry) - out_path = self.get_paths(dirpath, entry, compress=compress) + out_path, meta_file = self.get_paths(dirpath, entry, compress=compress) log.info("Writing %s to %s", entry.did, out_path) io_args = dict(encoding='utf-8', errors='ignore') - with IO.writer(out_path, **io_args) as out: + has_meta = None # None -> True/False on first row ;then ensure it is consistent + with IO.writer(out_path, **io_args) as out, IO.writer(meta_file, **io_args) as out_meta: count, skips = 0, 0 - for sentence in parser.read_segs(): - if isinstance(sentence, (list, tuple)) and len(sentence) == 1: - sentence = sentence[0] # flatten list + for row in parser.read_segs(): + if has_meta is None: # first row only + has_meta = bool(isinstance(row, (list, tuple)) and len(row) > 1) + sentence = row + if isinstance(row, (list, tuple)): + sentence = row[0] # flatten list assert isinstance(sentence, str), f'str sentence expected. found: {type(sentence)}; entry: {entry.did}' sentence = sentence and sentence.strip() if not sentence: @@ -268,12 +273,19 @@ def add_mono_entry(self, dirpath, entry: Entry, compress=False): continue sentence = sentence.replace('\n', ' ').replace('\t', ' ').replace('\r', ' ') out.write(f'{sentence}\n') + if has_meta: + assert len(row) >= 2, f'Expected 2 fields, found {len(row)}: {row}' + meta = json.dumps(row[1], ensure_ascii=False, indent=None) + meta = meta.replace('\n', ' ').replace('\t', ' ').replace('\r', ' ') + out_meta.write(f'{meta}\n') count += 1 msg = f'Looks like an error. {count} segs are valid {skips} are invalid: {entry}' assert count > 0, msg if skips > count: log.warning(msg) log.info(f"{entry}: Skips : {skips:,}/{count:,} => {100 * skips / count:.4f}%") + if not has_meta and meta_file.exists(): + meta_file.unlink() # remove empty meta file; file may not be empty if its .gz compressed flag_file.touch() return count, skips @@ -314,6 +326,14 @@ def read_parallel(cls, file1: Path, file2: Path): raise Exception(f'{file1} {file2} have unequal num of lines. This is an error.') yield seg1.strip(), seg2.strip() + @classmethod + def read_parallel2(cls, *files: Path): + streams = [IO.get_lines(f) for f in files] + for row in zip_longest(*streams): + if any(seg is None for seg in row): + raise Exception(f'{files} have unequal num of lines. This is an error.') + yield tuple(seg.strip() for seg in row) + def add_test_entries(self, entries): self.add_parts(self.tests_dir, entries, drop_noise=self.drop_test_noise, desc='Held-out sets', fail_on_error=self.fail_on_error) @@ -324,7 +344,7 @@ def add_test_entries(self, entries): def link_to_part(self, entry, data_dir, link_name): """Create link such as test, dev""" l1, l2 = self.langs - l1_path, l2_path = self.get_paths(data_dir, entry) + l1_path, l2_path, meta_file = self.get_paths(data_dir, entry) l1_path, l2_path = l1_path.relative_to(self.dir), l2_path.relative_to(self.dir) l1_link = self.dir / f'{link_name}.{l1.lang}' l2_link = self.dir / f'{link_name}.{l2.lang}' @@ -351,7 +371,7 @@ def add_dev_entries(self, entries): out_paths = (self.dir / f'dev.{l1.lang}', self.dir / f'dev.{l2.lang}') in_paths = [] for ent in entries: - e1, e2 = self.get_paths(self.tests_dir, ent) + e1, e2, meta_file = self.get_paths(self.tests_dir, ent) compat, swapped = BCP47Tag.check_compat_swap(self.langs, ent.did.langs) if not compat: raise Exception(f"Unable to unify language IDs {self.langs} x {ent.did.langs}") @@ -433,7 +453,7 @@ def add_parts_sequential(self, dir_path, entries, drop_noise=False, compress=Fal self.errors_file.open('a').write(f"{ent.did}\t{msg}\n") @classmethod - def get_paths(cls, dir_path: Path, entry: Entry, compress=False) -> Union[Path, Tuple[Path, Path]]: + def get_paths(cls, dir_path: Path, entry: Entry, compress=False) -> Union[Tuple[Path, Path], Tuple[Path, Path, Path]]: """ Gets file path for entry in given directory. Return single path for monolingual entry and tuple of two paths for bilingual entry. @@ -442,12 +462,15 @@ def get_paths(cls, dir_path: Path, entry: Entry, compress=False) -> Union[Path, compress_ext = compress and f'.{DEF_COMPRESS}' or '' l1_ext = str(entry.did.langs[0]) + compress_ext l1 = dir_path / f'{entry.did}.{l1_ext}' + # always compress meta, because we use jsonl format which is very verbose + # and we dont need to read meta often; so worth compressing + meta_path = dir_path / f'{entry.did}.meta.jsonl.{DEF_COMPRESS}' if len(entry.did.langs) == 1: # monolingual - return l1 + return l1, meta_path else: # biling l2_ext = str(entry.did.langs[1]) + compress_ext l2 = dir_path / f'{entry.did}.{l2_ext}' - return l1, l2 + return l1, l2, meta_path def add_part(self, dir_path: Path, entry: Entry, drop_noise=False, compress=False): flag_file = dir_path / f'.valid.{entry.did}' @@ -459,20 +482,22 @@ def add_part(self, dir_path: Path, entry: Entry, drop_noise=False, compress=Fals parser = Parser(path, ext=entry.in_ext or None, ent=entry) # langs = '_'.join(str(lang) for lang in self.langs) # Check that files are written in correct order - l1, l2 = self.get_paths(dir_path, entry, compress=compress) + l1, l2, meta_file = self.get_paths(dir_path, entry, compress=compress) io_args = dict(encoding='utf-8', errors='ignore') - with IO.writer(l1, **io_args) as f1, IO.writer(l2, **io_args) as f2: + has_meta = None + with IO.writer(l1, **io_args) as f1, IO.writer(l2, **io_args) as f2, IO.writer(meta_file, **io_args) as f3: count, skips, noise = 0, 0, 0 for rec in parser.read_segs(): - rec = rec[:2] # get the first two recs - if len(rec) != 2: + if has_meta is None: + has_meta = bool(len(rec) > 2) + if len(rec) < 2: skips += 1 continue if drop_noise and entry.is_noisy(seg1=rec[0], seg2=rec[1]): skips += 1 noise += 1 continue - sent1, sent2 = [s.strip() for s in rec] + sent1, sent2 = [s.strip() for s in rec[:2]] if not sent1 or not sent2: skips += 1 continue @@ -480,6 +505,10 @@ def add_part(self, dir_path: Path, entry: Entry, drop_noise=False, compress=Fals sent2 = sent2.replace('\n', ' ').replace('\t', ' ').replace('\r', ' ') f1.write(f'{sent1}\n') f2.write(f'{sent2}\n') + if has_meta: + assert len(rec) >= 3, f'Expected 3 fields, found {len(rec)}: {rec}' + meta = json.dumps(rec[2], ensure_ascii=False, indent=None) + f3.write(f'{meta}\n') count += 1 msg = f'Looks like an error. {count} segs are valid {skips} are invalid: {entry}' assert count > 0, msg @@ -488,6 +517,8 @@ def add_part(self, dir_path: Path, entry: Entry, drop_noise=False, compress=Fals if noise > 0: log.info(f"{entry}: Noise : {noise:,}/{count:,} => {100 * noise / count:.4f}%") log.info(f"wrote {count} lines to {l1} == {l2}") + if not has_meta and meta_file.exists(): + meta_file.unlink() flag_file.touch() return count, skips diff --git a/mtdata/index/huggingface.py b/mtdata/index/huggingface.py index 87cefac..99f0a96 100644 --- a/mtdata/index/huggingface.py +++ b/mtdata/index/huggingface.py @@ -53,9 +53,9 @@ def load_all(index: Index): if data_id in index: log.warning(f"Duplicate dataset id: {data_id}") continue - params = dict(langs=orig_langs, split=config["split"]) url = "https://huggingface.co/datasets/" + data['id'] - meta = dict(config=config['name'], orig_id=data['id'], split=config["split"]) + fields = dict(source="source", target="target", doc_id="document_id", domain="domain", seg_id="segment_id") + meta = dict(config=config['name'], orig_id=data['id'], split=config["split"], fields=fields) in_paths = config["paths"] cite = None entry = Entry(did=data_id, url=url, in_paths=in_paths, cite=cite, ext=HF_EXT, in_ext=HF_EXT, meta=meta) diff --git a/mtdata/index/statmt.py b/mtdata/index/statmt.py index 1d3b50a..d116e48 100644 --- a/mtdata/index/statmt.py +++ b/mtdata/index/statmt.py @@ -257,10 +257,11 @@ def load_parallel(index: Index): # ==== Europarl v10 EP_v10 = "http://www.statmt.org/europarl/v10/training/europarl-v10.%s-%s.tsv.gz" - for pair in ['cs en', 'cs pl', 'de en', 'de fr', 'es pt', 'fi en', 'fr en', 'lt en', 'pl en']: - l1, l2 = pair.split() + ep_meta = dict(fields=dict(source=0, target=1, doc_id=2, para_id=3, speaker_id=4, speaker_name=5, affiliation=6)) + ep_pairs = "cs-pl de-fr es-pt".split() + [f"{x}-en" for x in "bg cs da de el es et fi fr hu it lt lv nl pl pt ro sk sl sv".split()] + for l1, l2 in (x.split('-') for x in ep_pairs): index.add_entry(Entry(did=DatasetId(group=GROUP_ID, name=f'europarl', version='10', langs=(l1, l2)), - url=EP_v10 % (l1, l2), cite=wmt20_cite)) + url=EP_v10 % (l1, l2), cite=wmt20_cite, meta=ep_meta)) # ==== PMIndia V1 PMINDIA_v1 = "http://data.statmt.org/pmindia/v1/parallel/pmindia.v1.%s-%s.tsv" @@ -370,12 +371,12 @@ def load_parallel(index: Index): wmt22_cite = ('kocmi-etal-2022-findings',) index.add_entry(Entry( did=DatasetId(group=GROUP_ID, name='yandex', version='wmt22', langs=('en', 'ru')), cite=wmt22_cite, - ext='zip', in_ext='tsv', cols=(0,1), in_paths=[f'WMT2022-data-main/en-ru/en-ru.1m_{i}.tsv' for i in range(1, 11)], + ext='zip', in_ext='tsv', cols=(0,1), in_paths=[f'WMT2022-data-main/en-ru/en-ru.1m_{i}.tsv' for i in range(1, 11)], url='https://github.com/mashashma/WMT2022-data/archive/refs/heads/main.zip')) index.add_entry(Entry( did=DatasetId(group=GROUP_ID, name='yakut', version='wmt22', langs=('sah', 'rus')), cite=wmt22_cite, - ext='zip', in_ext='tsv', cols=(0, 1), in_paths=['yakut/sah-ru.parallel.uniq.tsv'], + ext='zip', in_ext='tsv', cols=(0, 1), in_paths=['yakut/sah-ru.parallel.uniq.tsv'], url='http://data.statmt.org/wmt22/translation-task/yakut.zip')) @@ -420,10 +421,12 @@ def load_mono(index: Index): in_ext='txt', cite=wmt22_cite) # 3. Europarl 10 + ep_meta = dict(fields=dict(source=0, doc_id=1, para_id=2, speaker_id=3, speaker_name=4)) for lang in 'cs de en es fi fr lt pl pt'.split(): version = '10' url = f'https://www.statmt.org/europarl/v{version}/training-monolingual/europarl-v{version}.{lang}.tsv.gz' - index += Entry(DatasetId(GROUP_ID, 'europarl', version, (lang,)), url=url, in_ext='tsv', cols=(0,), cite=wmt22_cite) + index += Entry(DatasetId(GROUP_ID, 'europarl', version, (lang,)), url=url, in_ext='tsv', cols=(0,), + cite=wmt22_cite, meta=ep_meta) # 4. News Commentary for version in '14 15 16 17 18 18.1'.split(): diff --git a/mtdata/main.py b/mtdata/main.py index 2085f7e..8193a52 100644 --- a/mtdata/main.py +++ b/mtdata/main.py @@ -76,10 +76,12 @@ def echo_data(did:DatasetId, delim='\t'): if isinstance(row, str): rec = row elif isinstance(row, (list, tuple)): - rec = [col.replace(delim, ' ').replace('\n', ' ') for col in row[:2]] - if len(row) == 3: - meta = json.dumps(row[2], indent=None, ensure_ascii=False) - rec.append(meta) + rec = [] + for col in row: + if isinstance(col, Mapping): + col = json.dumps(col, indent=None, ensure_ascii=False) + col = col.replace(delim, ' ').replace('\n', ' ') + rec.append(col) rec = delim.join(rec) else: raise ValueError(f'Unknown row type: {type(row)}. Expected str or list/tuple') diff --git a/mtdata/parser.py b/mtdata/parser.py index 5fb293c..f4c9563 100644 --- a/mtdata/parser.py +++ b/mtdata/parser.py @@ -69,14 +69,15 @@ def read_segs(self): reader = OpusXcesParser.read(align, lang1_dir, lang2_dir, preprocessing=preprocessing) readers.append(reader) else: + meta_fields = self.ent.meta and self.ent.meta.get('fields') or None for p in self.paths: if 'tsv' in self.ext: cols = (0, 1) #extract first two columns if self.ent and self.ent.cols: cols = self.ent.cols - readers.append(self.read_tsv(p, cols=cols)) + readers.append(self.read_tsv(p, cols=cols, meta_fields=meta_fields)) elif 'csvwithheader' in self.ext: - readers.append(self.read_tsv(p, delim=',', skipheader=True)) + readers.append(self.read_tsv(p, delim=',', skipheader=True, meta_fields=meta_fields)) elif 'raw' in self.ext or 'txt' in self.ext: readers.append(self.read_plain(p)) elif 'tmx' in self.ext: @@ -122,7 +123,7 @@ def read_plain(self, path): log.warning(f'Error reading file {path}') raise - def read_tsv(self, path, delim='\t', cols=None, skipheader=False): + def read_tsv(self, path, delim='\t', cols=None, skipheader=False, meta_fields=None): """ Read data from TSV file :param path: path to TSV file @@ -136,19 +137,36 @@ def read_tsv(self, path, delim='\t', cols=None, skipheader=False): line = stream.readline() for line in stream: row = [x.strip() for x in line.rstrip('\n').split(delim)] + out_row = row if cols: - row = [row[idx] for idx in cols] - yield row + out_row = [row[idx] for idx in cols] + if meta_fields: + metadata = {} + for key, idx in meta_fields.items(): + if key in ("source", "target") or idx >= len(row) or row[idx] in ("", None): + continue + metadata[key] = row[idx] + if metadata: + out_row.append(metadata) + yield out_row def read_hfds(self, ds): """ Read data from huggingface Dataset :param ds: huggingface dataset :return: generator of segments """ + fields = self.ent.meta["fields"] # expects dictionary + src_field = fields['source'] + tgt_field = fields.get('target', None) + rev_map = {v: k for k, v in fields.items()} + # fields map is a forward map of "dest: orig", and meant to pick a subset of fields from the row + # in the current version, I am going to retain all fields to see what all fields exist, + # and map the subset of fields as per the dict; so, created rev_map.get(orig,orig) for row in ds: - src = row['source'] - tgt = row['target'] - #doc_id = row.get('document_id') - #seg_id = row.get('segment_id') - meta = dict(doc_id=row.get('document_id'), seg_id=row.get('segment_id'), domain=row.get('domain')) - yield src, tgt, meta \ No newline at end of file + out_row = [row.pop(src_field)] + if tgt_field is not None: + out_row.append(row.pop(tgt_field)) + # remap meta fields if necessary + metadata = {rev_map.get(k, k): v for k, v in row.items() if k not in (src_field, tgt_field)} + out_row.append(metadata) + yield out_row diff --git a/mtdata/utils.py b/mtdata/utils.py index d15d5fc..8af5274 100644 --- a/mtdata/utils.py +++ b/mtdata/utils.py @@ -90,7 +90,7 @@ def writer(cls, path, text=True, append=False, **kwargs): def get_lines(cls, path, col=0, delim='\t', line_mapper=None, newline_fix=True): with cls.reader(path) as inp: if newline_fix and delim != '\r': - inp = (line.replace(b'\r', b'') for line in inp) + inp = (line.replace('\r', '') for line in inp) if col >= 0: inp = (line.split(delim)[col].strip() for line in inp) if line_mapper: diff --git a/tests/test_cli.py b/tests/test_cli.py index d3732cc..0a259b2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,9 +1,12 @@ +import json import subprocess from typing import List, Union from mtdata.index import INDEX as index from pathlib import Path from tempfile import TemporaryDirectory +from mtdata.utils import IO +from . import MTDATA_CMD def shrun(cmd: Union[str, List[str]], capture_output=False): p = subprocess.run(cmd, shell=isinstance(cmd, str), @@ -13,25 +16,44 @@ def shrun(cmd: Union[str, List[str]], capture_output=False): return p.returncode def test_cli_help(): - assert shrun('python -m mtdata --help') == 0 + assert shrun(f'{MTDATA_CMD} --help') == 0 def test_cli_list(): - code, out = shrun('python -m mtdata list --id', capture_output=True) + code, out = shrun(f'{MTDATA_CMD} list --id', capture_output=True) assert code == 0 assert len(out.splitlines()) >= len(index.entries) def test_cli_get(): with TemporaryDirectory() as out_dir: did = 'OPUS-gnome-v1-eng-kan' - assert shrun(f'python -m mtdata get -l eng-kan -tr {did} -o {out_dir}') == 0 + assert shrun(f'{MTDATA_CMD} get -l eng-kan -tr {did} -o {out_dir}') == 0 assert (Path(out_dir) / 'mtdata.signature.txt').exists() def test_cache(): - code = shrun('python -m mtdata cache -ri tg01_2to1_test -j3', capture_output=False) + code = shrun(f'{MTDATA_CMD} cache -ri tg01_2to1_test -j3', capture_output=False) assert code == 0 def test_get_recipe(): - with TemporaryDirectory() as out_dir: - code = shrun(f'python -m mtdata get-recipe -ri tg01_2to1_test -o {out_dir}', capture_output=False) - assert code == 0 \ No newline at end of file + with TemporaryDirectory() as out_dir: + code = shrun(f'{MTDATA_CMD} get-recipe -ri tg01_2to1_test -o {out_dir}', capture_output=False) + assert code == 0 + +def test_metadata(): + with TemporaryDirectory() as out_dir: + out_dir = Path(out_dir) + did = "Statmt-europarl-10-slv-eng" + code = shrun(f'{MTDATA_CMD} get -l slv-eng -tr {did} -o {out_dir}', capture_output=False) + assert code == 0 + assert (out_dir / 'mtdata.signature.txt').exists() + meta_file = out_dir / "train-parts" / f"{did}.meta.jsonl.gz" + + with IO.reader(meta_file) as inp: + count = 0 + for line in inp: + meta = json.loads(line) + assert meta.get('doc_id') is not None + count += 1 + seg_file = out_dir / "train-parts" / f"{did}.eng" + seg_count = sum(1 for _ in IO.get_lines(seg_file)) + assert seg_count == count, f"Seg count {seg_count} != meta count {count}" From f4a83c3ff83b9a2234754c93da70b7f4bf2d8e23 Mon Sep 17 00:00:00 2001 From: Thamme Gowda Date: Thu, 13 Mar 2025 01:12:03 -0700 Subject: [PATCH 08/13] Update OPUS index --- mtdata/resource/opus_index.tsv | 4740 ++++++++++++++++++++++++++++++++ 1 file changed, 4740 insertions(+) diff --git a/mtdata/resource/opus_index.tsv b/mtdata/resource/opus_index.tsv index e5a67b2..86e633b 100644 --- a/mtdata/resource/opus_index.tsv +++ b/mtdata/resource/opus_index.tsv @@ -24697,6 +24697,56 @@ HPLT v1.1 en sq HPLT v1.1 en sr HPLT v1.1 en sw HPLT v1.1 en zh_hant +HPLT v2 af en +HPLT v2 ar en +HPLT v2 az en +HPLT v2 be en +HPLT v2 bg en +HPLT v2 bn en +HPLT v2 bs en +HPLT v2 ca en +HPLT v2 cy en +HPLT v2 en eo +HPLT v2 en et +HPLT v2 en eu +HPLT v2 en fa +HPLT v2 en fi +HPLT v2 en ga +HPLT v2 en gl +HPLT v2 en gu +HPLT v2 en he +HPLT v2 en hi +HPLT v2 en hr +HPLT v2 en is +HPLT v2 en ja +HPLT v2 en kk +HPLT v2 en kn +HPLT v2 en ko +HPLT v2 en lt +HPLT v2 en lv +HPLT v2 en mk +HPLT v2 en ml +HPLT v2 en mr +HPLT v2 en ms +HPLT v2 en mt +HPLT v2 en nb +HPLT v2 en ne +HPLT v2 en nn +HPLT v2 en si +HPLT v2 en sk +HPLT v2 en sl +HPLT v2 en sq +HPLT v2 en sr +HPLT v2 en sw +HPLT v2 en ta +HPLT v2 en te +HPLT v2 en th +HPLT v2 en tr +HPLT v2 en uk +HPLT v2 en ur +HPLT v2 en uz +HPLT v2 en vi +HPLT v2 en xh IITB v2.0 en hi JESC v2019-12-05 en ja JParaCrawl v3.0 en ja @@ -42007,6 +42057,1281 @@ MultiHPLT v1.1 sq zh_hant MultiHPLT v1.1 sr sw MultiHPLT v1.1 sr zh_hant MultiHPLT v1.1 sw zh_hant +MultiHPLT v2 af ar +MultiHPLT v2 af az +MultiHPLT v2 af be +MultiHPLT v2 af bg +MultiHPLT v2 af bn +MultiHPLT v2 af bs +MultiHPLT v2 af ca +MultiHPLT v2 af cy +MultiHPLT v2 af en +MultiHPLT v2 af eo +MultiHPLT v2 af et +MultiHPLT v2 af eu +MultiHPLT v2 af fa +MultiHPLT v2 af fi +MultiHPLT v2 af ga +MultiHPLT v2 af gl +MultiHPLT v2 af gu +MultiHPLT v2 af he +MultiHPLT v2 af hi +MultiHPLT v2 af hr +MultiHPLT v2 af is +MultiHPLT v2 af ja +MultiHPLT v2 af kk +MultiHPLT v2 af kn +MultiHPLT v2 af ko +MultiHPLT v2 af lt +MultiHPLT v2 af lv +MultiHPLT v2 af mk +MultiHPLT v2 af ml +MultiHPLT v2 af mr +MultiHPLT v2 af ms +MultiHPLT v2 af mt +MultiHPLT v2 af nb +MultiHPLT v2 af ne +MultiHPLT v2 af nn +MultiHPLT v2 af si +MultiHPLT v2 af sk +MultiHPLT v2 af sl +MultiHPLT v2 af sq +MultiHPLT v2 af sr +MultiHPLT v2 af sw +MultiHPLT v2 af ta +MultiHPLT v2 af te +MultiHPLT v2 af th +MultiHPLT v2 af tr +MultiHPLT v2 af uk +MultiHPLT v2 af ur +MultiHPLT v2 af uz +MultiHPLT v2 af vi +MultiHPLT v2 af xh +MultiHPLT v2 ar az +MultiHPLT v2 ar be +MultiHPLT v2 ar bg +MultiHPLT v2 ar bn +MultiHPLT v2 ar bs +MultiHPLT v2 ar ca +MultiHPLT v2 ar cy +MultiHPLT v2 ar en +MultiHPLT v2 ar eo +MultiHPLT v2 ar et +MultiHPLT v2 ar eu +MultiHPLT v2 ar fa +MultiHPLT v2 ar fi +MultiHPLT v2 ar ga +MultiHPLT v2 ar gl +MultiHPLT v2 ar gu +MultiHPLT v2 ar he +MultiHPLT v2 ar hi +MultiHPLT v2 ar hr +MultiHPLT v2 ar is +MultiHPLT v2 ar ja +MultiHPLT v2 ar kk +MultiHPLT v2 ar kn +MultiHPLT v2 ar ko +MultiHPLT v2 ar lt +MultiHPLT v2 ar lv +MultiHPLT v2 ar mk +MultiHPLT v2 ar ml +MultiHPLT v2 ar mr +MultiHPLT v2 ar ms +MultiHPLT v2 ar mt +MultiHPLT v2 ar nb +MultiHPLT v2 ar ne +MultiHPLT v2 ar nn +MultiHPLT v2 ar si +MultiHPLT v2 ar sk +MultiHPLT v2 ar sl +MultiHPLT v2 ar sq +MultiHPLT v2 ar sr +MultiHPLT v2 ar sw +MultiHPLT v2 ar ta +MultiHPLT v2 ar te +MultiHPLT v2 ar th +MultiHPLT v2 ar tr +MultiHPLT v2 ar uk +MultiHPLT v2 ar ur +MultiHPLT v2 ar uz +MultiHPLT v2 ar vi +MultiHPLT v2 ar xh +MultiHPLT v2 az be +MultiHPLT v2 az bg +MultiHPLT v2 az bn +MultiHPLT v2 az bs +MultiHPLT v2 az ca +MultiHPLT v2 az cy +MultiHPLT v2 az en +MultiHPLT v2 az eo +MultiHPLT v2 az et +MultiHPLT v2 az eu +MultiHPLT v2 az fa +MultiHPLT v2 az fi +MultiHPLT v2 az ga +MultiHPLT v2 az gl +MultiHPLT v2 az gu +MultiHPLT v2 az he +MultiHPLT v2 az hi +MultiHPLT v2 az hr +MultiHPLT v2 az is +MultiHPLT v2 az ja +MultiHPLT v2 az kk +MultiHPLT v2 az kn +MultiHPLT v2 az ko +MultiHPLT v2 az lt +MultiHPLT v2 az lv +MultiHPLT v2 az mk +MultiHPLT v2 az ml +MultiHPLT v2 az mr +MultiHPLT v2 az ms +MultiHPLT v2 az mt +MultiHPLT v2 az nb +MultiHPLT v2 az ne +MultiHPLT v2 az nn +MultiHPLT v2 az si +MultiHPLT v2 az sk +MultiHPLT v2 az sl +MultiHPLT v2 az sq +MultiHPLT v2 az sr +MultiHPLT v2 az sw +MultiHPLT v2 az ta +MultiHPLT v2 az te +MultiHPLT v2 az th +MultiHPLT v2 az tr +MultiHPLT v2 az uk +MultiHPLT v2 az ur +MultiHPLT v2 az uz +MultiHPLT v2 az vi +MultiHPLT v2 az xh +MultiHPLT v2 be bg +MultiHPLT v2 be bn +MultiHPLT v2 be bs +MultiHPLT v2 be ca +MultiHPLT v2 be cy +MultiHPLT v2 be en +MultiHPLT v2 be eo +MultiHPLT v2 be et +MultiHPLT v2 be eu +MultiHPLT v2 be fa +MultiHPLT v2 be fi +MultiHPLT v2 be ga +MultiHPLT v2 be gl +MultiHPLT v2 be gu +MultiHPLT v2 be he +MultiHPLT v2 be hi +MultiHPLT v2 be hr +MultiHPLT v2 be is +MultiHPLT v2 be ja +MultiHPLT v2 be kk +MultiHPLT v2 be kn +MultiHPLT v2 be ko +MultiHPLT v2 be lt +MultiHPLT v2 be lv +MultiHPLT v2 be mk +MultiHPLT v2 be ml +MultiHPLT v2 be mr +MultiHPLT v2 be ms +MultiHPLT v2 be mt +MultiHPLT v2 be nb +MultiHPLT v2 be ne +MultiHPLT v2 be nn +MultiHPLT v2 be si +MultiHPLT v2 be sk +MultiHPLT v2 be sl +MultiHPLT v2 be sq +MultiHPLT v2 be sr +MultiHPLT v2 be sw +MultiHPLT v2 be ta +MultiHPLT v2 be te +MultiHPLT v2 be th +MultiHPLT v2 be tr +MultiHPLT v2 be uk +MultiHPLT v2 be ur +MultiHPLT v2 be uz +MultiHPLT v2 be vi +MultiHPLT v2 be xh +MultiHPLT v2 bg bn +MultiHPLT v2 bg bs +MultiHPLT v2 bg ca +MultiHPLT v2 bg cy +MultiHPLT v2 bg en +MultiHPLT v2 bg eo +MultiHPLT v2 bg et +MultiHPLT v2 bg eu +MultiHPLT v2 bg fa +MultiHPLT v2 bg fi +MultiHPLT v2 bg ga +MultiHPLT v2 bg gl +MultiHPLT v2 bg gu +MultiHPLT v2 bg he +MultiHPLT v2 bg hi +MultiHPLT v2 bg hr +MultiHPLT v2 bg is +MultiHPLT v2 bg ja +MultiHPLT v2 bg kk +MultiHPLT v2 bg kn +MultiHPLT v2 bg ko +MultiHPLT v2 bg lt +MultiHPLT v2 bg lv +MultiHPLT v2 bg mk +MultiHPLT v2 bg ml +MultiHPLT v2 bg mr +MultiHPLT v2 bg ms +MultiHPLT v2 bg mt +MultiHPLT v2 bg nb +MultiHPLT v2 bg ne +MultiHPLT v2 bg nn +MultiHPLT v2 bg si +MultiHPLT v2 bg sk +MultiHPLT v2 bg sl +MultiHPLT v2 bg sq +MultiHPLT v2 bg sr +MultiHPLT v2 bg sw +MultiHPLT v2 bg ta +MultiHPLT v2 bg te +MultiHPLT v2 bg th +MultiHPLT v2 bg tr +MultiHPLT v2 bg uk +MultiHPLT v2 bg ur +MultiHPLT v2 bg uz +MultiHPLT v2 bg vi +MultiHPLT v2 bg xh +MultiHPLT v2 bn bs +MultiHPLT v2 bn ca +MultiHPLT v2 bn cy +MultiHPLT v2 bn en +MultiHPLT v2 bn eo +MultiHPLT v2 bn et +MultiHPLT v2 bn eu +MultiHPLT v2 bn fa +MultiHPLT v2 bn fi +MultiHPLT v2 bn ga +MultiHPLT v2 bn gl +MultiHPLT v2 bn gu +MultiHPLT v2 bn he +MultiHPLT v2 bn hi +MultiHPLT v2 bn hr +MultiHPLT v2 bn is +MultiHPLT v2 bn ja +MultiHPLT v2 bn kk +MultiHPLT v2 bn kn +MultiHPLT v2 bn ko +MultiHPLT v2 bn lt +MultiHPLT v2 bn lv +MultiHPLT v2 bn mk +MultiHPLT v2 bn ml +MultiHPLT v2 bn mr +MultiHPLT v2 bn ms +MultiHPLT v2 bn mt +MultiHPLT v2 bn nb +MultiHPLT v2 bn ne +MultiHPLT v2 bn nn +MultiHPLT v2 bn si +MultiHPLT v2 bn sk +MultiHPLT v2 bn sl +MultiHPLT v2 bn sq +MultiHPLT v2 bn sr +MultiHPLT v2 bn sw +MultiHPLT v2 bn ta +MultiHPLT v2 bn te +MultiHPLT v2 bn th +MultiHPLT v2 bn tr +MultiHPLT v2 bn uk +MultiHPLT v2 bn ur +MultiHPLT v2 bn uz +MultiHPLT v2 bn vi +MultiHPLT v2 bn xh +MultiHPLT v2 bs ca +MultiHPLT v2 bs cy +MultiHPLT v2 bs en +MultiHPLT v2 bs eo +MultiHPLT v2 bs et +MultiHPLT v2 bs eu +MultiHPLT v2 bs fa +MultiHPLT v2 bs fi +MultiHPLT v2 bs ga +MultiHPLT v2 bs gl +MultiHPLT v2 bs gu +MultiHPLT v2 bs he +MultiHPLT v2 bs hi +MultiHPLT v2 bs hr +MultiHPLT v2 bs is +MultiHPLT v2 bs ja +MultiHPLT v2 bs kk +MultiHPLT v2 bs kn +MultiHPLT v2 bs ko +MultiHPLT v2 bs lt +MultiHPLT v2 bs lv +MultiHPLT v2 bs mk +MultiHPLT v2 bs ml +MultiHPLT v2 bs mr +MultiHPLT v2 bs ms +MultiHPLT v2 bs mt +MultiHPLT v2 bs nb +MultiHPLT v2 bs ne +MultiHPLT v2 bs nn +MultiHPLT v2 bs si +MultiHPLT v2 bs sk +MultiHPLT v2 bs sl +MultiHPLT v2 bs sq +MultiHPLT v2 bs sr +MultiHPLT v2 bs sw +MultiHPLT v2 bs ta +MultiHPLT v2 bs te +MultiHPLT v2 bs th +MultiHPLT v2 bs tr +MultiHPLT v2 bs uk +MultiHPLT v2 bs ur +MultiHPLT v2 bs uz +MultiHPLT v2 bs vi +MultiHPLT v2 bs xh +MultiHPLT v2 ca cy +MultiHPLT v2 ca en +MultiHPLT v2 ca eo +MultiHPLT v2 ca et +MultiHPLT v2 ca eu +MultiHPLT v2 ca fa +MultiHPLT v2 ca fi +MultiHPLT v2 ca ga +MultiHPLT v2 ca gl +MultiHPLT v2 ca gu +MultiHPLT v2 ca he +MultiHPLT v2 ca hi +MultiHPLT v2 ca hr +MultiHPLT v2 ca is +MultiHPLT v2 ca ja +MultiHPLT v2 ca kk +MultiHPLT v2 ca kn +MultiHPLT v2 ca ko +MultiHPLT v2 ca lt +MultiHPLT v2 ca lv +MultiHPLT v2 ca mk +MultiHPLT v2 ca ml +MultiHPLT v2 ca mr +MultiHPLT v2 ca ms +MultiHPLT v2 ca mt +MultiHPLT v2 ca nb +MultiHPLT v2 ca ne +MultiHPLT v2 ca nn +MultiHPLT v2 ca si +MultiHPLT v2 ca sk +MultiHPLT v2 ca sl +MultiHPLT v2 ca sq +MultiHPLT v2 ca sr +MultiHPLT v2 ca sw +MultiHPLT v2 ca ta +MultiHPLT v2 ca te +MultiHPLT v2 ca th +MultiHPLT v2 ca tr +MultiHPLT v2 ca uk +MultiHPLT v2 ca ur +MultiHPLT v2 ca uz +MultiHPLT v2 ca vi +MultiHPLT v2 ca xh +MultiHPLT v2 cy en +MultiHPLT v2 cy eo +MultiHPLT v2 cy et +MultiHPLT v2 cy eu +MultiHPLT v2 cy fa +MultiHPLT v2 cy fi +MultiHPLT v2 cy ga +MultiHPLT v2 cy gl +MultiHPLT v2 cy gu +MultiHPLT v2 cy he +MultiHPLT v2 cy hi +MultiHPLT v2 cy hr +MultiHPLT v2 cy is +MultiHPLT v2 cy ja +MultiHPLT v2 cy kk +MultiHPLT v2 cy kn +MultiHPLT v2 cy ko +MultiHPLT v2 cy lt +MultiHPLT v2 cy lv +MultiHPLT v2 cy mk +MultiHPLT v2 cy ml +MultiHPLT v2 cy mr +MultiHPLT v2 cy ms +MultiHPLT v2 cy mt +MultiHPLT v2 cy nb +MultiHPLT v2 cy ne +MultiHPLT v2 cy nn +MultiHPLT v2 cy si +MultiHPLT v2 cy sk +MultiHPLT v2 cy sl +MultiHPLT v2 cy sq +MultiHPLT v2 cy sr +MultiHPLT v2 cy sw +MultiHPLT v2 cy ta +MultiHPLT v2 cy te +MultiHPLT v2 cy th +MultiHPLT v2 cy tr +MultiHPLT v2 cy uk +MultiHPLT v2 cy ur +MultiHPLT v2 cy uz +MultiHPLT v2 cy vi +MultiHPLT v2 cy xh +MultiHPLT v2 en eo +MultiHPLT v2 en et +MultiHPLT v2 en eu +MultiHPLT v2 en fa +MultiHPLT v2 en fi +MultiHPLT v2 en ga +MultiHPLT v2 en gl +MultiHPLT v2 en gu +MultiHPLT v2 en he +MultiHPLT v2 en hi +MultiHPLT v2 en hr +MultiHPLT v2 en is +MultiHPLT v2 en ja +MultiHPLT v2 en kk +MultiHPLT v2 en kn +MultiHPLT v2 en ko +MultiHPLT v2 en lt +MultiHPLT v2 en lv +MultiHPLT v2 en mk +MultiHPLT v2 en ml +MultiHPLT v2 en mr +MultiHPLT v2 en ms +MultiHPLT v2 en mt +MultiHPLT v2 en nb +MultiHPLT v2 en ne +MultiHPLT v2 en nn +MultiHPLT v2 en si +MultiHPLT v2 en sk +MultiHPLT v2 en sl +MultiHPLT v2 en sq +MultiHPLT v2 en sr +MultiHPLT v2 en sw +MultiHPLT v2 en ta +MultiHPLT v2 en te +MultiHPLT v2 en th +MultiHPLT v2 en tr +MultiHPLT v2 en uk +MultiHPLT v2 en ur +MultiHPLT v2 en uz +MultiHPLT v2 en vi +MultiHPLT v2 en xh +MultiHPLT v2 eo et +MultiHPLT v2 eo eu +MultiHPLT v2 eo fa +MultiHPLT v2 eo fi +MultiHPLT v2 eo ga +MultiHPLT v2 eo gl +MultiHPLT v2 eo gu +MultiHPLT v2 eo he +MultiHPLT v2 eo hi +MultiHPLT v2 eo hr +MultiHPLT v2 eo is +MultiHPLT v2 eo ja +MultiHPLT v2 eo kk +MultiHPLT v2 eo kn +MultiHPLT v2 eo ko +MultiHPLT v2 eo lt +MultiHPLT v2 eo lv +MultiHPLT v2 eo mk +MultiHPLT v2 eo ml +MultiHPLT v2 eo mr +MultiHPLT v2 eo ms +MultiHPLT v2 eo mt +MultiHPLT v2 eo nb +MultiHPLT v2 eo ne +MultiHPLT v2 eo nn +MultiHPLT v2 eo si +MultiHPLT v2 eo sk +MultiHPLT v2 eo sl +MultiHPLT v2 eo sq +MultiHPLT v2 eo sr +MultiHPLT v2 eo sw +MultiHPLT v2 eo ta +MultiHPLT v2 eo te +MultiHPLT v2 eo th +MultiHPLT v2 eo tr +MultiHPLT v2 eo uk +MultiHPLT v2 eo ur +MultiHPLT v2 eo uz +MultiHPLT v2 eo vi +MultiHPLT v2 eo xh +MultiHPLT v2 et eu +MultiHPLT v2 et fa +MultiHPLT v2 et fi +MultiHPLT v2 et ga +MultiHPLT v2 et gl +MultiHPLT v2 et gu +MultiHPLT v2 et he +MultiHPLT v2 et hi +MultiHPLT v2 et hr +MultiHPLT v2 et is +MultiHPLT v2 et ja +MultiHPLT v2 et kk +MultiHPLT v2 et kn +MultiHPLT v2 et ko +MultiHPLT v2 et lt +MultiHPLT v2 et lv +MultiHPLT v2 et mk +MultiHPLT v2 et ml +MultiHPLT v2 et mr +MultiHPLT v2 et ms +MultiHPLT v2 et mt +MultiHPLT v2 et nb +MultiHPLT v2 et ne +MultiHPLT v2 et nn +MultiHPLT v2 et si +MultiHPLT v2 et sk +MultiHPLT v2 et sl +MultiHPLT v2 et sq +MultiHPLT v2 et sr +MultiHPLT v2 et sw +MultiHPLT v2 et ta +MultiHPLT v2 et te +MultiHPLT v2 et th +MultiHPLT v2 et tr +MultiHPLT v2 et uk +MultiHPLT v2 et ur +MultiHPLT v2 et uz +MultiHPLT v2 et vi +MultiHPLT v2 et xh +MultiHPLT v2 eu fa +MultiHPLT v2 eu fi +MultiHPLT v2 eu ga +MultiHPLT v2 eu gl +MultiHPLT v2 eu gu +MultiHPLT v2 eu he +MultiHPLT v2 eu hi +MultiHPLT v2 eu hr +MultiHPLT v2 eu is +MultiHPLT v2 eu ja +MultiHPLT v2 eu kk +MultiHPLT v2 eu kn +MultiHPLT v2 eu ko +MultiHPLT v2 eu lt +MultiHPLT v2 eu lv +MultiHPLT v2 eu mk +MultiHPLT v2 eu ml +MultiHPLT v2 eu mr +MultiHPLT v2 eu ms +MultiHPLT v2 eu mt +MultiHPLT v2 eu nb +MultiHPLT v2 eu ne +MultiHPLT v2 eu nn +MultiHPLT v2 eu si +MultiHPLT v2 eu sk +MultiHPLT v2 eu sl +MultiHPLT v2 eu sq +MultiHPLT v2 eu sr +MultiHPLT v2 eu sw +MultiHPLT v2 eu ta +MultiHPLT v2 eu te +MultiHPLT v2 eu th +MultiHPLT v2 eu tr +MultiHPLT v2 eu uk +MultiHPLT v2 eu ur +MultiHPLT v2 eu uz +MultiHPLT v2 eu vi +MultiHPLT v2 eu xh +MultiHPLT v2 fa fi +MultiHPLT v2 fa ga +MultiHPLT v2 fa gl +MultiHPLT v2 fa gu +MultiHPLT v2 fa he +MultiHPLT v2 fa hi +MultiHPLT v2 fa hr +MultiHPLT v2 fa is +MultiHPLT v2 fa ja +MultiHPLT v2 fa kk +MultiHPLT v2 fa kn +MultiHPLT v2 fa ko +MultiHPLT v2 fa lt +MultiHPLT v2 fa lv +MultiHPLT v2 fa mk +MultiHPLT v2 fa ml +MultiHPLT v2 fa mr +MultiHPLT v2 fa ms +MultiHPLT v2 fa mt +MultiHPLT v2 fa nb +MultiHPLT v2 fa ne +MultiHPLT v2 fa nn +MultiHPLT v2 fa si +MultiHPLT v2 fa sk +MultiHPLT v2 fa sl +MultiHPLT v2 fa sq +MultiHPLT v2 fa sr +MultiHPLT v2 fa sw +MultiHPLT v2 fa ta +MultiHPLT v2 fa te +MultiHPLT v2 fa th +MultiHPLT v2 fa tr +MultiHPLT v2 fa uk +MultiHPLT v2 fa ur +MultiHPLT v2 fa uz +MultiHPLT v2 fa vi +MultiHPLT v2 fa xh +MultiHPLT v2 fi ga +MultiHPLT v2 fi gl +MultiHPLT v2 fi gu +MultiHPLT v2 fi he +MultiHPLT v2 fi hi +MultiHPLT v2 fi hr +MultiHPLT v2 fi is +MultiHPLT v2 fi ja +MultiHPLT v2 fi kk +MultiHPLT v2 fi kn +MultiHPLT v2 fi ko +MultiHPLT v2 fi lt +MultiHPLT v2 fi lv +MultiHPLT v2 fi mk +MultiHPLT v2 fi ml +MultiHPLT v2 fi mr +MultiHPLT v2 fi ms +MultiHPLT v2 fi mt +MultiHPLT v2 fi nb +MultiHPLT v2 fi ne +MultiHPLT v2 fi nn +MultiHPLT v2 fi si +MultiHPLT v2 fi sk +MultiHPLT v2 fi sl +MultiHPLT v2 fi sq +MultiHPLT v2 fi sr +MultiHPLT v2 fi sw +MultiHPLT v2 fi ta +MultiHPLT v2 fi te +MultiHPLT v2 fi th +MultiHPLT v2 fi tr +MultiHPLT v2 fi uk +MultiHPLT v2 fi ur +MultiHPLT v2 fi uz +MultiHPLT v2 fi vi +MultiHPLT v2 fi xh +MultiHPLT v2 ga gl +MultiHPLT v2 ga gu +MultiHPLT v2 ga he +MultiHPLT v2 ga hi +MultiHPLT v2 ga hr +MultiHPLT v2 ga is +MultiHPLT v2 ga ja +MultiHPLT v2 ga kk +MultiHPLT v2 ga kn +MultiHPLT v2 ga ko +MultiHPLT v2 ga lt +MultiHPLT v2 ga lv +MultiHPLT v2 ga mk +MultiHPLT v2 ga ml +MultiHPLT v2 ga mr +MultiHPLT v2 ga ms +MultiHPLT v2 ga mt +MultiHPLT v2 ga nb +MultiHPLT v2 ga ne +MultiHPLT v2 ga nn +MultiHPLT v2 ga si +MultiHPLT v2 ga sk +MultiHPLT v2 ga sl +MultiHPLT v2 ga sq +MultiHPLT v2 ga sr +MultiHPLT v2 ga sw +MultiHPLT v2 ga ta +MultiHPLT v2 ga te +MultiHPLT v2 ga th +MultiHPLT v2 ga tr +MultiHPLT v2 ga uk +MultiHPLT v2 ga ur +MultiHPLT v2 ga uz +MultiHPLT v2 ga vi +MultiHPLT v2 ga xh +MultiHPLT v2 gl gu +MultiHPLT v2 gl he +MultiHPLT v2 gl hi +MultiHPLT v2 gl hr +MultiHPLT v2 gl is +MultiHPLT v2 gl ja +MultiHPLT v2 gl kk +MultiHPLT v2 gl kn +MultiHPLT v2 gl ko +MultiHPLT v2 gl lt +MultiHPLT v2 gl lv +MultiHPLT v2 gl mk +MultiHPLT v2 gl ml +MultiHPLT v2 gl mr +MultiHPLT v2 gl ms +MultiHPLT v2 gl mt +MultiHPLT v2 gl nb +MultiHPLT v2 gl ne +MultiHPLT v2 gl nn +MultiHPLT v2 gl si +MultiHPLT v2 gl sk +MultiHPLT v2 gl sl +MultiHPLT v2 gl sq +MultiHPLT v2 gl sr +MultiHPLT v2 gl sw +MultiHPLT v2 gl ta +MultiHPLT v2 gl te +MultiHPLT v2 gl th +MultiHPLT v2 gl tr +MultiHPLT v2 gl uk +MultiHPLT v2 gl ur +MultiHPLT v2 gl uz +MultiHPLT v2 gl vi +MultiHPLT v2 gl xh +MultiHPLT v2 gu he +MultiHPLT v2 gu hi +MultiHPLT v2 gu hr +MultiHPLT v2 gu is +MultiHPLT v2 gu ja +MultiHPLT v2 gu kk +MultiHPLT v2 gu kn +MultiHPLT v2 gu ko +MultiHPLT v2 gu lt +MultiHPLT v2 gu lv +MultiHPLT v2 gu mk +MultiHPLT v2 gu ml +MultiHPLT v2 gu mr +MultiHPLT v2 gu ms +MultiHPLT v2 gu mt +MultiHPLT v2 gu nb +MultiHPLT v2 gu ne +MultiHPLT v2 gu nn +MultiHPLT v2 gu si +MultiHPLT v2 gu sk +MultiHPLT v2 gu sl +MultiHPLT v2 gu sq +MultiHPLT v2 gu sr +MultiHPLT v2 gu sw +MultiHPLT v2 gu ta +MultiHPLT v2 gu te +MultiHPLT v2 gu th +MultiHPLT v2 gu tr +MultiHPLT v2 gu uk +MultiHPLT v2 gu ur +MultiHPLT v2 gu uz +MultiHPLT v2 gu vi +MultiHPLT v2 gu xh +MultiHPLT v2 he hi +MultiHPLT v2 he hr +MultiHPLT v2 he is +MultiHPLT v2 he ja +MultiHPLT v2 he kk +MultiHPLT v2 he kn +MultiHPLT v2 he ko +MultiHPLT v2 he lt +MultiHPLT v2 he lv +MultiHPLT v2 he mk +MultiHPLT v2 he ml +MultiHPLT v2 he mr +MultiHPLT v2 he ms +MultiHPLT v2 he mt +MultiHPLT v2 he nb +MultiHPLT v2 he ne +MultiHPLT v2 he nn +MultiHPLT v2 he si +MultiHPLT v2 he sk +MultiHPLT v2 he sl +MultiHPLT v2 he sq +MultiHPLT v2 he sr +MultiHPLT v2 he sw +MultiHPLT v2 he ta +MultiHPLT v2 he te +MultiHPLT v2 he th +MultiHPLT v2 he tr +MultiHPLT v2 he uk +MultiHPLT v2 he ur +MultiHPLT v2 he uz +MultiHPLT v2 he vi +MultiHPLT v2 he xh +MultiHPLT v2 hi hr +MultiHPLT v2 hi is +MultiHPLT v2 hi ja +MultiHPLT v2 hi kk +MultiHPLT v2 hi kn +MultiHPLT v2 hi ko +MultiHPLT v2 hi lt +MultiHPLT v2 hi lv +MultiHPLT v2 hi mk +MultiHPLT v2 hi ml +MultiHPLT v2 hi mr +MultiHPLT v2 hi ms +MultiHPLT v2 hi mt +MultiHPLT v2 hi nb +MultiHPLT v2 hi ne +MultiHPLT v2 hi nn +MultiHPLT v2 hi si +MultiHPLT v2 hi sk +MultiHPLT v2 hi sl +MultiHPLT v2 hi sq +MultiHPLT v2 hi sr +MultiHPLT v2 hi sw +MultiHPLT v2 hi ta +MultiHPLT v2 hi te +MultiHPLT v2 hi th +MultiHPLT v2 hi tr +MultiHPLT v2 hi uk +MultiHPLT v2 hi ur +MultiHPLT v2 hi uz +MultiHPLT v2 hi vi +MultiHPLT v2 hi xh +MultiHPLT v2 hr is +MultiHPLT v2 hr ja +MultiHPLT v2 hr kk +MultiHPLT v2 hr kn +MultiHPLT v2 hr ko +MultiHPLT v2 hr lt +MultiHPLT v2 hr lv +MultiHPLT v2 hr mk +MultiHPLT v2 hr ml +MultiHPLT v2 hr mr +MultiHPLT v2 hr ms +MultiHPLT v2 hr mt +MultiHPLT v2 hr nb +MultiHPLT v2 hr ne +MultiHPLT v2 hr nn +MultiHPLT v2 hr si +MultiHPLT v2 hr sk +MultiHPLT v2 hr sl +MultiHPLT v2 hr sq +MultiHPLT v2 hr sr +MultiHPLT v2 hr sw +MultiHPLT v2 hr ta +MultiHPLT v2 hr te +MultiHPLT v2 hr th +MultiHPLT v2 hr tr +MultiHPLT v2 hr uk +MultiHPLT v2 hr ur +MultiHPLT v2 hr uz +MultiHPLT v2 hr vi +MultiHPLT v2 hr xh +MultiHPLT v2 is ja +MultiHPLT v2 is kk +MultiHPLT v2 is kn +MultiHPLT v2 is ko +MultiHPLT v2 is lt +MultiHPLT v2 is lv +MultiHPLT v2 is mk +MultiHPLT v2 is ml +MultiHPLT v2 is mr +MultiHPLT v2 is ms +MultiHPLT v2 is mt +MultiHPLT v2 is nb +MultiHPLT v2 is ne +MultiHPLT v2 is nn +MultiHPLT v2 is si +MultiHPLT v2 is sk +MultiHPLT v2 is sl +MultiHPLT v2 is sq +MultiHPLT v2 is sr +MultiHPLT v2 is sw +MultiHPLT v2 is ta +MultiHPLT v2 is te +MultiHPLT v2 is th +MultiHPLT v2 is tr +MultiHPLT v2 is uk +MultiHPLT v2 is ur +MultiHPLT v2 is uz +MultiHPLT v2 is vi +MultiHPLT v2 is xh +MultiHPLT v2 ja kk +MultiHPLT v2 ja kn +MultiHPLT v2 ja ko +MultiHPLT v2 ja lt +MultiHPLT v2 ja lv +MultiHPLT v2 ja mk +MultiHPLT v2 ja ml +MultiHPLT v2 ja mr +MultiHPLT v2 ja ms +MultiHPLT v2 ja mt +MultiHPLT v2 ja nb +MultiHPLT v2 ja ne +MultiHPLT v2 ja nn +MultiHPLT v2 ja si +MultiHPLT v2 ja sk +MultiHPLT v2 ja sl +MultiHPLT v2 ja sq +MultiHPLT v2 ja sr +MultiHPLT v2 ja sw +MultiHPLT v2 ja ta +MultiHPLT v2 ja te +MultiHPLT v2 ja th +MultiHPLT v2 ja tr +MultiHPLT v2 ja uk +MultiHPLT v2 ja ur +MultiHPLT v2 ja uz +MultiHPLT v2 ja vi +MultiHPLT v2 ja xh +MultiHPLT v2 kk kn +MultiHPLT v2 kk ko +MultiHPLT v2 kk lt +MultiHPLT v2 kk lv +MultiHPLT v2 kk mk +MultiHPLT v2 kk ml +MultiHPLT v2 kk mr +MultiHPLT v2 kk ms +MultiHPLT v2 kk mt +MultiHPLT v2 kk nb +MultiHPLT v2 kk ne +MultiHPLT v2 kk nn +MultiHPLT v2 kk si +MultiHPLT v2 kk sk +MultiHPLT v2 kk sl +MultiHPLT v2 kk sq +MultiHPLT v2 kk sr +MultiHPLT v2 kk sw +MultiHPLT v2 kk ta +MultiHPLT v2 kk te +MultiHPLT v2 kk th +MultiHPLT v2 kk tr +MultiHPLT v2 kk uk +MultiHPLT v2 kk ur +MultiHPLT v2 kk uz +MultiHPLT v2 kk vi +MultiHPLT v2 kk xh +MultiHPLT v2 kn ko +MultiHPLT v2 kn lt +MultiHPLT v2 kn lv +MultiHPLT v2 kn mk +MultiHPLT v2 kn ml +MultiHPLT v2 kn mr +MultiHPLT v2 kn ms +MultiHPLT v2 kn mt +MultiHPLT v2 kn nb +MultiHPLT v2 kn ne +MultiHPLT v2 kn nn +MultiHPLT v2 kn si +MultiHPLT v2 kn sk +MultiHPLT v2 kn sl +MultiHPLT v2 kn sq +MultiHPLT v2 kn sr +MultiHPLT v2 kn sw +MultiHPLT v2 kn ta +MultiHPLT v2 kn te +MultiHPLT v2 kn th +MultiHPLT v2 kn tr +MultiHPLT v2 kn uk +MultiHPLT v2 kn ur +MultiHPLT v2 kn uz +MultiHPLT v2 kn vi +MultiHPLT v2 kn xh +MultiHPLT v2 ko lt +MultiHPLT v2 ko lv +MultiHPLT v2 ko mk +MultiHPLT v2 ko ml +MultiHPLT v2 ko mr +MultiHPLT v2 ko ms +MultiHPLT v2 ko mt +MultiHPLT v2 ko nb +MultiHPLT v2 ko ne +MultiHPLT v2 ko nn +MultiHPLT v2 ko si +MultiHPLT v2 ko sk +MultiHPLT v2 ko sl +MultiHPLT v2 ko sq +MultiHPLT v2 ko sr +MultiHPLT v2 ko sw +MultiHPLT v2 ko ta +MultiHPLT v2 ko te +MultiHPLT v2 ko th +MultiHPLT v2 ko tr +MultiHPLT v2 ko uk +MultiHPLT v2 ko ur +MultiHPLT v2 ko uz +MultiHPLT v2 ko vi +MultiHPLT v2 ko xh +MultiHPLT v2 lt lv +MultiHPLT v2 lt mk +MultiHPLT v2 lt ml +MultiHPLT v2 lt mr +MultiHPLT v2 lt ms +MultiHPLT v2 lt mt +MultiHPLT v2 lt nb +MultiHPLT v2 lt ne +MultiHPLT v2 lt nn +MultiHPLT v2 lt si +MultiHPLT v2 lt sk +MultiHPLT v2 lt sl +MultiHPLT v2 lt sq +MultiHPLT v2 lt sr +MultiHPLT v2 lt sw +MultiHPLT v2 lt ta +MultiHPLT v2 lt te +MultiHPLT v2 lt th +MultiHPLT v2 lt tr +MultiHPLT v2 lt uk +MultiHPLT v2 lt ur +MultiHPLT v2 lt uz +MultiHPLT v2 lt vi +MultiHPLT v2 lt xh +MultiHPLT v2 lv mk +MultiHPLT v2 lv ml +MultiHPLT v2 lv mr +MultiHPLT v2 lv ms +MultiHPLT v2 lv mt +MultiHPLT v2 lv nb +MultiHPLT v2 lv ne +MultiHPLT v2 lv nn +MultiHPLT v2 lv si +MultiHPLT v2 lv sk +MultiHPLT v2 lv sl +MultiHPLT v2 lv sq +MultiHPLT v2 lv sr +MultiHPLT v2 lv sw +MultiHPLT v2 lv ta +MultiHPLT v2 lv te +MultiHPLT v2 lv th +MultiHPLT v2 lv tr +MultiHPLT v2 lv uk +MultiHPLT v2 lv ur +MultiHPLT v2 lv uz +MultiHPLT v2 lv vi +MultiHPLT v2 lv xh +MultiHPLT v2 mk ml +MultiHPLT v2 mk mr +MultiHPLT v2 mk ms +MultiHPLT v2 mk mt +MultiHPLT v2 mk nb +MultiHPLT v2 mk ne +MultiHPLT v2 mk nn +MultiHPLT v2 mk si +MultiHPLT v2 mk sk +MultiHPLT v2 mk sl +MultiHPLT v2 mk sq +MultiHPLT v2 mk sr +MultiHPLT v2 mk sw +MultiHPLT v2 mk ta +MultiHPLT v2 mk te +MultiHPLT v2 mk th +MultiHPLT v2 mk tr +MultiHPLT v2 mk uk +MultiHPLT v2 mk ur +MultiHPLT v2 mk uz +MultiHPLT v2 mk vi +MultiHPLT v2 mk xh +MultiHPLT v2 ml mr +MultiHPLT v2 ml ms +MultiHPLT v2 ml mt +MultiHPLT v2 ml nb +MultiHPLT v2 ml ne +MultiHPLT v2 ml nn +MultiHPLT v2 ml si +MultiHPLT v2 ml sk +MultiHPLT v2 ml sl +MultiHPLT v2 ml sq +MultiHPLT v2 ml sr +MultiHPLT v2 ml sw +MultiHPLT v2 ml ta +MultiHPLT v2 ml te +MultiHPLT v2 ml th +MultiHPLT v2 ml tr +MultiHPLT v2 ml uk +MultiHPLT v2 ml ur +MultiHPLT v2 ml uz +MultiHPLT v2 ml vi +MultiHPLT v2 ml xh +MultiHPLT v2 mr ms +MultiHPLT v2 mr mt +MultiHPLT v2 mr nb +MultiHPLT v2 mr ne +MultiHPLT v2 mr nn +MultiHPLT v2 mr si +MultiHPLT v2 mr sk +MultiHPLT v2 mr sl +MultiHPLT v2 mr sq +MultiHPLT v2 mr sr +MultiHPLT v2 mr sw +MultiHPLT v2 mr ta +MultiHPLT v2 mr te +MultiHPLT v2 mr th +MultiHPLT v2 mr tr +MultiHPLT v2 mr uk +MultiHPLT v2 mr ur +MultiHPLT v2 mr uz +MultiHPLT v2 mr vi +MultiHPLT v2 mr xh +MultiHPLT v2 ms mt +MultiHPLT v2 ms nb +MultiHPLT v2 ms ne +MultiHPLT v2 ms nn +MultiHPLT v2 ms si +MultiHPLT v2 ms sk +MultiHPLT v2 ms sl +MultiHPLT v2 ms sq +MultiHPLT v2 ms sr +MultiHPLT v2 ms sw +MultiHPLT v2 ms ta +MultiHPLT v2 ms te +MultiHPLT v2 ms th +MultiHPLT v2 ms tr +MultiHPLT v2 ms uk +MultiHPLT v2 ms ur +MultiHPLT v2 ms uz +MultiHPLT v2 ms vi +MultiHPLT v2 ms xh +MultiHPLT v2 mt nb +MultiHPLT v2 mt ne +MultiHPLT v2 mt nn +MultiHPLT v2 mt si +MultiHPLT v2 mt sk +MultiHPLT v2 mt sl +MultiHPLT v2 mt sq +MultiHPLT v2 mt sr +MultiHPLT v2 mt sw +MultiHPLT v2 mt ta +MultiHPLT v2 mt te +MultiHPLT v2 mt th +MultiHPLT v2 mt tr +MultiHPLT v2 mt uk +MultiHPLT v2 mt ur +MultiHPLT v2 mt uz +MultiHPLT v2 mt vi +MultiHPLT v2 mt xh +MultiHPLT v2 nb ne +MultiHPLT v2 nb nn +MultiHPLT v2 nb si +MultiHPLT v2 nb sk +MultiHPLT v2 nb sl +MultiHPLT v2 nb sq +MultiHPLT v2 nb sr +MultiHPLT v2 nb sw +MultiHPLT v2 nb ta +MultiHPLT v2 nb te +MultiHPLT v2 nb th +MultiHPLT v2 nb tr +MultiHPLT v2 nb uk +MultiHPLT v2 nb ur +MultiHPLT v2 nb uz +MultiHPLT v2 nb vi +MultiHPLT v2 nb xh +MultiHPLT v2 ne nn +MultiHPLT v2 ne si +MultiHPLT v2 ne sk +MultiHPLT v2 ne sl +MultiHPLT v2 ne sq +MultiHPLT v2 ne sr +MultiHPLT v2 ne sw +MultiHPLT v2 ne ta +MultiHPLT v2 ne te +MultiHPLT v2 ne th +MultiHPLT v2 ne tr +MultiHPLT v2 ne uk +MultiHPLT v2 ne ur +MultiHPLT v2 ne uz +MultiHPLT v2 ne vi +MultiHPLT v2 ne xh +MultiHPLT v2 nn si +MultiHPLT v2 nn sk +MultiHPLT v2 nn sl +MultiHPLT v2 nn sq +MultiHPLT v2 nn sr +MultiHPLT v2 nn sw +MultiHPLT v2 nn ta +MultiHPLT v2 nn te +MultiHPLT v2 nn th +MultiHPLT v2 nn tr +MultiHPLT v2 nn uk +MultiHPLT v2 nn ur +MultiHPLT v2 nn uz +MultiHPLT v2 nn vi +MultiHPLT v2 nn xh +MultiHPLT v2 si sk +MultiHPLT v2 si sl +MultiHPLT v2 si sq +MultiHPLT v2 si sr +MultiHPLT v2 si sw +MultiHPLT v2 si ta +MultiHPLT v2 si te +MultiHPLT v2 si th +MultiHPLT v2 si tr +MultiHPLT v2 si uk +MultiHPLT v2 si ur +MultiHPLT v2 si uz +MultiHPLT v2 si vi +MultiHPLT v2 si xh +MultiHPLT v2 sk sl +MultiHPLT v2 sk sq +MultiHPLT v2 sk sr +MultiHPLT v2 sk sw +MultiHPLT v2 sk ta +MultiHPLT v2 sk te +MultiHPLT v2 sk th +MultiHPLT v2 sk tr +MultiHPLT v2 sk uk +MultiHPLT v2 sk ur +MultiHPLT v2 sk uz +MultiHPLT v2 sk vi +MultiHPLT v2 sk xh +MultiHPLT v2 sl sq +MultiHPLT v2 sl sr +MultiHPLT v2 sl sw +MultiHPLT v2 sl ta +MultiHPLT v2 sl te +MultiHPLT v2 sl th +MultiHPLT v2 sl tr +MultiHPLT v2 sl uk +MultiHPLT v2 sl ur +MultiHPLT v2 sl uz +MultiHPLT v2 sl vi +MultiHPLT v2 sl xh +MultiHPLT v2 sq sr +MultiHPLT v2 sq sw +MultiHPLT v2 sq ta +MultiHPLT v2 sq te +MultiHPLT v2 sq th +MultiHPLT v2 sq tr +MultiHPLT v2 sq uk +MultiHPLT v2 sq ur +MultiHPLT v2 sq uz +MultiHPLT v2 sq vi +MultiHPLT v2 sq xh +MultiHPLT v2 sr sw +MultiHPLT v2 sr ta +MultiHPLT v2 sr te +MultiHPLT v2 sr th +MultiHPLT v2 sr tr +MultiHPLT v2 sr uk +MultiHPLT v2 sr ur +MultiHPLT v2 sr uz +MultiHPLT v2 sr vi +MultiHPLT v2 sr xh +MultiHPLT v2 sw ta +MultiHPLT v2 sw te +MultiHPLT v2 sw th +MultiHPLT v2 sw tr +MultiHPLT v2 sw uk +MultiHPLT v2 sw ur +MultiHPLT v2 sw uz +MultiHPLT v2 sw vi +MultiHPLT v2 sw xh +MultiHPLT v2 ta te +MultiHPLT v2 ta th +MultiHPLT v2 ta tr +MultiHPLT v2 ta uk +MultiHPLT v2 ta ur +MultiHPLT v2 ta uz +MultiHPLT v2 ta vi +MultiHPLT v2 ta xh +MultiHPLT v2 te th +MultiHPLT v2 te tr +MultiHPLT v2 te uk +MultiHPLT v2 te ur +MultiHPLT v2 te uz +MultiHPLT v2 te vi +MultiHPLT v2 te xh +MultiHPLT v2 th tr +MultiHPLT v2 th uk +MultiHPLT v2 th ur +MultiHPLT v2 th uz +MultiHPLT v2 th vi +MultiHPLT v2 th xh +MultiHPLT v2 tr uk +MultiHPLT v2 tr ur +MultiHPLT v2 tr uz +MultiHPLT v2 tr vi +MultiHPLT v2 tr xh +MultiHPLT v2 uk ur +MultiHPLT v2 uk uz +MultiHPLT v2 uk vi +MultiHPLT v2 uk xh +MultiHPLT v2 ur uz +MultiHPLT v2 ur vi +MultiHPLT v2 ur xh +MultiHPLT v2 uz vi +MultiHPLT v2 uz xh +MultiHPLT v2 vi xh MultiMaCoCu v2 bg bs_Cyrl MultiMaCoCu v2 bg bs_Latn MultiMaCoCu v2 bg ca @@ -53949,6 +55274,3398 @@ OpenSubtitles v2018 ze_en zh_tw OpenSubtitles v2018 ze_zh zh_cn OpenSubtitles v2018 ze_zh zh_tw OpenSubtitles v2018 zh_cn zh_tw +OpenSubtitles v2024 af ar +OpenSubtitles v2024 af az +OpenSubtitles v2024 af bg +OpenSubtitles v2024 af bn +OpenSubtitles v2024 af bs +OpenSubtitles v2024 af ca +OpenSubtitles v2024 af cs +OpenSubtitles v2024 af da +OpenSubtitles v2024 af de +OpenSubtitles v2024 af el +OpenSubtitles v2024 af en +OpenSubtitles v2024 af en_ze +OpenSubtitles v2024 af eo +OpenSubtitles v2024 af es +OpenSubtitles v2024 af es_149 +OpenSubtitles v2024 af es_419 +OpenSubtitles v2024 af et +OpenSubtitles v2024 af eu +OpenSubtitles v2024 af fa +OpenSubtitles v2024 af fi +OpenSubtitles v2024 af fr +OpenSubtitles v2024 af he +OpenSubtitles v2024 af hi +OpenSubtitles v2024 af hr +OpenSubtitles v2024 af hu +OpenSubtitles v2024 af hy +OpenSubtitles v2024 af id +OpenSubtitles v2024 af is +OpenSubtitles v2024 af it +OpenSubtitles v2024 af ja +OpenSubtitles v2024 af ka +OpenSubtitles v2024 af km +OpenSubtitles v2024 af ko +OpenSubtitles v2024 af ku +OpenSubtitles v2024 af lt +OpenSubtitles v2024 af lv +OpenSubtitles v2024 af mk +OpenSubtitles v2024 af ml +OpenSubtitles v2024 af mn +OpenSubtitles v2024 af ms +OpenSubtitles v2024 af my +OpenSubtitles v2024 af nl +OpenSubtitles v2024 af no +OpenSubtitles v2024 af pl +OpenSubtitles v2024 af pt +OpenSubtitles v2024 af pt_BR +OpenSubtitles v2024 af ro +OpenSubtitles v2024 af ru +OpenSubtitles v2024 af sd +OpenSubtitles v2024 af si +OpenSubtitles v2024 af sk +OpenSubtitles v2024 af sl +OpenSubtitles v2024 af sq +OpenSubtitles v2024 af sr +OpenSubtitles v2024 af sv +OpenSubtitles v2024 af te +OpenSubtitles v2024 af th +OpenSubtitles v2024 af tl +OpenSubtitles v2024 af tr +OpenSubtitles v2024 af uk +OpenSubtitles v2024 af ur +OpenSubtitles v2024 af vi +OpenSubtitles v2024 af zh_CN +OpenSubtitles v2024 af zh_TW +OpenSubtitles v2024 af zh_ze +OpenSubtitles v2024 am ar +OpenSubtitles v2024 am bg +OpenSubtitles v2024 am bn +OpenSubtitles v2024 am cs +OpenSubtitles v2024 am da +OpenSubtitles v2024 am de +OpenSubtitles v2024 am el +OpenSubtitles v2024 am en +OpenSubtitles v2024 am es +OpenSubtitles v2024 am et +OpenSubtitles v2024 am fa +OpenSubtitles v2024 am fi +OpenSubtitles v2024 am fr +OpenSubtitles v2024 am he +OpenSubtitles v2024 am hr +OpenSubtitles v2024 am hu +OpenSubtitles v2024 am id +OpenSubtitles v2024 am is +OpenSubtitles v2024 am it +OpenSubtitles v2024 am ko +OpenSubtitles v2024 am ku +OpenSubtitles v2024 am mk +OpenSubtitles v2024 am ml +OpenSubtitles v2024 am ms +OpenSubtitles v2024 am nl +OpenSubtitles v2024 am no +OpenSubtitles v2024 am pl +OpenSubtitles v2024 am pt +OpenSubtitles v2024 am pt_BR +OpenSubtitles v2024 am ro +OpenSubtitles v2024 am ru +OpenSubtitles v2024 am si +OpenSubtitles v2024 am sk +OpenSubtitles v2024 am sl +OpenSubtitles v2024 am sr +OpenSubtitles v2024 am sv +OpenSubtitles v2024 am th +OpenSubtitles v2024 am tr +OpenSubtitles v2024 am uk +OpenSubtitles v2024 am vi +OpenSubtitles v2024 am zh_CN +OpenSubtitles v2024 am zh_TW +OpenSubtitles v2024 am zh_ze +OpenSubtitles v2024 an ar +OpenSubtitles v2024 an bg +OpenSubtitles v2024 an bn +OpenSubtitles v2024 an cs +OpenSubtitles v2024 an da +OpenSubtitles v2024 an de +OpenSubtitles v2024 an el +OpenSubtitles v2024 an en +OpenSubtitles v2024 an es +OpenSubtitles v2024 an es_149 +OpenSubtitles v2024 an es_419 +OpenSubtitles v2024 an es_ES +OpenSubtitles v2024 an et +OpenSubtitles v2024 an eu +OpenSubtitles v2024 an fa +OpenSubtitles v2024 an fi +OpenSubtitles v2024 an fr +OpenSubtitles v2024 an he +OpenSubtitles v2024 an hr +OpenSubtitles v2024 an hu +OpenSubtitles v2024 an id +OpenSubtitles v2024 an it +OpenSubtitles v2024 an ko +OpenSubtitles v2024 an lt +OpenSubtitles v2024 an ml +OpenSubtitles v2024 an ms +OpenSubtitles v2024 an nl +OpenSubtitles v2024 an pl +OpenSubtitles v2024 an pt +OpenSubtitles v2024 an pt_BR +OpenSubtitles v2024 an ro +OpenSubtitles v2024 an ru +OpenSubtitles v2024 an sl +OpenSubtitles v2024 an sr +OpenSubtitles v2024 an sv +OpenSubtitles v2024 an th +OpenSubtitles v2024 an tr +OpenSubtitles v2024 an uk +OpenSubtitles v2024 an vi +OpenSubtitles v2024 an zh_CN +OpenSubtitles v2024 an zh_TW +OpenSubtitles v2024 ar as +OpenSubtitles v2024 ar ast +OpenSubtitles v2024 ar az +OpenSubtitles v2024 ar be +OpenSubtitles v2024 ar bg +OpenSubtitles v2024 ar bn +OpenSubtitles v2024 ar br +OpenSubtitles v2024 ar bs +OpenSubtitles v2024 ar ca +OpenSubtitles v2024 ar cs +OpenSubtitles v2024 ar cy +OpenSubtitles v2024 ar da +OpenSubtitles v2024 ar de +OpenSubtitles v2024 ar el +OpenSubtitles v2024 ar en +OpenSubtitles v2024 ar en_ze +OpenSubtitles v2024 ar eo +OpenSubtitles v2024 ar es +OpenSubtitles v2024 ar es_149 +OpenSubtitles v2024 ar es_419 +OpenSubtitles v2024 ar es_ES +OpenSubtitles v2024 ar et +OpenSubtitles v2024 ar eu +OpenSubtitles v2024 ar fa +OpenSubtitles v2024 ar fi +OpenSubtitles v2024 ar fr +OpenSubtitles v2024 ar ga +OpenSubtitles v2024 ar gd +OpenSubtitles v2024 ar gl +OpenSubtitles v2024 ar he +OpenSubtitles v2024 ar hi +OpenSubtitles v2024 ar hr +OpenSubtitles v2024 ar hu +OpenSubtitles v2024 ar hy +OpenSubtitles v2024 ar id +OpenSubtitles v2024 ar ig +OpenSubtitles v2024 ar is +OpenSubtitles v2024 ar it +OpenSubtitles v2024 ar ja +OpenSubtitles v2024 ar ka +OpenSubtitles v2024 ar kk +OpenSubtitles v2024 ar km +OpenSubtitles v2024 ar kn +OpenSubtitles v2024 ar ko +OpenSubtitles v2024 ar ku +OpenSubtitles v2024 ar lb +OpenSubtitles v2024 ar lt +OpenSubtitles v2024 ar lv +OpenSubtitles v2024 ar mk +OpenSubtitles v2024 ar ml +OpenSubtitles v2024 ar mn +OpenSubtitles v2024 ar ms +OpenSubtitles v2024 ar my +OpenSubtitles v2024 ar ne +OpenSubtitles v2024 ar nl +OpenSubtitles v2024 ar nn +OpenSubtitles v2024 ar no +OpenSubtitles v2024 ar oc +OpenSubtitles v2024 ar or +OpenSubtitles v2024 ar pl +OpenSubtitles v2024 ar pt +OpenSubtitles v2024 ar pt_BR +OpenSubtitles v2024 ar pt_MZ +OpenSubtitles v2024 ar ro +OpenSubtitles v2024 ar ru +OpenSubtitles v2024 ar sd +OpenSubtitles v2024 ar si +OpenSubtitles v2024 ar sk +OpenSubtitles v2024 ar sl +OpenSubtitles v2024 ar so +OpenSubtitles v2024 ar sq +OpenSubtitles v2024 ar sr +OpenSubtitles v2024 ar sv +OpenSubtitles v2024 ar sw +OpenSubtitles v2024 ar ta +OpenSubtitles v2024 ar te +OpenSubtitles v2024 ar th +OpenSubtitles v2024 ar tl +OpenSubtitles v2024 ar tr +OpenSubtitles v2024 ar tt +OpenSubtitles v2024 ar uk +OpenSubtitles v2024 ar ur +OpenSubtitles v2024 ar vi +OpenSubtitles v2024 ar yue +OpenSubtitles v2024 ar zh_CN +OpenSubtitles v2024 ar zh_TW +OpenSubtitles v2024 ar zh_ze +OpenSubtitles v2024 as bg +OpenSubtitles v2024 as bn +OpenSubtitles v2024 as bs +OpenSubtitles v2024 as cs +OpenSubtitles v2024 as da +OpenSubtitles v2024 as de +OpenSubtitles v2024 as el +OpenSubtitles v2024 as en +OpenSubtitles v2024 as es +OpenSubtitles v2024 as es_149 +OpenSubtitles v2024 as es_419 +OpenSubtitles v2024 as es_ES +OpenSubtitles v2024 as et +OpenSubtitles v2024 as fa +OpenSubtitles v2024 as fi +OpenSubtitles v2024 as fr +OpenSubtitles v2024 as he +OpenSubtitles v2024 as hr +OpenSubtitles v2024 as hu +OpenSubtitles v2024 as id +OpenSubtitles v2024 as is +OpenSubtitles v2024 as it +OpenSubtitles v2024 as ko +OpenSubtitles v2024 as ku +OpenSubtitles v2024 as mk +OpenSubtitles v2024 as ml +OpenSubtitles v2024 as ms +OpenSubtitles v2024 as nl +OpenSubtitles v2024 as no +OpenSubtitles v2024 as oc +OpenSubtitles v2024 as pl +OpenSubtitles v2024 as pt +OpenSubtitles v2024 as pt_BR +OpenSubtitles v2024 as ro +OpenSubtitles v2024 as ru +OpenSubtitles v2024 as sd +OpenSubtitles v2024 as sk +OpenSubtitles v2024 as sl +OpenSubtitles v2024 as sr +OpenSubtitles v2024 as sv +OpenSubtitles v2024 as te +OpenSubtitles v2024 as th +OpenSubtitles v2024 as tr +OpenSubtitles v2024 as uk +OpenSubtitles v2024 as ur +OpenSubtitles v2024 as vi +OpenSubtitles v2024 as zh_CN +OpenSubtitles v2024 ast bg +OpenSubtitles v2024 ast bn +OpenSubtitles v2024 ast bs +OpenSubtitles v2024 ast ca +OpenSubtitles v2024 ast cs +OpenSubtitles v2024 ast cy +OpenSubtitles v2024 ast da +OpenSubtitles v2024 ast de +OpenSubtitles v2024 ast el +OpenSubtitles v2024 ast en +OpenSubtitles v2024 ast en_ze +OpenSubtitles v2024 ast eo +OpenSubtitles v2024 ast es +OpenSubtitles v2024 ast es_149 +OpenSubtitles v2024 ast es_419 +OpenSubtitles v2024 ast et +OpenSubtitles v2024 ast eu +OpenSubtitles v2024 ast fa +OpenSubtitles v2024 ast fi +OpenSubtitles v2024 ast fr +OpenSubtitles v2024 ast he +OpenSubtitles v2024 ast hi +OpenSubtitles v2024 ast hr +OpenSubtitles v2024 ast hu +OpenSubtitles v2024 ast id +OpenSubtitles v2024 ast is +OpenSubtitles v2024 ast it +OpenSubtitles v2024 ast ja +OpenSubtitles v2024 ast ka +OpenSubtitles v2024 ast ko +OpenSubtitles v2024 ast lt +OpenSubtitles v2024 ast lv +OpenSubtitles v2024 ast mk +OpenSubtitles v2024 ast ml +OpenSubtitles v2024 ast ms +OpenSubtitles v2024 ast my +OpenSubtitles v2024 ast nl +OpenSubtitles v2024 ast no +OpenSubtitles v2024 ast pl +OpenSubtitles v2024 ast pt +OpenSubtitles v2024 ast pt_BR +OpenSubtitles v2024 ast ro +OpenSubtitles v2024 ast ru +OpenSubtitles v2024 ast si +OpenSubtitles v2024 ast sk +OpenSubtitles v2024 ast sl +OpenSubtitles v2024 ast sq +OpenSubtitles v2024 ast sr +OpenSubtitles v2024 ast sv +OpenSubtitles v2024 ast ta +OpenSubtitles v2024 ast th +OpenSubtitles v2024 ast tr +OpenSubtitles v2024 ast uk +OpenSubtitles v2024 ast ur +OpenSubtitles v2024 ast vi +OpenSubtitles v2024 ast zh_CN +OpenSubtitles v2024 ast zh_TW +OpenSubtitles v2024 ast zh_ze +OpenSubtitles v2024 az be +OpenSubtitles v2024 az bg +OpenSubtitles v2024 az bn +OpenSubtitles v2024 az bs +OpenSubtitles v2024 az ca +OpenSubtitles v2024 az cs +OpenSubtitles v2024 az da +OpenSubtitles v2024 az de +OpenSubtitles v2024 az el +OpenSubtitles v2024 az en +OpenSubtitles v2024 az en_ze +OpenSubtitles v2024 az eo +OpenSubtitles v2024 az es +OpenSubtitles v2024 az es_149 +OpenSubtitles v2024 az es_419 +OpenSubtitles v2024 az es_ES +OpenSubtitles v2024 az et +OpenSubtitles v2024 az eu +OpenSubtitles v2024 az fa +OpenSubtitles v2024 az fi +OpenSubtitles v2024 az fr +OpenSubtitles v2024 az gl +OpenSubtitles v2024 az he +OpenSubtitles v2024 az hi +OpenSubtitles v2024 az hr +OpenSubtitles v2024 az hu +OpenSubtitles v2024 az id +OpenSubtitles v2024 az is +OpenSubtitles v2024 az it +OpenSubtitles v2024 az ja +OpenSubtitles v2024 az ka +OpenSubtitles v2024 az kk +OpenSubtitles v2024 az km +OpenSubtitles v2024 az ko +OpenSubtitles v2024 az ku +OpenSubtitles v2024 az lt +OpenSubtitles v2024 az lv +OpenSubtitles v2024 az mk +OpenSubtitles v2024 az ml +OpenSubtitles v2024 az mn +OpenSubtitles v2024 az ms +OpenSubtitles v2024 az my +OpenSubtitles v2024 az nl +OpenSubtitles v2024 az no +OpenSubtitles v2024 az pl +OpenSubtitles v2024 az pt +OpenSubtitles v2024 az pt_BR +OpenSubtitles v2024 az ro +OpenSubtitles v2024 az ru +OpenSubtitles v2024 az sd +OpenSubtitles v2024 az si +OpenSubtitles v2024 az sk +OpenSubtitles v2024 az sl +OpenSubtitles v2024 az sq +OpenSubtitles v2024 az sr +OpenSubtitles v2024 az sv +OpenSubtitles v2024 az ta +OpenSubtitles v2024 az te +OpenSubtitles v2024 az th +OpenSubtitles v2024 az tr +OpenSubtitles v2024 az tt +OpenSubtitles v2024 az uk +OpenSubtitles v2024 az ur +OpenSubtitles v2024 az vi +OpenSubtitles v2024 az yue +OpenSubtitles v2024 az zh_CN +OpenSubtitles v2024 az zh_TW +OpenSubtitles v2024 az zh_ze +OpenSubtitles v2024 be bg +OpenSubtitles v2024 be bn +OpenSubtitles v2024 be bs +OpenSubtitles v2024 be ca +OpenSubtitles v2024 be cs +OpenSubtitles v2024 be cy +OpenSubtitles v2024 be da +OpenSubtitles v2024 be de +OpenSubtitles v2024 be el +OpenSubtitles v2024 be en +OpenSubtitles v2024 be eo +OpenSubtitles v2024 be es +OpenSubtitles v2024 be es_149 +OpenSubtitles v2024 be es_419 +OpenSubtitles v2024 be es_ES +OpenSubtitles v2024 be et +OpenSubtitles v2024 be eu +OpenSubtitles v2024 be fa +OpenSubtitles v2024 be fi +OpenSubtitles v2024 be fr +OpenSubtitles v2024 be gl +OpenSubtitles v2024 be he +OpenSubtitles v2024 be hi +OpenSubtitles v2024 be hr +OpenSubtitles v2024 be hu +OpenSubtitles v2024 be hy +OpenSubtitles v2024 be id +OpenSubtitles v2024 be is +OpenSubtitles v2024 be it +OpenSubtitles v2024 be ja +OpenSubtitles v2024 be ka +OpenSubtitles v2024 be kk +OpenSubtitles v2024 be km +OpenSubtitles v2024 be kn +OpenSubtitles v2024 be ko +OpenSubtitles v2024 be ku +OpenSubtitles v2024 be lb +OpenSubtitles v2024 be lt +OpenSubtitles v2024 be lv +OpenSubtitles v2024 be mk +OpenSubtitles v2024 be ml +OpenSubtitles v2024 be mn +OpenSubtitles v2024 be ms +OpenSubtitles v2024 be my +OpenSubtitles v2024 be nl +OpenSubtitles v2024 be no +OpenSubtitles v2024 be pl +OpenSubtitles v2024 be pt +OpenSubtitles v2024 be pt_BR +OpenSubtitles v2024 be ro +OpenSubtitles v2024 be ru +OpenSubtitles v2024 be si +OpenSubtitles v2024 be sk +OpenSubtitles v2024 be sl +OpenSubtitles v2024 be sq +OpenSubtitles v2024 be sr +OpenSubtitles v2024 be sv +OpenSubtitles v2024 be ta +OpenSubtitles v2024 be te +OpenSubtitles v2024 be th +OpenSubtitles v2024 be tl +OpenSubtitles v2024 be tr +OpenSubtitles v2024 be tt +OpenSubtitles v2024 be uk +OpenSubtitles v2024 be ur +OpenSubtitles v2024 be uz +OpenSubtitles v2024 be vi +OpenSubtitles v2024 be yue +OpenSubtitles v2024 be zh_CN +OpenSubtitles v2024 be zh_TW +OpenSubtitles v2024 be zh_ze +OpenSubtitles v2024 bg bn +OpenSubtitles v2024 bg br +OpenSubtitles v2024 bg bs +OpenSubtitles v2024 bg ca +OpenSubtitles v2024 bg cs +OpenSubtitles v2024 bg cy +OpenSubtitles v2024 bg da +OpenSubtitles v2024 bg de +OpenSubtitles v2024 bg el +OpenSubtitles v2024 bg en +OpenSubtitles v2024 bg en_ze +OpenSubtitles v2024 bg eo +OpenSubtitles v2024 bg es +OpenSubtitles v2024 bg es_149 +OpenSubtitles v2024 bg es_419 +OpenSubtitles v2024 bg es_ES +OpenSubtitles v2024 bg et +OpenSubtitles v2024 bg eu +OpenSubtitles v2024 bg fa +OpenSubtitles v2024 bg fi +OpenSubtitles v2024 bg fr +OpenSubtitles v2024 bg ga +OpenSubtitles v2024 bg gd +OpenSubtitles v2024 bg gl +OpenSubtitles v2024 bg he +OpenSubtitles v2024 bg hi +OpenSubtitles v2024 bg hr +OpenSubtitles v2024 bg hu +OpenSubtitles v2024 bg hy +OpenSubtitles v2024 bg id +OpenSubtitles v2024 bg ig +OpenSubtitles v2024 bg is +OpenSubtitles v2024 bg it +OpenSubtitles v2024 bg ja +OpenSubtitles v2024 bg ka +OpenSubtitles v2024 bg kk +OpenSubtitles v2024 bg km +OpenSubtitles v2024 bg kn +OpenSubtitles v2024 bg ko +OpenSubtitles v2024 bg ku +OpenSubtitles v2024 bg lb +OpenSubtitles v2024 bg lt +OpenSubtitles v2024 bg lv +OpenSubtitles v2024 bg mk +OpenSubtitles v2024 bg ml +OpenSubtitles v2024 bg mn +OpenSubtitles v2024 bg ms +OpenSubtitles v2024 bg my +OpenSubtitles v2024 bg ne +OpenSubtitles v2024 bg nl +OpenSubtitles v2024 bg nn +OpenSubtitles v2024 bg no +OpenSubtitles v2024 bg oc +OpenSubtitles v2024 bg or +OpenSubtitles v2024 bg pl +OpenSubtitles v2024 bg pt +OpenSubtitles v2024 bg pt_BR +OpenSubtitles v2024 bg pt_MZ +OpenSubtitles v2024 bg ro +OpenSubtitles v2024 bg ru +OpenSubtitles v2024 bg sd +OpenSubtitles v2024 bg si +OpenSubtitles v2024 bg sk +OpenSubtitles v2024 bg sl +OpenSubtitles v2024 bg so +OpenSubtitles v2024 bg sq +OpenSubtitles v2024 bg sr +OpenSubtitles v2024 bg sv +OpenSubtitles v2024 bg sw +OpenSubtitles v2024 bg ta +OpenSubtitles v2024 bg te +OpenSubtitles v2024 bg th +OpenSubtitles v2024 bg tl +OpenSubtitles v2024 bg tr +OpenSubtitles v2024 bg tt +OpenSubtitles v2024 bg uk +OpenSubtitles v2024 bg ur +OpenSubtitles v2024 bg vi +OpenSubtitles v2024 bg yue +OpenSubtitles v2024 bg zh_CN +OpenSubtitles v2024 bg zh_TW +OpenSubtitles v2024 bg zh_ze +OpenSubtitles v2024 bn br +OpenSubtitles v2024 bn bs +OpenSubtitles v2024 bn ca +OpenSubtitles v2024 bn cs +OpenSubtitles v2024 bn cy +OpenSubtitles v2024 bn da +OpenSubtitles v2024 bn de +OpenSubtitles v2024 bn el +OpenSubtitles v2024 bn en +OpenSubtitles v2024 bn en_ze +OpenSubtitles v2024 bn eo +OpenSubtitles v2024 bn es +OpenSubtitles v2024 bn es_149 +OpenSubtitles v2024 bn es_419 +OpenSubtitles v2024 bn es_ES +OpenSubtitles v2024 bn et +OpenSubtitles v2024 bn eu +OpenSubtitles v2024 bn fa +OpenSubtitles v2024 bn fi +OpenSubtitles v2024 bn fr +OpenSubtitles v2024 bn ga +OpenSubtitles v2024 bn gd +OpenSubtitles v2024 bn gl +OpenSubtitles v2024 bn he +OpenSubtitles v2024 bn hi +OpenSubtitles v2024 bn hr +OpenSubtitles v2024 bn hu +OpenSubtitles v2024 bn hy +OpenSubtitles v2024 bn id +OpenSubtitles v2024 bn ig +OpenSubtitles v2024 bn is +OpenSubtitles v2024 bn it +OpenSubtitles v2024 bn ja +OpenSubtitles v2024 bn ka +OpenSubtitles v2024 bn kk +OpenSubtitles v2024 bn km +OpenSubtitles v2024 bn kn +OpenSubtitles v2024 bn ko +OpenSubtitles v2024 bn ku +OpenSubtitles v2024 bn lt +OpenSubtitles v2024 bn lv +OpenSubtitles v2024 bn mk +OpenSubtitles v2024 bn ml +OpenSubtitles v2024 bn mn +OpenSubtitles v2024 bn ms +OpenSubtitles v2024 bn my +OpenSubtitles v2024 bn ne +OpenSubtitles v2024 bn nl +OpenSubtitles v2024 bn no +OpenSubtitles v2024 bn oc +OpenSubtitles v2024 bn or +OpenSubtitles v2024 bn pl +OpenSubtitles v2024 bn pt +OpenSubtitles v2024 bn pt_BR +OpenSubtitles v2024 bn pt_MZ +OpenSubtitles v2024 bn ro +OpenSubtitles v2024 bn ru +OpenSubtitles v2024 bn sd +OpenSubtitles v2024 bn si +OpenSubtitles v2024 bn sk +OpenSubtitles v2024 bn sl +OpenSubtitles v2024 bn so +OpenSubtitles v2024 bn sq +OpenSubtitles v2024 bn sr +OpenSubtitles v2024 bn sv +OpenSubtitles v2024 bn sw +OpenSubtitles v2024 bn ta +OpenSubtitles v2024 bn te +OpenSubtitles v2024 bn th +OpenSubtitles v2024 bn tl +OpenSubtitles v2024 bn tr +OpenSubtitles v2024 bn tt +OpenSubtitles v2024 bn uk +OpenSubtitles v2024 bn ur +OpenSubtitles v2024 bn uz +OpenSubtitles v2024 bn vi +OpenSubtitles v2024 bn yue +OpenSubtitles v2024 bn zh_CN +OpenSubtitles v2024 bn zh_TW +OpenSubtitles v2024 bn zh_ze +OpenSubtitles v2024 br bs +OpenSubtitles v2024 br cs +OpenSubtitles v2024 br da +OpenSubtitles v2024 br de +OpenSubtitles v2024 br el +OpenSubtitles v2024 br en +OpenSubtitles v2024 br eo +OpenSubtitles v2024 br es +OpenSubtitles v2024 br es_149 +OpenSubtitles v2024 br es_419 +OpenSubtitles v2024 br et +OpenSubtitles v2024 br eu +OpenSubtitles v2024 br fa +OpenSubtitles v2024 br fi +OpenSubtitles v2024 br fr +OpenSubtitles v2024 br gl +OpenSubtitles v2024 br he +OpenSubtitles v2024 br hr +OpenSubtitles v2024 br hu +OpenSubtitles v2024 br id +OpenSubtitles v2024 br is +OpenSubtitles v2024 br it +OpenSubtitles v2024 br mk +OpenSubtitles v2024 br ml +OpenSubtitles v2024 br nl +OpenSubtitles v2024 br no +OpenSubtitles v2024 br pl +OpenSubtitles v2024 br pt +OpenSubtitles v2024 br pt_BR +OpenSubtitles v2024 br ro +OpenSubtitles v2024 br ru +OpenSubtitles v2024 br sk +OpenSubtitles v2024 br sl +OpenSubtitles v2024 br sr +OpenSubtitles v2024 br sv +OpenSubtitles v2024 br tr +OpenSubtitles v2024 br uk +OpenSubtitles v2024 br vi +OpenSubtitles v2024 br zh_CN +OpenSubtitles v2024 br zh_TW +OpenSubtitles v2024 bs ca +OpenSubtitles v2024 bs cs +OpenSubtitles v2024 bs cy +OpenSubtitles v2024 bs da +OpenSubtitles v2024 bs de +OpenSubtitles v2024 bs el +OpenSubtitles v2024 bs en +OpenSubtitles v2024 bs en_ze +OpenSubtitles v2024 bs eo +OpenSubtitles v2024 bs es +OpenSubtitles v2024 bs es_149 +OpenSubtitles v2024 bs es_419 +OpenSubtitles v2024 bs es_ES +OpenSubtitles v2024 bs et +OpenSubtitles v2024 bs eu +OpenSubtitles v2024 bs fa +OpenSubtitles v2024 bs fi +OpenSubtitles v2024 bs fr +OpenSubtitles v2024 bs ga +OpenSubtitles v2024 bs gd +OpenSubtitles v2024 bs gl +OpenSubtitles v2024 bs he +OpenSubtitles v2024 bs hi +OpenSubtitles v2024 bs hr +OpenSubtitles v2024 bs hu +OpenSubtitles v2024 bs hy +OpenSubtitles v2024 bs id +OpenSubtitles v2024 bs is +OpenSubtitles v2024 bs it +OpenSubtitles v2024 bs ja +OpenSubtitles v2024 bs ka +OpenSubtitles v2024 bs kk +OpenSubtitles v2024 bs km +OpenSubtitles v2024 bs kn +OpenSubtitles v2024 bs ko +OpenSubtitles v2024 bs ku +OpenSubtitles v2024 bs lb +OpenSubtitles v2024 bs lt +OpenSubtitles v2024 bs lv +OpenSubtitles v2024 bs mk +OpenSubtitles v2024 bs ml +OpenSubtitles v2024 bs mn +OpenSubtitles v2024 bs ms +OpenSubtitles v2024 bs my +OpenSubtitles v2024 bs ne +OpenSubtitles v2024 bs nl +OpenSubtitles v2024 bs nn +OpenSubtitles v2024 bs no +OpenSubtitles v2024 bs oc +OpenSubtitles v2024 bs pl +OpenSubtitles v2024 bs pt +OpenSubtitles v2024 bs pt_BR +OpenSubtitles v2024 bs pt_MZ +OpenSubtitles v2024 bs ro +OpenSubtitles v2024 bs ru +OpenSubtitles v2024 bs sd +OpenSubtitles v2024 bs si +OpenSubtitles v2024 bs sk +OpenSubtitles v2024 bs sl +OpenSubtitles v2024 bs so +OpenSubtitles v2024 bs sq +OpenSubtitles v2024 bs sr +OpenSubtitles v2024 bs sv +OpenSubtitles v2024 bs sw +OpenSubtitles v2024 bs ta +OpenSubtitles v2024 bs te +OpenSubtitles v2024 bs th +OpenSubtitles v2024 bs tl +OpenSubtitles v2024 bs tr +OpenSubtitles v2024 bs tt +OpenSubtitles v2024 bs uk +OpenSubtitles v2024 bs ur +OpenSubtitles v2024 bs vi +OpenSubtitles v2024 bs yue +OpenSubtitles v2024 bs zh_CN +OpenSubtitles v2024 bs zh_TW +OpenSubtitles v2024 bs zh_ze +OpenSubtitles v2024 ca cs +OpenSubtitles v2024 ca cy +OpenSubtitles v2024 ca da +OpenSubtitles v2024 ca de +OpenSubtitles v2024 ca el +OpenSubtitles v2024 ca en +OpenSubtitles v2024 ca en_ze +OpenSubtitles v2024 ca eo +OpenSubtitles v2024 ca es +OpenSubtitles v2024 ca es_149 +OpenSubtitles v2024 ca es_419 +OpenSubtitles v2024 ca es_ES +OpenSubtitles v2024 ca et +OpenSubtitles v2024 ca eu +OpenSubtitles v2024 ca fa +OpenSubtitles v2024 ca fi +OpenSubtitles v2024 ca fr +OpenSubtitles v2024 ca gl +OpenSubtitles v2024 ca he +OpenSubtitles v2024 ca hi +OpenSubtitles v2024 ca hr +OpenSubtitles v2024 ca hu +OpenSubtitles v2024 ca hy +OpenSubtitles v2024 ca id +OpenSubtitles v2024 ca is +OpenSubtitles v2024 ca it +OpenSubtitles v2024 ca ja +OpenSubtitles v2024 ca ka +OpenSubtitles v2024 ca kk +OpenSubtitles v2024 ca km +OpenSubtitles v2024 ca kn +OpenSubtitles v2024 ca ko +OpenSubtitles v2024 ca ku +OpenSubtitles v2024 ca lt +OpenSubtitles v2024 ca lv +OpenSubtitles v2024 ca mk +OpenSubtitles v2024 ca ml +OpenSubtitles v2024 ca mn +OpenSubtitles v2024 ca ms +OpenSubtitles v2024 ca my +OpenSubtitles v2024 ca nl +OpenSubtitles v2024 ca no +OpenSubtitles v2024 ca oc +OpenSubtitles v2024 ca pl +OpenSubtitles v2024 ca pt +OpenSubtitles v2024 ca pt_BR +OpenSubtitles v2024 ca pt_MZ +OpenSubtitles v2024 ca ro +OpenSubtitles v2024 ca ru +OpenSubtitles v2024 ca sd +OpenSubtitles v2024 ca si +OpenSubtitles v2024 ca sk +OpenSubtitles v2024 ca sl +OpenSubtitles v2024 ca sq +OpenSubtitles v2024 ca sr +OpenSubtitles v2024 ca sv +OpenSubtitles v2024 ca sw +OpenSubtitles v2024 ca ta +OpenSubtitles v2024 ca te +OpenSubtitles v2024 ca th +OpenSubtitles v2024 ca tl +OpenSubtitles v2024 ca tr +OpenSubtitles v2024 ca tt +OpenSubtitles v2024 ca uk +OpenSubtitles v2024 ca ur +OpenSubtitles v2024 ca vi +OpenSubtitles v2024 ca yue +OpenSubtitles v2024 ca zh_CN +OpenSubtitles v2024 ca zh_TW +OpenSubtitles v2024 ca zh_ze +OpenSubtitles v2024 cs cy +OpenSubtitles v2024 cs da +OpenSubtitles v2024 cs de +OpenSubtitles v2024 cs el +OpenSubtitles v2024 cs en +OpenSubtitles v2024 cs en_ze +OpenSubtitles v2024 cs eo +OpenSubtitles v2024 cs es +OpenSubtitles v2024 cs es_149 +OpenSubtitles v2024 cs es_419 +OpenSubtitles v2024 cs es_ES +OpenSubtitles v2024 cs et +OpenSubtitles v2024 cs eu +OpenSubtitles v2024 cs fa +OpenSubtitles v2024 cs fi +OpenSubtitles v2024 cs fr +OpenSubtitles v2024 cs ga +OpenSubtitles v2024 cs gd +OpenSubtitles v2024 cs gl +OpenSubtitles v2024 cs he +OpenSubtitles v2024 cs hi +OpenSubtitles v2024 cs hr +OpenSubtitles v2024 cs hu +OpenSubtitles v2024 cs hy +OpenSubtitles v2024 cs id +OpenSubtitles v2024 cs ig +OpenSubtitles v2024 cs is +OpenSubtitles v2024 cs it +OpenSubtitles v2024 cs ja +OpenSubtitles v2024 cs ka +OpenSubtitles v2024 cs kk +OpenSubtitles v2024 cs km +OpenSubtitles v2024 cs kn +OpenSubtitles v2024 cs ko +OpenSubtitles v2024 cs ku +OpenSubtitles v2024 cs lb +OpenSubtitles v2024 cs lt +OpenSubtitles v2024 cs lv +OpenSubtitles v2024 cs mk +OpenSubtitles v2024 cs ml +OpenSubtitles v2024 cs mn +OpenSubtitles v2024 cs ms +OpenSubtitles v2024 cs my +OpenSubtitles v2024 cs ne +OpenSubtitles v2024 cs nl +OpenSubtitles v2024 cs nn +OpenSubtitles v2024 cs no +OpenSubtitles v2024 cs oc +OpenSubtitles v2024 cs or +OpenSubtitles v2024 cs pl +OpenSubtitles v2024 cs pt +OpenSubtitles v2024 cs pt_BR +OpenSubtitles v2024 cs pt_MZ +OpenSubtitles v2024 cs ro +OpenSubtitles v2024 cs ru +OpenSubtitles v2024 cs sd +OpenSubtitles v2024 cs se +OpenSubtitles v2024 cs si +OpenSubtitles v2024 cs sk +OpenSubtitles v2024 cs sl +OpenSubtitles v2024 cs so +OpenSubtitles v2024 cs sq +OpenSubtitles v2024 cs sr +OpenSubtitles v2024 cs sv +OpenSubtitles v2024 cs sw +OpenSubtitles v2024 cs ta +OpenSubtitles v2024 cs te +OpenSubtitles v2024 cs th +OpenSubtitles v2024 cs tl +OpenSubtitles v2024 cs tr +OpenSubtitles v2024 cs tt +OpenSubtitles v2024 cs uk +OpenSubtitles v2024 cs ur +OpenSubtitles v2024 cs vi +OpenSubtitles v2024 cs yue +OpenSubtitles v2024 cs zh_CN +OpenSubtitles v2024 cs zh_TW +OpenSubtitles v2024 cs zh_ze +OpenSubtitles v2024 cy da +OpenSubtitles v2024 cy de +OpenSubtitles v2024 cy el +OpenSubtitles v2024 cy en +OpenSubtitles v2024 cy eo +OpenSubtitles v2024 cy es +OpenSubtitles v2024 cy es_149 +OpenSubtitles v2024 cy es_419 +OpenSubtitles v2024 cy et +OpenSubtitles v2024 cy eu +OpenSubtitles v2024 cy fa +OpenSubtitles v2024 cy fi +OpenSubtitles v2024 cy fr +OpenSubtitles v2024 cy ga +OpenSubtitles v2024 cy gd +OpenSubtitles v2024 cy he +OpenSubtitles v2024 cy hr +OpenSubtitles v2024 cy hu +OpenSubtitles v2024 cy id +OpenSubtitles v2024 cy is +OpenSubtitles v2024 cy it +OpenSubtitles v2024 cy ja +OpenSubtitles v2024 cy ko +OpenSubtitles v2024 cy lb +OpenSubtitles v2024 cy lt +OpenSubtitles v2024 cy lv +OpenSubtitles v2024 cy mk +OpenSubtitles v2024 cy ml +OpenSubtitles v2024 cy ms +OpenSubtitles v2024 cy my +OpenSubtitles v2024 cy nl +OpenSubtitles v2024 cy no +OpenSubtitles v2024 cy pl +OpenSubtitles v2024 cy pt +OpenSubtitles v2024 cy pt_BR +OpenSubtitles v2024 cy ro +OpenSubtitles v2024 cy ru +OpenSubtitles v2024 cy si +OpenSubtitles v2024 cy sk +OpenSubtitles v2024 cy sl +OpenSubtitles v2024 cy sq +OpenSubtitles v2024 cy sr +OpenSubtitles v2024 cy sv +OpenSubtitles v2024 cy th +OpenSubtitles v2024 cy tr +OpenSubtitles v2024 cy uk +OpenSubtitles v2024 cy ur +OpenSubtitles v2024 cy vi +OpenSubtitles v2024 cy zh_CN +OpenSubtitles v2024 cy zh_TW +OpenSubtitles v2024 cy zh_ze +OpenSubtitles v2024 da de +OpenSubtitles v2024 da el +OpenSubtitles v2024 da en +OpenSubtitles v2024 da en_ze +OpenSubtitles v2024 da eo +OpenSubtitles v2024 da es +OpenSubtitles v2024 da es_149 +OpenSubtitles v2024 da es_419 +OpenSubtitles v2024 da es_ES +OpenSubtitles v2024 da et +OpenSubtitles v2024 da eu +OpenSubtitles v2024 da fa +OpenSubtitles v2024 da fi +OpenSubtitles v2024 da fr +OpenSubtitles v2024 da ga +OpenSubtitles v2024 da gd +OpenSubtitles v2024 da gl +OpenSubtitles v2024 da he +OpenSubtitles v2024 da hi +OpenSubtitles v2024 da hr +OpenSubtitles v2024 da hu +OpenSubtitles v2024 da hy +OpenSubtitles v2024 da id +OpenSubtitles v2024 da ig +OpenSubtitles v2024 da is +OpenSubtitles v2024 da it +OpenSubtitles v2024 da ja +OpenSubtitles v2024 da ka +OpenSubtitles v2024 da kk +OpenSubtitles v2024 da km +OpenSubtitles v2024 da kn +OpenSubtitles v2024 da ko +OpenSubtitles v2024 da ku +OpenSubtitles v2024 da lb +OpenSubtitles v2024 da lt +OpenSubtitles v2024 da lv +OpenSubtitles v2024 da mk +OpenSubtitles v2024 da ml +OpenSubtitles v2024 da mn +OpenSubtitles v2024 da ms +OpenSubtitles v2024 da my +OpenSubtitles v2024 da ne +OpenSubtitles v2024 da nl +OpenSubtitles v2024 da nn +OpenSubtitles v2024 da no +OpenSubtitles v2024 da oc +OpenSubtitles v2024 da or +OpenSubtitles v2024 da pl +OpenSubtitles v2024 da pt +OpenSubtitles v2024 da pt_BR +OpenSubtitles v2024 da pt_MZ +OpenSubtitles v2024 da ro +OpenSubtitles v2024 da ru +OpenSubtitles v2024 da sd +OpenSubtitles v2024 da se +OpenSubtitles v2024 da si +OpenSubtitles v2024 da sk +OpenSubtitles v2024 da sl +OpenSubtitles v2024 da so +OpenSubtitles v2024 da sq +OpenSubtitles v2024 da sr +OpenSubtitles v2024 da sv +OpenSubtitles v2024 da sw +OpenSubtitles v2024 da ta +OpenSubtitles v2024 da te +OpenSubtitles v2024 da th +OpenSubtitles v2024 da tl +OpenSubtitles v2024 da tr +OpenSubtitles v2024 da tt +OpenSubtitles v2024 da uk +OpenSubtitles v2024 da ur +OpenSubtitles v2024 da vi +OpenSubtitles v2024 da yue +OpenSubtitles v2024 da zh_CN +OpenSubtitles v2024 da zh_TW +OpenSubtitles v2024 da zh_ze +OpenSubtitles v2024 de el +OpenSubtitles v2024 de en +OpenSubtitles v2024 de en_ze +OpenSubtitles v2024 de eo +OpenSubtitles v2024 de es +OpenSubtitles v2024 de es_419 +OpenSubtitles v2024 de es_ES +OpenSubtitles v2024 de et +OpenSubtitles v2024 de eu +OpenSubtitles v2024 de fa +OpenSubtitles v2024 de fi +OpenSubtitles v2024 de fr +OpenSubtitles v2024 de ga +OpenSubtitles v2024 de gd +OpenSubtitles v2024 de gl +OpenSubtitles v2024 de he +OpenSubtitles v2024 de hi +OpenSubtitles v2024 de hr +OpenSubtitles v2024 de hu +OpenSubtitles v2024 de hy +OpenSubtitles v2024 de id +OpenSubtitles v2024 de ig +OpenSubtitles v2024 de is +OpenSubtitles v2024 de it +OpenSubtitles v2024 de ja +OpenSubtitles v2024 de ka +OpenSubtitles v2024 de kk +OpenSubtitles v2024 de km +OpenSubtitles v2024 de kn +OpenSubtitles v2024 de ko +OpenSubtitles v2024 de ku +OpenSubtitles v2024 de lb +OpenSubtitles v2024 de lt +OpenSubtitles v2024 de lv +OpenSubtitles v2024 de mk +OpenSubtitles v2024 de ml +OpenSubtitles v2024 de mn +OpenSubtitles v2024 de ms +OpenSubtitles v2024 de my +OpenSubtitles v2024 de ne +OpenSubtitles v2024 de nl +OpenSubtitles v2024 de nn +OpenSubtitles v2024 de no +OpenSubtitles v2024 de oc +OpenSubtitles v2024 de or +OpenSubtitles v2024 de pl +OpenSubtitles v2024 de pt +OpenSubtitles v2024 de pt_BR +OpenSubtitles v2024 de pt_MZ +OpenSubtitles v2024 de ro +OpenSubtitles v2024 de ru +OpenSubtitles v2024 de sd +OpenSubtitles v2024 de si +OpenSubtitles v2024 de sk +OpenSubtitles v2024 de sl +OpenSubtitles v2024 de so +OpenSubtitles v2024 de sq +OpenSubtitles v2024 de sr +OpenSubtitles v2024 de sv +OpenSubtitles v2024 de sw +OpenSubtitles v2024 de ta +OpenSubtitles v2024 de te +OpenSubtitles v2024 de th +OpenSubtitles v2024 de tl +OpenSubtitles v2024 de tr +OpenSubtitles v2024 de tt +OpenSubtitles v2024 de uk +OpenSubtitles v2024 de ur +OpenSubtitles v2024 de vi +OpenSubtitles v2024 de yue +OpenSubtitles v2024 de zh_CN +OpenSubtitles v2024 de zh_TW +OpenSubtitles v2024 de zh_ze +OpenSubtitles v2024 el en +OpenSubtitles v2024 el en_ze +OpenSubtitles v2024 el eo +OpenSubtitles v2024 el es +OpenSubtitles v2024 el es_419 +OpenSubtitles v2024 el es_ES +OpenSubtitles v2024 el et +OpenSubtitles v2024 el eu +OpenSubtitles v2024 el fa +OpenSubtitles v2024 el fi +OpenSubtitles v2024 el fr +OpenSubtitles v2024 el ga +OpenSubtitles v2024 el gd +OpenSubtitles v2024 el gl +OpenSubtitles v2024 el he +OpenSubtitles v2024 el hi +OpenSubtitles v2024 el hr +OpenSubtitles v2024 el hu +OpenSubtitles v2024 el hy +OpenSubtitles v2024 el id +OpenSubtitles v2024 el ig +OpenSubtitles v2024 el is +OpenSubtitles v2024 el it +OpenSubtitles v2024 el ja +OpenSubtitles v2024 el ka +OpenSubtitles v2024 el kk +OpenSubtitles v2024 el km +OpenSubtitles v2024 el kn +OpenSubtitles v2024 el ko +OpenSubtitles v2024 el ku +OpenSubtitles v2024 el lb +OpenSubtitles v2024 el lt +OpenSubtitles v2024 el lv +OpenSubtitles v2024 el mk +OpenSubtitles v2024 el ml +OpenSubtitles v2024 el mn +OpenSubtitles v2024 el ms +OpenSubtitles v2024 el my +OpenSubtitles v2024 el ne +OpenSubtitles v2024 el nl +OpenSubtitles v2024 el nn +OpenSubtitles v2024 el no +OpenSubtitles v2024 el oc +OpenSubtitles v2024 el or +OpenSubtitles v2024 el pl +OpenSubtitles v2024 el pt +OpenSubtitles v2024 el pt_BR +OpenSubtitles v2024 el pt_MZ +OpenSubtitles v2024 el ro +OpenSubtitles v2024 el ru +OpenSubtitles v2024 el sd +OpenSubtitles v2024 el si +OpenSubtitles v2024 el sk +OpenSubtitles v2024 el sl +OpenSubtitles v2024 el so +OpenSubtitles v2024 el sq +OpenSubtitles v2024 el sr +OpenSubtitles v2024 el sv +OpenSubtitles v2024 el sw +OpenSubtitles v2024 el ta +OpenSubtitles v2024 el te +OpenSubtitles v2024 el th +OpenSubtitles v2024 el tl +OpenSubtitles v2024 el tr +OpenSubtitles v2024 el tt +OpenSubtitles v2024 el uk +OpenSubtitles v2024 el ur +OpenSubtitles v2024 el vi +OpenSubtitles v2024 el yue +OpenSubtitles v2024 el zh_CN +OpenSubtitles v2024 el zh_TW +OpenSubtitles v2024 el zh_ze +OpenSubtitles v2024 en en_ze +OpenSubtitles v2024 en eo +OpenSubtitles v2024 en es +OpenSubtitles v2024 en es_419 +OpenSubtitles v2024 en es_ES +OpenSubtitles v2024 en et +OpenSubtitles v2024 en eu +OpenSubtitles v2024 en fa +OpenSubtitles v2024 en fi +OpenSubtitles v2024 en fr +OpenSubtitles v2024 en ga +OpenSubtitles v2024 en gd +OpenSubtitles v2024 en gl +OpenSubtitles v2024 en he +OpenSubtitles v2024 en hi +OpenSubtitles v2024 en hr +OpenSubtitles v2024 en hu +OpenSubtitles v2024 en hy +OpenSubtitles v2024 en id +OpenSubtitles v2024 en ig +OpenSubtitles v2024 en is +OpenSubtitles v2024 en it +OpenSubtitles v2024 en ja +OpenSubtitles v2024 en ka +OpenSubtitles v2024 en kk +OpenSubtitles v2024 en km +OpenSubtitles v2024 en kn +OpenSubtitles v2024 en ko +OpenSubtitles v2024 en ku +OpenSubtitles v2024 en lb +OpenSubtitles v2024 en lt +OpenSubtitles v2024 en lv +OpenSubtitles v2024 en mk +OpenSubtitles v2024 en ml +OpenSubtitles v2024 en mn +OpenSubtitles v2024 en mr +OpenSubtitles v2024 en ms +OpenSubtitles v2024 en my +OpenSubtitles v2024 en ne +OpenSubtitles v2024 en nl +OpenSubtitles v2024 en nn +OpenSubtitles v2024 en no +OpenSubtitles v2024 en oc +OpenSubtitles v2024 en or +OpenSubtitles v2024 en pl +OpenSubtitles v2024 en pt +OpenSubtitles v2024 en pt_BR +OpenSubtitles v2024 en pt_MZ +OpenSubtitles v2024 en ro +OpenSubtitles v2024 en ru +OpenSubtitles v2024 en sd +OpenSubtitles v2024 en se +OpenSubtitles v2024 en si +OpenSubtitles v2024 en sk +OpenSubtitles v2024 en sl +OpenSubtitles v2024 en so +OpenSubtitles v2024 en sq +OpenSubtitles v2024 en sr +OpenSubtitles v2024 en sv +OpenSubtitles v2024 en sw +OpenSubtitles v2024 en ta +OpenSubtitles v2024 en te +OpenSubtitles v2024 en th +OpenSubtitles v2024 en tl +OpenSubtitles v2024 en tr +OpenSubtitles v2024 en tt +OpenSubtitles v2024 en uk +OpenSubtitles v2024 en ur +OpenSubtitles v2024 en vi +OpenSubtitles v2024 en yue +OpenSubtitles v2024 en zh_CN +OpenSubtitles v2024 en zh_TW +OpenSubtitles v2024 en zh_ze +OpenSubtitles v2024 en_ze es +OpenSubtitles v2024 en_ze es_419 +OpenSubtitles v2024 en_ze es_ES +OpenSubtitles v2024 en_ze et +OpenSubtitles v2024 en_ze eu +OpenSubtitles v2024 en_ze fa +OpenSubtitles v2024 en_ze fi +OpenSubtitles v2024 en_ze fr +OpenSubtitles v2024 en_ze gl +OpenSubtitles v2024 en_ze he +OpenSubtitles v2024 en_ze hi +OpenSubtitles v2024 en_ze hr +OpenSubtitles v2024 en_ze hu +OpenSubtitles v2024 en_ze id +OpenSubtitles v2024 en_ze is +OpenSubtitles v2024 en_ze it +OpenSubtitles v2024 en_ze ja +OpenSubtitles v2024 en_ze km +OpenSubtitles v2024 en_ze ko +OpenSubtitles v2024 en_ze ku +OpenSubtitles v2024 en_ze lt +OpenSubtitles v2024 en_ze lv +OpenSubtitles v2024 en_ze mk +OpenSubtitles v2024 en_ze ml +OpenSubtitles v2024 en_ze mn +OpenSubtitles v2024 en_ze ms +OpenSubtitles v2024 en_ze my +OpenSubtitles v2024 en_ze nl +OpenSubtitles v2024 en_ze no +OpenSubtitles v2024 en_ze pl +OpenSubtitles v2024 en_ze pt +OpenSubtitles v2024 en_ze pt_BR +OpenSubtitles v2024 en_ze ro +OpenSubtitles v2024 en_ze ru +OpenSubtitles v2024 en_ze si +OpenSubtitles v2024 en_ze sk +OpenSubtitles v2024 en_ze sl +OpenSubtitles v2024 en_ze sq +OpenSubtitles v2024 en_ze sr +OpenSubtitles v2024 en_ze sv +OpenSubtitles v2024 en_ze ta +OpenSubtitles v2024 en_ze th +OpenSubtitles v2024 en_ze tl +OpenSubtitles v2024 en_ze tr +OpenSubtitles v2024 en_ze tt +OpenSubtitles v2024 en_ze uk +OpenSubtitles v2024 en_ze ur +OpenSubtitles v2024 en_ze vi +OpenSubtitles v2024 en_ze yue +OpenSubtitles v2024 en_ze zh_CN +OpenSubtitles v2024 en_ze zh_TW +OpenSubtitles v2024 en_ze zh_ze +OpenSubtitles v2024 eo es +OpenSubtitles v2024 eo es_419 +OpenSubtitles v2024 eo es_ES +OpenSubtitles v2024 eo et +OpenSubtitles v2024 eo eu +OpenSubtitles v2024 eo fa +OpenSubtitles v2024 eo fi +OpenSubtitles v2024 eo fr +OpenSubtitles v2024 eo ga +OpenSubtitles v2024 eo gd +OpenSubtitles v2024 eo gl +OpenSubtitles v2024 eo he +OpenSubtitles v2024 eo hi +OpenSubtitles v2024 eo hr +OpenSubtitles v2024 eo hu +OpenSubtitles v2024 eo hy +OpenSubtitles v2024 eo id +OpenSubtitles v2024 eo is +OpenSubtitles v2024 eo it +OpenSubtitles v2024 eo ja +OpenSubtitles v2024 eo ka +OpenSubtitles v2024 eo kk +OpenSubtitles v2024 eo ko +OpenSubtitles v2024 eo ku +OpenSubtitles v2024 eo lb +OpenSubtitles v2024 eo lt +OpenSubtitles v2024 eo lv +OpenSubtitles v2024 eo mk +OpenSubtitles v2024 eo ml +OpenSubtitles v2024 eo mn +OpenSubtitles v2024 eo ms +OpenSubtitles v2024 eo my +OpenSubtitles v2024 eo nl +OpenSubtitles v2024 eo no +OpenSubtitles v2024 eo pl +OpenSubtitles v2024 eo pt +OpenSubtitles v2024 eo pt_BR +OpenSubtitles v2024 eo ro +OpenSubtitles v2024 eo ru +OpenSubtitles v2024 eo sd +OpenSubtitles v2024 eo si +OpenSubtitles v2024 eo sk +OpenSubtitles v2024 eo sl +OpenSubtitles v2024 eo sq +OpenSubtitles v2024 eo sr +OpenSubtitles v2024 eo sv +OpenSubtitles v2024 eo te +OpenSubtitles v2024 eo th +OpenSubtitles v2024 eo tl +OpenSubtitles v2024 eo tr +OpenSubtitles v2024 eo tt +OpenSubtitles v2024 eo uk +OpenSubtitles v2024 eo ur +OpenSubtitles v2024 eo vi +OpenSubtitles v2024 eo zh_CN +OpenSubtitles v2024 eo zh_TW +OpenSubtitles v2024 es es_419 +OpenSubtitles v2024 es es_ES +OpenSubtitles v2024 es et +OpenSubtitles v2024 es eu +OpenSubtitles v2024 es fa +OpenSubtitles v2024 es fi +OpenSubtitles v2024 es fr +OpenSubtitles v2024 es ga +OpenSubtitles v2024 es gd +OpenSubtitles v2024 es gl +OpenSubtitles v2024 es he +OpenSubtitles v2024 es hi +OpenSubtitles v2024 es hr +OpenSubtitles v2024 es hu +OpenSubtitles v2024 es hy +OpenSubtitles v2024 es id +OpenSubtitles v2024 es ig +OpenSubtitles v2024 es is +OpenSubtitles v2024 es it +OpenSubtitles v2024 es ja +OpenSubtitles v2024 es ka +OpenSubtitles v2024 es kk +OpenSubtitles v2024 es km +OpenSubtitles v2024 es kn +OpenSubtitles v2024 es ko +OpenSubtitles v2024 es ku +OpenSubtitles v2024 es lb +OpenSubtitles v2024 es lt +OpenSubtitles v2024 es lv +OpenSubtitles v2024 es mk +OpenSubtitles v2024 es ml +OpenSubtitles v2024 es mn +OpenSubtitles v2024 es ms +OpenSubtitles v2024 es my +OpenSubtitles v2024 es ne +OpenSubtitles v2024 es nl +OpenSubtitles v2024 es nn +OpenSubtitles v2024 es no +OpenSubtitles v2024 es oc +OpenSubtitles v2024 es or +OpenSubtitles v2024 es pl +OpenSubtitles v2024 es pt +OpenSubtitles v2024 es pt_BR +OpenSubtitles v2024 es pt_MZ +OpenSubtitles v2024 es ro +OpenSubtitles v2024 es ru +OpenSubtitles v2024 es sd +OpenSubtitles v2024 es si +OpenSubtitles v2024 es sk +OpenSubtitles v2024 es sl +OpenSubtitles v2024 es sq +OpenSubtitles v2024 es sr +OpenSubtitles v2024 es sv +OpenSubtitles v2024 es sw +OpenSubtitles v2024 es ta +OpenSubtitles v2024 es te +OpenSubtitles v2024 es th +OpenSubtitles v2024 es tl +OpenSubtitles v2024 es tr +OpenSubtitles v2024 es tt +OpenSubtitles v2024 es uk +OpenSubtitles v2024 es ur +OpenSubtitles v2024 es vi +OpenSubtitles v2024 es yue +OpenSubtitles v2024 es zh_CN +OpenSubtitles v2024 es zh_TW +OpenSubtitles v2024 es zh_ze +OpenSubtitles v2024 es_419 es_ES +OpenSubtitles v2024 es_419 et +OpenSubtitles v2024 es_419 eu +OpenSubtitles v2024 es_419 fa +OpenSubtitles v2024 es_419 fi +OpenSubtitles v2024 es_419 fr +OpenSubtitles v2024 es_419 ga +OpenSubtitles v2024 es_419 gd +OpenSubtitles v2024 es_419 gl +OpenSubtitles v2024 es_419 he +OpenSubtitles v2024 es_419 hi +OpenSubtitles v2024 es_419 hr +OpenSubtitles v2024 es_419 hu +OpenSubtitles v2024 es_419 id +OpenSubtitles v2024 es_419 ig +OpenSubtitles v2024 es_419 is +OpenSubtitles v2024 es_419 it +OpenSubtitles v2024 es_419 ja +OpenSubtitles v2024 es_419 ka +OpenSubtitles v2024 es_419 kk +OpenSubtitles v2024 es_419 km +OpenSubtitles v2024 es_419 kn +OpenSubtitles v2024 es_419 ko +OpenSubtitles v2024 es_419 ku +OpenSubtitles v2024 es_419 lb +OpenSubtitles v2024 es_419 lt +OpenSubtitles v2024 es_419 lv +OpenSubtitles v2024 es_419 mk +OpenSubtitles v2024 es_419 ml +OpenSubtitles v2024 es_419 mn +OpenSubtitles v2024 es_419 ms +OpenSubtitles v2024 es_419 my +OpenSubtitles v2024 es_419 ne +OpenSubtitles v2024 es_419 nl +OpenSubtitles v2024 es_419 nn +OpenSubtitles v2024 es_419 no +OpenSubtitles v2024 es_419 oc +OpenSubtitles v2024 es_419 pl +OpenSubtitles v2024 es_419 pt +OpenSubtitles v2024 es_419 pt_BR +OpenSubtitles v2024 es_419 pt_MZ +OpenSubtitles v2024 es_419 ro +OpenSubtitles v2024 es_419 ru +OpenSubtitles v2024 es_419 sd +OpenSubtitles v2024 es_419 si +OpenSubtitles v2024 es_419 sk +OpenSubtitles v2024 es_419 sl +OpenSubtitles v2024 es_419 sq +OpenSubtitles v2024 es_419 sr +OpenSubtitles v2024 es_419 sv +OpenSubtitles v2024 es_419 sw +OpenSubtitles v2024 es_419 ta +OpenSubtitles v2024 es_419 te +OpenSubtitles v2024 es_419 th +OpenSubtitles v2024 es_419 tl +OpenSubtitles v2024 es_419 tr +OpenSubtitles v2024 es_419 tt +OpenSubtitles v2024 es_419 uk +OpenSubtitles v2024 es_419 ur +OpenSubtitles v2024 es_419 vi +OpenSubtitles v2024 es_419 yue +OpenSubtitles v2024 es_419 zh_CN +OpenSubtitles v2024 es_419 zh_TW +OpenSubtitles v2024 es_419 zh_ze +OpenSubtitles v2024 es_ES et +OpenSubtitles v2024 es_ES eu +OpenSubtitles v2024 es_ES fa +OpenSubtitles v2024 es_ES fi +OpenSubtitles v2024 es_ES fr +OpenSubtitles v2024 es_ES ga +OpenSubtitles v2024 es_ES gl +OpenSubtitles v2024 es_ES he +OpenSubtitles v2024 es_ES hi +OpenSubtitles v2024 es_ES hr +OpenSubtitles v2024 es_ES hu +OpenSubtitles v2024 es_ES id +OpenSubtitles v2024 es_ES is +OpenSubtitles v2024 es_ES it +OpenSubtitles v2024 es_ES ja +OpenSubtitles v2024 es_ES ka +OpenSubtitles v2024 es_ES kk +OpenSubtitles v2024 es_ES km +OpenSubtitles v2024 es_ES ko +OpenSubtitles v2024 es_ES ku +OpenSubtitles v2024 es_ES lb +OpenSubtitles v2024 es_ES lt +OpenSubtitles v2024 es_ES lv +OpenSubtitles v2024 es_ES mk +OpenSubtitles v2024 es_ES ml +OpenSubtitles v2024 es_ES mn +OpenSubtitles v2024 es_ES ms +OpenSubtitles v2024 es_ES my +OpenSubtitles v2024 es_ES nl +OpenSubtitles v2024 es_ES nn +OpenSubtitles v2024 es_ES no +OpenSubtitles v2024 es_ES oc +OpenSubtitles v2024 es_ES pl +OpenSubtitles v2024 es_ES pt +OpenSubtitles v2024 es_ES pt_BR +OpenSubtitles v2024 es_ES ro +OpenSubtitles v2024 es_ES ru +OpenSubtitles v2024 es_ES sd +OpenSubtitles v2024 es_ES si +OpenSubtitles v2024 es_ES sk +OpenSubtitles v2024 es_ES sl +OpenSubtitles v2024 es_ES sq +OpenSubtitles v2024 es_ES sr +OpenSubtitles v2024 es_ES sv +OpenSubtitles v2024 es_ES ta +OpenSubtitles v2024 es_ES te +OpenSubtitles v2024 es_ES th +OpenSubtitles v2024 es_ES tl +OpenSubtitles v2024 es_ES tr +OpenSubtitles v2024 es_ES tt +OpenSubtitles v2024 es_ES uk +OpenSubtitles v2024 es_ES ur +OpenSubtitles v2024 es_ES vi +OpenSubtitles v2024 es_ES yue +OpenSubtitles v2024 es_ES zh_CN +OpenSubtitles v2024 es_ES zh_TW +OpenSubtitles v2024 es_ES zh_ze +OpenSubtitles v2024 et eu +OpenSubtitles v2024 et fa +OpenSubtitles v2024 et fi +OpenSubtitles v2024 et fr +OpenSubtitles v2024 et ga +OpenSubtitles v2024 et gd +OpenSubtitles v2024 et gl +OpenSubtitles v2024 et he +OpenSubtitles v2024 et hi +OpenSubtitles v2024 et hr +OpenSubtitles v2024 et hu +OpenSubtitles v2024 et hy +OpenSubtitles v2024 et id +OpenSubtitles v2024 et ig +OpenSubtitles v2024 et is +OpenSubtitles v2024 et it +OpenSubtitles v2024 et ja +OpenSubtitles v2024 et ka +OpenSubtitles v2024 et kk +OpenSubtitles v2024 et km +OpenSubtitles v2024 et kn +OpenSubtitles v2024 et ko +OpenSubtitles v2024 et ku +OpenSubtitles v2024 et lb +OpenSubtitles v2024 et lt +OpenSubtitles v2024 et lv +OpenSubtitles v2024 et mk +OpenSubtitles v2024 et ml +OpenSubtitles v2024 et mn +OpenSubtitles v2024 et ms +OpenSubtitles v2024 et my +OpenSubtitles v2024 et ne +OpenSubtitles v2024 et nl +OpenSubtitles v2024 et nn +OpenSubtitles v2024 et no +OpenSubtitles v2024 et oc +OpenSubtitles v2024 et or +OpenSubtitles v2024 et pl +OpenSubtitles v2024 et pt +OpenSubtitles v2024 et pt_BR +OpenSubtitles v2024 et pt_MZ +OpenSubtitles v2024 et ro +OpenSubtitles v2024 et ru +OpenSubtitles v2024 et sd +OpenSubtitles v2024 et si +OpenSubtitles v2024 et sk +OpenSubtitles v2024 et sl +OpenSubtitles v2024 et so +OpenSubtitles v2024 et sq +OpenSubtitles v2024 et sr +OpenSubtitles v2024 et sv +OpenSubtitles v2024 et sw +OpenSubtitles v2024 et ta +OpenSubtitles v2024 et te +OpenSubtitles v2024 et th +OpenSubtitles v2024 et tl +OpenSubtitles v2024 et tr +OpenSubtitles v2024 et tt +OpenSubtitles v2024 et uk +OpenSubtitles v2024 et ur +OpenSubtitles v2024 et vi +OpenSubtitles v2024 et yue +OpenSubtitles v2024 et zh_CN +OpenSubtitles v2024 et zh_TW +OpenSubtitles v2024 et zh_ze +OpenSubtitles v2024 eu fa +OpenSubtitles v2024 eu fi +OpenSubtitles v2024 eu fr +OpenSubtitles v2024 eu gl +OpenSubtitles v2024 eu he +OpenSubtitles v2024 eu hi +OpenSubtitles v2024 eu hr +OpenSubtitles v2024 eu hu +OpenSubtitles v2024 eu hy +OpenSubtitles v2024 eu id +OpenSubtitles v2024 eu is +OpenSubtitles v2024 eu it +OpenSubtitles v2024 eu ja +OpenSubtitles v2024 eu ka +OpenSubtitles v2024 eu kk +OpenSubtitles v2024 eu km +OpenSubtitles v2024 eu kn +OpenSubtitles v2024 eu ko +OpenSubtitles v2024 eu ku +OpenSubtitles v2024 eu lt +OpenSubtitles v2024 eu lv +OpenSubtitles v2024 eu mk +OpenSubtitles v2024 eu ml +OpenSubtitles v2024 eu mn +OpenSubtitles v2024 eu ms +OpenSubtitles v2024 eu my +OpenSubtitles v2024 eu ne +OpenSubtitles v2024 eu nl +OpenSubtitles v2024 eu no +OpenSubtitles v2024 eu oc +OpenSubtitles v2024 eu pl +OpenSubtitles v2024 eu pt +OpenSubtitles v2024 eu pt_BR +OpenSubtitles v2024 eu ro +OpenSubtitles v2024 eu ru +OpenSubtitles v2024 eu sd +OpenSubtitles v2024 eu si +OpenSubtitles v2024 eu sk +OpenSubtitles v2024 eu sl +OpenSubtitles v2024 eu sq +OpenSubtitles v2024 eu sr +OpenSubtitles v2024 eu sv +OpenSubtitles v2024 eu sw +OpenSubtitles v2024 eu ta +OpenSubtitles v2024 eu te +OpenSubtitles v2024 eu th +OpenSubtitles v2024 eu tl +OpenSubtitles v2024 eu tr +OpenSubtitles v2024 eu tt +OpenSubtitles v2024 eu uk +OpenSubtitles v2024 eu ur +OpenSubtitles v2024 eu vi +OpenSubtitles v2024 eu yue +OpenSubtitles v2024 eu zh_CN +OpenSubtitles v2024 eu zh_TW +OpenSubtitles v2024 eu zh_ze +OpenSubtitles v2024 fa fi +OpenSubtitles v2024 fa fr +OpenSubtitles v2024 fa ga +OpenSubtitles v2024 fa gd +OpenSubtitles v2024 fa gl +OpenSubtitles v2024 fa he +OpenSubtitles v2024 fa hi +OpenSubtitles v2024 fa hr +OpenSubtitles v2024 fa hu +OpenSubtitles v2024 fa hy +OpenSubtitles v2024 fa id +OpenSubtitles v2024 fa ig +OpenSubtitles v2024 fa is +OpenSubtitles v2024 fa it +OpenSubtitles v2024 fa ja +OpenSubtitles v2024 fa ka +OpenSubtitles v2024 fa kk +OpenSubtitles v2024 fa km +OpenSubtitles v2024 fa kn +OpenSubtitles v2024 fa ko +OpenSubtitles v2024 fa ku +OpenSubtitles v2024 fa lb +OpenSubtitles v2024 fa lt +OpenSubtitles v2024 fa lv +OpenSubtitles v2024 fa mk +OpenSubtitles v2024 fa ml +OpenSubtitles v2024 fa mn +OpenSubtitles v2024 fa ms +OpenSubtitles v2024 fa my +OpenSubtitles v2024 fa ne +OpenSubtitles v2024 fa nl +OpenSubtitles v2024 fa nn +OpenSubtitles v2024 fa no +OpenSubtitles v2024 fa oc +OpenSubtitles v2024 fa or +OpenSubtitles v2024 fa pl +OpenSubtitles v2024 fa pt +OpenSubtitles v2024 fa pt_BR +OpenSubtitles v2024 fa pt_MZ +OpenSubtitles v2024 fa ro +OpenSubtitles v2024 fa ru +OpenSubtitles v2024 fa sd +OpenSubtitles v2024 fa si +OpenSubtitles v2024 fa sk +OpenSubtitles v2024 fa sl +OpenSubtitles v2024 fa so +OpenSubtitles v2024 fa sq +OpenSubtitles v2024 fa sr +OpenSubtitles v2024 fa sv +OpenSubtitles v2024 fa sw +OpenSubtitles v2024 fa ta +OpenSubtitles v2024 fa te +OpenSubtitles v2024 fa th +OpenSubtitles v2024 fa tl +OpenSubtitles v2024 fa tr +OpenSubtitles v2024 fa tt +OpenSubtitles v2024 fa uk +OpenSubtitles v2024 fa ur +OpenSubtitles v2024 fa vi +OpenSubtitles v2024 fa yue +OpenSubtitles v2024 fa zh_CN +OpenSubtitles v2024 fa zh_TW +OpenSubtitles v2024 fa zh_ze +OpenSubtitles v2024 fi fr +OpenSubtitles v2024 fi ga +OpenSubtitles v2024 fi gd +OpenSubtitles v2024 fi gl +OpenSubtitles v2024 fi he +OpenSubtitles v2024 fi hi +OpenSubtitles v2024 fi hr +OpenSubtitles v2024 fi hu +OpenSubtitles v2024 fi hy +OpenSubtitles v2024 fi id +OpenSubtitles v2024 fi ig +OpenSubtitles v2024 fi is +OpenSubtitles v2024 fi it +OpenSubtitles v2024 fi ja +OpenSubtitles v2024 fi ka +OpenSubtitles v2024 fi kk +OpenSubtitles v2024 fi km +OpenSubtitles v2024 fi kn +OpenSubtitles v2024 fi ko +OpenSubtitles v2024 fi ku +OpenSubtitles v2024 fi lb +OpenSubtitles v2024 fi lt +OpenSubtitles v2024 fi lv +OpenSubtitles v2024 fi mk +OpenSubtitles v2024 fi ml +OpenSubtitles v2024 fi mn +OpenSubtitles v2024 fi ms +OpenSubtitles v2024 fi my +OpenSubtitles v2024 fi ne +OpenSubtitles v2024 fi nl +OpenSubtitles v2024 fi nn +OpenSubtitles v2024 fi no +OpenSubtitles v2024 fi oc +OpenSubtitles v2024 fi or +OpenSubtitles v2024 fi pl +OpenSubtitles v2024 fi pt +OpenSubtitles v2024 fi pt_BR +OpenSubtitles v2024 fi pt_MZ +OpenSubtitles v2024 fi ro +OpenSubtitles v2024 fi ru +OpenSubtitles v2024 fi sd +OpenSubtitles v2024 fi se +OpenSubtitles v2024 fi si +OpenSubtitles v2024 fi sk +OpenSubtitles v2024 fi sl +OpenSubtitles v2024 fi so +OpenSubtitles v2024 fi sq +OpenSubtitles v2024 fi sr +OpenSubtitles v2024 fi sv +OpenSubtitles v2024 fi sw +OpenSubtitles v2024 fi ta +OpenSubtitles v2024 fi te +OpenSubtitles v2024 fi th +OpenSubtitles v2024 fi tl +OpenSubtitles v2024 fi tr +OpenSubtitles v2024 fi tt +OpenSubtitles v2024 fi uk +OpenSubtitles v2024 fi ur +OpenSubtitles v2024 fi vi +OpenSubtitles v2024 fi yue +OpenSubtitles v2024 fi zh_CN +OpenSubtitles v2024 fi zh_TW +OpenSubtitles v2024 fi zh_ze +OpenSubtitles v2024 fr ga +OpenSubtitles v2024 fr gd +OpenSubtitles v2024 fr gl +OpenSubtitles v2024 fr he +OpenSubtitles v2024 fr hi +OpenSubtitles v2024 fr hr +OpenSubtitles v2024 fr hu +OpenSubtitles v2024 fr hy +OpenSubtitles v2024 fr id +OpenSubtitles v2024 fr ig +OpenSubtitles v2024 fr is +OpenSubtitles v2024 fr it +OpenSubtitles v2024 fr ja +OpenSubtitles v2024 fr ka +OpenSubtitles v2024 fr kk +OpenSubtitles v2024 fr km +OpenSubtitles v2024 fr kn +OpenSubtitles v2024 fr ko +OpenSubtitles v2024 fr ku +OpenSubtitles v2024 fr lb +OpenSubtitles v2024 fr lt +OpenSubtitles v2024 fr lv +OpenSubtitles v2024 fr mk +OpenSubtitles v2024 fr ml +OpenSubtitles v2024 fr mn +OpenSubtitles v2024 fr ms +OpenSubtitles v2024 fr my +OpenSubtitles v2024 fr ne +OpenSubtitles v2024 fr nl +OpenSubtitles v2024 fr nn +OpenSubtitles v2024 fr no +OpenSubtitles v2024 fr oc +OpenSubtitles v2024 fr or +OpenSubtitles v2024 fr pl +OpenSubtitles v2024 fr pt +OpenSubtitles v2024 fr pt_BR +OpenSubtitles v2024 fr pt_MZ +OpenSubtitles v2024 fr ro +OpenSubtitles v2024 fr ru +OpenSubtitles v2024 fr sd +OpenSubtitles v2024 fr si +OpenSubtitles v2024 fr sk +OpenSubtitles v2024 fr sl +OpenSubtitles v2024 fr sq +OpenSubtitles v2024 fr sr +OpenSubtitles v2024 fr sv +OpenSubtitles v2024 fr sw +OpenSubtitles v2024 fr ta +OpenSubtitles v2024 fr te +OpenSubtitles v2024 fr th +OpenSubtitles v2024 fr tl +OpenSubtitles v2024 fr tr +OpenSubtitles v2024 fr tt +OpenSubtitles v2024 fr uk +OpenSubtitles v2024 fr ur +OpenSubtitles v2024 fr uz +OpenSubtitles v2024 fr vi +OpenSubtitles v2024 fr yue +OpenSubtitles v2024 fr zh_CN +OpenSubtitles v2024 fr zh_TW +OpenSubtitles v2024 fr zh_ze +OpenSubtitles v2024 ga gd +OpenSubtitles v2024 ga he +OpenSubtitles v2024 ga hi +OpenSubtitles v2024 ga hr +OpenSubtitles v2024 ga hu +OpenSubtitles v2024 ga id +OpenSubtitles v2024 ga is +OpenSubtitles v2024 ga it +OpenSubtitles v2024 ga ja +OpenSubtitles v2024 ga ko +OpenSubtitles v2024 ga lb +OpenSubtitles v2024 ga lt +OpenSubtitles v2024 ga lv +OpenSubtitles v2024 ga mk +OpenSubtitles v2024 ga ms +OpenSubtitles v2024 ga my +OpenSubtitles v2024 ga nl +OpenSubtitles v2024 ga no +OpenSubtitles v2024 ga pl +OpenSubtitles v2024 ga pt +OpenSubtitles v2024 ga pt_BR +OpenSubtitles v2024 ga ro +OpenSubtitles v2024 ga ru +OpenSubtitles v2024 ga si +OpenSubtitles v2024 ga sk +OpenSubtitles v2024 ga sl +OpenSubtitles v2024 ga sr +OpenSubtitles v2024 ga sv +OpenSubtitles v2024 ga ta +OpenSubtitles v2024 ga te +OpenSubtitles v2024 ga th +OpenSubtitles v2024 ga tr +OpenSubtitles v2024 ga uk +OpenSubtitles v2024 ga ur +OpenSubtitles v2024 ga vi +OpenSubtitles v2024 ga zh_CN +OpenSubtitles v2024 ga zh_TW +OpenSubtitles v2024 ga zh_ze +OpenSubtitles v2024 gd he +OpenSubtitles v2024 gd hi +OpenSubtitles v2024 gd hr +OpenSubtitles v2024 gd hu +OpenSubtitles v2024 gd id +OpenSubtitles v2024 gd is +OpenSubtitles v2024 gd it +OpenSubtitles v2024 gd ja +OpenSubtitles v2024 gd ko +OpenSubtitles v2024 gd lb +OpenSubtitles v2024 gd lt +OpenSubtitles v2024 gd lv +OpenSubtitles v2024 gd mk +OpenSubtitles v2024 gd ms +OpenSubtitles v2024 gd my +OpenSubtitles v2024 gd nl +OpenSubtitles v2024 gd no +OpenSubtitles v2024 gd pl +OpenSubtitles v2024 gd pt +OpenSubtitles v2024 gd pt_BR +OpenSubtitles v2024 gd ro +OpenSubtitles v2024 gd ru +OpenSubtitles v2024 gd sk +OpenSubtitles v2024 gd sl +OpenSubtitles v2024 gd sq +OpenSubtitles v2024 gd sr +OpenSubtitles v2024 gd sv +OpenSubtitles v2024 gd th +OpenSubtitles v2024 gd tr +OpenSubtitles v2024 gd uk +OpenSubtitles v2024 gd ur +OpenSubtitles v2024 gd vi +OpenSubtitles v2024 gd zh_CN +OpenSubtitles v2024 gd zh_TW +OpenSubtitles v2024 gd zh_ze +OpenSubtitles v2024 gl he +OpenSubtitles v2024 gl hi +OpenSubtitles v2024 gl hr +OpenSubtitles v2024 gl hu +OpenSubtitles v2024 gl hy +OpenSubtitles v2024 gl id +OpenSubtitles v2024 gl ig +OpenSubtitles v2024 gl is +OpenSubtitles v2024 gl it +OpenSubtitles v2024 gl ja +OpenSubtitles v2024 gl ka +OpenSubtitles v2024 gl km +OpenSubtitles v2024 gl kn +OpenSubtitles v2024 gl ko +OpenSubtitles v2024 gl ku +OpenSubtitles v2024 gl lt +OpenSubtitles v2024 gl lv +OpenSubtitles v2024 gl mk +OpenSubtitles v2024 gl ml +OpenSubtitles v2024 gl mn +OpenSubtitles v2024 gl ms +OpenSubtitles v2024 gl my +OpenSubtitles v2024 gl nl +OpenSubtitles v2024 gl no +OpenSubtitles v2024 gl oc +OpenSubtitles v2024 gl pl +OpenSubtitles v2024 gl pt +OpenSubtitles v2024 gl pt_BR +OpenSubtitles v2024 gl ro +OpenSubtitles v2024 gl ru +OpenSubtitles v2024 gl sd +OpenSubtitles v2024 gl si +OpenSubtitles v2024 gl sk +OpenSubtitles v2024 gl sl +OpenSubtitles v2024 gl sq +OpenSubtitles v2024 gl sr +OpenSubtitles v2024 gl sv +OpenSubtitles v2024 gl ta +OpenSubtitles v2024 gl te +OpenSubtitles v2024 gl th +OpenSubtitles v2024 gl tl +OpenSubtitles v2024 gl tr +OpenSubtitles v2024 gl tt +OpenSubtitles v2024 gl uk +OpenSubtitles v2024 gl ur +OpenSubtitles v2024 gl vi +OpenSubtitles v2024 gl yue +OpenSubtitles v2024 gl zh_CN +OpenSubtitles v2024 gl zh_TW +OpenSubtitles v2024 gl zh_ze +OpenSubtitles v2024 he hi +OpenSubtitles v2024 he hr +OpenSubtitles v2024 he hu +OpenSubtitles v2024 he hy +OpenSubtitles v2024 he id +OpenSubtitles v2024 he ig +OpenSubtitles v2024 he is +OpenSubtitles v2024 he it +OpenSubtitles v2024 he ja +OpenSubtitles v2024 he ka +OpenSubtitles v2024 he kk +OpenSubtitles v2024 he km +OpenSubtitles v2024 he kn +OpenSubtitles v2024 he ko +OpenSubtitles v2024 he ku +OpenSubtitles v2024 he lb +OpenSubtitles v2024 he lt +OpenSubtitles v2024 he lv +OpenSubtitles v2024 he mk +OpenSubtitles v2024 he ml +OpenSubtitles v2024 he mn +OpenSubtitles v2024 he ms +OpenSubtitles v2024 he my +OpenSubtitles v2024 he ne +OpenSubtitles v2024 he nl +OpenSubtitles v2024 he nn +OpenSubtitles v2024 he no +OpenSubtitles v2024 he oc +OpenSubtitles v2024 he or +OpenSubtitles v2024 he pl +OpenSubtitles v2024 he pt +OpenSubtitles v2024 he pt_BR +OpenSubtitles v2024 he pt_MZ +OpenSubtitles v2024 he ro +OpenSubtitles v2024 he ru +OpenSubtitles v2024 he sd +OpenSubtitles v2024 he si +OpenSubtitles v2024 he sk +OpenSubtitles v2024 he sl +OpenSubtitles v2024 he so +OpenSubtitles v2024 he sq +OpenSubtitles v2024 he sr +OpenSubtitles v2024 he sv +OpenSubtitles v2024 he sw +OpenSubtitles v2024 he ta +OpenSubtitles v2024 he te +OpenSubtitles v2024 he th +OpenSubtitles v2024 he tl +OpenSubtitles v2024 he tr +OpenSubtitles v2024 he tt +OpenSubtitles v2024 he uk +OpenSubtitles v2024 he ur +OpenSubtitles v2024 he vi +OpenSubtitles v2024 he yue +OpenSubtitles v2024 he zh_CN +OpenSubtitles v2024 he zh_TW +OpenSubtitles v2024 he zh_ze +OpenSubtitles v2024 hi hr +OpenSubtitles v2024 hi hu +OpenSubtitles v2024 hi hy +OpenSubtitles v2024 hi id +OpenSubtitles v2024 hi is +OpenSubtitles v2024 hi it +OpenSubtitles v2024 hi ja +OpenSubtitles v2024 hi ka +OpenSubtitles v2024 hi kk +OpenSubtitles v2024 hi km +OpenSubtitles v2024 hi kn +OpenSubtitles v2024 hi ko +OpenSubtitles v2024 hi ku +OpenSubtitles v2024 hi lt +OpenSubtitles v2024 hi lv +OpenSubtitles v2024 hi mk +OpenSubtitles v2024 hi ml +OpenSubtitles v2024 hi mn +OpenSubtitles v2024 hi ms +OpenSubtitles v2024 hi my +OpenSubtitles v2024 hi nl +OpenSubtitles v2024 hi no +OpenSubtitles v2024 hi oc +OpenSubtitles v2024 hi pl +OpenSubtitles v2024 hi pt +OpenSubtitles v2024 hi pt_BR +OpenSubtitles v2024 hi pt_MZ +OpenSubtitles v2024 hi ro +OpenSubtitles v2024 hi ru +OpenSubtitles v2024 hi sd +OpenSubtitles v2024 hi si +OpenSubtitles v2024 hi sk +OpenSubtitles v2024 hi sl +OpenSubtitles v2024 hi sq +OpenSubtitles v2024 hi sr +OpenSubtitles v2024 hi sv +OpenSubtitles v2024 hi sw +OpenSubtitles v2024 hi ta +OpenSubtitles v2024 hi te +OpenSubtitles v2024 hi th +OpenSubtitles v2024 hi tl +OpenSubtitles v2024 hi tr +OpenSubtitles v2024 hi tt +OpenSubtitles v2024 hi uk +OpenSubtitles v2024 hi ur +OpenSubtitles v2024 hi vi +OpenSubtitles v2024 hi yue +OpenSubtitles v2024 hi zh_CN +OpenSubtitles v2024 hi zh_TW +OpenSubtitles v2024 hi zh_ze +OpenSubtitles v2024 hr hu +OpenSubtitles v2024 hr hy +OpenSubtitles v2024 hr id +OpenSubtitles v2024 hr ig +OpenSubtitles v2024 hr is +OpenSubtitles v2024 hr it +OpenSubtitles v2024 hr ja +OpenSubtitles v2024 hr ka +OpenSubtitles v2024 hr kk +OpenSubtitles v2024 hr km +OpenSubtitles v2024 hr kn +OpenSubtitles v2024 hr ko +OpenSubtitles v2024 hr ku +OpenSubtitles v2024 hr lb +OpenSubtitles v2024 hr lt +OpenSubtitles v2024 hr lv +OpenSubtitles v2024 hr mk +OpenSubtitles v2024 hr ml +OpenSubtitles v2024 hr mn +OpenSubtitles v2024 hr ms +OpenSubtitles v2024 hr my +OpenSubtitles v2024 hr ne +OpenSubtitles v2024 hr nl +OpenSubtitles v2024 hr nn +OpenSubtitles v2024 hr no +OpenSubtitles v2024 hr oc +OpenSubtitles v2024 hr or +OpenSubtitles v2024 hr pl +OpenSubtitles v2024 hr pt +OpenSubtitles v2024 hr pt_BR +OpenSubtitles v2024 hr pt_MZ +OpenSubtitles v2024 hr ro +OpenSubtitles v2024 hr ru +OpenSubtitles v2024 hr sd +OpenSubtitles v2024 hr se +OpenSubtitles v2024 hr si +OpenSubtitles v2024 hr sk +OpenSubtitles v2024 hr sl +OpenSubtitles v2024 hr so +OpenSubtitles v2024 hr sq +OpenSubtitles v2024 hr sr +OpenSubtitles v2024 hr sv +OpenSubtitles v2024 hr sw +OpenSubtitles v2024 hr ta +OpenSubtitles v2024 hr te +OpenSubtitles v2024 hr th +OpenSubtitles v2024 hr tl +OpenSubtitles v2024 hr tr +OpenSubtitles v2024 hr tt +OpenSubtitles v2024 hr uk +OpenSubtitles v2024 hr ur +OpenSubtitles v2024 hr vi +OpenSubtitles v2024 hr yue +OpenSubtitles v2024 hr zh_CN +OpenSubtitles v2024 hr zh_TW +OpenSubtitles v2024 hr zh_ze +OpenSubtitles v2024 hu hy +OpenSubtitles v2024 hu id +OpenSubtitles v2024 hu ig +OpenSubtitles v2024 hu is +OpenSubtitles v2024 hu it +OpenSubtitles v2024 hu ja +OpenSubtitles v2024 hu ka +OpenSubtitles v2024 hu kk +OpenSubtitles v2024 hu km +OpenSubtitles v2024 hu kn +OpenSubtitles v2024 hu ko +OpenSubtitles v2024 hu ku +OpenSubtitles v2024 hu lb +OpenSubtitles v2024 hu lt +OpenSubtitles v2024 hu lv +OpenSubtitles v2024 hu mk +OpenSubtitles v2024 hu ml +OpenSubtitles v2024 hu mn +OpenSubtitles v2024 hu ms +OpenSubtitles v2024 hu my +OpenSubtitles v2024 hu ne +OpenSubtitles v2024 hu nl +OpenSubtitles v2024 hu nn +OpenSubtitles v2024 hu no +OpenSubtitles v2024 hu oc +OpenSubtitles v2024 hu or +OpenSubtitles v2024 hu pl +OpenSubtitles v2024 hu pt +OpenSubtitles v2024 hu pt_BR +OpenSubtitles v2024 hu pt_MZ +OpenSubtitles v2024 hu ro +OpenSubtitles v2024 hu ru +OpenSubtitles v2024 hu sd +OpenSubtitles v2024 hu si +OpenSubtitles v2024 hu sk +OpenSubtitles v2024 hu sl +OpenSubtitles v2024 hu so +OpenSubtitles v2024 hu sq +OpenSubtitles v2024 hu sr +OpenSubtitles v2024 hu sv +OpenSubtitles v2024 hu sw +OpenSubtitles v2024 hu ta +OpenSubtitles v2024 hu te +OpenSubtitles v2024 hu th +OpenSubtitles v2024 hu tl +OpenSubtitles v2024 hu tr +OpenSubtitles v2024 hu tt +OpenSubtitles v2024 hu uk +OpenSubtitles v2024 hu ur +OpenSubtitles v2024 hu vi +OpenSubtitles v2024 hu yue +OpenSubtitles v2024 hu zh_CN +OpenSubtitles v2024 hu zh_TW +OpenSubtitles v2024 hu zh_ze +OpenSubtitles v2024 hy id +OpenSubtitles v2024 hy it +OpenSubtitles v2024 hy ka +OpenSubtitles v2024 hy ko +OpenSubtitles v2024 hy mk +OpenSubtitles v2024 hy ml +OpenSubtitles v2024 hy ms +OpenSubtitles v2024 hy nl +OpenSubtitles v2024 hy no +OpenSubtitles v2024 hy pl +OpenSubtitles v2024 hy pt +OpenSubtitles v2024 hy pt_BR +OpenSubtitles v2024 hy ro +OpenSubtitles v2024 hy ru +OpenSubtitles v2024 hy sk +OpenSubtitles v2024 hy sl +OpenSubtitles v2024 hy sq +OpenSubtitles v2024 hy sr +OpenSubtitles v2024 hy sv +OpenSubtitles v2024 hy ta +OpenSubtitles v2024 hy th +OpenSubtitles v2024 hy tl +OpenSubtitles v2024 hy tr +OpenSubtitles v2024 hy uk +OpenSubtitles v2024 hy vi +OpenSubtitles v2024 hy zh_CN +OpenSubtitles v2024 hy zh_TW +OpenSubtitles v2024 id ig +OpenSubtitles v2024 id is +OpenSubtitles v2024 id it +OpenSubtitles v2024 id ja +OpenSubtitles v2024 id ka +OpenSubtitles v2024 id kk +OpenSubtitles v2024 id km +OpenSubtitles v2024 id kn +OpenSubtitles v2024 id ko +OpenSubtitles v2024 id ku +OpenSubtitles v2024 id lb +OpenSubtitles v2024 id lt +OpenSubtitles v2024 id lv +OpenSubtitles v2024 id mk +OpenSubtitles v2024 id ml +OpenSubtitles v2024 id mn +OpenSubtitles v2024 id ms +OpenSubtitles v2024 id my +OpenSubtitles v2024 id ne +OpenSubtitles v2024 id nl +OpenSubtitles v2024 id nn +OpenSubtitles v2024 id no +OpenSubtitles v2024 id oc +OpenSubtitles v2024 id or +OpenSubtitles v2024 id pl +OpenSubtitles v2024 id pt +OpenSubtitles v2024 id pt_BR +OpenSubtitles v2024 id pt_MZ +OpenSubtitles v2024 id ro +OpenSubtitles v2024 id ru +OpenSubtitles v2024 id sd +OpenSubtitles v2024 id si +OpenSubtitles v2024 id sk +OpenSubtitles v2024 id sl +OpenSubtitles v2024 id so +OpenSubtitles v2024 id sq +OpenSubtitles v2024 id sr +OpenSubtitles v2024 id sv +OpenSubtitles v2024 id sw +OpenSubtitles v2024 id ta +OpenSubtitles v2024 id te +OpenSubtitles v2024 id th +OpenSubtitles v2024 id tl +OpenSubtitles v2024 id tr +OpenSubtitles v2024 id tt +OpenSubtitles v2024 id uk +OpenSubtitles v2024 id ur +OpenSubtitles v2024 id vi +OpenSubtitles v2024 id yue +OpenSubtitles v2024 id zh_CN +OpenSubtitles v2024 id zh_TW +OpenSubtitles v2024 id zh_ze +OpenSubtitles v2024 ig is +OpenSubtitles v2024 ig it +OpenSubtitles v2024 ig km +OpenSubtitles v2024 ig ko +OpenSubtitles v2024 ig lv +OpenSubtitles v2024 ig mk +OpenSubtitles v2024 ig ml +OpenSubtitles v2024 ig ms +OpenSubtitles v2024 ig my +OpenSubtitles v2024 ig nl +OpenSubtitles v2024 ig no +OpenSubtitles v2024 ig pl +OpenSubtitles v2024 ig pt +OpenSubtitles v2024 ig pt_BR +OpenSubtitles v2024 ig ro +OpenSubtitles v2024 ig ru +OpenSubtitles v2024 ig sq +OpenSubtitles v2024 ig sr +OpenSubtitles v2024 ig sv +OpenSubtitles v2024 ig th +OpenSubtitles v2024 ig tr +OpenSubtitles v2024 ig tt +OpenSubtitles v2024 ig vi +OpenSubtitles v2024 ig zh_CN +OpenSubtitles v2024 ig zh_TW +OpenSubtitles v2024 is it +OpenSubtitles v2024 is ja +OpenSubtitles v2024 is ka +OpenSubtitles v2024 is kk +OpenSubtitles v2024 is km +OpenSubtitles v2024 is ko +OpenSubtitles v2024 is ku +OpenSubtitles v2024 is lb +OpenSubtitles v2024 is lt +OpenSubtitles v2024 is lv +OpenSubtitles v2024 is mk +OpenSubtitles v2024 is ml +OpenSubtitles v2024 is mn +OpenSubtitles v2024 is ms +OpenSubtitles v2024 is my +OpenSubtitles v2024 is ne +OpenSubtitles v2024 is nl +OpenSubtitles v2024 is nn +OpenSubtitles v2024 is no +OpenSubtitles v2024 is oc +OpenSubtitles v2024 is or +OpenSubtitles v2024 is pl +OpenSubtitles v2024 is pt +OpenSubtitles v2024 is pt_BR +OpenSubtitles v2024 is ro +OpenSubtitles v2024 is ru +OpenSubtitles v2024 is sd +OpenSubtitles v2024 is si +OpenSubtitles v2024 is sk +OpenSubtitles v2024 is sl +OpenSubtitles v2024 is sq +OpenSubtitles v2024 is sr +OpenSubtitles v2024 is sv +OpenSubtitles v2024 is sw +OpenSubtitles v2024 is ta +OpenSubtitles v2024 is te +OpenSubtitles v2024 is th +OpenSubtitles v2024 is tl +OpenSubtitles v2024 is tr +OpenSubtitles v2024 is tt +OpenSubtitles v2024 is uk +OpenSubtitles v2024 is ur +OpenSubtitles v2024 is vi +OpenSubtitles v2024 is yue +OpenSubtitles v2024 is zh_CN +OpenSubtitles v2024 is zh_TW +OpenSubtitles v2024 is zh_ze +OpenSubtitles v2024 it ja +OpenSubtitles v2024 it ka +OpenSubtitles v2024 it kk +OpenSubtitles v2024 it km +OpenSubtitles v2024 it kn +OpenSubtitles v2024 it ko +OpenSubtitles v2024 it ku +OpenSubtitles v2024 it lb +OpenSubtitles v2024 it lt +OpenSubtitles v2024 it lv +OpenSubtitles v2024 it mk +OpenSubtitles v2024 it ml +OpenSubtitles v2024 it mn +OpenSubtitles v2024 it ms +OpenSubtitles v2024 it my +OpenSubtitles v2024 it ne +OpenSubtitles v2024 it nl +OpenSubtitles v2024 it nn +OpenSubtitles v2024 it no +OpenSubtitles v2024 it oc +OpenSubtitles v2024 it or +OpenSubtitles v2024 it pl +OpenSubtitles v2024 it pt +OpenSubtitles v2024 it pt_BR +OpenSubtitles v2024 it pt_MZ +OpenSubtitles v2024 it ro +OpenSubtitles v2024 it ru +OpenSubtitles v2024 it sd +OpenSubtitles v2024 it si +OpenSubtitles v2024 it sk +OpenSubtitles v2024 it sl +OpenSubtitles v2024 it sq +OpenSubtitles v2024 it sr +OpenSubtitles v2024 it sv +OpenSubtitles v2024 it sw +OpenSubtitles v2024 it ta +OpenSubtitles v2024 it te +OpenSubtitles v2024 it th +OpenSubtitles v2024 it tl +OpenSubtitles v2024 it tr +OpenSubtitles v2024 it tt +OpenSubtitles v2024 it uk +OpenSubtitles v2024 it ur +OpenSubtitles v2024 it vi +OpenSubtitles v2024 it yue +OpenSubtitles v2024 it zh_CN +OpenSubtitles v2024 it zh_TW +OpenSubtitles v2024 it zh_ze +OpenSubtitles v2024 ja ka +OpenSubtitles v2024 ja kk +OpenSubtitles v2024 ja km +OpenSubtitles v2024 ja ko +OpenSubtitles v2024 ja ku +OpenSubtitles v2024 ja lt +OpenSubtitles v2024 ja lv +OpenSubtitles v2024 ja mk +OpenSubtitles v2024 ja ml +OpenSubtitles v2024 ja mn +OpenSubtitles v2024 ja ms +OpenSubtitles v2024 ja my +OpenSubtitles v2024 ja ne +OpenSubtitles v2024 ja nl +OpenSubtitles v2024 ja no +OpenSubtitles v2024 ja oc +OpenSubtitles v2024 ja pl +OpenSubtitles v2024 ja pt +OpenSubtitles v2024 ja pt_BR +OpenSubtitles v2024 ja pt_MZ +OpenSubtitles v2024 ja ro +OpenSubtitles v2024 ja ru +OpenSubtitles v2024 ja sd +OpenSubtitles v2024 ja si +OpenSubtitles v2024 ja sk +OpenSubtitles v2024 ja sl +OpenSubtitles v2024 ja sq +OpenSubtitles v2024 ja sr +OpenSubtitles v2024 ja sv +OpenSubtitles v2024 ja ta +OpenSubtitles v2024 ja te +OpenSubtitles v2024 ja th +OpenSubtitles v2024 ja tl +OpenSubtitles v2024 ja tr +OpenSubtitles v2024 ja tt +OpenSubtitles v2024 ja uk +OpenSubtitles v2024 ja ur +OpenSubtitles v2024 ja vi +OpenSubtitles v2024 ja yue +OpenSubtitles v2024 ja zh_CN +OpenSubtitles v2024 ja zh_TW +OpenSubtitles v2024 ja zh_ze +OpenSubtitles v2024 ka kk +OpenSubtitles v2024 ka km +OpenSubtitles v2024 ka ko +OpenSubtitles v2024 ka ku +OpenSubtitles v2024 ka lt +OpenSubtitles v2024 ka lv +OpenSubtitles v2024 ka mk +OpenSubtitles v2024 ka ml +OpenSubtitles v2024 ka ms +OpenSubtitles v2024 ka my +OpenSubtitles v2024 ka nl +OpenSubtitles v2024 ka no +OpenSubtitles v2024 ka pl +OpenSubtitles v2024 ka pt +OpenSubtitles v2024 ka pt_BR +OpenSubtitles v2024 ka ro +OpenSubtitles v2024 ka ru +OpenSubtitles v2024 ka sd +OpenSubtitles v2024 ka si +OpenSubtitles v2024 ka sk +OpenSubtitles v2024 ka sl +OpenSubtitles v2024 ka sq +OpenSubtitles v2024 ka sr +OpenSubtitles v2024 ka sv +OpenSubtitles v2024 ka ta +OpenSubtitles v2024 ka te +OpenSubtitles v2024 ka th +OpenSubtitles v2024 ka tl +OpenSubtitles v2024 ka tr +OpenSubtitles v2024 ka tt +OpenSubtitles v2024 ka uk +OpenSubtitles v2024 ka ur +OpenSubtitles v2024 ka vi +OpenSubtitles v2024 ka yue +OpenSubtitles v2024 ka zh_CN +OpenSubtitles v2024 ka zh_TW +OpenSubtitles v2024 ka zh_ze +OpenSubtitles v2024 kk km +OpenSubtitles v2024 kk ko +OpenSubtitles v2024 kk ku +OpenSubtitles v2024 kk lt +OpenSubtitles v2024 kk lv +OpenSubtitles v2024 kk mk +OpenSubtitles v2024 kk ml +OpenSubtitles v2024 kk mn +OpenSubtitles v2024 kk ms +OpenSubtitles v2024 kk my +OpenSubtitles v2024 kk nl +OpenSubtitles v2024 kk no +OpenSubtitles v2024 kk pl +OpenSubtitles v2024 kk pt +OpenSubtitles v2024 kk pt_BR +OpenSubtitles v2024 kk ro +OpenSubtitles v2024 kk ru +OpenSubtitles v2024 kk si +OpenSubtitles v2024 kk sk +OpenSubtitles v2024 kk sl +OpenSubtitles v2024 kk sq +OpenSubtitles v2024 kk sr +OpenSubtitles v2024 kk sv +OpenSubtitles v2024 kk ta +OpenSubtitles v2024 kk th +OpenSubtitles v2024 kk tl +OpenSubtitles v2024 kk tr +OpenSubtitles v2024 kk tt +OpenSubtitles v2024 kk uk +OpenSubtitles v2024 kk ur +OpenSubtitles v2024 kk vi +OpenSubtitles v2024 kk zh_CN +OpenSubtitles v2024 kk zh_TW +OpenSubtitles v2024 km ko +OpenSubtitles v2024 km ku +OpenSubtitles v2024 km lt +OpenSubtitles v2024 km lv +OpenSubtitles v2024 km mk +OpenSubtitles v2024 km ml +OpenSubtitles v2024 km mn +OpenSubtitles v2024 km ms +OpenSubtitles v2024 km my +OpenSubtitles v2024 km ne +OpenSubtitles v2024 km nl +OpenSubtitles v2024 km no +OpenSubtitles v2024 km oc +OpenSubtitles v2024 km pl +OpenSubtitles v2024 km pt +OpenSubtitles v2024 km pt_BR +OpenSubtitles v2024 km ro +OpenSubtitles v2024 km ru +OpenSubtitles v2024 km si +OpenSubtitles v2024 km sk +OpenSubtitles v2024 km sl +OpenSubtitles v2024 km sq +OpenSubtitles v2024 km sr +OpenSubtitles v2024 km sv +OpenSubtitles v2024 km sw +OpenSubtitles v2024 km ta +OpenSubtitles v2024 km te +OpenSubtitles v2024 km th +OpenSubtitles v2024 km tl +OpenSubtitles v2024 km tr +OpenSubtitles v2024 km tt +OpenSubtitles v2024 km uk +OpenSubtitles v2024 km ur +OpenSubtitles v2024 km vi +OpenSubtitles v2024 km yue +OpenSubtitles v2024 km zh_CN +OpenSubtitles v2024 km zh_TW +OpenSubtitles v2024 km zh_ze +OpenSubtitles v2024 kn ko +OpenSubtitles v2024 kn ku +OpenSubtitles v2024 kn mk +OpenSubtitles v2024 kn ml +OpenSubtitles v2024 kn ms +OpenSubtitles v2024 kn nl +OpenSubtitles v2024 kn no +OpenSubtitles v2024 kn pl +OpenSubtitles v2024 kn pt +OpenSubtitles v2024 kn pt_BR +OpenSubtitles v2024 kn ro +OpenSubtitles v2024 kn ru +OpenSubtitles v2024 kn si +OpenSubtitles v2024 kn sk +OpenSubtitles v2024 kn sl +OpenSubtitles v2024 kn sr +OpenSubtitles v2024 kn sv +OpenSubtitles v2024 kn ta +OpenSubtitles v2024 kn te +OpenSubtitles v2024 kn th +OpenSubtitles v2024 kn tl +OpenSubtitles v2024 kn tr +OpenSubtitles v2024 kn uk +OpenSubtitles v2024 kn vi +OpenSubtitles v2024 kn zh_CN +OpenSubtitles v2024 kn zh_TW +OpenSubtitles v2024 ko ku +OpenSubtitles v2024 ko lb +OpenSubtitles v2024 ko lt +OpenSubtitles v2024 ko lv +OpenSubtitles v2024 ko mk +OpenSubtitles v2024 ko ml +OpenSubtitles v2024 ko mn +OpenSubtitles v2024 ko ms +OpenSubtitles v2024 ko my +OpenSubtitles v2024 ko ne +OpenSubtitles v2024 ko nl +OpenSubtitles v2024 ko no +OpenSubtitles v2024 ko oc +OpenSubtitles v2024 ko or +OpenSubtitles v2024 ko pl +OpenSubtitles v2024 ko pt +OpenSubtitles v2024 ko pt_BR +OpenSubtitles v2024 ko pt_MZ +OpenSubtitles v2024 ko ro +OpenSubtitles v2024 ko ru +OpenSubtitles v2024 ko sd +OpenSubtitles v2024 ko si +OpenSubtitles v2024 ko sk +OpenSubtitles v2024 ko sl +OpenSubtitles v2024 ko so +OpenSubtitles v2024 ko sq +OpenSubtitles v2024 ko sr +OpenSubtitles v2024 ko sv +OpenSubtitles v2024 ko sw +OpenSubtitles v2024 ko ta +OpenSubtitles v2024 ko te +OpenSubtitles v2024 ko th +OpenSubtitles v2024 ko tl +OpenSubtitles v2024 ko tr +OpenSubtitles v2024 ko tt +OpenSubtitles v2024 ko uk +OpenSubtitles v2024 ko ur +OpenSubtitles v2024 ko vi +OpenSubtitles v2024 ko yue +OpenSubtitles v2024 ko zh_CN +OpenSubtitles v2024 ko zh_TW +OpenSubtitles v2024 ko zh_ze +OpenSubtitles v2024 ku lt +OpenSubtitles v2024 ku lv +OpenSubtitles v2024 ku mk +OpenSubtitles v2024 ku ml +OpenSubtitles v2024 ku mn +OpenSubtitles v2024 ku ms +OpenSubtitles v2024 ku my +OpenSubtitles v2024 ku ne +OpenSubtitles v2024 ku nl +OpenSubtitles v2024 ku no +OpenSubtitles v2024 ku oc +OpenSubtitles v2024 ku pl +OpenSubtitles v2024 ku pt +OpenSubtitles v2024 ku pt_BR +OpenSubtitles v2024 ku ro +OpenSubtitles v2024 ku ru +OpenSubtitles v2024 ku sd +OpenSubtitles v2024 ku si +OpenSubtitles v2024 ku sk +OpenSubtitles v2024 ku sl +OpenSubtitles v2024 ku sq +OpenSubtitles v2024 ku sr +OpenSubtitles v2024 ku sv +OpenSubtitles v2024 ku sw +OpenSubtitles v2024 ku ta +OpenSubtitles v2024 ku te +OpenSubtitles v2024 ku th +OpenSubtitles v2024 ku tl +OpenSubtitles v2024 ku tr +OpenSubtitles v2024 ku tt +OpenSubtitles v2024 ku uk +OpenSubtitles v2024 ku ur +OpenSubtitles v2024 ku vi +OpenSubtitles v2024 ku yue +OpenSubtitles v2024 ku zh_CN +OpenSubtitles v2024 ku zh_TW +OpenSubtitles v2024 ku zh_ze +OpenSubtitles v2024 lb lt +OpenSubtitles v2024 lb lv +OpenSubtitles v2024 lb mk +OpenSubtitles v2024 lb mn +OpenSubtitles v2024 lb ms +OpenSubtitles v2024 lb my +OpenSubtitles v2024 lb nl +OpenSubtitles v2024 lb no +OpenSubtitles v2024 lb pl +OpenSubtitles v2024 lb pt +OpenSubtitles v2024 lb pt_BR +OpenSubtitles v2024 lb ro +OpenSubtitles v2024 lb ru +OpenSubtitles v2024 lb sk +OpenSubtitles v2024 lb sl +OpenSubtitles v2024 lb sq +OpenSubtitles v2024 lb sr +OpenSubtitles v2024 lb sv +OpenSubtitles v2024 lb th +OpenSubtitles v2024 lb tl +OpenSubtitles v2024 lb tr +OpenSubtitles v2024 lb uk +OpenSubtitles v2024 lb vi +OpenSubtitles v2024 lb zh_CN +OpenSubtitles v2024 lb zh_TW +OpenSubtitles v2024 lt lv +OpenSubtitles v2024 lt mk +OpenSubtitles v2024 lt ml +OpenSubtitles v2024 lt mn +OpenSubtitles v2024 lt ms +OpenSubtitles v2024 lt my +OpenSubtitles v2024 lt ne +OpenSubtitles v2024 lt nl +OpenSubtitles v2024 lt no +OpenSubtitles v2024 lt oc +OpenSubtitles v2024 lt or +OpenSubtitles v2024 lt pl +OpenSubtitles v2024 lt pt +OpenSubtitles v2024 lt pt_BR +OpenSubtitles v2024 lt pt_MZ +OpenSubtitles v2024 lt ro +OpenSubtitles v2024 lt ru +OpenSubtitles v2024 lt sd +OpenSubtitles v2024 lt si +OpenSubtitles v2024 lt sk +OpenSubtitles v2024 lt sl +OpenSubtitles v2024 lt sq +OpenSubtitles v2024 lt sr +OpenSubtitles v2024 lt sv +OpenSubtitles v2024 lt sw +OpenSubtitles v2024 lt ta +OpenSubtitles v2024 lt te +OpenSubtitles v2024 lt th +OpenSubtitles v2024 lt tl +OpenSubtitles v2024 lt tr +OpenSubtitles v2024 lt tt +OpenSubtitles v2024 lt uk +OpenSubtitles v2024 lt ur +OpenSubtitles v2024 lt vi +OpenSubtitles v2024 lt yue +OpenSubtitles v2024 lt zh_CN +OpenSubtitles v2024 lt zh_TW +OpenSubtitles v2024 lt zh_ze +OpenSubtitles v2024 lv mk +OpenSubtitles v2024 lv ml +OpenSubtitles v2024 lv mn +OpenSubtitles v2024 lv ms +OpenSubtitles v2024 lv my +OpenSubtitles v2024 lv nl +OpenSubtitles v2024 lv no +OpenSubtitles v2024 lv oc +OpenSubtitles v2024 lv or +OpenSubtitles v2024 lv pl +OpenSubtitles v2024 lv pt +OpenSubtitles v2024 lv pt_BR +OpenSubtitles v2024 lv ro +OpenSubtitles v2024 lv ru +OpenSubtitles v2024 lv sd +OpenSubtitles v2024 lv si +OpenSubtitles v2024 lv sk +OpenSubtitles v2024 lv sl +OpenSubtitles v2024 lv sq +OpenSubtitles v2024 lv sr +OpenSubtitles v2024 lv sv +OpenSubtitles v2024 lv sw +OpenSubtitles v2024 lv ta +OpenSubtitles v2024 lv te +OpenSubtitles v2024 lv th +OpenSubtitles v2024 lv tl +OpenSubtitles v2024 lv tr +OpenSubtitles v2024 lv tt +OpenSubtitles v2024 lv uk +OpenSubtitles v2024 lv ur +OpenSubtitles v2024 lv vi +OpenSubtitles v2024 lv yue +OpenSubtitles v2024 lv zh_CN +OpenSubtitles v2024 lv zh_TW +OpenSubtitles v2024 lv zh_ze +OpenSubtitles v2024 mk ml +OpenSubtitles v2024 mk mn +OpenSubtitles v2024 mk ms +OpenSubtitles v2024 mk my +OpenSubtitles v2024 mk ne +OpenSubtitles v2024 mk nl +OpenSubtitles v2024 mk no +OpenSubtitles v2024 mk oc +OpenSubtitles v2024 mk pl +OpenSubtitles v2024 mk pt +OpenSubtitles v2024 mk pt_BR +OpenSubtitles v2024 mk pt_MZ +OpenSubtitles v2024 mk ro +OpenSubtitles v2024 mk ru +OpenSubtitles v2024 mk sd +OpenSubtitles v2024 mk si +OpenSubtitles v2024 mk sk +OpenSubtitles v2024 mk sl +OpenSubtitles v2024 mk sq +OpenSubtitles v2024 mk sr +OpenSubtitles v2024 mk sv +OpenSubtitles v2024 mk sw +OpenSubtitles v2024 mk ta +OpenSubtitles v2024 mk te +OpenSubtitles v2024 mk th +OpenSubtitles v2024 mk tl +OpenSubtitles v2024 mk tr +OpenSubtitles v2024 mk tt +OpenSubtitles v2024 mk uk +OpenSubtitles v2024 mk ur +OpenSubtitles v2024 mk vi +OpenSubtitles v2024 mk yue +OpenSubtitles v2024 mk zh_CN +OpenSubtitles v2024 mk zh_TW +OpenSubtitles v2024 mk zh_ze +OpenSubtitles v2024 ml mn +OpenSubtitles v2024 ml ms +OpenSubtitles v2024 ml my +OpenSubtitles v2024 ml nl +OpenSubtitles v2024 ml no +OpenSubtitles v2024 ml oc +OpenSubtitles v2024 ml pl +OpenSubtitles v2024 ml pt +OpenSubtitles v2024 ml pt_BR +OpenSubtitles v2024 ml pt_MZ +OpenSubtitles v2024 ml ro +OpenSubtitles v2024 ml ru +OpenSubtitles v2024 ml sd +OpenSubtitles v2024 ml si +OpenSubtitles v2024 ml sk +OpenSubtitles v2024 ml sl +OpenSubtitles v2024 ml so +OpenSubtitles v2024 ml sq +OpenSubtitles v2024 ml sr +OpenSubtitles v2024 ml sv +OpenSubtitles v2024 ml sw +OpenSubtitles v2024 ml ta +OpenSubtitles v2024 ml te +OpenSubtitles v2024 ml th +OpenSubtitles v2024 ml tl +OpenSubtitles v2024 ml tr +OpenSubtitles v2024 ml tt +OpenSubtitles v2024 ml uk +OpenSubtitles v2024 ml ur +OpenSubtitles v2024 ml vi +OpenSubtitles v2024 ml yue +OpenSubtitles v2024 ml zh_CN +OpenSubtitles v2024 ml zh_TW +OpenSubtitles v2024 ml zh_ze +OpenSubtitles v2024 mn ms +OpenSubtitles v2024 mn my +OpenSubtitles v2024 mn nl +OpenSubtitles v2024 mn no +OpenSubtitles v2024 mn pl +OpenSubtitles v2024 mn pt +OpenSubtitles v2024 mn pt_BR +OpenSubtitles v2024 mn pt_MZ +OpenSubtitles v2024 mn ro +OpenSubtitles v2024 mn ru +OpenSubtitles v2024 mn si +OpenSubtitles v2024 mn sk +OpenSubtitles v2024 mn sl +OpenSubtitles v2024 mn sq +OpenSubtitles v2024 mn sr +OpenSubtitles v2024 mn sv +OpenSubtitles v2024 mn ta +OpenSubtitles v2024 mn te +OpenSubtitles v2024 mn th +OpenSubtitles v2024 mn tl +OpenSubtitles v2024 mn tr +OpenSubtitles v2024 mn tt +OpenSubtitles v2024 mn uk +OpenSubtitles v2024 mn ur +OpenSubtitles v2024 mn vi +OpenSubtitles v2024 mn yue +OpenSubtitles v2024 mn zh_CN +OpenSubtitles v2024 mn zh_TW +OpenSubtitles v2024 mn zh_ze +OpenSubtitles v2024 ms my +OpenSubtitles v2024 ms ne +OpenSubtitles v2024 ms nl +OpenSubtitles v2024 ms nn +OpenSubtitles v2024 ms no +OpenSubtitles v2024 ms oc +OpenSubtitles v2024 ms or +OpenSubtitles v2024 ms pl +OpenSubtitles v2024 ms pt +OpenSubtitles v2024 ms pt_BR +OpenSubtitles v2024 ms pt_MZ +OpenSubtitles v2024 ms ro +OpenSubtitles v2024 ms ru +OpenSubtitles v2024 ms sd +OpenSubtitles v2024 ms si +OpenSubtitles v2024 ms sk +OpenSubtitles v2024 ms sl +OpenSubtitles v2024 ms so +OpenSubtitles v2024 ms sq +OpenSubtitles v2024 ms sr +OpenSubtitles v2024 ms sv +OpenSubtitles v2024 ms sw +OpenSubtitles v2024 ms ta +OpenSubtitles v2024 ms te +OpenSubtitles v2024 ms th +OpenSubtitles v2024 ms tl +OpenSubtitles v2024 ms tr +OpenSubtitles v2024 ms tt +OpenSubtitles v2024 ms uk +OpenSubtitles v2024 ms ur +OpenSubtitles v2024 ms vi +OpenSubtitles v2024 ms yue +OpenSubtitles v2024 ms zh_CN +OpenSubtitles v2024 ms zh_TW +OpenSubtitles v2024 ms zh_ze +OpenSubtitles v2024 my ne +OpenSubtitles v2024 my nl +OpenSubtitles v2024 my no +OpenSubtitles v2024 my oc +OpenSubtitles v2024 my pl +OpenSubtitles v2024 my pt +OpenSubtitles v2024 my pt_BR +OpenSubtitles v2024 my ro +OpenSubtitles v2024 my ru +OpenSubtitles v2024 my sd +OpenSubtitles v2024 my si +OpenSubtitles v2024 my sk +OpenSubtitles v2024 my sl +OpenSubtitles v2024 my sq +OpenSubtitles v2024 my sr +OpenSubtitles v2024 my sv +OpenSubtitles v2024 my sw +OpenSubtitles v2024 my ta +OpenSubtitles v2024 my te +OpenSubtitles v2024 my th +OpenSubtitles v2024 my tl +OpenSubtitles v2024 my tr +OpenSubtitles v2024 my tt +OpenSubtitles v2024 my uk +OpenSubtitles v2024 my ur +OpenSubtitles v2024 my vi +OpenSubtitles v2024 my yue +OpenSubtitles v2024 my zh_CN +OpenSubtitles v2024 my zh_TW +OpenSubtitles v2024 my zh_ze +OpenSubtitles v2024 ne nl +OpenSubtitles v2024 ne no +OpenSubtitles v2024 ne pl +OpenSubtitles v2024 ne pt +OpenSubtitles v2024 ne pt_BR +OpenSubtitles v2024 ne ro +OpenSubtitles v2024 ne ru +OpenSubtitles v2024 ne si +OpenSubtitles v2024 ne sk +OpenSubtitles v2024 ne sl +OpenSubtitles v2024 ne sq +OpenSubtitles v2024 ne sr +OpenSubtitles v2024 ne sv +OpenSubtitles v2024 ne ta +OpenSubtitles v2024 ne th +OpenSubtitles v2024 ne tr +OpenSubtitles v2024 ne uk +OpenSubtitles v2024 ne ur +OpenSubtitles v2024 ne vi +OpenSubtitles v2024 ne zh_CN +OpenSubtitles v2024 ne zh_TW +OpenSubtitles v2024 nl nn +OpenSubtitles v2024 nl no +OpenSubtitles v2024 nl oc +OpenSubtitles v2024 nl or +OpenSubtitles v2024 nl pl +OpenSubtitles v2024 nl pt +OpenSubtitles v2024 nl pt_BR +OpenSubtitles v2024 nl pt_MZ +OpenSubtitles v2024 nl ro +OpenSubtitles v2024 nl ru +OpenSubtitles v2024 nl sd +OpenSubtitles v2024 nl si +OpenSubtitles v2024 nl sk +OpenSubtitles v2024 nl sl +OpenSubtitles v2024 nl so +OpenSubtitles v2024 nl sq +OpenSubtitles v2024 nl sr +OpenSubtitles v2024 nl sv +OpenSubtitles v2024 nl sw +OpenSubtitles v2024 nl ta +OpenSubtitles v2024 nl te +OpenSubtitles v2024 nl th +OpenSubtitles v2024 nl tl +OpenSubtitles v2024 nl tr +OpenSubtitles v2024 nl tt +OpenSubtitles v2024 nl uk +OpenSubtitles v2024 nl ur +OpenSubtitles v2024 nl vi +OpenSubtitles v2024 nl yue +OpenSubtitles v2024 nl zh_CN +OpenSubtitles v2024 nl zh_TW +OpenSubtitles v2024 nl zh_ze +OpenSubtitles v2024 nn no +OpenSubtitles v2024 nn pl +OpenSubtitles v2024 nn pt +OpenSubtitles v2024 nn pt_BR +OpenSubtitles v2024 nn ro +OpenSubtitles v2024 nn ru +OpenSubtitles v2024 nn sk +OpenSubtitles v2024 nn sl +OpenSubtitles v2024 nn sr +OpenSubtitles v2024 nn sv +OpenSubtitles v2024 nn th +OpenSubtitles v2024 nn tr +OpenSubtitles v2024 nn vi +OpenSubtitles v2024 nn zh_CN +OpenSubtitles v2024 no oc +OpenSubtitles v2024 no or +OpenSubtitles v2024 no pl +OpenSubtitles v2024 no pt +OpenSubtitles v2024 no pt_BR +OpenSubtitles v2024 no pt_MZ +OpenSubtitles v2024 no ro +OpenSubtitles v2024 no ru +OpenSubtitles v2024 no sd +OpenSubtitles v2024 no se +OpenSubtitles v2024 no si +OpenSubtitles v2024 no sk +OpenSubtitles v2024 no sl +OpenSubtitles v2024 no so +OpenSubtitles v2024 no sq +OpenSubtitles v2024 no sr +OpenSubtitles v2024 no sv +OpenSubtitles v2024 no sw +OpenSubtitles v2024 no ta +OpenSubtitles v2024 no te +OpenSubtitles v2024 no th +OpenSubtitles v2024 no tl +OpenSubtitles v2024 no tr +OpenSubtitles v2024 no tt +OpenSubtitles v2024 no uk +OpenSubtitles v2024 no ur +OpenSubtitles v2024 no vi +OpenSubtitles v2024 no yue +OpenSubtitles v2024 no zh_CN +OpenSubtitles v2024 no zh_TW +OpenSubtitles v2024 no zh_ze +OpenSubtitles v2024 oc pl +OpenSubtitles v2024 oc pt +OpenSubtitles v2024 oc pt_BR +OpenSubtitles v2024 oc ro +OpenSubtitles v2024 oc ru +OpenSubtitles v2024 oc sd +OpenSubtitles v2024 oc si +OpenSubtitles v2024 oc sk +OpenSubtitles v2024 oc sl +OpenSubtitles v2024 oc sq +OpenSubtitles v2024 oc sr +OpenSubtitles v2024 oc sv +OpenSubtitles v2024 oc ta +OpenSubtitles v2024 oc te +OpenSubtitles v2024 oc th +OpenSubtitles v2024 oc tr +OpenSubtitles v2024 oc uk +OpenSubtitles v2024 oc ur +OpenSubtitles v2024 oc vi +OpenSubtitles v2024 oc zh_CN +OpenSubtitles v2024 oc zh_TW +OpenSubtitles v2024 or pl +OpenSubtitles v2024 or pt +OpenSubtitles v2024 or pt_BR +OpenSubtitles v2024 or ro +OpenSubtitles v2024 or ru +OpenSubtitles v2024 or sk +OpenSubtitles v2024 or sr +OpenSubtitles v2024 or sv +OpenSubtitles v2024 or ta +OpenSubtitles v2024 or tr +OpenSubtitles v2024 or uk +OpenSubtitles v2024 or vi +OpenSubtitles v2024 or zh_CN +OpenSubtitles v2024 or zh_TW +OpenSubtitles v2024 pl pt +OpenSubtitles v2024 pl pt_BR +OpenSubtitles v2024 pl pt_MZ +OpenSubtitles v2024 pl ro +OpenSubtitles v2024 pl ru +OpenSubtitles v2024 pl sd +OpenSubtitles v2024 pl se +OpenSubtitles v2024 pl si +OpenSubtitles v2024 pl sk +OpenSubtitles v2024 pl sl +OpenSubtitles v2024 pl sq +OpenSubtitles v2024 pl sr +OpenSubtitles v2024 pl sv +OpenSubtitles v2024 pl sw +OpenSubtitles v2024 pl ta +OpenSubtitles v2024 pl te +OpenSubtitles v2024 pl th +OpenSubtitles v2024 pl tl +OpenSubtitles v2024 pl tr +OpenSubtitles v2024 pl tt +OpenSubtitles v2024 pl uk +OpenSubtitles v2024 pl ur +OpenSubtitles v2024 pl vi +OpenSubtitles v2024 pl yue +OpenSubtitles v2024 pl zh_CN +OpenSubtitles v2024 pl zh_TW +OpenSubtitles v2024 pl zh_ze +OpenSubtitles v2024 pt pt_BR +OpenSubtitles v2024 pt pt_MZ +OpenSubtitles v2024 pt ro +OpenSubtitles v2024 pt ru +OpenSubtitles v2024 pt sd +OpenSubtitles v2024 pt si +OpenSubtitles v2024 pt sk +OpenSubtitles v2024 pt sl +OpenSubtitles v2024 pt so +OpenSubtitles v2024 pt sq +OpenSubtitles v2024 pt sr +OpenSubtitles v2024 pt sv +OpenSubtitles v2024 pt sw +OpenSubtitles v2024 pt ta +OpenSubtitles v2024 pt te +OpenSubtitles v2024 pt th +OpenSubtitles v2024 pt tl +OpenSubtitles v2024 pt tr +OpenSubtitles v2024 pt tt +OpenSubtitles v2024 pt uk +OpenSubtitles v2024 pt ur +OpenSubtitles v2024 pt uz +OpenSubtitles v2024 pt vi +OpenSubtitles v2024 pt yue +OpenSubtitles v2024 pt zh_CN +OpenSubtitles v2024 pt zh_TW +OpenSubtitles v2024 pt zh_ze +OpenSubtitles v2024 pt_BR pt_MZ +OpenSubtitles v2024 pt_BR ro +OpenSubtitles v2024 pt_BR ru +OpenSubtitles v2024 pt_BR sd +OpenSubtitles v2024 pt_BR si +OpenSubtitles v2024 pt_BR sk +OpenSubtitles v2024 pt_BR sl +OpenSubtitles v2024 pt_BR so +OpenSubtitles v2024 pt_BR sq +OpenSubtitles v2024 pt_BR sr +OpenSubtitles v2024 pt_BR sv +OpenSubtitles v2024 pt_BR sw +OpenSubtitles v2024 pt_BR ta +OpenSubtitles v2024 pt_BR te +OpenSubtitles v2024 pt_BR th +OpenSubtitles v2024 pt_BR tl +OpenSubtitles v2024 pt_BR tr +OpenSubtitles v2024 pt_BR tt +OpenSubtitles v2024 pt_BR uk +OpenSubtitles v2024 pt_BR ur +OpenSubtitles v2024 pt_BR vi +OpenSubtitles v2024 pt_BR yue +OpenSubtitles v2024 pt_BR zh_CN +OpenSubtitles v2024 pt_BR zh_TW +OpenSubtitles v2024 pt_BR zh_ze +OpenSubtitles v2024 pt_MZ ro +OpenSubtitles v2024 pt_MZ ru +OpenSubtitles v2024 pt_MZ si +OpenSubtitles v2024 pt_MZ sk +OpenSubtitles v2024 pt_MZ sl +OpenSubtitles v2024 pt_MZ sq +OpenSubtitles v2024 pt_MZ sr +OpenSubtitles v2024 pt_MZ sv +OpenSubtitles v2024 pt_MZ th +OpenSubtitles v2024 pt_MZ tr +OpenSubtitles v2024 pt_MZ uk +OpenSubtitles v2024 pt_MZ vi +OpenSubtitles v2024 pt_MZ zh_CN +OpenSubtitles v2024 pt_MZ zh_TW +OpenSubtitles v2024 pt_MZ zh_ze +OpenSubtitles v2024 ro ru +OpenSubtitles v2024 ro sd +OpenSubtitles v2024 ro si +OpenSubtitles v2024 ro sk +OpenSubtitles v2024 ro sl +OpenSubtitles v2024 ro so +OpenSubtitles v2024 ro sq +OpenSubtitles v2024 ro sr +OpenSubtitles v2024 ro sv +OpenSubtitles v2024 ro sw +OpenSubtitles v2024 ro ta +OpenSubtitles v2024 ro te +OpenSubtitles v2024 ro th +OpenSubtitles v2024 ro tl +OpenSubtitles v2024 ro tr +OpenSubtitles v2024 ro tt +OpenSubtitles v2024 ro uk +OpenSubtitles v2024 ro ur +OpenSubtitles v2024 ro vi +OpenSubtitles v2024 ro yue +OpenSubtitles v2024 ro zh_CN +OpenSubtitles v2024 ro zh_TW +OpenSubtitles v2024 ro zh_ze +OpenSubtitles v2024 ru sd +OpenSubtitles v2024 ru si +OpenSubtitles v2024 ru sk +OpenSubtitles v2024 ru sl +OpenSubtitles v2024 ru sq +OpenSubtitles v2024 ru sr +OpenSubtitles v2024 ru sv +OpenSubtitles v2024 ru sw +OpenSubtitles v2024 ru ta +OpenSubtitles v2024 ru te +OpenSubtitles v2024 ru th +OpenSubtitles v2024 ru tl +OpenSubtitles v2024 ru tr +OpenSubtitles v2024 ru tt +OpenSubtitles v2024 ru uk +OpenSubtitles v2024 ru ur +OpenSubtitles v2024 ru vi +OpenSubtitles v2024 ru yue +OpenSubtitles v2024 ru zh_CN +OpenSubtitles v2024 ru zh_TW +OpenSubtitles v2024 ru zh_ze +OpenSubtitles v2024 sd si +OpenSubtitles v2024 sd sk +OpenSubtitles v2024 sd sl +OpenSubtitles v2024 sd sq +OpenSubtitles v2024 sd sr +OpenSubtitles v2024 sd sv +OpenSubtitles v2024 sd sw +OpenSubtitles v2024 sd te +OpenSubtitles v2024 sd th +OpenSubtitles v2024 sd tl +OpenSubtitles v2024 sd tr +OpenSubtitles v2024 sd uk +OpenSubtitles v2024 sd ur +OpenSubtitles v2024 sd vi +OpenSubtitles v2024 sd zh_CN +OpenSubtitles v2024 sd zh_TW +OpenSubtitles v2024 sd zh_ze +OpenSubtitles v2024 se sv +OpenSubtitles v2024 se tr +OpenSubtitles v2024 si sk +OpenSubtitles v2024 si sl +OpenSubtitles v2024 si so +OpenSubtitles v2024 si sq +OpenSubtitles v2024 si sr +OpenSubtitles v2024 si sv +OpenSubtitles v2024 si sw +OpenSubtitles v2024 si ta +OpenSubtitles v2024 si te +OpenSubtitles v2024 si th +OpenSubtitles v2024 si tl +OpenSubtitles v2024 si tr +OpenSubtitles v2024 si tt +OpenSubtitles v2024 si uk +OpenSubtitles v2024 si ur +OpenSubtitles v2024 si vi +OpenSubtitles v2024 si yue +OpenSubtitles v2024 si zh_CN +OpenSubtitles v2024 si zh_TW +OpenSubtitles v2024 si zh_ze +OpenSubtitles v2024 sk sl +OpenSubtitles v2024 sk so +OpenSubtitles v2024 sk sq +OpenSubtitles v2024 sk sr +OpenSubtitles v2024 sk sv +OpenSubtitles v2024 sk sw +OpenSubtitles v2024 sk ta +OpenSubtitles v2024 sk te +OpenSubtitles v2024 sk th +OpenSubtitles v2024 sk tl +OpenSubtitles v2024 sk tr +OpenSubtitles v2024 sk tt +OpenSubtitles v2024 sk uk +OpenSubtitles v2024 sk ur +OpenSubtitles v2024 sk vi +OpenSubtitles v2024 sk yue +OpenSubtitles v2024 sk zh_CN +OpenSubtitles v2024 sk zh_TW +OpenSubtitles v2024 sk zh_ze +OpenSubtitles v2024 sl sq +OpenSubtitles v2024 sl sr +OpenSubtitles v2024 sl sv +OpenSubtitles v2024 sl sw +OpenSubtitles v2024 sl ta +OpenSubtitles v2024 sl te +OpenSubtitles v2024 sl th +OpenSubtitles v2024 sl tl +OpenSubtitles v2024 sl tr +OpenSubtitles v2024 sl tt +OpenSubtitles v2024 sl uk +OpenSubtitles v2024 sl ur +OpenSubtitles v2024 sl vi +OpenSubtitles v2024 sl yue +OpenSubtitles v2024 sl zh_CN +OpenSubtitles v2024 sl zh_TW +OpenSubtitles v2024 sl zh_ze +OpenSubtitles v2024 so th +OpenSubtitles v2024 so zh_TW +OpenSubtitles v2024 sq sr +OpenSubtitles v2024 sq sv +OpenSubtitles v2024 sq sw +OpenSubtitles v2024 sq ta +OpenSubtitles v2024 sq te +OpenSubtitles v2024 sq th +OpenSubtitles v2024 sq tl +OpenSubtitles v2024 sq tr +OpenSubtitles v2024 sq tt +OpenSubtitles v2024 sq uk +OpenSubtitles v2024 sq ur +OpenSubtitles v2024 sq vi +OpenSubtitles v2024 sq yue +OpenSubtitles v2024 sq zh_CN +OpenSubtitles v2024 sq zh_TW +OpenSubtitles v2024 sq zh_ze +OpenSubtitles v2024 sr sv +OpenSubtitles v2024 sr sw +OpenSubtitles v2024 sr ta +OpenSubtitles v2024 sr te +OpenSubtitles v2024 sr th +OpenSubtitles v2024 sr tl +OpenSubtitles v2024 sr tr +OpenSubtitles v2024 sr tt +OpenSubtitles v2024 sr uk +OpenSubtitles v2024 sr ur +OpenSubtitles v2024 sr vi +OpenSubtitles v2024 sr yue +OpenSubtitles v2024 sr zh_CN +OpenSubtitles v2024 sr zh_TW +OpenSubtitles v2024 sr zh_ze +OpenSubtitles v2024 sv sw +OpenSubtitles v2024 sv ta +OpenSubtitles v2024 sv te +OpenSubtitles v2024 sv th +OpenSubtitles v2024 sv tl +OpenSubtitles v2024 sv tr +OpenSubtitles v2024 sv tt +OpenSubtitles v2024 sv uk +OpenSubtitles v2024 sv ur +OpenSubtitles v2024 sv vi +OpenSubtitles v2024 sv yue +OpenSubtitles v2024 sv zh_CN +OpenSubtitles v2024 sv zh_TW +OpenSubtitles v2024 sv zh_ze +OpenSubtitles v2024 sw ta +OpenSubtitles v2024 sw te +OpenSubtitles v2024 sw th +OpenSubtitles v2024 sw tl +OpenSubtitles v2024 sw tr +OpenSubtitles v2024 sw uk +OpenSubtitles v2024 sw ur +OpenSubtitles v2024 sw vi +OpenSubtitles v2024 sw zh_CN +OpenSubtitles v2024 sw zh_TW +OpenSubtitles v2024 ta te +OpenSubtitles v2024 ta th +OpenSubtitles v2024 ta tl +OpenSubtitles v2024 ta tr +OpenSubtitles v2024 ta tt +OpenSubtitles v2024 ta uk +OpenSubtitles v2024 ta ur +OpenSubtitles v2024 ta vi +OpenSubtitles v2024 ta yue +OpenSubtitles v2024 ta zh_CN +OpenSubtitles v2024 ta zh_TW +OpenSubtitles v2024 ta zh_ze +OpenSubtitles v2024 te th +OpenSubtitles v2024 te tl +OpenSubtitles v2024 te tr +OpenSubtitles v2024 te uk +OpenSubtitles v2024 te ur +OpenSubtitles v2024 te vi +OpenSubtitles v2024 te yue +OpenSubtitles v2024 te zh_CN +OpenSubtitles v2024 te zh_TW +OpenSubtitles v2024 te zh_ze +OpenSubtitles v2024 th tl +OpenSubtitles v2024 th tr +OpenSubtitles v2024 th tt +OpenSubtitles v2024 th uk +OpenSubtitles v2024 th ur +OpenSubtitles v2024 th vi +OpenSubtitles v2024 th yue +OpenSubtitles v2024 th zh_CN +OpenSubtitles v2024 th zh_TW +OpenSubtitles v2024 th zh_ze +OpenSubtitles v2024 tl tr +OpenSubtitles v2024 tl tt +OpenSubtitles v2024 tl uk +OpenSubtitles v2024 tl ur +OpenSubtitles v2024 tl vi +OpenSubtitles v2024 tl yue +OpenSubtitles v2024 tl zh_CN +OpenSubtitles v2024 tl zh_TW +OpenSubtitles v2024 tl zh_ze +OpenSubtitles v2024 tr tt +OpenSubtitles v2024 tr uk +OpenSubtitles v2024 tr ur +OpenSubtitles v2024 tr vi +OpenSubtitles v2024 tr yue +OpenSubtitles v2024 tr zh_CN +OpenSubtitles v2024 tr zh_TW +OpenSubtitles v2024 tr zh_ze +OpenSubtitles v2024 tt uk +OpenSubtitles v2024 tt ur +OpenSubtitles v2024 tt vi +OpenSubtitles v2024 tt yue +OpenSubtitles v2024 tt zh_CN +OpenSubtitles v2024 tt zh_TW +OpenSubtitles v2024 tt zh_ze +OpenSubtitles v2024 uk ur +OpenSubtitles v2024 uk uz +OpenSubtitles v2024 uk vi +OpenSubtitles v2024 uk yue +OpenSubtitles v2024 uk zh_CN +OpenSubtitles v2024 uk zh_TW +OpenSubtitles v2024 uk zh_ze +OpenSubtitles v2024 ur vi +OpenSubtitles v2024 ur yue +OpenSubtitles v2024 ur zh_CN +OpenSubtitles v2024 ur zh_TW +OpenSubtitles v2024 ur zh_ze +OpenSubtitles v2024 vi yue +OpenSubtitles v2024 vi zh_CN +OpenSubtitles v2024 vi zh_TW +OpenSubtitles v2024 vi zh_ze +OpenSubtitles v2024 yue zh_CN +OpenSubtitles v2024 yue zh_TW +OpenSubtitles v2024 yue zh_ze +OpenSubtitles v2024 zh_CN zh_TW +OpenSubtitles v2024 zh_CN zh_ze +OpenSubtitles v2024 zh_TW zh_ze PHP v1 cs de PHP v1 cs en PHP v1 cs es @@ -54390,6 +59107,29 @@ ParaCrawl v9 en zh ParaCrawl v9 es eu ParaCrawl v9 es gl ParaCrawl v9 fr nl +ParaCrawl-Bonus v9 az en +ParaCrawl-Bonus v9 cs pl +ParaCrawl-Bonus v9 de pl +ParaCrawl-Bonus v9 en hi +ParaCrawl-Bonus v9 en hy +ParaCrawl-Bonus v9 en id +ParaCrawl-Bonus v9 en km +ParaCrawl-Bonus v9 en ko +ParaCrawl-Bonus v9 en lo +ParaCrawl-Bonus v9 en my +ParaCrawl-Bonus v9 en ne +ParaCrawl-Bonus v9 en ps +ParaCrawl-Bonus v9 en ru +ParaCrawl-Bonus v9 en si +ParaCrawl-Bonus v9 en so +ParaCrawl-Bonus v9 en sw +ParaCrawl-Bonus v9 en tg +ParaCrawl-Bonus v9 en th +ParaCrawl-Bonus v9 en tl +ParaCrawl-Bonus v9 en uk +ParaCrawl-Bonus v9 en vi +ParaCrawl-Bonus v9 en zh +ParaCrawl-Bonus v9 fr nl QED v2.0a aa ab QED v2.0a aa ae QED v2.0a aa af From eccfefa37055db5ee9f9f477131708f6efa3fbe6 Mon Sep 17 00:00:00 2001 From: Thamme Gowda Date: Sun, 23 Mar 2025 19:55:47 -0700 Subject: [PATCH 09/13] "mtdata score" -- score QE metrics; pymarian integration (#168) * mtdata score: qe scoring via pymarian * scorer: fix minor issues * add mapper for mapping many small files with single subprocess * mtdata score: optimize using streamlining all parts * minor fixes --- CHANGELOG.md | 6 +- README.md | 9 +- mtdata/main.py | 38 +++++++- mtdata/map.py | 232 +++++++++++++++++++++++++++++++++++++++++++++++ mtdata/scorer.py | 124 +++++++++++++++++++++++++ 5 files changed, 402 insertions(+), 7 deletions(-) create mode 100644 mtdata/map.py create mode 100644 mtdata/scorer.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d71251b..ee710f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,10 @@ * Add preliminary support for huggingface datasets; currently wmt24++ is the only supported dataset * Update setup.py -> pyproject.toml; hf datasets is optional dependency * Add mtdata index subcommand. deprecate `mtdata --reindex ` -* Add meta, a dictionary field to Entry to store arbitrary key-vals which maybe useful for downloading and parsing datasets. -* Preliminary support for document id , (currently, one among the many in meta fields) +* Add a field named `meta` of type dictionary to the Entry class; stores arbitrary key-vals which maybe useful for downloading and parsing datasets. +* Support for document id , (currently, one among the many in meta fields)in `.meta.jsonl.gz` +* OPUS index updated +* `mtdata score` sub command added; support QE scoring via pymarian ## v0.4.2 diff --git a/README.md b/README.md index c1454d1..da8f3c4 100644 --- a/README.md +++ b/README.md @@ -53,12 +53,12 @@ These are the summary of datasets from various sources (Updated: Feb 2022). | Source | Dataset Count | |-------------:|--------------:| -| OPUS | 151,753| +| OPUS | 156,257| | Flores | 51,714| | Microsoft | 8,128| | Leipzig | 5,893| | Neulab | 4,455| -| Statmt | 1,784| +| Statmt | 1,798| | Facebook | 1,617| | AllenAi | 1,611| | ELRC | 1,575| @@ -69,18 +69,19 @@ These are the summary of datasets from various sources (Updated: Feb 2022). | AI4Bharath | 192| | ParaCrawl | 127| | Lindat | 56| +| Google | 55| | UN | 30| | JoshuaDec | 29| | StanfordNLP | 15| | ParIce | 8| | LangUk | 5| +| KECL | 4| | Phontron | 4| | NRC_CA | 4| -| KECL | 3| | IITB | 3| | WAT | 3| | Masakhane | 2| -| **Total** | **231,157** | +| **Total** | **235,731**| ## Usecases diff --git a/mtdata/main.py b/mtdata/main.py index 8193a52..d723539 100644 --- a/mtdata/main.py +++ b/mtdata/main.py @@ -12,7 +12,7 @@ import mtdata from mtdata import log, __version__, cache_dir as CACHE_DIR, cached_index_file from mtdata import pbar_man -from mtdata.entry import DatasetId, Langs +from mtdata.entry import DatasetId, Langs, LangPair from mtdata.utils import IO, format_byte_size DEF_N_JOBS = 1 @@ -202,6 +202,19 @@ def index_datasets(): # importing the index will recreate the index from mtdata.index import INDEX as index +def score_datasets(scorer: str, metric: str, langs: LangPair, out_dir: Path, fp16: bool=False, **kwargs): + """ + Score the datasets using the specified scorer + """ + assert scorer in ['pymarian'], f'Unknown scorer: {scorer}' + if scorer == "pymarian": + from .scorer import PyMarianScorer + marian_args = {arg: kwargs[arg] for arg in ['mini_batch', 'maxi_batch', 'workspace'] if kwargs.get(arg)} + scorer = PyMarianScorer(metric, langs=langs, quiet=not mtdata.debug_mode, fp16=fp16, **marian_args) + scorer.score_all(out_dir) + else: + raise Exception(f"scorer={scorer} is not implemented.") + class MyFormatter(argparse.ArgumentDefaultsHelpFormatter): def _split_lines(self, text, width: int): @@ -245,6 +258,8 @@ def parse_args(): "get-recipe" - Get the datasets used in the specified experiment from "list-recipe" "stats" - Get stats of dataset" "cache" - download datasets into cache +"report" - Generate a report of the datasets +"score" - score the datasets using the specified scorer ''') index_p = sub_ps.add_parser( 'index', formatter_class=MyFormatter, @@ -324,6 +339,25 @@ def add_getter_args(parser): cache_p.add_argument('-di', '--dataset-id', type=DatasetId.parse, nargs='*', help='Dataset ID') cache_p.add_argument('-j', '--n-jobs', type=int, help="Number of worker jobs (processes)", default=DEF_N_JOBS) + score_p = sub_ps.add_parser('score', formatter_class=MyFormatter + , help="Score the datasets using the specified scorer") + score_p.add_argument('-s', '--scorer', type=str, choices=["pymarian"], default="pymarian", help='Scorer name') + score_p.add_argument('-m', '--metric', type=str, default="wmt22-cometkiwi-da", + help='QE metric name. See scorer help for a list of supported metrics. \ + For pymarian supported QE metrics are: wmt20-comet-qe-da, wmt22-cometkiwi-da, wmt23-cometkiwi-da-xl, wmt23-cometkiwi-da-xxl') + score_p.add_argument('-fp16', '--fp16', action='store_true', default=False, + help='Use fp16 model for faster scoring') + score_p.add_argument("-l", "--langs", type=Langs, required=True, + help="Language pair") + score_p.add_argument("-mn", "--mini-batch", type=int, default=16, + help="mini batch size") + #score_p.add_argument('-mx', '--maxi-batch', type=int, default=1000, + # help="Maxi batch size") + score_p.add_argument("-ws", "--workspace", type=int, default=-8000, + help="Workspace memory for pymarian. Recommended: Total VRAM - memory for model") + score_p.add_argument('-o', '--out', dest='out_dir', type=Path, required=True, + help='Output directory name; this should be the output of "get"/"get-recipe" command') + args = p.parse_args() if args.log_level: log_level = log._nameToLevel[args.log_level] @@ -366,6 +400,8 @@ def main(): elif args.task == 'cache': assert args.recipe_id or args.dataset_id, "Need at least one of --recipe-id or --dataset-id" cache_datasets(recipes=args.recipe_id, dids=args.dataset_id, n_jobs=args.n_jobs) + elif args.task == 'score': + score_datasets(**vars(args)) else: raise Exception(f'task={args.task} not implemented') diff --git a/mtdata/map.py b/mtdata/map.py new file mode 100644 index 0000000..5cc3836 --- /dev/null +++ b/mtdata/map.py @@ -0,0 +1,232 @@ +import argparse +from pathlib import Path +from typing import List, Tuple, Union, Iterator +from itertools import zip_longest + +import subprocess as sp +import multiprocessing as mp +import threading as mt +import sys +import queue + +from mtdata import log +from mtdata.utils import IO + + +DELIM = '\t' +SENTINEL = None + + +def read_paths(paths: Iterator[List[Path]]) -> Iterator[Union[dict,list]]: + """ + Reads streams of data from the given paths + + :paths: stream of list of paths. each item should have atleast two paths where first is input and last is output. If more than 2 paths are given, the first n-1 paths are "pasted" to form TSV records + """ + n_data = 0 + n_ctrls = 0 + counter = -1 + for ps in paths: + try: + assert len(ps) >= 2, f"Expected two or more paths" + ps = [Path(p) for p in ps] + inps = ps[:-1] + outp = ps[-1] + yield dict(inputs=inps, output=outp, last_count=counter) # control record + n_ctrls += 1 + counter = 0 + #assert len(inps) == 1, f'Currently only single input is suppprted. TODO: support pasting' + streams = [IO.get_lines(p, col=-1) for p in inps] + for rec in zip_longest(*streams): + if len(inps) > 1 and any(x is None for x in rec): + raise ValueError(f"Unequal number of lines detected in {inps} @ count: {counter}") + rec = '\t'.join(x.strip().replace('\t', ' ') for x in rec) + yield rec + counter += 1 + n_data += counter + log.info(f"Producer: end of {inps}; count: {counter}") + except Exception as e: + log.exception(f"Producer: error in {inps}: {e}") + log.info(f"Producer: finishing... n_ctrls: {n_ctrls}; n_data: {n_data:,}") + + +class SubprocMapper: + + def __init__(self, cmdline: str, max_qsize=1024*1024, shell=True): + self.cmdline = cmdline + self._subproc_args = dict(shell=shell) + self.ctrl_queue = None + self.data_queue = None + self.proc = None + self._started = False + self.max_qsize = max_qsize + self._stop_event = mt.Event() + + def start(self): + assert not self._started, f'Already started' + self.ctrl_queue = mp.Queue(maxsize=self.max_qsize) + self.data_queue = mp.Queue(maxsize=self.max_qsize) + log.info(f"RUN:\n\t{self.cmdline}") + self.proc = sp.Popen(self.cmdline, stdin=sp.PIPE, stdout=sp.PIPE, text=True, **self._subproc_args) + self._stop_event.clear() + + def stdin_writer(self, stream: Iterator): + log.info("STDIN Writer: starting...") + n_ctrls = 0 + n_data = 0 + try: + for rec in stream: + if self._stop_event.is_set(): + break + if isinstance(rec, dict): # control record + self.ctrl_queue.put(rec) + n_ctrls += 1 + else: # data record + assert isinstance(rec, str), f"string expected, got {type(rec)}" + self.proc.stdin.write(rec + '\n') + n_data += 1 + except Exception as e: + log.error(f"STDIN Writer encountered an error: {e}") + self._stop_event.set() + finally: + log.info(f"STDIN Writer: finishing... n_ctrls: {n_ctrls:,}; n_data: {n_data:,}") + self.proc.stdin.close() + self.ctrl_queue.put(SENTINEL) + + def stdout_reader(self): + log.info("STDOUT reader: starting...") + i = 0 + try: + for line in self.proc.stdout: + if self._stop_event.is_set(): + break + self.data_queue.put(line.rstrip('\n')) + i += 1 + except Exception as e: + log.error(f"STDOUT Reader encountered an error: {e}") + self._stop_event.set() + finally: + log.info(f"STDOUT reader: finishing... n_data: {i:,}") + self.data_queue.put(SENTINEL) + + def iterator(self) -> Iterator: + log.info("Data and ctrl merger: starting...") + yield self.ctrl_queue.get() # the first ctrl msg; last_count=0 + n_ctrls = 1 + n_data = 0 + + ctrl_msg = None + line_count = 0 + try: + while not self._stop_event.is_set(): + if ctrl_msg is None and not self.ctrl_queue.empty(): + ctrl_msg = self.ctrl_queue.get() + if ctrl_msg is not None: + if line_count == ctrl_msg['last_count']: + yield ctrl_msg + n_ctrls += 1 + line_count = 0 + ctrl_msg = None + continue + else: + # line_count should not go beyond file + assert line_count < ctrl_msg['last_count'] + + rec = self.data_queue.get() + if rec is SENTINEL: # end of queue + break + yield rec + line_count += 1 + n_data += 1 + except Exception as e: + log.error(f"Iterator encountered an error: {e}") + self._stop_event.set() + finally: + log.info(f"Data and ctrl merger: finishing... n_ctrls: {n_ctrls:,}; n_data: {n_data:,}") + + def map(self, stream: Iterator) -> Iterator: + try: + self.start() + writer_t = mt.Thread(target=self.stdin_writer, args=(stream,)) + writer_t.start() + reader_t = mt.Thread(target=self.stdout_reader) + reader_t.start() + yield from self.iterator() + writer_t.join(timeout=10) + reader_t.join(timeout=10) + assert self.ctrl_queue.empty(), 'ctrl_queue is not empty' + assert self.data_queue.empty(), 'data_queue is not empty' + except Exception as e: + log.error(f"Mapper encountered an error: {e}") + self._stop_event.set() + raise + finally: + self.close() + + def __call__(self, stream: Iterator) -> Iterator: + return self.map(stream) + + def close(self): + self._stop_event.set() + if self.proc and self.proc.poll() is None: + log.info(f"terminating subprocess {self.proc.pid}") + self.proc.terminate() + + +def read_input_paths(input, delim=DELIM): + for line in input: + parts = line.rstrip("\n").split(delim) + parts = [Path(p) for p in parts] + yield parts + + +def trim_stream(stream, skip=0, limit=0): + assert skip > 0 or limit > 0 + i = 0 + for rec in stream: + if skip > 0: + if i > 0: # assumption: i == 0 is a ctrl msg, dont skip it + skip -= 1 + continue + yield rec + i += 1 + if limit > 0 and i == limit: + break + + +def main(): + args = vars(parse_args()) + paths = read_input_paths(args['input'], delim=args['delim']) + stream = read_paths(paths) + + n_skip = args.get('skip', 0) + n_limit = args.get('limit', 0) + if n_skip > 0 or n_limit > 0: + log.warning(f"Trimming the stream: skip: {n_skip}; limit:{n_limit}") + stream = trim_stream(stream, skip=n_skip, limit=n_limit) + + mapper = SubprocMapper(cmdline=args['cmdline']) + try: + out_stream = mapper(stream) + for rec in out_stream: + print(rec) + except Exception as e: + mapper.close() + raise + + +def parse_args(): + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('-c', '--cmd', dest='cmdline', type=str, required=True, + help="Mapper command that maps line-by-line, maintains 1:1 mapping and the input order. For example: 'cat'") + parser.add_argument('-i', '--input', type=argparse.FileType('r'), default=sys.stdin, + help="Listing file containing file paths. Atleast two paths per line is expected first one is input and last one is output") + parser.add_argument('-d', '--delim', type=str, default=DELIM, help="delimiter for paths in input") + parser.add_argument('-l', '--limit', type=int, default=0, + help="Limit data stream to these many lines. Score: for debugging and testing") + parser.add_argument('-s', '--skip', type=int, default=0, + help="Skip the first n records. Scope: for debugging and testing") + return parser.parse_args() + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/mtdata/scorer.py b/mtdata/scorer.py new file mode 100644 index 0000000..cc2f6bb --- /dev/null +++ b/mtdata/scorer.py @@ -0,0 +1,124 @@ +from pathlib import Path +from typing import List, Iterator, Tuple +from itertools import zip_longest +import subprocess as sp +import os + +from pymarian import Evaluator +from pymarian.utils import get_model_path, get_vocab_path + + +from . import log, pbar_man, Defaults +from .entry import LangPair +from .utils import IO +from .map import SubprocMapper, read_paths + + +class LocalDataset: + + def __init__(self, root, langs:LangPair, compress=True): + self.langs = langs + self.root = Path(root) + self.compress = compress + assert self.langs is not None, "Language pair must be specified" + assert self.root.exists(), f"Dataset root {self.root} does not exist" + assert self.root.is_dir(), f"Dataset root {self.root} is not a directory" + + def list_parallel_parts(self, subdir="train-parts") -> List[Tuple[Path, Path]]: + """ + List the parallel parts of the dataset. + """ + # * is added to match script or country tags + src_ext = f'{self.langs[0]}*' + (self.compress and ".gz" or "") + tgt_ext = f'{self.langs[1]}*' + (self.compress and ".gz" or "") + parts = [] + for src_file in self.root.glob(f'{subdir}/*.{src_ext}'): + n_exts_parts = len(src_ext.split('.')) + did = '.'.join(src_file.name.split('.')[:-n_exts_parts]) + tgt_files = list(src_file.parent.glob(f'{did}.{tgt_ext}')) + if len(tgt_files) != 1: + log.warning(f"Expected 1 target file for {src_file.parent}/{did}.{tgt_ext}, found {len(tgt_files)}") + tgt_file = tgt_files[0] if len(tgt_files) == 1 else None + parts.append((did, src_file, tgt_file)) + if len(parts) == 0: + log.warning(f"No parallel parts found in {subdir} for {self.langs}") + else: + log.info(f"Found {len(parts)} parallel parts in {subdir} for {self.langs}") + return parts + + +class PyMarianScorer(): + + def __init__(self, metric:str, langs:Tuple[str,str], quiet=False, fp16=False, **kwargs): + self.metric = metric + self.langs = langs + self.quiet = quiet + self.fp16 = fp16 + self.eval_args = kwargs + + def score_all(self, work_dir: Path): + """ + Score the translations in the work_dir using the specified metric. + """ + log.info(f'Scoring {work_dir} with {self.metric}') + dataset = LocalDataset(work_dir, self.langs) + parts = dataset.list_parallel_parts() + fp16 = self.fp16 and ".fp16" or "" + log.info(f'Found {len(parts)} parts') + all_paths = [] + for part_num, (did, src_file, tgt_file) in enumerate(parts): + out_file = src_file.parent / f'{did}.{self.metric}.score{fp16}.gz' + if out_file.exists() and out_file.stat().st_size > 0: + log.info(f'Skipping {src_file.name} {tgt_file.name} (already scored)') + continue + all_paths.append((src_file, tgt_file, out_file)) + log.info(f"Found {len(all_paths)}") + if not all_paths: + log.warning("No parts to score; skipping") + return + cmdline = f"pymarian-eval --stdin -m {self.metric} -a skip" + if not self.quiet: + cmdline += " --debug" + if self.fp16: + cmdline += " --fp16" + for k,v in self.eval_args.items(): + k = k.replace('_', '-') + cmdline += f" --{k} {v}" + if os.getenv('PYMARIAN_CACHE'): + cmdline += f" --cache {os.getenv('PYMARIAN_CACHE')}" + + # get internal command which is purely c++ and a bit more efficient than python wrapper + cmdline = sp.check_output(cmdline + " --print-cmd", text=True, shell=True).strip() + if cmdline.startswith("marian "): + # older version used "marian evaluate", make it "pymarian evaluate" + cmdline = f"py{cmdline}" + stream = read_paths(all_paths) + mapper = SubprocMapper(cmdline=cmdline) + try: + out_stream = mapper(stream) + out_path = None + tmp_path = None + writer = None + desc = f'Scoring {len(all_paths)} parts with {self.metric}' + with pbar_man.counter(unit='it', desc=desc, min_delta=Defaults.PBAR_REFRESH_INTERVAL, + autorefresh=True) as pbar: + for rec in out_stream: + if isinstance(rec, dict): + if writer is not None: + writer.close() # close the previous writer + tmp_path.rename(out_path) # rename the tmp file to the final output file + log.info(f"Renamed {tmp_path} to {out_path}") + out_path = rec['output'] + tmp_path = out_path.with_name('.tmp.' + out_path.name) + log.info(f"Writing to {tmp_path}") + tmp_path.unlink(missing_ok=True) # remove the tmp file if it exists + tmp_path.parent.mkdir(parents=True, exist_ok=True) + writer = IO.writer(tmp_path).__enter__() + else: + writer.write(rec + '\n') + pbar.update(1) + if writer is not None: + writer.close() + tmp_path.rename(out_path) # rename the tmp file to the final output file + finally: + mapper.close() From 5cf8fbe5ab8e9a15c9a5fdfb3f2cd0075293a1b0 Mon Sep 17 00:00:00 2001 From: Thamme Gowda Date: Wed, 26 Mar 2025 19:06:20 -0700 Subject: [PATCH 10/13] Fix: NTREX paths inside tarball: _ -> -; improve error message --- mtdata/index/other.py | 7 +++++-- mtdata/utils.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mtdata/index/other.py b/mtdata/index/other.py index 650e8ea..006906b 100644 --- a/mtdata/index/other.py +++ b/mtdata/index/other.py @@ -134,10 +134,13 @@ def load_all(index: Index): " swe tah tam tat tel tgk_Cyrl tha tir ton tsn tuk tur uig ukr urd uzb ven vie wol xho yor" " yue zho_CN zho_TW zul").split() # NOTE: fij tgk_Cyrl urd vie wol zul had extra lines for l2 in ntrex128_langs: + l2_ext = l2.replace('_', "-") index += Entry(did=f"Microsoft-ntrex-128-eng-{l2}", url=_url, filename="NTREX-52b9c57c.tar.gz", - in_ext='txt', in_paths=["*/NTREX-128/newstest2019-src.eng.txt", f"*/NTREX-128/newstest2019-ref.{l2}.txt"]) + in_ext='txt', in_paths=["*/NTREX-128/newstest2019-src.eng.txt", f"*/NTREX-128/newstest2019-ref.{l2_ext}.txt"]) for i, l1 in enumerate(ntrex128_langs): + l1_ext = l1.replace('_', "-") for l2 in ntrex128_langs[i+1:]: + l2_ext = l2.replace('_', "-") index += Entry(did=f"Microsoft-ntrex-128-{l1}-{l2}", url=_url, filename="NTREX-52b9c57c.tar.gz", - in_ext='txt', in_paths=[f"*/NTREX-128/newstest2019-ref.{l1}.txt", f"*/NTREX-128/newstest2019-ref.{l2}.txt"]) + in_ext='txt', in_paths=[f"*/NTREX-128/newstest2019-ref.{l1_ext}.txt", f"*/NTREX-128/newstest2019-ref.{l2_ext}.txt"]) diff --git a/mtdata/utils.py b/mtdata/utils.py index 8af5274..d425be0 100644 --- a/mtdata/utils.py +++ b/mtdata/utils.py @@ -181,7 +181,7 @@ def __post_init__(self): self.ext_dir = self.extract() matches = list(self.ext_dir.glob(self.name)) if len(matches) != 1: - raise Exception(f'expected to find exactly one path inside tarball, but found {matches}') + raise Exception(f'expected to find exactly one path inside tarball @ {self.ext_dir}/{self.name}, but found {matches}') self.child = matches[0] self.open = self.child.open From c18c6882ac909be677204bdaef6ed8c092e799a9 Mon Sep 17 00:00:00 2001 From: Thamme Gowda Date: Sun, 30 Mar 2025 21:44:21 -0700 Subject: [PATCH 11/13] Simnplify scorer interface. support any subproces command --- README.md | 19 ++++++++++++++++++ mtdata/main.py | 48 +++++++++++++++++++++++---------------------- mtdata/map.py | 15 +++++++++++++- mtdata/scorer.py | 51 ++++++++---------------------------------------- 4 files changed, 66 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index da8f3c4..947cad6 100644 --- a/README.md +++ b/README.md @@ -279,6 +279,25 @@ mtdata list-recipe # see all recipes mtdata get-recipe -ri -o # get recipe, recreate dataset ``` +## QE Scoring + +> Since v0.4.3 (WMT25) + +We support scoring parallel segments using any (QE) metric using a subprocess invocation. +The command for subprocess has to satisfy the following three assumptions +1. STDIN->STDOUT mapping such that reads `source\ttarget` lines from STDIN and prints score to stdout +2. 1:1 mapping. i.e. number of output lines match the number of input lines +3. Preserves input order + +Here is an example with `pymarian` + +```bash +pip install pymarian +metric="wmt22-cometkiwi-da" +cmd="pymarian-eval --stdin --fields src mt --workspace -8000 --model wmt22-cometkiwi-da --mini-batch 64" +python -m mtdata score -l eng-isl -o wmt25-eng-isl -c "$cmd" -n "$metric" +``` + ## Language Name Standardization ### ISO 639 3 Internally, all language codes are mapped to ISO-639 3 codes. diff --git a/mtdata/main.py b/mtdata/main.py index d723539..be7f46f 100644 --- a/mtdata/main.py +++ b/mtdata/main.py @@ -8,6 +8,7 @@ from typing import List, Dict, Mapping, Tuple, Optional import json import fnmatch +import subprocess as sp import mtdata from mtdata import log, __version__, cache_dir as CACHE_DIR, cached_index_file @@ -202,18 +203,22 @@ def index_datasets(): # importing the index will recreate the index from mtdata.index import INDEX as index -def score_datasets(scorer: str, metric: str, langs: LangPair, out_dir: Path, fp16: bool=False, **kwargs): +def score_datasets(cmd: str, langs: LangPair, out_dir: Path, metric_name: str): """ Score the datasets using the specified scorer """ - assert scorer in ['pymarian'], f'Unknown scorer: {scorer}' - if scorer == "pymarian": - from .scorer import PyMarianScorer - marian_args = {arg: kwargs[arg] for arg in ['mini_batch', 'maxi_batch', 'workspace'] if kwargs.get(arg)} - scorer = PyMarianScorer(metric, langs=langs, quiet=not mtdata.debug_mode, fp16=fp16, **marian_args) - scorer.score_all(out_dir) - else: - raise Exception(f"scorer={scorer} is not implemented.") + optimize_pymarian = True + # get an optimized command which is a bit more efficient than the python wrapper + if optimize_pymarian and cmd.startswith("pymarian-eval "): + optim_cmd = sp.check_output(cmd + " --print-cmd", text=True, shell=True).strip() + if optim_cmd.startswith("marian "): + # older version used "marian evaluate", make it "pymarian evaluate" + optim_cmd = f"py{optim_cmd}" + cmd = optim_cmd + from .scorer import LocalDataset + dataset = LocalDataset(out_dir, langs) + dataset.score_all(cmd, metric_name) + class MyFormatter(argparse.ArgumentDefaultsHelpFormatter): @@ -341,20 +346,16 @@ def add_getter_args(parser): score_p = sub_ps.add_parser('score', formatter_class=MyFormatter , help="Score the datasets using the specified scorer") - score_p.add_argument('-s', '--scorer', type=str, choices=["pymarian"], default="pymarian", help='Scorer name') - score_p.add_argument('-m', '--metric', type=str, default="wmt22-cometkiwi-da", - help='QE metric name. See scorer help for a list of supported metrics. \ - For pymarian supported QE metrics are: wmt20-comet-qe-da, wmt22-cometkiwi-da, wmt23-cometkiwi-da-xl, wmt23-cometkiwi-da-xxl') - score_p.add_argument('-fp16', '--fp16', action='store_true', default=False, - help='Use fp16 model for faster scoring') score_p.add_argument("-l", "--langs", type=Langs, required=True, - help="Language pair") - score_p.add_argument("-mn", "--mini-batch", type=int, default=16, - help="mini batch size") - #score_p.add_argument('-mx', '--maxi-batch', type=int, default=1000, - # help="Maxi batch size") - score_p.add_argument("-ws", "--workspace", type=int, default=-8000, - help="Workspace memory for pymarian. Recommended: Total VRAM - memory for model") + help="Language pair. Used to correctly identify source and target files.") + + DEF_SCORER_METRIC = "wmt22-cometkiwi-da" + DEF_SCORE_CMD = f"pymarian-eval --stdin --fields src mt --workspace -8000 --model {DEF_SCORER_METRIC} --mini-batch 64" + score_p.add_argument('-c', '--cmd', default=DEF_SCORE_CMD, help='Scorer command. Assumptions: \ + (1). STDIN->STDOUT where each line in STDIN has sourcetarget, and the corresponding score is written to STDOUT. \ + (2). Maintains input order (3). 1:1 mapping.') + score_p.add_argument('-n', '--name', dest='metric_name', default=DEF_SCORER_METRIC, + help='Metric name which will be added to all files') score_p.add_argument('-o', '--out', dest='out_dir', type=Path, required=True, help='Output directory name; this should be the output of "get"/"get-recipe" command') @@ -401,7 +402,8 @@ def main(): assert args.recipe_id or args.dataset_id, "Need at least one of --recipe-id or --dataset-id" cache_datasets(recipes=args.recipe_id, dids=args.dataset_id, n_jobs=args.n_jobs) elif args.task == 'score': - score_datasets(**vars(args)) + score_datasets(cmd=args.cmd, langs=args.langs, out_dir=args.out_dir, + metric_name=args.metric_name) else: raise Exception(f'task={args.task} not implemented') diff --git a/mtdata/map.py b/mtdata/map.py index 5cc3836..3297a64 100644 --- a/mtdata/map.py +++ b/mtdata/map.py @@ -1,3 +1,13 @@ +#!/usr/bin/env python +# +# Created by TG on March 2025 +# +# This is util help map a bunch of small files using a single subproc, +# under a few assumptions +# maps line-by-line STDIN->STDOUT, maintains 1:1 mapping and the input order. +# This util was created to optimize time for scoring many small files using +# a subprocess that takes much time to intialize (e.g. loading a large QE metric model) + import argparse from pathlib import Path from typing import List, Tuple, Union, Iterator @@ -7,7 +17,6 @@ import multiprocessing as mp import threading as mt import sys -import queue from mtdata import log from mtdata.utils import IO @@ -172,6 +181,10 @@ def close(self): log.info(f"terminating subprocess {self.proc.pid}") self.proc.terminate() + @classmethod + def read_stream(cls, paths: Iterator[List[Path]]) -> Iterator[Union[dict,list]]: + return read_paths(paths) + def read_input_paths(input, delim=DELIM): for line in input: diff --git a/mtdata/scorer.py b/mtdata/scorer.py index cc2f6bb..ba6f13e 100644 --- a/mtdata/scorer.py +++ b/mtdata/scorer.py @@ -1,17 +1,10 @@ from pathlib import Path -from typing import List, Iterator, Tuple -from itertools import zip_longest -import subprocess as sp -import os - -from pymarian import Evaluator -from pymarian.utils import get_model_path, get_vocab_path - +from typing import List, Tuple from . import log, pbar_man, Defaults from .entry import LangPair from .utils import IO -from .map import SubprocMapper, read_paths +from .map import SubprocMapper class LocalDataset: @@ -46,28 +39,16 @@ def list_parallel_parts(self, subdir="train-parts") -> List[Tuple[Path, Path]]: log.info(f"Found {len(parts)} parallel parts in {subdir} for {self.langs}") return parts - -class PyMarianScorer(): - - def __init__(self, metric:str, langs:Tuple[str,str], quiet=False, fp16=False, **kwargs): - self.metric = metric - self.langs = langs - self.quiet = quiet - self.fp16 = fp16 - self.eval_args = kwargs - - def score_all(self, work_dir: Path): + def score_all(self, cmdline, metric_name:str): """ Score the translations in the work_dir using the specified metric. """ - log.info(f'Scoring {work_dir} with {self.metric}') - dataset = LocalDataset(work_dir, self.langs) - parts = dataset.list_parallel_parts() - fp16 = self.fp16 and ".fp16" or "" + log.info(f'Scoring {self.root} with {metric_name}') + parts = self.list_parallel_parts() log.info(f'Found {len(parts)} parts') all_paths = [] for part_num, (did, src_file, tgt_file) in enumerate(parts): - out_file = src_file.parent / f'{did}.{self.metric}.score{fp16}.gz' + out_file = src_file.parent / f'{did}.{metric_name}.score.gz' if out_file.exists() and out_file.stat().st_size > 0: log.info(f'Skipping {src_file.name} {tgt_file.name} (already scored)') continue @@ -76,30 +57,14 @@ def score_all(self, work_dir: Path): if not all_paths: log.warning("No parts to score; skipping") return - cmdline = f"pymarian-eval --stdin -m {self.metric} -a skip" - if not self.quiet: - cmdline += " --debug" - if self.fp16: - cmdline += " --fp16" - for k,v in self.eval_args.items(): - k = k.replace('_', '-') - cmdline += f" --{k} {v}" - if os.getenv('PYMARIAN_CACHE'): - cmdline += f" --cache {os.getenv('PYMARIAN_CACHE')}" - - # get internal command which is purely c++ and a bit more efficient than python wrapper - cmdline = sp.check_output(cmdline + " --print-cmd", text=True, shell=True).strip() - if cmdline.startswith("marian "): - # older version used "marian evaluate", make it "pymarian evaluate" - cmdline = f"py{cmdline}" - stream = read_paths(all_paths) mapper = SubprocMapper(cmdline=cmdline) try: + stream = mapper.read_stream(all_paths) out_stream = mapper(stream) out_path = None tmp_path = None writer = None - desc = f'Scoring {len(all_paths)} parts with {self.metric}' + desc = f'Scoring {len(all_paths)} parts with {metric_name}' with pbar_man.counter(unit='it', desc=desc, min_delta=Defaults.PBAR_REFRESH_INTERVAL, autorefresh=True) as pbar: for rec in out_stream: From c37f511098c1ff02c8456ce6450214cd5210ebe0 Mon Sep 17 00:00:00 2001 From: Thamme Gowda Date: Sun, 30 Mar 2025 21:57:31 -0700 Subject: [PATCH 12/13] Add MTDataUserError; simplify error messages --- mtdata/__init__.py | 12 ++++++++ mtdata/data.py | 4 +-- mtdata/main.py | 73 +++++++++++++++++++++++++--------------------- 3 files changed, 53 insertions(+), 36 deletions(-) diff --git a/mtdata/__init__.py b/mtdata/__init__.py index 6c1ed92..eecedf6 100644 --- a/mtdata/__init__.py +++ b/mtdata/__init__.py @@ -27,6 +27,18 @@ class MTDataException(Exception): pass + +class MTDataUserError(MTDataException): + """ + This exception is for the cases where printing the whole stack trace is bad UI + and we want to show a user-friendly message. https://github.com/thammegowda/mtdata/issues/162 + """ + def __init__(self, msg, exitcode=1, *args): + super().__init__(*args) + self.msg = msg + self.exitcode = exitcode + + class Defaults: FILE_LOCK_TIMEOUT = 2 * 60 * 60 # 2 hours PBAR_REFRESH_INTERVAL = 1 # seconds \ No newline at end of file diff --git a/mtdata/data.py b/mtdata/data.py index 144f96c..b70aab7 100644 --- a/mtdata/data.py +++ b/mtdata/data.py @@ -12,7 +12,7 @@ import portalocker -from mtdata import Defaults +from mtdata import Defaults, MTDataUserError from mtdata import cache_dir as CACHE_DIR from mtdata import log, pbar_man from mtdata.cache import Cache @@ -116,7 +116,7 @@ def prepare(cls, langs, out_dir: Path, dataset_ids=Dict[str, List[DatasetId]], all_entries = cls.resolve_entries(all_dids) for ent in all_entries: if not ent.is_compatible(langs): - raise Exception(f'Given languages: {langs} and dataset: {ent.did} are not compatible') + raise MTDataUserError(f'Dataset {ent.did} is not compatible for {"-".join(map(str, langs))}') if n_jobs > 1: cls.parallel_download(all_entries, Cache(cache_dir), n_jobs=n_jobs) diff --git a/mtdata/main.py b/mtdata/main.py index be7f46f..ab48a4e 100644 --- a/mtdata/main.py +++ b/mtdata/main.py @@ -9,6 +9,7 @@ import json import fnmatch import subprocess as sp +import sys import mtdata from mtdata import log, __version__, cache_dir as CACHE_DIR, cached_index_file @@ -41,7 +42,7 @@ def get_data(langs, out_dir, merge_train=False, compress=False, if kwargs.get(old_name): dataset_ids[name] = kwargs.pop(old_name) if kwargs: - log.info(f"Args are ignored: {kwargs}") + log.debug(f"Args are ignored: {kwargs}") assert any(bool(ids) for ids in dataset_ids.values()),\ f'Required at least one of --train --test --dev --mono-train --mono-test --mono-dev \n given={dataset_ids.keys()}' dataset = Dataset.prepare( @@ -139,7 +140,7 @@ def get_recipe(recipe_id, out_dir: Path, compress=False, drop_dupes=False, drop_tests=False, fail_on_error=False, n_jobs=DEF_N_JOBS, merge_train=True, **kwargs): if kwargs: - log.warning(f"Args are ignored: {kwargs}") + log.debug(f"Args are ignored: {kwargs}") from mtdata.recipe import RECIPES recipe = RECIPES.get(recipe_id) if not recipe: @@ -374,38 +375,42 @@ def add_getter_args(parser): def main(): - args = parse_args() - if args.reindex and cached_index_file.exists(): - log.warning(f"--reindex flag is deprecated and will be removed in the future. Use 'index' subcommand instead") - index_datasets() - - if args.task == 'index': - index_datasets() - elif args.task == 'list': - list_data(args.langs, args.names, not_names=args.not_names, full=args.full, - groups=args.groups, not_groups=args.not_groups, id_only=args.id) - elif args.task == 'get': - get_data(**vars(args)) - elif args.task == 'echo': - # disable progress bar for echo; it sometimes insert new lines in the output - pbar_man.enabled = False - echo_data(did=args.dataset_id) - elif args.task == 'list-recipe': - list_recipes(id_only=args.id, format=args.format) - elif args.task == 'get-recipe': - get_recipe(**vars(args)) - elif args.task == 'stats': - show_stats(*args.did, quick=args.quick) - elif args.task == 'report': - generate_report(args.langs, names=args.names, not_names=args.not_names) - elif args.task == 'cache': - assert args.recipe_id or args.dataset_id, "Need at least one of --recipe-id or --dataset-id" - cache_datasets(recipes=args.recipe_id, dids=args.dataset_id, n_jobs=args.n_jobs) - elif args.task == 'score': - score_datasets(cmd=args.cmd, langs=args.langs, out_dir=args.out_dir, - metric_name=args.metric_name) - else: - raise Exception(f'task={args.task} not implemented') + try: + args = parse_args() + if args.reindex and cached_index_file.exists(): + log.warning(f"--reindex flag is deprecated and will be removed in the future. Use 'index' subcommand instead") + index_datasets() + + if args.task == 'index': + index_datasets() + elif args.task == 'list': + list_data(args.langs, args.names, not_names=args.not_names, full=args.full, + groups=args.groups, not_groups=args.not_groups, id_only=args.id) + elif args.task == 'get': + get_data(**vars(args)) + elif args.task == 'echo': + # disable progress bar for echo; it sometimes insert new lines in the output + pbar_man.enabled = False + echo_data(did=args.dataset_id) + elif args.task == 'list-recipe': + list_recipes(id_only=args.id, format=args.format) + elif args.task == 'get-recipe': + get_recipe(**vars(args)) + elif args.task == 'stats': + show_stats(*args.did, quick=args.quick) + elif args.task == 'report': + generate_report(args.langs, names=args.names, not_names=args.not_names) + elif args.task == 'cache': + assert args.recipe_id or args.dataset_id, "Need at least one of --recipe-id or --dataset-id" + cache_datasets(recipes=args.recipe_id, dids=args.dataset_id, n_jobs=args.n_jobs) + elif args.task == 'score': + score_datasets(cmd=args.cmd, langs=args.langs, out_dir=args.out_dir, + metric_name=args.metric_name) + else: + raise Exception(f'task={args.task} not implemented') + except mtdata.MTDataUserError as e: + print("Error: " + e.msg, file=sys.stderr) + sys.exit(e.exitcode) if __name__ == '__main__': From 454654f922d4f96b7b2e240f7f9f2558f8decea7 Mon Sep 17 00:00:00 2001 From: Thamme Gowda Date: Sun, 30 Mar 2025 22:03:25 -0700 Subject: [PATCH 13/13] version 0.4.3 --- CHANGELOG.md | 2 +- mtdata/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee710f5..87f0d76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log -## 0.4.3 - WIP +## 0.4.3 - 20250330 * Add preliminary support for huggingface datasets; currently wmt24++ is the only supported dataset * Update setup.py -> pyproject.toml; hf datasets is optional dependency * Add mtdata index subcommand. deprecate `mtdata --reindex ` diff --git a/mtdata/__init__.py b/mtdata/__init__.py index eecedf6..4e7cf93 100644 --- a/mtdata/__init__.py +++ b/mtdata/__init__.py @@ -4,7 +4,7 @@ # Created: 4/4/20 -__version__ = '0.4.3-dev' +__version__ = '0.4.3' __description__ = 'mtdata is a tool to download datasets for machine translation' __author__ = 'Thamme Gowda'