Merge branch 'master' into sc-translate

README.md

@@ -15,6 +15,8 @@ Language pair | SC model BLEU | SC Flores200 BLEU | Google BLEU | Meta NLLB200 B
 |Catalan-German | 28.5 |25.4 |32.9 |29.1|15.8| 3142257 | [cat-deu-2022-11-16.zip](https://www.softcatala.org/pub/softcatala/opennmt/models/2022-11-22/cat-deu-2022-11-16.zip)
 |English-Catalan | 46.9 |43.8 |46.0 |41.7|29.8| 7856208 | [eng-cat-2023-10-30.zip](https://www.softcatala.org/pub/softcatala/opennmt/models/2022-11-22/eng-cat-2023-10-30.zip)
 |Catalan-English | 47.4 |43.5 |47.0 |48.0|29.6| 7856208 | [cat-eng-2023-10-29.zip](https://www.softcatala.org/pub/softcatala/opennmt/models/2022-11-22/cat-eng-2023-10-29.zip)
+|Basque-Catalan | 38.8 |24.9 |29.6 |N/A|N/A| 9546180 | [eus-cat-2024-08-09.zip](https://www.softcatala.org/pub/softcatala/opennmt/models/2022-11-22/eus-cat-2024-08-09.zip)
+|Catalan-Basque | 27.3 |17.1 |18.0 |N/A|N/A| 9546180 | [cat-eus-2024-08-12.zip](https://www.softcatala.org/pub/softcatala/opennmt/models/2022-11-22/cat-eus-2024-08-12.zip)
 |French-Catalan | 41.3 |31.6 |37.3 |33.3|27.2| 2566302 | [fra-cat-2022-11-09.zip](https://www.softcatala.org/pub/softcatala/opennmt/models/2022-11-22/fra-cat-2022-11-09.zip)
 |Catalan-French | 41.4 |35.4 |41.7 |39.6|27.9| 2566302 | [cat-fra-2022-11-14.zip](https://www.softcatala.org/pub/softcatala/opennmt/models/2022-11-22/cat-fra-2022-11-14.zip)
 |Galician-Catalan | 74.1 |31.4 |36.5 |33.2|N/A| 2710149 | [glg-cat-2022-11-17.zip](https://www.softcatala.org/pub/softcatala/opennmt/models/2022-11-22/glg-cat-2022-11-17.zip)

data-processing-tools/join-single-file.py

@@ -9,6 +9,8 @@
 import datetime
 import unicodedata
 from optparse import OptionParser
+import resource
+import re
 
 g_configuration = None
 
@@ -160,6 +162,9 @@ def _create_dir_if_does_exist(directory):
     if not os.path.exists(directory):
        os.makedirs(directory)
 
+def _clean_for_dup_detection(string):
+    return re.sub(r'\s+', '', string)
+
 # https://arxiv.org/abs/1907.01279 contains an overview of some of the techniques used here
 def split_in_six_files(src_filename, tgt_filename, directory, source_lang, target_lang):
 
@@ -226,7 +231,8 @@ def split_in_six_files(src_filename, tgt_filename, directory, source_lang, targe
                 equal += 1
                 continue
 
-            pair = src + trg
+            with_no_spaces = _clean_for_dup_detection(src + trg)
+            pair = hash(with_no_spaces)
             if pair in pairs:
                 duplicated = duplicated + 1
                 continue
@@ -279,6 +285,7 @@ def split_in_six_files(src_filename, tgt_filename, directory, source_lang, targe
             cnt_steps_val += 1
             cnt_steps_test += 1
 
+    max_rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024
     pduplicated = duplicated * 100 / strings
     pdots = dots * 100 / strings
     pclean_src = clean_src * 100 / strings
@@ -289,6 +296,7 @@ def split_in_six_files(src_filename, tgt_filename, directory, source_lang, targe
     print(f"Cleaned acute accents. src: {clean_src} ({pclean_src:.2f}%), tgt: {clean_trg} ({pclean_trg:.2f}%)")
     print(f"Empty sentences or diff len too long: {bad_length} ({pbad_length:.2f}%)")
     print(f"Dots: {dots} ({pdots:.2f}%), equal: {equal} ({pequal:.2f}%)")
+    print(f"max_rss {max_rss:.2f} MB")
 
 def append_lines_from_file(src_filename, trg_file, max_lines):
     lines = 0

data-processing-tools/tests/testjoin-single-file.py

@@ -118,5 +118,10 @@ def test__get_val_test_split_lines(self):
         self.assertEquals(1000, steps_val)
         self.assertEquals(500, steps_test)
 
+    def test___clean_for_dup_detection(self):
+        self.assertEquals("Word1Word2", join_single_file._clean_for_dup_detection("Word1 Word2\n"))
+        self.assertEquals("Word1Word2", join_single_file._clean_for_dup_detection("Word1\tWord2\r"))
+        self.assertEquals("Word1Word2.", join_single_file._clean_for_dup_detection("Word1  Word2.\r"))
+
 if __name__ == '__main__':
     unittest.main()

evaluate/google-bleu.json

@@ -16,5 +16,7 @@
     "jpn-cat": "23.4",
     "cat-jpn": "32.5",
     "glg-cat": "36.5",
-    "cat-glg": "33.1"
-}
+    "cat-glg": "33.1",
+    "eus-cat": "29.6",
+    "cat-eus": "18.0"
+}

evaluate/google-translate.py

@@ -160,6 +160,9 @@ def main():
 
         "gl-ca" : ["glg", "cat"],
         "ca-gl" : ["cat", "glg"],
+
+        "eu-ca" : ["eus", "cat"],
+        "ca-eu" : ["cat", "eus"],
     }
 
     blue_scores = {}

languages/eus-cat/bleu.sh

@@ -0,0 +1 @@
+../master-language/bleu.sh

languages/eus-cat/corpus.yml

@@ -0,0 +1,13 @@
+source_files:
+    - corpus-raw/eus-cat-1.eu
+    - corpus-raw/eus-cat-2.eu
+    - corpus-raw/eus-cat-3.eu
+    - corpus-raw/eus-cat-4.eu
+    - corpus-raw/eus-cat-5.eu
+
+target_files:
+    - corpus-raw/eus-cat-1.ca
+    - corpus-raw/eus-cat-2.ca
+    - corpus-raw/eus-cat-3.ca
+    - corpus-raw/eus-cat-4.ca
+    - corpus-raw/eus-cat-5.ca

languages/eus-cat/data.yml

@@ -0,0 +1 @@
+../master-language/data.yml

languages/eus-cat/export.sh

@@ -0,0 +1 @@
+../master-language/export.sh

languages/eus-cat/model.py

@@ -0,0 +1 @@
+../master-language/model.py

languages/eus-cat/preprocess.sh

@@ -0,0 +1 @@
+../master-language/preprocess.sh

languages/eus-cat/train.sh

@@ -0,0 +1 @@
+../master-language/train.sh

languages/eus-cat/voc.sh

@@ -0,0 +1 @@
+../master-language/voc.sh

languages/get-corpuses.sh

@@ -3,7 +3,7 @@
 git clone --depth=1 https://github.com/Softcatala/parallel-catalan-corpus corpus-raw
 cd corpus-raw
 
-declare -a arr=("eng-cat" "deu-cat" "ita-cat" "fra-cat" "spa-cat" "nld-cat" "por-cat" "jpn-cat" "glg-cat" "oci-cat")
+declare -a arr=("eng-cat" "deu-cat" "ita-cat" "fra-cat" "spa-cat" "nld-cat" "por-cat" "jpn-cat" "glg-cat" "oci-cat" "eus-cat")
 
 for dirname in "${arr[@]}"; do
     echo Copying $dirname

languages/preprocess-all.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-declare -a arr=("ita-cat" "fra-cat" "spa-cat" "por-cat" "eng-cat" "deu-cat" "nld-cat" "jpn-cat" "glg-cat" "oci-cat")
+declare -a arr=("ita-cat" "fra-cat" "spa-cat" "por-cat" "eng-cat" "deu-cat" "nld-cat" "jpn-cat" "glg-cat" "oci-cat" "eus-cat")
 
 for dirname in "${arr[@]}"; do
     echo Processing $dirname

languages/train-all-from-cat.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-declare -a arr=("cat-eng" "cat-deu" "cat-fra" "cat-ita" "cat-spa" "cat-por" "cat-nld" "cat-jpn" "cat-glg" "cat-oci")
+declare -a arr=("cat-eng" "cat-deu" "cat-fra" "cat-ita" "cat-spa" "cat-por" "cat-nld" "cat-jpn" "cat-glg" "cat-oci" "cat-eus")
 #declare -a arr=("cat-eng")
 
 for dirname in "${arr[@]}"; do

languages/train-all-to-cat.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-declare -a arr=("eng-cat" "deu-cat" "fra-cat" "ita-cat" "spa-cat" "por-cat" "nld-cat" "jpn-cat" "glg-cat" "oci-cat")
+declare -a arr=("eng-cat" "deu-cat" "fra-cat" "ita-cat" "spa-cat" "por-cat" "nld-cat" "jpn-cat" "glg-cat" "oci-cat" "eus-cat")
 #declare -a arr=("eng-cat")
 
 for dirname in "${arr[@]}"; do

models-table.py

@@ -96,6 +96,7 @@ def convert_iso_639_3_to_string(language_pair):
         "jpn" : "Japanese",
         "glg" : "Galician",
         "oci" : "Occitan",
+        "eus" : "Basque",
     }
 
     for iso in languages.keys():