Merge branch 'master' into exp1

Softcatala · Aug 13, 2024 · 6084578 · 6084578
2 parents 2014792 + 4319440
commit 6084578
Show file tree

Hide file tree

Showing 20 changed files with 3,071 additions and 8 deletions.
diff --git a/README.md b/README.md
@@ -15,6 +15,8 @@ Language pair | SC model BLEU | SC Flores200 BLEU | Google BLEU | Meta NLLB200 B
 |Catalan-German | 28.5 |25.4 |32.9 |29.1|15.8| 3142257 | [cat-deu-2022-11-16.zip](https://www.softcatala.org/pub/softcatala/opennmt/models/2022-11-22/cat-deu-2022-11-16.zip)
 |English-Catalan | 46.9 |43.8 |46.0 |41.7|29.8| 7856208 | [eng-cat-2023-10-30.zip](https://www.softcatala.org/pub/softcatala/opennmt/models/2022-11-22/eng-cat-2023-10-30.zip)
 |Catalan-English | 47.4 |43.5 |47.0 |48.0|29.6| 7856208 | [cat-eng-2023-10-29.zip](https://www.softcatala.org/pub/softcatala/opennmt/models/2022-11-22/cat-eng-2023-10-29.zip)
+|Basque-Catalan | 38.8 |24.9 |29.6 |N/A|N/A| 9546180 | [eus-cat-2024-08-09.zip](https://www.softcatala.org/pub/softcatala/opennmt/models/2022-11-22/eus-cat-2024-08-09.zip)
+|Catalan-Basque | 27.3 |17.1 |18.0 |N/A|N/A| 9546180 | [cat-eus-2024-08-12.zip](https://www.softcatala.org/pub/softcatala/opennmt/models/2022-11-22/cat-eus-2024-08-12.zip)
 |French-Catalan | 41.3 |31.6 |37.3 |33.3|27.2| 2566302 | [fra-cat-2022-11-09.zip](https://www.softcatala.org/pub/softcatala/opennmt/models/2022-11-22/fra-cat-2022-11-09.zip)
 |Catalan-French | 41.4 |35.4 |41.7 |39.6|27.9| 2566302 | [cat-fra-2022-11-14.zip](https://www.softcatala.org/pub/softcatala/opennmt/models/2022-11-22/cat-fra-2022-11-14.zip)
 |Galician-Catalan | 74.1 |31.4 |36.5 |33.2|N/A| 2710149 | [glg-cat-2022-11-17.zip](https://www.softcatala.org/pub/softcatala/opennmt/models/2022-11-22/glg-cat-2022-11-17.zip)

diff --git a/evaluate/flores200.eus b/evaluate/flores200.eus
diff --git a/evaluate/google-bleu.json b/evaluate/google-bleu.json
@@ -16,5 +16,7 @@
     "jpn-cat": "23.4",
     "cat-jpn": "32.5",
     "glg-cat": "36.5",
-    "cat-glg": "33.1"
-}
+    "cat-glg": "33.1",
+    "eus-cat": "29.6",
+    "cat-eus": "18.0"
+}
diff --git a/evaluate/google-translate.py b/evaluate/google-translate.py
@@ -160,6 +160,9 @@ def main():
 
         "gl-ca" : ["glg", "cat"],
         "ca-gl" : ["cat", "glg"],
+
+        "eu-ca" : ["eus", "cat"],
+        "ca-eu" : ["cat", "eus"],
     }
 
     blue_scores = {}

diff --git a/evaluate/google-translate/flores200-cat-eus.eus b/evaluate/google-translate/flores200-cat-eus.eus
diff --git a/evaluate/google-translate/flores200-eus-cat.cat b/evaluate/google-translate/flores200-eus-cat.cat
diff --git a/languages/eus-cat/bleu.sh b/languages/eus-cat/bleu.sh
@@ -0,0 +1 @@
+../master-language/bleu.sh
diff --git a/languages/eus-cat/corpus.yml b/languages/eus-cat/corpus.yml
@@ -0,0 +1,13 @@
+source_files:
+    - corpus-raw/eus-cat-1.eu
+    - corpus-raw/eus-cat-2.eu
+    - corpus-raw/eus-cat-3.eu
+    - corpus-raw/eus-cat-4.eu
+    - corpus-raw/eus-cat-5.eu
+
+target_files:
+    - corpus-raw/eus-cat-1.ca
+    - corpus-raw/eus-cat-2.ca
+    - corpus-raw/eus-cat-3.ca
+    - corpus-raw/eus-cat-4.ca
+    - corpus-raw/eus-cat-5.ca
diff --git a/languages/eus-cat/data.yml b/languages/eus-cat/data.yml
@@ -0,0 +1 @@
+../master-language/data.yml
diff --git a/languages/eus-cat/export.sh b/languages/eus-cat/export.sh
@@ -0,0 +1 @@
+../master-language/export.sh
diff --git a/languages/eus-cat/model.py b/languages/eus-cat/model.py
@@ -0,0 +1 @@
+../master-language/model.py
diff --git a/languages/eus-cat/preprocess.sh b/languages/eus-cat/preprocess.sh
@@ -0,0 +1 @@
+../master-language/preprocess.sh
diff --git a/languages/eus-cat/train.sh b/languages/eus-cat/train.sh
@@ -0,0 +1 @@
+../master-language/train.sh
diff --git a/languages/eus-cat/voc.sh b/languages/eus-cat/voc.sh
@@ -0,0 +1 @@
+../master-language/voc.sh
diff --git a/languages/get-corpuses.sh b/languages/get-corpuses.sh
@@ -4,7 +4,7 @@
 git clone --single-branch --branch eng-cat-aina --depth 1 https://github.com/Softcatala/parallel-catalan-corpus corpus-raw
 cd corpus-raw
 
-declare -a arr=("eng-cat" "deu-cat" "ita-cat" "fra-cat" "spa-cat" "nld-cat" "por-cat" "jpn-cat" "glg-cat" "oci-cat")
+declare -a arr=("eng-cat" "deu-cat" "ita-cat" "fra-cat" "spa-cat" "nld-cat" "por-cat" "jpn-cat" "glg-cat" "oci-cat" "eus-cat")
 
 for dirname in "${arr[@]}"; do
     echo Copying $dirname

diff --git a/languages/master-language/export.sh b/languages/master-language/export.sh
@@ -16,8 +16,7 @@ onmt-main --config data.yml --auto_config export --export_dir $exportDir/tensorf
 echo "Model description: $modelName" >> $modelDescription
 echo "Date: $currentDate" >> $modelDescription
 python3 ../../install-scripts/stack-versions.py >> $modelDescription
-wc corpus/$modelName/src-train.txt -l > $exportDir/metadata/inputs_used.txt
-ls corpus/$modelName/*.txt -l >> $exportDir/metadata/inputs_used.txt
+wc src-train.txt -l > $exportDir/metadata/inputs_used.txt
 cp *.model $exportDir/tokenizer/
 cp corpus.yml data.yml model.py $exportDir/metadata/
 cp sp-vocab.txt.token $exportDir/tensorflow/assets/

diff --git a/languages/preprocess-all.sh b/languages/preprocess-all.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-declare -a arr=("ita-cat" "fra-cat" "spa-cat" "por-cat" "eng-cat" "deu-cat" "nld-cat" "jpn-cat" "glg-cat" "oci-cat")
+declare -a arr=("ita-cat" "fra-cat" "spa-cat" "por-cat" "eng-cat" "deu-cat" "nld-cat" "jpn-cat" "glg-cat" "oci-cat" "eus-cat")
 
 for dirname in "${arr[@]}"; do
     echo Processing $dirname

diff --git a/languages/train-all-from-cat.sh b/languages/train-all-from-cat.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-declare -a arr=("cat-eng" "cat-deu" "cat-fra" "cat-ita" "cat-spa" "cat-por" "cat-nld" "cat-jpn" "cat-glg" "cat-oci")
+declare -a arr=("cat-eng" "cat-deu" "cat-fra" "cat-ita" "cat-spa" "cat-por" "cat-nld" "cat-jpn" "cat-glg" "cat-oci" "cat-eus")
 #declare -a arr=("cat-eng")
 
 for dirname in "${arr[@]}"; do

diff --git a/languages/train-all-to-cat.sh b/languages/train-all-to-cat.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-declare -a arr=("eng-cat" "deu-cat" "fra-cat" "ita-cat" "spa-cat" "por-cat" "nld-cat" "jpn-cat" "glg-cat" "oci-cat")
+declare -a arr=("eng-cat" "deu-cat" "fra-cat" "ita-cat" "spa-cat" "por-cat" "nld-cat" "jpn-cat" "glg-cat" "oci-cat" "eus-cat")
 #declare -a arr=("eng-cat")
 
 for dirname in "${arr[@]}"; do

diff --git a/models-table.py b/models-table.py
@@ -96,6 +96,7 @@ def convert_iso_639_3_to_string(language_pair):
         "jpn" : "Japanese",
         "glg" : "Galician",
         "oci" : "Occitan",
+        "eus" : "Basque",
     }
 
     for iso in languages.keys():