Datasets - add languages

biolab · Nov 4, 2022 · 9914fa7 · 9914fa7
1 parent e69636c
commit 9914fa7
Show file tree

Hide file tree

Showing 16 changed files with 33 additions and 1 deletion.
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,4 +1,4 @@
-recursive-include orangecontrib/text/datasets *.tab *.txt
+recursive-include orangecontrib/text/datasets *.tab *.txt *.metadata
 recursive-include orangecontrib/text/models *.ftz
 recursive-include orangecontrib/text/sentiment *.txt
 recursive-include orangecontrib/text/tests *.txt *.json *.pkl

diff --git a/orangecontrib/text/datasets/20newsgroups-test.tab.metadata b/orangecontrib/text/datasets/20newsgroups-test.tab.metadata
@@ -0,0 +1 @@
+language: en
diff --git a/orangecontrib/text/datasets/20newsgroups-train.tab.metadata b/orangecontrib/text/datasets/20newsgroups-train.tab.metadata
@@ -0,0 +1 @@
+language: en
diff --git a/orangecontrib/text/datasets/andersen.tab.metadata b/orangecontrib/text/datasets/andersen.tab.metadata
@@ -0,0 +1 @@
+language: en
diff --git a/orangecontrib/text/datasets/book-excerpts.tab.metadata b/orangecontrib/text/datasets/book-excerpts.tab.metadata
@@ -0,0 +1 @@
+language: en
diff --git a/orangecontrib/text/datasets/deerwester.tab.metadata b/orangecontrib/text/datasets/deerwester.tab.metadata
@@ -0,0 +1 @@
+language: en
diff --git a/orangecontrib/text/datasets/election-tweets-2016.tab.metadata b/orangecontrib/text/datasets/election-tweets-2016.tab.metadata
@@ -0,0 +1 @@
+language: en
diff --git a/orangecontrib/text/datasets/friends-transcripts.tab.metadata b/orangecontrib/text/datasets/friends-transcripts.tab.metadata
@@ -0,0 +1 @@
+language: en
diff --git a/orangecontrib/text/datasets/grimm-tales-selected.tab.metadata b/orangecontrib/text/datasets/grimm-tales-selected.tab.metadata
@@ -0,0 +1 @@
+language: en
diff --git a/orangecontrib/text/datasets/grimm-tales.tab.metadata b/orangecontrib/text/datasets/grimm-tales.tab.metadata
@@ -0,0 +1 @@
+language: en
diff --git a/orangecontrib/text/datasets/reuters-r52-test.tab.metadata b/orangecontrib/text/datasets/reuters-r52-test.tab.metadata
@@ -0,0 +1 @@
+language: en
diff --git a/orangecontrib/text/datasets/reuters-r52-train.tab.metadata b/orangecontrib/text/datasets/reuters-r52-train.tab.metadata
@@ -0,0 +1 @@
+language: en
diff --git a/orangecontrib/text/datasets/reuters-r8-test.tab.metadata b/orangecontrib/text/datasets/reuters-r8-test.tab.metadata
@@ -0,0 +1 @@
+language: en
diff --git a/orangecontrib/text/datasets/reuters-r8-train.tab.metadata b/orangecontrib/text/datasets/reuters-r8-train.tab.metadata
@@ -0,0 +1 @@
+language: en
diff --git a/orangecontrib/text/datasets/slo-opinion-corpus.tab.metadata b/orangecontrib/text/datasets/slo-opinion-corpus.tab.metadata
@@ -0,0 +1 @@
+language: sl
diff --git a/orangecontrib/text/tests/test_datasets.py b/orangecontrib/text/tests/test_datasets.py
@@ -0,0 +1,18 @@
+import os
+import unittest
+
+from orangecontrib.text import Corpus
+
+
+class TestDatasets(unittest.TestCase):
+    def test_languages(self):
+        cur_file = os.path.dirname(os.path.abspath(__file__))
+
+        for file in os.listdir(os.path.join(cur_file, "..", "datasets")):
+            if file.endswith((".tab", ".xlsx", ".pkl", ".csv")):
+                c = Corpus.from_file(file)
+                self.assertIsNotNone(c.language)
+
+
+if __name__ == "__main__":
+    unittest.main()