Yatter2

.github/workflows/testgha.yml

@@ -20,12 +20,11 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install setuptools wheel pytest DeepDiff
-        pip install -r requirements.txt
+        pip install poetry
+        poetry update
     - name: Build yatter
       run: |
-        cd src
-        pip install .
+        poetry install --with test
     - name: Test with pytest
       run: |
-        pytest
+        poetry run pytest

.gitignore

@@ -2,4 +2,24 @@
 lib/__pychache__
 .idea
 venv
-.pytest_cache
+.pytest_cache
+tempReadme.md
+src/yatter/__pycache__/termmap.cpython-311.pyc
+src/yatter/__pycache__/target.cpython-311.pyc
+src/yatter/__pycache__/subject.cpython-311.pyc
+src/yatter/__pycache__/source.cpython-311.pyc
+src/yatter/__pycache__/predicateobject.cpython-311.pyc
+src/yatter/__pycache__/normalization.cpython-311.pyc
+src/yatter/__pycache__/mapping.cpython-311.pyc
+src/yatter/__pycache__/graph.cpython-311.pyc
+src/yatter/__pycache__/function.cpython-311.pyc
+src/yatter/__pycache__/constants.cpython-311.pyc
+src/yatter/__pycache__/__main__.cpython-311.pyc
+src/yatter/__pycache__/__init__.cpython-311.pyc
+src/yatter/__init__.py
+test/normalize/NORMKEYS-01/mapping_normalized.yml
+test/normalize/NORMKEYS-01/mapping.yml
+mapping.rml.ttl
+test/rml-io/YARRRMLTC-0036/__pycache__/test_yarrrmltc0036.cpython-311-pytest-8.3.3.pyc
+*.pyc
+/src/outputNEW.ttl

pyproject.toml

@@ -0,0 +1,24 @@
+[tool.poetry]
+name = "yatter"
+version = "1.1.4"
+description = "The tool translates mapping rules from YARRRML in a turtle-based serialization of RML or R2RML."
+authors = ["David Chaves <[email protected]>", "Rubén Buján <[email protected]>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.8"
+ruamel-yaml = "^0.17.21"
+rdflib = "^6.2.0"
+coloredlogs = "^15.0.1"
+
+[tool.poetry.group.test.dependencies]
+pytest = "^8.3.3"
+setuptools = "^75.1.0"
+wheel = "^0.44.0"
+DeepDiff = "^8.0.1"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+

src/yatter/__init__.py

@@ -1,29 +1,32 @@
 from .constants import *
 from .mapping import add_prefix, add_mapping, add_inverse_prefix, get_non_asserted_mappings, merge_mapping_section_by_key
-from .source import get_initial_sources, add_source, generate_database_connections, add_table, add_inverse_source
+from .source import add_source, generate_database_connections, add_table, add_inverse_source
 from .subject import add_subject, add_inverse_subject
 from .predicateobject import add_predicate_object_maps, add_inverse_pom
 from .target import add_logical_targets
 from .function import add_functions
+from .normalization import normalize
 import rdflib
 import ruamel.yaml as yaml
 
 
 def translate(yarrrml_data, mapping_format=RML_URI):
     logger.info("Translating YARRRML mapping to [R2]RML")
-
-    list_initial_sources = get_initial_sources(yarrrml_data)
+    external_sources = {}
+    external_targets = {}
+    yarrrml_data = normalize(yarrrml_data, external_sources, external_targets)
     rml_mapping = [add_prefix(yarrrml_data)]
-    rml_mapping.extend(generate_database_connections(yarrrml_data, list_initial_sources))
-    rml_mapping.extend(add_logical_targets(yarrrml_data))
+    rml_mapping.extend(generate_database_connections(yarrrml_data, external_sources))
+    rml_mapping.extend(add_logical_targets(yarrrml_data, external_targets))
     rml_mapping.extend(add_functions(yarrrml_data))
+
     try:
         mappings, mapping_format = get_non_asserted_mappings(yarrrml_data, mapping_format)
         for mapping in yarrrml_data.get(YARRRML_MAPPINGS):
             if mapping_format == R2RML_URI:
-                source_list = add_table(yarrrml_data, mapping, list_initial_sources)
+                source_list = add_table(yarrrml_data, mapping)
             else:
-                source_list = add_source(yarrrml_data, mapping, list_initial_sources)
+                source_list = add_source(yarrrml_data, mapping, external_sources)
             subject_list = add_subject(yarrrml_data, mapping, mapping_format)
             pred = add_predicate_object_maps(yarrrml_data, mapping, mapping_format)
             it = 0

src/yatter/constants.py

@@ -110,7 +110,6 @@
 
 YARRRML_PREFIXES = 'prefixes'
 YARRRML_SOURCES = 'sources'
-YARRRML_SOURCE = 'source'
 YARRRML_TABLE = 'table'
 YARRRML_ACCESS = 'access'
 YARRRML_QUERY = 'query'
@@ -124,35 +123,25 @@
 
 YARRRML_MAPPINGS = 'mappings' # used for mappings in conditions and mappings main key
 YARRRML_MAPPING = 'mapping'
-YARRRRL_MAPPINGS_SHORTCUT = 'm'
 
-YARRRML_SUBJECTS_SHORTCUT = 's'
-YARRRML_SUBJECT = 'subject'
 YARRRML_SUBJECTS = 'subjects'
-
+YARRRML_AUTHORS = 'authors'
 YARRRML_GRAPHS = 'graphs'
-YARRRML_GRAPH = 'graph'
-YARRRML_GRAPH_SHORTCUT = 'g'
 
 YARRRML_PREDICATEOBJECT = 'predicateobjects'
-YARRRML_PREDICATEOBJECT_SHORTCUT = 'po'
+YARRRML_PREDICATEOBJECT_SHORTCUT = 'po' #eliminar
 
 YARRRML_PREDICATES = 'predicates'
-YARRRML_PREDICATE = 'predicate'
-YARRRML_PREDICATES_SHORTCUT = 'p'
 YARRRML_OBJECTS = 'objects'
-YARRRML_OBJECT = 'objects'
-YARRRML_OBJECT_SHORTCUT = 'o'
 YARRRML_VALUE = 'value'
-YARRRML_VALUE_SHORTCUT = 'v'
 YARRRML_DATATYPE = 'datatype'
 YARRRML_LANGUAGE = 'language'
 
 YARRRML_CONDITION = 'condition'
 YARRRML_EQUAL = 'equal'
 YARRRML_JOIN = 'join'
 YARRRML_PARAMETERS = 'parameters' #used for conditions and functions
-YARRRML_PARAMETERS_SHORTCUT = 'pms'
+
 
 YARRRML_IRI = '~iri'
 YARRRML_LANG = '~lang'
@@ -166,22 +155,19 @@
 YARRRML_COMPRESSION = 'compression'
 
 YARRRML_FUNCTION = 'function'
-YARRRML_FUNCTION_SHORTCUT_A = 'fn'
-YARRRML_FUNCTION_SHORTCUT_B = 'f'
 
 YARRRML_PARAMETER = 'parameter'
-YARRRML_PARAMETER_SHORTCUT = 'p'
 
-YARRRML_MAPPING_KEYS = [YARRRML_MAPPINGS, YARRRML_MAPPING, YARRRRL_MAPPINGS_SHORTCUT]
-YARRRML_SUBJECT_KEYS = [YARRRML_SUBJECTS, YARRRML_SUBJECT, YARRRML_SUBJECTS_SHORTCUT]
-YARRRML_POM_KEYS = [YARRRML_PREDICATEOBJECT, YARRRML_PREDICATEOBJECT_SHORTCUT]
-YARRRML_GRAPH_KEYS = [YARRRML_GRAPHS, YARRRML_GRAPH, YARRRML_GRAPH_SHORTCUT]
-YARRRML_PREDICATE_KEYS = [YARRRML_PREDICATES, YARRRML_PREDICATE, YARRRML_PREDICATES_SHORTCUT]
-YARRRML_OBJECT_KEYS = [YARRRML_OBJECTS, YARRRML_OBJECT, YARRRML_OBJECT_SHORTCUT]
-YARRRML_FUNCTION_KEYS = [YARRRML_FUNCTION, YARRRML_FUNCTION_SHORTCUT_A, YARRRML_FUNCTION_SHORTCUT_B]
-YARRRML_PARAMETERS_KEYS = [YARRRML_PARAMETERS, YARRRML_PARAMETERS_SHORTCUT]
-YARRRML_PARAMETER_KEYS = [YARRRML_PARAMETER, YARRRML_PARAMETER_SHORTCUT]
-YARRRML_VALUE_KEYS = [YARRRML_VALUE, YARRRML_VALUE_SHORTCUT]
+YARRRML_MAPPING_KEYS = [YARRRML_MAPPINGS, YARRRML_MAPPING]
+YARRRML_SUBJECT_KEYS = [YARRRML_SUBJECTS]
+YARRRML_POM_KEYS = [YARRRML_PREDICATEOBJECT]
+YARRRML_GRAPH_KEYS = [YARRRML_GRAPHS]
+YARRRML_PREDICATE_KEYS = [YARRRML_PREDICATES]
+YARRRML_OBJECT_KEYS = [YARRRML_OBJECTS]
+YARRRML_FUNCTION_KEYS = [YARRRML_FUNCTION]
+YARRRML_PARAMETERS_KEYS = [YARRRML_PARAMETERS]
+YARRRML_PARAMETER_KEYS = [YARRRML_PARAMETER]
+YARRRML_VALUE_KEYS = [YARRRML_VALUE]
 
 
 YARRRML_OUTPUT_FORMAT = {
@@ -226,4 +212,4 @@
 }
 
 logger = logging.getLogger(__name__)
-coloredlogs.install(level='DEBUG', fmt='%(asctime)s,%(msecs)03d | %(levelname)s: %(message)s')
+coloredlogs.install(level='DEBUG', fmt='%(asctime)s,%(msecs)03d | %(levelname)s: %(message)s')

src/yatter/function.py

@@ -140,8 +140,14 @@ def generate_extended_join(yarrrml_data):
     for value in equals:
         parameters = []
         conditions = value.replace("(","",1).rsplit(")",1)[0].split(",")
-        parameters.append(['str1',conditions[0]])
-        parameters.append(['str2', conditions[1]])
+        expanded_param = dict()
+        expanded_param[YARRRML_PARAMETER] = 'str1'
+        expanded_param[YARRRML_VALUE] = conditions[0]
+        parameters.append(expanded_param)
+        expanded_param = dict()
+        expanded_param[YARRRML_PARAMETER] = 'str2'
+        expanded_param[YARRRML_VALUE] = conditions[1]
+        parameters.append(expanded_param)
         extended_join[YARRRML_CONDITION].append({YARRRML_FUNCTION: YARRRML_EQUAL, YARRRML_PARAMETERS: parameters})
 
     return extended_join

src/yatter/mapping.py

@@ -1,6 +1,8 @@
-from .import *
+from . import *
+
 prefixes = {}
 
+
 def add_mapping(mapping, mappings, it):
     map_template = "<" + mapping + "_" + str(it) + "> a "
     if mappings[mapping]:
@@ -58,7 +60,6 @@ def add_prefix(data):
 
 
 def check_common_prefixes(prefix_uri, common_prefixes):
-
     if prefix_uri == R2RML_URI:
         common_prefixes.append("r2rml")
     elif prefix_uri == RML_URI:
@@ -106,8 +107,8 @@ def get_non_asserted_mappings(yarrrml_data, mapping_format):
                 for value in yarrrml_data.get(YARRRML_MAPPINGS).get(mapping).get(key):
                     if type(value) is dict:
                         star_data = [value]
-                        if YARRRML_OBJECT_SHORTCUT in value:
-                            star_data = value[YARRRML_OBJECT_SHORTCUT]
+                        if YARRRML_OBJECTS in value:
+                            star_data = value[YARRRML_OBJECTS]
                         for val in star_data:
                             if YARRRML_NON_ASSERTED in val:
                                 mappings[val[YARRRML_NON_ASSERTED]] = "non_asserted"
@@ -118,9 +119,8 @@ def get_non_asserted_mappings(yarrrml_data, mapping_format):
     return mappings, mapping_format
 
 
-
-def merge_mapping_section_by_key(key,yarrrml_list):
-    output = {key:{}}
+def merge_mapping_section_by_key(key, yarrrml_list):
+    output = {key: {}}
     for yarrrml_mapping in yarrrml_list:
         output[key] = output[key] | yarrrml_mapping
-    return output
+    return output