Normalize msmarco-v2-vector queries and switch to dot-product (elasti…

…c#578)

* msmarco-v2-vector: add vector normalization to parse_documents.py and use dot-product on ingestion mappings

* msmarco-v2-vector: update dataset file sizes to match new normalised dataset files

* msmarco-v2-vector: normalize queries

msmarco-v2-vector/_tools/parse_queries.py

@@ -3,6 +3,8 @@
 from os import environ
 
 import ir_datasets
+import numpy
+import vg
 from cohere import AsyncClient
 
 DATASET_NAME: str = "msmarco-passage-v2/train"
@@ -12,7 +14,7 @@
 
 async def retrieve_embed_for_query(co, text):
     response = await co.embed(texts=[text], model="embed-english-v3.0", input_type="search_query")
-    return response.embeddings[0]
+    return vg.normalize(numpy.array(response.embeddings[0])).tolist()
 
 
 async def output_queries(queries_file):

msmarco-v2-vector/_tools/requirements.txt

@@ -1,3 +1,4 @@
 cohere
 datasets
 ir-datasets
+vg

msmarco-v2-vector/index-vectors-only-mapping.json

@@ -19,7 +19,7 @@
         "element_type": "float",
         "dims": 1024,
         "index": true,
-        "similarity": "max_inner_product",
+        "similarity": "dot_product",
         "index_options": {
           "type": {{ vector_index_type | default("int8_hnsw") | tojson }}
         }

msmarco-v2-vector/index-vectors-with-text-mapping.json

@@ -24,7 +24,7 @@
         "element_type": "float",
         "dims": 1024,
         "index": true,
-        "similarity": "max_inner_product",
+        "similarity": "dot_product",
         "index_options": {
           "type": {{ vector_index_type | default("int8_hnsw") |tojson }}
         }