From 026ff04a067022c39b7a31d66c749943b0c72bd8 Mon Sep 17 00:00:00 2001 From: Alessandro Benedetti Date: Fri, 20 Dec 2024 16:58:07 +0000 Subject: [PATCH] SOLR-17525 backport to 9.x (#2921) * SOLR-17525: Text Embedder Query Parser (#2809) * added support to store Large Language Models external endpoints config for text to vector encoding * added a query parser that can use such external models to encode text to vector and run a KNearest Neighbour search --------- Co-authored-by: Christine Poerschke Co-authored-by: Christos Malliaridis Co-authored-by: David Smiley (cherry picked from commit e8803e6f707b4600cec1e7e70f1d52f895bae0a6) --- .../randomization/policies/solr-tests.policy | 3 + settings.gradle | 1 + solr/CHANGES.txt | 2 + .../apache/solr/search/neural/KnnQParser.java | 4 +- .../licenses/converter-jackson-2.9.0.jar.sha1 | 1 + .../converter-jackson-LICENSE-ASL.txt | 202 +++++++++ solr/licenses/converter-jackson-NOTICE.txt | 13 + solr/licenses/jtokkit-1.1.0.jar.sha1 | 1 + solr/licenses/jtokkit-LICENSE-MIT.txt | 21 + solr/licenses/langchain4j-LICENSE-ASL.txt | 202 +++++++++ solr/licenses/langchain4j-NOTICE.txt | 13 + .../langchain4j-cohere-0.35.0.jar.sha1 | 1 + .../licenses/langchain4j-core-0.35.0.jar.sha1 | 1 + .../langchain4j-hugging-face-0.35.0.jar.sha1 | 1 + .../langchain4j-mistral-ai-0.35.0.jar.sha1 | 1 + .../langchain4j-open-ai-0.35.0.jar.sha1 | 1 + solr/licenses/okhttp-sse-4.12.0.jar.sha1 | 1 + solr/licenses/openai4j-0.22.0.jar.sha1 | 1 + solr/licenses/openai4j-LICENSE-ASL.txt | 201 +++++++++ solr/licenses/openai4j-NOTICE.txt | 11 + solr/licenses/retrofit-2.9.0.jar.sha1 | 1 + solr/licenses/retrofit-LICENSE-ASL.txt | 202 +++++++++ solr/licenses/retrofit-NOTICE.txt | 13 + solr/modules/llm/README.md | 21 + solr/modules/llm/build.gradle | 41 ++ .../model/SolrTextToVectorModel.java | 185 +++++++++ .../llm/texttovector/model/package-info.java | 19 + .../search/TextToVectorQParserPlugin.java | 136 ++++++ .../llm/texttovector/search/package-info.java | 19 + .../store/TextToVectorModelException.java | 30 ++ .../store/TextToVectorModelStore.java | 67 +++ .../llm/texttovector/store/package-info.java | 19 + .../rest/ManagedTextToVectorModelStore.java | 200 +++++++++ .../texttovector/store/rest/package-info.java | 19 + solr/modules/llm/src/java/overview.html | 21 + solr/modules/llm/src/test-files/log4j2.xml | 43 ++ .../modelExamples/cohere-model.json | 13 + .../modelExamples/dummy-model-ambiguous.json | 8 + .../dummy-model-unsupported.json | 8 + .../test-files/modelExamples/dummy-model.json | 7 + .../modelExamples/huggingface-model.json | 8 + .../modelExamples/mistralai-model.json | 13 + .../modelExamples/openai-model.json | 13 + .../solr/collection1/conf/schema.xml | 52 +++ .../solr/collection1/conf/solrconfig-llm.xml | 57 +++ solr/modules/llm/src/test-files/solr/solr.xml | 42 ++ .../test/org/apache/solr/llm/TestLlmBase.java | 180 ++++++++ .../model/DummyEmbeddingModel.java | 91 +++++ .../model/DummyEmbeddingModelTest.java | 48 +++ .../search/TextToVectorQParserTest.java | 386 ++++++++++++++++++ .../store/rest/TestModelManager.java | 222 ++++++++++ .../rest/TestModelManagerPersistence.java | 125 ++++++ solr/server/etc/security.policy | 10 +- .../pages/dense-vector-search.adoc | 74 +++- .../query-guide/pages/text-to-vector.adoc | 270 ++++++++++++ versions.lock | 20 +- versions.props | 1 + 57 files changed, 3356 insertions(+), 10 deletions(-) create mode 100644 solr/licenses/converter-jackson-2.9.0.jar.sha1 create mode 100644 solr/licenses/converter-jackson-LICENSE-ASL.txt create mode 100644 solr/licenses/converter-jackson-NOTICE.txt create mode 100644 solr/licenses/jtokkit-1.1.0.jar.sha1 create mode 100644 solr/licenses/jtokkit-LICENSE-MIT.txt create mode 100644 solr/licenses/langchain4j-LICENSE-ASL.txt create mode 100644 solr/licenses/langchain4j-NOTICE.txt create mode 100644 solr/licenses/langchain4j-cohere-0.35.0.jar.sha1 create mode 100644 solr/licenses/langchain4j-core-0.35.0.jar.sha1 create mode 100644 solr/licenses/langchain4j-hugging-face-0.35.0.jar.sha1 create mode 100644 solr/licenses/langchain4j-mistral-ai-0.35.0.jar.sha1 create mode 100644 solr/licenses/langchain4j-open-ai-0.35.0.jar.sha1 create mode 100644 solr/licenses/okhttp-sse-4.12.0.jar.sha1 create mode 100644 solr/licenses/openai4j-0.22.0.jar.sha1 create mode 100644 solr/licenses/openai4j-LICENSE-ASL.txt create mode 100644 solr/licenses/openai4j-NOTICE.txt create mode 100644 solr/licenses/retrofit-2.9.0.jar.sha1 create mode 100644 solr/licenses/retrofit-LICENSE-ASL.txt create mode 100644 solr/licenses/retrofit-NOTICE.txt create mode 100644 solr/modules/llm/README.md create mode 100644 solr/modules/llm/build.gradle create mode 100644 solr/modules/llm/src/java/org/apache/solr/llm/texttovector/model/SolrTextToVectorModel.java create mode 100644 solr/modules/llm/src/java/org/apache/solr/llm/texttovector/model/package-info.java create mode 100644 solr/modules/llm/src/java/org/apache/solr/llm/texttovector/search/TextToVectorQParserPlugin.java create mode 100644 solr/modules/llm/src/java/org/apache/solr/llm/texttovector/search/package-info.java create mode 100644 solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/TextToVectorModelException.java create mode 100644 solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/TextToVectorModelStore.java create mode 100644 solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/package-info.java create mode 100644 solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/rest/ManagedTextToVectorModelStore.java create mode 100644 solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/rest/package-info.java create mode 100644 solr/modules/llm/src/java/overview.html create mode 100644 solr/modules/llm/src/test-files/log4j2.xml create mode 100644 solr/modules/llm/src/test-files/modelExamples/cohere-model.json create mode 100644 solr/modules/llm/src/test-files/modelExamples/dummy-model-ambiguous.json create mode 100644 solr/modules/llm/src/test-files/modelExamples/dummy-model-unsupported.json create mode 100644 solr/modules/llm/src/test-files/modelExamples/dummy-model.json create mode 100644 solr/modules/llm/src/test-files/modelExamples/huggingface-model.json create mode 100644 solr/modules/llm/src/test-files/modelExamples/mistralai-model.json create mode 100644 solr/modules/llm/src/test-files/modelExamples/openai-model.json create mode 100644 solr/modules/llm/src/test-files/solr/collection1/conf/schema.xml create mode 100644 solr/modules/llm/src/test-files/solr/collection1/conf/solrconfig-llm.xml create mode 100644 solr/modules/llm/src/test-files/solr/solr.xml create mode 100644 solr/modules/llm/src/test/org/apache/solr/llm/TestLlmBase.java create mode 100644 solr/modules/llm/src/test/org/apache/solr/llm/texttovector/model/DummyEmbeddingModel.java create mode 100644 solr/modules/llm/src/test/org/apache/solr/llm/texttovector/model/DummyEmbeddingModelTest.java create mode 100644 solr/modules/llm/src/test/org/apache/solr/llm/texttovector/search/TextToVectorQParserTest.java create mode 100644 solr/modules/llm/src/test/org/apache/solr/llm/texttovector/store/rest/TestModelManager.java create mode 100644 solr/modules/llm/src/test/org/apache/solr/llm/texttovector/store/rest/TestModelManagerPersistence.java create mode 100644 solr/solr-ref-guide/modules/query-guide/pages/text-to-vector.adoc diff --git a/gradle/testing/randomization/policies/solr-tests.policy b/gradle/testing/randomization/policies/solr-tests.policy index 8183cf1e1e7..7024b0579cb 100644 --- a/gradle/testing/randomization/policies/solr-tests.policy +++ b/gradle/testing/randomization/policies/solr-tests.policy @@ -56,6 +56,9 @@ grant { // needed by randomizedtesting runner to identify test methods. permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; + permission java.lang.reflect.ReflectPermission "newProxyInPackage.dev.langchain4j.model.cohere"; + permission java.lang.reflect.ReflectPermission "newProxyInPackage.dev.ai4j.openai4j"; + permission java.lang.reflect.ReflectPermission "newProxyInPackage.dev.langchain4j.model.huggingface"; permission java.lang.RuntimePermission "accessDeclaredMembers"; // needed by certain tests to redirect sysout/syserr: permission java.lang.RuntimePermission "setIO"; diff --git a/settings.gradle b/settings.gradle index 8cc382ddc8d..2f1d1098146 100644 --- a/settings.gradle +++ b/settings.gradle @@ -52,6 +52,7 @@ include "solr:modules:hdfs" include "solr:modules:jaegertracer-configurator" include "solr:modules:jwt-auth" include "solr:modules:langid" +include "solr:modules:llm" include "solr:modules:ltr" include "solr:modules:s3-repository" include "solr:modules:scripting" diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 08c21bfd7c0..83ba7f4cdc5 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -13,6 +13,8 @@ New Features * SOLR-17150: Implement `memAllowed` parameter to limit per-thread memory allocations during request processing. (Andrzej Bialecki, Gus Heck) +* SOLR-17525: Added knn_text_to_vector query parser to encode text to vector at query time through external LLM services. (Alessandro Benedetti) + Improvements --------------------- * SOLR-17158: Users using query limits (timeAllowed, cpuTimeAllowed) for whom partial results are uninteresting diff --git a/solr/core/src/java/org/apache/solr/search/neural/KnnQParser.java b/solr/core/src/java/org/apache/solr/search/neural/KnnQParser.java index 166dada5b7f..b6d9f2541cd 100644 --- a/solr/core/src/java/org/apache/solr/search/neural/KnnQParser.java +++ b/solr/core/src/java/org/apache/solr/search/neural/KnnQParser.java @@ -26,8 +26,8 @@ public class KnnQParser extends AbstractVectorQParserBase { // retrieve the top K results based on the distance similarity function - static final String TOP_K = "topK"; - static final int DEFAULT_TOP_K = 10; + protected static final String TOP_K = "topK"; + protected static final int DEFAULT_TOP_K = 10; public KnnQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { super(qstr, localParams, params, req); diff --git a/solr/licenses/converter-jackson-2.9.0.jar.sha1 b/solr/licenses/converter-jackson-2.9.0.jar.sha1 new file mode 100644 index 00000000000..20ab4394574 --- /dev/null +++ b/solr/licenses/converter-jackson-2.9.0.jar.sha1 @@ -0,0 +1 @@ +19b4010914e747601e26f46c6a403044bbe0b2bf diff --git a/solr/licenses/converter-jackson-LICENSE-ASL.txt b/solr/licenses/converter-jackson-LICENSE-ASL.txt new file mode 100644 index 00000000000..7a4a3ea2424 --- /dev/null +++ b/solr/licenses/converter-jackson-LICENSE-ASL.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/solr/licenses/converter-jackson-NOTICE.txt b/solr/licenses/converter-jackson-NOTICE.txt new file mode 100644 index 00000000000..25fdd5926ad --- /dev/null +++ b/solr/licenses/converter-jackson-NOTICE.txt @@ -0,0 +1,13 @@ +Copyright 2013 Square, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/solr/licenses/jtokkit-1.1.0.jar.sha1 b/solr/licenses/jtokkit-1.1.0.jar.sha1 new file mode 100644 index 00000000000..a86a8ae6804 --- /dev/null +++ b/solr/licenses/jtokkit-1.1.0.jar.sha1 @@ -0,0 +1 @@ +b7370f801db3eb8c7c6a2c2c06231909ac6de0b0 diff --git a/solr/licenses/jtokkit-LICENSE-MIT.txt b/solr/licenses/jtokkit-LICENSE-MIT.txt new file mode 100644 index 00000000000..9409cee80de --- /dev/null +++ b/solr/licenses/jtokkit-LICENSE-MIT.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Knuddels, Philip Müller + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/solr/licenses/langchain4j-LICENSE-ASL.txt b/solr/licenses/langchain4j-LICENSE-ASL.txt new file mode 100644 index 00000000000..a12e3073373 --- /dev/null +++ b/solr/licenses/langchain4j-LICENSE-ASL.txt @@ -0,0 +1,202 @@ +Apache License + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/solr/licenses/langchain4j-NOTICE.txt b/solr/licenses/langchain4j-NOTICE.txt new file mode 100644 index 00000000000..be32f8a2f46 --- /dev/null +++ b/solr/licenses/langchain4j-NOTICE.txt @@ -0,0 +1,13 @@ +The goal of LangChain4j is to simplify integrating LLMs into Java applications. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/solr/licenses/langchain4j-cohere-0.35.0.jar.sha1 b/solr/licenses/langchain4j-cohere-0.35.0.jar.sha1 new file mode 100644 index 00000000000..a2a66c77142 --- /dev/null +++ b/solr/licenses/langchain4j-cohere-0.35.0.jar.sha1 @@ -0,0 +1 @@ +737ebd43f49e1820b17b18b6e3154bf94e0b2853 diff --git a/solr/licenses/langchain4j-core-0.35.0.jar.sha1 b/solr/licenses/langchain4j-core-0.35.0.jar.sha1 new file mode 100644 index 00000000000..04931abbf53 --- /dev/null +++ b/solr/licenses/langchain4j-core-0.35.0.jar.sha1 @@ -0,0 +1 @@ +1ac1bf8b9aa2864ac3fcae7d46a5d9f80a582b30 diff --git a/solr/licenses/langchain4j-hugging-face-0.35.0.jar.sha1 b/solr/licenses/langchain4j-hugging-face-0.35.0.jar.sha1 new file mode 100644 index 00000000000..789b74e63dc --- /dev/null +++ b/solr/licenses/langchain4j-hugging-face-0.35.0.jar.sha1 @@ -0,0 +1 @@ +4eb72d70b9e26473d3e02f9199ccdf3a9bc24cbf diff --git a/solr/licenses/langchain4j-mistral-ai-0.35.0.jar.sha1 b/solr/licenses/langchain4j-mistral-ai-0.35.0.jar.sha1 new file mode 100644 index 00000000000..f37eb1b33db --- /dev/null +++ b/solr/licenses/langchain4j-mistral-ai-0.35.0.jar.sha1 @@ -0,0 +1 @@ +4f1165e6ee19430dcadc6cb260d798a40fd49ef0 diff --git a/solr/licenses/langchain4j-open-ai-0.35.0.jar.sha1 b/solr/licenses/langchain4j-open-ai-0.35.0.jar.sha1 new file mode 100644 index 00000000000..35cdfb38385 --- /dev/null +++ b/solr/licenses/langchain4j-open-ai-0.35.0.jar.sha1 @@ -0,0 +1 @@ +d77645377c517e18fd5e0ca2ca56d4a38e21495a diff --git a/solr/licenses/okhttp-sse-4.12.0.jar.sha1 b/solr/licenses/okhttp-sse-4.12.0.jar.sha1 new file mode 100644 index 00000000000..60d422fb2c6 --- /dev/null +++ b/solr/licenses/okhttp-sse-4.12.0.jar.sha1 @@ -0,0 +1 @@ +eca9c68c54ae7fd18d465beba65d80a44e9667e4 diff --git a/solr/licenses/openai4j-0.22.0.jar.sha1 b/solr/licenses/openai4j-0.22.0.jar.sha1 new file mode 100644 index 00000000000..506a2fd66b4 --- /dev/null +++ b/solr/licenses/openai4j-0.22.0.jar.sha1 @@ -0,0 +1 @@ +a9c8a0cdbf43cd31a08ee7ce957ab84991b67427 diff --git a/solr/licenses/openai4j-LICENSE-ASL.txt b/solr/licenses/openai4j-LICENSE-ASL.txt new file mode 100644 index 00000000000..f49a4e16e68 --- /dev/null +++ b/solr/licenses/openai4j-LICENSE-ASL.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/solr/licenses/openai4j-NOTICE.txt b/solr/licenses/openai4j-NOTICE.txt new file mode 100644 index 00000000000..60508a114b6 --- /dev/null +++ b/solr/licenses/openai4j-NOTICE.txt @@ -0,0 +1,11 @@ +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/solr/licenses/retrofit-2.9.0.jar.sha1 b/solr/licenses/retrofit-2.9.0.jar.sha1 new file mode 100644 index 00000000000..8939819d52c --- /dev/null +++ b/solr/licenses/retrofit-2.9.0.jar.sha1 @@ -0,0 +1 @@ +d8fdfbd5da952141a665a403348b74538efc05ff diff --git a/solr/licenses/retrofit-LICENSE-ASL.txt b/solr/licenses/retrofit-LICENSE-ASL.txt new file mode 100644 index 00000000000..7a4a3ea2424 --- /dev/null +++ b/solr/licenses/retrofit-LICENSE-ASL.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/solr/licenses/retrofit-NOTICE.txt b/solr/licenses/retrofit-NOTICE.txt new file mode 100644 index 00000000000..25fdd5926ad --- /dev/null +++ b/solr/licenses/retrofit-NOTICE.txt @@ -0,0 +1,13 @@ +Copyright 2013 Square, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/solr/modules/llm/README.md b/solr/modules/llm/README.md new file mode 100644 index 00000000000..2467434a14b --- /dev/null +++ b/solr/modules/llm/README.md @@ -0,0 +1,21 @@ + + +The Large Language Model module for Solr provides a set of mechanisms for plugging in third party LLM implementations. +It currently provides text vectorisation through langChain4j. + +See https://solr.apache.org/guide/solr/latest/query-guide/text-to-vector.html for how to get started. diff --git a/solr/modules/llm/build.gradle b/solr/modules/llm/build.gradle new file mode 100644 index 00000000000..9f8ef218f66 --- /dev/null +++ b/solr/modules/llm/build.gradle @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +apply plugin: 'java-library' + +description = 'Indexing/Query time integration with LLM' + +dependencies { + compileOnly 'com.github.stephenc.jcip:jcip-annotations' + implementation project(':solr:core') + implementation project(':solr:solrj') + + implementation 'org.apache.lucene:lucene-core' + + implementation 'dev.langchain4j:langchain4j-core' + runtimeOnly 'dev.langchain4j:langchain4j-hugging-face' + runtimeOnly 'dev.langchain4j:langchain4j-mistral-ai' + runtimeOnly 'dev.langchain4j:langchain4j-open-ai' + runtimeOnly 'dev.langchain4j:langchain4j-cohere' + + + implementation 'org.slf4j:slf4j-api' + + testImplementation project(':solr:test-framework') + testImplementation 'junit:junit' + testImplementation 'commons-io:commons-io' +} diff --git a/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/model/SolrTextToVectorModel.java b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/model/SolrTextToVectorModel.java new file mode 100644 index 00000000000..f798f27db5d --- /dev/null +++ b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/model/SolrTextToVectorModel.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.llm.texttovector.model; + +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.model.embedding.EmbeddingModel; +import java.lang.reflect.Method; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Map; +import java.util.Objects; +import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.RamUsageEstimator; +import org.apache.solr.core.SolrResourceLoader; +import org.apache.solr.llm.texttovector.store.TextToVectorModelException; +import org.apache.solr.llm.texttovector.store.rest.ManagedTextToVectorModelStore; + +/** + * This object wraps a {@link dev.langchain4j.model.embedding.EmbeddingModel} to encode text to + * vector. It's meant to be used as a managed resource with the {@link + * ManagedTextToVectorModelStore} + */ +public class SolrTextToVectorModel implements Accountable { + private static final long BASE_RAM_BYTES = + RamUsageEstimator.shallowSizeOfInstance(SolrTextToVectorModel.class); + private static final String TIMEOUT_PARAM = "timeout"; + private static final String MAX_SEGMENTS_PER_BATCH_PARAM = "maxSegmentsPerBatch"; + private static final String MAX_RETRIES_PARAM = "maxRetries"; + + private final String name; + private final Map params; + private final EmbeddingModel textToVector; + private final int hashCode; + + public static SolrTextToVectorModel getInstance( + SolrResourceLoader solrResourceLoader, + String className, + String name, + Map params) + throws TextToVectorModelException { + try { + /* + * The idea here is to build a {@link dev.langchain4j.model.embedding.EmbeddingModel} using inversion + * of control. + * Each model has its own list of parameters we don't know beforehand, but each {@link dev.langchain4j.model.embedding.EmbeddingModel} class + * has its own builder that uses setters with the same name of the parameter in input. + * */ + EmbeddingModel textToVector; + Class modelClass = solrResourceLoader.findClass(className, EmbeddingModel.class); + var builder = modelClass.getMethod("builder").invoke(null); + if (params != null) { + /* + * This block of code has the responsibility of instantiate a {@link + * dev.langchain4j.model.embedding.EmbeddingModel} using the params provided.classes have + * params of The specific implementation of {@link + * dev.langchain4j.model.embedding.EmbeddingModel} is not known beforehand. So we benefit of + * the design choice in langchain4j that each subclass implementing {@link + * dev.langchain4j.model.embedding.EmbeddingModel} uses setters with the same name of the + * param. + */ + for (String paramName : params.keySet()) { + /* + * When a param is not primitive, we need to instantiate the object explicitly and then call the + * setter method. + * N.B. when adding support to new models, pay attention to all the parameters they + * support, some of them may require to be handled in here as separate switch cases + */ + switch (paramName) { + case TIMEOUT_PARAM: + Duration timeOut = Duration.ofSeconds((Long) params.get(paramName)); + builder.getClass().getMethod(paramName, Duration.class).invoke(builder, timeOut); + break; + case MAX_SEGMENTS_PER_BATCH_PARAM: + builder + .getClass() + .getMethod(paramName, Integer.class) + .invoke(builder, ((Long) params.get(paramName)).intValue()); + break; + case MAX_RETRIES_PARAM: + builder + .getClass() + .getMethod(paramName, Integer.class) + .invoke(builder, ((Long) params.get(paramName)).intValue()); + break; + /* + * For primitive params if there's only one setter available, we call it. + * If there's choice we default to the string one + */ + default: + ArrayList paramNameMatches = new ArrayList<>(); + for (var method : builder.getClass().getMethods()) { + if (paramName.equals(method.getName()) && method.getParameterCount() == 1) { + paramNameMatches.add(method); + } + } + if (paramNameMatches.size() == 1) { + paramNameMatches.get(0).invoke(builder, params.get(paramName)); + } else { + builder + .getClass() + .getMethod(paramName, String.class) + .invoke(builder, params.get(paramName).toString()); + } + } + } + } + textToVector = (EmbeddingModel) builder.getClass().getMethod("build").invoke(builder); + return new SolrTextToVectorModel(name, textToVector, params); + } catch (final Exception e) { + throw new TextToVectorModelException("Model loading failed for " + className, e); + } + } + + public SolrTextToVectorModel( + String name, EmbeddingModel textToVector, Map params) { + this.name = name; + this.textToVector = textToVector; + this.params = params; + this.hashCode = calculateHashCode(); + } + + public float[] vectorise(String text) { + Embedding vector = textToVector.embed(text).content(); + return vector.vector(); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "(name=" + getName() + ")"; + } + + @Override + public long ramBytesUsed() { + return BASE_RAM_BYTES + + RamUsageEstimator.sizeOfObject(name) + + RamUsageEstimator.sizeOfObject(textToVector); + } + + @Override + public int hashCode() { + return hashCode; + } + + private int calculateHashCode() { + final int prime = 31; + int result = 1; + result = (prime * result) + Objects.hashCode(name); + result = (prime * result) + Objects.hashCode(textToVector); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (!(obj instanceof SolrTextToVectorModel)) return false; + final SolrTextToVectorModel other = (SolrTextToVectorModel) obj; + return Objects.equals(textToVector, other.textToVector) && Objects.equals(name, other.name); + } + + public String getName() { + return name; + } + + public String getEmbeddingModelClassName() { + return textToVector.getClass().getName(); + } + + public Map getParams() { + return params; + } +} diff --git a/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/model/package-info.java b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/model/package-info.java new file mode 100644 index 00000000000..64e50f6f88b --- /dev/null +++ b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/model/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** APIs and classes for implementing text to vector logic. */ +package org.apache.solr.llm.texttovector.model; diff --git a/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/search/TextToVectorQParserPlugin.java b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/search/TextToVectorQParserPlugin.java new file mode 100644 index 00000000000..17749eba7b6 --- /dev/null +++ b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/search/TextToVectorQParserPlugin.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.llm.texttovector.search; + +import java.io.IOException; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.ResourceLoader; +import org.apache.lucene.util.ResourceLoaderAware; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.SolrResourceLoader; +import org.apache.solr.llm.texttovector.model.SolrTextToVectorModel; +import org.apache.solr.llm.texttovector.store.rest.ManagedTextToVectorModelStore; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.rest.ManagedResource; +import org.apache.solr.rest.ManagedResourceObserver; +import org.apache.solr.schema.DenseVectorField; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.search.QParser; +import org.apache.solr.search.QParserPlugin; +import org.apache.solr.search.SyntaxError; +import org.apache.solr.search.neural.KnnQParser; + +/** + * A neural query parser that encode the query to a vector and then run K-nearest neighbors search + * on Dense Vector fields. See Wiki page + * https://solr.apache.org/guide/solr/latest/query-guide/dense-vector-search.html + */ +public class TextToVectorQParserPlugin extends QParserPlugin + implements ResourceLoaderAware, ManagedResourceObserver { + public static final String EMBEDDING_MODEL_PARAM = "model"; + private ManagedTextToVectorModelStore modelStore = null; + + @Override + public QParser createParser( + String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { + return new TextToVectorQParser(qstr, localParams, params, req); + } + + @Override + public void inform(ResourceLoader loader) throws IOException { + final SolrResourceLoader solrResourceLoader = (SolrResourceLoader) loader; + ManagedTextToVectorModelStore.registerManagedTextToVectorModelStore(solrResourceLoader, this); + } + + @Override + public void onManagedResourceInitialized(NamedList args, ManagedResource res) + throws SolrException { + if (res instanceof ManagedTextToVectorModelStore) { + modelStore = (ManagedTextToVectorModelStore) res; + } + if (modelStore != null) { + modelStore.loadStoredModels(); + } + } + + public class TextToVectorQParser extends KnnQParser { + + public TextToVectorQParser( + String queryString, SolrParams localParams, SolrParams params, SolrQueryRequest req) { + super(queryString, localParams, params, req); + } + + @Override + public Query parse() throws SyntaxError { + checkParam(qstr, "Query string is empty, nothing to vectorise"); + final String embeddingModelName = localParams.get(EMBEDDING_MODEL_PARAM); + checkParam(embeddingModelName, "The 'model' parameter is missing"); + SolrTextToVectorModel textToVector = modelStore.getModel(embeddingModelName); + + if (textToVector != null) { + final SchemaField schemaField = req.getCore().getLatestSchema().getField(getFieldName()); + final DenseVectorField denseVectorType = getCheckedFieldType(schemaField); + int fieldDimensions = denseVectorType.getDimension(); + VectorEncoding vectorEncoding = denseVectorType.getVectorEncoding(); + final int topK = localParams.getInt(TOP_K, DEFAULT_TOP_K); + + switch (vectorEncoding) { + case FLOAT32: + { + float[] vectorToSearch = textToVector.vectorise(qstr); + checkVectorDimension(vectorToSearch.length, fieldDimensions); + return new KnnFloatVectorQuery( + schemaField.getName(), vectorToSearch, topK, getFilterQuery()); + } + default: + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Vector Encoding not supported : " + vectorEncoding); + } + } else { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "The model requested '" + + embeddingModelName + + "' can't be found in the store: " + + ManagedTextToVectorModelStore.REST_END_POINT); + } + } + } + + private void checkVectorDimension(int inputVectorDimension, int fieldVectorDimension) { + if (inputVectorDimension != fieldVectorDimension) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "incorrect vector dimension." + + " The vector value has size " + + inputVectorDimension + + " while it is expected a vector with size " + + fieldVectorDimension); + } + } + + private void checkParam(String value, String message) { + if (value == null || value.isBlank()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, message); + } + } +} diff --git a/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/search/package-info.java b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/search/package-info.java new file mode 100644 index 00000000000..9fbf84e62c6 --- /dev/null +++ b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/search/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** APIs and classes for implementing text to vector QueryParsers. */ +package org.apache.solr.llm.texttovector.search; diff --git a/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/TextToVectorModelException.java b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/TextToVectorModelException.java new file mode 100644 index 00000000000..076f2b45f9b --- /dev/null +++ b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/TextToVectorModelException.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.llm.texttovector.store; + +public class TextToVectorModelException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public TextToVectorModelException(String message) { + super(message); + } + + public TextToVectorModelException(String message, Exception cause) { + super(message, cause); + } +} diff --git a/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/TextToVectorModelStore.java b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/TextToVectorModelStore.java new file mode 100644 index 00000000000..cf1db239d44 --- /dev/null +++ b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/TextToVectorModelStore.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.llm.texttovector.store; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.solr.llm.texttovector.model.SolrTextToVectorModel; + +/** Simple store to manage CRUD operations on the {@link SolrTextToVectorModel} */ +public class TextToVectorModelStore { + + private final Map availableModels; + + public TextToVectorModelStore() { + availableModels = Collections.synchronizedMap(new LinkedHashMap<>()); + } + + public SolrTextToVectorModel getModel(String name) { + return availableModels.get(name); + } + + public void clear() { + availableModels.clear(); + } + + public List getModels() { + synchronized (availableModels) { + final List availableModelsValues = + new ArrayList(availableModels.values()); + return Collections.unmodifiableList(availableModelsValues); + } + } + + @Override + public String toString() { + return "ModelStore [availableModels=" + availableModels.keySet() + "]"; + } + + public SolrTextToVectorModel delete(String modelName) { + return availableModels.remove(modelName); + } + + public void addModel(SolrTextToVectorModel modeldata) throws TextToVectorModelException { + final String name = modeldata.getName(); + if (availableModels.putIfAbsent(modeldata.getName(), modeldata) != null) { + throw new TextToVectorModelException( + "model '" + name + "' already exists. Please use a different name"); + } + } +} diff --git a/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/package-info.java b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/package-info.java new file mode 100644 index 00000000000..630ac6085a8 --- /dev/null +++ b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Contains model store related classes. */ +package org.apache.solr.llm.texttovector.store; diff --git a/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/rest/ManagedTextToVectorModelStore.java b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/rest/ManagedTextToVectorModelStore.java new file mode 100644 index 00000000000..0652ec54d77 --- /dev/null +++ b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/rest/ManagedTextToVectorModelStore.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.llm.texttovector.store.rest; + +import java.lang.invoke.MethodHandles; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import net.jcip.annotations.ThreadSafe; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SolrResourceLoader; +import org.apache.solr.llm.texttovector.model.SolrTextToVectorModel; +import org.apache.solr.llm.texttovector.store.TextToVectorModelException; +import org.apache.solr.llm.texttovector.store.TextToVectorModelStore; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.rest.BaseSolrResource; +import org.apache.solr.rest.ManagedResource; +import org.apache.solr.rest.ManagedResourceObserver; +import org.apache.solr.rest.ManagedResourceStorage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Managed Resource wrapper for the the {@link TextToVectorModelStore} to expose it via REST */ +@ThreadSafe +public class ManagedTextToVectorModelStore extends ManagedResource + implements ManagedResource.ChildResourceSupport { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + /** the model store rest endpoint */ + public static final String REST_END_POINT = "/schema/text-to-vector-model-store"; + + /** Managed model store: the name of the attribute containing all the models of a model store */ + private static final String MODELS_JSON_FIELD = "models"; + + /** name of the attribute containing a class */ + static final String CLASS_KEY = "class"; + + /** name of the attribute containing a name */ + static final String NAME_KEY = "name"; + + /** name of the attribute containing parameters */ + static final String PARAMS_KEY = "params"; + + public static void registerManagedTextToVectorModelStore( + SolrResourceLoader solrResourceLoader, ManagedResourceObserver managedResourceObserver) { + solrResourceLoader + .getManagedResourceRegistry() + .registerManagedResource( + REST_END_POINT, ManagedTextToVectorModelStore.class, managedResourceObserver); + } + + public static ManagedTextToVectorModelStore getManagedModelStore(SolrCore core) { + return (ManagedTextToVectorModelStore) core.getRestManager().getManagedResource(REST_END_POINT); + } + + /** + * Returns the available models as a list of Maps objects. After an update the managed resources + * needs to return the resources in this format in order to store in json somewhere (zookeeper, + * disk...) + * + * @return the available models as a list of Maps objects + */ + private static List modelsAsManagedResources(List models) { + return models.stream() + .map(ManagedTextToVectorModelStore::toModelMap) + .collect(Collectors.toList()); + } + + @SuppressWarnings("unchecked") + public static SolrTextToVectorModel fromModelMap( + SolrResourceLoader solrResourceLoader, Map embeddingModel) { + return SolrTextToVectorModel.getInstance( + solrResourceLoader, + (String) embeddingModel.get(CLASS_KEY), // modelClassName + (String) embeddingModel.get(NAME_KEY), // modelName + (Map) embeddingModel.get(PARAMS_KEY)); + } + + private static LinkedHashMap toModelMap(SolrTextToVectorModel model) { + final LinkedHashMap modelMap = new LinkedHashMap<>(5, 1.0f); + modelMap.put(NAME_KEY, model.getName()); + modelMap.put(CLASS_KEY, model.getEmbeddingModelClassName()); + modelMap.put(PARAMS_KEY, model.getParams()); + return modelMap; + } + + private final TextToVectorModelStore store; + private Object managedData; + + public ManagedTextToVectorModelStore( + String resourceId, SolrResourceLoader loader, ManagedResourceStorage.StorageIO storageIO) + throws SolrException { + super(resourceId, loader, storageIO); + store = new TextToVectorModelStore(); + } + + @Override + protected ManagedResourceStorage createStorage( + ManagedResourceStorage.StorageIO storageIO, SolrResourceLoader loader) throws SolrException { + return new ManagedResourceStorage.JsonStorage(storageIO, loader, -1); + } + + @Override + protected void onManagedDataLoadedFromStorage(NamedList managedInitArgs, Object managedData) + throws SolrException { + store.clear(); + this.managedData = managedData; + } + + public void loadStoredModels() { + log.info("------ managed models ~ loading ------"); + + if ((managedData != null) && (managedData instanceof List)) { + @SuppressWarnings({"unchecked"}) + final List> textToVectorModels = (List>) managedData; + for (final Map textToVectorModel : textToVectorModels) { + addModelFromMap(textToVectorModel); + } + } + } + + private void addModelFromMap(Map modelMap) { + try { + addModel(fromModelMap(solrResourceLoader, modelMap)); + } catch (final TextToVectorModelException e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); + } + } + + public void addModel(SolrTextToVectorModel model) throws TextToVectorModelException { + try { + if (log.isInfoEnabled()) { + log.info("adding model {}", model.getName()); + } + store.addModel(model); + } catch (final TextToVectorModelException e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); + } + } + + @SuppressWarnings("unchecked") + @Override + protected Object applyUpdatesToManagedData(Object updates) { + if (updates instanceof List) { + final List> textToVectorModels = (List>) updates; + for (final Map textToVectorModel : textToVectorModels) { + addModelFromMap(textToVectorModel); + } + } + + if (updates instanceof Map) { + final Map map = (Map) updates; + addModelFromMap(map); + } + + return modelsAsManagedResources(store.getModels()); + } + + @Override + public void doDeleteChild(BaseSolrResource endpoint, String childId) { + store.delete(childId); + storeManagedData(applyUpdatesToManagedData(null)); + } + + /** + * Called to retrieve a named part (the given childId) of the resource at the given endpoint. + * Note: since we have a unique child managed store we ignore the childId. + */ + @Override + public void doGet(BaseSolrResource endpoint, String childId) { + final SolrQueryResponse response = endpoint.getSolrResponse(); + response.add(MODELS_JSON_FIELD, modelsAsManagedResources(store.getModels())); + } + + public SolrTextToVectorModel getModel(String modelName) { + return store.getModel(modelName); + } + + @Override + public String toString() { + return "ManagedModelStore [store=" + store + "]"; + } +} diff --git a/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/rest/package-info.java b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/rest/package-info.java new file mode 100644 index 00000000000..56ae30f2ebe --- /dev/null +++ b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/rest/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Contains the {@link org.apache.solr.rest.ManagedResource} that encapsulate the model stores. */ +package org.apache.solr.llm.texttovector.store.rest; diff --git a/solr/modules/llm/src/java/overview.html b/solr/modules/llm/src/java/overview.html new file mode 100644 index 00000000000..29daef11a7b --- /dev/null +++ b/solr/modules/llm/src/java/overview.html @@ -0,0 +1,21 @@ + + + +Apache Solr Search Server: text to vector module + + diff --git a/solr/modules/llm/src/test-files/log4j2.xml b/solr/modules/llm/src/test-files/log4j2.xml new file mode 100644 index 00000000000..5e696935965 --- /dev/null +++ b/solr/modules/llm/src/test-files/log4j2.xml @@ -0,0 +1,43 @@ + + + + + + + + + %maxLen{%-4r %-5p (%t) [%notEmpty{n:%X{node_name}}%notEmpty{ c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}%notEmpty{ t:%X{trace_id}}] %c{1.} %m%notEmpty{ + =>%ex{short}}}{10240}%n + + + + + + + + + + + + + + + + + + diff --git a/solr/modules/llm/src/test-files/modelExamples/cohere-model.json b/solr/modules/llm/src/test-files/modelExamples/cohere-model.json new file mode 100644 index 00000000000..5eef83667b5 --- /dev/null +++ b/solr/modules/llm/src/test-files/modelExamples/cohere-model.json @@ -0,0 +1,13 @@ +{ + "class": "dev.langchain4j.model.cohere.CohereEmbeddingModel", + "name": "cohere-1", + "params": { + "baseUrl": "https://api.cohere.ai/v1/", + "apiKey": "apiKey-cohere", + "modelName": "embed-english-light-v3.0", + "inputType": "search_document", + "timeout": 60, + "logRequests": true, + "logResponses": true + } +} diff --git a/solr/modules/llm/src/test-files/modelExamples/dummy-model-ambiguous.json b/solr/modules/llm/src/test-files/modelExamples/dummy-model-ambiguous.json new file mode 100644 index 00000000000..43de925cf9d --- /dev/null +++ b/solr/modules/llm/src/test-files/modelExamples/dummy-model-ambiguous.json @@ -0,0 +1,8 @@ +{ + "class": "org.apache.solr.llm.texttovector.model.DummyEmbeddingModel", + "name": "dummy-1", + "params": { + "embedding": [1.0, 2.0, 3.0, 4.0], + "ambiguous": 10 + } +} diff --git a/solr/modules/llm/src/test-files/modelExamples/dummy-model-unsupported.json b/solr/modules/llm/src/test-files/modelExamples/dummy-model-unsupported.json new file mode 100644 index 00000000000..9af02f14003 --- /dev/null +++ b/solr/modules/llm/src/test-files/modelExamples/dummy-model-unsupported.json @@ -0,0 +1,8 @@ +{ + "class": "org.apache.solr.llm.texttovector.model.DummyEmbeddingModel", + "name": "dummy-1", + "params": { + "embedding": [1.0, 2.0, 3.0, 4.0], + "unsupported": 10 + } +} diff --git a/solr/modules/llm/src/test-files/modelExamples/dummy-model.json b/solr/modules/llm/src/test-files/modelExamples/dummy-model.json new file mode 100644 index 00000000000..00603b8369b --- /dev/null +++ b/solr/modules/llm/src/test-files/modelExamples/dummy-model.json @@ -0,0 +1,7 @@ +{ + "class": "org.apache.solr.llm.texttovector.model.DummyEmbeddingModel", + "name": "dummy-1", + "params": { + "embedding": [1.0, 2.0, 3.0, 4.0] + } +} diff --git a/solr/modules/llm/src/test-files/modelExamples/huggingface-model.json b/solr/modules/llm/src/test-files/modelExamples/huggingface-model.json new file mode 100644 index 00000000000..f0a72cdf758 --- /dev/null +++ b/solr/modules/llm/src/test-files/modelExamples/huggingface-model.json @@ -0,0 +1,8 @@ +{ + "class": "dev.langchain4j.model.huggingface.HuggingFaceEmbeddingModel", + "name": "huggingface-1", + "params": { + "accessToken": "apiKey-huggingface", + "modelId": "sentence-transformers/all-MiniLM-L6-v2" + } +} diff --git a/solr/modules/llm/src/test-files/modelExamples/mistralai-model.json b/solr/modules/llm/src/test-files/modelExamples/mistralai-model.json new file mode 100644 index 00000000000..082a1a434f1 --- /dev/null +++ b/solr/modules/llm/src/test-files/modelExamples/mistralai-model.json @@ -0,0 +1,13 @@ +{ + "class": "dev.langchain4j.model.mistralai.MistralAiEmbeddingModel", + "name": "mistralai-1", + "params": { + "baseUrl": "https://api.mistral.ai/v1", + "apiKey": "apiKey-mistralAI", + "modelName": "mistral-embed", + "timeout": 60, + "logRequests": true, + "logResponses": true, + "maxRetries": 5 + } +} diff --git a/solr/modules/llm/src/test-files/modelExamples/openai-model.json b/solr/modules/llm/src/test-files/modelExamples/openai-model.json new file mode 100644 index 00000000000..a90d5bd6285 --- /dev/null +++ b/solr/modules/llm/src/test-files/modelExamples/openai-model.json @@ -0,0 +1,13 @@ +{ + "class": "dev.langchain4j.model.openai.OpenAiEmbeddingModel", + "name": "openai-1", + "params": { + "baseUrl": "https://api.openai.com/v1", + "apiKey": "apiKey-openAI", + "modelName": "text-embedding-3-small", + "timeout": 60, + "logRequests": true, + "logResponses": true, + "maxRetries": 5 + } +} diff --git a/solr/modules/llm/src/test-files/solr/collection1/conf/schema.xml b/solr/modules/llm/src/test-files/solr/collection1/conf/schema.xml new file mode 100644 index 00000000000..92aa0e76591 --- /dev/null +++ b/solr/modules/llm/src/test-files/solr/collection1/conf/schema.xml @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + diff --git a/solr/modules/llm/src/test-files/solr/collection1/conf/solrconfig-llm.xml b/solr/modules/llm/src/test-files/solr/collection1/conf/solrconfig-llm.xml new file mode 100644 index 00000000000..3a1d05285fc --- /dev/null +++ b/solr/modules/llm/src/test-files/solr/collection1/conf/solrconfig-llm.xml @@ -0,0 +1,57 @@ + + + + + ${tests.luceneMatchVersion:LATEST} + ${solr.data.dir:} + + + + + + + + + + + + + + + + + + 15000 + false + + + 1000 + + + ${solr.data.dir:} + + + + + + + explicit + json + true + id + + + + diff --git a/solr/modules/llm/src/test-files/solr/solr.xml b/solr/modules/llm/src/test-files/solr/solr.xml new file mode 100644 index 00000000000..c8c3ebeb30a --- /dev/null +++ b/solr/modules/llm/src/test-files/solr/solr.xml @@ -0,0 +1,42 @@ + + + + + + ${shareSchema:false} + ${configSetBaseDir:configsets} + ${coreRootDirectory:.} + + + ${urlScheme:} + ${socketTimeout:90000} + ${connTimeout:15000} + + + + 127.0.0.1 + ${hostPort:8983} + ${hostContext:solr} + ${solr.zkclienttimeout:30000} + ${genericCoreNodeNames:true} + ${leaderVoteWait:10000} + ${distribUpdateConnTimeout:45000} + ${distribUpdateSoTimeout:340000} + + + diff --git a/solr/modules/llm/src/test/org/apache/solr/llm/TestLlmBase.java b/solr/modules/llm/src/test/org/apache/solr/llm/TestLlmBase.java new file mode 100644 index 00000000000..6fb391ad364 --- /dev/null +++ b/solr/modules/llm/src/test/org/apache/solr/llm/TestLlmBase.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.llm; + +import java.lang.invoke.MethodHandles; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.apache.commons.io.file.PathUtils; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.llm.texttovector.store.rest.ManagedTextToVectorModelStore; +import org.apache.solr.util.RestTestBase; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestLlmBase extends RestTestBase { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + protected static Path tmpSolrHome; + protected static Path tmpConfDir; + + public static final String MODEL_FILE_NAME = "_schema_text-to-vector-model-store.json"; + protected static final String COLLECTION = "collection1"; + protected static final String CONF_DIR = COLLECTION + "/conf"; + + protected static Path embeddingModelStoreFile = null; + + protected static String IDField = "id"; + protected static String vectorField = "vector"; + protected static String vectorField2 = "vector2"; + protected static String vectorFieldByteEncoding = "vector_byte_encoding"; + + protected static void setupTest( + String solrconfig, String schema, boolean buildIndex, boolean persistModelStore) + throws Exception { + initFolders(persistModelStore); + createJettyAndHarness( + tmpSolrHome.toAbsolutePath().toString(), solrconfig, schema, "/solr", true, null); + if (buildIndex) prepareIndex(); + } + + protected static void initFolders(boolean isPersistent) throws Exception { + tmpSolrHome = createTempDir(); + tmpConfDir = tmpSolrHome.resolve(CONF_DIR); + tmpConfDir.toFile().deleteOnExit(); + PathUtils.copyDirectory(TEST_PATH(), tmpSolrHome.toAbsolutePath()); + + final Path modelStore = tmpConfDir.resolve(MODEL_FILE_NAME); + + if (isPersistent) { + embeddingModelStoreFile = modelStore; + } + + if (Files.exists(modelStore)) { + if (log.isInfoEnabled()) { + log.info("remove model store config file in {}", modelStore.toAbsolutePath()); + } + Files.delete(modelStore); + } + + System.setProperty("managed.schema.mutable", "true"); + } + + protected static void afterTest() throws Exception { + if (null != restTestHarness) { + restTestHarness.close(); + restTestHarness = null; + } + solrClientTestRule.reset(); + if (null != tmpSolrHome) { + PathUtils.deleteDirectory(tmpSolrHome); + tmpSolrHome = null; + } + System.clearProperty("managed.schema.mutable"); + } + + public static void loadModel(String fileName, String status) throws Exception { + final URL url = TestLlmBase.class.getResource("/modelExamples/" + fileName); + final String multipleModels = Files.readString(Path.of(url.toURI()), StandardCharsets.UTF_8); + + assertJPut( + ManagedTextToVectorModelStore.REST_END_POINT, + multipleModels, + "/responseHeader/status==" + status); + } + + public static void loadModel(String fileName) throws Exception { + final URL url = TestLlmBase.class.getResource("/modelExamples/" + fileName); + final String multipleModels = Files.readString(Path.of(url.toURI()), StandardCharsets.UTF_8); + + assertJPut( + ManagedTextToVectorModelStore.REST_END_POINT, multipleModels, "/responseHeader/status==0"); + } + + protected static void prepareIndex() throws Exception { + List docsToIndex = prepareDocs(); + for (SolrInputDocument doc : docsToIndex) { + assertU(adoc(doc)); + } + + assertU(commit()); + } + + private static List prepareDocs() { + int docsCount = 13; + List docs = new ArrayList<>(docsCount); + for (int i = 1; i < docsCount + 1; i++) { + SolrInputDocument doc = new SolrInputDocument(); + doc.addField(IDField, i); + docs.add(doc); + } + + docs.get(0) + .addField(vectorField, Arrays.asList(1f, 2f, 3f, 4f)); // cosine distance vector1= 1.0 + docs.get(1) + .addField( + vectorField, Arrays.asList(1.5f, 2.5f, 3.5f, 4.5f)); // cosine distance vector1= 0.998 + docs.get(2) + .addField( + vectorField, + Arrays.asList(7.5f, 15.5f, 17.5f, 22.5f)); // cosine distance vector1= 0.992 + docs.get(3) + .addField( + vectorField, Arrays.asList(1.4f, 2.4f, 3.4f, 4.4f)); // cosine distance vector1= 0.999 + docs.get(4) + .addField(vectorField, Arrays.asList(30f, 22f, 35f, 20f)); // cosine distance vector1= 0.862 + docs.get(5) + .addField(vectorField, Arrays.asList(40f, 1f, 1f, 200f)); // cosine distance vector1= 0.756 + docs.get(6) + .addField(vectorField, Arrays.asList(5f, 10f, 20f, 40f)); // cosine distance vector1= 0.970 + docs.get(7) + .addField( + vectorField, Arrays.asList(120f, 60f, 30f, 15f)); // cosine distance vector1= 0.515 + docs.get(8) + .addField( + vectorField, Arrays.asList(200f, 50f, 100f, 25f)); // cosine distance vector1= 0.554 + docs.get(9) + .addField( + vectorField, Arrays.asList(1.8f, 2.5f, 3.7f, 4.9f)); // cosine distance vector1= 0.997 + docs.get(10) + .addField(vectorField2, Arrays.asList(1f, 2f, 3f, 4f)); // cosine distance vector2= 1 + docs.get(11) + .addField( + vectorField2, + Arrays.asList(7.5f, 15.5f, 17.5f, 22.5f)); // cosine distance vector2= 0.992 + docs.get(12) + .addField( + vectorField2, Arrays.asList(1.5f, 2.5f, 3.5f, 4.5f)); // cosine distance vector2= 0.998 + + docs.get(0).addField(vectorFieldByteEncoding, Arrays.asList(1, 2, 3, 4)); + docs.get(1).addField(vectorFieldByteEncoding, Arrays.asList(2, 2, 1, 4)); + docs.get(2).addField(vectorFieldByteEncoding, Arrays.asList(1, 2, 1, 2)); + docs.get(3).addField(vectorFieldByteEncoding, Arrays.asList(7, 2, 1, 3)); + docs.get(4).addField(vectorFieldByteEncoding, Arrays.asList(19, 2, 4, 4)); + docs.get(5).addField(vectorFieldByteEncoding, Arrays.asList(19, 2, 4, 4)); + docs.get(6).addField(vectorFieldByteEncoding, Arrays.asList(18, 2, 4, 4)); + docs.get(7).addField(vectorFieldByteEncoding, Arrays.asList(8, 3, 2, 4)); + + return docs; + } +} diff --git a/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/model/DummyEmbeddingModel.java b/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/model/DummyEmbeddingModel.java new file mode 100644 index 00000000000..00edcba114b --- /dev/null +++ b/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/model/DummyEmbeddingModel.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.llm.texttovector.model; + +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.embedding.EmbeddingModel; +import dev.langchain4j.model.output.Response; +import java.util.ArrayList; +import java.util.List; + +public class DummyEmbeddingModel implements EmbeddingModel { + final float[] embedding; + + public DummyEmbeddingModel(float[] embedding) { + this.embedding = embedding; + } + + @Override + public Response embed(String text) { + Embedding dummy = new Embedding(embedding); + return new Response(dummy); + } + + @Override + public Response embed(TextSegment textSegment) { + Embedding dummy = new Embedding(embedding); + return new Response(dummy); + } + + @Override + public Response> embedAll(List textSegments) { + return null; + } + + @Override + public int dimension() { + return embedding.length; + } + + public static DummyEmbeddingModelBuilder builder() { + return new DummyEmbeddingModelBuilder(); + } + + public static class DummyEmbeddingModelBuilder { + private float[] builderEmbeddings; + private int intValue; + + public DummyEmbeddingModelBuilder() {} + + public DummyEmbeddingModelBuilder embedding(ArrayList embeddings) { + this.builderEmbeddings = new float[embeddings.size()]; + for (int i = 0; i < embeddings.size(); i++) { + this.builderEmbeddings[i] = embeddings.get(i).floatValue(); + } + return this; + } + + public DummyEmbeddingModelBuilder unsupported(Integer input) { + return this; + } + + public DummyEmbeddingModelBuilder ambiguous(int input) { + this.intValue = input; + return this; + } + + public DummyEmbeddingModelBuilder ambiguous(String input) { + this.intValue = Integer.valueOf(input); + return this; + } + + public DummyEmbeddingModel build() { + return new DummyEmbeddingModel(this.builderEmbeddings); + } + } +} diff --git a/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/model/DummyEmbeddingModelTest.java b/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/model/DummyEmbeddingModelTest.java new file mode 100644 index 00000000000..823f591fb95 --- /dev/null +++ b/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/model/DummyEmbeddingModelTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.llm.texttovector.model; + +import org.apache.solr.SolrTestCase; +import org.junit.Test; + +public class DummyEmbeddingModelTest extends SolrTestCase { + + @Test + public void constructAndEmbed() throws Exception { + assertEquals( + "[1.0, 2.0, 3.0, 4.0]", + new DummyEmbeddingModel(new float[] {1, 2, 3, 4}) + .embed("hello") + .content() + .vectorAsList() + .toString()); + assertEquals( + "[8.0, 7.0, 6.0, 5.0]", + new DummyEmbeddingModel(new float[] {8, 7, 6, 5}) + .embed("world") + .content() + .vectorAsList() + .toString()); + assertEquals( + "[0.0, 0.0, 4.0, 2.0]", + new DummyEmbeddingModel(new float[] {0, 0, 4, 2}) + .embed("answer") + .content() + .vectorAsList() + .toString()); + } +} diff --git a/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/search/TextToVectorQParserTest.java b/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/search/TextToVectorQParserTest.java new file mode 100644 index 00000000000..5c406f08217 --- /dev/null +++ b/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/search/TextToVectorQParserTest.java @@ -0,0 +1,386 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.llm.texttovector.search; + +import java.util.Arrays; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.llm.TestLlmBase; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TextToVectorQParserTest extends TestLlmBase { + @BeforeClass + public static void init() throws Exception { + setupTest("solrconfig-llm.xml", "schema.xml", true, false); + loadModel("dummy-model.json"); + } + + @Test + public void notExistentModel_shouldThrowException() throws Exception { + final String solrQuery = "{!knn_text_to_vector model=not-exist f=vector topK=5}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/error/msg=='The model requested \\'not-exist\\' can\\'t be found in the store: /schema/text-to-vector-model-store'", + "/error/code==400"); + } + + @Test + public void missingModelParam_shouldThrowException() throws Exception { + final String solrQuery = "{!knn_text_to_vector f=vector topK=5}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/error/msg=='The \\'model\\' parameter is missing'", + "/error/code==400"); + } + + @Test + public void incorrectVectorFieldType_shouldThrowException() throws Exception { + final String solrQuery = "{!knn_text_to_vector model=dummy-1 f=id topK=5}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/error/msg=='only DenseVectorField is compatible with Vector Query Parsers'", + "/error/code==400"); + } + + @Test + public void undefinedVectorField_shouldThrowException() throws Exception { + final String solrQuery = "{!knn_text_to_vector model=dummy-1 f=notExistent topK=5}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/error/msg=='undefined field: \"notExistent\"'", + "/error/code==400"); + } + + @Test + public void missingVectorFieldParam_shouldThrowException() throws Exception { + final String solrQuery = "{!knn_text_to_vector model=dummy-1 topK=5}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/error/msg=='the Dense Vector field \\'f\\' is missing'", + "/error/code==400"); + } + + @Test + public void vectorByteEncodingField_shouldRaiseException() throws Exception { + final String solrQuery = + "{!knn_text_to_vector model=dummy-1 f=vector_byte_encoding topK=5}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/error/msg=='Vector Encoding not supported : BYTE'", + "/error/code==500"); + } + + @Test + public void missingQueryToEmbed_shouldThrowException() throws Exception { + final String solrQuery = "{!knn_text_to_vector model=dummy-1 f=vector topK=5}"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/error/msg=='Query string is empty, nothing to vectorise'", + "/error/code==400"); + } + + @Test + public void incorrectVectorToSearchDimension_shouldThrowException() throws Exception { + final String solrQuery = + "{!knn_text_to_vector model=dummy-1 f=2048_float_vector topK=5}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/error/msg=='incorrect vector dimension. The vector value has size 4 while it is expected a vector with size 2048'", + "/error/code==400"); + } + + @Test + public void topK_shouldEmbedAndReturnOnlyTopKResults() throws Exception { + final String solrQuery = "{!knn_text_to_vector model=dummy-1 f=vector topK=5}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/response/numFound==5]", + "/response/docs/[0]/id=='1'", + "/response/docs/[1]/id=='4'", + "/response/docs/[2]/id=='2'", + "/response/docs/[3]/id=='10'", + "/response/docs/[4]/id=='3'"); + } + + @Test + public void vectorFieldParam_shouldSearchOnThatField() throws Exception { + final String solrQuery = "{!knn_text_to_vector model=dummy-1 f=vector2 topK=5}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/response/numFound==3]", + "/response/docs/[0]/id=='11'", + "/response/docs/[1]/id=='13'", + "/response/docs/[2]/id=='12'"); + } + + @Test + public void embeddedQuery_shouldRankBySimilarityFunction() throws Exception { + final String solrQuery = "{!knn_text_to_vector model=dummy-1 f=vector topK=10}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/response/numFound==10]", + "/response/docs/[0]/id=='1'", + "/response/docs/[1]/id=='4'", + "/response/docs/[2]/id=='2'", + "/response/docs/[3]/id=='10'", + "/response/docs/[4]/id=='3'", + "/response/docs/[5]/id=='7'", + "/response/docs/[6]/id=='5'", + "/response/docs/[7]/id=='6'", + "/response/docs/[8]/id=='9'", + "/response/docs/[9]/id=='8'"); + } + + @Test + public void embeddedQueryUsedInFilter_shouldFilterResultsBeforeTheQueryExecution() + throws Exception { + final String solrQuery = "{!knn_text_to_vector model=dummy-1 f=vector topK=4}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery("id:(3 4 9 2)"); + query.setFilterQueries(solrQuery); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/response/numFound==2]", + "/response/docs/[0]/id=='2'", + "/response/docs/[1]/id=='4'"); + } + + @Test + public void embeddedQueryUsedInFilters_shouldFilterResultsBeforeTheQueryExecution() + throws Exception { + final String solrQuery = "{!knn_text_to_vector model=dummy-1 f=vector topK=4}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery("id:(3 4 9 2)"); + query.setFilterQueries(solrQuery, "id:(4 20 9)"); + query.add("fl", "id"); + + // topK=4 -> 1,4,2,10 + assertJQ( + "/query" + query.toQueryString(), "/response/numFound==1]", "/response/docs/[0]/id=='4'"); + } + + @Test + public void embeddedQueryUsedInFiltersWithPreFilter_shouldFilterResultsBeforeTheQueryExecution() + throws Exception { + final String solrQuery = + "{!knn_text_to_vector model=dummy-1 f=vector topK=4 preFilter='id:(1 4 7 8 9)'}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery("id:(3 4 9 2)"); + query.setFilterQueries(solrQuery, "id:(4 20 9)"); + query.add("fl", "id"); + + // topK=4 w/localparam preFilter -> 1,4,7,9 + assertJQ( + "/query" + query.toQueryString(), + "/response/numFound==2]", + "/response/docs/[0]/id=='4'", + "/response/docs/[1]/id=='9'"); + } + + @Test + public void embeddedQueryUsedInFilters_rejectIncludeExclude() throws Exception { + for (String fq : + Arrays.asList( + "{!knn_text_to_vector model=dummy-1 f=vector topK=5 includeTags=xxx}hello world", + "{!knn_text_to_vector model=dummy-1 f=vector topK=5 excludeTags=xxx}hello world")) { + final SolrQuery query = new SolrQuery(); + query.setQuery("*:*"); + query.setFilterQueries(fq); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/error/msg=='Knn Query Parser used as a filter does not support includeTags or excludeTags localparams'", + "/error/code==400"); + } + } + + @Test + public void embeddedQueryAsSubQuery() throws Exception { + final String solrQuery = + "*:* AND {!knn_text_to_vector model=dummy-1 f=vector topK=5 v='hello world'}"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.setFilterQueries("id:(2 4 7 9 8 20 3)"); + query.add("fl", "id"); + + // When knn parser is a subquery, it should not pre-filter on any global fq params + // topK -> 1,4,2,10,3 -> fq -> 4,2,3 + assertJQ( + "/query" + query.toQueryString(), + "/response/numFound==3]", + "/response/docs/[0]/id=='4'", + "/response/docs/[1]/id=='2'", + "/response/docs/[2]/id=='3'"); + } + + @Test + public void embeddedQueryAsSubQuery_withPreFilter() throws Exception { + final String solrQuery = + "*:* AND {!knn_text_to_vector model=dummy-1 f=vector topK=5 preFilter='id:(2 4 7 9 8 20 3)' v='hello world'}"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.add("fl", "id"); + + // knn subquery should still accept `preFilter` local param + // filt -> topK -> 4,2,3,7,9 + assertJQ( + "/query" + query.toQueryString(), + "/response/numFound==5]", + "/response/docs/[0]/id=='4'", + "/response/docs/[1]/id=='2'", + "/response/docs/[2]/id=='3'", + "/response/docs/[3]/id=='7'", + "/response/docs/[4]/id=='9'"); + } + + @Test + public void embeddedQueryAsSubQuery_rejectIncludeExclude() throws Exception { + for (String q : + Arrays.asList( + "{!knn_text_to_vector model=dummy-1 f=vector topK=5 includeTags=xxx}hello world", + "{!knn_text_to_vector model=dummy-1 f=vector topK=5 excludeTags=xxx}hello world")) { + final SolrQuery query = new SolrQuery(); + query.setQuery("*:* OR " + q); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/error/msg=='Knn Query Parser used as a sub-query does not support includeTags or excludeTags localparams'", + "/error/code==400"); + } + } + + @Test + public void embeddedQueryWithCostlyFq_shouldPerformKnnSearchWithPostFilter() throws Exception { + final String solrQuery = "{!knn_text_to_vector model=dummy-1 f=vector topK=10}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.setFilterQueries("{!frange cache=false l=0.99}$q"); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/response/numFound==5]", + "/response/docs/[0]/id=='1'", + "/response/docs/[1]/id=='4'", + "/response/docs/[2]/id=='2'", + "/response/docs/[3]/id=='10'", + "/response/docs/[4]/id=='3'"); + } + + @Test + public void embeddedQueryWithFilterQueries_shouldPerformKnnSearchWithPreFiltersAndPostFilters() + throws Exception { + final String solrQuery = "{!knn_text_to_vector model=dummy-1 f=vector topK=4}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.setFilterQueries("id:(3 4 9 2)", "{!frange cache=false l=0.99}$q"); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/response/numFound==2]", + "/response/docs/[0]/id=='4'", + "/response/docs/[1]/id=='2'"); + } + + @Test + public void embeddedQueryWithNegativeFilterQuery_shouldPerformKnnSearchInPreFilteredResults() + throws Exception { + final String solrQuery = "{!knn_text_to_vector model=dummy-1 f=vector topK=4}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.setFilterQueries("-id:4"); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/response/numFound==4]", + "/response/docs/[0]/id=='1'", + "/response/docs/[1]/id=='2'", + "/response/docs/[2]/id=='10'", + "/response/docs/[3]/id=='3'"); + } + + /** + * See {@link org.apache.solr.search.ReRankQParserPlugin.ReRankQueryRescorer#combine(float, + * boolean, float)}} for more details. + */ + @Test + public void embeddedQueryAsRerank_shouldAddSimilarityFunctionScore() throws Exception { + final SolrQuery query = new SolrQuery(); + query.set("rq", "{!rerank reRankQuery=$rqq reRankDocs=4 reRankWeight=1}"); + query.set("rqq", "{!knn_text_to_vector model=dummy-1 f=vector topK=4}hello world"); + query.setQuery("id:(3 4 9 2)"); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/response/numFound==4]", + "/response/docs/[0]/id=='4'", + "/response/docs/[1]/id=='2'", + "/response/docs/[2]/id=='3'", + "/response/docs/[3]/id=='9'"); + } +} diff --git a/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/store/rest/TestModelManager.java b/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/store/rest/TestModelManager.java new file mode 100644 index 00000000000..37d40b3f6c4 --- /dev/null +++ b/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/store/rest/TestModelManager.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.llm.texttovector.store.rest; + +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.SolrResourceLoader; +import org.apache.solr.llm.TestLlmBase; +import org.apache.solr.llm.texttovector.search.TextToVectorQParserPlugin; +import org.apache.solr.rest.ManagedResource; +import org.apache.solr.rest.ManagedResourceStorage; +import org.apache.solr.rest.RestManager; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestModelManager extends TestLlmBase { + + @BeforeClass + public static void init() throws Exception { + setupTest("solrconfig-llm.xml", "schema.xml", false, false); + } + + @Test + public void test() throws Exception { + final SolrResourceLoader loader = new SolrResourceLoader(tmpSolrHome); + + final RestManager.Registry registry = loader.getManagedResourceRegistry(); + assertNotNull( + "Expected a non-null RestManager.Registry from the SolrResourceLoader!", registry); + + final String resourceId = "/schema/mstore1"; + registry.registerManagedResource( + resourceId, ManagedTextToVectorModelStore.class, new TextToVectorQParserPlugin()); + + final NamedList initArgs = new NamedList<>(); + + final RestManager restManager = new RestManager(); + restManager.init(loader, initArgs, new ManagedResourceStorage.InMemoryStorageIO()); + + final ManagedResource res = restManager.getManagedResource(resourceId); + assertTrue(res instanceof ManagedTextToVectorModelStore); + assertEquals(res.getResourceId(), resourceId); + } + + @Test + public void testRestManagerEndpoints() throws Exception { + assertJQ("/schema/managed", "/responseHeader/status==0"); + + final String cohereModelClassName = "dev.langchain4j.model.cohere.CohereEmbeddingModel"; + + // Add models + String model = "{ \"name\":\"testModel1\", \"class\":\"" + cohereModelClassName + "\"}"; + // fails since it does not have params + assertJPut(ManagedTextToVectorModelStore.REST_END_POINT, model, "/responseHeader/status==400"); + // success + model = + "{ name:\"testModel2\", class:\"" + + cohereModelClassName + + "\"," + + "params:{" + + "baseUrl:\"https://api.cohere.ai/v1/\"," + + "apiKey:\"cohereApiKey2\"," + + "modelName:\"embed-english-light-v3.0\"," + + "inputType:\"search_document\"," + + "logRequests:true," + + "logResponses:false" + + "}}"; + assertJPut(ManagedTextToVectorModelStore.REST_END_POINT, model, "/responseHeader/status==0"); + // success + final String multipleModels = + "[{ name:\"testModel3\", class:\"" + + cohereModelClassName + + "\"," + + "params:{baseUrl:\"https://api.cohere.ai/v1/\"," + + "apiKey:\"cohereApiKey3\"," + + "modelName:\"embed-english-light-v3.0\"," + + "inputType:\"search_document\"," + + "logRequests:true," + + "logResponses:false" + + "}}\n" + + ",{ name:\"testModel4\", class:\"" + + cohereModelClassName + + "\"," + + "params:{baseUrl:\"https://api.cohere.ai/v1/\"," + + "apiKey:\"cohereApiKey4\"," + + "modelName:\"embed-english-light-v3.0\"," + + "inputType:\"search_document\"," + + "logRequests:true," + + "logResponses:false" + + "}}]"; + assertJPut( + ManagedTextToVectorModelStore.REST_END_POINT, multipleModels, "/responseHeader/status==0"); + final String qryResult = JQ(ManagedTextToVectorModelStore.REST_END_POINT); + + assertTrue( + qryResult.contains("\"name\":\"testModel2\"") + && qryResult.contains("\"name\":\"testModel3\"") + && qryResult.contains("\"name\":\"testModel4\"")); + + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/name=='testModel2'"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[1]/name=='testModel3'"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[2]/name=='testModel4'"); + restTestHarness.delete(ManagedTextToVectorModelStore.REST_END_POINT + "/testModel2"); + restTestHarness.delete(ManagedTextToVectorModelStore.REST_END_POINT + "/testModel3"); + restTestHarness.delete(ManagedTextToVectorModelStore.REST_END_POINT + "/testModel4"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models==[]'"); + } + + @Test + public void loadModel_cohere_shouldLoadModelConfig() throws Exception { + loadModel("cohere-model.json"); + + final String modelName = "cohere-1"; + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/name=='" + modelName + "'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/baseUrl=='https://api.cohere.ai/v1/'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/apiKey=='apiKey-cohere'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/modelName=='embed-english-light-v3.0'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/inputType=='search_document'"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/timeout==60"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/logRequests==true"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/logResponses==true"); + + restTestHarness.delete(ManagedTextToVectorModelStore.REST_END_POINT + "/" + modelName); + } + + @Test + public void loadModel_openAi_shouldLoadModelConfig() throws Exception { + loadModel("openai-model.json"); + + final String modelName = "openai-1"; + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/name=='" + modelName + "'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/baseUrl=='https://api.openai.com/v1'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/apiKey=='apiKey-openAI'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/modelName=='text-embedding-3-small'"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/timeout==60"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/logRequests==true"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/logResponses==true"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/maxRetries==5"); + + restTestHarness.delete(ManagedTextToVectorModelStore.REST_END_POINT + "/" + modelName); + } + + @Test + public void loadModel_mistralAi_shouldLoadModelConfig() throws Exception { + loadModel("mistralai-model.json"); + + final String modelName = "mistralai-1"; + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/name=='" + modelName + "'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/baseUrl=='https://api.mistral.ai/v1'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/apiKey=='apiKey-mistralAI'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/modelName=='mistral-embed'"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/timeout==60"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/logRequests==true"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/logResponses==true"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/maxRetries==5"); + + restTestHarness.delete(ManagedTextToVectorModelStore.REST_END_POINT + "/" + modelName); + } + + @Test + public void loadModel_huggingface_shouldLoadModelConfig() throws Exception { + loadModel("huggingface-model.json"); + + final String modelName = "huggingface-1"; + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/name=='" + modelName + "'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/accessToken=='apiKey-huggingface'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/modelId=='sentence-transformers/all-MiniLM-L6-v2'"); + + restTestHarness.delete(ManagedTextToVectorModelStore.REST_END_POINT + "/" + modelName); + } + + @Test + public void loadModel_dummyUnsupportedParam_shouldRaiseError() throws Exception { + loadModel("dummy-model-unsupported.json", "400"); + } + + @Test + public void loadModel_dummyAmbiguousParam_shouldDefaultToString() throws Exception { + loadModel("dummy-model-ambiguous.json"); + + final String modelName = "dummy-1"; + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/name=='" + modelName + "'"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/ambiguous==10"); + + restTestHarness.delete(ManagedTextToVectorModelStore.REST_END_POINT + "/" + modelName); + } +} diff --git a/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/store/rest/TestModelManagerPersistence.java b/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/store/rest/TestModelManagerPersistence.java new file mode 100644 index 00000000000..798e2f091b6 --- /dev/null +++ b/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/store/rest/TestModelManagerPersistence.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.llm.texttovector.store.rest; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import org.apache.solr.common.util.Utils; +import org.apache.solr.llm.TestLlmBase; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class TestModelManagerPersistence extends TestLlmBase { + + @Before + public void init() throws Exception { + setupTest("solrconfig-llm.xml", "schema.xml", false, true); + } + + @After + public void cleanup() throws Exception { + afterTest(); + } + + @Test + public void testModelAreStoredCompact() throws Exception { + loadModel("cohere-model.json"); + + final String JSONOnDisk = Files.readString(embeddingModelStoreFile, StandardCharsets.UTF_8); + Object objectFromDisk = Utils.fromJSONString(JSONOnDisk); + assertEquals(new String(Utils.toJSON(objectFromDisk, -1), UTF_8), JSONOnDisk); + } + + @Test + public void testModelStorePersistence() throws Exception { + // check models are empty + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/==[]"); + + // load models and features from files + loadModel("cohere-model.json"); + + final String modelName = "cohere-1"; + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/name=='" + modelName + "'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/baseUrl=='https://api.cohere.ai/v1/'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/apiKey=='apiKey-cohere'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/modelName=='embed-english-light-v3.0'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/inputType=='search_document'"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/timeout==60"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/logRequests==true"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/logResponses==true"); + + // check persistence after reload + restTestHarness.reload(); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/name=='" + modelName + "'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/baseUrl=='https://api.cohere.ai/v1/'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/apiKey=='apiKey-cohere'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/modelName=='embed-english-light-v3.0'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/inputType=='search_document'"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/timeout==60"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/logRequests==true"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/logResponses==true"); + + // check persistence after restart + getJetty().stop(); + getJetty().start(); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/name=='" + modelName + "'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/baseUrl=='https://api.cohere.ai/v1/'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/apiKey=='apiKey-cohere'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/modelName=='embed-english-light-v3.0'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/inputType=='search_document'"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/timeout==60"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/logRequests==true"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/logResponses==true"); + + // delete loaded models and features + restTestHarness.delete(ManagedTextToVectorModelStore.REST_END_POINT + "/" + modelName); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/==[]"); + + // check persistence after reload + restTestHarness.reload(); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/==[]"); + + // check persistence after restart + getJetty().stop(); + getJetty().start(); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/==[]"); + } +} diff --git a/solr/server/etc/security.policy b/solr/server/etc/security.policy index c898ac8dbfe..1536424cd58 100644 --- a/solr/server/etc/security.policy +++ b/solr/server/etc/security.policy @@ -60,6 +60,15 @@ grant { // needed by randomizedtesting runner to identify test methods. permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; + permission java.lang.reflect.ReflectPermission "newProxyInPackage.dev.langchain4j.model.cohere"; + permission java.lang.reflect.ReflectPermission "newProxyInPackage.dev.ai4j.openai4j"; + permission java.lang.reflect.ReflectPermission "newProxyInPackage.dev.langchain4j.model.huggingface"; + permission java.net.SocketPermission "api.cohere.ai", "accept,listen,connect,resolve"; + permission java.net.SocketPermission "api.openai.com", "accept,listen,connect,resolve"; + permission java.net.SocketPermission "api.mistral.ai", "accept,listen,connect,resolve"; + permission java.net.SocketPermission "api-inference.huggingface.co", "accept,listen,connect,resolve"; + permission java.net.SocketPermission "langchain4j.dev", "accept,listen,connect,resolve"; + permission java.lang.RuntimePermission "accessDeclaredMembers"; // needed by certain tests to redirect sysout/syserr: permission java.lang.RuntimePermission "setIO"; @@ -108,7 +117,6 @@ grant { permission java.lang.RuntimePermission "getProtectionDomain"; // needed by aws s3 sdk (Apache HTTP Client) permission java.lang.RuntimePermission "accessClassInPackage.jdk.internal.reflect"; - // These two *have* to be spelled out a separate permission java.lang.management.ManagementPermission "control"; permission java.lang.management.ManagementPermission "monitor"; diff --git a/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc b/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc index 63f5b016a04..379e502fe1a 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc @@ -242,9 +242,9 @@ client.add(Arrays.asList(d1, d2)); == Query Time -Apache Solr provides two query parsers that work with dense vector fields, that each support different ways of matching documents based on vector similarity: The `knn` query parser, and the `vectorSimilarity` query parser. +Apache Solr provides three query parsers that work with dense vector fields, that each support different ways of matching documents based on vector similarity: The `knn` query parser, the `vectorSimilarity` query parser and the `knn_text_to_vector` query parser. -Both parsers return scores for retrieved documents that are the approximate distance to the target vector (defined by the similarityFunction configured at indexing time) and both support "Pre-Filtering" the document graph to reduce the number of candidate vectors evaluated (with out needing to compute their vector similarity distances). +All parsers return scores for retrieved documents that are the approximate distance to the target vector (defined by the similarityFunction configured at indexing time) and both support "Pre-Filtering" the document graph to reduce the number of candidate vectors evaluated (without needing to compute their vector similarity distances). Common parameters for both query parsers are: @@ -308,6 +308,40 @@ Here's an example of a simple `knn` search: The search results retrieved are the k=10 nearest documents to the vector in input `[1.0, 2.0, 3.0, 4.0]`, ranked by the `similarityFunction` configured at indexing time. + +=== knn_text_to_vector Query Parser + +The `knn_text_to_vector` query parser encode a textual query to a vector using a dedicated Large Language Model(fine tuned for the task of encoding text to vector for sentence similarity) and matches k-nearest neighbours documents to such query vector. + +In addition to the parameters in common with the other dense-retrieval query parsers, it takes the following: + +`model`:: ++ +[%autowidth,frame=none] +|=== +s|Required |Default: none +|=== ++ +The model to use to encode the text to a vector. Must reference an existing model loaded into the `/schema/text-to-vector-model-store`. + +`topK`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: 10 +|=== ++ +How many k-nearest results to return. + +Here's an example of a simple `knn_text_to_vector` search: + +[source,text] +?q={!knn_text_to_vector model=a-model f=vector topK=10}hello world query + +The search results retrieved are the k=10 nearest documents to the vector encoded from the query `hello world query`, using the model `a-model`. + +For more details on how to work with vectorise text in Apache Solr, please refer to the dedicated page: xref:text-to-vector.adoc[Text to Vector] + === vectorSimilarity Query Parser The `vectorSimilarity` vector similarity query parser matches documents whose similarity with the target vector is a above a minimum threshold. @@ -341,6 +375,42 @@ Here's an example of a simple `vectorSimilarity` search: The search results retrieved are all documents whose similarity with the input vector `[1.0, 2.0, 3.0, 4.0]` is at least `0.7` based on the `similarityFunction` configured at indexing time +=== Which one to use? + +Let's see when to use each of the dense retrieval query parsers available: + +== knn Query Parser + +You should use the `knn` query parser when: + +* you search for the top-K closest vectors to a query vector +* you work directly with vectors (no text encoding is involved) +* you want to a have a fine-grained control over the way you encode text to vector and prefer to do it outside of Apache Solr + + +== knn_text_to_vector Query Parser + +You should use the `knn_text_to_vector` query parser when: + +* you search for the top-K closest vectors to a query text +* you work directly with text and want Solr to handle the encoding to vector behind the scenes +* you are building demos/prototypes + +[IMPORTANT] +==== +Apache Solr uses https://github.com/langchain4j/langchain4j[LangChain4j] to interact with Large Language Models. +The integration is experimental and we are going to improve our stress-test and benchmarking coverage of this query parser in future iterations: if you care about raw performance you may prefer to encode the text outside of Solr +==== + +== vectorSimilarity Query Parser + +You should use the `vectorSimilarity` query parser when: + +* you search for the closest vectors to a query vector within a similarity threshold +* you work directly with vectors (no text encoding is involved) +* you want to a have a fine-grained control over the way you encode text to vector and prefer to do it outside of Apache Solr + + === Graph Pre-Filtering Pre-Filtering the set of candidate documents considered when walking the graph can be specified either explicitly, or implicitly (based on existing `fq` params) depending on how and when these dense vector query parsers are used. diff --git a/solr/solr-ref-guide/modules/query-guide/pages/text-to-vector.adoc b/solr/solr-ref-guide/modules/query-guide/pages/text-to-vector.adoc new file mode 100644 index 00000000000..89d7d4a743e --- /dev/null +++ b/solr/solr-ref-guide/modules/query-guide/pages/text-to-vector.adoc @@ -0,0 +1,270 @@ += Text to Vector +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +This module brings the power of *Large Language Models* (*LLM*s) to Solr. More specifically, it provides a text-to-vector capability, used on documents or queries, via integrating with popular external services that do this. The state-of-the-art of such services use an LLM, hence the name of this module. +_Without_ this module, vectors must be supplied _to_ Solr for indexing & searching, possibly coordinating with such services. + + +== Main Concepts + +=== From Text to Vector + +The aim of encoding text to numerical vectors is to represent text in a way that semantically similar sentences are encoded to vectors close in a vector space. +Vector distance metrics (algorithms) can then be used to compute a pairwise similarity, producing a score. + + +=== Large Language Models + +Specific Large Language Models are able to encode text to a numerical vector. + +For additional information you can refer to this https://sease.io/2021/12/using-bert-to-improve-search-relevance.html[blog post]. + +==== Text to Vector Online Services + +Training, fine-tuning and operating such Large Language Models is expensive. + +Many companies focus on this aspect and let users access APIs to encode the text (at the price of a license fee). + +Apache Solr uses https://github.com/langchain4j/langchain4j[LangChain4j] to connect to such apis. + +[IMPORTANT] +==== +This module sends your documents and queries off to some hosted service on the internet. +There are cost, privacy, performance, and service availability implications on such a strong dependency that should be diligently examined before employing this module in a serious way. + +==== + +At the moment a subset of the text vectorisation services supported by LangChain4j is supported by Solr. + +*Disclaimer*: Apache Solr is *in no way* affiliated to any of these corporations or services. + +If you want to add support for additional services or improve the support for the existing ones, feel free to contribute: + +* https://github.com/apache/solr/blob/main/CONTRIBUTING.md[Contributing to Solr] + +== Module + +This is provided via the `llm` xref:configuration-guide:solr-modules.adoc[Solr Module] that needs to be enabled before use. + +== LLM Configuration + +You need to register / configure the plugins provided by the LLM module that you want to use. This is done in `solrconfig.xml`. + +* Declaration of the `knn_text_to_vector` query parser. ++ +[source,xml] +---- + +---- + +== Text to Vector Lifecycle + + +=== Models + +* A model encodes text to a vector. +* A model in Solr is a reference to an external API that runs the Large Language Model responsible for text vectorisation. + +[IMPORTANT] +==== +the Solr vectorisation model specifies the parameters to access the APIs, the model doesn't run internally in Solr + +==== + +A model is described by these parameters: + + +`class`:: ++ +[%autowidth,frame=none] +|=== +s|Required |Default: none +|=== ++ +The model implementation. +Accepted values: + +* `dev.langchain4j.model.huggingface.HuggingFaceEmbeddingModel`. +* `dev.langchain4j.model.mistralai.MistralAiEmbeddingModel`. +* `dev.langchain4j.model.openai.OpenAiEmbeddingModel`. +* `dev.langchain4j.model.cohere.CohereEmbeddingModel`. + + +`name`:: ++ +[%autowidth,frame=none] +|=== +s|Required |Default: none +|=== ++ +The identifier of your model, this is used by any component that intends to use the model (`knn_text_to_vector` query parser). + +`params`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +Each model class has potentially different params. +Many are shared but for the full set of parameters of the model you are interested in please refer to the official documentation of the LangChain4j version included in Solr: https://docs.langchain4j.dev/category/embedding-models[Vectorisationm Models in LangChain4j]. + + +=== Supported Models +Apache Solr uses https://github.com/langchain4j/langchain4j[LangChain4j] to support text vectorisation. +The models currently supported are: + +[tabs#supported-models] +====== +Hugging Face:: ++ +==== + +[source,json] +---- +{ + "class": "dev.langchain4j.model.huggingface.HuggingFaceEmbeddingModel", + "name": "", + "params": { + "accessToken": "", + "modelId": "" + } +} +---- +==== + +MistralAI:: ++ +==== +[source,json] +---- +{ + "class": "dev.langchain4j.model.mistralai.MistralAiEmbeddingModel", + "name": "", + "params": { + "baseUrl": "https://api.mistral.ai/v1", + "apiKey": "", + "modelName": "", + "timeout": 60, + "logRequests": true, + "logResponses": true, + "maxRetries": 5 + } +} +---- +==== + +OpenAI:: ++ +==== +[source,json] +---- +{ + "class": "dev.langchain4j.model.openai.OpenAiEmbeddingModel", + "name": "", + "params": { + "baseUrl": "https://api.openai.com/v1", + "apiKey": "", + "modelName": "", + "timeout": 60, + "logRequests": true, + "logResponses": true, + "maxRetries": 5 + } +} +---- +==== + +Cohere:: ++ +==== +[source,json] +---- +{ + "class": "dev.langchain4j.model.cohere.CohereEmbeddingModel", + "name": "", + "params": { + "baseUrl": "https://api.cohere.ai/v1/", + "apiKey": "", + "modelName": "", + "inputType": "search_document", + "timeout": 60, + "logRequests": true, + "logResponses": true + } +} +---- +==== +====== + +=== Uploading a Model + +To upload the model in a `/path/myModel.json` file, please run: + +[source,bash] +---- +curl -XPUT 'http://localhost:8983/solr/techproducts/schema/vectorisation-model-store' --data-binary "@/path/myModel.json" -H 'Content-type:application/json' +---- + + +To view all models: + +[source,text] +http://localhost:8983/solr/techproducts/schema/vectorisation-model-store + +To delete the `currentModel` model: + +[source,bash] +---- +curl -XDELETE 'http://localhost:8983/solr/techproducts/schema/vectorisation-model-store/currentModel' +---- + + +To view the model you just uploaded please open the following URL in a browser: + +[source,text] +http://localhost:8983/solr/techproducts/schema/vectorisation-model-store + +.Example: /path/myModel.json +[source,json] +---- +{ + "class": "dev.langchain4j.model.openai.OpenAiEmbeddingModel", + "name": "openai-1", + "params": { + "baseUrl": "https://api.openai.com/v1", + "apiKey": "apiKey-openAI", + "modelName": "text-embedding-3-small", + "timeout": 60, + "logRequests": true, + "logResponses": true, + "maxRetries": 5 + } +} + +---- + +=== Running a Text-to-Vector Query +To run a query that embeds your query text, using a model you previously uploaded is simple: + +[source,text] +?q={!knn_text_to_vector model=a-model f=vector topK=10}hello world query + +The search results retrieved are the k=10 nearest documents to the vector encoded from the query `hello world query`, using the model `a-model`. + +For more details on how to work with vector search query parsers in Apache Solr, please refer to the dedicated page: xref:dense-vector-search.adoc[Dense Vector Search] \ No newline at end of file diff --git a/versions.lock b/versions.lock index c1f31bda592..b9780503c73 100644 --- a/versions.lock +++ b/versions.lock @@ -9,7 +9,7 @@ com.epam:parso:2.0.14 (1 constraints: 8e0c750e) com.fasterxml.jackson:jackson-bom:2.18.0 (13 constraints: 2614e9f5) com.fasterxml.jackson.core:jackson-annotations:2.18.0 (13 constraints: abfd0034) com.fasterxml.jackson.core:jackson-core:2.18.0 (16 constraints: 7443a4d1) -com.fasterxml.jackson.core:jackson-databind:2.18.0 (28 constraints: 31054ffd) +com.fasterxml.jackson.core:jackson-databind:2.18.0 (32 constraints: c03ffc6b) com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.18.0 (2 constraints: 641ca1f1) com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.18.0 (2 constraints: ec19ff7a) com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.18.0 (1 constraints: bb0eb266) @@ -45,7 +45,7 @@ com.google.cloud:google-cloud-core:2.40.0 (3 constraints: 3e2f1a54) com.google.cloud:google-cloud-core-grpc:2.40.0 (1 constraints: 1a1001a6) com.google.cloud:google-cloud-core-http:2.40.0 (1 constraints: 1a1001a6) com.google.cloud:google-cloud-storage:2.40.1 (2 constraints: cf1cc626) -com.google.code.gson:gson:2.11.0 (7 constraints: 88614316) +com.google.code.gson:gson:2.11.0 (8 constraints: 566fc864) com.google.errorprone:error_prone_annotations:2.31.0 (15 constraints: a5c51259) com.google.guava:failureaccess:1.0.2 (2 constraints: fb19bf37) com.google.guava:guava:33.1.0-jre (26 constraints: 0280374a) @@ -69,15 +69,19 @@ com.helger:profiler:1.1.1 (1 constraints: e21053b8) com.ibm.icu:icu4j:74.2 (1 constraints: ae0f2484) com.j256.simplemagic:simplemagic:1.17 (1 constraints: dd04f830) com.jayway.jsonpath:json-path:2.9.0 (2 constraints: 6f12c62c) +com.knuddels:jtokkit:1.1.0 (1 constraints: 9d0e146a) com.lmax:disruptor:3.4.4 (1 constraints: 0d050a36) com.mchange:c3p0:0.9.5.5 (1 constraints: c80c571b) com.mchange:mchange-commons-java:0.2.19 (1 constraints: 84075b75) com.pff:java-libpst:0.9.3 (1 constraints: 630cfa01) com.rometools:rome:1.18.0 (1 constraints: 910c870e) com.rometools:rome-utils:1.18.0 (1 constraints: 10095d96) -com.squareup.okhttp3:okhttp:4.12.0 (3 constraints: 11345236) +com.squareup.okhttp3:okhttp:4.12.0 (9 constraints: b987c5a4) +com.squareup.okhttp3:okhttp-sse:4.12.0 (2 constraints: 13189cad) com.squareup.okio:okio:3.6.0 (1 constraints: 530c38fd) com.squareup.okio:okio-jvm:3.6.0 (1 constraints: 500ad3b9) +com.squareup.retrofit2:converter-jackson:2.9.0 (4 constraints: ce36e7ea) +com.squareup.retrofit2:retrofit:2.9.0 (5 constraints: 4a48510b) com.sun.activation:jakarta.activation:1.2.2 (1 constraints: ba0dac35) com.sun.istack:istack-commons-runtime:3.0.12 (1 constraints: eb0d9a43) com.tdunning:t-digest:3.3 (1 constraints: aa04232c) @@ -93,6 +97,12 @@ commons-digester:commons-digester:2.1 (1 constraints: 550fb664) commons-io:commons-io:2.15.1 (10 constraints: ff7571ba) commons-validator:commons-validator:1.7 (1 constraints: a10a3dd2) de.l3s.boilerpipe:boilerpipe:1.1.0 (1 constraints: 590ce401) +dev.ai4j:openai4j:0.22.0 (1 constraints: cf0ee778) +dev.langchain4j:langchain4j-cohere:0.35.0 (1 constraints: 3a053a3b) +dev.langchain4j:langchain4j-core:0.35.0 (5 constraints: 9543f68f) +dev.langchain4j:langchain4j-hugging-face:0.35.0 (1 constraints: 3a053a3b) +dev.langchain4j:langchain4j-mistral-ai:0.35.0 (1 constraints: 3a053a3b) +dev.langchain4j:langchain4j-open-ai:0.35.0 (1 constraints: 3a053a3b) edu.ucar:cdm:4.5.5 (3 constraints: 9d1abd7d) edu.ucar:grib:4.5.5 (1 constraints: 650c0402) edu.ucar:httpservices:4.5.5 (2 constraints: 8f122834) @@ -361,7 +371,7 @@ org.jetbrains:annotations:13.0 (1 constraints: df0e795c) org.jetbrains.kotlin:kotlin-stdlib:1.9.10 (4 constraints: 5c405537) org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10 (3 constraints: 3a2c8a72) org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10 (1 constraints: e210ffd2) -org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10 (4 constraints: e23809e7) +org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10 (8 constraints: 49773549) org.locationtech.jts:jts-core:1.19.0 (2 constraints: a31de760) org.locationtech.jts.io:jts-io-common:1.19.0 (1 constraints: 930d513a) org.locationtech.proj4j:proj4j:1.2.2 (1 constraints: 5d0daf2c) @@ -387,7 +397,7 @@ org.scala-lang.modules:scala-java8-compat_2.13:1.0.2 (1 constraints: fa0af8e7) org.semver4j:semver4j:5.3.0 (1 constraints: 0a050d36) org.slf4j:jcl-over-slf4j:2.0.13 (3 constraints: fa17e8b5) org.slf4j:jul-to-slf4j:2.0.13 (3 constraints: 54285f5f) -org.slf4j:slf4j-api:2.0.13 (74 constraints: 61e60edc) +org.slf4j:slf4j-api:2.0.13 (77 constraints: c90b7945) org.tallison:isoparser:1.9.41.7 (1 constraints: fb0c5528) org.tallison:jmatio:1.5 (1 constraints: ff0b57e9) org.tallison:metadata-extractor:2.17.1.0 (1 constraints: f00c3b28) diff --git a/versions.props b/versions.props index 8931dc3386f..0be5c899dbf 100644 --- a/versions.props +++ b/versions.props @@ -21,6 +21,7 @@ commons-cli:commons-cli=1.9.0 commons-codec:commons-codec=1.17.1 commons-collections:commons-collections=3.2.2 commons-io:commons-io=2.15.1 +dev.langchain4j:langchain4j-*=0.35.0 io.dropwizard.metrics:*=4.2.26 io.grpc:grpc-*=1.65.1 io.jaegertracing:*=1.8.1