adds neo4j option

by default it will extract entities from the docs and store them
in the neo4j graph

Makefile

@@ -1,11 +1,12 @@
+SPRING_PROFILES_ACTIVE ?= dev
 
 .PHONY dev:
 dev:
 	@echo "Starting the application..."
 	@echo "Make sure you have a .env file in the root directory"
 	@echo "Exporting environment variables..."
 	$(call setup_env, .env)
-	SPRING_PROFILES_ACTIVE=dev,${SPRING_PROFILES_ACTIVE} ./gradlew bootRun
+	SPRING_PROFILES_ACTIVE=${SPRING_PROFILES_ACTIVE} ./gradlew bootRun
 
 .PHONY dev-docker:
 dev-docker:

build.gradle

@@ -24,6 +24,14 @@ test {
      }
  }
 
+ configurations {
+    all {
+        //exclude group: 'org.springframework.boot', module: 'spring-boot-starter-logging'
+        //exclude group: 'ch.qos.logback', module: 'logback-classic'
+        exclude group: 'org.apache.logging.log4j', module: 'log4j-to-slf4j'
+    }
+}
+
 dependencies {
 	implementation platform('software.amazon.awssdk:bom:2.21.5')
 	implementation 'software.amazon.awssdk:s3'
@@ -66,6 +74,8 @@ dependencies {
 
 	implementation 'org.opensearch.client:opensearch-java:2.9.1'
 
+    implementation 'org.springframework.boot:spring-boot-starter-data-neo4j'
+	implementation 'org.neo4j:neo4j-cypher-dsl:2023.9.7'
 
 
 	runtimeOnly 'com.h2database:h2'

src/main/java/net/savantly/mainbot/api/DocumentApi.java

@@ -3,6 +3,7 @@
 import java.util.List;
 
 import org.springframework.http.ResponseEntity;
+import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
@@ -37,11 +38,9 @@ public ResponseEntity<List<String>> addDocument(@RequestBody DocumentAddRequest
         return ResponseEntity.ok(documentService.addDocument(document));
     }
 
+    @PreAuthorize("${authorization.addDocsExpression}")
     private boolean isUnauthorized(UserDto currentUser, DocumentAddRequest document) {
         log.info("checking authorization for user: {}", currentUser);
-        if (currentUser.getGroups().contains(authorizationConfig.getAdminGroup())) {
-            return false;
-        }
         if (currentUser.getGroups().contains(document.getNamespace())) {
             return false;
         }

src/main/java/net/savantly/mainbot/dom/documents/DocumentsConfiguration.java

@@ -54,7 +54,8 @@ public ChunkingConfiguration chunkingConfiguration() {
     @JsonFormat(shape = JsonFormat.Shape.STRING)
     static enum DocumentServiceImplementationType {
         LC4J,
-        OPENSEARCH
+        OPENSEARCH,
+        NEO4J
     }
 
 }

src/main/java/net/savantly/mainbot/dom/neo4j/Neo4JConfig.java

@@ -0,0 +1,58 @@
+package net.savantly.mainbot.dom.neo4j;
+
+import org.neo4j.cypherdsl.core.renderer.Dialect;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import net.savantly.mainbot.dom.documents.DocumentService;
+import net.savantly.mainbot.dom.documents.impl.LC4JDocumentService;
+import net.savantly.mainbot.dom.documents.processing.DocumentProcessorManager;
+import net.savantly.mainbot.dom.embeddingstore.DocumentEmbedder;
+import net.savantly.mainbot.dom.embeddingstore.EmbeddingStoreProvider;
+import net.savantly.mainbot.dom.neo4j.dto.Neo4JEntityPersistor;
+import net.savantly.mainbot.dom.neo4j.entity.PersonRepository;
+import net.savantly.mainbot.dom.neo4j.entity.PlaceRepository;
+import net.savantly.mainbot.dom.neo4j.entity.ThingRepository;
+
+@Configuration
+@ConfigurationProperties(prefix = "neo4j")
+@ConditionalOnProperty(name = "documents.implementation", havingValue = "NEO4J")
+public class Neo4JConfig {
+
+    @Bean
+    org.neo4j.cypherdsl.core.renderer.Configuration cypherDslConfiguration() {
+        return org.neo4j.cypherdsl.core.renderer.Configuration.newConfig()
+                .withDialect(Dialect.NEO4J_5).build();
+    }
+
+    @Bean
+    @ConditionalOnMissingBean(Neo4JEntityExtractor.class)
+    public Neo4JEntityExtractor neo4JEntityExtractor() {
+        return new Neo4JEntityExtractor() {
+        };
+    }
+
+    @Bean
+    @ConditionalOnMissingBean(Neo4JEntityPersistor.class)
+    public Neo4JEntityPersistor neo4JEntityPersistor(PersonRepository personRepository, PlaceRepository placeRepository,
+            ThingRepository thingRepository) {
+        return new Neo4JEntityPersistor(personRepository, placeRepository, thingRepository);
+    }
+
+    @Bean
+    public DocumentService documentService(
+            Neo4JEntityExtractor entityExtractor,
+            Neo4JEntityPersistor entityPersistor,
+            DocumentEmbedder embedder,
+            EmbeddingStoreProvider storeProvider,
+            DocumentProcessorManager processorManager) {
+
+        var lc4jDocumentService = new LC4JDocumentService(embedder, storeProvider, processorManager);
+
+        return new Neo4JDocumentService(entityExtractor, entityPersistor, lc4jDocumentService);
+    }
+
+}

src/main/java/net/savantly/mainbot/dom/neo4j/Neo4JDocumentService.java

@@ -0,0 +1,36 @@
+package net.savantly.mainbot.dom.neo4j;
+
+import java.util.List;
+
+import net.savantly.mainbot.dom.documents.DocumentAddRequest;
+import net.savantly.mainbot.dom.documents.DocumentQuery;
+import net.savantly.mainbot.dom.documents.DocumentSearchResult;
+import net.savantly.mainbot.dom.documents.DocumentService;
+import net.savantly.mainbot.dom.documents.impl.LC4JDocumentService;
+import net.savantly.mainbot.dom.neo4j.dto.Neo4JEntityPersistor;
+
+public class Neo4JDocumentService implements DocumentService {
+
+    private final Neo4JEntityExtractor entityExtractor;
+    private final Neo4JEntityPersistor entityPersistor;
+    private final LC4JDocumentService lc4jDocumentService;
+
+    public Neo4JDocumentService(Neo4JEntityExtractor entityExtractor, Neo4JEntityPersistor entityPersistor, LC4JDocumentService lc4jDocumentService) {
+        this.entityExtractor = entityExtractor;
+        this.entityPersistor = entityPersistor;
+        this.lc4jDocumentService = lc4jDocumentService;
+    }
+
+    @Override
+    public List<String> addDocument(DocumentAddRequest document) {
+        var extractedEntities = entityExtractor.extractEntities(document.getText());
+        entityPersistor.persist(extractedEntities);
+        return lc4jDocumentService.addDocument(document);
+    }
+
+    @Override
+    public List<DocumentSearchResult> search(DocumentQuery query, String namespace) {
+        return lc4jDocumentService.search(query, namespace);
+    }
+
+}

src/main/java/net/savantly/mainbot/dom/neo4j/Neo4JEntityExtractor.java

@@ -0,0 +1,36 @@
+package net.savantly.mainbot.dom.neo4j;
+
+import net.savantly.mainbot.dom.neo4j.dto.EntityExtraction;
+import net.savantly.mainbot.dom.neo4j.entity.PersonNode;
+import net.savantly.mainbot.dom.neo4j.entity.PlaceNode;
+import net.savantly.mainbot.dom.neo4j.entity.ThingNode;
+
+public interface Neo4JEntityExtractor {
+
+    default EntityExtraction extractEntities(String text) {
+
+        var person1 = new PersonNode().setName("John Doe");
+        var person2 = new PersonNode().setName("Jane Doe");
+
+        person1.getKnows().add(person2);
+
+        var place1 = new PlaceNode().setName("New York");
+        var place2 = new PlaceNode().setName("Los Angeles");
+
+        person1.setLivesIn(place1);
+        person2.setLivesIn(place2);
+
+        person1.setHailsFrom(place2);
+
+        var car = new ThingNode().setName("Car");
+        var house = new ThingNode().setName("House");
+        person1.setHas(car);
+        person2.setHas(house);
+
+        var extracted = new EntityExtraction();
+        extracted.setPeople(java.util.List.of(person1, person2));
+        extracted.setPlaces(java.util.List.of(place1, place2));
+        extracted.setThings(java.util.List.of(car, house));
+        return extracted;
+    };
+}

src/main/java/net/savantly/mainbot/dom/neo4j/dto/EntityExtraction.java

@@ -0,0 +1,18 @@
+package net.savantly.mainbot.dom.neo4j.dto;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.Data;
+import net.savantly.mainbot.dom.neo4j.entity.PersonNode;
+import net.savantly.mainbot.dom.neo4j.entity.PlaceNode;
+import net.savantly.mainbot.dom.neo4j.entity.ThingNode;
+
+@Data
+public class EntityExtraction {
+
+    private List<PersonNode> people = new ArrayList<>();
+    private List<PlaceNode> places = new ArrayList<>();
+    private List<ThingNode> things = new ArrayList<>();
+
+}

src/main/java/net/savantly/mainbot/dom/neo4j/dto/Neo4JEntityPersistor.java

@@ -0,0 +1,22 @@
+package net.savantly.mainbot.dom.neo4j.dto;
+
+import lombok.RequiredArgsConstructor;
+import net.savantly.mainbot.dom.neo4j.entity.PersonRepository;
+import net.savantly.mainbot.dom.neo4j.entity.PlaceRepository;
+import net.savantly.mainbot.dom.neo4j.entity.ThingRepository;
+
+@RequiredArgsConstructor
+public class Neo4JEntityPersistor {
+
+    private final PersonRepository personRepository;
+    private final PlaceRepository placeRepository;
+    private final ThingRepository thingRepository;
+
+    public void persist(EntityExtraction entityExtraction) {
+        personRepository.saveAll(entityExtraction.getPeople());
+        placeRepository.saveAll(entityExtraction.getPlaces());
+        thingRepository.saveAll(entityExtraction.getThings());
+
+    }
+
+}

src/main/java/net/savantly/mainbot/dom/neo4j/dto/Neo4JEntitySearch.java

@@ -0,0 +1,26 @@
+package net.savantly.mainbot.dom.neo4j.dto;
+
+import java.util.List;
+
+import lombok.RequiredArgsConstructor;
+import net.savantly.mainbot.dom.documents.DocumentQuery;
+import net.savantly.mainbot.dom.neo4j.dto.query.Neo4JQueryGenerator;
+import net.savantly.mainbot.dom.neo4j.entity.PersonRepository;
+import net.savantly.mainbot.dom.neo4j.entity.PlaceRepository;
+import net.savantly.mainbot.dom.neo4j.entity.ThingRepository;
+
+@RequiredArgsConstructor
+public class Neo4JEntitySearch {
+
+    private final Neo4JQueryGenerator queryGenerator;
+    private final PersonRepository personRepository;
+    private final PlaceRepository placeRepository;
+    private final ThingRepository thingRepository;
+
+    public List<NodeDTO> search(DocumentQuery query, String namespace) {
+
+        var nodeQuery = queryGenerator.generateQuery(query.getText());
+
+        return null;
+    }
+}

src/main/java/net/savantly/mainbot/dom/neo4j/dto/NodeDTO.java

@@ -0,0 +1,36 @@
+package net.savantly.mainbot.dom.neo4j.dto;
+
+import java.util.Map;
+
+import lombok.Data;
+import lombok.experimental.Accessors;
+
+//** TODO: NOT USED YET */
+@Data
+@Accessors(chain = true)
+public class NodeDTO {
+    private String id;
+     /**
+      * The label of the node
+      * Person, Place, Thing, etc.
+      */
+    private String label;
+
+    /**
+     * The name of the node
+     * John Doe, New York, etc.
+     */
+    private String name;
+
+    /**
+     * The description of the node
+     * A person, a city, etc.
+     */
+    private String description;
+
+    /**
+     * The properties of the node
+     * key-value pairs
+     */
+    private Map<String, Object> properties = new java.util.HashMap<>();
+}

src/main/java/net/savantly/mainbot/dom/neo4j/dto/Relationship.java

@@ -0,0 +1,25 @@
+package net.savantly.mainbot.dom.neo4j.dto;
+
+import lombok.Data;
+import lombok.experimental.Accessors;
+
+@Data
+@Accessors(chain = true)
+public class Relationship {
+
+    /**
+     * The start node of the relationship
+     */
+    private NodeDTO startNode;
+
+    /**
+     * The end node of the relationship
+     */
+    private NodeDTO endNode;
+
+    /**
+     * The type of the relationship
+     * KNOWS, LIVES_IN, etc.
+     */
+    private String type;
+}

src/main/java/net/savantly/mainbot/dom/neo4j/dto/query/Neo4JNodeQuery.java

@@ -0,0 +1,22 @@
+package net.savantly.mainbot.dom.neo4j.dto.query;
+
+import lombok.Data;
+
+@Data
+public class Neo4JNodeQuery {
+
+    /**
+     * The label of the node (Person, Place, etc)
+     */
+    private String label;
+
+    /**
+     * The key to search using the value
+     */
+    private String key;
+
+    /**
+     * The value to search for
+     */
+    private String value;
+}

src/main/java/net/savantly/mainbot/dom/neo4j/dto/query/Neo4JQueryGenerator.java

@@ -0,0 +1,10 @@
+package net.savantly.mainbot.dom.neo4j.dto.query;
+
+import dev.langchain4j.service.SystemMessage;
+
+public interface Neo4JQueryGenerator {
+
+    @SystemMessage("Generate a Cypher query from the given schema and context, for Neo4J database.")
+    public String generateQuery(String text);
+
+}