Skip to content

Commit

Permalink
on subtemplate's update, trigger indexation for every md referencing it
Browse files Browse the repository at this point in the history 
rely on metadataIndexer.batchIndexInThreadPool for async indexing
test threadPoolIndexation starts for expected md
query for 10000 metadata to index
  • Loading branch information
@cmangeat
cmangeat committed Feb 15, 2024
1 parent f5a0710 commit ff7cf0b
Show file tree
Hide file tree
Showing 3 changed files with 86 additions and 10 deletions.
38 changes: 35 additions & 3 deletions core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@

package org.fao.geonet.kernel.datamanager.base;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Function;
import com.google.common.base.Optional;
import com.google.common.base.Predicate;
Expand All @@ -33,6 +35,7 @@
import jeeves.transaction.TransactionTask;
import jeeves.xlink.Processor;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.action.search.SearchResponse;
import org.fao.geonet.ApplicationContextHolder;
import org.fao.geonet.constants.Edit;
import org.fao.geonet.constants.Geonet;
Expand All @@ -45,7 +48,6 @@
import org.fao.geonet.kernel.schema.SchemaPlugin;
import org.fao.geonet.kernel.search.EsSearchManager;
import org.fao.geonet.kernel.search.IndexingMode;
import org.fao.geonet.kernel.search.MetaSearcher;
import org.fao.geonet.kernel.search.index.BatchOpsMetadataReindexer;
import org.fao.geonet.kernel.setting.SettingManager;
import org.fao.geonet.kernel.setting.Settings;
Expand Down Expand Up @@ -104,8 +106,6 @@ public class BaseMetadataManager implements IMetadataManager {
@Autowired
private GroupRepository groupRepository;
@Autowired
private MetadataStatusRepository metadataStatusRepository;
@Autowired
private MetadataValidationRepository metadataValidationRepository;
@Autowired
private MetadataRepository metadataRepository;
Expand Down Expand Up @@ -747,6 +747,38 @@ public synchronized AbstractMetadata updateMetadata(final ServiceContext context
getSearchManager().delete(String.format("+uuid:\"%s\"", uuidBeforeUfo));
}
metadataIndexer.indexMetadata(metadataId, true, indexingMode);

if (metadata.getDataInfo().getType() == MetadataType.SUB_TEMPLATE) {
ObjectMapper objectMapper = new ObjectMapper();

JsonNode query = objectMapper.createObjectNode()
.set("bool", objectMapper.createObjectNode().set("must", objectMapper.createArrayNode()
.add(objectMapper.createObjectNode().set("query_string",
objectMapper.createObjectNode().put("query", String.format("xlink:*%s*", metadata.getUuid()))))
)
);
/*
verify needs for these parameters
parameters.addContent(new Element(Geonet.SearchResult.BUILD_SUMMARY).setText("false"));
parameters.addContent(new Element(SearchParameter.ISADMIN).addContent("true"));
parameters.addContent(new Element(SearchParameter.ISTEMPLATE).addContent("y or n"));
*/

SearchResponse response = this.searchManager.query(query, Collections.singleton("_id"), 0, 10000, null);
ArrayList<String> toIndex = new ArrayList<>();
response.getHits().forEach(consumer -> {
String consumerUuid = consumer.getId();
try {
String consumerId = this.metadataUtils.getMetadataId(consumerUuid);
if (consumerId != null) {
toIndex.add(consumerId);
}
} catch (Exception e) {
throw new RuntimeException(e);
}
});
metadataIndexer.batchIndexInThreadPool(context, toIndex);
}
}
}

Expand Down
53 changes: 46 additions & 7 deletions core/src/test/java/org/fao/geonet/kernel/LocalXLinksUpdateDeleteTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,14 @@
package org.fao.geonet.kernel;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import jeeves.server.context.ServiceContext;
import org.elasticsearch.action.search.SearchResponse;
import org.fao.geonet.AbstractCoreIntegrationTest;
import org.fao.geonet.domain.AbstractMetadata;
import org.fao.geonet.kernel.datamanager.IMetadataManager;
import org.fao.geonet.kernel.datamanager.base.BaseMetadataIndexer;
import org.fao.geonet.kernel.search.EsSearchManager;
import org.fao.geonet.kernel.search.IndexingMode;
import org.fao.geonet.kernel.setting.SettingManager;
import org.fao.geonet.kernel.setting.Settings;
Expand All @@ -12,30 +17,41 @@
import org.jdom.Element;
import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;

import java.net.URL;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

import static org.fao.geonet.domain.MetadataType.SUB_TEMPLATE;
import static org.fao.geonet.domain.MetadataType.TEMPLATE;
import static org.fao.geonet.schema.iso19139.ISO19139Namespaces.GCO;
import static org.fao.geonet.schema.iso19139.ISO19139Namespaces.GMD;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.when;


public class LocalXLinksUpdateDeleteTest extends AbstractIntegrationTestWithMockedSingletons {
@Autowired
private IMetadataManager metadataManager;

@Autowired
private SettingManager settingManager;

@Autowired
private EsSearchManager searchManager;

private ServiceContext serviceContext;

@Autowired
private BaseMetadataIndexer metadataIndexer;

@Before
public void setUp() throws Exception {
serviceContext = createServiceContext();
Expand All @@ -48,10 +64,8 @@ public void updateHasToRegisterReferrersForIndexation() throws Exception {
Element contactElement = Xml.loadStream(contactResource.openStream());
AbstractMetadata contactMetadata = insertContact(contactElement);
AbstractMetadata vicinityMapMetadata = insertVicinityMap(contactMetadata);

// TODOES
// Object document = searchForMetadataTagged("babar");
// assertFalse(context.getBean(IndexingList.class).getIdentifiers().contains(vicinityMapMetadata.getId()));
assertEquals(1,searchForMetadataTagged(vicinityMapMetadata.getUuid(), "babar").getHits().getHits().length);
assertEquals(0,searchForMetadataTagged(vicinityMapMetadata.getUuid(), "momo").getHits().getHits().length);

Xml.selectElement(contactElement, "gmd:individualName/gco:CharacterString", Arrays.asList(GMD, GCO)).setText("momo");
metadataManager.updateMetadata(serviceContext,
Expand All @@ -64,8 +78,24 @@ public void updateHasToRegisterReferrersForIndexation() throws Exception {
false,
IndexingMode.full);

// TODOES assertEquals(vicinityMapMetadata.getUuid(), document.getField("_uuid").stringValue());
// assertTrue(context.getBean(IndexingList.class).getIdentifiers().contains(vicinityMapMetadata.getId()));
ArgumentCaptor<List> toIndexCaptor = ArgumentCaptor.forClass(List.class);
Mockito.verify(metadataIndexer).batchIndexInThreadPool(Mockito.anyObject(), toIndexCaptor.capture());
assertTrue(toIndexCaptor.getValue().contains(Integer.toString(vicinityMapMetadata.getId())));
}

private SearchResponse searchForMetadataTagged(String mdUuid, String individual) throws Exception {
ObjectMapper objectMapper = new ObjectMapper();

JsonNode query = objectMapper.createObjectNode()
.set("nested", objectMapper.createObjectNode()
.put("path", "contactForResource")
.set("query", objectMapper.createObjectNode()
.set("bool", objectMapper.createObjectNode().set("must", objectMapper.createArrayNode()
.add(objectMapper.createObjectNode().set("match", objectMapper.createObjectNode().put("_id", mdUuid)))
.add(objectMapper.createObjectNode().set("match", objectMapper.createObjectNode().put("contactForResource.individual", individual))))))
);

return this.searchManager.query(query, Collections.emptySet(), 0, 10, null);
}

@Test
Expand Down Expand Up @@ -124,4 +154,13 @@ private AbstractMetadata insertContact(Element contactElement) throws Exception
when(springLocalServiceInvoker.invoke(any(String.class))).thenReturn(contactElement);
return contactMetadata;
}

static public class SpyFactory {
@Autowired
BaseMetadataIndexer baseMetadataIndexer;

BaseMetadataIndexer build() {
return Mockito.spy(baseMetadataIndexer);
}
}
}
5 changes: 5 additions & 0 deletions core/src/test/resources/mocked-core-repository-test-context.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,9 @@
<bean id="localInvokerMock" class="org.mockito.Mockito" factory-method="mock" primary="true">
<constructor-arg value="org.fao.geonet.kernel.SpringLocalServiceInvoker" />
</bean>

<bean id="spyFactory" class="org.fao.geonet.kernel.LocalXLinksUpdateDeleteTest.SpyFactory" autowire = "byType"/>

<bean id="baseMetadataIndexerSpy" factory-bean="spyFactory" factory-method="build" primary="true" />

</beans>

0 comments on commit ff7cf0b

Please sign in to comment.