Commit ad2d79a0 authored by Maxym Borodenko's avatar Maxym Borodenko

ElasticTrigger and ElasticLoader

parent 5feb3d1c
......@@ -18,8 +18,8 @@ package org.genesys2.server.component.elastic;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.concurrent.BlockingQueue;
......@@ -141,14 +141,36 @@ public class ElasticJPAListener implements InitializingBean {
return;
}
HashSet<Object> visited = new HashSet<Object>();
if (toReindex instanceof Iterable<?>) {
// Iterate and test
Iterable<?> it = (Iterable<?>) toReindex;
List<ElasticReindex> list = new ArrayList<>();
it.forEach((item) -> addNotNull(list, forReindexing(item)));
elasticReindexQueue.addAll(list);
it.forEach((item) -> {
maybeReindex(item, visited);
});
} else {
addNotNull(elasticReindexQueue, forReindexing(toReindex));
maybeReindex(toReindex, visited);
}
}
private void maybeReindex(Object toReindex, HashSet<Object> visited) {
if (toReindex == null || visited.contains(toReindex)) {
return;
}
visited.add(toReindex);
ElasticReindex fri = forReindexing(toReindex);
if (fri != null) {
LOG.trace("Scheduling {}#{}", fri.getClazz(), fri.getId());
addNotNull(elasticReindexQueue, fri);
}
if (toReindex instanceof ElasticTrigger) {
Object[] rels = ((ElasticTrigger) toReindex).reindexedEntities();
if (rels != null) {
for (Object related : rels) {
maybeReindex(related, visited);
}
}
}
}
......@@ -166,7 +188,7 @@ public class ElasticJPAListener implements InitializingBean {
EmptyModel entity = (EmptyModel) toReindex;
return new ElasticReindex(clazz.getName(), entity.getId());
} else {
LOG.warn("Don't know how to index {}. Not a BasicModel.", clazz.getName());
LOG.warn("Don't know how to index {}. Not an EmptyModel.", clazz.getName());
}
}
return null;
......
/*
* Copyright 2020 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys2.server.component.elastic;
/**
* Prepare entity for indexing.
*
* This allows for custom lazy-loading.
*
* @author Matija Obreza
*/
public interface ElasticLoader {
/**
* Lazy-load and initialize before sending to ES for indexing
*/
void prepareForIndexing();
}
/*
* Copyright 2020 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys2.server.component.elastic;
/**
* ElasticTrigger allows us to specify which related entities
* need reindexing.
*/
public interface ElasticTrigger {
/**
* List entities related to this object that need reindexing
* on persist/remove/merge.
*
* @return the object[] null values allowed and ignored.
*/
Object[] reindexedEntities();
}
......@@ -34,6 +34,7 @@ import javax.persistence.UniqueConstraint;
import com.fasterxml.jackson.annotation.JsonIgnore;
import org.apache.commons.lang3.StringUtils;
import org.genesys2.server.component.elastic.ElasticTrigger;
import org.genesys2.server.model.GlobalVersionedAuditedModel;
import org.genesys2.server.model.grin.TaxonomySpecies;
import org.genesys2.server.model.json.Api1Constants;
......@@ -42,7 +43,6 @@ import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldType;
import com.fasterxml.jackson.annotation.JsonIdentityInfo;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
......@@ -55,7 +55,7 @@ import com.fasterxml.jackson.annotation.ObjectIdGenerators;
@Table(name = "taxonomy2", uniqueConstraints = { @UniqueConstraint(name="UK_taxonomy2", columnNames = { "genus", "species", "spAuthor", "subtaxa", "subtAuthor" }) })
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonIdentityInfo(generator = ObjectIdGenerators.PropertyGenerator.class, property = "id", scope = Taxonomy2.class)
public class Taxonomy2 extends GlobalVersionedAuditedModel {
public class Taxonomy2 extends GlobalVersionedAuditedModel implements ElasticTrigger {
private static final long serialVersionUID = 8881324404490162933L;
......@@ -106,6 +106,11 @@ public class Taxonomy2 extends GlobalVersionedAuditedModel {
@Field(type = FieldType.Object)
private TaxonomySpecies currentTaxonomySpecies;
@ManyToOne(cascade = {}, fetch = FetchType.EAGER, optional = true)
@JoinColumn(name = "overrideTaxonomySpecies")
@JsonIgnore
private TaxonomySpecies overrideTaxonomySpecies;
@OneToMany(cascade = {}, fetch = FetchType.LAZY, mappedBy = "taxonomy")
@JsonIgnore
private List<Accession> accessions;
......@@ -241,12 +246,6 @@ public class Taxonomy2 extends GlobalVersionedAuditedModel {
spAuthor, subtaxa, subtAuthor);
}
@ManyToOne(cascade = {}, fetch = FetchType.EAGER, optional = true)
@JoinColumn(name = "overrideTaxonomySpecies")
@JsonIgnore
private TaxonomySpecies overrideTaxonomySpecies;
/**
* Clean up nulls and stuff...
*
......@@ -312,4 +311,9 @@ public class Taxonomy2 extends GlobalVersionedAuditedModel {
public void setOverrideTaxonomySpecies(TaxonomySpecies overrideTaxonomySpecies) {
this.overrideTaxonomySpecies = overrideTaxonomySpecies;
}
@Override
public Object[] reindexedEntities() {
return this.accessions == null ? null : this.accessions.toArray();
}
}
......@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
......@@ -92,6 +93,7 @@ import org.genesys.blocks.model.Publishable;
import org.genesys.blocks.model.VersionedModel;
import org.genesys.blocks.model.filters.EmptyModelFilter;
import org.genesys.custom.elasticsearch.CustomMapping;
import org.genesys2.server.component.elastic.ElasticLoader;
import org.genesys2.server.component.elastic.ElasticQueryBuilder;
import org.genesys2.server.model.json.Api0Constants.Accession;
import org.genesys2.server.model.json.Api1Constants;
......@@ -360,7 +362,9 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
@Override
public List<Class<?>> getIndexedEntities() {
return ListUtils.unmodifiableList(new ArrayList<>(this.indexedEntities));
ArrayList<Class<?>> entities = new ArrayList<>(this.indexedEntities);
entities.sort(Comparator.comparing(Class::getName));
return ListUtils.unmodifiableList(entities);
}
@Override
......@@ -511,8 +515,6 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
return;
}
ensureWriteAlias(clazz);
taskExecutor.execute(() -> {
LOG.debug("Running scheduled reindex of {} size={}", clazz.getName(), copy.size());
try {
......@@ -553,6 +555,10 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
LOG.trace("Indexing {} {}", clazz.getName(), x);
EmptyModel bm = (EmptyModel) x;
if (x instanceof ElasticLoader) {
// Prepare entity for indexing (e.g. lazy-load)
((ElasticLoader) x).prepareForIndexing();
}
try {
jsons.put(bm.getId().toString(), mapper.mapToString(bm));
} catch (IOException e) {
......
......@@ -46,6 +46,7 @@ import org.genesys2.server.component.elastic.ElasticJacksonAnnotationIntrospecto
import org.genesys2.server.component.elastic.ElasticReindex;
import org.genesys2.server.component.elastic.ElasticReindexProcessor;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.Taxonomy2;
import org.genesys2.server.model.grin.TaxonomySpecies;
import org.genesys2.server.model.impl.ActivityPost;
import org.genesys2.server.model.impl.Article;
......@@ -236,6 +237,7 @@ public class ElasticsearchConfig extends ElasticsearchConfigurationSupport {
es.indexEntity(ActivityPost.class);
es.indexEntity(TaxonomySpecies.class); // Index GRIN Taxonomy
es.indexEntity(Taxonomy2.class);
return es;
}
......
......@@ -22,12 +22,15 @@ import com.google.common.collect.Lists;
import org.genesys.test.config.ApplicationConfig;
import org.genesys2.server.api.model.AccessionHeaderJson;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.AccessionId;
import org.genesys2.server.model.genesys.Taxonomy2;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.persistence.AccessionHistoricRepository;
import org.genesys2.server.persistence.AccessionIdRepository;
import org.genesys2.server.persistence.AccessionRepository;
import org.genesys2.server.persistence.FaoInstituteRepository;
import org.genesys2.server.service.InstituteService;
import org.genesys2.server.service.TaxonomyService;
import org.genesys2.server.service.worker.AccessionUploader;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
......@@ -53,6 +56,8 @@ public abstract class AbstractServiceTest extends AbstractTest {
protected AccessionHistoricRepository accessionHistoricRepository;
@Autowired
protected FaoInstituteRepository instituteRepository;
@Autowired
public TaxonomyService taxonomyService;
protected FaoInstitute setupInstitute(final String code){
final FaoInstitute inputI = new FaoInstitute();
......@@ -61,6 +66,12 @@ public abstract class AbstractServiceTest extends AbstractTest {
}
protected Accession upsertAccession(final String instCode, final String acceNumb, final String genus) {
Taxonomy2 taxon = new Taxonomy2();
taxon.setGenus(genus);
return upsertAccession(instCode, acceNumb, taxonomyService.ensureTaxonomy(taxon));
}
protected Accession upsertAccession(final String instCode, final String acceNumb, final Taxonomy2 taxonomy2) {
FaoInstitute institute = instituteService.getInstitute(instCode);
if (institute == null) {
institute = new FaoInstitute();
......@@ -69,17 +80,15 @@ public abstract class AbstractServiceTest extends AbstractTest {
instituteService.update(Lists.newArrayList(institute));
institute = instituteService.getInstitute(instCode);
}
Accession a = new Accession();
AccessionId accessionId = new AccessionId();
a.setAccessionId(accessionId);
a.setInstitute(institute);
a.setAccessionNumber(acceNumb);
a.setTaxonomy(taxonomyService.ensureTaxonomy(taxonomy2));
accessionRepository.save(a);
ObjectMapper objectMapper = new ObjectMapper();
ArrayNode updates = objectMapper.createArrayNode();
ObjectNode accession = updates.addObject();
accession.put("instituteCode", instCode);
accession.put("accessionNumber", acceNumb);
ObjectNode taxa = accession.putObject("taxonomy");
taxa.put("genus", genus);
accessionUploader.upsertAccessions(institute, updates);
return accessionRepository.findOne(institute, null, acceNumb, genus);
return accessionRepository.findOne(institute, null, acceNumb, taxonomy2.getGenus());
}
protected void deleteAccession(final String instCode, final String acceNumb, final String genus) {
......
......@@ -30,6 +30,8 @@ import org.genesys.catalog.model.vocab.ControlledVocabulary;
import org.genesys.catalog.model.vocab.VocabularyTerm;
import org.genesys.custom.elasticsearch.EmbeddedNodeFactoryBean;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.Taxonomy2;
import org.genesys2.server.model.grin.TaxonomySpecies;
import org.genesys2.server.model.impl.ActivityPost;
import org.genesys2.server.model.impl.Article;
import org.genesys2.server.model.impl.Country;
......@@ -146,6 +148,9 @@ public class TestElasticsearchConfig extends ElasticsearchConfigurationSupport i
es.indexEntity(Article.class);
es.indexEntity(ActivityPost.class);
es.indexEntity(TaxonomySpecies.class); // Index GRIN Taxonomy
es.indexEntity(Taxonomy2.class);
return es;
}
......
/*
* Copyright 2020 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.test.server.services;
import org.genesys2.server.component.elastic.ElasticJPAListener;
import org.genesys2.server.component.elastic.ElasticReindexProcessor;
import org.genesys2.server.service.ElasticsearchService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.EnableAspectJAutoProxy;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.ContextHierarchy;
@ContextHierarchy(@ContextConfiguration(name = "services", classes = { AbstractElasticServicesTest.Config.class }))
public abstract class AbstractElasticServicesTest extends AbstractServicesTest {
@Configuration
@EnableAspectJAutoProxy
public static class Config { // extends ApplicationConfig.BaseConfig {
@Bean
public ElasticJPAListener listener() {
return new ElasticJPAListener();
}
@Bean
public ElasticReindexProcessor reindexProcessor() {
return new ElasticReindexProcessor();
}
}
@Autowired
protected ElasticsearchService elasticsearchService;
}
......@@ -30,7 +30,6 @@ import org.genesys2.server.service.GenesysFilterService;
import org.genesys2.server.service.HtmlSanitizer;
import org.genesys2.server.service.InstituteService;
import org.genesys2.server.service.MappingService;
import org.genesys2.server.service.TaxonomyService;
import org.genesys2.server.service.TeamService;
import org.genesys2.server.service.TokenVerificationService;
import org.genesys2.server.service.UserService;
......@@ -60,9 +59,6 @@ public abstract class AbstractServicesTest extends AbstractServiceTest {
@Autowired
public HtmlSanitizer htmlSanitizer;
@Autowired
public TaxonomyService taxonomyService;
@Autowired
public TokenVerificationService tokenVerificationService;
......
/*
* Copyright 2020 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.test.server.services;
import static org.hamcrest.Matchers.*;
import static org.hamcrest.MatcherAssert.assertThat;
import java.util.Date;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.Taxonomy2;
import org.genesys2.server.model.grin.CooperatorOwnedModel;
import org.genesys2.server.model.grin.TaxonomyGenus;
import org.genesys2.server.model.grin.TaxonomySpecies;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.persistence.Taxonomy2Repository;
import org.genesys2.server.persistence.grin.TaxonomyGenusRepository;
import org.genesys2.server.persistence.grin.TaxonomySpeciesRepository;
import org.genesys2.server.service.filter.AccessionFilter;
import org.genesys2.server.service.impl.SearchException;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.transaction.annotation.Transactional;
public class AccessionSearchTest extends AbstractElasticServicesTest {
private static final String INST_CODE = "XYZ001";
private static final String ACCE_NUMB = "A1";
private static final String GENUS = "Abelia";
private static final String SPECIES = "biflora";
private static final String SPECIES_AUTHORITY = "Turcz.";
private static final Long TEST_GRIN_USER_ID = 1L;
@Autowired
private Taxonomy2Repository taxonomy2Repository;
@Autowired
private TaxonomyGenusRepository taxonomyGenusRepository;
@Autowired
private TaxonomySpeciesRepository taxonomySpeciesRepository;
@Before
@Transactional
@Override
public void beforeTest() throws Exception {
super.beforeTest();
FaoInstitute institute = setupInstitute(INST_CODE);
assertThat(institute, notNullValue());
}
@After
@Transactional
@Override
public void cleanup() throws Exception {
accessionRepository.deleteAll();
accessionHistoricRepository.deleteAll();
instituteRepository.deleteAll();
accessionIdRepository.deleteAll();
taxonomy2Repository.deleteAll();
taxonomySpeciesRepository.deleteAll();
taxonomyGenusRepository.deleteAll();
super.cleanup();
}
@Test
public void reindexAccessionWhenTaxonomyUpdatedTest() throws SearchException {
// Add Taxonomy2
Taxonomy2 taxonomy2 = new Taxonomy2();
taxonomy2.setGenus(GENUS);
taxonomy2.setSpecies(SPECIES);
taxonomy2 = taxonomyService.ensureTaxonomy(taxonomy2);
assertThat(taxonomy2, notNullValue());
assertThat(taxonomy2Repository.findByGenusAndSpecies(GENUS, SPECIES), notNullValue());
// Add Accession for added Taxonomy2
Accession accession = upsertAccession(INST_CODE, ACCE_NUMB, taxonomy2);
assertThat(accession, notNullValue());
assertThat(accessionRepository.count(), is(1L));
elasticsearchService.waitForCount(Accession.class, null, 1);
Page<Accession> results = elasticsearchService.findAll(Accession.class, null, PageRequest.of(0, 2));
assertThat(results.getNumberOfElements(), is(1));
AccessionFilter filter = new AccessionFilter();
results = elasticsearchService.findAll(Accession.class, filter, PageRequest.of(0, 10));
assertThat(results.getNumberOfElements(), is(1));
filter._text = "thisdoesntexist";
results = elasticsearchService.findAll(Accession.class, filter, PageRequest.of(0, 10));
assertThat(results.getNumberOfElements(), is(0));
filter._text = SPECIES;
results = elasticsearchService.findAll(Accession.class, filter, PageRequest.of(0, 10));
assertThat(results.getNumberOfElements(), is(1));
// Add GRIN TaxonomyGenus
TaxonomyGenus taxonomyGenus = new TaxonomyGenus(taxonomy2.getId());
taxonomyGenus.setName(GENUS);
taxonomyGenus.setGenusName(GENUS);
populateAuditData(taxonomyGenus);
taxonomyGenus = taxonomyGenusRepository.save(taxonomyGenus);
assertThat(taxonomyGenusRepository.count(), is(1L));
// Add GRIN TaxonomyGenus
TaxonomySpecies taxonomySpecies = new TaxonomySpecies(taxonomy2.getId());
taxonomySpecies.setName(SPECIES);
taxonomySpecies.setSpeciesName(SPECIES);
taxonomySpecies.setTaxonomyGenus(taxonomyGenus);
taxonomySpecies.setSpeciesAuthority(SPECIES_AUTHORITY);
populateAuditData(taxonomySpecies);
taxonomySpecies = taxonomySpeciesRepository.save(taxonomySpecies);
assertThat(taxonomySpeciesRepository.count(), is(1L));
// Update reference to GRIN TaxonomySpecies
taxonomyService.setGrinSpecies(taxonomy2.getId(), taxonomySpecies.getId());
filter._text = SPECIES_AUTHORITY;
assertThat("Related Accession must be reindexed", elasticsearchService.waitForCount(Accession.class, filter, 1), is(1L));
results = elasticsearchService.findAll(Accession.class, filter, PageRequest.of(0, 2));
assertThat(results.getNumberOfElements(), is(1));
}
private void populateAuditData(CooperatorOwnedModel entity) {
entity.setOwnedDate(new Date());
entity.setOwnedById(TEST_GRIN_USER_ID);
entity.setCreatedDate(new Date());
entity.setCreatedById(TEST_GRIN_USER_ID);
}
}
......@@ -48,7 +48,6 @@ public class AccessionServiceTest extends AbstractServicesTest {
private static final String INST_CODE = "XYZ001";
private static final String ACCE_NUMB = "A1";
private FaoInstitute institute;
@Autowired
private RepositoryService repositoryService;
......@@ -63,7 +62,7 @@ public class AccessionServiceTest extends AbstractServicesTest {
public void beforeTest() throws Exception {
super.beforeTest();
institute = setupInstitute(INST_CODE);
FaoInstitute institute = setupInstitute(INST_CODE);
assertNotNull(institute);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment