Skip to content

GitLab

Commit 4e2af4ba authored by Matija Obreza's avatar Matija Obreza
Browse files

Java config for ES, embedded ES 

NonUniqueAccessionException
parent 37308a84
Hide whitespace changes
Inline Side-by-side
pom.xml
View file @ 4e2af4ba
......@@ -419,60 +419,6 @@
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-elasticsearch</artifactId>
<version>1.0.2.RELEASE</version>
<exclusions>
<!-- <exclusion>
<artifactId>lucene-core</artifactId>
<groupId>org.apache.lucene</groupId>
</exclusion> -->
<exclusion>
<artifactId>lucene-sandbox</artifactId>
<groupId>org.apache.lucene</groupId>
</exclusion>
<exclusion>
<artifactId>lucene-queries</artifactId>
<groupId>org.apache.lucene</groupId>
</exclusion>
<exclusion>
<artifactId>lucene-codecs</artifactId>
<groupId>org.apache.lucene</groupId>
</exclusion>
<exclusion>
<artifactId>lucene-grouping</artifactId>
<groupId>org.apache.lucene</groupId>
</exclusion>
<exclusion>
<artifactId>lucene-memory</artifactId>
<groupId>org.apache.lucene</groupId>
</exclusion>
<exclusion>
<artifactId>lucene-spatial</artifactId>
<groupId>org.apache.lucene</groupId>
</exclusion>
<exclusion>
<artifactId>lucene-suggest</artifactId>
<groupId>org.apache.lucene</groupId>
</exclusion>
<exclusion>
<artifactId>lucene-queryparser</artifactId>
<groupId>org.apache.lucene</groupId>
</exclusion>
<exclusion>
<artifactId>lucene-misc</artifactId>
<groupId>org.apache.lucene</groupId>
</exclusion>
<exclusion>
<artifactId>lucene-join</artifactId>
<groupId>org.apache.lucene</groupId>
</exclusion>
<exclusion>
<artifactId>lucene-highlighter</artifactId>
<groupId>org.apache.lucene</groupId>
</exclusion>
<exclusion>
<artifactId>lucene-analyzers-common</artifactId>
<groupId>org.apache.lucene</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
......
src/main/java/org/genesys2/server/model/elastic/AccessionDetails.java
View file @ 4e2af4ba
......@@ -24,7 +24,7 @@ import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldIndex;
import org.springframework.data.elasticsearch.annotations.FieldType;
@Document(indexName = "genesys", type = "mcpd", refreshInterval = "-1")
@Document(indexName = "genesys", type = "mcpd", refreshInterval = "60")
public class AccessionDetails {
private static final Logger LOG = Logger.getLogger(AccessionDetails.class);
......
src/main/java/org/genesys2/server/service/BatchRESTService.java
View file @ 4e2af4ba
......@@ -30,7 +30,7 @@ public interface BatchRESTService {
boolean upsertAccessionData(FaoInstitute institute, Map<AccessionHeaderJson, ObjectNode> batch) throws RESTApiException;
void upsertAccessionNames(FaoInstitute institute, List<AccessionNamesJson> batch);
void upsertAccessionNames(FaoInstitute institute, List<AccessionNamesJson> batch)throws RESTApiException;
int deleteAccessions(FaoInstitute institute, List<AccessionHeaderJson> batch);
int deleteAccessions(FaoInstitute institute, List<AccessionHeaderJson> batch) throws RESTApiException;
}
src/main/java/org/genesys2/server/service/ElasticService.java
View file @ 4e2af4ba
......@@ -16,6 +16,8 @@
package org.genesys2.server.service;
import java.util.Collection;
import org.genesys2.server.model.elastic.AccessionDetails;
import org.genesys2.server.service.impl.SearchException;
import org.springframework.data.domain.Page;
......@@ -33,6 +35,8 @@ public interface ElasticService {
void update(String className, long id);
void updateAll(String className, Collection<Long> bucket);
void refreshIndex(String className);
}
src/main/java/org/genesys2/server/service/GenesysService.java
View file @ 4e2af4ba
......@@ -41,6 +41,7 @@ import org.genesys2.server.model.impl.Country;
import org.genesys2.server.model.impl.Crop;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.model.impl.Organization;
import org.genesys2.server.service.impl.NonUniqueAccessionException;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
......@@ -96,9 +97,9 @@ public interface GenesysService {
List<Accession> listAccessions(List<? extends AccessionIdentifier3> accns);
Accession getAccession(AccessionIdentifier3 aid3);
Accession getAccession(AccessionIdentifier3 aid3) throws NonUniqueAccessionException;
Accession getAccession(String instCode, String acceNumb, String genus);
Accession getAccession(String instCode, String acceNumb, String genus) throws NonUniqueAccessionException;
Page<Object[]> statisticsGenusByInstitute(FaoInstitute faoInstitute, Pageable pageable);
......@@ -125,7 +126,7 @@ public interface GenesysService {
List<Accession> saveAccessions(FaoInstitute institute, List<Accession> matching);
void saveSvalbards(List<SvalbardData> svalbards);
List<SvalbardData> saveSvalbards(List<SvalbardData> svalbards);
long countAvailable(Set<Long> accessionIds);
......@@ -137,13 +138,13 @@ public interface GenesysService {
SvalbardData getSvalbardData(Accession accession);
void saveCollecting(List<AccessionCollect> all);
List<AccessionCollect> saveCollecting(List<AccessionCollect> all);
void saveGeo(List<AccessionGeo> all);
List<AccessionGeo> saveGeo(List<AccessionGeo> all);
void saveBreeding(List<AccessionBreeding> all);
List<AccessionBreeding> saveBreeding(List<AccessionBreeding> all);
void saveExchange(List<AccessionExchange> all);
List<AccessionExchange> saveExchange(List<AccessionExchange> all);
void refreshMetadataMethods();
......@@ -151,13 +152,13 @@ public interface GenesysService {
void writeAccessions(String jsonFilter, OutputStream outputStream) throws IOException;
void saveAliases(List<AccessionAlias> aliases);
List<AccessionAlias> saveAliases(List<AccessionAlias> aliases);
void removeAliases(List<AccessionAlias> aliases);
List<AccessionAlias> removeAliases(List<AccessionAlias> aliases);
void upsertAliases(long accessionId, String acceNames, String otherIds);
void removeAliases(Set<Long> toRemove);
Set<Long> removeAliases(Set<Long> toRemove);
List<AccessionGeo> listAccessionsGeo(Set<Long> copy);
......@@ -175,8 +176,9 @@ public interface GenesysService {
* @param instCode
* @param acceNumb
* @return the 1 accession
* @throws NonUniqueAccessionException
*/
Accession getAccession(String instCode, String acceNumb);
Accession getAccession(String instCode, String acceNumb) throws NonUniqueAccessionException;
List<FaoInstitute> findHoldingInstitutes(Set<Long> accessionIds);
......@@ -184,9 +186,9 @@ public interface GenesysService {
int countAccessions(String jsonFilter);
void saveRemarks(List<AccessionRemark> toSaveRemarks);
List<AccessionRemark> saveRemarks(List<AccessionRemark> toSaveRemarks);
void removeRemarks(List<AccessionRemark> toRemoveRemarks);
List<AccessionRemark> removeRemarks(List<AccessionRemark> toRemoveRemarks);
AccessionDetails getAccessionDetails(long accessionId);
......
src/main/java/org/genesys2/server/service/impl/BatchRESTServiceImpl.java
View file @ 4e2af4ba
......@@ -85,7 +85,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
@Transactional
@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#institute, 'WRITE') or hasPermission(#institute, 'CREATE')")
public boolean upsertAccessionData(FaoInstitute institute, Map<AccessionHeaderJson, ObjectNode> batch) throws RESTApiException {
LOG.info("Batch processing " + batch.size() + " entries for " + institute);
LOG.info("Batch processing " + batch.size() + " entries for " + institute.getCode());
final boolean useUniqueAcceNumbs = institute.hasUniqueAcceNumbs();
......@@ -112,10 +112,15 @@ public class BatchRESTServiceImpl implements BatchRESTService {
}
Accession accession = null;
if (useUniqueAcceNumbs) {
accession = genesysService.getAccession(dataJson.instCode, dataJson.acceNumb);
} else {
accession = genesysService.getAccession(dataJson.instCode, dataJson.acceNumb, dataJson.genus);
try {
if (useUniqueAcceNumbs) {
accession = genesysService.getAccession(dataJson.instCode, dataJson.acceNumb);
} else {
accession = genesysService.getAccession(dataJson.instCode, dataJson.acceNumb, dataJson.genus);
}
} catch (NonUniqueAccessionException e) {
LOG.warn(e.getMessage());
throw new RESTApiException(e.getMessage());
}
boolean updated = false;
......@@ -744,7 +749,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
@Override
@Transactional
@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#institute, 'WRITE') or hasPermission(#institute, 'CREATE')")
public void upsertAccessionNames(FaoInstitute institute, List<AccessionNamesJson> batch) {
public void upsertAccessionNames(FaoInstitute institute, List<AccessionNamesJson> batch) throws RESTApiException {
LOG.info("Batch processing " + batch.size() + " entries for " + institute);
final List<AccessionAlias> toSave = new ArrayList<AccessionAlias>();
......@@ -753,10 +758,15 @@ public class BatchRESTServiceImpl implements BatchRESTService {
for (final AccessionNamesJson dataJson : batch) {
Accession accession = null;
if (useUniqueAcceNumbs) {
accession = genesysService.getAccession(institute.getCode(), dataJson.acceNumb);
} else {
accession = genesysService.getAccession(institute.getCode(), dataJson.acceNumb, dataJson.genus);
try {
if (useUniqueAcceNumbs) {
accession = genesysService.getAccession(institute.getCode(), dataJson.acceNumb);
} else {
accession = genesysService.getAccession(institute.getCode(), dataJson.acceNumb, dataJson.genus);
}
} catch (NonUniqueAccessionException e) {
LOG.warn(e.getMessage());
throw new RESTApiException(e.getMessage());
}
if (accession == null) {
......@@ -832,7 +842,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
@Override
@Transactional
@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#institute, 'DELETE') or hasPermission(#institute, 'MANAGE')")
public int deleteAccessions(FaoInstitute institute, List<AccessionHeaderJson> batch) {
public int deleteAccessions(FaoInstitute institute, List<AccessionHeaderJson> batch) throws RESTApiException {
LOG.info("Batch deleting " + batch.size() + " entries for " + institute);
final List<Accession> toDelete = new ArrayList<Accession>(batch.size());
final boolean useUniqueAcceNumbs = institute.hasUniqueAcceNumbs();
......@@ -847,10 +857,15 @@ public class BatchRESTServiceImpl implements BatchRESTService {
}
Accession accession;
if (useUniqueAcceNumbs) {
accession = genesysService.getAccession(institute.getCode(), dataJson.acceNumb);
} else {
accession = genesysService.getAccession(institute.getCode(), dataJson.acceNumb, dataJson.genus);
try {
if (useUniqueAcceNumbs) {
accession = genesysService.getAccession(institute.getCode(), dataJson.acceNumb);
} else {
accession = genesysService.getAccession(institute.getCode(), dataJson.acceNumb, dataJson.genus);
}
} catch (NonUniqueAccessionException e) {
LOG.warn(e.getMessage());
throw new RESTApiException(e.getMessage());
}
if (accession != null) {
......
src/main/java/org/genesys2/server/service/impl/ElasticsearchSearchServiceImpl.java
View file @ 4e2af4ba
......@@ -2,6 +2,7 @@ package org.genesys2.server.service.impl;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
......@@ -16,6 +17,7 @@ import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.OrganizationService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
......@@ -31,7 +33,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
@Service
public class ElasticsearchSearchServiceImpl implements ElasticService {
public class ElasticsearchSearchServiceImpl implements ElasticService, InitializingBean {
private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchSearchServiceImpl.class);
@Autowired
......@@ -143,10 +145,41 @@ public class ElasticsearchSearchServiceImpl implements ElasticService {
iq.setId(String.valueOf(id));
iq.setObject(eo);
LOG.info("Indexing " + className + " id=" + id);
if (LOG.isDebugEnabled()) {
LOG.debug("Indexing " + className + " id=" + id);
}
elasticsearchTemplate.index(iq);
}
@Override
public void updateAll(String className, Collection<Long> ids) {
if (!clazzMap.containsKey(className)) {
return;
}
LOG.info("Updating " + className + " bulk_size=" + ids.size());
List<IndexQuery> queries = new ArrayList<IndexQuery>();
for (Long id : ids) {
if (id == null)
continue; // Skip null id
Object eo = toElasticObject(className, id);
if (eo == null)
continue; // Skip null results, TODO perhaps delete?
IndexQuery iq = new IndexQuery();
iq.setId(String.valueOf(id));
iq.setObject(eo);
queries.add(iq);
}
if (LOG.isInfoEnabled() && ! queries.isEmpty()) {
LOG.info("Indexing " + className + " count=" + queries.size());
elasticsearchTemplate.bulkIndex(queries);
}
}
@Override
public void remove(String className, long id) {
Class<?> clazz2 = clazzMap.get(className);
......@@ -174,7 +207,7 @@ public class ElasticsearchSearchServiceImpl implements ElasticService {
return;
}
LOG.info("Refresing index " + clazz2);
LOG.info("Refreshing index " + clazz2);
elasticsearchTemplate.refresh(clazz2, true);
}
......@@ -255,4 +288,14 @@ public class ElasticsearchSearchServiceImpl implements ElasticService {
LOG.info("Done.");
}
@Override
public void afterPropertiesSet() throws Exception {
LOG.info("Initializing index");
elasticsearchTemplate.createIndex(AccessionDetails.class);
LOG.info("Putting mapping");
elasticsearchTemplate.putMapping(AccessionDetails.class);
LOG.info("Refreshing");
elasticsearchTemplate.refresh(AccessionDetails.class, true);
}
}
src/main/java/org/genesys2/server/service/impl/GenesysServiceImpl.java
View file @ 4e2af4ba
......@@ -262,30 +262,45 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
}
@Override
public Accession getAccession(AccessionIdentifier3 aid3) {
Accession accession = accessionRepository.findOne(aid3.getHoldingInstitute(), aid3.getAccessionName(), aid3.getGenus());
if (accession != null)
accession.getStoRage().size();
return accession;
public Accession getAccession(AccessionIdentifier3 aid3) throws NonUniqueAccessionException {
try {
Accession accession = accessionRepository.findOne(aid3.getHoldingInstitute(), aid3.getAccessionName(), aid3.getGenus());
if (accession != null)
accession.getStoRage().size();
return accession;
} catch (IncorrectResultSizeDataAccessException e) {
LOG.error("Duplicate accession name instCode=" + aid3.getHoldingInstitute() + " acceNumb=" + aid3.getAccessionName() + " genus=" + aid3.getGenus());
throw new NonUniqueAccessionException(aid3.getHoldingInstitute(), aid3.getAccessionName(), aid3.getGenus());
}
}
@Override
public Accession getAccession(String instCode, String acceNumb) {
Accession accession = accessionRepository.findByInstituteCodeAndAccessionName(instCode, acceNumb);
if (accession != null)
accession.getStoRage().size();
return accession;
public Accession getAccession(String instCode, String acceNumb) throws NonUniqueAccessionException {
try {
Accession accession = accessionRepository.findByInstituteCodeAndAccessionName(instCode, acceNumb);
if (accession != null)
accession.getStoRage().size();
return accession;
} catch (IncorrectResultSizeDataAccessException e) {
LOG.error("Duplicate accession name instCode=" + instCode + " acceNumb=" + acceNumb);
throw new NonUniqueAccessionException(instCode, acceNumb);
}
}
@Override
public Accession getAccession(String instCode, String acceNumb, String genus) {
public Accession getAccession(String instCode, String acceNumb, String genus) throws NonUniqueAccessionException {
if (genus == null) {
throw new NullPointerException("Genus is required to load accession by instCode, acceNumb and genus");
}
Accession accession = accessionRepository.findOne(instCode, acceNumb, genus);
if (accession != null)
accession.getStoRage().size();
return accession;
try {
Accession accession = accessionRepository.findOne(instCode, acceNumb, genus);
if (accession != null)
accession.getStoRage().size();
return accession;
} catch (IncorrectResultSizeDataAccessException e) {
LOG.error("Duplicate accession name instCode=" + instCode + " acceNumb=" + acceNumb);
throw new NonUniqueAccessionException(instCode, acceNumb);
}
}
@Override
......@@ -641,44 +656,51 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
// Worker threads don't carry this information
// @PreAuthorize("hasRole('ADMINISTRATOR')")
@Transactional(readOnly = false)
public void saveSvalbards(List<SvalbardData> svalbards) {
public List<SvalbardData> saveSvalbards(List<SvalbardData> svalbards) {
svalbardRepository.save(svalbards);
return svalbards;
}
@Override
@Transactional(readOnly = false)
public void saveCollecting(List<AccessionCollect> all) {
public List<AccessionCollect> saveCollecting(List<AccessionCollect> all) {
accessionCollectRepository.save(all);
return all;
}
@Override
@Transactional(readOnly = false)
public void saveGeo(List<AccessionGeo> all) {
public List<AccessionGeo> saveGeo(List<AccessionGeo> all) {
accessionGeoRepository.save(all);
return all;
}
@Override
@Transactional(readOnly = false)
public void saveBreeding(List<AccessionBreeding> all) {
public List<AccessionBreeding> saveBreeding(List<AccessionBreeding> all) {
accessionBreedingRepository.save(all);
return all;
}
@Override
@Transactional(readOnly = false)
public void saveExchange(List<AccessionExchange> all) {
public List<AccessionExchange> saveExchange(List<AccessionExchange> all) {
accessionExchangeRepository.save(all);
return all;
}
@Override
@Transactional(readOnly = false)
public void saveRemarks(List<AccessionRemark> toSaveRemarks) {
public List<AccessionRemark> saveRemarks(List<AccessionRemark> toSaveRemarks) {
accessionRemarkRepository.save(toSaveRemarks);
return toSaveRemarks;
}
@Override
@Transactional(readOnly = false)
public void removeRemarks(List<AccessionRemark> toRemoveRemarks) {
public List<AccessionRemark> removeRemarks(List<AccessionRemark> toRemoveRemarks) {
accessionRemarkRepository.delete(toRemoveRemarks);
return toRemoveRemarks;
}
@Override
......@@ -1352,18 +1374,20 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
@Override
@Transactional
public void saveAliases(List<AccessionAlias> aliases) {
public List<AccessionAlias> saveAliases(List<AccessionAlias> aliases) {
if (aliases.size() > 0) {
accessionAliasRepository.save(aliases);
}
return aliases;
}
@Override
@Transactional
public void removeAliases(List<AccessionAlias> aliases) {
public List<AccessionAlias> removeAliases(List<AccessionAlias> aliases) {
if (aliases.size() > 0) {
accessionAliasRepository.delete(aliases);
}
return aliases;
}
@Override
......@@ -1460,10 +1484,11 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
@Override
@Transactional
public void removeAliases(Set<Long> toRemove) {
public Set<Long> removeAliases(Set<Long> toRemove) {
for (final Long id : toRemove) {
this.accessionAliasRepository.delete(id);
}
return toRemove;
}
@Override
......
src/main/java/org/genesys2/server/service/impl/NonUniqueAccessionException.java 0 → 100644
View file @ 4e2af4ba
/**
* Copyright 2014 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
package org.genesys2.server.service.impl;
public class NonUniqueAccessionException extends Exception {
private String instCode;
private String acceNumb;
private String genus;
public NonUniqueAccessionException(String instCode, String acceNumb) {
this.instCode = instCode;
this.acceNumb = acceNumb;
}
public NonUniqueAccessionException(String instCode, String acceNumb, String genus) {
this.instCode = instCode;
this.acceNumb = acceNumb;
this.genus = genus;
}
public String getInstCode() {
return instCode;
}
public String getAcceNumb() {
return acceNumb;
}
@Override
public String getMessage() {
return "Non-unique accession instCode=" + instCode + " acceNumb=" + acceNumb + (genus == null ? "" : " genus=" + genus);
}
}
src/main/java/org/genesys2/server/service/worker/ElasticUpdater.java
View file @ 4e2af4ba
......@@ -59,7 +59,6 @@ public class ElasticUpdater {
* @param ids
*/
public void remove(Class<?> clazz, Long id) {
LOG.info("Removing " + clazz + " " + id);
ElasticNode node = new ElasticNode(clazz, id);
elasticUpdateQueue.remove(node);
elasticRemoveQueue.add(node);
......@@ -87,11 +86,10 @@ public class ElasticUpdater {
* @param ids
*/
public void update(Class<?> clazz, Long id) {
LOG.info("Updating " + clazz + " " + id);
ElasticNode node = new ElasticNode(clazz, id);
elasticRemoveQueue.remove(node);