Java config for ES, embedded ES

NonUniqueAccessionException

pom.xml

@@ -419,60 +419,6 @@
 			<groupId>org.springframework.data</groupId>
 			<artifactId>spring-data-elasticsearch</artifactId>
 			<version>1.0.2.RELEASE</version>
-			<exclusions>
-				<!-- <exclusion>
-					<artifactId>lucene-core</artifactId>
-					<groupId>org.apache.lucene</groupId>
-				</exclusion> -->
-				<exclusion>
-					<artifactId>lucene-sandbox</artifactId>
-					<groupId>org.apache.lucene</groupId>
-				</exclusion>
-				<exclusion>
-					<artifactId>lucene-queries</artifactId>
-					<groupId>org.apache.lucene</groupId>
-				</exclusion>
-				<exclusion>
-					<artifactId>lucene-codecs</artifactId>
-					<groupId>org.apache.lucene</groupId>
-				</exclusion>
-				<exclusion>
-					<artifactId>lucene-grouping</artifactId>
-					<groupId>org.apache.lucene</groupId>
-				</exclusion>
-				<exclusion>
-					<artifactId>lucene-memory</artifactId>
-					<groupId>org.apache.lucene</groupId>
-				</exclusion>
-				<exclusion>
-					<artifactId>lucene-spatial</artifactId>
-					<groupId>org.apache.lucene</groupId>
-				</exclusion>
-				<exclusion>
-					<artifactId>lucene-suggest</artifactId>
-					<groupId>org.apache.lucene</groupId>
-				</exclusion>
-				<exclusion>
-					<artifactId>lucene-queryparser</artifactId>
-					<groupId>org.apache.lucene</groupId>
-				</exclusion>
-				<exclusion>
-					<artifactId>lucene-misc</artifactId>
-					<groupId>org.apache.lucene</groupId>
-				</exclusion>
-				<exclusion>
-					<artifactId>lucene-join</artifactId>
-					<groupId>org.apache.lucene</groupId>
-				</exclusion>
-				<exclusion>
-					<artifactId>lucene-highlighter</artifactId>
-					<groupId>org.apache.lucene</groupId>
-				</exclusion>
-				<exclusion>
-					<artifactId>lucene-analyzers-common</artifactId>
-					<groupId>org.apache.lucene</groupId>
-				</exclusion>
-			</exclusions>
 		</dependency>
 		
 		<dependency>

src/main/java/org/genesys2/server/model/elastic/AccessionDetails.java

@@ -24,7 +24,7 @@ import org.springframework.data.elasticsearch.annotations.Field;
 import org.springframework.data.elasticsearch.annotations.FieldIndex;
 import org.springframework.data.elasticsearch.annotations.FieldType;
 
-@Document(indexName = "genesys", type = "mcpd", refreshInterval = "-1")
+@Document(indexName = "genesys", type = "mcpd", refreshInterval = "60")
 public class AccessionDetails {
 	private static final Logger LOG = Logger.getLogger(AccessionDetails.class);
 

src/main/java/org/genesys2/server/service/BatchRESTService.java

@@ -30,7 +30,7 @@ public interface BatchRESTService {
 
 	boolean upsertAccessionData(FaoInstitute institute, Map<AccessionHeaderJson, ObjectNode> batch) throws RESTApiException;
 
-	void upsertAccessionNames(FaoInstitute institute, List<AccessionNamesJson> batch);
+	void upsertAccessionNames(FaoInstitute institute, List<AccessionNamesJson> batch)throws RESTApiException;
 
-	int deleteAccessions(FaoInstitute institute, List<AccessionHeaderJson> batch);
+	int deleteAccessions(FaoInstitute institute, List<AccessionHeaderJson> batch) throws RESTApiException;
 }

src/main/java/org/genesys2/server/service/ElasticService.java

@@ -16,6 +16,8 @@
 
 package org.genesys2.server.service;
 
+import java.util.Collection;
+
 import org.genesys2.server.model.elastic.AccessionDetails;
 import org.genesys2.server.service.impl.SearchException;
 import org.springframework.data.domain.Page;
@@ -33,6 +35,8 @@ public interface ElasticService {
 
 	void update(String className, long id);
 
+	void updateAll(String className, Collection<Long> bucket);
+
 	void refreshIndex(String className);
 
 }

src/main/java/org/genesys2/server/service/GenesysService.java

@@ -41,6 +41,7 @@ import org.genesys2.server.model.impl.Country;
 import org.genesys2.server.model.impl.Crop;
 import org.genesys2.server.model.impl.FaoInstitute;
 import org.genesys2.server.model.impl.Organization;
+import org.genesys2.server.service.impl.NonUniqueAccessionException;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
 
@@ -96,9 +97,9 @@ public interface GenesysService {
 
 	List<Accession> listAccessions(List<? extends AccessionIdentifier3> accns);
 
-	Accession getAccession(AccessionIdentifier3 aid3);
+	Accession getAccession(AccessionIdentifier3 aid3) throws NonUniqueAccessionException;
 
-	Accession getAccession(String instCode, String acceNumb, String genus);
+	Accession getAccession(String instCode, String acceNumb, String genus) throws NonUniqueAccessionException;
 
 	Page<Object[]> statisticsGenusByInstitute(FaoInstitute faoInstitute, Pageable pageable);
 
@@ -125,7 +126,7 @@ public interface GenesysService {
 
 	List<Accession> saveAccessions(FaoInstitute institute, List<Accession> matching);
 
-	void saveSvalbards(List<SvalbardData> svalbards);
+	List<SvalbardData> saveSvalbards(List<SvalbardData> svalbards);
 
 	long countAvailable(Set<Long> accessionIds);
 
@@ -137,13 +138,13 @@ public interface GenesysService {
 
 	SvalbardData getSvalbardData(Accession accession);
 
-	void saveCollecting(List<AccessionCollect> all);
+	List<AccessionCollect> saveCollecting(List<AccessionCollect> all);
 
-	void saveGeo(List<AccessionGeo> all);
+	List<AccessionGeo> saveGeo(List<AccessionGeo> all);
 
-	void saveBreeding(List<AccessionBreeding> all);
+	List<AccessionBreeding> saveBreeding(List<AccessionBreeding> all);
 
-	void saveExchange(List<AccessionExchange> all);
+	List<AccessionExchange> saveExchange(List<AccessionExchange> all);
 
 	void refreshMetadataMethods();
 
@@ -151,13 +152,13 @@ public interface GenesysService {
 
 	void writeAccessions(String jsonFilter, OutputStream outputStream) throws IOException;
 
-	void saveAliases(List<AccessionAlias> aliases);
+	List<AccessionAlias> saveAliases(List<AccessionAlias> aliases);
 
-	void removeAliases(List<AccessionAlias> aliases);
+	List<AccessionAlias> removeAliases(List<AccessionAlias> aliases);
 
 	void upsertAliases(long accessionId, String acceNames, String otherIds);
 
-	void removeAliases(Set<Long> toRemove);
+	Set<Long> removeAliases(Set<Long> toRemove);
 
 	List<AccessionGeo> listAccessionsGeo(Set<Long> copy);
 
@@ -175,8 +176,9 @@ public interface GenesysService {
 	 * @param instCode
 	 * @param acceNumb
 	 * @return the 1 accession
+	 * @throws NonUniqueAccessionException
 	 */
-	Accession getAccession(String instCode, String acceNumb);
+	Accession getAccession(String instCode, String acceNumb) throws NonUniqueAccessionException;
 
 	List<FaoInstitute> findHoldingInstitutes(Set<Long> accessionIds);
 
@@ -184,9 +186,9 @@ public interface GenesysService {
 
 	int countAccessions(String jsonFilter);
 
-	void saveRemarks(List<AccessionRemark> toSaveRemarks);
+	List<AccessionRemark> saveRemarks(List<AccessionRemark> toSaveRemarks);
 
-	void removeRemarks(List<AccessionRemark> toRemoveRemarks);
+	List<AccessionRemark> removeRemarks(List<AccessionRemark> toRemoveRemarks);
 
 	AccessionDetails getAccessionDetails(long accessionId);
 

src/main/java/org/genesys2/server/service/impl/BatchRESTServiceImpl.java

@@ -85,7 +85,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
 	@Transactional
 	@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#institute, 'WRITE') or hasPermission(#institute, 'CREATE')")
 	public boolean upsertAccessionData(FaoInstitute institute, Map<AccessionHeaderJson, ObjectNode> batch) throws RESTApiException {
-		LOG.info("Batch processing " + batch.size() + " entries for " + institute);
+		LOG.info("Batch processing " + batch.size() + " entries for " + institute.getCode());
 
 		final boolean useUniqueAcceNumbs = institute.hasUniqueAcceNumbs();
 
@@ -112,10 +112,15 @@ public class BatchRESTServiceImpl implements BatchRESTService {
 			}
 
 			Accession accession = null;
-			if (useUniqueAcceNumbs) {
-				accession = genesysService.getAccession(dataJson.instCode, dataJson.acceNumb);
-			} else {
-				accession = genesysService.getAccession(dataJson.instCode, dataJson.acceNumb, dataJson.genus);
+			try {
+				if (useUniqueAcceNumbs) {
+					accession = genesysService.getAccession(dataJson.instCode, dataJson.acceNumb);
+				} else {
+					accession = genesysService.getAccession(dataJson.instCode, dataJson.acceNumb, dataJson.genus);
+				}
+			} catch (NonUniqueAccessionException e) {
+				LOG.warn(e.getMessage());
+				throw new RESTApiException(e.getMessage());
 			}
 
 			boolean updated = false;
@@ -744,7 +749,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
 	@Override
 	@Transactional
 	@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#institute, 'WRITE') or hasPermission(#institute, 'CREATE')")
-	public void upsertAccessionNames(FaoInstitute institute, List<AccessionNamesJson> batch) {
+	public void upsertAccessionNames(FaoInstitute institute, List<AccessionNamesJson> batch) throws RESTApiException {
 		LOG.info("Batch processing " + batch.size() + " entries for " + institute);
 
 		final List<AccessionAlias> toSave = new ArrayList<AccessionAlias>();
@@ -753,10 +758,15 @@ public class BatchRESTServiceImpl implements BatchRESTService {
 
 		for (final AccessionNamesJson dataJson : batch) {
 			Accession accession = null;
-			if (useUniqueAcceNumbs) {
-				accession = genesysService.getAccession(institute.getCode(), dataJson.acceNumb);
-			} else {
-				accession = genesysService.getAccession(institute.getCode(), dataJson.acceNumb, dataJson.genus);
+			try {
+				if (useUniqueAcceNumbs) {
+					accession = genesysService.getAccession(institute.getCode(), dataJson.acceNumb);
+				} else {
+					accession = genesysService.getAccession(institute.getCode(), dataJson.acceNumb, dataJson.genus);
+				}
+			} catch (NonUniqueAccessionException e) {
+				LOG.warn(e.getMessage());
+				throw new RESTApiException(e.getMessage());
 			}
 
 			if (accession == null) {
@@ -832,7 +842,7 @@ public class BatchRESTServiceImpl implements BatchRESTService {
 	@Override
 	@Transactional
 	@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#institute, 'DELETE') or hasPermission(#institute, 'MANAGE')")
-	public int deleteAccessions(FaoInstitute institute, List<AccessionHeaderJson> batch) {
+	public int deleteAccessions(FaoInstitute institute, List<AccessionHeaderJson> batch) throws RESTApiException {
 		LOG.info("Batch deleting " + batch.size() + " entries for " + institute);
 		final List<Accession> toDelete = new ArrayList<Accession>(batch.size());
 		final boolean useUniqueAcceNumbs = institute.hasUniqueAcceNumbs();
@@ -847,10 +857,15 @@ public class BatchRESTServiceImpl implements BatchRESTService {
 			}
 
 			Accession accession;
-			if (useUniqueAcceNumbs) {
-				accession = genesysService.getAccession(institute.getCode(), dataJson.acceNumb);
-			} else {
-				accession = genesysService.getAccession(institute.getCode(), dataJson.acceNumb, dataJson.genus);
+			try {
+				if (useUniqueAcceNumbs) {
+					accession = genesysService.getAccession(institute.getCode(), dataJson.acceNumb);
+				} else {
+					accession = genesysService.getAccession(institute.getCode(), dataJson.acceNumb, dataJson.genus);
+				}
+			} catch (NonUniqueAccessionException e) {
+				LOG.warn(e.getMessage());
+				throw new RESTApiException(e.getMessage());
 			}
 
 			if (accession != null) {

src/main/java/org/genesys2/server/service/impl/ElasticsearchSearchServiceImpl.java

@@ -2,6 +2,7 @@ package org.genesys2.server.service.impl;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -16,6 +17,7 @@ import org.genesys2.server.service.GenesysService;
 import org.genesys2.server.service.OrganizationService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.PageRequest;
@@ -31,7 +33,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 
 @Service
-public class ElasticsearchSearchServiceImpl implements ElasticService {
+public class ElasticsearchSearchServiceImpl implements ElasticService, InitializingBean {
 	private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchSearchServiceImpl.class);
 
 	@Autowired
@@ -143,10 +145,41 @@ public class ElasticsearchSearchServiceImpl implements ElasticService {
 		iq.setId(String.valueOf(id));
 		iq.setObject(eo);
 
-		LOG.info("Indexing " + className + " id=" + id);
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("Indexing " + className + " id=" + id);
+		}
 		elasticsearchTemplate.index(iq);
 	}
 
+	@Override
+	public void updateAll(String className, Collection<Long> ids) {
+		if (!clazzMap.containsKey(className)) {
+			return;
+		}
+		LOG.info("Updating " + className + " bulk_size=" + ids.size());
+		List<IndexQuery> queries = new ArrayList<IndexQuery>();
+
+		for (Long id : ids) {
+			if (id == null)
+				continue; // Skip null id
+
+			Object eo = toElasticObject(className, id);
+			if (eo == null)
+				continue; // Skip null results, TODO perhaps delete?
+
+			IndexQuery iq = new IndexQuery();
+			iq.setId(String.valueOf(id));
+			iq.setObject(eo);
+
+			queries.add(iq);
+		}
+
+		if (LOG.isInfoEnabled() && ! queries.isEmpty()) {
+			LOG.info("Indexing " + className + " count=" + queries.size());
+			elasticsearchTemplate.bulkIndex(queries);
+		}
+	}
+
 	@Override
 	public void remove(String className, long id) {
 		Class<?> clazz2 = clazzMap.get(className);
@@ -174,7 +207,7 @@ public class ElasticsearchSearchServiceImpl implements ElasticService {
 
 			return;
 		}
-		LOG.info("Refresing index " + clazz2);
+		LOG.info("Refreshing index " + clazz2);
 		elasticsearchTemplate.refresh(clazz2, true);
 	}
 
@@ -255,4 +288,14 @@ public class ElasticsearchSearchServiceImpl implements ElasticService {
 		LOG.info("Done.");
 	}
 
+	@Override
+	public void afterPropertiesSet() throws Exception {
+		LOG.info("Initializing index");
+		elasticsearchTemplate.createIndex(AccessionDetails.class);
+		LOG.info("Putting mapping");
+		elasticsearchTemplate.putMapping(AccessionDetails.class);
+		LOG.info("Refreshing");
+		elasticsearchTemplate.refresh(AccessionDetails.class, true);
+	}
+
 }

src/main/java/org/genesys2/server/service/impl/GenesysServiceImpl.java

@@ -262,30 +262,45 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
 	}
 
 	@Override
-	public Accession getAccession(AccessionIdentifier3 aid3) {
-		Accession accession = accessionRepository.findOne(aid3.getHoldingInstitute(), aid3.getAccessionName(), aid3.getGenus());
-		if (accession != null)
-			accession.getStoRage().size();
-		return accession;
+	public Accession getAccession(AccessionIdentifier3 aid3) throws NonUniqueAccessionException {
+		try {
+			Accession accession = accessionRepository.findOne(aid3.getHoldingInstitute(), aid3.getAccessionName(), aid3.getGenus());
+			if (accession != null)
+				accession.getStoRage().size();
+			return accession;
+		} catch (IncorrectResultSizeDataAccessException e) {
+			LOG.error("Duplicate accession name instCode=" + aid3.getHoldingInstitute() + " acceNumb=" + aid3.getAccessionName() + " genus=" + aid3.getGenus());
+			throw new NonUniqueAccessionException(aid3.getHoldingInstitute(), aid3.getAccessionName(), aid3.getGenus());
+		}
 	}
 
 	@Override
-	public Accession getAccession(String instCode, String acceNumb) {
-		Accession accession = accessionRepository.findByInstituteCodeAndAccessionName(instCode, acceNumb);
-		if (accession != null)
-			accession.getStoRage().size();
-		return accession;
+	public Accession getAccession(String instCode, String acceNumb) throws NonUniqueAccessionException {
+		try {
+			Accession accession = accessionRepository.findByInstituteCodeAndAccessionName(instCode, acceNumb);
+			if (accession != null)
+				accession.getStoRage().size();
+			return accession;
+		} catch (IncorrectResultSizeDataAccessException e) {
+			LOG.error("Duplicate accession name instCode=" + instCode + " acceNumb=" + acceNumb);
+			throw new NonUniqueAccessionException(instCode, acceNumb);
+		}
 	}
 
 	@Override
-	public Accession getAccession(String instCode, String acceNumb, String genus) {
+	public Accession getAccession(String instCode, String acceNumb, String genus) throws NonUniqueAccessionException {
 		if (genus == null) {
 			throw new NullPointerException("Genus is required to load accession by instCode, acceNumb and genus");
 		}
-		Accession accession = accessionRepository.findOne(instCode, acceNumb, genus);
-		if (accession != null)
-			accession.getStoRage().size();
-		return accession;
+		try {
+			Accession accession = accessionRepository.findOne(instCode, acceNumb, genus);
+			if (accession != null)
+				accession.getStoRage().size();
+			return accession;
+		} catch (IncorrectResultSizeDataAccessException e) {
+			LOG.error("Duplicate accession name instCode=" + instCode + " acceNumb=" + acceNumb);
+			throw new NonUniqueAccessionException(instCode, acceNumb);
+		}
 	}
 
 	@Override
@@ -641,44 +656,51 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
 	// Worker threads don't carry this information
 	// @PreAuthorize("hasRole('ADMINISTRATOR')")
 	@Transactional(readOnly = false)
-	public void saveSvalbards(List<SvalbardData> svalbards) {
+	public List<SvalbardData> saveSvalbards(List<SvalbardData> svalbards) {
 		svalbardRepository.save(svalbards);
+		return svalbards;
 	}
 
 	@Override
 	@Transactional(readOnly = false)
-	public void saveCollecting(List<AccessionCollect> all) {
+	public List<AccessionCollect> saveCollecting(List<AccessionCollect> all) {
 		accessionCollectRepository.save(all);
+		return all;
 	}
 
 	@Override
 	@Transactional(readOnly = false)
-	public void saveGeo(List<AccessionGeo> all) {
+	public List<AccessionGeo> saveGeo(List<AccessionGeo> all) {
 		accessionGeoRepository.save(all);
+		return all;
 	}
 
 	@Override
 	@Transactional(readOnly = false)
-	public void saveBreeding(List<AccessionBreeding> all) {
+	public List<AccessionBreeding> saveBreeding(List<AccessionBreeding> all) {
 		accessionBreedingRepository.save(all);
+		return all;
 	}
 
 	@Override
 	@Transactional(readOnly = false)
-	public void saveExchange(List<AccessionExchange> all) {
+	public List<AccessionExchange> saveExchange(List<AccessionExchange> all) {
 		accessionExchangeRepository.save(all);
+		return all;
 	}
 
 	@Override
 	@Transactional(readOnly = false)
-	public void saveRemarks(List<AccessionRemark> toSaveRemarks) {
+	public List<AccessionRemark> saveRemarks(List<AccessionRemark> toSaveRemarks) {
 		accessionRemarkRepository.save(toSaveRemarks);
+		return toSaveRemarks;
 	}
 
 	@Override
 	@Transactional(readOnly = false)
-	public void removeRemarks(List<AccessionRemark> toRemoveRemarks) {
+	public List<AccessionRemark> removeRemarks(List<AccessionRemark> toRemoveRemarks) {
 		accessionRemarkRepository.delete(toRemoveRemarks);
+		return toRemoveRemarks;
 	}
 
 	@Override
@@ -1352,18 +1374,20 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
 
 	@Override
 	@Transactional
-	public void saveAliases(List<AccessionAlias> aliases) {
+	public List<AccessionAlias> saveAliases(List<AccessionAlias> aliases) {
 		if (aliases.size() > 0) {
 			accessionAliasRepository.save(aliases);
 		}
+		return aliases;
 	}
 
 	@Override
 	@Transactional
-	public void removeAliases(List<AccessionAlias> aliases) {
+	public List<AccessionAlias> removeAliases(List<AccessionAlias> aliases) {
 		if (aliases.size() > 0) {
 			accessionAliasRepository.delete(aliases);
 		}
+		return aliases;
 	}
 
 	@Override
@@ -1460,10 +1484,11 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
 
 	@Override
 	@Transactional
-	public void removeAliases(Set<Long> toRemove) {
+	public Set<Long> removeAliases(Set<Long> toRemove) {
 		for (final Long id : toRemove) {
 			this.accessionAliasRepository.delete(id);
 		}
+		return toRemove;
 	}
 
 	@Override

src/main/java/org/genesys2/server/service/impl/NonUniqueAccessionException.java

+/**
+ * Copyright 2014 Global Crop Diversity Trust
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+package org.genesys2.server.service.impl;
+
+public class NonUniqueAccessionException extends Exception {
+
+	private String instCode;
+	private String acceNumb;
+	private String genus;
+
+	public NonUniqueAccessionException(String instCode, String acceNumb) {
+		this.instCode = instCode;
+		this.acceNumb = acceNumb;
+	}
+
+	public NonUniqueAccessionException(String instCode, String acceNumb, String genus) {
+		this.instCode = instCode;
+		this.acceNumb = acceNumb;
+		this.genus = genus;
+	}
+
+	public String getInstCode() {
+		return instCode;
+	}
+
+	public String getAcceNumb() {
+		return acceNumb;
+	}
+
+	@Override
+	public String getMessage() {
+		return "Non-unique accession instCode=" + instCode + " acceNumb=" + acceNumb + (genus == null ? "" : " genus=" + genus);
+	}
+
+}

src/main/java/org/genesys2/server/service/worker/ElasticUpdater.java

@@ -59,7 +59,6 @@ public class ElasticUpdater {
 	 * @param ids
 	 */
 	public void remove(Class<?> clazz, Long id) {
-		LOG.info("Removing " + clazz + " " + id);
 		ElasticNode node = new ElasticNode(clazz, id);
 		elasticUpdateQueue.remove(node);
 		elasticRemoveQueue.add(node);
@@ -87,11 +86,10 @@ public class ElasticUpdater {
 	 * @param ids
 	 */