Load AccessionDetails in batches

src/main/java/org/genesys2/server/service/ElasticService.java

@@ -32,8 +32,6 @@ public interface ElasticService {
 
 	void remove(String className, long id);
 
-	void update(String className, long id);
-
 	void updateAll(String className, Collection<Long> bucket);
 
 	void refreshIndex(String className);

src/main/java/org/genesys2/server/service/GenesysService.java

@@ -191,8 +191,6 @@ public interface GenesysService {
 
 	List<AccessionBreeding> removeBreeding(List<AccessionBreeding> toRemove);
 
-	Set<AccessionDetails> getAccessionDetails(Collection<Accession> accessions);
-
 	List<Long> listAccessionsIds(Taxonomy2 taxonomy);
 
 	public static class AllStuff {
@@ -208,7 +206,10 @@ public interface GenesysService {
 		public List<AccessionAlias> names = null;
 		public AccessionExchange exch = null;
 		public List<AccessionRemark> remarks = null;
+		public SvalbardData svalbard = null;
 	}
 
-	List<AllStuff> loadAllStuff(List<Long> accessionIds);
+	List<AllStuff> loadAllStuff(Collection<Long> accessionIds);
+
+	Set<AccessionDetails> getAccessionDetails(Collection<Long> accessionIds);
 }

src/main/java/org/genesys2/server/service/impl/ElasticsearchSearchServiceImpl.java

@@ -284,50 +284,39 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
 	}
 
 	@Override
-	public void update(String className, long id) {
+	public void updateAll(String className, Collection<Long> ids) {
 		if (!clazzMap.containsKey(className)) {
 			return;
 		}
-
-		Object eo = toElasticObject(className, id);
-		if (eo == null)
+		if (ids.isEmpty()) {
+			LOG.info("Skipping empty updateAll.");
 			return;
-
-		IndexQuery iq = new IndexQuery();
-		iq.setId(String.valueOf(id));
-		iq.setObject(eo);
-
-		if (LOG.isDebugEnabled()) {
-			LOG.debug("Indexing " + className + " id=" + id);
 		}
-		elasticsearchTemplate.index(iq);
-	}
 
-	@Override
-	public void updateAll(String className, Collection<Long> ids) {
-		if (!clazzMap.containsKey(className)) {
+		if (LOG.isDebugEnabled())
+			LOG.debug("Updating " + className + " bulk_size=" + ids.size());
+		List<IndexQuery> queries = new ArrayList<IndexQuery>();
+
+		if (!Accession.class.getName().equals(className)) {
+			LOG.warn("Unsupported class " + className);
 			return;
 		}
-		LOG.info("Updating " + className + " bulk_size=" + ids.size());
-		List<IndexQuery> queries = new ArrayList<IndexQuery>();
 
-		for (Long id : ids) {
-			if (id == null)
-				continue; // Skip null id
+		Set<AccessionDetails> ads = genesysService.getAccessionDetails(ids);
 
-			Object eo = toElasticObject(className, id);
-			if (eo == null)
-				continue; // Skip null results, TODO perhaps delete?
+		for (AccessionDetails ad : ads) {
+			if (ad == null)
+				continue; // Skip null id
 
 			IndexQuery iq = new IndexQuery();
-			iq.setId(String.valueOf(id));
-			iq.setObject(eo);
+			iq.setId(String.valueOf(ad.getId()));
+			iq.setObject(ad);
 
 			queries.add(iq);
 		}
 
 		if (LOG.isInfoEnabled() && !queries.isEmpty()) {
-			LOG.info("Indexing " + className + " count=" + queries.size());
+			LOG.info("Indexing " + className + " count=" + queries.size() + " of provided objects count=" + ids.size());
 			elasticsearchTemplate.bulkIndex(queries);
 		}
 	}
@@ -343,15 +332,6 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
 		elasticsearchTemplate.delete(clazz2, String.valueOf(id));
 	}
 
-	private Object toElasticObject(String className, long id) {
-		if (Accession.class.getName().equals(className)) {
-			return genesysService.getAccessionDetails(id);
-		}
-
-		LOG.warn("Unsupported class " + className);
-		return null;
-	}
-
 	@Override
 	public void refreshIndex(String className) {
 		Class<?> clazz2 = clazzMap.get(className);

src/main/java/org/genesys2/server/service/impl/GenesysServiceImpl.java

@@ -89,6 +89,7 @@ import org.genesys2.server.service.CropService;
 import org.genesys2.server.service.DatasetService;
 import org.genesys2.server.service.GenesysService;
 import org.genesys2.server.service.HtmlSanitizer;
+import org.genesys2.server.service.OrganizationService;
 import org.genesys2.server.service.TaxonomyService;
 import org.genesys2.server.service.impl.FilterHandler.AppliedFilters;
 import org.genesys2.spring.SecurityContextUtil;
@@ -152,7 +153,7 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
 	@Autowired
 	private CropTaxonomyRepository cropTaxonomyRepository;
 	@Autowired
-	private OrganizationRepository organizationRepository;
+	private OrganizationService organizationService;
 	@Autowired
 	private FaoInstituteRepository instituteRepository;
 	@Autowired
@@ -311,29 +312,37 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
 		if (accession == null)
 			return null;
 
-		AccessionDetails ad = AccessionDetails.from(accession);
-		ad.networks(organizationRepository.getOrganizations(accession.getInstitute()));
-		ad.aliases(listAccessionAliases(accession));
-		ad.exch(listAccessionExchange(accession));
-		ad.collect(listAccessionCollect(accession));
-		ad.breeding(listAccessionBreeding(accession));
-		ad.geo(listAccessionGeo(accession));
-		ad.svalbard(getSvalbardData(accession));
-		ad.remarks(listAccessionRemarks(accession));
+		return toAccessionDetails(loadAllStuff(accession));
+	}
+
+	private AccessionDetails toAccessionDetails(AllStuff all) {
+		if (all == null || all.accession == null)
+			return null;
+
+		AccessionDetails ad = AccessionDetails.from(all.accession);
+		ad.networks(organizationService.getOrganizations(all.accession.getInstitute()));
+		ad.aliases(all.names);
+		ad.exch(all.exch);
+		ad.collect(all.collect);
+		ad.breeding(all.bred);
+		ad.geo(all.geo);
+		ad.svalbard(all.svalbard);
+		ad.remarks(all.remarks);
 		// ad.traits(listMethods(accession),
 		// getAccessionTraitValues(accession));
-		ad.crops(cropService.getCrops(accession.getTaxonomy()));
+		ad.crops(cropService.getCrops(all.accession.getTaxonomy()));
 		return ad;
 	}
 
 	@Override
-	public Set<AccessionDetails> getAccessionDetails(Collection<Accession> accessions) {
-		Set<AccessionDetails> set = new HashSet<AccessionDetails>(accessions.size());
-		for (Accession accn : accessions) {
-			if (accn == null)
+	public Set<AccessionDetails> getAccessionDetails(Collection<Long> accessionIds) {
+		Set<AccessionDetails> set = new HashSet<AccessionDetails>(accessionIds.size());
+		List<AllStuff> alls = loadAllStuff(accessionIds);
+		for (AllStuff all : alls) {
+			if (all == null)
 				continue;
 
-			set.add(getAccessionDetails(accn.getId()));
+			set.add(toAccessionDetails(all));
 		}
 		return set;
 	}
@@ -412,15 +421,23 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
 	public AllStuff loadAllStuff(Accession accession) {
 		if (accession == null)
 			return null;
-		AllStuff all = new AllStuff(1l);
+		AllStuff all = new AllStuff(accession.getId());
+		all.bred = listAccessionBreeding(accession);
+		all.collect = listAccessionCollect(accession);
+		all.exch = listAccessionExchange(accession);
+		all.geo = listAccessionGeo(accession);
 		all.names = listAccessionAliases(accession);
 		all.remarks = listAccessionRemarks(accession);
+		all.svalbard = getSvalbardData(accession);
 		return all;
 	}
 
 	@Override
-	public List<AllStuff> loadAllStuff(List<Long> accessionIds) {
+	public List<AllStuff> loadAllStuff(Collection<Long> accessionIds) {
 		List<AllStuff> alls = new ArrayList<AllStuff>(accessionIds.size());
+		if (accessionIds == null || accessionIds.size() == 0) {
+			return alls;
+		}
 		HashMap<Long, AllStuff> map = new HashMap<Long, AllStuff>();
 
 		for (Long accessionId : accessionIds) {
@@ -457,12 +474,19 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
 			all.exch = e;
 		}
 
+		for (SvalbardData e : svalbardRepository.findAllFor(accessionIds)) {
+			AllStuff all = map.get(e.getId());
+			all.svalbard = e;
+		}
+
 		for (AccessionAlias aa : accessionAliasRepository.findAllFor(accessionIds)) {
+			if (aa == null) {
+				System.err.println("aa is null");
+				continue;
+			}
 			AllStuff all = map.get(aa.getAccession().getId());
 			if (all == null) {
 				System.err.println(aa.getAccession().getId() + " not in the list! " + aa);
-			} else if (aa == null) {
-				System.err.println("aa is null");
 			} else {
 				all.names.add(aa);
 			}
@@ -579,9 +603,9 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
 
 	@Override
 	public Page<Accession> listAccessionsByOrganization(Organization organization, Pageable pageable) {
-		final List<FaoInstitute> members = organizationRepository.findInstitutesByOrganization(organization);
+		final List<FaoInstitute> members = organizationService.getMembers(organization);
 		if (members == null || members.size() == 0) {
-			return new PageImpl<Accession>(ListUtils.EMPTY_LIST);
+			return new PageImpl<Accession>(Accession.EMPTY_LIST);
 		}
 		return accessionRepository.findByInstitute(members, pageable);
 	}
@@ -649,14 +673,14 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
 	@Transactional(readOnly = false)
 	@CacheEvict(value = "statistics", allEntries = true)
 	public Accession saveAccession(Accession accession) {
-			if (LOG.isDebugEnabled())
+		if (LOG.isDebugEnabled())
 			LOG.debug("Updating " + accession);
 
 		System.err.println("Saving " + accession + " ST=" + accession.getStorage());
 		Accession res = accessionRepository.save(accession);
 		updateAccessionCount(accession.getInstitute());
 		return res;
-		}
+	}
 
 	@Override
 	@Transactional(readOnly = false)

src/main/java/org/genesys2/server/service/worker/ElasticUpdaterAspect.java

@@ -16,6 +16,7 @@
 
 package org.genesys2.server.service.worker;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
@@ -151,8 +152,13 @@ public class ElasticUpdaterAspect {
 			LOG.debug("Returning from " + joinPoint.toLongString());
 		}
 
+		Collection<Long> accessionIds = new ArrayList<Long>(accessions.size());
+		for (Accession a : accessions) {
+			if (a != null && a.getId() != null)
+				accessionIds.add(a.getId());
+		}
 		// Need to load AccessionDetails before deleting
-		Set<AccessionDetails> deletedAccessions = genesysService.getAccessionDetails(accessions);
+		Set<AccessionDetails> deletedAccessions = genesysService.getAccessionDetails(accessionIds);
 
 		try {
 			Object res = joinPoint.proceed();