Commit 4e432938 authored by Matija Obreza's avatar Matija Obreza
Browse files

Load AccessionDetails in batches

parent 0065da86
......@@ -32,8 +32,6 @@ public interface ElasticService {
void remove(String className, long id);
void update(String className, long id);
void updateAll(String className, Collection<Long> bucket);
void refreshIndex(String className);
......
......@@ -191,8 +191,6 @@ public interface GenesysService {
List<AccessionBreeding> removeBreeding(List<AccessionBreeding> toRemove);
Set<AccessionDetails> getAccessionDetails(Collection<Accession> accessions);
List<Long> listAccessionsIds(Taxonomy2 taxonomy);
public static class AllStuff {
......@@ -208,7 +206,10 @@ public interface GenesysService {
public List<AccessionAlias> names = null;
public AccessionExchange exch = null;
public List<AccessionRemark> remarks = null;
public SvalbardData svalbard = null;
}
List<AllStuff> loadAllStuff(List<Long> accessionIds);
List<AllStuff> loadAllStuff(Collection<Long> accessionIds);
Set<AccessionDetails> getAccessionDetails(Collection<Long> accessionIds);
}
......@@ -284,50 +284,39 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
}
@Override
public void update(String className, long id) {
public void updateAll(String className, Collection<Long> ids) {
if (!clazzMap.containsKey(className)) {
return;
}
Object eo = toElasticObject(className, id);
if (eo == null)
if (ids.isEmpty()) {
LOG.info("Skipping empty updateAll.");
return;
IndexQuery iq = new IndexQuery();
iq.setId(String.valueOf(id));
iq.setObject(eo);
if (LOG.isDebugEnabled()) {
LOG.debug("Indexing " + className + " id=" + id);
}
elasticsearchTemplate.index(iq);
}
@Override
public void updateAll(String className, Collection<Long> ids) {
if (!clazzMap.containsKey(className)) {
if (LOG.isDebugEnabled())
LOG.debug("Updating " + className + " bulk_size=" + ids.size());
List<IndexQuery> queries = new ArrayList<IndexQuery>();
if (!Accession.class.getName().equals(className)) {
LOG.warn("Unsupported class " + className);
return;
}
LOG.info("Updating " + className + " bulk_size=" + ids.size());
List<IndexQuery> queries = new ArrayList<IndexQuery>();
for (Long id : ids) {
if (id == null)
continue; // Skip null id
Set<AccessionDetails> ads = genesysService.getAccessionDetails(ids);
Object eo = toElasticObject(className, id);
if (eo == null)
continue; // Skip null results, TODO perhaps delete?
for (AccessionDetails ad : ads) {
if (ad == null)
continue; // Skip null id
IndexQuery iq = new IndexQuery();
iq.setId(String.valueOf(id));
iq.setObject(eo);
iq.setId(String.valueOf(ad.getId()));
iq.setObject(ad);
queries.add(iq);
}
if (LOG.isInfoEnabled() && !queries.isEmpty()) {
LOG.info("Indexing " + className + " count=" + queries.size());
LOG.info("Indexing " + className + " count=" + queries.size() + " of provided objects count=" + ids.size());
elasticsearchTemplate.bulkIndex(queries);
}
}
......@@ -343,15 +332,6 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
elasticsearchTemplate.delete(clazz2, String.valueOf(id));
}
private Object toElasticObject(String className, long id) {
if (Accession.class.getName().equals(className)) {
return genesysService.getAccessionDetails(id);
}
LOG.warn("Unsupported class " + className);
return null;
}
@Override
public void refreshIndex(String className) {
Class<?> clazz2 = clazzMap.get(className);
......
......@@ -89,6 +89,7 @@ import org.genesys2.server.service.CropService;
import org.genesys2.server.service.DatasetService;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.HtmlSanitizer;
import org.genesys2.server.service.OrganizationService;
import org.genesys2.server.service.TaxonomyService;
import org.genesys2.server.service.impl.FilterHandler.AppliedFilters;
import org.genesys2.spring.SecurityContextUtil;
......@@ -152,7 +153,7 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
@Autowired
private CropTaxonomyRepository cropTaxonomyRepository;
@Autowired
private OrganizationRepository organizationRepository;
private OrganizationService organizationService;
@Autowired
private FaoInstituteRepository instituteRepository;
@Autowired
......@@ -311,29 +312,37 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
if (accession == null)
return null;
AccessionDetails ad = AccessionDetails.from(accession);
ad.networks(organizationRepository.getOrganizations(accession.getInstitute()));
ad.aliases(listAccessionAliases(accession));
ad.exch(listAccessionExchange(accession));
ad.collect(listAccessionCollect(accession));
ad.breeding(listAccessionBreeding(accession));
ad.geo(listAccessionGeo(accession));
ad.svalbard(getSvalbardData(accession));
ad.remarks(listAccessionRemarks(accession));
return toAccessionDetails(loadAllStuff(accession));
}
private AccessionDetails toAccessionDetails(AllStuff all) {
if (all == null || all.accession == null)
return null;
AccessionDetails ad = AccessionDetails.from(all.accession);
ad.networks(organizationService.getOrganizations(all.accession.getInstitute()));
ad.aliases(all.names);
ad.exch(all.exch);
ad.collect(all.collect);
ad.breeding(all.bred);
ad.geo(all.geo);
ad.svalbard(all.svalbard);
ad.remarks(all.remarks);
// ad.traits(listMethods(accession),
// getAccessionTraitValues(accession));
ad.crops(cropService.getCrops(accession.getTaxonomy()));
ad.crops(cropService.getCrops(all.accession.getTaxonomy()));
return ad;
}
@Override
public Set<AccessionDetails> getAccessionDetails(Collection<Accession> accessions) {
Set<AccessionDetails> set = new HashSet<AccessionDetails>(accessions.size());
for (Accession accn : accessions) {
if (accn == null)
public Set<AccessionDetails> getAccessionDetails(Collection<Long> accessionIds) {
Set<AccessionDetails> set = new HashSet<AccessionDetails>(accessionIds.size());
List<AllStuff> alls = loadAllStuff(accessionIds);
for (AllStuff all : alls) {
if (all == null)
continue;
set.add(getAccessionDetails(accn.getId()));
set.add(toAccessionDetails(all));
}
return set;
}
......@@ -412,15 +421,23 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
public AllStuff loadAllStuff(Accession accession) {
if (accession == null)
return null;
AllStuff all = new AllStuff(1l);
AllStuff all = new AllStuff(accession.getId());
all.bred = listAccessionBreeding(accession);
all.collect = listAccessionCollect(accession);
all.exch = listAccessionExchange(accession);
all.geo = listAccessionGeo(accession);
all.names = listAccessionAliases(accession);
all.remarks = listAccessionRemarks(accession);
all.svalbard = getSvalbardData(accession);
return all;
}
@Override
public List<AllStuff> loadAllStuff(List<Long> accessionIds) {
public List<AllStuff> loadAllStuff(Collection<Long> accessionIds) {
List<AllStuff> alls = new ArrayList<AllStuff>(accessionIds.size());
if (accessionIds == null || accessionIds.size() == 0) {
return alls;
}
HashMap<Long, AllStuff> map = new HashMap<Long, AllStuff>();
for (Long accessionId : accessionIds) {
......@@ -457,12 +474,19 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
all.exch = e;
}
for (SvalbardData e : svalbardRepository.findAllFor(accessionIds)) {
AllStuff all = map.get(e.getId());
all.svalbard = e;
}
for (AccessionAlias aa : accessionAliasRepository.findAllFor(accessionIds)) {
if (aa == null) {
System.err.println("aa is null");
continue;
}
AllStuff all = map.get(aa.getAccession().getId());
if (all == null) {
System.err.println(aa.getAccession().getId() + " not in the list! " + aa);
} else if (aa == null) {
System.err.println("aa is null");
} else {
all.names.add(aa);
}
......@@ -579,9 +603,9 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
@Override
public Page<Accession> listAccessionsByOrganization(Organization organization, Pageable pageable) {
final List<FaoInstitute> members = organizationRepository.findInstitutesByOrganization(organization);
final List<FaoInstitute> members = organizationService.getMembers(organization);
if (members == null || members.size() == 0) {
return new PageImpl<Accession>(ListUtils.EMPTY_LIST);
return new PageImpl<Accession>(Accession.EMPTY_LIST);
}
return accessionRepository.findByInstitute(members, pageable);
}
......@@ -649,14 +673,14 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
@Transactional(readOnly = false)
@CacheEvict(value = "statistics", allEntries = true)
public Accession saveAccession(Accession accession) {
if (LOG.isDebugEnabled())
if (LOG.isDebugEnabled())
LOG.debug("Updating " + accession);
System.err.println("Saving " + accession + " ST=" + accession.getStorage());
Accession res = accessionRepository.save(accession);
updateAccessionCount(accession.getInstitute());
return res;
}
}
@Override
@Transactional(readOnly = false)
......
......@@ -16,6 +16,7 @@
package org.genesys2.server.service.worker;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
......@@ -151,8 +152,13 @@ public class ElasticUpdaterAspect {
LOG.debug("Returning from " + joinPoint.toLongString());
}
Collection<Long> accessionIds = new ArrayList<Long>(accessions.size());
for (Accession a : accessions) {
if (a != null && a.getId() != null)
accessionIds.add(a.getId());
}
// Need to load AccessionDetails before deleting
Set<AccessionDetails> deletedAccessions = genesysService.getAccessionDetails(accessions);
Set<AccessionDetails> deletedAccessions = genesysService.getAccessionDetails(accessionIds);
try {
Object res = joinPoint.proceed();
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment