Commit f71bcc45 authored by Matija Obreza's avatar Matija Obreza

Fix: updated low-level repository for performance

parent 62c8b10d
/**
* Copyright 2014 Global Crop Diversity Trust
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -12,13 +12,12 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
*/
package org.genesys2.server.persistence;
import java.util.Collection;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import org.genesys2.server.model.genesys.Accession;
......@@ -26,22 +25,64 @@ import org.genesys2.server.model.genesys.AccessionData;
import org.genesys2.server.model.impl.AccessionIdentifier3;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.service.filter.AccessionFilter;
import org.genesys2.server.service.impl.NonUniqueAccessionException;
import org.springframework.data.domain.Pageable;
/**
* Custom accession repository methods.
*
* @author Matija Obreza
*/
public interface AccessionRepositoryCustom {
/**
* Find active and historic.
*
* @param accessionUuids the accession uuids
* @return the list
*/
List<AccessionData> findActiveAndHistoric(Collection<UUID> accessionUuids);
/**
* Find.
*
* @param accessionUuids the accession uuids
* @return the list
*/
List<Accession> find(Collection<UUID> accessionUuids);
/**
* Find.
*
* @param accessions the accessions
* @return the list
*/
List<Accession> find(List<Accession> accessions);
/**
* Find by id.
*
* @param identifiers the identifiers
* @return the list
*/
List<Accession> findById(List<? extends AccessionIdentifier3> identifiers);
List<Accession> find(FaoInstitute institute, Set<? extends AccessionIdentifier3> accessionIds) throws NonUniqueAccessionException;
/**
* Find one.
*
* @param institute the institute
* @param doi the doi
* @param acceNumb the acce numb
* @param genus the genus
* @return the accession
*/
Accession findOne(FaoInstitute institute, String doi, String acceNumb, String genus);
/**
* Find all.
*
* @param filter the filter
* @param page the page
* @return the list
*/
List<Accession> findAll(AccessionFilter filter, Pageable page);
}
......@@ -19,7 +19,6 @@ package org.genesys2.server.persistence;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.UUID;
......@@ -30,6 +29,7 @@ import javax.persistence.PersistenceContext;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Join;
import javax.persistence.criteria.Path;
import javax.persistence.criteria.Predicate;
import javax.persistence.criteria.Root;
......@@ -41,7 +41,6 @@ import org.genesys2.server.model.genesys.Taxonomy2;
import org.genesys2.server.model.impl.AccessionIdentifier3;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.service.filter.AccessionFilter;
import org.genesys2.server.service.impl.NonUniqueAccessionException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
......@@ -84,11 +83,11 @@ public class AccessionRepositoryCustomImpl implements AccessionRepositoryCustom,
if (forUpdate == null || forUpdate.isEmpty()) {
return Collections.emptyList();
}
CriteriaQuery<Accession> cq = criteriaBuilder.createQuery(Accession.class);
CriteriaQuery<Long> cq = criteriaBuilder.createQuery(Long.class);
Root<Accession> root = cq.from(Accession.class);
cq.distinct(true);
cq.select(root);
cq.select(root.get("id"));
Join<Accession, Taxonomy2> tax = root.join("taxonomy");
Join<Accession, FaoInstitute> inst = root.join("institute");
......@@ -97,24 +96,29 @@ public class AccessionRepositoryCustomImpl implements AccessionRepositoryCustom,
Set<String> uniqueDois = forUpdate.stream().map(aid -> aid.getDoi()).filter(doi -> doi != null).distinct().collect(Collectors.toSet());
Set<String> acceNumbs = forUpdate.stream().map(aid -> aid.getAccessionNumber()).filter(acceNumb -> acceNumb != null).distinct().collect(Collectors.toSet());
Path<Object> theDoi = root.get("doi");
Path<Object> theInstCode = inst.get("code");
Path<Object> theAcceNumb = root.get("accessionNumber");
Path<Object> theGenus = tax.get("genus");
if (uniqueDois.size() > 0) {
restrictions.add(root.get("doi").in(uniqueDois));
restrictions.add(theDoi.in(uniqueDois));
LOG.trace("*** {} dois={}", uniqueDois);
}
// A lot of .. (acceNumb=? and genus=?)
for (Accession ah : forUpdate) {
restrictions.add(criteriaBuilder.and(criteriaBuilder.equal(inst.get("code"), ah.getInstituteCode()), criteriaBuilder.equal(root.get("accessionNumber"), ah
.getAccessionNumber()), criteriaBuilder.equal(tax.get("genus"), ah.getTaxonomy().getGenus())));
restrictions.add(criteriaBuilder.and(criteriaBuilder.equal(theInstCode, ah.getInstituteCode()), criteriaBuilder.equal(theAcceNumb, ah
.getAccessionNumber()), criteriaBuilder.equal(theGenus, ah.getTaxonomy().getGenus())));
}
cq.where(criteriaBuilder.or(restrictions.toArray(EMPTY_PREDICATE_ARRAY)));
List<Accession> res = em.createQuery(cq).getResultList();
List<Long> res = em.createQuery(cq).getResultList();
if (LOG.isDebugEnabled())
LOG.trace("*** Loaded accessions {} of {}", res.size(), acceNumbs.size());
return res;
return jpaQueryFactory.selectFrom(QAccession.accession).where(QAccession.accession.id.in(res)).fetch();
}
@Override
......@@ -123,13 +127,13 @@ public class AccessionRepositoryCustomImpl implements AccessionRepositoryCustom,
return Collections.emptyList();
}
CriteriaQuery<Accession> cq = criteriaBuilder.createQuery(Accession.class);
CriteriaQuery<Long> cq = criteriaBuilder.createQuery(Long.class);
Root<Accession> root = cq.from(Accession.class);
cq.distinct(true);
cq.select(root);
cq.select(root.get("id"));
Join<Accession, Taxonomy2> tax = root.join("taxonomy");
Join<Accession, FaoInstitute> inst = root.join("institute");
Join<Accession, Taxonomy2> tax = root.join("taxonomy");
List<Predicate> restrictions = new ArrayList<Predicate>();
Set<String> uniqueDois = identifiers.stream().map(aid -> aid.getDoi()).filter(doi -> doi != null).distinct().collect(Collectors.toSet());
......@@ -140,75 +144,23 @@ public class AccessionRepositoryCustomImpl implements AccessionRepositoryCustom,
LOG.trace("*** {} dois={}", uniqueDois);
}
Path<Object> theInstCode = inst.get("code");
Path<Object> theAcceNumb = root.get("accessionNumber");
Path<Object> theGenus = tax.get("genus");
// A lot of .. (acceNumb=? and genus=?)
for (AccessionIdentifier3 aid3 : identifiers) {
restrictions.add(criteriaBuilder.and(criteriaBuilder.equal(inst.get("code"), aid3.getHoldingInstitute()), criteriaBuilder.equal(root.get("accessionNumber"), aid3
.getAccessionNumber()), criteriaBuilder.equal(tax.get("genus"), aid3.getGenus())));
restrictions.add(criteriaBuilder.and(criteriaBuilder.equal(theInstCode, aid3.getHoldingInstitute()), criteriaBuilder.equal(theAcceNumb, aid3
.getAccessionNumber()), criteriaBuilder.equal(theGenus, aid3.getGenus())));
}
cq.where(criteriaBuilder.or(restrictions.toArray(EMPTY_PREDICATE_ARRAY)));
List<Accession> res = em.createQuery(cq).getResultList();
List<Long> res = em.createQuery(cq).getResultList();
if (LOG.isDebugEnabled())
LOG.trace("*** Loaded accessions {} of {}", res.size(), acceNumbs.size());
return res;
}
@Override
public List<Accession> find(FaoInstitute institute, Set<? extends AccessionIdentifier3> accessionIds) throws NonUniqueAccessionException {
boolean uniqueAcceNumbs = institute.hasUniqueAcceNumbs();
CriteriaQuery<Accession> cq = criteriaBuilder.createQuery(Accession.class);
Root<Accession> root = cq.from(Accession.class);
cq.distinct(true);
cq.select(root);
// root.fetch("stoRage", JoinType.LEFT);
Join<Accession, Taxonomy2> tax = null;
if (!uniqueAcceNumbs) {
tax = root.join("taxonomy");
}
List<Predicate> restrictions = new ArrayList<Predicate>();
Set<String> uniqueDois = accessionIds.stream().map(aid -> aid.getDoi()).filter(doi -> doi != null).distinct().collect(Collectors.toSet());
Set<String> acceNumbs = accessionIds.stream().map(aid -> aid.getAccessionNumber()).filter(acceNumb -> acceNumb != null).distinct().collect(Collectors.toSet());
if (uniqueDois.size() > 0) {
restrictions.add(root.get("doi").in(uniqueDois));
LOG.debug("*** {} dois={}", institute.getCode(), uniqueDois);
}
if (uniqueAcceNumbs) {
restrictions.add(root.get("accessionNumber").in(acceNumbs));
LOG.trace("*** {} accenumbs={}", institute.getCode(), acceNumbs);
} else {
// A lot of .. (acceNumb=? and genus=?)
for (AccessionIdentifier3 ah : accessionIds) {
restrictions.add(criteriaBuilder.and(criteriaBuilder.equal(root.get("accessionNumber"), ah.getAccessionNumber()), criteriaBuilder.equal(tax.get("genus"), ah
.getGenus())));
}
}
cq.where(criteriaBuilder.and(criteriaBuilder.equal(root.get("institute"), institute), criteriaBuilder.or(restrictions.toArray(EMPTY_PREDICATE_ARRAY))));
List<Accession> res = em.createQuery(cq).getResultList();
LOG.trace("*** Loaded accessions {} of {}", res.size(), acceNumbs.size());
// Check for duplicate names if institute is using unique acceNumbs
if (uniqueAcceNumbs) {
Set<String> s = new HashSet<String>();
for (Accession a : res) {
if (s.contains(a.getAccessionNumber())) {
LOG.error("Duplicate accession name instCode=" + a.getInstituteCode() + " acceNumb=" + a.getAccessionNumber());
throw new NonUniqueAccessionException(a.getInstituteCode(), a.getAccessionNumber());
}
s.add(a.getAccessionNumber());
}
}
return res;
return jpaQueryFactory.selectFrom(QAccession.accession).where(QAccession.accession.id.in(res)).fetch();
}
@Override
......
......@@ -46,9 +46,9 @@ public class AccessionCounter implements InitializingBean, DisposableBean {
private static final Logger LOG = LoggerFactory.getLogger(AccessionCounter.class);
// Interval time (in ms) for scanning the queue for expired updates
private static final int DELAY_BETWEEN_QUEUESCANS = 5000;
private static final int DELAY_BETWEEN_QUEUESCANS = 5000; // 5 sec
// If a request comes before this delay (in ms), reschedule
private static final long MINTIME_BETWEEN_UPDATES = 30000;
private static final long MINTIME_BETWEEN_UPDATES = 60 * 60 * 1000; // 1 min
private DelayQueue<DelayedObject<String>> instituteQueue;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment