Commit 304f3ec2 authored by Matija Obreza's avatar Matija Obreza

Use JPA version of AccessionFilter for performance

parent 86ea7355
......@@ -23,16 +23,11 @@ import java.util.UUID;
import org.genesys.blocks.model.filters.NumberFilter;
import org.genesys.blocks.model.filters.StringFilter;
import org.genesys.blocks.model.filters.UuidModelFilter;
import org.genesys.catalog.model.dataset.QDataset;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.QAccession;
import org.genesys2.server.model.genesys.QAccessionCollect;
import org.genesys2.server.model.genesys.QAccessionGeo;
import org.genesys2.server.model.genesys.QAccessionId;
import org.genesys2.server.model.genesys.QPDCI;
import org.genesys2.server.model.genesys.QTaxonomy2;
import org.genesys2.server.model.impl.QCountry;
import org.genesys2.server.model.impl.QCrop;
import org.genesys2.server.model.impl.QFaoInstitute;
import org.genesys2.server.model.impl.QSubset;
import com.hazelcast.util.CollectionUtil;
import com.querydsl.core.BooleanBuilder;
......@@ -130,46 +125,49 @@ public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession>
}
public <T> JPQLQuery<T> buildJpaQuery(JPQLQuery<T> query, QAccession accession) {
QAccessionId qAccessionId = new QAccessionId("accessionId");
query.innerJoin(accession.accessionId, qAccessionId);
QAccessionId qAccessionId = accession.accessionId;
query.innerJoin(qAccessionId);
final BooleanBuilder and = new BooleanBuilder();
super.buildQuery(accession, accession._super._super, and);
if (taxonomy != null) {
QTaxonomy2 qTaxonomy = new QTaxonomy2("taxonomy");
query.innerJoin(accession.taxonomy, qTaxonomy);
and.and(taxonomy.buildQuery(qTaxonomy));
query.innerJoin(accession.taxonomy);
and.and(taxonomy.buildQuery(accession.taxonomy));
}
if (crop != null && !crop.isEmpty()) {
QCrop qCrop = new QCrop("crop");
query.leftJoin(accession.crop, qCrop);
and.and(qCrop.shortName.in(crop));
query.leftJoin(accession.crop);
and.and(accession.crop.shortName.in(crop));
}
if (institute != null) {
QFaoInstitute qInstitute = new QFaoInstitute("institute");
query.innerJoin(accession.institute, qInstitute);
and.and(institute.buildQuery(qInstitute));
query.innerJoin(accession.institute);
and.and(institute.buildQuery(accession.institute));
}
if (countryOfOrigin != null) {
QCountry qOrigin = new QCountry("countryOfOrigin");
query.leftJoin(accession.countryOfOrigin, qOrigin);
and.and(countryOfOrigin.buildQuery(qOrigin));
query.leftJoin(accession.countryOfOrigin);
and.and(countryOfOrigin.buildQuery(accession.countryOfOrigin));
}
if (geo != null) {
QAccessionGeo qGeo = new QAccessionGeo("geo");
query.leftJoin(qAccessionId.geo, qGeo);
and.and(geo.buildQuery(qGeo));
query.leftJoin(qAccessionId.geo);
and.and(geo.buildQuery(qAccessionId.geo));
}
if (coll != null) {
QAccessionCollect qColl = new QAccessionCollect("coll");
query.leftJoin(qAccessionId.coll, qColl);
and.and(coll.buildQuery(qColl));
query.leftJoin(qAccessionId.coll);
and.and(coll.buildQuery(qAccessionId.coll));
}
if (pdci != null) {
QPDCI qPdci = new QPDCI("pdci");
query.leftJoin(qAccessionId.pdci, qPdci);
and.and(pdci.buildQuery(qPdci.score));
query.leftJoin(qAccessionId.pdci);
and.and(pdci.buildQuery(qAccessionId.pdci.score));
}
if (CollectionUtil.isNotEmpty(subsets)) {
QSubset qSubset = new QSubset("subsets");
query.leftJoin(accession.subsets, qSubset);
and.and(qSubset.uuid.in(subsets));
}
if (CollectionUtil.isNotEmpty(datasets)) {
QDataset qDataset = new QDataset("datasets");
query.leftJoin(accession.datasets, qDataset);
and.and(qDataset.uuid.in(datasets));
}
......@@ -214,7 +212,7 @@ public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession>
and.and(qAccessionId.uuid.in(uuid));
}
if (aliases != null) {
and.and(aliases.buildQuery(accession.accessionId.aliases.any().name));
and.and(aliases.buildQuery(qAccessionId.aliases.any().name));
}
// TODO Do we have to change this?
if (sgsv != null) {
......@@ -232,13 +230,6 @@ public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession>
}
}
if (CollectionUtil.isNotEmpty(subsets)) {
and.and(accession.subsets.any().uuid.in(subsets));
}
if (CollectionUtil.isNotEmpty(datasets)) {
and.and(accession.datasets.any().uuid.in(datasets));
}
query.where(and);
return query;
}
......
......@@ -70,7 +70,6 @@ import org.springframework.transaction.annotation.Transactional;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.querydsl.core.BooleanBuilder;
import com.querydsl.core.Tuple;
import com.querydsl.jpa.impl.JPAQuery;
import com.querydsl.jpa.impl.JPAQueryFactory;
......@@ -273,7 +272,7 @@ public class AccessionServiceImpl implements AccessionService {
if (total < 10000 && ! filter.isFulltextQuery()) {
// If total is below 10K and no full-text query, use actual count
total = accessionRepository.count(filter.buildPredicate());
total = accessionRepository.count(filter);
}
return total;
......@@ -511,8 +510,10 @@ public class AccessionServiceImpl implements AccessionService {
AccessionFilter suggestionFilter = filter.copy(AccessionFilter.class);
try {
suggestionFilter.clearFilter(suggestionKey);
} catch (NoSuchFieldException | IllegalAccessException e) {
LOG.error("Error while clearing filter: ", e.getMessage());
} catch (NoSuchFieldException e) {
LOG.debug("No field {} while clearing filter: ", suggestionKey, e.getMessage());
} catch (Throwable e) {
LOG.error("Error while clearing filter {}: ", suggestionKey, e.getMessage());
}
ElasticsearchService.TermResult suggestion = elasticsearchService.termStatisticsAuto(Accession.class, suggestionFilter, 100, suggestionKey);
......@@ -541,13 +542,14 @@ public class AccessionServiceImpl implements AccessionService {
final QAccession accession = QAccession.accession;
JPAQuery<Tuple> query = jpaQueryFactory.selectFrom(accession).select(accession.accessionId.geo.latitude.min(), accession.accessionId.geo.longitude.max(),
accession.accessionId.geo.latitude.max(), accession.accessionId.geo.longitude.min());
JPAQuery<Accession> query = jpaQueryFactory.selectFrom(accession);
if (filter != null)
query.where(filter.buildPredicate());
if (filter != null) {
filter.buildJpaQuery(query, accession);
}
Object[] results = query.fetchOne().toArray();
Object[] results = query.select(accession.accessionId.geo.latitude.min(), accession.accessionId.geo.longitude.max(),
accession.accessionId.geo.latitude.max(), accession.accessionId.geo.longitude.min()).fetchOne().toArray();
return new Number[][] { new Number[] { (Number) results[0], (Number) results[1] }, new Number[] { (Number) results[2], (Number) results[3] } };
}
......
......@@ -493,7 +493,7 @@ public class ElasticsearchServiceImpl implements ElasticsearchService, Initializ
} catch (IOException e) {
e.printStackTrace();
}
LOG.trace("JSON: {}", jsons.get(bm.getId().toString()));
// is found
notFoundIds.remove(bm.getId());
......
......@@ -19,6 +19,7 @@ package org.genesys2.server.service.impl;
import java.util.List;
import java.util.stream.Collectors;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.QAccession;
import org.genesys2.server.model.genesys.QAccessionGeo;
import org.genesys2.server.service.ElasticsearchService;
......@@ -69,16 +70,10 @@ public class GenesysFilterServiceImpl implements GenesysFilterService {
@Override
public List<Double[]> listGeoTile(AccessionFilter filter, Integer limit, int zoom, int xtile, int ytile) {
QAccession accession = QAccession.accession;
QAccessionGeo accessionGeo = accession.accessionId.geo;
JPAQuery<Tuple> query = jpaQueryFactory.selectFrom(accession).select(accessionGeo.longitude, accessionGeo.latitude).distinct();
QAccession qAccession = QAccession.accession;
QAccessionGeo qAccessionGeo = qAccession.accessionId.geo;
BooleanBuilder filt = new BooleanBuilder();
// filt.and(accessionGeo.isNotNull()).and(accessionGeo.latitude.isNotNull()).and(accessionGeo.longitude.isNotNull());
if (filter != null) {
filt.and(filter.buildPredicate());
}
// The tile query
{
......@@ -94,20 +89,31 @@ public class GenesysFilterServiceImpl implements GenesysFilterService {
LOG.debug("{} <= lat <= {} corr={}", latS, latN, diffLat * .2);
}
filt.and(accessionGeo.longitude.between(lonW - zoom * diffLon * .2, lonE + zoom * diffLon * .2));
filt.and(accessionGeo.latitude.between(latS - zoom * diffLat * .2, latN + zoom * diffLat * .2));
filt.and(qAccessionGeo.longitude.between(lonW - zoom * diffLon * .2, lonE + zoom * diffLon * .2));
filt.and(qAccessionGeo.latitude.between(latS - zoom * diffLat * .2, latN + zoom * diffLat * .2));
}
query.where(filt);
if (filter.isFulltextQuery()) {
if (filter != null) {
filt.and(filter.buildPredicate());
}
return elasticsearchService.distinctCoordinates(filt, filter._text);
} else {
List<Tuple> results = query.fetch();
// filt.and(accessionGeo.isNotNull()).and(accessionGeo.latitude.isNotNull()).and(accessionGeo.longitude.isNotNull());
JPAQuery<Accession> query = jpaQueryFactory.selectFrom(qAccession);
JPAQuery<Accession> x = query.innerJoin(qAccessionGeo);
System.err.println(">>>> SAme? " + (x == query));
if (filter != null) {
filter.buildJpaQuery(query, qAccession);
}
query.where(filt);
List<Tuple> results = query.select(qAccessionGeo.longitude, qAccessionGeo.latitude).distinct().fetch();
return results.stream()
.map(item -> new Double[]{
item.get(accessionGeo.longitude),
item.get(accessionGeo.latitude),
item.get(qAccessionGeo.longitude),
item.get(qAccessionGeo.latitude),
}).collect(Collectors.toList());
}
......
......@@ -73,7 +73,7 @@ public class AccessionProcessor {
if (filter.isFulltextQuery()) {
processEs(filter, action, null);
} else {
process(filter.buildPredicate(), action, null);
process(toQuery(filter), action, null);
}
}
......@@ -82,12 +82,11 @@ public class AccessionProcessor {
if (filter.isFulltextQuery()) {
processEs(filter, action, maxSize);
} else {
process(filter.buildPredicate(), action, maxSize);
process(toQuery(filter), action, maxSize);
}
}
@Transactional(readOnly = true, propagation = Propagation.REQUIRES_NEW)
public void process(Predicate predicate, IAccessionBatchAction action, Long maxSize) throws Exception {
private JPQLQuery<Long> toQuery(AccessionFilter filter) throws Exception {
PathBuilder<Accession> builder = new PathBuilderFactory().create(Accession.class);
Querydsl querydsl = new Querydsl(em, builder);
JPQLQuery<Long> query = querydsl.createQuery(QAccession.accession)
......@@ -97,9 +96,11 @@ public class AccessionProcessor {
.orderBy(QAccession.accession.id.asc());
// apply filter
query.where(predicate);
if (filter != null) {
filter.buildJpaQuery(query, QAccession.accession);
}
process(query, action, maxSize);
return query;
}
private void processEs(AccessionFilter filter, IAccessionBatchAction action, Long maxSize) throws Exception {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment