Commit 36cffdf9 authored by Matija Obreza's avatar Matija Obreza

Merge branch '439-use-es-for-text-queries' into 'master'

Resolve "Use ES for text queries"

Closes #439

See merge request genesys-pgr/genesys-server!392
parents 134b9d79 029ab3b4
...@@ -21,15 +21,14 @@ import java.util.HashSet; ...@@ -21,15 +21,14 @@ import java.util.HashSet;
import java.util.Set; import java.util.Set;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.genesys.blocks.model.filters.StringFilter; import org.genesys.blocks.model.filters.StringFilter;
import org.genesys.blocks.model.filters.UuidModelFilter; import org.genesys.blocks.model.filters.UuidModelFilter;
import org.genesys.catalog.model.dataset.Dataset; import org.genesys.catalog.model.dataset.Dataset;
import org.genesys2.server.model.PublishState;
import org.genesys2.server.service.filter.IFullTextFilter;
import com.querydsl.core.BooleanBuilder; import com.querydsl.core.BooleanBuilder;
import com.querydsl.core.types.Predicate; import com.querydsl.core.types.Predicate;
import org.genesys2.server.model.PublishState;
/** /**
* The Class DatasetFilter. * The Class DatasetFilter.
...@@ -37,7 +36,7 @@ import org.genesys2.server.model.PublishState; ...@@ -37,7 +36,7 @@ import org.genesys2.server.model.PublishState;
* @author Andrey Lugovskoy * @author Andrey Lugovskoy
* @author Matija Obreza * @author Matija Obreza
*/ */
public class DatasetFilter extends UuidModelFilter<DatasetFilter, Dataset> { public class DatasetFilter extends UuidModelFilter<DatasetFilter, Dataset> implements IFullTextFilter {
/** Any text. */ /** Any text. */
public String _text; public String _text;
...@@ -113,29 +112,29 @@ public class DatasetFilter extends UuidModelFilter<DatasetFilter, Dataset> { ...@@ -113,29 +112,29 @@ public class DatasetFilter extends UuidModelFilter<DatasetFilter, Dataset> {
and.and(dataset.rights.in(rights)); and.and(dataset.rights.in(rights));
} }
if (StringUtils.isNotBlank(_text)) { // if (StringUtils.isNotBlank(_text)) {
/*@formatter:off*/ // /*@formatter:off*/
and.andAnyOf( // and.andAnyOf(
ArrayUtils.addAll( // ArrayUtils.addAll(
FilterHelpers.equalsAny(_text, // FilterHelpers.equalsAny(_text,
dataset.versionTag, // dataset.versionTag,
dataset.owner.shortName, // dataset.owner.shortName,
dataset.crops.any(), // dataset.crops.any(),
dataset.descriptors.any().crop, // dataset.descriptors.any().crop,
dataset.accessionRefs.any().genus, dataset.accessionRefs.any().instCode, dataset.accessionRefs.any().acceNumb // dataset.accessionRefs.any().genus, dataset.accessionRefs.any().instCode, dataset.accessionRefs.any().acceNumb
), // ),
FilterHelpers.containsAll(_text, // FilterHelpers.containsAll(_text,
dataset.title, dataset.description, // dataset.title, dataset.description,
dataset.creators.any().fullName, // dataset.creators.any().fullName,
dataset.locations.any().verbatimLocality, // dataset.locations.any().verbatimLocality,
dataset.repositoryFiles.any().title, dataset.repositoryFiles.any().originalFilename, // dataset.repositoryFiles.any().title, dataset.repositoryFiles.any().originalFilename,
dataset.owner.name, // dataset.owner.name,
dataset.descriptors.any().title // dataset.descriptors.any().title
) // )
) // )
); // );
/*@formatter:on*/ // /*@formatter:on*/
} // }
return and; return and;
} }
...@@ -149,4 +148,9 @@ public class DatasetFilter extends UuidModelFilter<DatasetFilter, Dataset> { ...@@ -149,4 +148,9 @@ public class DatasetFilter extends UuidModelFilter<DatasetFilter, Dataset> {
} }
return this; return this;
} }
@Override
public String get_text() {
return _text;
}
} }
...@@ -20,14 +20,13 @@ import static org.genesys.catalog.model.traits.QDescriptor.descriptor; ...@@ -20,14 +20,13 @@ import static org.genesys.catalog.model.traits.QDescriptor.descriptor;
import java.util.HashSet; import java.util.HashSet;
import java.util.Set; import java.util.Set;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.genesys.blocks.model.filters.NumberFilter; import org.genesys.blocks.model.filters.NumberFilter;
import org.genesys.blocks.model.filters.StringFilter; import org.genesys.blocks.model.filters.StringFilter;
import org.genesys.blocks.model.filters.UuidModelFilter; import org.genesys.blocks.model.filters.UuidModelFilter;
import org.genesys.catalog.model.traits.Descriptor; import org.genesys.catalog.model.traits.Descriptor;
import org.genesys.catalog.model.traits.QDescriptor; import org.genesys.catalog.model.traits.QDescriptor;
import org.genesys2.server.model.PublishState; import org.genesys2.server.model.PublishState;
import org.genesys2.server.service.filter.IFullTextFilter;
import com.querydsl.core.BooleanBuilder; import com.querydsl.core.BooleanBuilder;
import com.querydsl.core.types.Predicate; import com.querydsl.core.types.Predicate;
...@@ -38,7 +37,7 @@ import com.querydsl.core.types.Predicate; ...@@ -38,7 +37,7 @@ import com.querydsl.core.types.Predicate;
* @author Andrey Lugovskoy * @author Andrey Lugovskoy
* @author Matija Obreza * @author Matija Obreza
*/ */
public class DescriptorFilter extends UuidModelFilter<DescriptorFilter, Descriptor> { public class DescriptorFilter extends UuidModelFilter<DescriptorFilter, Descriptor> implements IFullTextFilter {
/** Any text. */ /** Any text. */
public String _text; public String _text;
...@@ -157,24 +156,24 @@ public class DescriptorFilter extends UuidModelFilter<DescriptorFilter, Descript ...@@ -157,24 +156,24 @@ public class DescriptorFilter extends UuidModelFilter<DescriptorFilter, Descript
} }
} }
if (StringUtils.isNotBlank(_text)) { // if (StringUtils.isNotBlank(_text)) {
/*@formatter:off*/ // /*@formatter:off*/
and.andAnyOf( // and.andAnyOf(
ArrayUtils.addAll( // ArrayUtils.addAll(
FilterHelpers.equalsAny(_text, // FilterHelpers.equalsAny(_text,
descriptorPath.crop, descriptorPath.versionTag, descriptorPath.publisher, // descriptorPath.crop, descriptorPath.versionTag, descriptorPath.publisher,
descriptorPath.owner.shortName, // descriptorPath.owner.shortName,
descriptorPath.descriptorLists.any().publisher, descriptorPath.descriptorLists.any().versionTag, descriptorPath.descriptorLists.any().crop // descriptorPath.descriptorLists.any().publisher, descriptorPath.descriptorLists.any().versionTag, descriptorPath.descriptorLists.any().crop
), // ),
FilterHelpers.containsAll(_text, // FilterHelpers.containsAll(_text,
descriptorPath.title, descriptorPath.description, descriptorPath.bibliographicCitation, // descriptorPath.title, descriptorPath.description, descriptorPath.bibliographicCitation,
descriptorPath.owner.name, // descriptorPath.owner.name,
descriptorPath.descriptorLists.any().title // descriptorPath.descriptorLists.any().title
) // )
) // )
); // );
/*@formatter:on*/ // /*@formatter:on*/
} // }
return and; return and;
} }
...@@ -197,4 +196,8 @@ public class DescriptorFilter extends UuidModelFilter<DescriptorFilter, Descript ...@@ -197,4 +196,8 @@ public class DescriptorFilter extends UuidModelFilter<DescriptorFilter, Descript
} }
} }
@Override
public String get_text() {
return _text;
}
} }
...@@ -20,13 +20,12 @@ import static org.genesys.catalog.model.traits.QDescriptorList.descriptorList; ...@@ -20,13 +20,12 @@ import static org.genesys.catalog.model.traits.QDescriptorList.descriptorList;
import java.util.HashSet; import java.util.HashSet;
import java.util.Set; import java.util.Set;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.genesys.blocks.model.filters.StringFilter; import org.genesys.blocks.model.filters.StringFilter;
import org.genesys.blocks.model.filters.UuidModelFilter; import org.genesys.blocks.model.filters.UuidModelFilter;
import org.genesys.catalog.model.traits.DescriptorList; import org.genesys.catalog.model.traits.DescriptorList;
import org.genesys.catalog.model.traits.QDescriptorList; import org.genesys.catalog.model.traits.QDescriptorList;
import org.genesys2.server.model.PublishState; import org.genesys2.server.model.PublishState;
import org.genesys2.server.service.filter.IFullTextFilter;
import com.querydsl.core.BooleanBuilder; import com.querydsl.core.BooleanBuilder;
import com.querydsl.core.types.Predicate; import com.querydsl.core.types.Predicate;
...@@ -37,7 +36,7 @@ import com.querydsl.core.types.Predicate; ...@@ -37,7 +36,7 @@ import com.querydsl.core.types.Predicate;
* @author Andrey Lugovskoy * @author Andrey Lugovskoy
* @author Matija Obreza * @author Matija Obreza
*/ */
public class DescriptorListFilter extends UuidModelFilter<DescriptorListFilter, DescriptorList> { public class DescriptorListFilter extends UuidModelFilter<DescriptorListFilter, DescriptorList> implements IFullTextFilter {
/** Any text. */ /** Any text. */
public String _text; public String _text;
...@@ -98,24 +97,24 @@ public class DescriptorListFilter extends UuidModelFilter<DescriptorListFilter, ...@@ -98,24 +97,24 @@ public class DescriptorListFilter extends UuidModelFilter<DescriptorListFilter,
and.and(owner.buildQuery(descriptorList.owner)); and.and(owner.buildQuery(descriptorList.owner));
} }
if (StringUtils.isNotBlank(_text)) { // if (StringUtils.isNotBlank(_text)) {
/*@formatter:off*/ // /*@formatter:off*/
and.andAnyOf( // and.andAnyOf(
ArrayUtils.addAll( // ArrayUtils.addAll(
FilterHelpers.equalsAny(_text, // FilterHelpers.equalsAny(_text,
descriptorList.crop, descriptorList.versionTag, descriptorList.publisher, // descriptorList.crop, descriptorList.versionTag, descriptorList.publisher,
descriptorList.owner.shortName, // descriptorList.owner.shortName,
descriptorList.descriptors.any().crop, descriptorList.descriptors.any().versionTag, descriptorList.descriptors.any().publisher // descriptorList.descriptors.any().crop, descriptorList.descriptors.any().versionTag, descriptorList.descriptors.any().publisher
), // ),
FilterHelpers.containsAll(_text, // FilterHelpers.containsAll(_text,
descriptorList.title, descriptorList.description, descriptorList.bibliographicCitation, // descriptorList.title, descriptorList.description, descriptorList.bibliographicCitation,
descriptorList.owner.name, // descriptorList.owner.name,
descriptorList.descriptors.any().title // descriptorList.descriptors.any().title
) // )
) // )
); // );
/*@formatter:on*/ // /*@formatter:on*/
} // }
return and; return and;
} }
...@@ -129,4 +128,9 @@ public class DescriptorListFilter extends UuidModelFilter<DescriptorListFilter, ...@@ -129,4 +128,9 @@ public class DescriptorListFilter extends UuidModelFilter<DescriptorListFilter,
} }
return this; return this;
} }
@Override
public String get_text() {
return _text;
}
} }
...@@ -46,6 +46,7 @@ import org.genesys2.server.exception.NotFoundElement; ...@@ -46,6 +46,7 @@ import org.genesys2.server.exception.NotFoundElement;
import org.genesys2.server.model.PublishState; import org.genesys2.server.model.PublishState;
import org.genesys2.server.model.UserRole; import org.genesys2.server.model.UserRole;
import org.genesys2.server.service.DownloadService; import org.genesys2.server.service.DownloadService;
import org.genesys2.server.service.ElasticsearchService;
import org.genesys2.util.JPAUtils; import org.genesys2.util.JPAUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
...@@ -102,6 +103,9 @@ public class DescriptorServiceImpl implements DescriptorService { ...@@ -102,6 +103,9 @@ public class DescriptorServiceImpl implements DescriptorService {
@Autowired @Autowired
private VersionManager versionManager; private VersionManager versionManager;
@Autowired
private ElasticsearchService elasticsearchService;
/** /**
* {@inheritDoc} * {@inheritDoc}
*/ */
...@@ -301,8 +305,12 @@ public class DescriptorServiceImpl implements DescriptorService { ...@@ -301,8 +305,12 @@ public class DescriptorServiceImpl implements DescriptorService {
@Override @Override
public Page<Descriptor> listDescriptors(final DescriptorFilter descriptorFilter, final Pageable page) { public Page<Descriptor> listDescriptors(final DescriptorFilter descriptorFilter, final Pageable page) {
Pageable markdownSortPageRequest = JPAUtils.toMarkdownSort(page, "title"); Pageable markdownSortPageRequest = JPAUtils.toMarkdownSort(page, "title");
Page<Descriptor> res = descriptorRepository.findAll(new BooleanBuilder().and(descriptorFilter.buildPredicate()).and(QDescriptor.descriptor.state.in(PublishState.PUBLISHED)), markdownSortPageRequest);
return new PageImpl<>(res.getContent(), page, res.getTotalElements()); if (descriptorFilter.isFulltextQuery()) {
return elasticsearchService.findAll(Descriptor.class, descriptorFilter, descriptorFilter._text, markdownSortPageRequest);
} else {
return descriptorRepository.findAll(new BooleanBuilder().and(descriptorFilter.buildPredicate()).and(QDescriptor.descriptor.state.in(PublishState.PUBLISHED)), markdownSortPageRequest);
}
} }
/** /**
......
...@@ -185,15 +185,19 @@ public interface AccessionService { ...@@ -185,15 +185,19 @@ public interface AccessionService {
* @return the list of processed accessions * @return the list of processed accessions
*/ */
List<Accession> processAccessions(List<Long> accessionIds, IAccessionBatchAction action); List<Accession> processAccessions(List<Long> accessionIds, IAccessionBatchAction action);
public static interface IAccessionBatchAction { public static interface IBatchAction<T> {
/** /**
* Run action on batch of Accessions * Run action on batch of Accessions
* *
* @param a the accession * @param a the accession
* @return must return the resulting {@link Accession} * @return must return the resulting {@link Accession}
*/ */
List<Accession> apply(List<Accession> a) throws Exception; List<T> apply(List<T> a) throws Exception;
}
public static interface IAccessionBatchAction extends IBatchAction<Accession> {
} }
class AccessionDetails { class AccessionDetails {
......
...@@ -24,7 +24,12 @@ import java.util.Set; ...@@ -24,7 +24,12 @@ import java.util.Set;
import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilder;
import org.genesys.blocks.model.BasicModel; import org.genesys.blocks.model.BasicModel;
import org.genesys.blocks.model.filters.BasicModelFilter; import org.genesys.blocks.model.filters.BasicModelFilter;
import org.genesys2.server.service.AccessionService.IBatchAction;
import org.genesys2.server.service.impl.SearchException; import org.genesys2.server.service.impl.SearchException;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import com.querydsl.core.types.Predicate;
/** /**
* The Interface ElasticsearchService. * The Interface ElasticsearchService.
...@@ -199,6 +204,8 @@ public interface ElasticsearchService { ...@@ -199,6 +204,8 @@ public interface ElasticsearchService {
<T extends BasicModel> List<T> find(Class<T> clazz, BasicModelFilter<?, ?> filter); <T extends BasicModel> List<T> find(Class<T> clazz, BasicModelFilter<?, ?> filter);
<T extends BasicModel> Page<T> findAll(Class<T> clazz, BasicModelFilter<?, ?> filter, String text, Pageable page);
/** /**
* Wrapper for search results * Wrapper for search results
*/ */
...@@ -218,4 +225,18 @@ public interface ElasticsearchService { ...@@ -218,4 +225,18 @@ public interface ElasticsearchService {
return sr; return sr;
} }
} }
List<Double[]> distinctCoordinates(Predicate filt, String _text);
<T extends BasicModel> void process(Class<T> clazz, BasicModelFilter<?, ?> filter, IBatchAction<T> action, Long maxSize) throws Exception;
/**
* Wait until X records match specified filter in ES.
*
* @param clazz
* @param filter
* @param mustHaveCount
* @throws InterruptedException
*/
void waitForCount(Class<? extends BasicModel> clazz, BasicModelFilter<?, ?> filter, int mustHaveCount) throws InterruptedException;
} }
...@@ -42,10 +42,13 @@ import com.querydsl.jpa.JPQLQuery; ...@@ -42,10 +42,13 @@ import com.querydsl.jpa.JPQLQuery;
/** /**
* Filters for {@link Accession}. * Filters for {@link Accession}.
*/ */
public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession> implements Serializable { public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession> implements Serializable, IFullTextFilter {
private static final long serialVersionUID = -1441103961567816877L; private static final long serialVersionUID = -1441103961567816877L;
/** Any text. */
public String _text;
/** The historic. */ /** The historic. */
public Boolean historic; public Boolean historic;
...@@ -404,4 +407,7 @@ public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession> ...@@ -404,4 +407,7 @@ public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession>
return doi; return doi;
} }
public String get_text() {
return _text;
}
} }
/*
* Copyright 2019 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys2.server.service.filter;
import org.apache.commons.lang3.StringUtils;
import com.fasterxml.jackson.annotation.JsonIgnore;
/**
* IFullTextFilter is used to mark filters that use full-text search with ES
*/
public interface IFullTextFilter {
/**
* Gets the text.
*
* @return the text
*/
String get_text();
/**
* Does the filter require full-text search?
*
* @return true if {@link #get_text()} is non-blank
*/
@JsonIgnore
default boolean isFulltextQuery() {
return StringUtils.isNotBlank(get_text());
}
}
...@@ -28,8 +28,6 @@ import java.util.UUID; ...@@ -28,8 +28,6 @@ import java.util.UUID;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.genesys.blocks.auditlog.service.AuditTrailService; import org.genesys.blocks.auditlog.service.AuditTrailService;
import org.genesys.catalog.model.dataset.Dataset; import org.genesys.catalog.model.dataset.Dataset;
...@@ -69,6 +67,8 @@ import org.springframework.stereotype.Service; ...@@ -69,6 +67,8 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.querydsl.core.BooleanBuilder; import com.querydsl.core.BooleanBuilder;
import com.querydsl.core.Tuple; import com.querydsl.core.Tuple;
import com.querydsl.jpa.impl.JPAQuery; import com.querydsl.jpa.impl.JPAQuery;
...@@ -269,8 +269,8 @@ public class AccessionServiceImpl implements AccessionService { ...@@ -269,8 +269,8 @@ public class AccessionServiceImpl implements AccessionService {
public long countAccessions(AccessionFilter filter) { public long countAccessions(AccessionFilter filter) {
long total = elasticsearchService.count(Accession.class, filter); long total = elasticsearchService.count(Accession.class, filter);
if (total < 10000) { if (total < 10000 && ! filter.isFulltextQuery()) {
// If total is below 10K, use actual count // If total is below 10K and no full-text query, use actual count
total = accessionRepository.count(filter.buildPredicate()); total = accessionRepository.count(filter.buildPredicate());
} }
...@@ -368,6 +368,10 @@ public class AccessionServiceImpl implements AccessionService { ...@@ -368,6 +368,10 @@ public class AccessionServiceImpl implements AccessionService {
@Override @Override
// @Cacheable(value = "apiResponses.accessionApi1.list", unless = "#result == null", keyGenerator = "shortFilterKeyGenerator") // @Cacheable(value = "apiResponses.accessionApi1.list", unless = "#result == null", keyGenerator = "shortFilterKeyGenerator")
public Page<Accession> list(AccessionFilter filter, Pageable page) { public Page<Accession> list(AccessionFilter filter, Pageable page) {
if (filter.isFulltextQuery()) {
return elasticsearchService.findAll(Accession.class, filter, filter._text, page);
}
List<Accession> content = accessionRepository.findAll(filter, page); List<Accession> content = accessionRepository.findAll(filter, page);
long total = countAccessions(filter); long total = countAccessions(filter);
...@@ -486,6 +490,11 @@ public class AccessionServiceImpl implements AccessionService { ...@@ -486,6 +490,11 @@ public class AccessionServiceImpl implements AccessionService {
@Override @Override
public Number[][] getGeoBounds(AccessionFilter filter) { public Number[][] getGeoBounds(AccessionFilter filter) {
if (filter.isFulltextQuery()) {
return AccessionService.DEFAULT_GEOBOUNDS;
}
final QAccession accession = QAccession.accession; final QAccession accession = QAccession.accession;
JPAQuery<Tuple> query = jpaQueryFactory.selectFrom(accession).select(accession.accessionId.geo.latitude.min(), accession.accessionId.geo.longitude.max(), JPAQuery<Tuple> query = jpaQueryFactory.selectFrom(accession).select(accession.accessionId.geo.latitude.min(), accession.accessionId.geo.longitude.max(),
......
...@@ -198,6 +198,11 @@ public class DownloadServiceImpl implements DownloadService { ...@@ -198,6 +198,11 @@ public class DownloadServiceImpl implements DownloadService {
@Override @Override
public void writeXlsxMCPD(AccessionFilter filter, OutputStream outputStream) throws IOException { public void writeXlsxMCPD(AccessionFilter filter, OutputStream outputStream) throws IOException {
if (filter.isFulltextQuery()) {
throw new IOException("Cannot download MCPD when using full-text queries.");
}
final String dataSource = baseUrl + "/explore?filter=" + filter.toString(); final String dataSource = baseUrl + "/explore?filter=" + filter.toString();
writeXlsxMCPD(filter.buildPredicate(), outputStream, filter.toString(), dataSource); writeXlsxMCPD(filter.buildPredicate(), outputStream, filter.toString(), dataSource);
} }
...@@ -768,6 +773,11 @@ public class DownloadServiceImpl implements DownloadService { ...@@ -768,6 +773,11 @@ public class DownloadServiceImpl implements DownloadService {
@Override @Override
public void writeXlsxPDCI(final AccessionFilter filter, final OutputStream outputStream) throws IOException { public void writeXlsxPDCI(final AccessionFilter filter, final OutputStream outputStream) throws IOException {
if (filter.isFulltextQuery()) {
throw new IOException("Cannot download MCPD when using full-text queries.");
}
XSSFWorkbook template = new XSSFWorkbook(getClass().getResourceAsStream("/template/download/PDCI.xlsx")); XSSFWorkbook template = new XSSFWorkbook(getClass().getResourceAsStream("/template/download/PDCI.xlsx"));
// keep 1000 rows in memory, exceeding rows will be flushed to disk // keep 1000 rows in memory, exceeding rows will be flushed to disk
......
...@@ -474,14 +474,19 @@ public class GenesysFilterServiceImpl implements GenesysFilterService { ...@@ -474,14 +474,19 @@ public class GenesysFilterServiceImpl implements GenesysFilterService {
filt.and(accessionGeo.latitude.between(latS - zoom * diffLat * .2, latN + zoom * diffLat * .2)); filt.and(accessionGeo.latitude.between(latS - zoom * diffLat * .2, latN + zoom * diffLat * .2));
} }
query.where(filt); query.where(filt);
if (filter.isFulltextQuery()) {
return elasticsearchService.distinctCoordinates(filt, filter._text);
} else {
List<Tuple> results = query.fetch();
return results.stream()
.map(item -> new Double[]{
item.get(accessionGeo.longitude),
item.get(accessionGeo.latitude),
}).collect(Collectors.toList());
}
List<Tuple> results = query.fetch();
return results.stream()
.map(item -> new Double[]{
item.get(accessionGeo.longitude),
item.get(accessionGeo.latitude),
}).collect(Collectors.toList());
} }
@Override @Override
......
...@@ -1051,6 +1051,11 @@ public class GenesysServiceImpl implements GenesysService, DatasetService { ...@@ -1051,6 +1051,11 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
@Override @Override
// TODO FIXME Need proper term URLs // TODO FIXME Need proper term URLs
public void writeAccessions(final AccessionFilter filter, final OutputStream outputStream) throws Exception { public void writeAccessions(final AccessionFilter filter, final OutputStream outputStream) throws Exception {
if (filter.isFulltextQuery()) {
throw new IOException("Cannot download DWC-A when using full-text queries.");
}
// UTF8 is used for encoding entry names // UTF8 is used for encoding entry names
final ZipOutputStream zos = new ZipOutputStream(outputStream); final ZipOutputStream zos = new ZipOutputStream(outputStream);
zos.setComment("Genesys Accessions filter=" + filter); zos.setComment("Genesys Accessions filter=" + filter);
......
...@@ -25,6 +25,7 @@ import org.genesys2.server.model.genesys.Accession; ...@@ -25,6 +25,7 @@ import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.QAccession; import org.genesys2.server.model.genesys.QAccession;
import org.genesys2.server.persistence.AccessionRepository; import org.genesys2.server.persistence.AccessionRepository;
import org.genesys2.server.service.AccessionService; import org.genesys2.server.service.AccessionService;
import org.genesys2.server.service.ElasticsearchService;
import org.genesys2.server.service.AccessionService.IAccessionBatchAction; import org.genesys2.server.service.AccessionService.IAccessionBatchAction;
import org.genesys2.server.service.filter.AccessionFilter; import org.genesys2.server.service.filter.AccessionFilter;
import org.slf4j.Logger; import org.slf4j.Logger;
...@@ -64,14 +65,25 @@ public class AccessionProcessor { ...@@ -64,14 +65,25 @@ public class AccessionProcessor {
/// Size of database batch scan for IDs /// Size of database batch scan for IDs
private int batchSize = 1000; private int batchSize = 1000;
@Autowired
private ElasticsearchService elasticSearchService;
@Transactional(readOnly = true, propagation = Propagation.REQUIRES_NEW) @Transactional(readOnly = true, propagation = Propagation.REQUIRES_NEW)