Commit 9e339bd9 authored by Matija Obreza's avatar Matija Obreza
Browse files

Merge branch '25-filter-accessions-by-georegion' into 'master'

Resolve "Filter accessions by GeoRegion"

Closes #25

See merge request !12
parents 2ee0ffbf 892b0ec1
......@@ -85,5 +85,8 @@ public interface FilterConstants {
public static final String PDCI = "pdci";
public static final String REGION_ORIGIN = "regionOrigin";
public static final String REGION_HOLD_INSTITUTE= "regionHoldInst";
}
......@@ -41,4 +41,5 @@ public interface GenesysFilterService {
void listGeoTile(boolean distinct, AppliedFilters filters, Integer limit, int zoom, int xtile, int ytile, RowCallbackHandler rowHandler);
AppliedFilters transformFiltersIfNeed(AppliedFilters appliedFilters);
}
......@@ -7,6 +7,7 @@ import org.xml.sax.SAXException;
import javax.xml.parsers.ParserConfigurationException;
import java.io.IOException;
import java.util.List;
import java.util.Locale;
public interface GeoRegionService {
......@@ -18,10 +19,16 @@ public interface GeoRegionService {
List<GeoRegion> findAll();
List<GeoRegion> findAll(Locale locale);
void updateGeoRegionData() throws IOException, ParserConfigurationException, SAXException;
GeoRegion getRegion(Country country);
List<GeoRegion> getGeoRegionsForFilter();
List<GeoRegion> conversionToSubRegionsList(GeoRegion parentGeo);
List<GeoRegion> getChildren(String regionIsoCode);
}
......@@ -99,6 +99,9 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
@Autowired
private FilterHandler filterHandler;
@Autowired
private GenesysFilterService filterService;
private final Map<String, Class<?>> clazzMap;
@Autowired
......@@ -160,6 +163,7 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
@Override
public String getBoundingBox(final AppliedFilters filters) {
filterService.transformFiltersIfNeed(filters);
AndFilterBuilder filterBuilder = getFilterBuilder(filters);
final FilterAggregationBuilder builder = AggregationBuilders
......@@ -223,6 +227,8 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
*/
@Override
public TermResult termStatisticsAuto(AppliedFilters appliedFilters, String term, int size) throws SearchException {
filterService.transformFiltersIfNeed(appliedFilters);
TermResult termResult = null;
int newSize = size;
do {
......
......@@ -81,6 +81,8 @@ public class FilterHandler {
this.availableFilters.add(new BasicFilter(FilterConstants.CROPNAME, DataType.STRING));
this.availableFilters.add(new BasicFilter(FilterConstants.LISTS, DataType.STRING));
this.availableFilters.add(new BasicFilter(FilterConstants.REGION_ORIGIN, DataType.STRING));
this.availableFilters.add(new BasicFilter(FilterConstants.REGION_HOLD_INSTITUTE, DataType.STRING));
this.availableFilters.add(new I18nListFilter<Integer>(FilterConstants.SAMPSTAT, DataType.NUMERIC).build("accession.sampleStatus",
new Integer[] { 100, 110, 120, 130, 200, 300, 400, 410, 411, 412, 413, 414, 415, 416, 420, 421, 422, 423, 500, 600, 999 }));
this.availableFilters.add(new AutocompleteFilter(FilterConstants.TAXONOMY_GENUS, "/explore/ac/" + FilterConstants.TAXONOMY_GENUS));
......@@ -89,7 +91,7 @@ public class FilterHandler {
this.availableFilters.add(new AutocompleteFilter(FilterConstants.TAXONOMY_SCINAME, "/explore/ac/" + FilterConstants.TAXONOMY_SCINAME));
this.availableFilters.add(new AutocompleteFilter(FilterConstants.ORGCTY_ISO3, "/explore/ac/" + FilterConstants.ORGCTY_ISO3));
this.availableFilters.add(new AutocompleteFilter(FilterConstants.INSTITUTE_COUNTRY_ISO3, "/explore/ac/" + FilterConstants.ORGCTY_ISO3));
this.availableFilters.add(new AutocompleteFilter(FilterConstants.INSTITUTE_COUNTRY_ISO3, "/explore/ac/" + FilterConstants.INSTITUTE_COUNTRY_ISO3));
this.availableFilters.add(new BasicFilter(FilterConstants.GEO_LATITUDE, DataType.NUMERIC));
this.availableFilters.add(new BasicFilter(FilterConstants.GEO_LONGITUDE, DataType.NUMERIC));
this.availableFilters.add(new BasicFilter(FilterConstants.GEO_ELEVATION, DataType.NUMERIC));
......@@ -135,6 +137,17 @@ public class FilterHandler {
return null;
}
public List<GenesysFilter> getAdditionalFilters(String[] selectedFilters) {
List<String> gmFilterList = new ArrayList<>();
for (String filter : selectedFilters) {
if (filter.startsWith("gm:")) {
gmFilterList.add(filter);
}
}
String[] gmFilters = gmFilterList.toArray(new String[gmFilterList.size()]);
return selectFilters(gmFilters);
}
public List<GenesysFilter> selectFilters(String[] selectedFilters) {
LOG.debug("Loading filter definitions sel=" + ArrayUtils.toString(selectedFilters));
final List<GenesysFilter> filters = new ArrayList<GenesysFilter>();
......
......@@ -20,6 +20,7 @@ import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
......@@ -33,6 +34,7 @@ import org.genesys2.server.model.genesys.Method;
import org.genesys2.server.model.impl.Country;
import org.genesys2.server.model.impl.Crop;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.model.impl.GeoRegion;
import org.genesys2.server.persistence.domain.AccessionRepository;
import org.genesys2.server.persistence.domain.MethodRepository;
import org.genesys2.server.service.CropService;
......@@ -40,6 +42,7 @@ import org.genesys2.server.service.ElasticService;
import org.genesys2.server.service.FilterConstants;
import org.genesys2.server.service.GenesysFilterService;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.GeoRegionService;
import org.genesys2.server.service.GeoService;
import org.genesys2.server.service.InstituteService;
import org.genesys2.server.service.TaxonomyService;
......@@ -99,6 +102,9 @@ public class GenesysFilterServiceImpl implements GenesysFilterService {
@Autowired
private ElasticService elasticService;
@Autowired
private GeoRegionService geoRegionService;
@Autowired
public void setDataSource(final DataSource dataSource) {
this.jdbcTemplate = new JdbcTemplate(dataSource);
......@@ -157,6 +163,8 @@ public class GenesysFilterServiceImpl implements GenesysFilterService {
}
}
transformFiltersIfNeed(filters);
if (pageable.getPageNumber() > PAGINATION_MAXPAGE_LIMIT) {
throw new RuntimeException("Genesys does not support requests for data beyond " + PAGINATION_MAXPAGE_LIMIT + "th page.");
}
......@@ -197,6 +205,7 @@ public class GenesysFilterServiceImpl implements GenesysFilterService {
LOG.debug("Autocomplete " + filter + " ac=" + ac);
}
transformFiltersIfNeed(filters);
Crop crop = null;
{
......@@ -236,19 +245,20 @@ public class GenesysFilterServiceImpl implements GenesysFilterService {
LOG.error("Error occurred during search", e);
}
if (termResult != null) {
for (final Country c : countries) {
String label = c.getCode3() + ", " + c.getName();
for (Term term : termResult.getTerms()) {
if (term.getTerm().equals(c.getCode3()) && term.getCount() > 0) {
label = label.concat(" (" + term.getCount() + ")");
completed.add(new LabelValue<String>(c.getCode3(), label));
break;
}
}
}
fillCountryCompleteListFromTerm(completed, termResult, countries);
} else if (FilterConstants.INSTITUTE_COUNTRY_ISO3.equalsIgnoreCase(filter)) {
final List<Country> countries = geoService.autocomplete(ac);
TermResult termResult = null;
try {
termResult = elasticService.termStatisticsAuto(filters, FilterConstants.INSTITUTE_COUNTRY_ISO3, 20000);
} catch (SearchException e) {
LOG.error("Error occurred during search", e);
}
} else if (FilterConstants.TAXONOMY_GENUS.equalsIgnoreCase(filter)) {
fillCountryCompleteListFromTerm(completed, termResult, countries);
}
else if (FilterConstants.TAXONOMY_GENUS.equalsIgnoreCase(filter)) {
final List<String> genera = taxonomyService.autocompleteGenus(ac, crop);
TermResult termResult = null;
try {
......@@ -257,18 +267,8 @@ public class GenesysFilterServiceImpl implements GenesysFilterService {
LOG.error("Error occurred during search", e);
}
if (termResult != null) {
for (final String value : genera) {
String label = value;
for (Term term : termResult.getTerms()) {
if (term.getTerm().equals(value) && term.getCount() > 0) {
label = label.concat(" (" + term.getCount() + ")");
completed.add(new LabelValue<String>(value, label));
break;
}
}
}
}
fillCompleteListFromTerm(completed, termResult, genera);
} else if (FilterConstants.TAXONOMY_SPECIES.equalsIgnoreCase(filter)) {
List<String> genus = new ArrayList<>();
......@@ -287,18 +287,8 @@ public class GenesysFilterServiceImpl implements GenesysFilterService {
LOG.error("Error occurred during search", e);
}
if (termResult != null) {
for (final String value : species) {
String label = value;
for (Term term : termResult.getTerms()) {
if (term.getTerm().equals(value) && term.getCount() > 0) {
label = label.concat(" (" + term.getCount() + ")");
completed.add(new LabelValue<String>(value, label));
break;
}
}
}
}
fillCompleteListFromTerm(completed, termResult, species);
} else if (FilterConstants.TAXONOMY_SUBTAXA.equalsIgnoreCase(filter)) {
List<String> genus = new ArrayList<>();
List<String> species = new ArrayList<>();
......@@ -324,18 +314,8 @@ public class GenesysFilterServiceImpl implements GenesysFilterService {
LOG.error("Error occurred during search", e);
}
if (termResult != null) {
for (final String value : subtaxa) {
String label = value;
for (Term term : termResult.getTerms()) {
if (term.getTerm().equals(value) && term.getCount() > 0) {
label = label.concat(" (" + term.getCount() + ")");
completed.add(new LabelValue<String>(value, label));
break;
}
}
}
}
fillCompleteListFromTerm(completed, termResult, subtaxa);
} else if (FilterConstants.TAXONOMY_SCINAME.equalsIgnoreCase(filter)) {
final List<String> taxa = taxonomyService.autocompleteTaxonomy(ac);
TermResult termResult = null;
......@@ -344,23 +324,41 @@ public class GenesysFilterServiceImpl implements GenesysFilterService {
} catch (SearchException e) {
LOG.error("Error occurred during search", e);
}
fillCompleteListFromTerm(completed, termResult, taxa);
} else {
throw new RuntimeException("No autocompleter for " + filter);
}
return completed;
}
if (termResult != null) {
for (final String taxonomy : taxa) {
String label = taxonomy;
for (Term term : termResult.getTerms()) {
if (term.getTerm().equals(taxonomy) && term.getCount() > 0) {
label = label.concat(" (" + term.getCount() + ")");
completed.add(new LabelValue<String>(taxonomy, label));
break;
}
private void fillCountryCompleteListFromTerm(final List<LabelValue<String>> completed, final TermResult termResult, final List<Country> countries) {
if (termResult != null) {
for (final Country c : countries) {
String label = c.getCode3() + ", " + c.getName();
for (Term term : termResult.getTerms()) {
if (term.getTerm().equals(c.getCode3()) && term.getCount() > 0) {
label = label.concat(" (" + term.getCount() + ")");
completed.add(new LabelValue<String>(c.getCode3(), label));
break;
}
}
}
}
}
private void fillCompleteListFromTerm(final List<LabelValue<String>> completed, final TermResult termResult, final List<String> list) {
if (termResult != null) {
for (final String val : list) {
String label = val;
for (Term term : termResult.getTerms()) {
if (term.getTerm().equals(val) && term.getCount() > 0) {
label = label.concat(" (" + term.getCount() + ")");
completed.add(new LabelValue<String>(val, label));
break;
}
}
}
} else {
throw new RuntimeException("No autocompleter for " + filter);
}
return completed;
}
public static class LabelValue<T> {
......@@ -426,4 +424,132 @@ public class GenesysFilterServiceImpl implements GenesysFilterService {
}, rowHandler);
}
@Override
public AppliedFilters transformFiltersIfNeed(final AppliedFilters appliedFilters) {
AppliedFilter regionOriginFilter = appliedFilters.stream()
.filter(filter -> filter.getFilterName().equals("regionOrigin")).findFirst().orElse(null);
AppliedFilter regionHoldInstFilter = appliedFilters.stream()
.filter(filter -> filter.getFilterName().equals("regionHoldInst")).findFirst().orElse(null);
AppliedFilter countryOriginFilter = appliedFilters.stream()
.filter(filter -> filter.getFilterName().equals("orgCty.iso3")).findFirst().orElse(null);
AppliedFilter countryHoldInstFilter = appliedFilters.stream()
.filter(filter -> filter.getFilterName().equals("institute.country.iso3")).findFirst().orElse(null);
List<String> regionHoldInstValues = new ArrayList<>();
if (regionOriginFilter !=null && countryOriginFilter!=null && !countryOriginFilter.getValues().isEmpty()) {
regionHoldInstValues.addAll(getFilterValues(countryOriginFilter));
}
List<String> instituteCountryValues = new ArrayList<>();
if (regionHoldInstFilter !=null && countryHoldInstFilter!=null && !countryHoldInstFilter.getValues().isEmpty()) {
instituteCountryValues.addAll(getFilterValues(countryHoldInstFilter));
}
conversionFiltersFromGeoRegions(appliedFilters);
transformFilterListIfExistsRegionAndCountryInFilters(appliedFilters, regionOriginFilter, countryOriginFilter, regionHoldInstValues);
transformFilterListIfExistsRegionAndCountryInFilters(appliedFilters, regionHoldInstFilter, countryHoldInstFilter, instituteCountryValues);
return appliedFilters;
}
private void transformFilterListIfExistsRegionAndCountryInFilters(final AppliedFilters appliedFilters,
final AppliedFilter regionFilter,
final AppliedFilter countryFilter,
final List<String> regionValues) {
if (regionFilter != null && countryFilter != null && !regionValues.isEmpty()) {
AppliedFilter newCountryFilter = new AppliedFilter();
newCountryFilter.setFilterName(countryFilter.getFilterName());
for (String value : regionValues) {
newCountryFilter.addFilterValue(new FilterHandler.LiteralValueFilter(value));
}
appliedFilters.removeIf(appliedFilter1 -> appliedFilter1.getFilterName().equals(countryFilter.getFilterName()));
appliedFilters.add(newCountryFilter);
}
}
private AppliedFilters conversionFiltersFromGeoRegions(final AppliedFilters appliedFilters) {
AppliedFilter regionOriginFilter = appliedFilters.stream()
.filter(filter -> filter.getFilterName().equals("regionOrigin")).findFirst().orElse(null);
AppliedFilter regionHoldInstFilter = appliedFilters.stream()
.filter(filter -> filter.getFilterName().equals("regionHoldInst")).findFirst().orElse(null);
if (regionOriginFilter != null && !regionOriginFilter.getValues().isEmpty()) {
conversionGeoRegionsFilterToAnotherFilter(appliedFilters, regionOriginFilter, "orgCty.iso3");
}
if (regionHoldInstFilter != null && !regionHoldInstFilter.getValues().isEmpty()) {
conversionGeoRegionsFilterToAnotherFilter(appliedFilters, regionHoldInstFilter, "institute.country.iso3");
}
return appliedFilters;
}
private AppliedFilters conversionGeoRegionsFilterToAnotherFilter(final AppliedFilters appliedFilters,
final AppliedFilter oldFilter,
final String anotherFilterName) {
List<GeoRegion> geoRegionList = new ArrayList<>();
List<Country> countryList = new ArrayList<>();
Set<String> countryIsoList = new HashSet<>();
for (String isoCode: getFilterValues(oldFilter)) {
GeoRegion geoRegion = geoRegionService.find(isoCode);
if (geoRegion != null) {
if (geoRegion.getParentRegion().getName().equals("Americas") ||
geoRegion.getParentRegion().getName().equals("World")) {
geoRegionList.addAll(geoRegionService.conversionToSubRegionsList(geoRegion));
} else {
geoRegionList.add(geoRegionService.find(isoCode));
}
}
}
if (!geoRegionList.isEmpty()) {
for (GeoRegion geoRegion: geoRegionList) {
countryList.addAll(geoRegion.getCountries());
}
}
if (!countryList.isEmpty()) {
for (Country country: countryList) {
countryIsoList.add(country.getCode3());
}
}
if (!countryIsoList.isEmpty()) {
AppliedFilter orgCtyFilter = appliedFilters.stream()
.filter(filter -> filter.getFilterName().equals(anotherFilterName))
.findFirst().orElse(new AppliedFilter().setFilterName(anotherFilterName));
if (!orgCtyFilter.getValues().isEmpty())
countryIsoList.addAll(getFilterValues(orgCtyFilter));
for (String isoCode: countryIsoList) {
orgCtyFilter.addFilterValue(new FilterHandler.LiteralValueFilter(isoCode));
}
appliedFilters.removeIf(appliedFilter1 -> appliedFilter1.getFilterName().equals(oldFilter.getFilterName()));
if (!appliedFilters.hasFilter(anotherFilterName))
appliedFilters.add(orgCtyFilter);
}
return appliedFilters;
}
private Set<String> getFilterValues(final AppliedFilter appliedFilter) {
Set<String> literals = new HashSet<>();
for (FilterHandler.FilterValue filterValue : appliedFilter.getValues()) {
if (filterValue instanceof FilterHandler.LiteralValueFilter) {
FilterHandler.LiteralValueFilter literal = (FilterHandler.LiteralValueFilter) filterValue;
literals.add(literal.getValue().toString());
}
}
return literals;
}
}
......@@ -2,7 +2,10 @@ package org.genesys2.server.service.impl;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Locale;
import java.util.Optional;
import java.util.stream.Collectors;
......@@ -17,6 +20,7 @@ import org.genesys2.server.service.worker.GeoRegionDataCLDR;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.i18n.LocaleContextHolder;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.xml.sax.SAXException;
......@@ -83,6 +87,18 @@ public class GeoRegionServiceImpl implements GeoRegionService {
return geoRegionRepository.findAll();
}
@Override
public List<GeoRegion> findAll(final Locale locale) {
final List<GeoRegion> all = findAll();
Collections.sort(all, new Comparator<GeoRegion>() {
@Override
public int compare(GeoRegion o1, GeoRegion o2) {
return o1.getName(locale).compareTo(o2.getName(locale));
}
});
return all;
}
@Override
@Transactional(readOnly = false)
public void updateGeoRegionData() throws IOException, ParserConfigurationException, SAXException {
......@@ -125,6 +141,40 @@ public class GeoRegionServiceImpl implements GeoRegionService {
}
}
@Override
public List<GeoRegion> getGeoRegionsForFilter() {
List<GeoRegion> geoRegionList = findAll();
geoRegionList.removeIf(geoRegion -> geoRegion.getName().equals("World") || geoRegion.getName().equals("Americas"));
List<GeoRegion> resultList = new ArrayList<>();
for (GeoRegion geoRegion: geoRegionList) {
if (geoRegion.getParentRegion().getName().equals("World") || geoRegion.getParentRegion().getName().equals("Americas")) {
geoRegion.setParentRegion(null);
resultList.add(geoRegion);
}
else resultList.add(geoRegion);
}
Locale locale = LocaleContextHolder.getLocale();
resultList.sort(Comparator.comparing(o -> o.getName(locale)));
return resultList;
}
@Override
public List<GeoRegion> conversionToSubRegionsList(final GeoRegion parentGeo) {
List<GeoRegion> geoRegionList = findAll();
List<GeoRegion> subRegionsList = new ArrayList<>();
geoRegionList.removeIf(geoRegion -> geoRegion.getName().equals("World"));
subRegionsList.addAll(geoRegionList.stream().
filter(geoRegion -> parentGeo.getName().equals(geoRegion.getParentRegion().getName()))
.collect(Collectors.toList()));
return subRegionsList;
}
private boolean isChild(GeoRegion region, String parentIsoCode) {
return region.getParentRegion() != null && (region.getParentRegion().getIsoCode().equals(parentIsoCode) || isChild(region.getParentRegion(), parentIsoCode));
......
......@@ -154,6 +154,7 @@ public class MappingServiceImpl implements MappingService {
@Override
@Cacheable(value = "tileserver", key = "'tile-' + #zoom + '-' + #xtile + '-' + #ytile + '-' + #filters")
public byte[] getTile(AppliedFilters filters, final int zoom, final int xtile, final int ytile) {
filterService.transformFiltersIfNeed(filters);
final BufferedImage bi = new BufferedImage(256, 256, BufferedImage.TYPE_INT_ARGB);
// final int pixelSize = (int) Math.round(1.0 + (1 << (zoom / 2))) / 2;
......
......@@ -53,12 +53,14 @@ import org.genesys2.server.model.genesys.Parameter;
import org.genesys2.server.model.genesys.ParameterCategory;
import org.genesys2.server.model.impl.Country;
import org.genesys2.server.model.impl.Crop;
import org.genesys2.server.model.impl.GeoRegion;
import org.genesys2.server.service.CropService;
import org.genesys2.server.service.DownloadService;
import org.genesys2.server.service.ElasticService;
import org.genesys2.server.service.FilterConstants;
import org.genesys2.server.service.GenesysFilterService;
import org.genesys2.server.service.GenesysService;
import org.genesys2.server.service.GeoRegionService;
import org.genesys2.server.service.GeoService;
import org.genesys2.server.service.MappingService;
import org.genesys2.server.service.TraitService;
......@@ -124,6 +126,9 @@ public class ExplorerController extends BaseController implements InitializingBe
@Autowired
private GeoService geoService;
@Autowired
private GeoRegionService geoRegionService;
private final ObjectMapper mapper = new ObjectMapper();
@Value("${base.url}")
......@@ -177,15 +182,12 @@ public class ExplorerController extends BaseController implements InitializingBe
* @throws IOException
*/
@RequestMapping("/explore")
public String viewFiltered(
HttpServletResponse response,
ModelMap model,
public String viewFiltered(HttpServletResponse response, ModelMap model,
@RequestParam(value = "page", required = false, defaultValue = "1") int page,
@RequestParam(value = "filter", required = true, defaultValue = "{}") String jsonFilter,
@RequestParam(value = "results", required = true, defaultValue = "50") int results,
@RequestParam(value = "columns", required = true, defaultValue = "") String[] columns,
@CookieValue(value = "columns", required = false) String[] cookieColumns
)
@CookieValue(value = "columns", required = false) String[] cookieColumns)
throws IOException, SearchException {
String[] selectedFilters = null;
......@@ -193,6 +195,7 @@ public class ExplorerController extends BaseController implements InitializingBe
_logger.debug("Filtering by: " + jsonFilter);
AppliedFilters appliedFilters = mapper.readValue(jsonFilter, AppliedFilters.class);
final AppliedFilters storedTransformedFilters = filterService.transformFiltersIfNeed(appliedFilters);
Crop crop = null;
{
String shortName = appliedFilters.getFirstLiteralValue(FilterConstants.CROPS, String.class);