Commit 19b9f806 authored by Matija Obreza's avatar Matija Obreza

Merge branch '481-accession-filter-active-records-by-default' into 'master'

Resolve "Accession filter: Active records by default"

Closes #481

See merge request genesys-pgr/genesys-server!487
parents f34f7b20 682f64ee
......@@ -62,12 +62,14 @@ public class ShortFilterServiceImpl implements ShortFilterService, InitializingB
private final ObjectMapper mapper;
private final ObjectMapper strictMapper;
private final ObjectMapper mapperWithAllFields;
@Value("${hashids.salt}")
private String hashidsSalt;
private Hashids hashids;
public ShortFilterServiceImpl() {
mapper = new ObjectMapper();
strictMapper = new ObjectMapper();
......@@ -83,6 +85,13 @@ public class ShortFilterServiceImpl implements ShortFilterService, InitializingB
// ignore outdated properties
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
strictMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true);
// upgrade to arrays
mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true);
// don't ignore Null fields in this mapper
mapperWithAllFields = mapper.copy().setSerializationInclusion(JsonInclude.Include.USE_DEFAULTS);
}
@Override
......@@ -92,18 +101,42 @@ public class ShortFilterServiceImpl implements ShortFilterService, InitializingB
}
@Override
@SuppressWarnings(value = "unchecked")
public String normalizeFilter(final BasicModelFilter<?, ?> filter) throws IOException {
// Defaults
BasicModelFilter<?, ?> defaultFilter = null;
Map<String, Object> defaultFilterMap = null;
try {
defaultFilter = filter.getClass().newInstance();
defaultFilterMap = (Map<String, Object>) mapper.readValue(mapper.writeValueAsString(defaultFilter), Object.class);
} catch (InstantiationException | IllegalAccessException e) {
LOG.warn("Class {} cannot be instantiated.", filter.getClass().getSimpleName());
}
// get serialized filter without invalid values; with sorted keys but unsorted
// values
final String serializedFilter = mapper.writeValueAsString(filter);
final String serializedFilter = mapper.writeValueAsString(BasicModelFilter.normalize(filter));
// we have to deserialize filter object to be able to sort values
final Object filterObject = mapper.readValue(serializedFilter, Object.class);
sortFilterValues(filterObject);
if (defaultFilterMap == null || defaultFilterMap.size() == 0) {
sortFilterValues(filterObject);
return mapper.writeValueAsString(filterObject);
}
// make filter object as a map with keys and values
final Map<String, Object> filterMap = (Map<String, Object>) filterObject;
// add default properties with NULL values to current filter
// if they aren't included after serialization
defaultFilterMap.keySet().forEach(key -> filterMap.putIfAbsent(key, null));
// get nice sorted filter
return mapper.writeValueAsString(filterObject);
sortFilterValues(filterMap);
return mapperWithAllFields.writeValueAsString(filterMap);
}
@SuppressWarnings(value = "unchecked")
......@@ -210,13 +243,13 @@ public class ShortFilterServiceImpl implements ShortFilterService, InitializingB
while (shortFilter.getRedirect() != null) {
if (codes.contains(shortFilter.getRedirect())) {
LOG.warn("Cyclic shortFilter redirect for {}", shortFilter.getRedirect());
return strictMapper.readValue(shortFilter.getJson(), clazz);
return BasicModelFilter.normalize(strictMapper.readValue(shortFilter.getJson(), clazz));
}
shortFilter = shortFilterRepository.findByCode(shortFilter.getRedirect());
codes.add(shortFilter.getCode());
}
return strictMapper.readValue(shortFilter.getJson(), clazz);
return BasicModelFilter.normalize(strictMapper.readValue(shortFilter.getJson(), clazz));
}
@Override
......@@ -225,8 +258,8 @@ public class ShortFilterServiceImpl implements ShortFilterService, InitializingB
FilterInfo<T> processedFilter = new FilterInfo<>();
if (filterCode != null) {
try {
processedFilter.filterCode = filterCode;
processedFilter.filter = filterByCode(filterCode, clazz);
processedFilter.filterCode = getCode(processedFilter.filter);
} catch (JsonMappingException e) {
LOG.warn("Stored filter does not match target {}", clazz);
......
......@@ -231,15 +231,10 @@ public class AccessionController {
public AccessionSuggestionPage<Accession> filter(@RequestParam(name = "f", required = false) String filterCode, final Pagination page,
@RequestBody(required = false) AccessionFilter filter) throws IOException, SearchException {
if (filterCode != null) {
filter = shortFilterService.filterByCode(filterCode, AccessionFilter.class);
} else {
filterCode = shortFilterService.getCode(filter);
}
FilterInfo<AccessionFilter> filterInfo = shortFilterService.processFilter(filterCode, filter, AccessionFilter.class);
FilteredPage<Accession> pageRes = new FilteredPage<>(filterInfo.filterCode, filterInfo.filter, accessionService.list(filterInfo.filter, page.toPageRequest(50, Sort.Direction.ASC, "seqNo")));
Map<String, TermResult> suggestionRes = accessionService.getSuggestions(filter);
Map<String, TermResult> suggestionRes = accessionService.getSuggestions(filterInfo.filter);
return new AccessionSuggestionPage<Accession>(pageRes, suggestionRes);
}
......@@ -257,17 +252,13 @@ public class AccessionController {
public AccessionOverview overview(@RequestParam(name = "f", required = false) String filterCode, @RequestBody(required = false) AccessionFilter filter)
throws IOException, SearchException {
if (filterCode != null) {
filter = shortFilterService.filterByCode(filterCode, AccessionFilter.class);
} else {
filterCode = shortFilterService.getCode(filter);
}
FilterInfo<AccessionFilter> filterInfo = shortFilterService.processFilter(filterCode, filter, AccessionFilter.class);
Map<String, TermResult> overview = elasticsearchService.termStatisticsAuto(Accession.class, filter, 10, terms.toArray(new String[] {}));
long accessionCount = accessionService.countAccessions(filter);
Map<String, TermResult> suggestions = accessionService.getSuggestions(filter);
Map<String, TermResult> overview = elasticsearchService.termStatisticsAuto(Accession.class, filterInfo.filter, 10, terms.toArray(new String[] {}));
long accessionCount = accessionService.countAccessions(filterInfo.filter);
Map<String, TermResult> suggestions = accessionService.getSuggestions(filterInfo.filter);
return AccessionOverview.from(filterCode, filter, overview, accessionCount, suggestions);
return AccessionOverview.from(filterInfo.filterCode, filterInfo.filter, overview, accessionCount, suggestions);
}
/**
......@@ -326,21 +317,18 @@ public class AccessionController {
@PostMapping(value = "/mapinfo", produces = MediaType.APPLICATION_JSON_VALUE)
@JsonView({ JsonViews.Public.class })
public AccessionMapInfo mapInfo(@RequestParam(value = "f", required = false) String filterCode, @RequestBody(required = false) AccessionFilter filter) throws IOException, SearchException {
if (filterCode != null) {
filter = shortFilterService.filterByCode(filterCode, AccessionFilter.class);
} else {
filterCode = shortFilterService.getCode(filter);
}
FilterInfo<AccessionFilter> filterInfo = shortFilterService.processFilter(filterCode, filter, AccessionFilter.class);
// Force only georeferenced accessions
AccessionFilter georefFilter = filter.copy(AccessionFilter.class);
AccessionFilter georefFilter = filterInfo.filter.copy(AccessionFilter.class);
georefFilter.geo().referenced = true;
AccessionMapInfo mapInfo = new AccessionMapInfo();
mapInfo.filterCode = filterCode;
mapInfo.filter = filter;
mapInfo.filterCode = filterInfo.filterCode;
mapInfo.filter = filterInfo.filter;
if (StringUtils.isBlank(filterCode)) {
if (StringUtils.isBlank(filterInfo.filterCode)) {
// Entire map
mapInfo.bounds = AccessionService.DEFAULT_GEOBOUNDS;
} else {
......@@ -348,7 +336,7 @@ public class AccessionController {
}
mapInfo.accessionCount = accessionService.countAccessions(georefFilter);
mapInfo.tileServers = cdnServers;
mapInfo.suggestions= accessionService.getSuggestions(filter);
mapInfo.suggestions= accessionService.getSuggestions(filterInfo.filter);
return mapInfo;
}
......@@ -410,11 +398,6 @@ public class AccessionController {
@RequestParam(value = "term", required = true) String term,
@RequestParam(name = "f", required = false) String filterCode, @RequestBody(required = false) AccessionFilter filter) throws IOException {
if (filterCode != null) {
filter = shortFilterService.filterByCode(filterCode, AccessionFilter.class);
} else {
filterCode = shortFilterService.getCode(filter);
}
FilterInfo<AccessionFilter> filterInfo = shortFilterService.processFilter(filterCode, filter, AccessionFilter.class);
return accessionService.autocomplete(filterInfo.filter, field, term);
}
......
......@@ -15,12 +15,12 @@
*/
package org.genesys2.server.service.filter;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.deser.std.UUIDDeserializer;
import com.hazelcast.util.CollectionUtil;
import com.querydsl.core.types.ExpressionUtils;
import com.querydsl.core.types.Predicate;
import com.querydsl.jpa.JPQLQuery;
import java.io.Serializable;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import org.genesys.blocks.model.filters.NumberFilter;
import org.genesys.blocks.model.filters.StringFilter;
import org.genesys.blocks.model.filters.UuidModelFilter;
......@@ -30,11 +30,12 @@ import org.genesys2.server.model.genesys.QAccession;
import org.genesys2.server.model.genesys.QAccessionId;
import org.genesys2.server.model.impl.QSubset;
import java.io.Serializable;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.deser.std.UUIDDeserializer;
import com.hazelcast.util.CollectionUtil;
import com.querydsl.core.types.ExpressionUtils;
import com.querydsl.core.types.Predicate;
import com.querydsl.jpa.JPQLQuery;
/**
* Filters for {@link Accession}.
......@@ -47,7 +48,7 @@ public class AccessionFilter extends UuidModelFilter<AccessionFilter, Accession>
public String _text;
/** The historic. */
public Boolean historic;
public Boolean historic = false;
/** The aegis. */
public Boolean aegis;
......
......@@ -519,7 +519,7 @@ public class GenesysServiceImpl implements GenesysService {
public List<Object[]> statisticsSpeciesByInstituteES(FaoInstitute institute, int size) {
AccessionFilter af = new AccessionFilter();
af.holder().code = Sets.newHashSet(institute.getCode());
af.historic = false;
af.historic(false);
ElasticsearchService.TermResult result = null;
try {
......
......@@ -78,7 +78,7 @@ public class StatisticsServiceImpl implements StatisticsService {
// @Cacheable(value = "statistics", key = "'stats.' + #root.methodName")
public long numberOfActiveAccessions() {
AccessionFilter af = new AccessionFilter();
af.historic = false;
af.historic(false);
return elasticsearchService.count(Accession.class, af);
}
......@@ -86,7 +86,7 @@ public class StatisticsServiceImpl implements StatisticsService {
// @Cacheable(value = "statistics", key = "'stats.' + #root.methodName")
public long numberOfHistoricAccessions() {
AccessionFilter af = new AccessionFilter();
af.historic = true;
af.historic(true);
return elasticsearchService.count(Accession.class, af);
}
......
......@@ -252,6 +252,8 @@ public class WebConfiguration extends WebMvcConfigurerAdapter {
mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
// explicit json views: every fields needs to be annotated, therefore enabled
mapper.enable(MapperFeature.DEFAULT_VIEW_INCLUSION);
// enable upgrading to arrays
mapper.enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY);
return mapper;
}
......
......@@ -23,6 +23,7 @@ import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
import static org.hamcrest.Matchers.startsWith;
import static org.junit.Assert.*;
import static org.springframework.restdocs.mockmvc.MockMvcRestDocumentation.*;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*;
......@@ -268,6 +269,122 @@ public class AccessionControllerTest extends AbstractApiTest {
/*@formatter:on*/
}
@Test
public void historicFilterTest() throws Exception {
Accession accession1 = addAccessionInDB(null);
Accession accession2 = addAccessionInDB(null);
Accession accessionH = addAccessionInDB(null);
accessionH.setHistoric(true);
accessionRepository.save(accession1);
accessionRepository.save(accession2);
accessionRepository.save(accessionH);
AccessionFilter filter = new AccessionFilter();
// filter.aegis = true;
/*@formatter:off*/
// Active accessions
mockMvc
.perform(post(AccessionController.CONTROLLER_URL + "/list")
.contentType(MediaType.APPLICATION_JSON)
.content(verboseMapper.writeValueAsString(filter)))
// .andDo(MockMvcResultHandlers.print())
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andExpect(jsonPath("$.totalElements", is(2)))
.andExpect(jsonPath("$.filter.historic", is(false)))
;
/*@formatter:on*/
filter.NOTNULL = Sets.newHashSet("historic"); // must be cleared
mockMvc
.perform(post(AccessionController.CONTROLLER_URL + "/list")
.contentType(MediaType.APPLICATION_JSON)
.content(verboseMapper.writeValueAsString(filter)))
// .andDo(MockMvcResultHandlers.print())
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andExpect(jsonPath("$.totalElements", is(3)))
.andExpect(jsonPath("$.filter.NOTNULL").isArray())
.andExpect(jsonPath("$.filter.NOTNULL", containsInAnyOrder("historic")))
;
/*@formatter:on*/
filter.NOTNULL = null;
filter.historic(true);
mockMvc
.perform(post(AccessionController.CONTROLLER_URL + "/list")
.contentType(MediaType.APPLICATION_JSON)
.content(verboseMapper.writeValueAsString(filter)))
// .andDo(MockMvcResultHandlers.print())
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andExpect(jsonPath("$.totalElements", is(1)))
.andExpect(jsonPath("$.filter.historic", is(true)))
.andExpect(jsonPath("$.content[0].uuid", is(accessionH.getUuid().toString())));
;
/*@formatter:on*/
}
@Test
public void historicNullFilterTest() throws Exception {
Accession accession1 = addAccessionInDB(null);
Accession accession2 = addAccessionInDB(null);
Accession accessionH = addAccessionInDB(null);
accessionH.setHistoric(true);
accessionRepository.save(accession1);
accessionRepository.save(accession2);
accessionRepository.save(accessionH);
AccessionFilter filter = new AccessionFilter();
filter.isNull().add("historic");
/*@formatter:off*/
mockMvc
.perform(post(AccessionController.CONTROLLER_URL + "/list")
.contentType(MediaType.APPLICATION_JSON)
.content(verboseMapper.writeValueAsString(filter)))
// .andDo(MockMvcResultHandlers.print())
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andExpect(jsonPath("$.totalElements", is(0)))
.andExpect(jsonPath("$.filter.historic", nullValue()))
.andExpect(jsonPath("$.filter.NULL", contains("historic")))
;
/*@formatter:on*/
}
@Test
public void historicFilterIsNullTest() throws Exception {
Accession accession1 = addAccessionInDB(null);
Accession accession2 = addAccessionInDB(null);
Accession accessionH = addAccessionInDB(null);
accessionH.setHistoric(true);
accessionRepository.save(accession1);
accessionRepository.save(accession2);
accessionRepository.save(accessionH);
AccessionFilter filter = new AccessionFilter();
assertNotNull(filter.historic);
assertFalse(filter.historic);
// set historic to NULL
filter.historic = null;
/*@formatter:off*/
mockMvc
.perform(post(AccessionController.CONTROLLER_URL + "/list")
.contentType(MediaType.APPLICATION_JSON)
.content(verboseMapper.writeValueAsString(filter)))
// .andDo(MockMvcResultHandlers.print())
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andExpect(jsonPath("$.filterCode", notNullValue()))
.andExpect(jsonPath("$.totalElements", is(3)))
.andExpect(jsonPath("$.filter.historic", nullValue()))
;
/*@formatter:on*/
}
@Test
public void aegisFilterTest() throws Exception {
Accession withAegis = addAccessionInDB(true);
......
/*
* Copyright 2019 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.test.simpletest;
import static com.google.common.collect.Sets.*;
......@@ -20,6 +35,8 @@ import org.slf4j.LoggerFactory;
import com.jayway.jsonpath.JsonPath;
import com.querydsl.core.types.Predicate;
import net.minidev.json.JSONArray;
public class EsQueryTest {
private static Logger LOG = LoggerFactory.getLogger(EsQueryTest.class);
......@@ -32,7 +49,8 @@ public class EsQueryTest {
af.accessionNumber.sw = "FOO";
af.geo = new AccessionGeoFilter();
af.geo.latitude = new NumberFilter<>();
af.geo.latitude.between = new Double[] { 10d, 20d };
af.geo.latitude.ge = 10d;
af.geo.latitude.le = 20d;
Predicate predicate = af.buildPredicate();
ElasticQueryBuilder visitor = new ElasticQueryBuilder();
predicate.accept(visitor, null);
......@@ -54,10 +72,10 @@ public class EsQueryTest {
QueryBuilder esQuery = visitor.getQuery();
LOG.debug("ES query: {}", visitor.getQuery());
Object storage = JsonPath.read(esQuery.toString(), "bool.filter.terms.storage");
assertThat(storage, notNullValue());
assertThat(((List<?>) storage), hasSize(1));
assertThat(((List<?>) storage), containsInAnyOrder(30));
JSONArray storages = JsonPath.read(esQuery.toString(), "bool.filter[*].terms.storage");
assertThat(storages.get(0), notNullValue());
assertThat(((List<?>) storages.get(0)), hasSize(1));
assertThat(((List<?>) storages.get(0)), containsInAnyOrder(30));
}
@Test
......@@ -70,10 +88,10 @@ public class EsQueryTest {
QueryBuilder esQuery = visitor.getQuery();
LOG.debug("ES query: {}", visitor.getQuery());
Object storage = JsonPath.read(esQuery.toString(), "bool.filter.terms.storage");
assertThat(storage, notNullValue());
assertThat(((List<?>) storage), hasSize(2));
assertThat(((List<?>) storage), containsInAnyOrder(20, 10));
JSONArray storages = JsonPath.read(esQuery.toString(), "bool.filter[*].terms.storage");
assertThat(storages.get(0), notNullValue());
assertThat(((List<?>) storages.get(0)), hasSize(2));
assertThat(((List<?>) storages.get(0)), containsInAnyOrder(20, 10));
}
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment