Commit 97b9a2c0 authored by Matija Obreza's avatar Matija Obreza Committed by Maxym Borodenko

Images: Solving slow response from server using offset

- Introduced FilteredSlice, OffsetRequest, ScrollPagination;
- Added new API endpoint to fetch accession images with offset;
- Added Unit test;
parent 0c2ea604
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys2.server.api;
import java.io.Serializable;
import java.util.List;
import org.genesys.blocks.model.filters.EmptyModelFilter;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Sort;
import com.fasterxml.jackson.annotation.JsonInclude;
/**
* The Class FilteredSlice.
*
* @param <T> the generic type
*/
public class FilteredSlice<T> implements Serializable {
private static final long serialVersionUID = 6965069448240229428L;
/** The data is serialized on the base JSON level. */
public List<T> content;
/** The filter. */
public EmptyModelFilter<?, ?> filter;
/** The filter code. */
@JsonInclude(JsonInclude.Include.NON_EMPTY)
public String filterCode;
public Sort sort;
public long totalElements;
public long offset;
/**
* Instantiates a new filtered page.
*
* @param filter the filter
* @param data the data
*/
public FilteredSlice(final EmptyModelFilter<?, ?> filter, final Page<T> data, final Long offset) {
this.filter = filter;
this.content = data.getContent();
this.sort = data.getSort();
this.totalElements = data.getTotalElements();
this.offset = offset == null ? 0 : offset;
}
/**
* Instantiates a new filtered page.
*
* @param filterCode the filter code
* @param filter the filter
* @param data the data
*/
public FilteredSlice(final String filterCode, final EmptyModelFilter<?, ?> filter, final Page<T> data, final Long offset) {
this.filterCode = filterCode;
this.filter = filter;
this.content = data.getContent();
this.sort = data.getSort();
this.totalElements = data.getTotalElements();
this.offset = offset == null ? 0 : offset;
}
}
/*
* Copyright 2019 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys2.server.api;
import java.io.Serializable;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import com.fasterxml.jackson.annotation.JsonIgnore;
/**
* OffsetRequest
*/
public class OffsetRequest implements Pageable, Serializable {
/** The Constant serialVersionUID. */
private static final long serialVersionUID = 3521785103051707272L;
/** The offset. */
private long offset;
/** The page size. */
private int pageSize;
/** The sort. */
private Sort sort;
/**
* Of.
*
* @param offset the offset
* @param pageSize the page size
* @param sort the sort
* @return the offset request
*/
public static OffsetRequest of(long offset, int pageSize, Sort sort) {
OffsetRequest or = new OffsetRequest();
or.offset = offset;
or.pageSize = pageSize;
or.sort = sort;
return or;
}
/**
* Of.
*
* @param offset the offset
* @param pageSize the page size
* @param direction the direction
* @param properties the properties
* @return the offset request
*/
public static OffsetRequest of(long offset, int pageSize, Sort.Direction direction, String... properties) {
return of(offset, pageSize, Sort.by(direction, properties));
}
/**
* Gets the page number.
*
* @return the page number
*/
@Override
@JsonIgnore
public int getPageNumber() {
return offset == 0 ? 0 : 1;
}
/**
* Gets the page size.
*
* @return the page size
*/
@Override
public int getPageSize() {
return pageSize;
}
/**
* Gets the offset.
*
* @return the offset
*/
@Override
public long getOffset() {
return offset;
}
/**
* Gets the sort.
*
* @return the sort
*/
@Override
public Sort getSort() {
return sort;
}
/**
* Next.
*
* @return the pageable
*/
@Override
@JsonIgnore
public Pageable next() {
return OffsetRequest.of(this.offset + pageSize, pageSize, sort);
}
/**
* Previous or first.
*
* @return the pageable
*/
@Override
@JsonIgnore
public Pageable previousOrFirst() {
return OffsetRequest.of(offset == 0 ? 0 : offset - pageSize, pageSize, sort);
}
/**
* First.
*
* @return the pageable
*/
@Override
@JsonIgnore
public Pageable first() {
return OffsetRequest.of(0, pageSize, sort);
}
/**
* Checks for previous.
*
* @return true, if successful
*/
@Override
@JsonIgnore
public boolean hasPrevious() {
return offset > 0;
}
}
/*
* Copyright 2019 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys2.server.api;
import java.util.Arrays;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.data.domain.Sort.Direction;
/**
* Data pagination request.
*
* @author Matija Obreza
*/
public class ScrollPagination {
/** The default sort properties. */
private final String[] DEFAULT_SORT_PROPERTIES = { "id" };
/** Offset (0-based). */
private Long o;
/** Page size (length). */
private Integer l;
/** Sort direction. */
private Sort.Direction d;
/** Sort properties. */
private String[] s;
/**
* Gets the o.
*
* @return the o
*/
public Long getO() {
return o;
}
/**
* Sets the o.
*
* @param o the new o
*/
public void setO(Long o) {
this.o = o;
}
/**
* Gets the l.
*
* @return the l
*/
public int getL() {
return l;
}
/**
* Sets the l.
*
* @param l the new l
*/
public void setL(int l) {
this.l = l;
}
/**
* Gets the d.
*
* @return the d
*/
public Sort.Direction getD() {
return d;
}
/**
* Sets the d.
*
* @param d the new d
*/
public void setD(Sort.Direction d) {
this.d = d;
}
/**
* Gets the s.
*
* @return the s
*/
public String[] getS() {
return s;
}
/**
* Sets the s.
*
* @param s the new s
*/
public void setS(String[] s) {
this.s = s;
}
/**
* To string.
*
* @return the string
*/
/*
* (non-Javadoc)
* @see java.lang.Object#toString()
*/
@Override
public String toString() {
return "Pagination offset=" + o + ", pageSize=" + l + ", dir=" + d + ", sort=" + Arrays.toString(s);
}
/**
* Get sort direction or {@link Sort.Direction#ASC} if null.
*
* @param defaultDir the default dir
* @return sort direction or {@link Sort.Direction#ASC}
*/
private Direction getDirection(Direction defaultDir) {
return d == null ? defaultDir : d;
}
/**
* Gets list of sort properties or provided defaults.
*
* @param defaultSortProps the default sort props
* @return provided properties or defaultSortProps
*/
private String[] getSortProperties(String[] defaultSortProps) {
return s == null || s.length == 0 ? defaultSortProps : s;
}
/**
* To page request using the {@link #DEFAULT_SORT_PROPERTIES} and ASC sort.
*
* @param maxPageSize the max page size
* @return the pageable
*/
public Pageable toPageRequest(int maxPageSize) {
return OffsetRequest.of(o == null ? 0 : o, Integer.min(l == null ? maxPageSize : l, maxPageSize), getDirection(Sort.Direction.ASC), getSortProperties(
DEFAULT_SORT_PROPERTIES));
}
/**
* To page request.
*
* @param maxPageSize the max page size
* @param defaultDir the default dir
* @param defaultSort the default sort
* @return the pageable
*/
public Pageable toPageRequest(int maxPageSize, Direction defaultDir, String... defaultSort) {
return OffsetRequest.of(o == null ? 0 : o, Integer.min(l == null ? maxPageSize : l, maxPageSize), getDirection(defaultDir), getSortProperties(defaultSort));
}
/**
* To page request.
*
* @param maxPageSize the max page size
* @param sort the sort
* @return the pageable
*/
public static Pageable toPageRequest(int maxPageSize, Sort sort) {
return OffsetRequest.of(0, maxPageSize, sort);
}
}
......@@ -38,7 +38,10 @@ import org.genesys.catalog.service.ShortFilterService;
import org.genesys.catalog.service.ShortFilterService.FilterInfo;
import org.genesys2.server.api.ApiBaseController;
import org.genesys2.server.api.FilteredPage;
import org.genesys2.server.api.FilteredSlice;
import org.genesys2.server.api.OffsetRequest;
import org.genesys2.server.api.Pagination;
import org.genesys2.server.api.ScrollPagination;
import org.genesys2.server.api.model.AccessionHeaderJson;
import org.genesys2.server.exception.InvalidApiUsageException;
import org.genesys2.server.model.genesys.Accession;
......@@ -50,6 +53,7 @@ import org.genesys2.server.service.AccessionService;
import org.genesys2.server.service.AccessionService.AccessionMapInfo;
import org.genesys2.server.service.AccessionService.AccessionOverview;
import org.genesys2.server.service.AccessionService.AccessionSuggestionPage;
import org.genesys2.server.service.AccessionService.AccessionSuggestionSlice;
import org.genesys2.server.service.DownloadService;
import org.genesys2.server.service.ElasticsearchService;
import org.genesys2.server.service.ElasticsearchService.TermResult;
......@@ -227,7 +231,7 @@ public class AccessionController {
* @throws SearchException
*/
@JsonView({ JsonViews.Root.class }) // same as getAccessionDetails so we get imageGallery!
@PostMapping(value = "/images", produces = { MediaType.APPLICATION_JSON_VALUE })
@PostMapping(value = "/images", params = { "p" }, produces = { MediaType.APPLICATION_JSON_VALUE })
public AccessionSuggestionPage<AccessionService.AccessionDetails> images(@RequestParam(name = "f", required = false) final String filterCode, final Pagination page,
@RequestBody(required = false) final AccessionFilter filter) throws IOException, SearchException {
......@@ -241,6 +245,21 @@ public class AccessionController {
return new AccessionSuggestionPage<>(pageRes, suggestionRes);
}
@JsonView({ JsonViews.Root.class })
@PostMapping(value = "/images", params = { "o" }, produces = { MediaType.APPLICATION_JSON_VALUE })
public AccessionSuggestionSlice<AccessionService.AccessionDetails> images(@RequestParam(name = "f", required = false) final String filterCode,
final ScrollPagination page, @RequestBody(required = false) final AccessionFilter filter) throws IOException, SearchException {
FilterInfo<AccessionFilter> filterInfo = shortFilterService.processFilter(filterCode, filter, AccessionFilter.class);
FilteredSlice<AccessionService.AccessionDetails> pageRes = new FilteredSlice<>(filterInfo.filterCode, filterInfo.filter, accessionService.withImages(filterInfo.filter, page.toPageRequest(20, Sort.Direction.ASC, "seqNo")), page.getO());
filterInfo.filter.images = true;
Map<String, TermResult> suggestionRes = accessionService.getSuggestions(filterInfo.filter);
return new AccessionSuggestionSlice<>(pageRes, suggestionRes);
}
/**
* List accessions by filterCode or filter
......
......@@ -28,6 +28,7 @@ import org.genesys.filerepository.model.ImageGallery;
import org.genesys.filerepository.model.RepositoryFile;
import org.genesys.filerepository.model.RepositoryFolder;
import org.genesys2.server.api.FilteredPage;
import org.genesys2.server.api.FilteredSlice;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.PDCI;
import org.genesys2.server.model.genesys.Taxonomy2;
......@@ -283,6 +284,17 @@ public interface AccessionService {
}
}
public static class AccessionSuggestionSlice<T> {
@JsonUnwrapped
public FilteredSlice<T> slice;
public Map<String, ElasticsearchService.TermResult> suggestions;
public AccessionSuggestionSlice(FilteredSlice<T> slice, Map<String, ElasticsearchService.TermResult> suggestions) {
this.slice = slice;
this.suggestions = suggestions;
}
}
/**
* List taxonomic records for filtered accession.
*
......
......@@ -26,6 +26,8 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Lists;
import org.apache.commons.lang3.RandomUtils;
import org.genesys.catalog.service.PartnerService;
import org.genesys.filerepository.persistence.ImageGalleryPersistence;
import org.genesys.filerepository.service.RepositoryService;
import org.genesys.test.base.AbstractApiTest;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.AccessionGeo;
......@@ -102,6 +104,10 @@ public abstract class AbstractAccessionControllerTest extends AbstractApiTest {
protected DiversityTreeRepository treeRepository;
@Autowired
protected DiversityTreeAccessionRefRepository diversityTreeAccessionRefRepository;
@Autowired
protected RepositoryService repositoryService;
@Autowired
protected ImageGalleryPersistence imageGalleryPersistence;
protected FaoInstitute institute;
protected AtomicInteger acceNumb = new AtomicInteger(1);
......@@ -111,6 +117,8 @@ public abstract class AbstractAccessionControllerTest extends AbstractApiTest {
@Transactional
public void cleanup() throws Exception {
treeRepository.deleteAll();
imageGalleryPersistence.deleteAll();
repositoryFileRepository.deleteAll();
accessionHistoricRepository.deleteAll();
accessionRepository.deleteAll();
accessionIdRepository.deleteAll();
......
......@@ -30,6 +30,8 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilder
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*;
import java.io.StringReader;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.Set;
import java.util.UUID;
......@@ -52,6 +54,7 @@ import org.genesys2.spring.CSVMessageConverter;
import org.hamcrest.Matchers;
import org.junit.Test;
import org.springframework.http.MediaType;
import org.springframework.security.test.context.support.WithMockUser;
import org.springframework.test.web.servlet.MvcResult;
import org.springframework.test.web.servlet.ResultActions;
......@@ -1487,7 +1490,128 @@ public class AccessionControllerTest extends AbstractAccessionControllerTest {
;
/*@formatter:on*/
}
@Test
@WithMockUser(username = "user", password = "user", roles = "ADMINISTRATOR")
public void listAccessionImagesWithOffsetTest() throws Exception {
List<Accession> accessions = accessionRepository.saveAll(Sets.newHashSet(setUpAccession(institute), setUpAccession(institute), setUpAccession(institute)));
assertThat(accessionRepository.count(), is(3L));
// Add 10 images for each accession
addDummyImagesToAccessions(accessions, 10);
assertThat(repositoryFileRepository.count(), is(30L));
AccessionFilter af = new AccessionFilter();
/*@formatter:off*/
mockMvc
.perform(post(AccessionController.CONTROLLER_URL + "/images?l=1")
.contentType(MediaType.APPLICATION_JSON)
.param("o", "")
.content(objectMapper.writeValueAsString(af))
)
// .andDo(org.springframework.test.web.servlet.result.MockMvcResultHandlers.print())
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE))
.andExpect(jsonPath("$", not(nullValue())))
.andExpect(jsonPath("$.content").isArray())
.andExpect(jsonPath("$.content", hasSize(1)))
.andExpect(jsonPath("$.content[0].details").isMap())
.andExpect(jsonPath("$.content[0].imageGallery").isMap())
.andExpect(jsonPath("$.content[0].imageGallery.images").isArray())
.andExpect(jsonPath("$.content[0].imageGallery.images", hasSize(10)))
.andExpect(jsonPath("$.suggestions").isMap())
.andExpect(jsonPath("$.filter").exists())
.andExpect(jsonPath("$.filterCode").exists())
// pageable
.andExpect(jsonPath("$.sort").isArray())
.andExpect(jsonPath("$.totalElements", is(3)))
.andExpect(jsonPath("$.offset", is(0))) // must be 0 by default
.andExpect(jsonPath("$.size").doesNotHaveJsonPath())
.andExpect(jsonPath("$.numberOfElements").doesNotHaveJsonPath())
.andExpect(jsonPath("$.number").doesNotHaveJsonPath())
.andExpect(jsonPath("$.first").doesNotHaveJsonPath())
.andExpect(jsonPath("$.last").doesNotHaveJsonPath())
.andExpect(jsonPath("$.totalPages").doesNotHaveJsonPath())
;
mockMvc
.perform(post(AccessionController.CONTROLLER_URL + "/images?o=1&l=1")
.contentType(MediaType.APPLICATION_JSON)
.param("o", "")
.content(objectMapper.writeValueAsString(af))
)
// .andDo(org.springframework.test.web.servlet.result.MockMvcResultHandlers.print())
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE))
.andExpect(jsonPath("$", not(nullValue())))
.andExpect(jsonPath("$.content").isArray())
.andExpect(jsonPath("$.content", hasSize(1)))
.andExpect(jsonPath("$.content[0].details").isMap())
.andExpect(jsonPath("$.content[0].imageGallery").isMap())
.andExpect(jsonPath("$.content[0].imageGallery.images").isArray())
.andExpect(jsonPath("$.content[0].imageGallery.images", hasSize(10)))
.andExpect(jsonPath("$.suggestions").isMap())
.andExpect(jsonPath("$.filter").exists())
.andExpect(jsonPath("$.filterCode").exists())
// pageable
.andExpect(jsonPath("$.sort").isArray())
.andExpect(jsonPath("$.totalElements", is(3)))
.andExpect(jsonPath("$.offset", is(1)))
.andExpect(jsonPath("$.size").doesNotHaveJsonPath())
.andExpect(jsonPath("$.numberOfElements").doesNotHaveJsonPath())
.andExpect(jsonPath("$.number").doesNotHaveJsonPath())