Commit f1d2b495 authored by Matija Obreza's avatar Matija Obreza

Merge branch '379-kpi-api-v1-observed-value-on-specific-date' into 'master'

Resolve "KPI API v1: Observed value on specific date"

Closes #379

See merge request genesys-pgr/genesys-server!301
parents 5632c427 8255583b
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys2.server.api.v1;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.genesys2.server.api.Pagination;
import org.genesys2.server.model.kpi.Execution;
import org.genesys2.server.model.kpi.Observation;
import org.genesys2.server.service.KPIService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.format.annotation.DateTimeFormat;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.swagger.annotations.Api;
/**
* @author Maxym Borodenko
*/
@RestController("kpiReadApi1")
@RequestMapping(value = { KPIController.CONTROLLER_URL })
@PreAuthorize("isAuthenticated()")
@Api(tags = { "kpi" })
public class KPIReadController {
@Autowired
private KPIService kpiService;
@Autowired
private ObjectMapper objectMapper;
@GetMapping(value="/observations")
public Page<String> listExecutions(final Pagination page) {
Pageable pagination = page.toPageRequest(100, Sort.Direction.ASC, "name");
Page<Execution> executions = kpiService.listExecutions(pagination);
return new PageImpl<String>(executions.getContent().stream().filter(exec -> exec != null).map(exec -> exec.getName()).collect(Collectors.toList()), pagination, executions.getTotalElements());
}
@PostMapping(value="/observations/{executionName}")
public ArrayNode observations(@PathVariable final String executionName,
@RequestParam(value="date", required = true) @DateTimeFormat(pattern="yyyyMMdd") final Date date,
@RequestBody(required = false) final Map<String, Set<String>> keys) {
ArrayNode l=objectMapper.createArrayNode();
kpiService.filterObservations(executionName, date, keys).forEach(observation -> {
l.add(toMap(observation));
});;
return l;
}
private ObjectNode toMap(Observation observation) {
ObjectNode m=objectMapper.createObjectNode();
m.put("value", observation.getValue());
m.put("stdDev", observation.getStdDev());
ObjectNode dims = m.putObject("dimensions");
observation.getDimensions().forEach(dim -> {
dims.put(dim.getName(), dim.getValue());
});
return m;
}
}
......@@ -71,6 +71,7 @@ public class Observation implements EntityId, Serializable {
@JsonIgnoreProperties({ "id" })
private List<DimensionKey> dimensions = new ArrayList<DimensionKey>();
@JsonIgnore
@Column
private int dimensionCount;
......
......@@ -16,6 +16,7 @@
package org.genesys2.server.service;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Set;
......@@ -79,6 +80,7 @@ public interface KPIService {
Page<Observation> listObservations(Execution execution, long dimensionKeyId, Pageable pageable);
List<Observation> filterObservations(String executionName, Date date, Map<String, Set<String>> keys);
}
......@@ -16,7 +16,12 @@
package org.genesys2.server.service.impl;
import static org.genesys2.server.model.kpi.QDimensionKey.dimensionKey;
import static org.genesys2.server.model.kpi.QExecutionRun.executionRun;
import static org.genesys2.server.model.kpi.QObservation.observation;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
......@@ -31,6 +36,7 @@ import javax.persistence.Query;
import javax.persistence.TypedQuery;
import javax.validation.Valid;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.genesys2.server.model.kpi.Dimension;
......@@ -59,6 +65,9 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.validation.annotation.Validated;
import com.querydsl.jpa.impl.JPAQuery;
import com.querydsl.jpa.impl.JPAQueryFactory;
@Service
@Transactional(readOnly = true)
@Validated
......@@ -86,6 +95,9 @@ public class KPIServiceImpl implements KPIService {
@Autowired
private ExecutionRunRepository executionRunRepository;
@Autowired
private JPAQueryFactory jpaQueryFactory;
@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#parameter, 'ADMINISTRATION')")
@Override
@Transactional
......@@ -514,6 +526,28 @@ public class KPIServiceImpl implements KPIService {
return res;
}
@Override
public List<Observation> filterObservations(final String executionName, final Date date, final Map<String, Set<String>> keys) {
JPAQuery<ExecutionRun> executionQuery = jpaQueryFactory.selectFrom(executionRun);
executionQuery.where(executionRun.execution.name.eq(executionName).and(executionRun.timestamp.before(date)));
executionQuery.orderBy(executionRun.timestamp.desc());
ExecutionRun executionRun = executionQuery.fetchFirst();
if (executionRun == null) {
return Collections.emptyList();
}
JPAQuery<Observation> observationQuery = jpaQueryFactory.selectFrom(observation).distinct();
if (!MapUtils.isEmpty(keys)) {
for (String name : keys.keySet()) {
observationQuery = observationQuery.join(observation.dimensions, dimensionKey).on(dimensionKey.name.eq(name).and(dimensionKey.value.in(keys.get(name))));
}
}
observationQuery.where(observation.executionRun.id.eq(executionRun.getId()));
List<Observation> observations = observationQuery.fetch();
observations.forEach(observation -> observation.getDimensions().size());
return observations;
}
private Set<DimensionKey> getDimensionKeys(Map<String, String> dimensionFilters) {
// TODO needs .equals()?
Set<DimensionKey> dks = new HashSet<DimensionKey>();
......
......@@ -54,7 +54,6 @@ import org.springframework.http.MediaType;
import org.springframework.restdocs.JUnitRestDocumentation;
import org.springframework.security.test.context.support.WithMockUser;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.result.MockMvcResultHandlers;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.context.WebApplicationContext;
......@@ -176,13 +175,13 @@ public class ApiImagesDocsTest extends AbstractApiTest {
LOG.info(objectMapper.writeValueAsString(ri));
sb.append(ri.getUuid());
})
.andDo(MockMvcResultHandlers.print())
// .andDo(MockMvcResultHandlers.print())
.andExpect(jsonPath("$.id", greaterThan(0)))
.andExpect(jsonPath("$.folder", is("/wiews/" + TEST_INSTCODE + "/acn/" + ACCENUMB)))
.andExpect(jsonPath("$.originalFilename", is(image1.getOriginalFilename())))
.andExpect(jsonPath("$.contentType", is("image/jpeg")))
.andDo(MockMvcResultHandlers.print())
// .andDo(MockMvcResultHandlers.print())
.andDo(document("img-instgallery-put",
pathParameters(
parameterWithName("instCode").description("Institute WIEWS code (e.g. NGA039)"),
......@@ -245,7 +244,7 @@ public class ApiImagesDocsTest extends AbstractApiTest {
.andDo(r -> {
LOG.info("Image metadata put: {}", r.getResponse().getContentAsString());
})
.andDo(MockMvcResultHandlers.print())
// .andDo(MockMvcResultHandlers.print())
.andDo(document("img-instgallery-metadata-put",
pathParameters(
parameterWithName("instCode").description("Institute WIEWS code (e.g. NGA039)"),
......
package org.genesys.test.server.api.v1;
import static org.hamcrest.Matchers.*;
import static org.junit.Assert.assertThat;
import static org.springframework.restdocs.mockmvc.RestDocumentationRequestBuilders.*;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.RandomUtils;
import org.genesys.test.base.AbstractApiTest;
import org.genesys2.server.api.v1.KPIController;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.model.kpi.BooleanDimension;
import org.genesys2.server.model.kpi.Execution;
import org.genesys2.server.model.kpi.ExecutionRun;
import org.genesys2.server.model.kpi.JpaDimension;
import org.genesys2.server.model.kpi.KPIParameter;
import org.genesys2.server.persistence.FaoInstituteRepository;
import org.genesys2.server.persistence.kpi.DimensionRepository;
import org.genesys2.server.persistence.kpi.ExecutionRepository;
import org.genesys2.server.persistence.kpi.ExecutionRunRepository;
import org.genesys2.server.persistence.kpi.KPIParameterRepository;
import org.genesys2.server.persistence.kpi.ObservationRepository;
import org.genesys2.server.service.KPIService;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
import org.springframework.transaction.annotation.Transactional;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Sets;
/**
* @author Maxym Borodenko
*/
public class KPIReadControllerTest extends AbstractApiTest {
@Autowired
private KPIService kpiService;
@Autowired
private KPIParameterRepository paramRepository;
@Autowired
private DimensionRepository dimRepository;
@Autowired
private ExecutionRepository execRepository;
@Autowired
private ExecutionRunRepository execRunRepository;
@Autowired
private ObservationRepository obsRepository;
@Autowired
private FaoInstituteRepository instituteRepository;
protected static final ObjectMapper objectMapper;
protected static final ObjectMapper verboseMapper = new ObjectMapper();
static {
objectMapper = new ObjectMapper();
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
objectMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true);
objectMapper.setSerializationInclusion(JsonInclude.Include.NON_EMPTY);
}
@Override
@Transactional
public void beforeTest() throws Exception {
super.beforeTest();
List<FaoInstitute> institutes = new ArrayList<>();
for (int i=0; i<10; i++) {
FaoInstitute institute=new FaoInstitute();
institute.setAccessionCount(100+i);
institute.setCode("INS"+i);
institute.setCurrent(true);
institute.setAllowMaterialRequests(i % 2 == 1);
institute.setElevation(RandomUtils.nextDouble(0, 2000));
// Replaced by code
institute.setvCode(i > 1 && RandomUtils.nextBoolean() ? null : "INS" + (i - 2));
// Generate type
institute.setType("TYPE" + i % 3);
institutes.add(instituteRepository.save(institute));
}
assertThat(instituteRepository.count(), is(10l));
}
@Override
@Transactional
public void cleanup() throws Exception {
execRepository.deleteAll();
paramRepository.deleteAll();
dimRepository.deleteAll();
obsRepository.deleteAll();
execRunRepository.deleteAll();
instituteRepository.deleteAll();
super.cleanup();
}
@Test
public void testList() throws Exception {
ExecutionRun executionRun = buildAndSave("institute.count");
assertThat(executionRun.getExecution().getName(), is("institute.count"));
mockMvc.perform(get(KPIController.CONTROLLER_URL.concat("/observations")))
// .andDo(MockMvcResultHandlers.print())
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andExpect(jsonPath("$.content", hasSize(1)));
}
@Test
public void testFilterObservations() throws Exception {
ExecutionRun executionRun = buildAndSave("institute.count");
assertThat(executionRun.getObservations().size(), is(10));
LocalDateTime ldt = LocalDateTime.now().plusMonths(1);
String searchDate = DateTimeFormatter.ofPattern("yyyyMMdd").format(ldt);
Map<String, Set<String>> filter = new HashMap<>();
filter.put("allowMaterialRequests", Sets.newHashSet("true"));
filter.put("code", Sets.newHashSet("INS1"));
mockMvc.perform(post(KPIController.CONTROLLER_URL.concat("/observations/{executionName}"), executionRun.getExecution().getName())
.param("date", searchDate)
.content(verboseMapper.writeValueAsString(filter))
.contentType(MediaType.APPLICATION_JSON_UTF8))
// .andDo(MockMvcResultHandlers.print())
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andExpect(jsonPath("$", hasSize(1)));
}
@Test
public void testEmptyListIfNotFound() throws Exception {
ExecutionRun executionRun = buildAndSave("institute.count");
LocalDateTime ldt = LocalDateTime.now().plusMonths(-10);
String searchDate = DateTimeFormatter.ofPattern("yyyyMMdd").format(ldt);
mockMvc.perform(post(KPIController.CONTROLLER_URL.concat("/observations/{executionName}"), executionRun.getExecution().getName())
.param("date", searchDate)
.contentType(MediaType.APPLICATION_JSON_UTF8))
// .andDo(MockMvcResultHandlers.print())
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andExpect(jsonPath("$", hasSize(0)));
}
@Test
public void testErrorWhenWrongDate() throws Exception {
ExecutionRun executionRun = buildAndSave("institute.count");
mockMvc.perform(post(KPIController.CONTROLLER_URL.concat("/observations/{executionName}"), executionRun.getExecution().getName())
.param("date", "wrong-date")
.contentType(MediaType.APPLICATION_JSON_UTF8))
// .andDo(MockMvcResultHandlers.print())
.andExpect(status().isBadRequest())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8));
}
protected ExecutionRun buildAndSave(final String executionName) {
KPIParameter param1 = kpiService.save(createParameter("institute", FaoInstitute.class, "Institute"));
Execution exec1 = new Execution();
exec1.setName(executionName);
exec1.setTitle("Institute count");
exec1.setParameter(param1);
exec1.setType(Execution.ExecutionType.AVERAGE);
exec1.setProperty("accessionCount");
JpaDimension dimInst = kpiService.save(createJpaDimension("institute.code", "Institute codes", FaoInstitute.class, "code"));
exec1.addDimension(dimInst, null, "code");
BooleanDimension dimBool = kpiService.save(createBoolDimension("yesno", "Boolean yes and no dimension"));
exec1.addDimension(dimBool, null, "allowMaterialRequests");
Execution loaded = kpiService.save(exec1);
assertThat(loaded.getProperty(), is("accessionCount"));
assertThat(loaded.getType(), is(Execution.ExecutionType.AVERAGE));
return kpiService.executeAndSave(loaded);
}
private BooleanDimension createBoolDimension(String name, String title) {
BooleanDimension dim = new BooleanDimension();
dim.setName(name);
dim.setTitle(title);
return dim;
}
private KPIParameter createParameter(String name, Class<?> entity, String title) {
KPIParameter param1 = new KPIParameter();
param1.setName(name);
param1.setEntity(entity.getSimpleName());
param1.setTitle(title);
return param1;
}
private JpaDimension createJpaDimension(String name, String title, Class<?> clazz, String property) {
JpaDimension dim = new JpaDimension();
dim.setName(name);
dim.setTitle(title);
dim.setEntity(clazz.getSimpleName());
dim.setField(property);
return dim;
}
}
......@@ -16,10 +16,16 @@
package org.genesys.test.server.services;
import static org.hamcrest.Matchers.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import java.util.Calendar;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
......@@ -58,7 +64,9 @@ import com.google.common.collect.Sets;
@WithMockUser(username = "admin", password = "admin", roles = "ADMINISTRATOR")
public class KPIServiceTest extends AbstractServicesTest {
private static final String EXECUTION_NAME = "institute.count";
@Autowired
private KPIService kpiService;
......@@ -103,7 +111,7 @@ public class KPIServiceTest extends AbstractServicesTest {
institutes.add(instituteRepository.save(institute));
}
assertThat(instituteRepository.count(), is(10l));
{
Organization org = new Organization();
org.setSlug("ORG1");
......@@ -353,6 +361,58 @@ public class KPIServiceTest extends AbstractServicesTest {
assertThat(obsRepository.count(), is(0l));
}
@Test
public void testFilterObservations() {
KPIParameter param1 = kpiService.save(createParameter("institute", FaoInstitute.class, "Institute"));
Execution exec1 = new Execution();
exec1.setName("institute.count");
exec1.setTitle("Institute count");
exec1.setParameter(param1);
exec1.setType(ExecutionType.AVERAGE);
exec1.setProperty("accessionCount");
JpaDimension dimInst = kpiService.save(createJpaDimension("institute.code", "Institute codes", FaoInstitute.class, "code"));
exec1.addDimension(dimInst, null, "code");
BooleanDimension dimBool = kpiService.save(createBoolDimension("yesno", "Boolean yes and no dimension"));
exec1.addDimension(dimBool, null, "allowMaterialRequests");
Execution loaded = kpiService.save(exec1);
assertThat(loaded.getProperty(), is("accessionCount"));
assertThat(loaded.getType(), is(Execution.ExecutionType.AVERAGE));
ExecutionRun run1 = kpiService.executeAndSave(loaded);
final Calendar dateOfRun1 = Calendar.getInstance();
//substract 1 month from current date
dateOfRun1.add(Calendar.MONTH, -1);
run1.setTimestamp(dateOfRun1.getTime());
run1 = execRunRepository.save(run1);
ExecutionRun run2 = kpiService.executeAndSave(loaded);
ExecutionRun run3 = kpiService.executeAndSave(loaded);
final Calendar dateOfRun3 = Calendar.getInstance();
//substract 1 month from current date
dateOfRun3.add(Calendar.MONTH, -2);
run3.setTimestamp(dateOfRun3.getTime());
run3 = execRunRepository.save(run3);
assertEquals(execRunRepository.findAll().size(), 3);
final Calendar searchDate = Calendar.getInstance();
//substract 1 month from current date
searchDate.add(Calendar.MONTH, -1);
searchDate.add(Calendar.DAY_OF_MONTH, 1);
Map<String, Set<String>> map = new HashMap<>();
map.put("allowMaterialRequests", Sets.newHashSet("true"));
map.put("code", Sets.newHashSet("INS1"));
List<Observation> observationList = kpiService.filterObservations(EXECUTION_NAME, searchDate.getTime(), map);
assertThat(observationList.size(), is(1));
assertTrue(run1.getObservations().stream().anyMatch(obs -> obs.getId().equals(observationList.get(0).getId())));
assertFalse(run2.getObservations().stream().anyMatch(obs -> obs.getId().equals(observationList.get(0).getId())));
assertFalse(run3.getObservations().stream().anyMatch(obs -> obs.getId().equals(observationList.get(0).getId())));
}
@Test
public void testExecutionAverageByBooleanAndJpa() {
......@@ -366,14 +426,14 @@ public class KPIServiceTest extends AbstractServicesTest {
JpaDimension dimInst = kpiService.save(createJpaDimension("institute.code", "Institute codes", FaoInstitute.class, "code"));
exec1.addDimension(dimInst, null, "code");
BooleanDimension dimBool = kpiService.save(createBoolDimension("yesno", "Boolean yes and no dimension"));
exec1.addDimension(dimBool, null, "allowMaterialRequests");
Execution loaded = kpiService.save(exec1);
assertThat(loaded.getProperty(), is("accessionCount"));
assertThat(loaded.getType(), is(Execution.ExecutionType.AVERAGE));
ExecutionRun run = kpiService.executeAndSave(loaded);
List<Observation> observations = run.getObservations();
assertThat(observations, notNullValue());
......@@ -388,7 +448,7 @@ public class KPIServiceTest extends AbstractServicesTest {
Page<Observation> obs = kpiService.listObservations(run, Maps.newHashMap(), new PageRequest(0, 20));
assertThat(obs.getTotalElements(), is(10l));
assertThat(obs.getContent(), hasSize(10));
kpiService.deleteObservations(run);
kpiService.delete(loaded);
assertThat(execRepository.count(), is(0l));
......@@ -414,7 +474,7 @@ public class KPIServiceTest extends AbstractServicesTest {
assertThat(loaded.getProperty(), is("slug"));
assertThat(loaded.getLink(), is("networks"));
assertThat(loaded.getType(), is(Execution.ExecutionType.COUNT));
List<Observation> results = kpiService.execute(loaded);
assertThat(results, notNullValue());
// results.forEach(r -> {
......@@ -422,7 +482,7 @@ public class KPIServiceTest extends AbstractServicesTest {
// });
assertThat(results, hasSize(1));
assertThat(results.stream().collect(Collectors.summingDouble(Observation::getValue)), is(2d)); // distinct slugs of organizations of all institutes
kpiService.delete(loaded);
assertThat(execRepository.count(), is(0l));
}
......@@ -447,7 +507,7 @@ public class KPIServiceTest extends AbstractServicesTest {
assertThat(loaded.getProperty(), is("slug"));
assertThat(loaded.getLink(), is("networks"));
assertThat(loaded.getType(), is(Execution.ExecutionType.COUNT));