Commit e5e0f60f authored by Matija Obreza's avatar Matija Obreza

Merge branch 'kpi-charts' into 'master'

Read KPI filtered observations for specified date range

See merge request genesys-pgr/genesys-server!370
parents 25c47d09 5de0f4c3
......@@ -16,16 +16,21 @@
package org.genesys2.server.api.v1;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.genesys2.server.api.Pagination;
import org.genesys2.server.model.kpi.Execution;
import org.genesys2.server.model.kpi.Observation;
import org.genesys2.server.service.KPIService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Sort;
import org.springframework.data.domain.Sort.Direction;
import org.springframework.format.annotation.DateTimeFormat;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.GetMapping;
......@@ -39,6 +44,7 @@ import org.springframework.web.bind.annotation.RestController;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.querydsl.core.Tuple;
import io.swagger.annotations.Api;
......@@ -67,7 +73,7 @@ public class KPIReadController {
return kpiService.listExecutions().stream().map(ex -> ex.getName()).collect(Collectors.toList());
}
@PostMapping(value="/observations/{executionName}")
@PostMapping(value="/observations/{executionName}", params = { "date" })
public ArrayNode observations(@PathVariable final String executionName,
@RequestParam(value="date", required = true) @DateTimeFormat(pattern="yyyyMMdd") final Date date,
@RequestBody(required = false) final Map<String, Set<String>> keys) {
......@@ -80,6 +86,35 @@ public class KPIReadController {
return l;
}
@PostMapping(value="/observations/range/{executionName}")
public Page<Object> observations2(@PathVariable final String executionName, @RequestParam(value = "days", required = false) final Integer days,
@RequestParam(value = "from", required = false) @DateTimeFormat(pattern = "yyyyMMdd") Date from,
@RequestParam(value = "to", required = false) @DateTimeFormat(pattern = "yyyyMMdd") Date to, final Pagination pagination, @RequestBody(required = false) final Map<String, Set<String>> keys) {
Execution execution = kpiService.getExecution(executionName);
if (days != null) {
Calendar startDate = Calendar.getInstance();
if (to != null) {
startDate.setTime(to);
} else {
to = startDate.getTime();
}
startDate.add(Calendar.DAY_OF_MONTH, -days);
from = startDate.getTime();
}
Page<Tuple> observations = kpiService.listObservations(execution, from, to, keys, pagination.toPageRequest(1000, Direction.DESC, "id"));
return observations.map(tuple -> toMap(tuple));
}
private ObjectNode toMap(Tuple observation) {
ObjectNode m = toMap(observation.get(1, Observation.class));
m.put("timestamp", observation.get(0, Date.class).getTime());
return m;
}
private ObjectNode toMap(Observation observation) {
ObjectNode m=objectMapper.createObjectNode();
m.put("value", observation.getValue());
......
......@@ -35,7 +35,9 @@ import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Repository;
import com.querydsl.core.types.OrderSpecifier;
import com.querydsl.core.types.dsl.BooleanExpression;
import com.querydsl.core.types.dsl.PathBuilder;
import com.querydsl.jpa.impl.JPAQuery;
import com.querydsl.jpa.impl.JPAQueryFactory;
......@@ -55,12 +57,22 @@ public class ObservationRepositoryCustomImpl implements ObservationCustomReposit
dks.forEach(dk -> {
filters.and(QObservation.observation.dimensions.any().eq(dk));
});
JPAQuery<Observation> q = jpaQueryFactory.select(QObservation.observation)
// where
.where(filters)
.where(filters);
if (page.getSort().iterator().hasNext()) {
PathBuilder<Observation> entityPath = new PathBuilder<>(Observation.class, "observation");
page.getSort().forEach(order -> {
// order
.orderBy(QObservation.observation.id.asc());
PathBuilder<?> path = entityPath.get(order.getProperty());
q.orderBy(new OrderSpecifier(com.querydsl.core.types.Order.valueOf(order.getDirection().name()), path));
});
} else {
// this was the default
q.orderBy(QObservation.observation.id.asc());
}
long total = q.fetchCount();
......
/*
* Copyright 2018 Global Crop Diversity Trust
* Copyright 2019 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -32,6 +32,8 @@ import org.genesys2.server.model.kpi.Observation;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import com.querydsl.core.Tuple;
public interface KPIService {
KPIParameter save(KPIParameter parameter);
......@@ -85,6 +87,19 @@ public interface KPIService {
Page<Observation> listObservations(Execution execution, long dimensionKeyId, Pageable pageable);
/**
* List selected observations of an execution with the executionRun#timestamp.
*
* @param execution the execution
* @param from the from
* @param to the to
* @param keys the keys
* @param page the page
* @return the page
*/
Page<Tuple> listObservations(Execution execution, Date from, Date to, Map<String, Set<String>> keys, Pageable page);
List<Observation> filterObservations(Execution execution, Date date, Map<String, Set<String>> keys);
/**
......@@ -98,4 +113,5 @@ public interface KPIService {
*/
SortedMap<Date, List<Observation>> calculateRunDiff(Execution execution, Date from, Date to, Map<String, Set<String>> keys);
}
......@@ -54,6 +54,7 @@ import org.genesys2.server.model.kpi.ExecutionRun;
import org.genesys2.server.model.kpi.JpaDimension;
import org.genesys2.server.model.kpi.KPIParameter;
import org.genesys2.server.model.kpi.Observation;
import org.genesys2.server.model.kpi.QObservation;
import org.genesys2.server.persistence.kpi.DimensionKeyRepository;
import org.genesys2.server.persistence.kpi.DimensionRepository;
import org.genesys2.server.persistence.kpi.ExecutionRepository;
......@@ -77,6 +78,9 @@ import org.springframework.transaction.annotation.Transactional;
import org.springframework.validation.annotation.Validated;
import com.google.api.client.util.Lists;
import com.querydsl.core.Tuple;
import com.querydsl.core.types.OrderSpecifier;
import com.querydsl.core.types.dsl.PathBuilder;
import com.querydsl.jpa.impl.JPAQuery;
import com.querydsl.jpa.impl.JPAQueryFactory;
......@@ -577,6 +581,42 @@ public class KPIServiceImpl implements KPIService {
return res;
}
@Override
@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#execution, 'READ')")
public Page<Tuple> listObservations(Execution execution, Date from, Date to, final Map<String, Set<String>> keys, Pageable page) {
final JPAQuery<Tuple> observationQuery = jpaQueryFactory.selectFrom(observation).select(observation.executionRun.timestamp, observation).distinct();
if (!MapUtils.isEmpty(keys)) {
for (String name : keys.keySet()) {
observationQuery.join(observation.dimensions, dimensionKey).on(dimensionKey.name.eq(name).and(dimensionKey.value.in(keys.get(name))));
}
}
observationQuery.where(observation.executionRun.execution.id.eq(execution.getId()));
observationQuery.where(observation.executionRun.timestamp.between(from, to));
long total = observationQuery.fetchCount();
if (page.getSort().iterator().hasNext()) {
PathBuilder<Observation> entityPath = new PathBuilder<>(Observation.class, "observation");
page.getSort().forEach(order -> {
// order
PathBuilder<?> path = entityPath.get(order.getProperty());
observationQuery.orderBy(new OrderSpecifier(com.querydsl.core.types.Order.valueOf(order.getDirection().name()), path));
});
} else {
// default sort by newest first
observationQuery.orderBy(QObservation.observation.id.desc());
}
observationQuery.offset(page.getOffset()).limit(page.getPageSize());
List<Tuple> observations = observationQuery.fetch();
observations.forEach(tuple -> tuple.get(observation).getDimensions().size());
return new PageImpl<>(observations, page, total);
}
@Override
@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#execution, 'READ')")
public List<Observation> filterObservations(final Execution execution, final Date date, final Map<String, Set<String>> keys) {
......
......@@ -58,6 +58,7 @@ import com.google.common.collect.Sets;
/**
* @author Maxym Borodenko
* @author Matija Obreza
*/
public class KPIReadControllerTest extends AbstractApiTest {
......@@ -125,7 +126,9 @@ public class KPIReadControllerTest extends AbstractApiTest {
@Test
@WithMockUser(username = "user", password = "user", roles = "ADMINISTRATOR")
public void testList() throws Exception {
ExecutionRun executionRun = buildAndSave("institute.count");
Execution execution = buildAndSave("institute.count");
ExecutionRun executionRun = kpiService.executeAndSave(execution);
assertThat(executionRun.getExecution().getName(), is("institute.count"));
mockMvc.perform(get(KPIController.CONTROLLER_URL.concat("/observations")))
......@@ -138,7 +141,9 @@ public class KPIReadControllerTest extends AbstractApiTest {
@Test
@WithMockUser(username = "user", password = "user", roles = "ADMINISTRATOR")
public void testFilterObservations() throws Exception {
ExecutionRun executionRun = buildAndSave("institute.count");
Execution execution = buildAndSave("institute.count");
ExecutionRun executionRun = kpiService.executeAndSave(execution);
assertThat(executionRun.getObservations().size(), is(10));
LocalDateTime ldt = LocalDateTime.now().plusMonths(1);
String searchDate = DateTimeFormatter.ofPattern("yyyyMMdd").format(ldt);
......@@ -157,10 +162,67 @@ public class KPIReadControllerTest extends AbstractApiTest {
.andExpect(jsonPath("$", hasSize(1)));
}
@Test
@WithMockUser(username = "user", password = "user", roles = "ADMINISTRATOR")
public void testFilterListObservations() throws Exception {
Execution execution = buildAndSave("institute.count");
kpiService.executeAndSave(execution);
kpiService.executeAndSave(execution);
ExecutionRun executionRun = kpiService.executeAndSave(execution);
assertThat(executionRun.getObservations().size(), is(10));
Map<String, Set<String>> filter = new HashMap<>();
filter.put("allowMaterialRequests", Sets.newHashSet("true"));
mockMvc.perform(post(KPIController.CONTROLLER_URL.concat("/observations/range/{executionName}"), executionRun.getExecution().getName())
.param("days", "20")
.content(verboseMapper.writeValueAsString(filter))
.contentType(MediaType.APPLICATION_JSON_UTF8))
// .andDo(MockMvcResultHandlers.print())
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andExpect(jsonPath("$.number", is(0)))
.andExpect(jsonPath("$.first", is(true)))
.andExpect(jsonPath("$.content", hasSize(15)))
.andExpect(jsonPath("$.content[0].timestamp", notNullValue()));
}
@Test
@WithMockUser(username = "user", password = "user", roles = "ADMINISTRATOR")
public void testFilterListObservationsPaged() throws Exception {
Execution execution = buildAndSave("institute.count");
kpiService.executeAndSave(execution);
kpiService.executeAndSave(execution);
ExecutionRun executionRun = kpiService.executeAndSave(execution);
assertThat(executionRun.getObservations().size(), is(10));
Map<String, Set<String>> filter = new HashMap<>();
filter.put("allowMaterialRequests", Sets.newHashSet("true"));
mockMvc.perform(post(KPIController.CONTROLLER_URL.concat("/observations/range/{executionName}"), executionRun.getExecution().getName())
.param("days", "20")
.param("p", "1")
.param("l", "5")
.content(verboseMapper.writeValueAsString(filter))
.contentType(MediaType.APPLICATION_JSON_UTF8))
// .andDo(MockMvcResultHandlers.print())
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andExpect(jsonPath("$.number", is(1)))
.andExpect(jsonPath("$.first", is(false)))
.andExpect(jsonPath("$.last", is(false)))
.andExpect(jsonPath("$.content", hasSize(5)))
.andExpect(jsonPath("$.content[0].timestamp", notNullValue()));
}
@Test
@WithMockUser(username = "user", password = "user", roles = "ADMINISTRATOR")
public void testEmptyListIfNotFound() throws Exception {
ExecutionRun executionRun = buildAndSave("institute.count");
Execution execution = buildAndSave("institute.count");
ExecutionRun executionRun = kpiService.executeAndSave(execution);
LocalDateTime ldt = LocalDateTime.now().plusMonths(-10);
String searchDate = DateTimeFormatter.ofPattern("yyyyMMdd").format(ldt);
......@@ -175,7 +237,8 @@ public class KPIReadControllerTest extends AbstractApiTest {
@Test
public void testErrorWhenWrongDate() throws Exception {
ExecutionRun executionRun = buildAndSave("institute.count");
Execution execution = buildAndSave("institute.count");
ExecutionRun executionRun = kpiService.executeAndSave(execution);
mockMvc.perform(post(KPIController.CONTROLLER_URL.concat("/observations/{executionName}"), executionRun.getExecution().getName())
.param("date", "wrong-date")
......@@ -185,7 +248,7 @@ public class KPIReadControllerTest extends AbstractApiTest {
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8));
}
protected ExecutionRun buildAndSave(final String executionName) {
protected Execution buildAndSave(final String executionName) {
KPIParameter param1 = kpiService.save(createParameter("institute", FaoInstitute.class, "Institute"));
Execution exec1 = new Execution();
exec1.setName(executionName);
......@@ -203,8 +266,7 @@ public class KPIReadControllerTest extends AbstractApiTest {
Execution loaded = kpiService.save(exec1);
assertThat(loaded.getProperty(), is("accessionCount"));
assertThat(loaded.getType(), is(Execution.ExecutionType.AVERAGE));
return kpiService.executeAndSave(loaded);
return loaded;
}
private BooleanDimension createBoolDimension(String name, String title) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment