Commit 64a8e983 authored by Matija Obreza's avatar Matija Obreza

Merge branch '364-kpi-execution' into 'master'

Resolve "KPI Execution"

Closes #364

See merge request genesys-pgr/genesys-server!265
parents 8e7c3d51 abdfca5e
......@@ -80,6 +80,7 @@
<querydsl.version>4.1.4</querydsl.version>
<hibernate.version>4.3.11.Final</hibernate.version>
<hibernate.validator.version>4.3.2.Final</hibernate.validator.version>
<hsqldb.version>2.3.6</hsqldb.version>
<ehcache.version>2.7.4</ehcache.version>
......@@ -608,6 +609,21 @@
<artifactId>jna</artifactId>
<version>4.5.2</version>
</dependency>
<dependency>
<groupId>javax.validation</groupId>
<artifactId>validation-api</artifactId>
<version>1.1.0.Final</version>
</dependency>
<dependency>
<groupId>org.hibernate</groupId>
<artifactId>hibernate-validator</artifactId>
<version>${hibernate.validator.version}</version>
</dependency>
<dependency>
<groupId>org.hibernate</groupId>
<artifactId>hibernate-validator-annotation-processor</artifactId>
<version>${hibernate.validator.version}</version>
</dependency>
</dependencies>
<build>
......
......@@ -25,6 +25,7 @@ import javax.xml.bind.ValidationException;
import org.genesys2.server.api.ApiBaseController;
import org.genesys2.server.api.ModelValidationException;
import org.genesys2.server.api.Pagination;
import org.genesys2.server.api.model.ExecutionDimensionJson;
import org.genesys2.server.api.model.ExecutionJson;
import org.genesys2.server.exception.AuthorizationException;
......@@ -32,10 +33,12 @@ import org.genesys2.server.model.kpi.BooleanDimension;
import org.genesys2.server.model.kpi.Dimension;
import org.genesys2.server.model.kpi.Execution;
import org.genesys2.server.model.kpi.ExecutionDimension;
import org.genesys2.server.model.kpi.ExecutionRun;
import org.genesys2.server.model.kpi.KPIParameter;
import org.genesys2.server.model.kpi.Observation;
import org.genesys2.server.service.KPIService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.http.MediaType;
import org.springframework.security.access.prepost.PreAuthorize;
......@@ -70,10 +73,10 @@ public class KPIController extends ApiBaseController {
* @throws AuthorizationException
*/
@RequestMapping(value = "/parameter/list", method = RequestMethod.GET, produces = { MediaType.APPLICATION_JSON_VALUE })
public @ResponseBody Map<String, String> listParameters() {
public @ResponseBody Map<String, String> listParameters(final Pagination page) {
LOG.info("Listing KPI parameters");
HashMap<String, String> m = new HashMap<>();
for (KPIParameter kpip : kpiService.listParameters()) {
for (KPIParameter kpip : kpiService.listParameters(page.toPageRequest(100))) {
m.put(kpip.getName(), kpip.getTitle() + "\n" + kpip.getDescription());
}
return m;
......@@ -127,10 +130,10 @@ public class KPIController extends ApiBaseController {
* @throws AuthorizationException
*/
@RequestMapping(value = "/dimension/list", method = RequestMethod.GET, produces = { MediaType.APPLICATION_JSON_VALUE })
public @ResponseBody HashMap<Long, String> listDimensions() {
public @ResponseBody HashMap<Long, String> listDimensions(final Pagination page) {
LOG.info("Listing KPI dimensions");
HashMap<Long, String> m = new HashMap<>();
for (Dimension<?> dim : kpiService.listDimensions()) {
for (Dimension<?> dim : kpiService.listDimensions(page.toPageRequest(100))) {
m.put(dim.getId(), dim.getName() + " " + dim.getTitle());
}
return m;
......@@ -187,7 +190,7 @@ public class KPIController extends ApiBaseController {
public @ResponseBody HashMap<String, String> listExecution() {
LOG.info("Listing KPI executions");
HashMap<String, String> m = new HashMap<>();
for (Execution exec : kpiService.listExecutions()) {
for (Execution exec : kpiService.listExecutions(new PageRequest(0, 100))) {
m.put(exec.getName(), exec.getTitle());
}
return m;
......@@ -225,7 +228,7 @@ public class KPIController extends ApiBaseController {
* @throws ValidationException
*/
@RequestMapping(value = "/observation/{executionName}/", method = { RequestMethod.POST }, produces = { MediaType.APPLICATION_JSON_VALUE })
public @ResponseBody List<Observation> listObservations(@PathVariable("executionName") String executionName,
public @ResponseBody Page<Observation> listObservations(@PathVariable("executionName") String executionName,
@RequestParam(value = "page", required = false, defaultValue = "1") int page, @RequestBody(required = false) Map<String, String> dimensionFilters) {
return kpiService.listObservations(kpiService.findLastExecutionRun(kpiService.getExecution(executionName)), dimensionFilters, new PageRequest(page - 1,
50));
......@@ -243,11 +246,9 @@ public class KPIController extends ApiBaseController {
public @ResponseBody List<Observation> execute(@PathVariable("executionName") String executionName) {
Execution execution = kpiService.getExecution(executionName);
LOG.info("Running execute on : {}", executionName);
List<Observation> res = kpiService.execute(execution);
LOG.info("Saving results: count={}", res.size());
List<Observation> x = kpiService.save(execution, res);
ExecutionRun run = kpiService.executeAndSave(execution);
LOG.info("Done saving results.");
return x;
return run.getObservations();
}
/**
......
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys2.server.api.v1;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.genesys.blocks.model.JsonViews;
import org.genesys2.server.api.ApiBaseController;
import org.genesys2.server.api.Pagination;
import org.genesys2.server.exception.NotFoundElement;
import org.genesys2.server.model.kpi.Dimension;
import org.genesys2.server.model.kpi.Execution;
import org.genesys2.server.model.kpi.ExecutionRun;
import org.genesys2.server.model.kpi.KPIParameter;
import org.genesys2.server.service.KPIService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.fasterxml.jackson.annotation.JsonView;
import io.swagger.annotations.Api;
@RestController("kpiApi1")
@RequestMapping(value = { KPIController.CONTROLLER_URL })
@Api(tags = { "kpi" })
public class KPIController {
public static final String CONTROLLER_URL = ApiBaseController.APIv1_BASE + "/kpi";
@Autowired
private KPIService kpiService;
/**
* First declare the KPI parameters: e.g. `Accession where `
*
* @param parameter
* @return
*/
@PostMapping(value = "/parameters/save")
public KPIParameter saveParameter(@RequestBody KPIParameter parameter) {
return kpiService.save(parameter);
}
@DeleteMapping(value = "/parameters/{name}")
public KPIParameter deleteParameter(@PathVariable String name) {
return kpiService.delete(kpiService.getParameter(name));
}
@GetMapping(value="/parameters")
public Page<KPIParameter> listParameters(final Pagination page) {
return kpiService.listParameters(page.toPageRequest(100));
}
/**
* Then declare the KPI dimensions: e.g. `FaoInstitute#code where accessionCount gt 0`
* Dimensions provide query parameters to KPI execution
*
* @param dimension
* @return
*/
@PostMapping(value = "/dimensions/save")
public Dimension<?> saveDimension(@RequestBody Dimension<?> dimension) {
return kpiService.save(dimension);
}
@DeleteMapping(value = "/dimensions/{name}")
public Dimension<?> deleteDimension(@PathVariable String name) {
return kpiService.delete(kpiService.getDimension(name));
}
@GetMapping(value="/dimensions")
public Page<Dimension<?>> listDimensions(final Pagination page) {
return kpiService.listDimensions(page.toPageRequest(100));
}
/**
* Declare execution with a {@link KPIParameter} and one or more {@link Dimension}
*
* @param execution
* @return
*/
@PostMapping(value = "/executions/save")
public Execution saveExecution(@RequestBody Execution execution) {
return kpiService.save(execution);
}
@DeleteMapping(value = "/executions/{name}")
public Execution deleteExecution(@PathVariable String name) {
return kpiService.save(kpiService.getExecution(name));
}
@GetMapping(value="/executions")
public Page<Execution> listExecution(final Pagination page) {
return kpiService.listExecutions(page.toPageRequest(100));
}
@PostMapping(value = "/executions/{name}/execute")
public ExecutionRun runExecution(@PathVariable String name) {
return kpiService.executeAndSave(kpiService.getExecution(name));
}
/**
* Execution details
*/
@GetMapping(value = "/executions/{name}")
public ExecutionDetails executionDetails(@PathVariable String name) {
Execution execution = kpiService.getExecution(name);
return ExecutionDetails.from(execution, kpiService.listExecutionRuns(execution, new PageRequest(0, 10)));
}
@GetMapping(value = "/executions/{name}/runs")
public List<ExecutionRun> executionRuns(final @PathVariable String name, final Pagination page) {
return kpiService.listExecutionRuns(kpiService.getExecution(name), page.toPageRequest(100));
}
@GetMapping(value = "/executions/{name}/runs/{id}")
public ExecutionRun executionRun(final @PathVariable String name, final long runId) {
ExecutionRun run = kpiService.getExecutionRun(runId);
if (! StringUtils.equals(run.getExecution().getName(), name)) {
throw new NotFoundElement("No run " + runId + " for execution " + name);
}
return run;
}
/**
* Details
*/
public static class ExecutionDetails {
public Execution execution;
public List<ExecutionRun> runs;
public static ExecutionDetails from(Execution execution, List<ExecutionRun> runs) {
ExecutionDetails ed=new ExecutionDetails();
ed.execution=execution;
ed.runs = runs;
return ed;
}
}
}
......@@ -60,7 +60,7 @@ public class BooleanDimension extends Dimension<Boolean> {
if (hasTrue()) {
b.add(Boolean.TRUE);
}
if ((mode ^ 2) > 0) {
if (hasFalse()) {
b.add(Boolean.FALSE);
}
return b;
......
......@@ -25,6 +25,8 @@ import javax.persistence.Entity;
import javax.persistence.Inheritance;
import javax.persistence.InheritanceType;
import javax.persistence.Table;
import javax.validation.constraints.NotNull;
import javax.validation.constraints.Size;
import org.genesys.blocks.model.AuditedVersionedModel;
import org.genesys2.server.model.genesys.Parameter;
......@@ -54,9 +56,15 @@ public abstract class Dimension<T> extends AuditedVersionedModel {
*
*/
private static final long serialVersionUID = 1672379271657218936L;
@Column(length = 100, nullable = false)
@NotNull
@Size(max = 100)
@Column(length = 100, unique = true, nullable = false)
private String name;
@Column(length = 100, nullable = false)
@NotNull
@Size(max = 100)
@Column(length = 100, unique = true, nullable = false)
private String title;
final public String getName() {
......
......@@ -27,11 +27,13 @@ import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.OneToMany;
import javax.persistence.Table;
import net.sf.oval.constraint.NotBlank;
import javax.validation.constraints.NotNull;
import javax.validation.constraints.Size;
import org.genesys.blocks.model.AuditedVersionedModel;
import com.fasterxml.jackson.annotation.JsonIgnore;
/**
* Evaluates {@link KPIParameter} by {@link Dimension}s.
*
......@@ -47,23 +49,44 @@ public class Execution extends AuditedVersionedModel {
*/
private static final long serialVersionUID = 1102563708369373562L;
public static enum ExecutionType {
COUNT, SUM, AVERAGE
}
/**
* This specifies the "key" under which observations are filed
*/
@NotBlank
@NotNull
@Size(max = 100)
@Column(length = 100, unique = true, nullable = false)
private String name;
@NotNull
@Column(nullable = false)
private ExecutionType type = ExecutionType.COUNT;
@Size(max = 100)
@Column(length = 100)
private String title;
@NotNull
@ManyToOne(cascade = {}, fetch = FetchType.EAGER, optional = false)
@JoinColumn(name = "parameterId")
private KPIParameter parameter;
/** This is the property of {@link #parameter} we're observing */
@NotNull
@Size(max = 30)
@Column(nullable = false, length = 30)
private String property = "id";
@OneToMany(orphanRemoval = true, fetch = FetchType.EAGER, cascade = { CascadeType.ALL })
@JoinColumn(name = "executionId")
private List<ExecutionDimension> dimensions = new ArrayList<ExecutionDimension>();
@Column(length = 100)
private String title;
@JsonIgnore
@OneToMany(mappedBy="execution", orphanRemoval = true, fetch = FetchType.LAZY, cascade = { CascadeType.REMOVE })
private List<ExecutionRun> runs;
public String getName() {
return name;
......@@ -81,8 +104,41 @@ public class Execution extends AuditedVersionedModel {
return parameter;
}
/**
* Defaults to {@link ExecutionType#COUNT}.
*
* @param type
*/
public void setType(ExecutionType type) {
this.type = type;
}
public ExecutionType getType() {
return type;
}
/**
* Set the property of the {{@link #parameter} that we're observing. Defaults to
* `id`, but you will use `accessionCount` or similar for AVERAGE
*
* @param property
*/
public void setProperty(String property) {
this.property = property;
}
public String getProperty() {
return property;
}
/**
* Order of dimensions matters!
*
* @param dimension
* @param link
* @param field
*/
public void addDimension(Dimension<?> dimension, String link, String field) {
// what do we do?
ExecutionDimension ped = new ExecutionDimension();
ped.setDimension(dimension);
ped.setLink(link);
......@@ -94,9 +150,23 @@ public class Execution extends AuditedVersionedModel {
public String query() {
StringBuffer sb = new StringBuffer(), where = new StringBuffer();
String alias = "_base";
sb.append("select count(distinct ");
sb.append(alias);
sb.append(") from ");
sb.append("select ");
switch (type) {
case SUM:
sb.append("sum(").append(alias).append(".").append(property).append(")");
break;
case AVERAGE:
sb.append("avg(").append(alias).append(".").append(property).append(")");
sb.append(", ");
sb.append("stddev(").append(alias).append(".").append(property).append(")");
break;
case COUNT:
default:
sb.append("count(distinct ").append(alias).append(".").append(property).append(")");
}
sb.append(" from ");
sb.append(parameter.getEntity());
sb.append(" ").append(alias);
......
......@@ -21,6 +21,8 @@ import javax.persistence.Entity;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.Table;
import javax.validation.constraints.NotNull;
import javax.validation.constraints.Size;
import org.genesys.blocks.model.BasicModel;
......@@ -33,12 +35,18 @@ public class ExecutionDimension extends BasicModel {
*/
private static final long serialVersionUID = 5401855589899745004L;
@NotNull
@ManyToOne(cascade = {}, optional = false)
@JoinColumn(name = "dimensionId")
private Dimension<?> dimension;
/** Linked entity from the Parameter to inner join directly (e.g. accession.institute) **/
@Size(max=100)
@Column(length = 100, nullable = true)
private String link;
/** Field in the Parameter (e.g. accession.instCode) or Linked entity ({@link #link} (e.g. accession.institute.code) **/
@Size(max=100)
@Column(length = 100, nullable = false)
private String field;
......@@ -71,4 +79,8 @@ public class ExecutionDimension extends BasicModel {
return "id=" + getId() + " link=" + link + " field=" + field + " dim=" + dimension.getName();
}
public String toName() {
return (link == null ? "" : link + ".") + field;
}
}
......@@ -17,18 +17,22 @@
package org.genesys2.server.model.kpi;
import java.util.Date;
import java.util.List;
import javax.persistence.CascadeType;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.ManyToOne;
import javax.persistence.OneToMany;
import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
import com.fasterxml.jackson.annotation.JsonIgnore;
import javax.validation.constraints.NotNull;
import org.genesys.blocks.model.BasicModel;
import com.fasterxml.jackson.annotation.JsonIgnore;
@Entity
@Table(name = "kpiexecutionrun")
public class ExecutionRun extends BasicModel {
......@@ -38,13 +42,18 @@ public class ExecutionRun extends BasicModel {
*/
private static final long serialVersionUID = -2475286586650646568L;
@NotNull
@Column(nullable = false)
@Temporal(TemporalType.TIMESTAMP)
private Date timestamp = new Date();
@NotNull
@JsonIgnore
@ManyToOne(cascade = {}, optional = false)
private Execution execution;
@OneToMany(mappedBy="executionRun", cascade = { CascadeType.PERSIST }, orphanRemoval = true)
private List<Observation> observations;
public Date getTimestamp() {
return timestamp;
......@@ -61,4 +70,12 @@ public class ExecutionRun extends BasicModel {
public void setExecution(Execution execution) {
this.execution = execution;
}
public List<Observation> getObservations() {
return observations;
}
public void setObservations(List<Observation> observations) {
this.observations = observations;
}
}
......@@ -20,20 +20,25 @@ import java.util.Set;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.validation.constraints.NotNull;
import javax.validation.constraints.Size;
import com.fasterxml.jackson.annotation.JsonIgnore;
@Entity
public class JpaDimension extends Dimension<Object> {
/**
*
*/
private static final long serialVersionUID = 4337653920991433997L;
@NotNull @Size(min=1, max=100)
@Column(length = 100)
private String entity;
@NotNull @Size(min=1, max=100)
@Column(length = 50)
private String field;
@Size(max=100)
@Column(name = "`condition`", length = 100)
private String condition;
......
......@@ -21,6 +21,8 @@ import javax.persistence.Entity;
import javax.persistence.Lob;
import javax.persistence.Table;
import javax.persistence.UniqueConstraint;
import javax.validation.constraints.NotNull;
import javax.validation.constraints.Size;
import org.genesys.blocks.model.AuditedVersionedModel;
import org.hibernate.annotations.Type;
......@@ -49,15 +51,22 @@ public class KPIParameter extends AuditedVersionedModel {
*/
private static final long serialVersionUID = 5773482399613068571L;
@NotNull
@Size(max = 100)
@Column(length = 100, nullable = false)
private String name;
@NotNull
@Size(max = 100)
@Column(length = 100, nullable = false)
private String title;
@NotNull
@Size(max = 100)
@Column(length = 100, nullable = false)
private String entity;
@Size(max = 300)
@Column(name = "`condition`", length = 300)
private String condition;
......
/**
* Copyright 2014 Global Crop Diversity Trust
*
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
*/
package org.genesys2.server.model.kpi;
import java.util.HashSet;
import java.util.Set;
import java.util.stream.Collectors;
import javax.persistence.CollectionTable;
import javax.persistence.Column;
......@@ -26,6 +26,9 @@ import javax.persistence.Entity;
import javax.persistence.JoinColumn;
import javax.persistence.Transient;
/**
* @author Matija Obreza
*/
@Entity
public class NumericListDimension extends FixedListDimension<Number> {
......@@ -49,11 +52,7 @@ public class NumericListDimension extends FixedListDimension<Number> {
if (list == null) {
this.values = null;
} else {
Set<Double> doubles = new HashSet<Double>();
for (Number n : list) {
doubles.add(n.doubleValue());
}
this.values = doubles;
this.values = list.stream().map(Number::doubleValue).collect(Collectors.toSet());
}
}
......@@ -62,11 +61,7 @@ public class NumericListDimension extends FixedListDimension<Number> {
if (values == null) {
return null;
}
Set<Number> numbers = new HashSet<Number>();
for (double d : values) {
numbers.add(toType(d));
}
return numbers;
return this.values.stream().map(this::toType).collect(Collectors.toSet());
}
private Number toType(Double d) {
......
......@@ -17,7 +17,8 @@
package org.genesys2.server.model.kpi;
import java.io.Serializable;
import java.util.HashSet;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import javax.persistence.Column;
......@@ -33,10 +34,10 @@ import javax.persistence.ManyToOne;
import javax.persistence.PrePersist;
import javax.persistence.Table;
import com.fasterxml.jackson.annotation.JsonIgnore;
import org.genesys.blocks.model.EntityId;
import com.fasterxml.jackson.annotation.JsonIgnore;
/**
* Holds results of {@link Execution} run.
*/
......@@ -56,6 +57,9 @@ public class Observation implements EntityId, Serializable {
@Column(name = "`value`")
private double value;