Commit 7a7c66c9 authored by Matija Obreza's avatar Matija Obreza

KPI Execution supports `group by` as additional dimensions

- alias field added
parent 3af55efa
......@@ -115,4 +115,10 @@ public class DimensionKey implements EntityId, Serializable {
return true;
}
public static DimensionKey createFor(String name, String value) {
DimensionKey dk = new DimensionKey();
dk.name = name;
dk.value = value;
return dk;
}
}
......@@ -20,7 +20,9 @@ import java.util.ArrayList;
import java.util.List;
import javax.persistence.CascadeType;
import javax.persistence.CollectionTable;
import javax.persistence.Column;
import javax.persistence.ElementCollection;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.JoinColumn;
......@@ -86,32 +88,42 @@ public class Execution extends AuditedVersionedModel implements SelfCleaning {
@JoinColumn(name = "parameterId")
private KPIParameter parameter;
/** This joins the _base.link in the query and allows us to count properties of collections */
/**
* This joins the _base.link in the query and allows us to count properties of
* collections
*/
@Size(min = 1, max = 50)
@Pattern(regexp="[a-z]([a-zA-Z0-9_]+)")
@Pattern(regexp = "[a-z]([a-zA-Z0-9_]+)")
@Column(length = 50)
private String link;
/** This is the property of {@link #parameter} we're observing */
@NotNull
@Size(max = 30)
@Column(nullable = false, length = 30)
private String property = "id";
@OneToMany(orphanRemoval = true, fetch = FetchType.EAGER, cascade = { CascadeType.ALL })
@OneToMany(orphanRemoval = true, fetch = FetchType.LAZY, cascade = { CascadeType.ALL })
@JoinColumn(name = "executionId")
private List<ExecutionDimension> dimensions = new ArrayList<ExecutionDimension>();
private List<ExecutionDimension> executionDimensions = new ArrayList<ExecutionDimension>();
@ElementCollection(fetch = FetchType.LAZY)
@CollectionTable(name = "execution_group", joinColumns = @JoinColumn(name = "executionId"))
private List<ExecutionGroup> groups = new ArrayList<ExecutionGroup>();
@JsonIgnore
@OneToMany(mappedBy="execution", orphanRemoval = true, fetch = FetchType.LAZY, cascade = { CascadeType.REMOVE })
@OneToMany(mappedBy = "execution", orphanRemoval = true, fetch = FetchType.LAZY, cascade = { CascadeType.REMOVE })
private List<ExecutionRun> runs;
@PrePersist
@PreUpdate
private void preupdate() {
trimStringsToNull();
if (groups != null) {
groups.forEach(group -> group.trimStringsToNull());
}
}
public String getName() {
return name;
}
......@@ -140,17 +152,17 @@ public class Execution extends AuditedVersionedModel implements SelfCleaning {
public ExecutionType getType() {
return type;
}
/**
* Set the name of the collection to join to the query. The {@link #property} then referes
* to the joined collection.
* Set the name of the collection to join to the query. The {@link #property}
* then referes to the joined collection.
*
* @param link Collection property of the entity to join
*/
public void setLink(String link) {
this.link = link;
}
public String getLink() {
return link;
}
......@@ -182,66 +194,80 @@ public class Execution extends AuditedVersionedModel implements SelfCleaning {
ped.setLink(link);
ped.setField(field);
dimensions.add(ped);
executionDimensions.add(ped);
}
public String query() {
StringBuffer sb = new StringBuffer(), where = new StringBuffer();
String alias = "_base";
final String aliasDereferenced = "PC";
final String aliasParameter = "PA";
sb.append("select ");
for (ExecutionGroup group : groups) {
sb.append(group.toJpa());
if (group.getAlias() != null) {
sb.append(" as ").append(group.toName());
}
sb.append(", ");
}
switch (type) {
case SUM:
sb.append("sum(").append(alias).append(".").append(property).append(")");
sb.append("sum(").append(link == null ? aliasParameter : aliasDereferenced).append(".").append(property).append(")");
break;
case AVERAGE:
sb.append("avg(").append(alias).append(".").append(property).append(")");
sb.append("avg(").append(link == null ? aliasParameter : aliasDereferenced).append(".").append(property).append(")");
sb.append(", ");
sb.append("stddev(").append(alias).append(".").append(property).append(")");
sb.append("stddev(").append(link == null ? aliasParameter : aliasDereferenced).append(".").append(property).append(")");
break;
case COUNT:
default:
sb.append("count(distinct ").append(alias).append(".").append(property).append(")");
sb.append("count(distinct ").append(link == null ? aliasParameter : aliasDereferenced).append(".").append(property).append(")");
}
sb.append(" from ");
sb.append(parameter.getEntity());
sb.append(" ").append(link == null ? alias : "X");
sb.append(" ").append(aliasParameter);
int pedC = 0;
for (ExecutionDimension ped : dimensions) {
pedC++;
if (ped.getLink() != null) {
int execDimCounter = 0;
for (ExecutionDimension execDim : executionDimensions) {
execDimCounter++;
// System.err.println("DIM" + execDimCounter + " " + execDim);
if (execDim.getLink() != null) {
sb.append(" inner join ");
sb.append(link == null ? alias : "X").append(".");
sb.append(ped.getLink());
sb.append(" _ped").append(pedC).append(" ");
sb.append(aliasParameter).append(".");
sb.append(execDim.getLink());
sb.append(" ED").append(execDimCounter).append(" ");
}
if (pedC > 1)
if (execDimCounter > 1)
where.append(" and ");
if (ped.getLink() == null) {
where.append("( ").append(link == null ? alias : "X").append(".").append(ped.getField()).append(" = ?").append(pedC).append(" )");
if (execDim.getLink() == null) {
where.append("( ").append(aliasParameter).append(".").append(execDim.getField()).append(" = ?").append(execDimCounter).append(
" )");
} else {
where.append("( _ped").append(pedC).append(".").append(ped.getField()).append(" = ?").append(pedC).append(" )");
where.append("( ED").append(execDimCounter).append(".").append(execDim.getField()).append(" = ?").append(execDimCounter).append(" )");
}
}
if (link != null) {
// We're joining a collection to count it's property
sb.append(" inner join ");
sb.append("X.");
sb.append(aliasParameter).append(".");
sb.append(link);
sb.append(" ").append(alias);
sb.append(" ").append(aliasDereferenced);
}
if (where.length() > 0 || parameter.getCondition() != null) {
sb.append(" where ");
if (parameter.getCondition() != null) {
sb.append(link == null ? alias : "X").append(".").append(parameter.getCondition());
sb.append(aliasParameter).append(".").append(parameter.getCondition());
}
if (dimensions.size() > 0) {
if (executionDimensions.size() > 0) {
if (parameter.getCondition() != null) {
sb.append(" and ");
}
......@@ -249,17 +275,33 @@ public class Execution extends AuditedVersionedModel implements SelfCleaning {
}
}
if (!groups.isEmpty()) {
sb.append(" group by ");
int groupCounter = 0;
for (ExecutionGroup group : groups) {
groupCounter++;
if (groupCounter > 1) {
sb.append(", ");
}
sb.append(group.toJpa());
}
}
return sb.toString();
}
public void setExecutionDimensions(List<ExecutionDimension> executionDimensions) {
this.executionDimensions = executionDimensions;
}
public List<ExecutionDimension> getExecutionDimensions() {
return dimensions;
return executionDimensions;
}
public Dimension<?> getDimension(int depth) {
if (depth >= dimensions.size())
if (depth >= executionDimensions.size())
return null;
return dimensions.get(depth).getDimension();
return executionDimensions.get(depth).getDimension();
}
public String getTitle() {
......@@ -270,4 +312,11 @@ public class Execution extends AuditedVersionedModel implements SelfCleaning {
this.title = title;
}
public List<ExecutionGroup> getGroups() {
return groups;
}
public void setGroups(List<ExecutionGroup> groups) {
this.groups = groups;
}
}
......@@ -49,12 +49,18 @@ public class ExecutionDimension extends BasicModel implements SelfCleaning {
@Pattern(regexp = "[a-z][a-zA-Z0-9_\\.\\(\\)]*")
@Column(length = 100, nullable = true)
private String link;
/** Field in the Parameter (e.g. accession.instCode) or Linked entity ({@link #link} (e.g. accession.institute.code) **/
@Size(max=100)
@Pattern(regexp = "[_a-z][a-zA-Z0-9_\\.\\(\\)]*")
@Column(length = 100, nullable = false)
private String field;
/** Alias: how we store the key name in the observation **/
@Size(max=100)
@Pattern(regexp = "[_a-z][a-zA-Z0-9_\\.]*")
@Column(length = 100)
private String alias;
@PrePersist
@PreUpdate
......@@ -85,14 +91,22 @@ public class ExecutionDimension extends BasicModel implements SelfCleaning {
public void setLink(String link) {
this.link = link;
}
public String getAlias() {
return alias;
}
public void setAlias(String alias) {
this.alias = alias;
}
@Override
public String toString() {
return "id=" + getId() + " link=" + link + " field=" + field + " dim=" + dimension.getName();
return "id=" + getId() + " link=" + link + " field=" + field + " alias=" + alias + " dim=" + dimension.getName();
}
public String toName() {
return (link == null ? "" : link + ".") + field;
return alias == null ? ((link == null ? "" : link + ".") + field) : alias;
}
}
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys2.server.model.kpi;
import java.io.Serializable;
import javax.persistence.Column;
import javax.persistence.Embeddable;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.PrePersist;
import javax.persistence.PreUpdate;
import javax.validation.constraints.NotNull;
import javax.validation.constraints.Pattern;
import javax.validation.constraints.Size;
import org.genesys.blocks.model.SelfCleaning;
// TODO: Auto-generated Javadoc
/**
* GroupBy for Executions.
*/
@Embeddable
public class ExecutionGroup implements SelfCleaning, Serializable {
private static final long serialVersionUID = 1390895207879490387L;
/**
* Supported operations.
*/
public static enum Operation {
/** The year. */
YEAR,
/** The month. */
MONTH,
/** The day. */
DAY,
/** The length. */
LENGTH
}
@NotNull
@Size(max = 100)
@Pattern(regexp = "[_a-z][a-zA-Z0-9_\\.\\(\\)]*")
@Column(length = 100, nullable = false)
private String field;
@Column(length = 20)
@Enumerated(EnumType.STRING)
private Operation op;
/** The alias. */
@Size(max = 100)
@Pattern(regexp = "[_a-z][a-zA-Z0-9_]+")
@Column(length = 100)
private String alias;
@PrePersist
@PreUpdate
private void preupdate() {
trimStringsToNull();
}
/**
* Gets the field.
*
* @return the field
*/
public String getField() {
return field;
}
/**
* Sets the field.
*
* @param field the new field
*/
public void setField(String field) {
this.field = field;
}
/**
* Gets the op.
*
* @return the op
*/
public Operation getOp() {
return op;
}
/**
* Sets the op.
*
* @param op the new op
*/
public void setOp(Operation op) {
this.op = op;
}
/**
* Sets the alias.
*
* @param alias the new alias
*/
public void setAlias(String alias) {
this.alias = alias;
}
/**
* Gets the alias.
*
* @return the alias
*/
public String getAlias() {
return alias;
}
/* (non-Javadoc)
* @see java.lang.Object#hashCode()
*/
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((alias == null) ? 0 : alias.hashCode());
result = prime * result + ((field == null) ? 0 : field.hashCode());
result = prime * result + ((op == null) ? 0 : op.hashCode());
return result;
}
/* (non-Javadoc)
* @see java.lang.Object#equals(java.lang.Object)
*/
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
ExecutionGroup other = (ExecutionGroup) obj;
if (alias == null) {
if (other.alias != null)
return false;
} else if (!alias.equals(other.alias))
return false;
if (field == null) {
if (other.field != null)
return false;
} else if (!field.equals(other.field))
return false;
if (op != other.op)
return false;
return true;
}
public String toName() {
return alias == null ? field : alias;
}
/**
* Generate JPA statement for this group
*
* @return JPA fragment
*/
public String toJpa() {
StringBuffer sb = new StringBuffer();
if (op != null) {
sb.append(op.toString()).append("(");
}
sb.append(field);
if (op != null) {
sb.append(")");
}
return sb.toString();
}
}
......@@ -36,6 +36,7 @@ import org.apache.commons.lang3.StringUtils;
import org.genesys2.server.model.kpi.Dimension;
import org.genesys2.server.model.kpi.DimensionKey;
import org.genesys2.server.model.kpi.Execution;
import org.genesys2.server.model.kpi.ExecutionGroup;
import org.genesys2.server.model.kpi.ExecutionRun;
import org.genesys2.server.model.kpi.JpaDimension;
import org.genesys2.server.model.kpi.KPIParameter;
......@@ -230,37 +231,66 @@ public class KPIServiceImpl implements KPIService {
}
}
private Observation getSingleObservation(Query query, Object... params) {
private List<Observation> getObservationResults(Execution execution, Query query, Object... params) {
for (int i = 0; i < params.length; i++) {
LOG.debug("\t?{} = {}", (i + 1), params[i]);
query.setParameter(i + 1, params[i]);
}
Object res = query.getSingleResult();
List<Object> results = query.getResultList();
List<Observation> observations = new ArrayList<>(results.size());
if (res == null) {
return null;
} else if (res instanceof Object[]) {
Object[] r = (Object[]) res;
Observation observation = new Observation();
if (r[0] != null) {
observation.setValue(((Number) r[0]).doubleValue());
}
if (r.length > 1 && r[1] != null) {
observation.setStdDev(((Number) r[1]).doubleValue());
}
return observation;
} else if (res instanceof Number) {
Number r = (Number) res;
Observation observation = new Observation();
observation.setValue(r.doubleValue());
return observation;
for (Object res : results) {
LOG.trace("Query result: {}", res);
if (res == null) {
continue;
} else if (res instanceof Object[]) {
// case of group by or AVERAGE+STDEV
Object[] r = (Object[]) res;
// LOG.trace("OBS: {}", r);
Observation observation = new Observation();
int pos=0;
{
// Query is designed to first return groupBy's
List<ExecutionGroup> groups = execution.getGroups();
for (ExecutionGroup group : groups) {
String groupValue = r[pos] == null ? "" : r[pos].toString();
DimensionKey dk = dimensionKeyRepository.findByNameAndValue(group.toName(), groupValue);
if (dk == null) {
dk = DimensionKey.createFor(group.toName(), groupValue);
}
observation.getDimensions().add(dk);
pos++;
}
}
if (r[pos] != null) {
observation.setValue(((Number) r[pos]).doubleValue());
}
pos++;
if (r.length > pos && r[pos] != null) {
observation.setStdDev(((Number) r[pos]).doubleValue());
}
} else {
throw new RuntimeException("Unrecognized return type " + res.getClass() + " for " + res);
observations.add(observation);
} else if (res instanceof Number) {
Number r = (Number) res;
Observation observation = new Observation();
observation.setValue(r.doubleValue());
observations.add(observation);
} else {
throw new RuntimeException("Unrecognized return type " + res.getClass() + " for " + res);
}
}
return observations;
}
// readonly mode
......@@ -403,8 +433,10 @@ public class KPIServiceImpl implements KPIService {
// execute
if (LOG.isDebugEnabled())
LOG.debug("Executing: {} params={}", execution.getName(), params);
Observation res = getSingleObservation(query, params.toArray());
registerObservation(res, execution, params.toArray(), results);
List<Observation> res = getObservationResults(execution, query, params.toArray());
res.forEach(observed -> {
registerObservation(observed, execution, params.toArray(), results);
});
} else {
// Recurse
Set<?> values = null;
......@@ -422,6 +454,10 @@ public class KPIServiceImpl implements KPIService {
LOG.debug("Observation is null, skipping");
return;
}
if (observed.getValue() == 0d) {
LOG.debug("Observation value is 0d, skipping");
return;
}
KPIParameter parameter = execution.getParameter();
if (LOG.isDebugEnabled()) {
......@@ -431,23 +467,18 @@ public class KPIServiceImpl implements KPIService {
}
for (int i = 0; i < conditions.length; i++) {
String name = execution.getDimension(i).getName();