KPI Execution supports `group by` as additional dimensions

- alias field added

src/main/java/org/genesys2/server/model/kpi/DimensionKey.java

@@ -115,4 +115,10 @@ public class DimensionKey implements EntityId, Serializable {
 		return true;
 	}
 
+	public static DimensionKey createFor(String name, String value) {
+		DimensionKey dk = new DimensionKey();
+		dk.name = name;
+		dk.value = value;
+		return dk;
+	}
 }

src/main/java/org/genesys2/server/model/kpi/Execution.java

@@ -20,7 +20,9 @@ import java.util.ArrayList;
 import java.util.List;
 
 import javax.persistence.CascadeType;
+import javax.persistence.CollectionTable;
 import javax.persistence.Column;
+import javax.persistence.ElementCollection;
 import javax.persistence.Entity;
 import javax.persistence.FetchType;
 import javax.persistence.JoinColumn;
@@ -86,32 +88,42 @@ public class Execution extends AuditedVersionedModel implements SelfCleaning {
 	@JoinColumn(name = "parameterId")
 	private KPIParameter parameter;
 
-	/** This joins the _base.link in the query and allows us to count properties of collections */
+	/**
+	 * This joins the _base.link in the query and allows us to count properties of
+	 * collections
+	 */
 	@Size(min = 1, max = 50)
-	@Pattern(regexp="[a-z]([a-zA-Z0-9_]+)")
+	@Pattern(regexp = "[a-z]([a-zA-Z0-9_]+)")
 	@Column(length = 50)
 	private String link;
-	
+
 	/** This is the property of {@link #parameter} we're observing */
 	@NotNull
 	@Size(max = 30)
 	@Column(nullable = false, length = 30)
 	private String property = "id";
 
-	@OneToMany(orphanRemoval = true, fetch = FetchType.EAGER, cascade = { CascadeType.ALL })
+	@OneToMany(orphanRemoval = true, fetch = FetchType.LAZY, cascade = { CascadeType.ALL })
 	@JoinColumn(name = "executionId")
-	private List<ExecutionDimension> dimensions = new ArrayList<ExecutionDimension>();
+	private List<ExecutionDimension> executionDimensions = new ArrayList<ExecutionDimension>();
+
+	@ElementCollection(fetch = FetchType.LAZY)
+	@CollectionTable(name = "execution_group", joinColumns = @JoinColumn(name = "executionId"))
+	private List<ExecutionGroup> groups = new ArrayList<ExecutionGroup>();
 
 	@JsonIgnore
-	@OneToMany(mappedBy="execution", orphanRemoval = true, fetch = FetchType.LAZY, cascade = { CascadeType.REMOVE })
+	@OneToMany(mappedBy = "execution", orphanRemoval = true, fetch = FetchType.LAZY, cascade = { CascadeType.REMOVE })
 	private List<ExecutionRun> runs;
 
 	@PrePersist
 	@PreUpdate
 	private void preupdate() {
 		trimStringsToNull();
+		if (groups != null) {
+			groups.forEach(group -> group.trimStringsToNull());
+		}
 	}
-	
+
 	public String getName() {
 		return name;
 	}
@@ -140,17 +152,17 @@ public class Execution extends AuditedVersionedModel implements SelfCleaning {
 	public ExecutionType getType() {
 		return type;
 	}
-	
+
 	/**
-	 * Set the name of the collection to join to the query. The {@link #property} then referes
-	 * to the joined collection.
+	 * Set the name of the collection to join to the query. The {@link #property}
+	 * then referes to the joined collection.
 	 * 
 	 * @param link Collection property of the entity to join
 	 */
 	public void setLink(String link) {
 		this.link = link;
 	}
-	
+
 	public String getLink() {
 		return link;
 	}
@@ -182,66 +194,80 @@ public class Execution extends AuditedVersionedModel implements SelfCleaning {
 		ped.setLink(link);
 		ped.setField(field);
 
-		dimensions.add(ped);
+		executionDimensions.add(ped);
 	}
 
 	public String query() {
 		StringBuffer sb = new StringBuffer(), where = new StringBuffer();
-		String alias = "_base";
+
+		final String aliasDereferenced = "PC";
+		final String aliasParameter = "PA";
+
 		sb.append("select ");
+
+		for (ExecutionGroup group : groups) {
+			sb.append(group.toJpa());
+			if (group.getAlias() != null) {
+				sb.append(" as ").append(group.toName());
+			}
+			sb.append(", ");
+		}
+
 		switch (type) {
 		case SUM:
-			sb.append("sum(").append(alias).append(".").append(property).append(")");
+			sb.append("sum(").append(link == null ? aliasParameter : aliasDereferenced).append(".").append(property).append(")");
 			break;
 
 		case AVERAGE:
-			sb.append("avg(").append(alias).append(".").append(property).append(")");
+			sb.append("avg(").append(link == null ? aliasParameter : aliasDereferenced).append(".").append(property).append(")");
 			sb.append(", ");
-			sb.append("stddev(").append(alias).append(".").append(property).append(")");
+			sb.append("stddev(").append(link == null ? aliasParameter : aliasDereferenced).append(".").append(property).append(")");
 			break;
 
 		case COUNT:
 		default:
-			sb.append("count(distinct ").append(alias).append(".").append(property).append(")");
+			sb.append("count(distinct ").append(link == null ? aliasParameter : aliasDereferenced).append(".").append(property).append(")");
 		}
 		sb.append(" from ");
 		sb.append(parameter.getEntity());
-		sb.append(" ").append(link == null ? alias : "X");
+		sb.append(" ").append(aliasParameter);
 
-		int pedC = 0;
-		for (ExecutionDimension ped : dimensions) {
-			pedC++;
-			if (ped.getLink() != null) {
+		int execDimCounter = 0;
+		for (ExecutionDimension execDim : executionDimensions) {
+			execDimCounter++;
+			// System.err.println("DIM" + execDimCounter + " " + execDim);
+			if (execDim.getLink() != null) {
 				sb.append(" inner join ");
-				sb.append(link == null ? alias : "X").append(".");
-				sb.append(ped.getLink());
-				sb.append(" _ped").append(pedC).append(" ");
+				sb.append(aliasParameter).append(".");
+				sb.append(execDim.getLink());
+				sb.append(" ED").append(execDimCounter).append(" ");
 			}
 
-			if (pedC > 1)
+			if (execDimCounter > 1)
 				where.append(" and ");
 
-			if (ped.getLink() == null) {
-				where.append("( ").append(link == null ? alias : "X").append(".").append(ped.getField()).append(" = ?").append(pedC).append(" )");
+			if (execDim.getLink() == null) {
+				where.append("( ").append(aliasParameter).append(".").append(execDim.getField()).append(" = ?").append(execDimCounter).append(
+					" )");
 			} else {
-				where.append("( _ped").append(pedC).append(".").append(ped.getField()).append(" = ?").append(pedC).append(" )");
+				where.append("( ED").append(execDimCounter).append(".").append(execDim.getField()).append(" = ?").append(execDimCounter).append(" )");
 			}
 		}
 
 		if (link != null) {
 			// We're joining a collection to count it's property
 			sb.append(" inner join ");
-			sb.append("X.");
+			sb.append(aliasParameter).append(".");
 			sb.append(link);
-			sb.append(" ").append(alias);
+			sb.append(" ").append(aliasDereferenced);
 		}
 
 		if (where.length() > 0 || parameter.getCondition() != null) {
 			sb.append(" where ");
 			if (parameter.getCondition() != null) {
-				sb.append(link == null ? alias : "X").append(".").append(parameter.getCondition());
+				sb.append(aliasParameter).append(".").append(parameter.getCondition());
 			}
-			if (dimensions.size() > 0) {
+			if (executionDimensions.size() > 0) {
 				if (parameter.getCondition() != null) {
 					sb.append(" and ");
 				}
@@ -249,17 +275,33 @@ public class Execution extends AuditedVersionedModel implements SelfCleaning {
 			}
 		}
 
+		if (!groups.isEmpty()) {
+			sb.append(" group by ");
+			int groupCounter = 0;
+			for (ExecutionGroup group : groups) {
+				groupCounter++;
+				if (groupCounter > 1) {
+					sb.append(", ");
+				}
+				sb.append(group.toJpa());
+			}
+		}
+
 		return sb.toString();
 	}
 
+	public void setExecutionDimensions(List<ExecutionDimension> executionDimensions) {
+		this.executionDimensions = executionDimensions;
+	}
+	
 	public List<ExecutionDimension> getExecutionDimensions() {
-		return dimensions;
+		return executionDimensions;
 	}
-
+	
 	public Dimension<?> getDimension(int depth) {
-		if (depth >= dimensions.size())
+		if (depth >= executionDimensions.size())
 			return null;
-		return dimensions.get(depth).getDimension();
+		return executionDimensions.get(depth).getDimension();
 	}
 
 	public String getTitle() {
@@ -270,4 +312,11 @@ public class Execution extends AuditedVersionedModel implements SelfCleaning {
 		this.title = title;
 	}
 
+	public List<ExecutionGroup> getGroups() {
+		return groups;
+	}
+
+	public void setGroups(List<ExecutionGroup> groups) {
+		this.groups = groups;
+	}
 }

src/main/java/org/genesys2/server/model/kpi/ExecutionDimension.java

@@ -49,12 +49,18 @@ public class ExecutionDimension extends BasicModel implements SelfCleaning {
 	@Pattern(regexp = "[a-z][a-zA-Z0-9_\\.\\(\\)]*")
 	@Column(length = 100, nullable = true)
 	private String link;
-	
+
 	/** Field in the Parameter (e.g. accession.instCode) or Linked entity ({@link #link} (e.g. accession.institute.code) **/
 	@Size(max=100)
 	@Pattern(regexp = "[_a-z][a-zA-Z0-9_\\.\\(\\)]*")
 	@Column(length = 100, nullable = false)
 	private String field;
+
+	/** Alias: how we store the key name in the observation **/
+	@Size(max=100)
+	@Pattern(regexp = "[_a-z][a-zA-Z0-9_\\.]*")
+	@Column(length = 100)
+	private String alias;
 	
 	@PrePersist
 	@PreUpdate
@@ -85,14 +91,22 @@ public class ExecutionDimension extends BasicModel implements SelfCleaning {
 	public void setLink(String link) {
 		this.link = link;
 	}
+	
+	public String getAlias() {
+		return alias;
+	}
+	
+	public void setAlias(String alias) {
+		this.alias = alias;
+	}
 
 	@Override
 	public String toString() {
-		return "id=" + getId() + " link=" + link + " field=" + field + " dim=" + dimension.getName();
+		return "id=" + getId() + " link=" + link + " field=" + field + " alias=" + alias + " dim=" + dimension.getName();
 	}
 
 	public String toName() {
-		return (link == null ? "" : link + ".") + field;
+		return alias == null ? ((link == null ? "" : link + ".") + field) : alias;
 	}
 
 }

src/main/java/org/genesys2/server/model/kpi/ExecutionGroup.java

+/*
+ * Copyright 2018 Global Crop Diversity Trust
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.genesys2.server.model.kpi;
+
+import java.io.Serializable;
+
+import javax.persistence.Column;
+import javax.persistence.Embeddable;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
+import javax.persistence.PrePersist;
+import javax.persistence.PreUpdate;
+import javax.validation.constraints.NotNull;
+import javax.validation.constraints.Pattern;
+import javax.validation.constraints.Size;
+
+import org.genesys.blocks.model.SelfCleaning;
+
+// TODO: Auto-generated Javadoc
+/**
+ * GroupBy for Executions.
+ */
+@Embeddable
+public class ExecutionGroup implements SelfCleaning, Serializable {
+
+	private static final long serialVersionUID = 1390895207879490387L;
+
+	/**
+	 * Supported operations.
+	 */
+	public static enum Operation {
+
+		/** The year. */
+		YEAR,
+		/** The month. */
+		MONTH,
+		/** The day. */
+		DAY,
+		/** The length. */
+		LENGTH
+	}
+
+	@NotNull
+	@Size(max = 100)
+	@Pattern(regexp = "[_a-z][a-zA-Z0-9_\\.\\(\\)]*")
+	@Column(length = 100, nullable = false)
+	private String field;
+
+	@Column(length = 20)
+	@Enumerated(EnumType.STRING)
+	private Operation op;
+
+	/** The alias. */
+	@Size(max = 100)
+	@Pattern(regexp = "[_a-z][a-zA-Z0-9_]+")
+	@Column(length = 100)
+	private String alias;
+
+	
+	@PrePersist
+	@PreUpdate
+	private void preupdate() {
+		trimStringsToNull();
+	}
+	
+	/**
+	 * Gets the field.
+	 *
+	 * @return the field
+	 */
+	public String getField() {
+		return field;
+	}
+
+	/**
+	 * Sets the field.
+	 *
+	 * @param field the new field
+	 */
+	public void setField(String field) {
+		this.field = field;
+	}
+
+	/**
+	 * Gets the op.
+	 *
+	 * @return the op
+	 */
+	public Operation getOp() {
+		return op;
+	}
+
+	/**
+	 * Sets the op.
+	 *
+	 * @param op the new op
+	 */
+	public void setOp(Operation op) {
+		this.op = op;
+	}
+	
+	/**
+	 * Sets the alias.
+	 *
+	 * @param alias the new alias
+	 */
+	public void setAlias(String alias) {
+		this.alias = alias;
+	}
+	
+	/**
+	 * Gets the alias.
+	 *
+	 * @return the alias
+	 */
+	public String getAlias() {
+		return alias;
+	}
+
+	/* (non-Javadoc)
+	 * @see java.lang.Object#hashCode()
+	 */
+	@Override
+	public int hashCode() {
+		final int prime = 31;
+		int result = 1;
+		result = prime * result + ((alias == null) ? 0 : alias.hashCode());
+		result = prime * result + ((field == null) ? 0 : field.hashCode());
+		result = prime * result + ((op == null) ? 0 : op.hashCode());
+		return result;
+	}
+
+	/* (non-Javadoc)
+	 * @see java.lang.Object#equals(java.lang.Object)
+	 */
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (obj == null)
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+		ExecutionGroup other = (ExecutionGroup) obj;
+		if (alias == null) {
+			if (other.alias != null)
+				return false;
+		} else if (!alias.equals(other.alias))
+			return false;
+		if (field == null) {
+			if (other.field != null)
+				return false;
+		} else if (!field.equals(other.field))
+			return false;
+		if (op != other.op)
+			return false;
+		return true;
+	}
+
+	public String toName() {
+		return alias == null ? field : alias;
+	}
+
+	/**
+	 * Generate JPA statement for this group
+	 * 
+	 * @return JPA fragment
+	 */
+	public String toJpa() {
+		StringBuffer sb = new StringBuffer();
+		if (op != null) {
+			sb.append(op.toString()).append("(");
+		}
+		sb.append(field);
+		if (op != null) {
+			sb.append(")");
+		}
+		return sb.toString();
+	}
+}

src/main/java/org/genesys2/server/service/impl/KPIServiceImpl.java

@@ -36,6 +36,7 @@ import org.apache.commons.lang3.StringUtils;
 import org.genesys2.server.model.kpi.Dimension;
 import org.genesys2.server.model.kpi.DimensionKey;
 import org.genesys2.server.model.kpi.Execution;
+import org.genesys2.server.model.kpi.ExecutionGroup;
 import org.genesys2.server.model.kpi.ExecutionRun;
 import org.genesys2.server.model.kpi.JpaDimension;
 import org.genesys2.server.model.kpi.KPIParameter;
@@ -230,37 +231,66 @@ public class KPIServiceImpl implements KPIService {
 		}
 	}
 
-	private Observation getSingleObservation(Query query, Object... params) {
+	private List<Observation> getObservationResults(Execution execution, Query query, Object... params) {
 		for (int i = 0; i < params.length; i++) {
 			LOG.debug("\t?{} = {}", (i + 1), params[i]);
 			query.setParameter(i + 1, params[i]);
 		}
 
-		Object res = query.getSingleResult();
+		List<Object> results = query.getResultList();
+		List<Observation> observations = new ArrayList<>(results.size());
 		
-		if (res == null) {
-			return null;
-		
-		} else if (res instanceof Object[]) {
-			Object[] r = (Object[]) res;
-			Observation observation = new Observation();
-			if (r[0] != null) {
-				observation.setValue(((Number) r[0]).doubleValue());
-			}
-			if (r.length > 1 && r[1] != null) {
-				observation.setStdDev(((Number) r[1]).doubleValue());
-			}
-			return observation;
-		
-		} else if (res instanceof Number) {
-			Number r = (Number) res;
-			Observation observation = new Observation();
-			observation.setValue(r.doubleValue());
-			return observation;
+		for (Object res : results) {
+			LOG.trace("Query result: {}", res);
+			
+			if (res == null) {
+				continue;
+			
+			} else if (res instanceof Object[]) {
+				// case of group by or AVERAGE+STDEV
+				Object[] r = (Object[]) res;
+				// LOG.trace("OBS: {}", r);
+
+				Observation observation = new Observation();
+				
+				int pos=0;
+				{
+					// Query is designed to first return groupBy's
+					List<ExecutionGroup> groups = execution.getGroups();
+					for (ExecutionGroup group : groups) {
+						String groupValue = r[pos] == null ? "" : r[pos].toString();
+						DimensionKey dk = dimensionKeyRepository.findByNameAndValue(group.toName(), groupValue);
+						if (dk == null) {
+							dk = DimensionKey.createFor(group.toName(), groupValue);
+						}
+						observation.getDimensions().add(dk);
+						pos++;
+					}
+				}
+				
+				if (r[pos] != null) {
+					observation.setValue(((Number) r[pos]).doubleValue());
+				}
+				pos++;
+				
+				if (r.length > pos && r[pos] != null) {
+					observation.setStdDev(((Number) r[pos]).doubleValue());
+				}
 
-		} else {
-			throw new RuntimeException("Unrecognized return type " + res.getClass() + " for " + res);
+				observations.add(observation);
+			
+			} else if (res instanceof Number) {
+				Number r = (Number) res;
+				Observation observation = new Observation();
+				observation.setValue(r.doubleValue());
+				observations.add(observation);
+	
+			} else {
+				throw new RuntimeException("Unrecognized return type " + res.getClass() + " for " + res);
+			}
 		}
+		
+		return observations;
 	}
 
 	// readonly mode
@@ -403,8 +433,10 @@ public class KPIServiceImpl implements KPIService {
 			// execute
 			if (LOG.isDebugEnabled())
 				LOG.debug("Executing: {} params={}", execution.getName(), params);
-			Observation res = getSingleObservation(query, params.toArray());
-			registerObservation(res, execution, params.toArray(), results);
+			List<Observation> res = getObservationResults(execution, query, params.toArray());
+			res.forEach(observed -> {
+				registerObservation(observed, execution, params.toArray(), results);
+			});
 		} else {
 			// Recurse
 			Set<?> values = null;
@@ -422,6 +454,10 @@ public class KPIServiceImpl implements KPIService {
 			LOG.debug("Observation is null, skipping");
 			return;
 		}
+		if (observed.getValue() == 0d) {
+			LOG.debug("Observation value is 0d, skipping");
+			return;
+		}
 
 		KPIParameter parameter = execution.getParameter();
 		if (LOG.isDebugEnabled()) {
@@ -431,23 +467,18 @@ public class KPIServiceImpl implements KPIService {
 		}
 
 		for (int i = 0; i < conditions.length; i++) {
-			String name = execution.getDimension(i).getName();