Skip to content

GitLab

Commit 67ccd57d authored by Matija Obreza's avatar Matija Obreza
Browse files

Extracted query building to DirectMysqlQuery class

parent b0e99e4b
Hide whitespace changes
Inline Side-by-side
src/main/java/org/genesys2/server/service/GenesysFilterService.java
View file @ 67ccd57d
......@@ -25,7 +25,6 @@ import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.jdbc.core.RowCallbackHandler;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
public interface GenesysFilterService {
......@@ -33,8 +32,8 @@ public interface GenesysFilterService {
String[] DEFAULT_FILTERS = { "crop", "genus", "taxon", "orgCty", "latitude", "longitude", "organization", "instCode", "acceNumb", "available", "mlsStat", "inTrust",
"inSvalbard" };
Page<Accession> listAccessions(JsonNode jsonTree, Pageable pageable);
Page<Accession> listAccessions(ObjectNode jsonTree, Pageable pageable);
List<GenesysFilter> listAvailableFilters();
public static interface GenesysFilter {
......@@ -59,4 +58,5 @@ public interface GenesysFilterService {
void listGeo(ObjectNode jsonTree, Integer limit, RowCallbackHandler rowHandler);
void listGeoTile(boolean distinct, ObjectNode jsonTree, Integer limit, int zoom, int xtile, int ytile, RowCallbackHandler rowHandler);
}
src/main/java/org/genesys2/server/service/impl/DirectMysqlQuery.java 0 → 100644
View file @ 67ccd57d
package org.genesys2.server.service.impl;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.genesys2.server.model.genesys.Method;
import org.genesys2.server.service.MappingService.CoordUtil;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort.Order;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.JsonNodeType;
import com.fasterxml.jackson.databind.node.ObjectNode;
public class DirectMysqlQuery {
private static final Log LOG = LogFactory.getLog(DirectMysqlQuery.class);
final Set<String> tables = new HashSet<String>();
final List<Object> params = new ArrayList<Object>();
final StringBuffer sb, whereBuffer = new StringBuffer(), sortBuffer = new StringBuffer();
public DirectMysqlQuery(String baseTable, String baseAlias) {
sb = new StringBuffer(300);
sb.append(" from ").append(baseTable).append(StringUtils.SPACE).append(baseAlias);
tables.add(baseTable);
}
public DirectMysqlQuery innerJoin(String table, String alias, String onExpr) {
if (tables.contains(table)) {
LOG.warn("Table already inner-joined: " + table);
return this;
}
if (LOG.isDebugEnabled()) {
LOG.debug("Inner-joining " + table + " " + alias + " on " + onExpr);
}
sb.append(" inner join ").append(table).append(StringUtils.SPACE);
tables.add(table);
if (StringUtils.isNotBlank(alias)) {
sb.append(alias).append(StringUtils.SPACE);
}
sb.append("on ").append(onExpr).append(StringUtils.SPACE);
return this;
}
public DirectMysqlQuery jsonFilter(ObjectNode jsonTree, MethodResolver methodResolver) {
return join(jsonTree).filter(jsonTree, methodResolver);
}
protected DirectMysqlQuery join(ObjectNode jsonTree) {
if (jsonTree.has("crop") || jsonTree.has("genus") || jsonTree.has("taxon")) {
innerJoin("taxonomy2", "t", "t.id=a.taxonomyId2");
if (jsonTree.has("crop")) {
innerJoin("croptaxonomy", "ct", "ct.taxonomyId=t.id");
innerJoin("crop", null, "crop.id=ct.cropId");
}
}
if (jsonTree.has("organization")) {
innerJoin("faoinstitute", "fao", "fao.id=a.instituteId");
innerJoin("organizationinstitute", "oi", "oi.instituteId=fao.id");
innerJoin("organization", "org", "org.id=oi.organizationId");
}
if (jsonTree.has("latitude") || jsonTree.has("longitude") || jsonTree.has("elevation")) {
innerJoin("accessiongeo", "geo", "geo.accessionId=a.id");
}
return this;
}
protected DirectMysqlQuery filter(ObjectNode jsonTree, MethodResolver methodResolver) {
createQuery(whereBuffer, "a.taxSpecies", jsonTree.get("species"), params);
createQuery(whereBuffer, "a.acceNumb", jsonTree.get("acceNumb"), params);
createQuery(whereBuffer, "a.orgCty", jsonTree.get("orgCty"), params);
createQuery(whereBuffer, "a.instCode", jsonTree.get("instCode"), params);
createQuery(whereBuffer, "a.inSGSV", jsonTree.get("inSvalbard"), params);
createQuery(whereBuffer, "a.mlsStat", jsonTree.get("mlsStat"), params);
createQuery(whereBuffer, "a.inTrust", jsonTree.get("inTrust"), params);
createQuery(whereBuffer, "a.available", jsonTree.get("available"), params);
createQuery(whereBuffer, "org.slug", jsonTree.get("organization"), params);
createQuery(whereBuffer, "t.genus", jsonTree.get("genus"), params);
createQuery(whereBuffer, "t.taxonName", jsonTree.get("taxon"), params);
createQuery(whereBuffer, "geo.longitude", jsonTree.get("longitude"), params);
createQuery(whereBuffer, "geo.latitude", jsonTree.get("latitude"), params);
createQuery(whereBuffer, "geo.elevation", jsonTree.get("elevation"), params);
createQuery(whereBuffer, "crop.shortName", jsonTree.get("crop"), params);
for (Iterator<String> it = jsonTree.fieldNames(); it.hasNext();) {
String fieldName = it.next();
if (fieldName.startsWith("gm:")) {
LOG.info("Filtering on C&E data, method=" + fieldName);
// Handle Genesys Method!
long methodId = Long.parseLong(fieldName.substring(3));
// Need to validate method
Method method = methodResolver.getMethod(methodId);
if (method != null) {
String alias = "gm" + methodId;
innerJoin("`" + methodId + "`", alias, alias + ".accessionId=a.id");
createQuery(whereBuffer, alias + ".`" + method.getFieldName() + "`", jsonTree.get(fieldName), params);
} else {
LOG.warn("No such method with id=" + methodId);
}
}
}
LOG.info("Parameter count: " + params.size());
LOG.info("Count query:\n" + sb.toString());
return this;
}
public void filterTile(int zoom, int xtile, int ytile) {
if (zoom >= 0) {
LOG.debug("ZOOM=" + zoom);
final double lonW = CoordUtil.tileToLon(zoom, xtile);
final double lonE = CoordUtil.tileToLon(zoom, xtile + 1);
final double diffLon = lonE - lonW;
final double latN = CoordUtil.tileToLat(zoom, ytile);
final double latS = CoordUtil.tileToLat(zoom, ytile + 1);
final double diffLat = latN - latS;
if (whereBuffer.length() == 0)
whereBuffer.append(" where ");
else
whereBuffer.append(" and ");
whereBuffer.append(" ((geo.longitude between ? and ?) and (geo.latitude between ? and ?)) ");
params.add(lonW - zoom * diffLon * .2);
params.add(lonE + zoom * diffLon * .2);
params.add(latS - zoom * diffLat * .2);
params.add(latN + zoom * diffLat * .2);
if (LOG.isDebugEnabled()) {
LOG.debug(lonW + " <= lon <= " + lonE + " corr=" + diffLon * .2);
LOG.debug(latS + " <= lat <= " + latN + " corr=" + diffLat * .2);
}
}
}
private void createQuery(StringBuffer sb, String dbName, JsonNode fieldQuery, List<Object> params) {
LOG.debug("Handling " + dbName);
if (fieldQuery != null) {
LOG.debug("Adding " + fieldQuery + " sz=" + fieldQuery.size() + " t=" + fieldQuery.getNodeType());
if (sb.length() == 0)
sb.append(" where ");
else
sb.append(" and ");
// Opening
sb.append(" ( ");
// Always an array of filters
if (fieldQuery.getNodeType() == JsonNodeType.ARRAY) {
// A filter value can be (a) explicit value or (b) an operation
// (a) explicit values are handled by =? or by IN (?,?,..)
int handledCount = handleExplicitValues(sb, dbName, fieldQuery.elements(), params);
// do we have more?
if (handledCount > 0 && fieldQuery.size() > handledCount) {
sb.append(" OR ");
}
handledCount += handleNullValues(sb, dbName, fieldQuery.elements(), params);
// do we have more?
if (handledCount > 0 && fieldQuery.size() > handledCount) {
sb.append(" OR ");
}
// (b) operations are expressed as {"min":12} or {"max":33} or
// {"range":[3,10]} or {"like":"test"}
handleOperations(sb, dbName, fieldQuery.elements(), params);
}
// closing
sb.append(" ) ");
}
}
private int handleOperations(StringBuffer sb, String dbName, Iterator<JsonNode> elements, List<Object> params) {
int counter = 0;
while (elements.hasNext()) {
JsonNode filterValue = elements.next();
LOG.debug("Inspecting " + dbName + " ... " + filterValue);
if (filterValue.isObject()) {
LOG.debug("Adding " + filterValue);
JsonNode range = filterValue.findValue("range");
if (range != null && range.isArray()) {
LOG.debug("Adding array: " + range);
if (counter > 0) {
sb.append(" or ");
}
counter++;
// must be an array
sb.append("\n ( ").append(dbName);
sb.append(" between ? and ? ) ");
addParam(params, range.get(0));
addParam(params, range.get(1));
}
JsonNode min = filterValue.findValue("min");
if (min != null && min.isNumber()) {
LOG.debug("Adding min number: " + min);
if (counter > 0) {
sb.append(" or ");
}
counter++;
// must be an number
sb.append("\n ( ").append(dbName);
sb.append(" >= ? ) ");
addParam(params, min);
}
JsonNode max = filterValue.findValue("max");
if (max != null && max.isNumber()) {
LOG.debug("Adding max number: " + max);
if (counter > 0) {
sb.append(" or ");
}
counter++;
// must be an number
sb.append("\n ( ").append(dbName);
sb.append(" <= ? ) ");
addParam(params, max);
}
JsonNode like = filterValue.findValue("like");
if (like != null && like.isTextual()) {
LOG.debug("Adding LIKE : " + like);
if (counter > 0) {
sb.append(" or ");
}
counter++;
// must be an number
sb.append("\n ( ").append(dbName);
sb.append(" LIKE ? ) ");
addParam(params, like);
}
}
}
return counter;
}
private int handleNullValues(StringBuffer sb, String dbName, Iterator<JsonNode> elements, List<Object> params) {
int counter = 0;
while (elements.hasNext()) {
JsonNode filterValue = elements.next();
LOG.debug("Inspecting " + dbName + " ... " + filterValue);
if (filterValue.isNull()) {
LOG.debug("Adding " + filterValue);
counter++;
}
}
if (counter > 0) {
sb.append("\n ( ").append(dbName);
sb.append(" is NULL ) ");
}
return counter;
}
private int handleExplicitValues(StringBuffer sb, String dbName, Iterator<JsonNode> elements, List<Object> params) {
int counter = 0;
while (elements.hasNext()) {
JsonNode filterValue = elements.next();
LOG.debug("Inspecting " + dbName + " ... " + filterValue);
if (filterValue.isNumber() || filterValue.isTextual() || filterValue.isBoolean()) {
LOG.debug("Adding " + filterValue);
counter++;
addParam(params, filterValue);
}
}
if (counter == 0) {
// Nothing..
} else if (counter == 1) {
sb.append("\n ( ").append(dbName);
sb.append(" = ? ) ");
} else {
sb.append("\n ( ").append(dbName);
sb.append(" IN ( ?");
for (int i = counter - 1; i > 0; i--) {
sb.append(",?");
}
sb.append(" ) )");
}
return counter;
}
private void addParam(List<Object> params, JsonNode value) {
if (JsonNodeType.STRING == value.getNodeType())
params.add(value.asText());
else if (JsonNodeType.BOOLEAN == value.getNodeType())
params.add(value.asBoolean());
else if (JsonNodeType.NUMBER == value.getNodeType())
params.add(value.asDouble());
}
public String getCountQuery(String countWhat) {
LOG.info("select count(" + countWhat + ") " + sb.toString() + " " + whereBuffer.toString());
return "select count(" + countWhat + ") " + sb.toString() + " " + whereBuffer.toString();
}
public Object[] getParameters() {
return params.toArray();
}
public DirectMysqlQuery pageable(Pageable pageable) {
if (sortBuffer.length() != 0) {
throw new RuntimeException("sortBuffer is not blank, invalid use of #pageable(Pageable)");
}
if (pageable == null) {
return this;
}
if (pageable.getSort() != null) {
sortBuffer.append("\n order by ");
for (Order o : pageable.getSort()) {
LOG.debug("Order: " + o);
// ClassMetadata md =
// sessionFactory.getClassMetadata(Accession.class);
// md.
// EntityType<Accession> x =
// entityManager.getMetamodel().entity(Accession.class);
// System.err.println(x.getAttribute(o.getProperty()).getName());
// sb.append(x.getAttribute(o.getProperty()).getName());
sortBuffer.append("a.").append(o.getProperty());
sortBuffer.append(" ").append(o.getDirection());
}
}
sortBuffer.append(" limit ");
LOG.debug("Pageable=" + pageable.getOffset() + " " + pageable.getPageNumber());
sortBuffer.append(pageable.getOffset());
sortBuffer.append(", ");
sortBuffer.append(pageable.getPageSize());
LOG.info("Filter query:\n" + sb.toString());
LOG.info("Parameter count: " + params.size());
LOG.info("Params: " + ArrayUtils.toString(params.toArray()));
return this;
}
public String getQuery(String what) {
LOG.info("select " + what + " " + sb.toString() + " " + whereBuffer.toString() + " " + sortBuffer.toString());
return "select " + what + " " + sb.toString() + " " + whereBuffer.toString() + " " + sortBuffer.toString();
}
public void limit(Integer limit) {
if (sortBuffer.length() != 0) {
throw new RuntimeException("sortBuffer is not blank, invalid use of #limit(Integer)");
}
if (limit != null) {
sortBuffer.append(" limit ").append(limit);
}
}
public static interface MethodResolver {
Method getMethod(long methodId);
}
}
src/main/java/org/genesys2/server/service/impl/GenesysFilterServiceImpl.java
View file @ 67ccd57d
......@@ -33,7 +33,6 @@ import javax.sql.DataSource;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.Predicate;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.genesys2.server.model.genesys.Accession;
......@@ -46,17 +45,16 @@ import org.genesys2.server.persistence.domain.MethodRepository;
import org.genesys2.server.persistence.domain.TraitValueRepository;
import org.genesys2.server.service.GenesysFilterService;
import org.genesys2.server.service.GenesysFilterService.GenesysFilter.DataType;
import org.genesys2.server.service.MappingService.CoordUtil;
import org.genesys2.server.service.InstituteService;
import org.genesys2.server.service.SearchService;
import org.genesys2.server.service.TaxonomyService;
import org.genesys2.server.service.TraitService;
import org.genesys2.server.service.impl.DirectMysqlQuery.MethodResolver;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.data.domain.Sort.Order;
import org.springframework.jdbc.core.ArgumentPreparedStatementSetter;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.PreparedStatementCreator;
......@@ -65,7 +63,6 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.JsonNodeType;
import com.fasterxml.jackson.databind.node.ObjectNode;
@Service
......@@ -203,104 +200,26 @@ public class GenesysFilterServiceImpl implements GenesysFilterService {
}
@Override
public Page<Accession> listAccessions(JsonNode jsonTree, Pageable pageable) {
public Page<Accession> listAccessions(ObjectNode jsonTree, Pageable pageable) {
Iterator<Entry<String, JsonNode>> fields = jsonTree.fields();
while (fields.hasNext()) {
Entry<String, JsonNode> entry = fields.next();
LOG.debug("Looking at " + entry.getKey() + " = " + entry.getValue());
}
List<Object> params = new ArrayList<Object>();
StringBuffer sb = new StringBuffer();
sb.append(" from accession a ");
if (jsonTree.has("crop") || jsonTree.has("genus") || jsonTree.has("taxon")) {
sb.append(" inner join taxonomy2 t on t.id=a.taxonomyId2 ");
if (jsonTree.has("crop")) {
sb.append(" inner join croptaxonomy ct on ct.taxonomyId=t.id inner join crop on crop.id=ct.cropId ");
}
}
if (jsonTree.has("latitude") || jsonTree.has("longitude") || jsonTree.has("elevation")) {
sb.append(" inner join accessiongeo geo on geo.accessionId=a.id ");
}
if (jsonTree.has("organization")) {
sb.append(" inner join faoinstitute fao on fao.id=a.instituteId inner join organizationinstitute oi on oi.instituteId=fao.id inner join organization org on org.id=oi.organizationId ");
}
StringBuffer sbf = new StringBuffer();
createQuery(sbf, "a.taxSpecies", jsonTree.get("species"), params);
createQuery(sbf, "a.acceNumb", jsonTree.get("acceNumb"), params);
createQuery(sbf, "a.orgCty", jsonTree.get("orgCty"), params);
createQuery(sbf, "a.instCode", jsonTree.get("instCode"), params);
createQuery(sbf, "a.inSGSV", jsonTree.get("inSvalbard"), params);
createQuery(sbf, "a.mlsStat", jsonTree.get("mlsStat"), params);
createQuery(sbf, "a.inTrust", jsonTree.get("inTrust"), params);
createQuery(sbf, "a.available", jsonTree.get("available"), params);
createQuery(sbf, "org.slug", jsonTree.get("organization"), params);
createQuery(sbf, "t.genus", jsonTree.get("genus"), params);
createQuery(sbf, "t.taxonName", jsonTree.get("taxon"), params);
createQuery(sbf, "geo.longitude", jsonTree.get("longitude"), params);
createQuery(sbf, "geo.latitude", jsonTree.get("latitude"), params);
createQuery(sbf, "geo.elevation", jsonTree.get("elevation"), params);
createQuery(sbf, "crop.shortName", jsonTree.get("crop"), params);
for (Iterator<String> it = jsonTree.fieldNames(); it.hasNext();) {
String fieldName = it.next();
if (fieldName.startsWith("gm:")) {
LOG.info("Filtering on C&E data, method=" + fieldName);
// Handle Genesys Method!
long methodId = Long.parseLong(fieldName.substring(3));
// Need to validate method
Method method = traitService.getMethod(methodId);
if (method != null) {
String alias = "gm" + methodId;
sb.append(" inner join `").append(methodId).append("` ").append(alias).append(" on ");
sb.append(alias).append(".accessionId=a.id");
createQuery(sbf, alias + ".`" + method.getFieldName() + "`", jsonTree.get(fieldName), params);
} else {
LOG.warn("No such method with id=" + methodId);
}
DirectMysqlQuery directQuery = new DirectMysqlQuery("accession", "a");
directQuery.jsonFilter(jsonTree, new MethodResolver() {
@Override
public Method getMethod(long methodId) {
return methodRepository.findOne(methodId);
}
}
sb.append(sbf.toString());
LOG.info("Parameter count: " + params.size());
LOG.info("Count query:\n" + sb.toString());
Long totalCount = this.jdbcTemplate.queryForObject("select count(a.id) " + sb.toString(), params.toArray(), Long.class);
});
directQuery.pageable(pageable);
Long totalCount = this.jdbcTemplate.queryForObject(directQuery.getCountQuery("a.id"), directQuery.getParameters(), Long.class);
LOG.info("Total count: " + totalCount);
if (pageable.getSort() != null) {
sb.append("\n order by ");
for (Order o : pageable.getSort()) {
LOG.debug("Order: " + o);
// ClassMetadata md =
// sessionFactory.getClassMetadata(Accession.class);
// md.
// EntityType<Accession> x =
// entityManager.getMetamodel().entity(Accession.class);
// System.err.println(x.getAttribute(o.getProperty()).getName());
// sb.append(x.getAttribute(o.getProperty()).getName());
sb.append("a.").append(o.getProperty());
sb.append(" ").append(o.getDirection());
}
}
sb.append(" limit ");
LOG.debug("Pageable=" + pageable.getOffset() + " " + pageable.getPageNumber());
sb.append(pageable.getOffset());
sb.append(", ");
sb.append(pageable.getPageSize());
LOG.info("Filter query:\n" + sb.toString());
LOG.info("Parameter count: " + params.size());
LOG.info("Params: " + ArrayUtils.toString(params.toArray()));
// List<Accession> results = this.jdbcTemplate.query(sb.toString(), new
// Object[] {}, new BeanPropertyRowMapper<Accession>(Accession.class));
LOG.info("Getting IDs");
List<Long> results = this.jdbcTemplate.queryForList("select a.id " + sb.toString(), params.toArray(), Long.class);
List<Long> results = this.jdbcTemplate.queryForList(directQuery.getQuery("a.id"), directQuery.getParameters(), Long.class);
LOG.info("Getting accessions");
return new PageImpl<Accession>(results.size() == 0 ? new ArrayList<Accession>() : accessionRepository.listById(results,
......@@ -310,173 +229,6 @@ public class GenesysFilterServiceImpl implements GenesysFilterService {
pageable, totalCount);
}
private void createQuery(StringBuffer sb, String dbName, JsonNode fieldQuery, List<Object> params) {
LOG.debug("Handling " + dbName);
if (fieldQuery != null) {
LOG.debug("Adding " + fieldQuery + " sz=" + fieldQuery.size() + " t=" + fieldQuery.getNodeType());
if (sb.length() == 0)
sb.append(" where ");
else
sb.append(" and ");
// Opening
sb.append(" ( ");
// Always an array of filters
if (fieldQuery.getNodeType() == JsonNodeType.ARRAY) {
// A filter value can be (a) explicit value or (b) an operation