Merge remote-tracking branch 'upstream/master' into 25-filter-accessions-by-georegion

Conflicts:
	src/main/java/org/genesys2/server/servlet/controller/ExplorerController.java

docker/jetty/resources/genesys.properties

@@ -18,5 +18,4 @@ db.showSql=false
 db.hbm2ddl=true
 hibernate.dialect=org.hibernate.dialect.MySQL5InnoDBDialect
 
-download.files.dir=/data/genesys/temp
-file.repository.dir=/data/genesys/persistent/repository
+data.dir=/data/genesys/

src/main/java/org/genesys2/server/service/ElasticService.java

@@ -43,4 +43,6 @@ public interface ElasticService {
 	List<String> autocompleteSearch(String query) throws SearchException;
 
 	void regenerateAccessionSequentialNumber();
+
+	String getBoundingBox(AppliedFilters filters);
 }

src/main/java/org/genesys2/server/service/GenesysFilterService.java

@@ -41,6 +41,4 @@ public interface GenesysFilterService {
 
 	void listGeoTile(boolean distinct, AppliedFilters filters, Integer limit, int zoom, int xtile, int ytile, RowCallbackHandler rowHandler);
 
-	void listGeoForBoundingBox(AppliedFilters filters, RowCallbackHandler rowHandler);
-
 }

src/main/java/org/genesys2/server/service/MappingService.java

@@ -68,15 +68,6 @@ public interface MappingService {
 	 */
 	byte[] getTile(AppliedFilters filters, int zoom, int xtile, int ytile);
 
-	/**
-	 * Gets coordinates of bounding box
-	 *
-	 * @param filters
-	 *            the filters
-	 * @return coordinates
-	 */
-	String getBoundingBox(AppliedFilters filters);
-
 	/**
 	 * Coordinate utilities
 	 */

src/main/java/org/genesys2/server/service/impl/ElasticsearchSearchServiceImpl.java

@@ -26,14 +26,23 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.index.query.AndFilterBuilder;
 import org.elasticsearch.index.query.FilterBuilders;
 import org.elasticsearch.index.query.OrFilterBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryStringQueryBuilder.Operator;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.bucket.filter.Filter;
+import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.max.InternalMax;
+import org.elasticsearch.search.aggregations.metrics.min.InternalMin;
 import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
@@ -74,6 +83,8 @@ import org.springframework.stereotype.Service;
 @Service
 public class ElasticsearchSearchServiceImpl implements ElasticService, InitializingBean {
 
+	private final ObjectMapper mapper = new ObjectMapper();
+
 	private static final Log LOG = LogFactory.getLog(ElasticsearchSearchServiceImpl.class);
 
 	@Autowired
@@ -147,6 +158,41 @@ public class ElasticsearchSearchServiceImpl implements ElasticService, Initializ
 		}
 	}
 
+	@Override
+	public String getBoundingBox(final AppliedFilters filters) {
+		AndFilterBuilder filterBuilder = getFilterBuilder(filters);
+
+		final FilterAggregationBuilder builder = AggregationBuilders
+				.filter("agg").filter(filterBuilder)
+				.subAggregation(AggregationBuilders.min("min_longitude").field("geo.longitude"))
+				.subAggregation(AggregationBuilders.max("max_longitude").field("geo.longitude"))
+				.subAggregation(AggregationBuilders.min("min_latitude").field("geo.latitude"))
+				.subAggregation(AggregationBuilders.max("max_latitude").field("geo.latitude"));
+
+		SearchQuery searchQuery =
+				new NativeSearchQueryBuilder()
+						.withIndices(IndexAliasConstants.INDEXALIAS_PASSPORT_WRITE)
+						.withTypes(IndexAliasConstants.PASSPORT_TYPE)
+						.addAggregation(builder)
+						.build();
+
+		Aggregations aggregations = elasticsearchTemplate.query(searchQuery, SearchResponse::getAggregations);
+
+		Filter agg = aggregations.get("agg");
+
+		InternalMin minLatitude = agg.getAggregations().get("min_latitude");
+		InternalMin minLongitude = agg.getAggregations().get("min_longitude");
+		InternalMax maxLatitude = agg.getAggregations().get("max_latitude");
+		InternalMax maxLongitude = agg.getAggregations().get("max_longitude");
+
+		final ObjectNode geoJson = mapper.createObjectNode();
+		geoJson.put("west", String.valueOf(minLatitude.getValue()));
+		geoJson.put("south", String.valueOf(minLongitude.getValue()));
+		geoJson.put("east", String.valueOf(maxLatitude.getValue()));
+		geoJson.put("north",String.valueOf(maxLongitude.getValue()));
+		return geoJson.toString();
+	}
+
 	@Override
 	public TermResult termStatistics(AppliedFilters appliedFilters, String term, int size) throws SearchException {
 

src/main/java/org/genesys2/server/service/impl/GenesysFilterServiceImpl.java

@@ -415,28 +415,4 @@ public class GenesysFilterServiceImpl implements GenesysFilterService {
 		}, rowHandler);
 	}
 
-	@Override
-	public void listGeoForBoundingBox(final AppliedFilters filters, final RowCallbackHandler rowHandler) {
-
-		final DirectMysqlQuery directQuery = new DirectMysqlQuery("accessiongeo", "geo");
-
-		if (!filters.isEmpty()) {
-			// Join accession table when we have filters or looking at details
-			directQuery.innerJoin("accession", "a", "a.id=geo.accessionId");
-		}
-
-		if (!filters.isEmpty()) {
-			directQuery.join(filters);
-			directQuery.filter(filters, methodId -> methodRepository.findOne(methodId));
-		}
-
-		this.jdbcTemplate.query(connection -> {
-            final PreparedStatement stmt = connection.prepareStatement(
-            		directQuery.getQuery("Max(geo.longitude), Min(geo.longitude), Max(geo.latitude), Min(geo.latitude)"));
-            // Set mysql JConnector to stream results
-            stmt.setFetchSize(Integer.MIN_VALUE);
-            new ArgumentPreparedStatementSetter(directQuery.getParameters()).setValues(stmt);
-            return stmt;
-        }, rowHandler);
-	}
 }

src/main/java/org/genesys2/server/service/impl/MappingServiceImpl.java

@@ -222,19 +222,4 @@ public class MappingServiceImpl implements MappingService {
 			throw new RuntimeException("Could not render image", e);
 		}
 	}
-
-	public String getBoundingBox(final AppliedFilters filters) {
-		final String[] result = {""};
-
-		filterService.listGeoForBoundingBox(filters, resultSet -> {
-            final ObjectNode geoJson = mapper.createObjectNode();
-            geoJson.put("west", String.valueOf(resultSet.getDouble("Min(geo.latitude)")));
-            geoJson.put("south", String.valueOf(resultSet.getDouble("Min(geo.longitude)")));
-            geoJson.put("east", String.valueOf(resultSet.getDouble("Max(geo.latitude)")));
-            geoJson.put("north", String.valueOf(resultSet.getDouble("Max(geo.longitude)")));
-            result[0] = geoJson.toString();
-        });
-		return result[0];
-	}
-
 }

src/main/java/org/genesys2/server/service/worker/WorldClimUpdater.java

@@ -90,8 +90,8 @@ public class WorldClimUpdater implements InitializingBean {
 	// http://biogeo.ucdavis.edu/data/climate/worldclim/1_4/grid/cur/bio_2-5m_bil.zip
 	// http://biogeo.ucdavis.edu/data/climate/worldclim/1_4/grid/cur/alt_2-5m_bil.zip
 
-	@Value("${download.files.dir}")
-	private String downloadDir;
+	@Value("${worldclim.dir}")
+	private String worldclimDir;
 
 	private File worldClimDir;
 
@@ -110,7 +110,7 @@ public class WorldClimUpdater implements InitializingBean {
 
 	@Override
 	public void afterPropertiesSet() throws Exception {
-		this.worldClimDir = new File(downloadDir, "worldclim");
+		this.worldClimDir = new File(worldclimDir);
 		this.worldClimDir.mkdirs();
 	}
 

src/main/java/org/genesys2/server/servlet/controller/ExplorerController.java

@@ -706,7 +706,7 @@ public class ExplorerController extends BaseController implements InitializingBe
 	}
 
 	/**
-	 *
+	 * 
 	 * @param cookieColumns
 	 *            validated selected columns
 	 * @param response
@@ -976,8 +976,7 @@ public class ExplorerController extends BaseController implements InitializingBe
 
 		AppliedFilters appliedFilters = mapper.readValue(jsonFilter, AppliedFilters.class);
 		transformFiltersIfNeed(appliedFilters);
-
-		return mappingService.getBoundingBox(appliedFilters);
+		return elasticService.getBoundingBox(appliedFilters);
 	}
 
 	@RequestMapping(value = "/explore/tile/{zoom}/{x}/{y}", produces = MediaType.IMAGE_PNG_VALUE)

src/main/java/org/genesys2/spring/config/ElasticsearchConfigDev.java

@@ -16,11 +16,14 @@
 
 package org.genesys2.spring.config;
 
+import static org.elasticsearch.node.NodeBuilder.*;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.settings.ImmutableSettings;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Profile;
-import org.springframework.data.elasticsearch.client.NodeClientFactoryBean;
 
 @Configuration
 @Profile({ "dev" })
@@ -35,11 +38,15 @@ public class ElasticsearchConfigDev {
 	@Value("${elasticsearch.enable.http}")
 	private boolean enableHttp;
 
+	@Value("${elasticsearch.data.path}")
+	private String dataPath;
+
 	@Bean
-	public NodeClientFactoryBean ncfb() {
-		NodeClientFactoryBean ncfb = new NodeClientFactoryBean(true);
-		ncfb.setClusterName(clusterName);
-		ncfb.setEnableHttp(enableHttp);
-		return ncfb;
+	public NodeClient nodeClient() {
+		ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder().put("http.enabled",
+				String.valueOf(this.enableHttp)).put("path.home", this.dataPath);
+
+		return (NodeClient) nodeBuilder().settings(settings).clusterName(this.clusterName).local(true).node()
+				.client();
 	}
 }

src/main/resources/spring/spring.properties

@@ -48,7 +48,8 @@ captcha.siteKey=${captcha.publicKey}
 captcha.publicKey=6Lfb5w4TAAAAAI6pD_4l5uIwXEaUJ6KQmhU9cyjx
 
 # paths
-download.files.dir=./data/
+data.dir=./data
+worldclim.dir=${data.dir}/worldclim
 
 # ITPGRFA Easy-SMTA account (if you have one)
 itpgrfa.easysmta.url=https://mls.planttreaty.org/itt/index.php?r=extsys/userinfo
@@ -148,6 +149,7 @@ elasticsearch.enable.http=true
 # Connect to external ES nodes. Comma separated list of hostname:port
 # elasticsearch.cluster.nodes=localhost:9300,otherhost:9300
 elasticsearch.cluster.nodes=
+elasticsearch.data.path=${data.dir}/elastic
 
 transifex.project=genesys-dev
 transifex.username=Alexandr19011990
@@ -166,7 +168,7 @@ transifex.content.template=<html xmlns="http://www.w3.org/1999/xhtml">\
 
 
 # File repository
-file.repository.dir=${download.files.dir}/repository
+file.repository.dir=${data.dir}/repository
 
 # File repository S3 parameter values
 s3.accessKey=

src/main/webapp/WEB-INF/jsp/accession/explore2.jsp

@@ -430,7 +430,7 @@
                 $(this).remove();
                 $('input[i-key="' + key + '"][value="' + value + '"]').prop('checked', false);
 
-                if (jsonData["crops"].length === 0) {
+                if (jsonData["crops"] !== undefined && jsonData["crops"].length === 0) {
                     delete jsonData["crops"];
                     $("#panel_additional_id").hide();
                 }

src/main/webapp/WEB-INF/jsp/accession/map.jsp

@@ -320,7 +320,7 @@
                 $(this).remove();
                 $('input[i-key=" + normKey + "][value=" + value + "]').prop('checked', false);
 
-                if (jsonData["crops"].length === 0) {
+                if (jsonData["crops"] !== undefined && jsonData["crops"].length === 0) {
                     delete jsonData["crops"];
                     $("#panel_additional_id").hide();
                 }

src/main/webapp/WEB-INF/jsp/accession/overview.jsp

@@ -277,21 +277,6 @@
                 jsonData[key] = [];
             });
 
-            $("body").on("click", ".filtval", function (event) {
-                event.preventDefault();
-                var key = $(this).attr("i-key");
-                var normKey = GenesysFilter.normKey(key);
-                var value = $(this).attr("x-key").replace(normKey, "");
-
-                if (value == "null") value = null;
-                GenesysFilterUtil.removeValue(value, key, jsonData);
-                applyFilters();
-                BrowseUtil.applySuggestions(jsonData, messages);
-
-                $(this).remove();
-                $('input[i-key=" + normKey + "][value=" + value + "]').prop('checked', false);
-            });
-
             $("#menuShareLink").on("click", function () {
                 if ($('#shortLink').val() === '') {
                     $.ajax({

src/test/resources/spring/spring.properties

@@ -38,7 +38,9 @@ captcha.publicKey=B
 lucene.indexDir=./lucene/
 
 # paths
-download.files.dir=./data/
+data.dir=./data
+elasticsearch.data.path=${data.dir}/elastic
+file.repository.dir=${data.dir}/repository
 
 # ITPGRFA Easy-SMTA account (if you have one)
 itpgrfa.easysmta.url=https://mls.planttreaty.org/itt/index.php?r=extsys/userinfo