Commit 816daf81 authored by Matija Obreza's avatar Matija Obreza

Merge branch '267-elasticsearch-update' into 'master'

Resolve "Elasticsearch update"

Closes #267

See merge request genesys-pgr/genesys-server!168
parents fbf47ce6 b19ca6fa
......@@ -412,17 +412,10 @@
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>1.7.6</version>
</dependency>
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-elasticsearch</artifactId>
<version>1.3.2.RELEASE</version>
<version>2.1.12.RELEASE</version>
</dependency>
<dependency>
......@@ -567,6 +560,11 @@
<version>${spring.framework.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna</artifactId>
<version>4.5.2</version>
</dependency>
</dependencies>
<build>
......@@ -867,10 +865,6 @@
</includes>
<filtering>true</filtering>
</resource>
<resource>
<directory>src/test/resources</directory>
<filtering>true</filtering>
</resource>
<resource>
<directory>${project.build.directory}/generated-resources</directory>
<filtering>false</filtering>
......
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.catalog.api.v0;
import static com.google.common.collect.Sets.*;
import static org.elasticsearch.index.query.QueryBuilders.*;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.genesys.blocks.model.BasicModel;
import org.genesys.blocks.model.JsonViews;
import org.genesys.catalog.exceptions.InvalidApiUsageException;
import org.genesys.catalog.model.Partner;
import org.genesys.catalog.model.dataset.AccessionIdentifier;
import org.genesys.catalog.model.dataset.Dataset;
import org.genesys.catalog.model.filters.DatasetFilter;
import org.genesys.catalog.model.traits.Descriptor;
import org.genesys2.server.model.impl.Crop;
import org.genesys2.server.service.ElasticsearchService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import com.fasterxml.jackson.annotation.JsonView;
/**
* API to search the Catalog.
*
* @author Matija Obreza
*/
@RestController("catalogSearchApi0")
@RequestMapping(value = { "/api/v0/search" })
public class SearchController {
private static final Logger LOG = LoggerFactory.getLogger(SearchController.class);
@Autowired(required = false)
private ElasticsearchService elasticsearch;
/**
* Get suggestions for dataset filters .
*
* @param filters the filters
* @param searchQuery the search query
* @return the map
*/
@JsonView({ JsonViews.Minimal.class })
@PostMapping(value = "/dataset/suggest", produces = MediaType.APPLICATION_JSON_VALUE)
public Map<String, SearchResults<?>> datasets(@RequestBody(required = false) final DatasetFilter filters, @RequestParam(value = "q", required = true) String searchQuery) {
LOG.trace("Incoming {}", searchQuery);
searchQuery = sanitizeQuery(searchQuery);
LOG.info("Suggestions for datasets for: {}", searchQuery);
if (StringUtils.isBlank(searchQuery)) {
throw new InvalidApiUsageException("No search query provided");
}
QueryBuilder extraFilters = filtersFromDataset(filters);
Map<Class<BasicModel>, List<BasicModel>> hitsByEntity = elasticsearch.search(extraFilters, newHashSet(Crop.class, Partner.class,
AccessionIdentifier.class, Descriptor.class), searchQuery);
Map<String, SearchResults<?>> suggestions = new HashMap<>();
suggestions.put("search.group.crop", SearchResults.from("code", Arrays.asList("crop"), hitsByEntity.get(Crop.class)));
suggestions.put("search.group.partner", SearchResults.from("uuid", Arrays.asList("owner.uuid"), hitsByEntity.get(Partner.class)));
suggestions.put("search.group.accession", SearchResults.from("doi", Arrays.asList("accessionIdentifier.doi"), hitsByEntity.get(AccessionIdentifier.class)));
suggestions.put("search.group.descriptor", SearchResults.from("uuid", Arrays.asList("descriptor.uuid"), hitsByEntity.get(Descriptor.class)));
// Search datasets
suggestions.put("search.matches", SearchResults.from("uuid", Arrays.asList("uuid"), elasticsearch.search(extraFilters, Dataset.class, searchQuery)));
return suggestions;
}
/// Try to enhance the search by adding known Dataset filters
private QueryBuilder filtersFromDataset(DatasetFilter filters) {
if (filters == null) {
return null;
}
BoolQueryBuilder q = boolQuery();
if (filters.crop != null && !filters.crop.isEmpty()) {
q.must(termsQuery("crop", filters.crop));
}
q.boost(3.0f);
return q.hasClauses() ? q : null;
}
/**
* Sanitize incoming search query
*
* @param searchQuery incoming
* @return sanitized search query
*/
private String sanitizeQuery(String searchQuery) {
if (StringUtils.isBlank(searchQuery)) {
return null;
}
return searchQuery.replaceAll("[^\\w\\d\\s]+", "").replaceAll("\\s\\s+", " ").trim();
}
}
/**
* Wrapper for search results
*/
class SearchResults<T extends BasicModel> {
public List<String> filters;
public String key = "uuid";
public List<T> hits;
public static <T extends BasicModel> SearchResults<T> from(String key, List<String> filters, List<T> list) {
if (list == null || list.isEmpty())
return null;
SearchResults<T> sr = new SearchResults<T>();
sr.filters = filters;
sr.key = key;
sr.hits = list;
return sr;
}
}
/**
* Copyright 2016 Global Crop Diversity Trust
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -12,67 +12,48 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
package org.genesys2.server.service;
*/
package org.genesys.catalog.custom.elasticsearch;
import org.genesys2.server.service.impl.FilterHandler.AppliedFilters;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.data.elasticsearch.ElasticsearchException;
public interface ElasticSearchManagementService {
/**
* Move alias to a different index
*
* @param aliasName
* @param indexName
*/
void realias(String aliasName, String indexName);
/**
* Delete an ES alias
*
* @param aliasName
*/
void deleteAlias(String aliasName);
/**
* Checks if alias exists
*
* @param aliasName
*/
boolean aliasExists(String aliasName);
/**
* Refreshes specified index
*
* @param className
*/
void refreshIndex(String className);
import java.io.IOException;
/**
* Create new indexes and fill them with data
*/
void regenerateIndexes(String indexName) throws ElasticsearchException;
import static org.genesys.catalog.custom.elasticsearch.CustomMappingBuilder.buildMapping;
/**
* Reindex part of accession database
*
* @param filters
*/
void reindex(AppliedFilters filters);
/**
* Reindex part of accession database
*
* @param type
*/
void reindex(String type);
/**
* Used to access the package-protected buildMapping functions.
*/
public class CustomMapping {
/** The Constant LOG. */
public static final Logger LOG = LoggerFactory.getLogger(CustomMapping.class);
/**
* Delete an unused index (no aliases)
* Spring data mapping.
*
* @param indexName
*/
void deleteIndex(String indexName);
* @param <T> the generic type
* @param clazz the clazz
* @param indexType the index type
* @return the object
*/
public static <T> Object springDataMapping(Class<T> clazz, String indexType) {
XContentBuilder xContentBuilder = null;
try {
xContentBuilder = buildMapping(clazz, indexType, "id", null);
} catch (Exception e) {
throw new ElasticsearchException("Failed to build mapping for " + clazz.getSimpleName(), e);
}
try {
LOG.info("Mapping for class={}/{}: {}", clazz, indexType, xContentBuilder.string());
} catch (IOException e) {
}
return xContentBuilder;
}
}
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.catalog.custom.elasticsearch;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.node.NodeClient;
import org.elasticsearch.common.settings.Settings;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.FactoryBean;
import org.springframework.beans.factory.InitializingBean;
import static org.elasticsearch.node.NodeBuilder.nodeBuilder;
/**
* Sets up an embedded ES.
*/
public class EmbeddedClientFactoryBean implements FactoryBean<Client>, InitializingBean, DisposableBean {
private String clusterName;
private boolean withHttp = true;
private String dataHome = "data/elasticsearch";
private NodeClient client;
/**
* Sets the cluster name.
*
* @param clusterName the new cluster name
*/
public void setClusterName(String clusterName) {
this.clusterName = clusterName;
}
/**
* Sets the with http.
*
* @param withHttp the new with http
*/
public void setWithHttp(boolean withHttp) {
this.withHttp = withHttp;
}
/**
* Sets the data home.
*
* @param dataHome the new data home
*/
public void setDataHome(String dataHome) {
this.dataHome = dataHome;
}
@Override
public void afterPropertiesSet() throws Exception {
final Settings.Builder settings = Settings.builder()
/*@formatter:off*/
.put("http.enabled", String.valueOf(withHttp))
.put("path.home", dataHome);
/*@formatter:on*/
this.client = (NodeClient) nodeBuilder().settings(settings).clusterName(clusterName).local(true).node().client();
}
@Override
public Client getObject() throws Exception {
return this.client;
}
@Override
public Class<?> getObjectType() {
return Client.class;
}
@Override
public boolean isSingleton() {
return true;
}
@Override
public void destroy() throws Exception {
try {
System.err.println("Closing elasticSearch client");
if (client != null) {
client.close();
}
} catch (final Exception e) {
System.err.println("Error closing ElasticSearch client: " + e.getMessage());
}
}
}
package org.genesys2.brapi.service.impl;
import java.util.Collections;
import java.util.UUID;
import java.util.stream.Collectors;
import org.elasticsearch.common.lang3.StringUtils;
import org.apache.commons.lang3.StringUtils;
import org.genesys2.brapi.model.Germplasm;
import org.genesys2.brapi.service.BrAPIService;
import org.genesys2.server.model.elastic.AccessionDetails;
import org.genesys2.server.model.elastic.Taxonomy;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.genesys.Taxonomy2;
import org.genesys2.server.model.impl.Crop;
import org.genesys2.server.service.CropService;
import org.genesys2.server.service.FilterConstants;
......
......@@ -34,7 +34,7 @@ import org.genesys2.server.model.elastic.AccessionDetails;
import org.genesys2.server.model.genesys.Accession;
import org.genesys2.server.model.impl.FaoInstitute;
import org.genesys2.server.model.json.AccessionJson;
import org.genesys2.server.service.ElasticService;
import org.genesys2.server.service.ElasticsearchService;
import org.genesys2.server.service.GenesysFilterService;
import org.genesys2.server.service.GenesysRESTService;
import org.genesys2.server.service.GenesysService;
......@@ -109,7 +109,7 @@ public class AccessionController extends ApiBaseController {
GenesysRESTService restService;
@Autowired
ElasticService elasticService;
ElasticsearchService elasticService;
/**
* Check if accessions exists in the system
......@@ -335,20 +335,21 @@ public class AccessionController extends ApiBaseController {
}
@RequestMapping(value = "/search", method = { RequestMethod.GET }, produces = { MediaType.APPLICATION_JSON_VALUE })
public @ResponseBody Page<AccessionDetails> search(@RequestParam("page") final int page, @RequestParam("query") final String query) throws SearchException {
return elasticService.search(StringUtils.defaultIfBlank(query, "*"), new PageRequest(page - 1, 20));
public @ResponseBody List<Accession> search(@RequestParam("page") final int page, @RequestParam("query") final String query) throws SearchException {
return elasticService.search(null, Accession.class, StringUtils.defaultIfBlank(query, "*")); //, new PageRequest(page - 1, 20));
}
// FIXME Not using institute...
@RequestMapping(value = "/{instCode}/list", method = { RequestMethod.GET }, produces = { MediaType.APPLICATION_JSON_VALUE })
public @ResponseBody Page<AccessionDetails> list(@PathVariable("instCode") String instCode, @RequestParam("page") int page, @RequestParam("query") String query)
public @ResponseBody List<Accession> list(@PathVariable("instCode") String instCode, @RequestParam("page") int page, @RequestParam("query") String query)
throws SearchException {
FaoInstitute institute = instituteService.getInstitute(instCode);
if (institute == null) {
throw new ResourceNotFoundException();
}
query = StringUtils.defaultIfBlank(query, "*");
return elasticService.search(query, new PageRequest(page - 1, 20));
// TODO filer by inst
return elasticService.search(null, Accession.class, query); // , new PageRequest(page - 1, 20));
}
@RequestMapping(value = "/{id}", method = { RequestMethod.GET }, produces = { MediaType.APPLICATION_JSON_VALUE })
......
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys2.server.component.elastic;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import javax.annotation.Resource;
import org.aspectj.lang.JoinPoint;
import org.aspectj.lang.annotation.AfterReturning;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Before;
import org.genesys.blocks.model.BasicModel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.stereotype.Component;
/**
* AspectJ powered listener on repository save and delete operations
* adds indexed entities to re-index queue, handled by {@link ElasticReindexProcessor}.
*/
@Aspect
@Component
public class ElasticJPAListener {
private final static Logger LOG = LoggerFactory.getLogger(ElasticJPAListener.class);
private Set<Object> includedClasses;
private Set<Object> ignoredClasses;
@Resource
private BlockingQueue<ElasticReindex> elasticReindexQueue;
/**
* Instantiates a new elastic JPA listener.
*/
public ElasticJPAListener() {
System.err.println("Making ElasticJPAListener");
ignoredClasses = Collections.synchronizedSet(new HashSet<>());
includedClasses = Collections.synchronizedSet(new HashSet<>());
}
/**
* After persist.
*
* @param joinPoint the join point
* @param result the result
*/
@AfterReturning(value = "execution(* org.springframework.data.jpa.repository.JpaRepository.save(..))", returning = "result")
public void afterPersist(final JoinPoint joinPoint, final Object result) {
LOG.debug("JPA afterPersist {} {}", joinPoint.toLongString(), joinPoint.getTarget());
try {
scheduleReindexing(result);
} catch (Throwable e) {
LOG.error(e.getMessage(), e);
}
}
/**
* Before remove.
*
* @param joinPoint the join point
*/
@Before(value = "execution(* org.springframework.data.jpa.repository.JpaRepository.delete(..)) || execution(* org.springframework.data.jpa.repository.JpaRepository.deleteInBatch(..))")
public void beforeRemove(final JoinPoint joinPoint) {
final Object[] args = joinPoint.getArgs();
try {
final Object removed = args[0];
LOG.debug("JPA afterRemove: {} {}", joinPoint.toLongString(), joinPoint.getTarget());
scheduleReindexing(removed);
} catch (Throwable e) {
LOG.error(e.getMessage(), e);
}
}